maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

Browse Source
This commit is contained in:
maksw committed 2026-08-11 18:38:36 +02:00
1 parent 8b254f84f4
commit 4add6789ba
177 files changed
+22565 -3070
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+1 -1
View File
@@ -2,7 +2,7 @@ build.obj
bin/
obj/
qemu.log
fatroot/
disk.vhd
usbfatroot/
code.zip
code/build/build.exe
OVMF/OVMF_VARS_4M.fd
BIN
View File
Binary file not shown.
OVMF/OVMF_VARS_4M_d.fd
BIN
View File
Binary file not shown.
code/boot.ini
+1
View File
@@ -1,5 +1,6 @@
[kernel]
; bitmask: 1 = debug prints, 2 = self check, 4 = file log, 4096 = checked intrusive lists
; 0x8000 reports MTRRs. Add 0x10000 for one temporary framebuffer WC experiment.
debug = 0xffff
[init]
commandline = apps/shell_native.exe
code/boot/uefi/main.c
+47 -17
View File
@@ -26,6 +26,11 @@ int _fltused = 0;
#define KERNEL_VIRT_BASE 0xFFFF800000000000ULL
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
#define PAGE_SIZE 4096ULL
#define BOOT_PAGE_PRESENT 0x01ULL
#define BOOT_PAGE_WRITE 0x02ULL
#define BOOT_PAGE_CACHE_WB 0x00ULL
#define BOOT_PAGE_CACHE_WC 0x08ULL
#define BOOT_PAGE_CACHE_UC 0x10ULL
// Implements the ascii Str To Unicode Str S operation.
static int AsciiStrToUnicodeStrS(const char* Source, uint16_t* Dest, size_t DestMax) {
@@ -168,6 +173,35 @@ static void MapRangeSmart(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags) {
}
}
// Returns whether a firmware descriptor describes memory that is safe to expose through the direct map.
static bool IsDirectMapRamType(EFI_MEMORY_TYPE type) {
switch (type) {
case EfiLoaderCode:
case EfiLoaderData:
case EfiBootServicesCode:
case EfiBootServicesData:
case EfiRuntimeServicesCode:
case EfiRuntimeServicesData:
case EfiConventionalMemory:
case EfiACPIReclaimMemory:
case EfiACPIMemoryNVS:
return true;
default:
return false;
}
}
// Converts firmware cache attributes into the kernel's canonical PAT policy.
static UINT64 DirectMapCacheFlags(const EFI_MEMORY_DESCRIPTOR* desc) {
if (!desc)
return BOOT_PAGE_CACHE_UC;
if (desc->Attribute & (EFI_MEMORY_UC | EFI_MEMORY_UCE | EFI_MEMORY_WT))
return BOOT_PAGE_CACHE_UC;
if (desc->Attribute & EFI_MEMORY_WC)
return BOOT_PAGE_CACHE_WC;
return BOOT_PAGE_CACHE_WB;
}
static void MapRangeSkipping(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags,
const UINT64* holes, UINTN holeCount) {
UINT64 curVirt = virt;
@@ -790,8 +824,8 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
for (UINT64 i = 0; i < fbBytes; i += PAGE_SIZE) {
UINT64 phys = gop->Mode->FrameBufferBase + i;
Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // P|W|PCD (write-combined MMIO)
Map4K(phys, phys, 0x13);
Map4K(PHYS_MAP_BASE + phys, phys, BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_WC);
Map4K(phys, phys, BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_WC);
}
}
@@ -866,21 +900,16 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
for (UINTN i = 0; i < mapSize / descSz; i++) {
EFI_MEMORY_DESCRIPTOR* desc = (EFI_MEMORY_DESCRIPTOR *)((UINT8 *)efiMap + i * descSz);
// Map normal firmware-described regions into the physical window.
// PCI/firmware MMIO is mapped explicitly by the owning driver.
if (desc->Type != EfiUnusableMemory &&
desc->Type != EfiMemoryMappedIO &&
desc->Type != EfiMemoryMappedIOPortSpace) {
// Only firmware-described RAM receives a direct-map alias. Reserved
// and MMIO descriptors stay unmapped until an owning subsystem maps
// them with an explicit PAT policy.
if (IsDirectMapRamType(desc->Type)) {
UINT64 start = desc->PhysicalStart;
UINT64 end = start + (desc->NumberOfPages * PAGE_SIZE);
UINT64 size = desc->NumberOfPages * PAGE_SIZE;
UINT64 flags = BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | DirectMapCacheFlags(desc);
// Add 2 pages (8KB) of padding to the end of the region.
// This catches ACPI tables that cross page boundaries into unmapped
// memory because of buggy UEFI size reporting.
end += 2 * PAGE_SIZE;
MapRangeSkipping(start, start, end - start, 0x03, idHoles, idHoleCount);
MapRangeSkipping(PHYS_MAP_BASE + start, start, end - start, 0x03, winHoles, winHoleCount);
MapRangeSkipping(start, start, size, flags, idHoles, idHoleCount);
MapRangeSkipping(PHYS_MAP_BASE + start, start, size, flags, winHoles, winHoleCount);
}
}
}
@@ -891,8 +920,9 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
// We explicitly map the region from 0xFEC00000 up through 0xFEE00FFF
// to guarantee the HAL can access them via the physical window.
for (UINT64 phys = 0xFEC00000; phys < 0xFEE01000; phys += PAGE_SIZE) {
Map4K(phys, phys, 0x13); // Identity Map, P|W|PCD
Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // Physical Window Map, P|W|PCD
UINT64 flags = BOOT_PAGE_PRESENT | BOOT_PAGE_WRITE | BOOT_PAGE_CACHE_UC;
Map4K(phys, phys, flags);
Map4K(PHYS_MAP_BASE + phys, phys, flags);
}
Print(L"8\n");
code/build/build.c
+273 -103
View File
@@ -21,6 +21,10 @@
#define LWIP_CFLAGS_LITERAL DRIVER_CFLAGS_LITERAL " /Icode\\drivers\\net\\lwip\\include /Icode\\drivers\\net\\lwip\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706"
#define LFLAGS_LITERAL "/nologo /NODEFAULTLIB /DEBUG"
// Partition offsets inside disk.vhd
#define VHD_PART1 "disk.vhd@@1M" // ESP / Partition 1 (Offset 1MB)
#define VHD_PART2 "disk.vhd@@65M" // Test/Data / Partition 2 (Offset 65MB)
const char *CommonCflags = COMMON_CFLAGS_LITERAL;
const char *KernelCflags = KERNEL_CFLAGS_LITERAL;
const char *UserCflags = USER_CFLAGS_LITERAL;
@@ -46,6 +50,7 @@ static const Driver_Target PCI_BUS_DRIVER =
static const Driver_Target STANDALONE_DRIVERS[] = {
DRIVER_TARGET("ps2", "obj\\drivers\\ps2", "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"),
DRIVER_TARGET("serial", "obj\\drivers\\serial", "code\\drivers\\input\\serial\\serial.c"),
DRIVER_TARGET_LIBS("hda", "obj\\drivers\\hda", "code\\drivers\\audio\\hda\\hda.c code\\drivers\\audio\\hda\\hda_transport.c code\\drivers\\audio\\hda\\hda_codec.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("ata", "obj\\drivers\\ata", "code\\drivers\\storage\\ata\\ata.c", "bin\\pci.lib"),
DRIVER_TARGET_LIBS("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c", "bin\\pci.lib"),
DRIVER_TARGET("fat", "obj\\drivers\\fat", "code\\drivers\\fs\\fat\\fat.c code\\drivers\\fs\\fat\\fat_mount.c"),
@@ -64,7 +69,7 @@ void create_layout() {
"obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop",
"obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\all_tests", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke"
};
@@ -91,44 +96,190 @@ Nob_Proc run_async(const char *cmd_str) {
return proc;
}
// Cleans all object output by replacing obj/ as a whole.
// Only obj/ is disposable. fatroot/ and usbfatroot/ are developer-owned vvfat roots.
bool clean_build_object_directories(void) {
const char *cmd = "if exist obj rmdir /s /q obj";
if (!run_sync(cmd)) return false;
return nob_mkdir_if_not_exists("obj");
// Deletes stale object files from a target directory before compiling into it.
bool clean_object_directory(const char *obj_dir) {
const char *cmd = nob_temp_sprintf("if exist %s\\*.obj del /q %s\\*.obj", obj_dir, obj_dir);
return run_sync(cmd);
}
// Creates the FAT root layout used by QEMU vvfat without deleting existing files.
// The vvfat root is also a convenient developer workspace, so packaging must be non-destructive.
bool create_fatroot_layout(void) {
// Cleans object directories that are linked through wildcards.
bool clean_build_object_directories(void) {
const char *dirs[] = {
"fatroot\\EFI", "fatroot\\EFI\\BOOT",
"fatroot\\system", "fatroot\\drivers", "fatroot\\apps", "fatroot\\tests"
"obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
"obj\\syscall", "obj\\runtime", "obj\\win32", "obj\\posix",
"obj\\init", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping",
"obj\\doom", "obj\\video", "obj\\memtop", "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_peb_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\all_tests", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_present", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke"
};
for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) {
if (!nob_mkdir_if_not_exists(dirs[i])) return false;
if (!clean_object_directory(dirs[i])) return false;
}
if (!clean_object_directory(PCI_BUS_DRIVER.obj_dir)) return false;
for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) {
if (!clean_object_directory(STANDALONE_DRIVERS[i].obj_dir)) return false;
}
return true;
}
// Creates the USB vvfat root once and leaves its contents alone between builds.
// Creates the USB fat root once and leaves its contents alone between builds.
bool create_usbfatroot_if_missing(void) {
return nob_mkdir_if_not_exists("usbfatroot");
}
// Utility to copy a local file into a FAT image using mcopy without admin rights.
bool mcopy_file(const char *src, const char *vhd_target, const char *dst_path) {
const char *cmd = nob_temp_sprintf("mcopy -o -i %s %s ::%s", vhd_target, src, dst_path);
return run_sync(cmd);
}
// Utility to create a directory in a FAT image using mmd.
bool mmd_dir(const char *vhd_target, const char *dir_path) {
const char *cmd = nob_temp_sprintf("mmd -i %s ::%s 2>nul", vhd_target, dir_path);
run_sync(cmd);
return true;
}
// Writes a 512-byte Connectix VHD Fixed Disk Footer at the end of the file.
static void write_vpc_footer(FILE *f, uint64_t disk_size_bytes) {
uint8_t footer[512] = {0};
memcpy(&footer[0], "conectix", 8);
// Features: 0x00000002 (Big Endian)
footer[11] = 0x02;
// File Format Version: 0x00010000
footer[13] = 0x01;
// Data Offset: 0xFFFFFFFFFFFFFFFF (Fixed disk)
memset(&footer[16], 0xFF, 8);
// Creator Application: "qemu"
memcpy(&footer[28], "qemu", 4);
// Creator Version: 0x00020000
footer[33] = 0x02;
// Creator Host OS: "Wi2k"
memcpy(&footer[36], "Wi2k", 4);
// Original & Current Size (uint64_t Big Endian)
for (int i = 0; i < 8; i++) {
uint8_t b = (uint8_t)(disk_size_bytes >> ((7 - i) * 8));
footer[40 + i] = b;
footer[48 + i] = b;
}
// Geometry for 512MB: 1024 Cylinders (0x0400), 16 Heads, 63 Sectors/Track
footer[56] = 0x04;
footer[57] = 0x00;
footer[58] = 16;
footer[59] = 63;
// Disk Type: 2 (Fixed Hard Disk)
footer[63] = 0x02;
// Unique ID GUID
static const uint8_t guid[16] = {
0xA1, 0xB2, 0xC3, 0xD4, 0xE5, 0xF6, 0x78, 0x90,
0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0
};
memcpy(&footer[68], guid, 16);
// Checksum: 1's complement sum of all bytes in footer with checksum cleared
uint32_t sum = 0;
for (int i = 0; i < 512; i++) {
sum += footer[i];
}
uint32_t checksum = ~sum;
footer[64] = (uint8_t)(checksum >> 24);
footer[65] = (uint8_t)(checksum >> 16);
footer[66] = (uint8_t)(checksum >> 8);
footer[67] = (uint8_t)(checksum & 0xFF);
fseek(f, (long)disk_size_bytes, SEEK_SET);
fwrite(footer, 1, 512, f);
}
// Creates a 512 MB partitioned disk.vhd image with a valid VPC footer.
bool create_disk_vhd_if_missing(void) {
if (nob_file_exists("disk.vhd")) return true;
nob_log(NOB_INFO, "disk.vhd missing, creating new 512 MB fixed VHD image...");
const uint64_t disk_size = (uint64_t)512 * 1024 * 1024; // 512 MB Payload
FILE *f = fopen("disk.vhd", "wb");
if (!f) {
nob_log(NOB_ERROR, "Failed to create disk.vhd file!");
return false;
}
// Expand file to 512MB payload
if (fseek(f, (long)(disk_size - 1), SEEK_SET) != 0) {
fclose(f);
return false;
}
fputc(0, f);
// Build standard MBR partition table (Sector 0)
uint8_t mbr[512] = {0};
mbr[510] = 0x55;
mbr[511] = 0xAA;
// Partition 1 Entry: 64MB FAT32 @ LBA 2048 (1MB offset)
uint8_t *p1 = &mbr[0x01BE];
p1[0] = 0x80; // Active / Bootable
p1[1] = 0x00; p1[2] = 0x02; p1[3] = 0x00; // CHS Start
p1[4] = 0x0C; // FAT32 LBA
p1[5] = 0xFE; p1[6] = 0xFF; p1[7] = 0xFF; // CHS End (LBA boundary)
uint32_t p1_lba = 2048;
uint32_t p1_sectors = 131072; // 64 MB
memcpy(&p1[8], &p1_lba, 4);
memcpy(&p1[12], &p1_sectors, 4);
// Partition 2 Entry: 256MB FAT32 @ LBA 133120 (65MB offset)
uint8_t *p2 = &mbr[0x01CE];
p2[0] = 0x00; // Inactive
p2[1] = 0x00; p2[2] = 0x01; p2[3] = 0x08; // CHS Start
p2[4] = 0x0C; // FAT32 LBA
p2[5] = 0xFE; p2[6] = 0xFF; p2[7] = 0xFF; // CHS End (LBA boundary)
uint32_t p2_lba = 133120;
uint32_t p2_sectors = 524288; // 256 MB
memcpy(&p2[8], &p2_lba, 4);
memcpy(&p2[12], &p2_sectors, 4);
// Write Sector 0
fseek(f, 0, SEEK_SET);
fwrite(mbr, 1, 512, f);
// Append 512-byte Connectix VPC Footer
write_vpc_footer(f, disk_size);
fclose(f);
nob_log(NOB_INFO, "Formatting disk.vhd partitions with mformat...");
if (!run_sync("mformat -i " VHD_PART1 " -F")) return false;
if (!run_sync("mformat -i " VHD_PART2 " -F")) return false;
return true;
}
// Runs the requested operation.
int main(int argc, char **argv) {
// The build tool rebuilds itself before doing any real work so the graph below matches the source.
NOB_GO_REBUILD_URSELF(argc, argv);
if (!clean_build_object_directories()) return 1;
create_layout();
if (!create_disk_vhd_if_missing()) return 1;
if (!create_usbfatroot_if_missing()) return 1;
create_layout();
if (!clean_build_object_directories()) return 1;
// Core images build sequentially because later images link against earlier import libraries.
nob_log(NOB_INFO, "--- Building Core Subsystems ---");
const char *core_tasks[] = {
@@ -165,7 +316,6 @@ int main(int argc, char **argv) {
if (!run_sync(core_tasks[i])) return 1;
}
// PCI builds before PCI function drivers because it exports the bus-driver import library.
nob_log(NOB_INFO, "--- Building Bus Drivers ---");
const char *pci_cmd = nob_temp_sprintf(
"cl %s /MP /c /Fo%s\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /IMPLIB:bin\\%s.lib /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\printf.obj obj\\libc.obj %s\\*.obj",
@@ -174,12 +324,9 @@ int main(int argc, char **argv) {
);
if (!run_sync(pci_cmd)) return 1;
// 4. Parallel Phase (Drivers and Applications).
nob_log(NOB_INFO, "--- Compiling Drivers and Apps in Parallel ---");
Nob_Procs procs = {0};
// Queue standalone drivers. Each driver is now a target with one or more
// translation units, but still links to one .sys image with DriverEntry.
for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) {
const Driver_Target *driver = &STANDALONE_DRIVERS[i];
const char *cmd = nob_temp_sprintf(
@@ -190,7 +337,6 @@ int main(int argc, char **argv) {
nob_da_append(&procs, run_async(cmd));
}
// Parse lwIP Sources and Queue it.
Nob_String_Builder sb = {0}, lwip_files = {0};
if (nob_read_entire_file("code\\drivers\\net\\lwip\\lwip_sources.txt", &sb)) {
Nob_String_View sv = nob_sv_from_parts(sb.items, sb.count);
@@ -206,9 +352,7 @@ int main(int argc, char **argv) {
nob_da_append(&procs, run_async(nob_temp_sprintf("cl %s /MP /c /Foobj\\lwip\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\lwip.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj obj\\lwip\\*.obj", LwipCflags, lwip_files.items)));
}
// Queue Subsystem Applications.
const char *apps[][4] = {
// Folder, Libs, Name, Optional Extra Link Args.
{"system\\init", NATIVE_INIT_LIBS_LITERAL, "init", "/ENTRY:_start"},
{"tests\\all_tests", NATIVE_EXE_LIBS_LITERAL, "all_tests", NATIVE_EXE_ENTRY_LITERAL},
{"apps\\shell_win32", WIN32_EXE_LIBS_LITERAL, "shell_win32", NATIVE_EXE_ENTRY_LITERAL},
@@ -239,6 +383,12 @@ int main(int argc, char **argv) {
{"tests\\win32_seh_smoke", WIN32_EXE_LIBS_LITERAL, "win32_seh_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\syscall_contract_smoke", NATIVE_EXE_LIBS_LITERAL, "syscall_contract_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\would_block_smoke", NATIVE_EXE_LIBS_LITERAL, "would_block_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\audio_stream_smoke", NATIVE_EXE_LIBS_LITERAL, "audio_stream_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\winmm_waveout_smoke", WIN32_EXE_LIBS_LITERAL, "winmm_waveout_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\win32_condition_variable_smoke", WIN32_EXE_LIBS_LITERAL, "win32_condition_variable_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\win32_mutex_smoke", WIN32_EXE_LIBS_LITERAL, "win32_mutex_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\win32_thread_suspend_smoke", WIN32_EXE_LIBS_LITERAL, "win32_thread_suspend_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\win32_module_lookup_smoke", WIN32_EXE_LIBS_LITERAL, "win32_module_lookup_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\cwd_peb_smoke", NATIVE_EXE_LIBS_LITERAL, "cwd_peb_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\mmap_smoke", NATIVE_EXE_LIBS_LITERAL, "mmap_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\pcache_smoke", NATIVE_EXE_LIBS_LITERAL, "pcache_smoke", NATIVE_EXE_ENTRY_LITERAL},
@@ -270,98 +420,118 @@ int main(int argc, char **argv) {
nob_da_append(&procs, run_async(cmd));
}
// Wait for everything to link simultaneously.
if (!nob_procs_wait(procs)) {
nob_log(NOB_ERROR, "One or more parallel build jobs failed!");
return 1;
}
// 4. Packing vvfat root.
nob_log(NOB_INFO, "--- Assembling fatroot ---");
if (!create_fatroot_layout()) return 1;
// 4. Formatting and Packing VHD via mtools (No Admin, Fast).
nob_log(NOB_INFO, "--- Formatting & Packing disk.vhd with mtools ---");
run_sync("mformat -i " VHD_PART1 " -F");
run_sync("mformat -i " VHD_PART2 " -F");
const char *copies[][2] = {
{"bin\\bootx64.efi", "fatroot\\EFI\\BOOT\\bootx64.efi"},
{"bin\\kernel.exe", "fatroot\\system\\colonel.exe"},
{"bin\\hal.dll", "fatroot\\system\\hal.dll"},
{"bin\\syscall.dll", "fatroot\\system\\syscall.dll"},
{"bin\\win32.dll", "fatroot\\system\\win32.dll"},
{"bin\\posix.dll", "fatroot\\system\\posix.dll"},
{"code\\settings.ini", "fatroot\\system\\settings.ini"},
{"bin\\init.exe", "fatroot\\system\\init.exe"},
mmd_dir(VHD_PART1, "apps");
mmd_dir(VHD_PART1, "system");
mmd_dir(VHD_PART1, "drivers");
mmd_dir(VHD_PART1, "EFI");
mmd_dir(VHD_PART1, "EFI/BOOT");
mmd_dir(VHD_PART2, "tests");
{"bin\\pci.sys", "fatroot\\drivers\\pci.sys"},
{"bin\\ps2.sys", "fatroot\\drivers\\ps2.sys"},
{"bin\\serial.sys", "fatroot\\drivers\\serial.sys"},
{"bin\\ata.sys", "fatroot\\drivers\\ata.sys"},
{"bin\\ahci.sys", "fatroot\\drivers\\ahci.sys"},
{"bin\\fat.sys", "fatroot\\drivers\\fat.sys"},
{"bin\\ntfs.sys", "fatroot\\drivers\\ntfs.sys"},
{"bin\\vmwaresvga.sys", "fatroot\\drivers\\vmwaresvga.sys"},
{"bin\\lwip.sys", "fatroot\\drivers\\lwip.sys"},
{"bin\\usb.sys", "fatroot\\drivers\\usb.sys"},
{"bin\\rtl816x.sys", "fatroot\\drivers\\rtl816x.sys"},
{"bin\\rtl8139.sys", "fatroot\\drivers\\rtl8139.sys"},
{"code\\drivers\\drivers.ini", "fatroot\\drivers\\drivers.ini"},
{"code\\boot.ini", "fatroot\\boot.ini"},
const char *copies[][3] = {
{"bin\\bootx64.efi", VHD_PART1, "EFI/BOOT/bootx64.efi"},
{"bin\\kernel.exe", VHD_PART1, "system/colonel.exe"},
{"bin\\hal.dll", VHD_PART1, "system/hal.dll"},
{"bin\\syscall.dll", VHD_PART1, "system/syscall.dll"},
{"bin\\win32.dll", VHD_PART1, "system/win32.dll"},
{"bin\\posix.dll", VHD_PART1, "system/posix.dll"},
{"code\\settings.ini", VHD_PART1, "system/settings.ini"},
{"bin\\init.exe", VHD_PART1, "system/init.exe"},
{"bin\\pci.sys", VHD_PART1, "drivers/pci.sys"},
{"bin\\ps2.sys", VHD_PART1, "drivers/ps2.sys"},
{"bin\\serial.sys", VHD_PART1, "drivers/serial.sys"},
{"bin\\hda.sys", VHD_PART1, "drivers/hda.sys"},
{"bin\\ata.sys", VHD_PART1, "drivers/ata.sys"},
{"bin\\ahci.sys", VHD_PART1, "drivers/ahci.sys"},
{"bin\\fat.sys", VHD_PART1, "drivers/fat.sys"},
{"bin\\ntfs.sys", VHD_PART1, "drivers/ntfs.sys"},
{"bin\\vmwaresvga.sys", VHD_PART1, "drivers/vmwaresvga.sys"},
{"bin\\lwip.sys", VHD_PART1, "drivers/lwip.sys"},
{"bin\\usb.sys", VHD_PART1, "drivers/usb.sys"},
{"bin\\rtl816x.sys", VHD_PART1, "drivers/rtl816x.sys"},
{"bin\\rtl8139.sys", VHD_PART1, "drivers/rtl8139.sys"},
{"code\\drivers\\drivers.ini", VHD_PART1, "drivers/drivers.ini"},
{"code\\boot.ini", VHD_PART1, "boot.ini"},
{"bin\\shell_win32.exe", "fatroot\\apps\\shell_win32.exe"},
{"bin\\shell_posix.exe", "fatroot\\apps\\shell_posix.exe"},
{"bin\\shell_native.exe", "fatroot\\apps\\shell_native.exe"},
{"bin\\edit.exe", "fatroot\\apps\\edit.exe"},
{"bin\\httpget.exe", "fatroot\\apps\\httpget.exe"},
{"bin\\tcpserve.exe", "fatroot\\apps\\tcpserve.exe"},
{"bin\\udp.exe", "fatroot\\apps\\udp.exe"},
{"bin\\resolve.exe", "fatroot\\apps\\resolve.exe"},
{"bin\\ping.exe", "fatroot\\apps\\ping.exe"},
{"bin\\doom.exe", "fatroot\\apps\\doom.exe"},
{"bin\\video.exe", "fatroot\\apps\\video.exe"},
{"bin\\memtop.exe", "fatroot\\apps\\memtop.exe"},
{"bin\\shell_win32.exe", VHD_PART1, "apps/shell_win32.exe"},
{"bin\\shell_posix.exe", VHD_PART1, "apps/shell_posix.exe"},
{"bin\\shell_native.exe", VHD_PART1, "apps/shell_native.exe"},
{"bin\\edit.exe", VHD_PART1, "apps/edit.exe"},
{"bin\\httpget.exe", VHD_PART1, "apps/httpget.exe"},
{"bin\\tcpserve.exe", VHD_PART1, "apps/tcpserve.exe"},
{"bin\\udp.exe", VHD_PART1, "apps/udp.exe"},
{"bin\\resolve.exe", VHD_PART1, "apps/resolve.exe"},
{"bin\\ping.exe", VHD_PART1, "apps/ping.exe"},
{"bin\\doom.exe", VHD_PART1, "apps/doom.exe"},
{"bin\\video.exe", VHD_PART1, "apps/video.exe"},
{"bin\\memtop.exe", VHD_PART1, "apps/memtop.exe"},
{"bin\\all_tests.exe", "fatroot\\tests\\all_tests.exe"},
{"bin\\async_apc.exe", "fatroot\\tests\\async_apc.exe"},
{"bin\\async_cancel.exe", "fatroot\\tests\\async_cancel.exe"},
{"bin\\apc_return_smoke.exe", "fatroot\\tests\\apc_return_smoke.exe"},
{"bin\\file_stat_smoke.exe", "fatroot\\tests\\file_stat_smoke.exe"},
{"bin\\handle_dup_smoke.exe", "fatroot\\tests\\handle_dup_smoke.exe"},
{"bin\\named_queue_lifetime_smoke.exe", "fatroot\\tests\\named_queue_lifetime_smoke.exe"},
{"bin\\input_api_smoke.exe", "fatroot\\tests\\input_api_smoke.exe"},
{"bin\\input_model_smoke.exe", "fatroot\\tests\\input_model_smoke.exe"},
{"bin\\timer_smoke.exe", "fatroot\\tests\\timer_smoke.exe"},
{"bin\\stats_abi_smoke.exe", "fatroot\\tests\\stats_abi_smoke.exe"},
{"bin\\realpath_smoke.exe", "fatroot\\tests\\realpath_smoke.exe"},
{"bin\\file_lock_smoke.exe", "fatroot\\tests\\file_lock_smoke.exe"},
{"bin\\process_protection_smoke.exe", "fatroot\\tests\\process_protection_smoke.exe"},
{"bin\\socket_addr_smoke.exe", "fatroot\\tests\\socket_addr_smoke.exe"},
{"bin\\net_irp_smoke.exe", "fatroot\\tests\\net_irp_smoke.exe"},
{"bin\\module_load_smoke.exe", "fatroot\\tests\\module_load_smoke.exe"},
{"bin\\posix_module_smoke.exe", "fatroot\\tests\\posix_module_smoke.exe"},
{"bin\\exception_dispatcher_smoke.exe", "fatroot\\tests\\exception_dispatcher_smoke.exe"},
{"bin\\native_exception_runtime_smoke.exe", "fatroot\\tests\\native_exception_runtime_smoke.exe"},
{"bin\\native_raise_exception_smoke.exe", "fatroot\\tests\\native_raise_exception_smoke.exe"},
{"bin\\exception_smoke.exe", "fatroot\\tests\\exception_smoke.exe"},
{"bin\\win32_exception_smoke.exe", "fatroot\\tests\\win32_exception_smoke.exe"},
{"bin\\win32_seh_smoke.exe", "fatroot\\tests\\win32_seh_smoke.exe"},
{"bin\\syscall_contract_smoke.exe", "fatroot\\tests\\syscall_contract_smoke.exe"},
{"bin\\would_block_smoke.exe", "fatroot\\tests\\would_block_smoke.exe"},
{"bin\\cwd_peb_smoke.exe", "fatroot\\tests\\cwd_peb_smoke.exe"},
{"bin\\mmap_smoke.exe", "fatroot\\tests\\mmap_smoke.exe"},
{"bin\\pcache_smoke.exe", "fatroot\\tests\\pcache_smoke.exe"},
{"bin\\pcache_pressure.exe", "fatroot\\tests\\pcache_pressure.exe"},
{"bin\\dirty_writeback.exe", "fatroot\\tests\\dirty_writeback.exe"},
{"bin\\shared_mmap.exe", "fatroot\\tests\\shared_mmap.exe"},
{"bin\\pcache_invalidate.exe", "fatroot\\tests\\pcache_invalidate.exe"},
{"bin\\pcache_mmap_read_reuse.exe", "fatroot\\tests\\pcache_mmap_read_reuse.exe"},
{"bin\\pcache_multithread.exe", "fatroot\\tests\\pcache_multithread.exe"},
{"bin\\surface_smoke.exe", "fatroot\\tests\\surface_smoke.exe"},
{"bin\\surface_invalid_rect.exe", "fatroot\\tests\\surface_invalid_rect.exe"},
{"bin\\surface_multithread_present.exe", "fatroot\\tests\\surface_multithread_present.exe"}
// Partition 2 Apps
{"bin\\all_tests.exe", VHD_PART2, "tests/all_tests.exe"},
{"bin\\async_apc.exe", VHD_PART2, "tests/async_apc.exe"},
{"bin\\async_cancel.exe", VHD_PART2, "tests/async_cancel.exe"},
{"bin\\apc_return_smoke.exe", VHD_PART2, "tests/apc_return_smoke.exe"},
{"bin\\file_stat_smoke.exe", VHD_PART2, "tests/file_stat_smoke.exe"},
{"bin\\handle_dup_smoke.exe", VHD_PART2, "tests/handle_dup_smoke.exe"},
{"bin\\named_queue_lifetime_smoke.exe", VHD_PART2, "tests/named_queue_lifetime_smoke.exe"},
{"bin\\input_api_smoke.exe", VHD_PART2, "tests/input_api_smoke.exe"},
{"bin\\input_model_smoke.exe", VHD_PART2, "tests/input_model_smoke.exe"},
{"bin\\timer_smoke.exe", VHD_PART2, "tests/timer_smoke.exe"},
{"bin\\stats_abi_smoke.exe", VHD_PART2, "tests/stats_abi_smoke.exe"},
{"bin\\realpath_smoke.exe", VHD_PART2, "tests/realpath_smoke.exe"},
{"bin\\file_lock_smoke.exe", VHD_PART2, "tests/file_lock_smoke.exe"},
{"bin\\process_protection_smoke.exe", VHD_PART2, "tests/process_protection_smoke.exe"},
{"bin\\socket_addr_smoke.exe", VHD_PART2, "tests/socket_addr_smoke.exe"},
{"bin\\net_irp_smoke.exe", VHD_PART2, "tests/net_irp_smoke.exe"},
{"bin\\module_load_smoke.exe", VHD_PART2, "tests/module_load_smoke.exe"},
{"bin\\posix_module_smoke.exe", VHD_PART2, "tests/posix_module_smoke.exe"},
{"bin\\exception_dispatcher_smoke.exe", VHD_PART2, "tests/exception_dispatcher_smoke.exe"},
{"bin\\native_exception_runtime_smoke.exe", VHD_PART2, "tests/native_exception_runtime_smoke.exe"},
{"bin\\native_raise_exception_smoke.exe", VHD_PART2, "tests/native_raise_exception_smoke.exe"},
{"bin\\exception_smoke.exe", VHD_PART2, "tests/exception_smoke.exe"},
{"bin\\win32_exception_smoke.exe", VHD_PART2, "tests/win32_exception_smoke.exe"},
{"bin\\win32_seh_smoke.exe", VHD_PART2, "tests/win32_seh_smoke.exe"},
{"bin\\syscall_contract_smoke.exe", VHD_PART2, "tests/syscall_contract_smoke.exe"},
{"bin\\would_block_smoke.exe", VHD_PART2, "tests/would_block_smoke.exe"},
{"bin\\audio_stream_smoke.exe", VHD_PART2, "tests/audio_stream_smoke.exe"},
{"bin\\winmm_waveout_smoke.exe", VHD_PART2, "tests/winmm_waveout_smoke.exe"},
{"bin\\win32_condition_variable_smoke.exe", VHD_PART2, "tests/win32_condition_variable_smoke.exe"},
{"bin\\win32_mutex_smoke.exe", VHD_PART2, "tests/win32_mutex_smoke.exe"},
{"bin\\win32_thread_suspend_smoke.exe", VHD_PART2, "tests/win32_thread_suspend_smoke.exe"},
{"bin\\win32_module_lookup_smoke.exe", VHD_PART2, "tests/win32_module_lookup_smoke.exe"},
{"bin\\cwd_peb_smoke.exe", VHD_PART2, "tests/cwd_peb_smoke.exe"},
{"bin\\mmap_smoke.exe", VHD_PART2, "tests/mmap_smoke.exe"},
{"bin\\pcache_smoke.exe", VHD_PART2, "tests/pcache_smoke.exe"},
{"bin\\pcache_pressure.exe", VHD_PART2, "tests/pcache_pressure.exe"},
{"bin\\dirty_writeback.exe", VHD_PART2, "tests/dirty_writeback.exe"},
{"bin\\shared_mmap.exe", VHD_PART2, "tests/shared_mmap.exe"},
{"bin\\pcache_invalidate.exe", VHD_PART2, "tests/pcache_invalidate.exe"},
{"bin\\pcache_mmap_read_reuse.exe", VHD_PART2, "tests/pcache_mmap_read_reuse.exe"},
{"bin\\pcache_multithread.exe", VHD_PART2, "tests/pcache_multithread.exe"},
{"bin\\surface_smoke.exe", VHD_PART2, "tests/surface_smoke.exe"},
{"bin\\surface_invalid_rect.exe", VHD_PART2, "tests/surface_invalid_rect.exe"},
{"bin\\surface_multithread_present.exe", VHD_PART2, "tests/surface_multithread_present.exe"}
};
for (size_t i = 0; i < NOB_ARRAY_LEN(copies); i++) {
if (!nob_copy_file(copies[i][0], copies[i][1])) return 1;
mcopy_file(copies[i][0], copies[i][1], copies[i][2]);
}
if (nob_file_exists("code\\user\\apps\\video\\bink2w64.dll")) mcopy_file("code\\user\\apps\\video\\bink2w64.dll", VHD_PART1, "apps/bink2w64.dll");
if (nob_file_exists("code\\user\\apps\\video\\video.bk2")) mcopy_file("code\\user\\apps\\video\\video.bk2", VHD_PART1, "apps/video.bk2");
if (nob_file_exists("code\\user\\apps\\doom\\doom1.wad")) mcopy_file("code\\user\\apps\\doom\\doom1.wad", VHD_PART1, "apps/doom1.wad");
nob_log(NOB_INFO, "Build complete!");
return 0;
}
code/common/libc.c
+25
View File
@@ -20,6 +20,31 @@ char* strncpy(char* dest, const char* src, size_t n) {
return dest;
}
char *strstr(const char *haystack, const char *needle) {
// An empty needle matches the beginning of haystack
if (!*needle) {
return (char *)haystack;
}
for (; *haystack != '\0'; haystack++) {
const char *h = haystack;
const char *n = needle;
// Match as many characters as possible
while (*h != '\0' && *n != '\0' && *h == *n) {
h++;
n++;
}
// If we reached the end of needle, we found a match
if (*n == '\0') {
return (char *)haystack;
}
}
return NULL;
}
// Implements the strcat operation.
char* strcat(char* dest, const char* src) {
char* ptr = dest;
code/drivers/audio/hda/hda.c
+403
View File
@@ -0,0 +1,403 @@
// Intel High Definition Audio PCI attachment and controller lifetime management.
#include "hda.h"
#include <string.h>
const NTL_DDK* gHdaDdk;
static DRIVER_OBJECT gHdaDriver;
static HDA_CONTROLLER* gHdaControllers[HDA_MAX_CONTROLLERS];
static uint32_t gHdaControllerCount;
static SPINLOCK gHdaRegistryLock = SPINLOCK_INIT;
static const IO_DEVICE_ID gHdaIds[] = {
{ IoBusPci, { "PCI\\VEN_8086&DEV_2668", sizeof("PCI\\VEN_8086&DEV_2668") - 1u } },
{ IoBusPci, { "PCI\\VEN_15AD&DEV_1977", sizeof("PCI\\VEN_15AD&DEV_1977") - 1u } },
{ IoBusPci, { "PCI\\CC_040300", sizeof("PCI\\CC_040300") - 1u } },
};
static const IO_DRIVER_ID_TABLE gHdaIdTable = {
gHdaIds,
sizeof(gHdaIds) / sizeof(gHdaIds[0]),
};
// Registers a controller in the shared interrupt scan set.
static KSTATUS HdaRegisterController(HDA_CONTROLLER* controller) {
if (!controller)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock);
for (uint32_t index = 0; index < gHdaControllerCount; index++) {
if (gHdaControllers[index] == controller) {
KeReleaseSpinlock(&gHdaRegistryLock, flags);
return NTLSTATUS_SUCCESS;
}
}
if (gHdaControllerCount == HDA_MAX_CONTROLLERS) {
KeReleaseSpinlock(&gHdaRegistryLock, flags);
return NTLSTATUS_NO_MEMORY;
}
gHdaControllers[gHdaControllerCount++] = controller;
KeReleaseSpinlock(&gHdaRegistryLock, flags);
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the shared interrupt scan set.
static void HdaUnregisterController(HDA_CONTROLLER* controller) {
if (!controller)
return;
uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock);
for (uint32_t index = 0; index < gHdaControllerCount; index++) {
if (gHdaControllers[index] != controller)
continue;
gHdaControllers[index] = gHdaControllers[gHdaControllerCount - 1u];
gHdaControllers[gHdaControllerCount - 1u] = (HDA_CONTROLLER*)0;
gHdaControllerCount--;
break;
}
KeReleaseSpinlock(&gHdaRegistryLock, flags);
}
// Dispatches a hardware interrupt to every live HDA controller.
static void HdaIsr(void) {
DEVICE_OBJECT* devices[HDA_MAX_CONTROLLERS];
uint32_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&gHdaRegistryLock);
count = gHdaControllerCount;
if (count > HDA_MAX_CONTROLLERS)
count = HDA_MAX_CONTROLLERS;
for (uint32_t index = 0; index < count; index++) {
HDA_CONTROLLER* controller = gHdaControllers[index];
devices[index] = controller ? controller->Fdo : (DEVICE_OBJECT*)0;
if (devices[index])
IoReferenceDeviceObject(devices[index]);
}
KeReleaseSpinlock(&gHdaRegistryLock, flags);
for (uint32_t index = 0; index < count; index++) {
DEVICE_OBJECT* device = devices[index];
if (!device)
continue;
HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension;
if (controller && controller->Magic == HDA_CONTROLLER_MAGIC)
HdaHandleInterrupt(controller);
IoDereferenceDeviceObject(device);
}
}
// Registers the global ISR once for each kernel IRQ slot used by HDA.
static KSTATUS HdaRegisterIrq(uint8_t irq) {
if (irq >= 24u)
return NTLSTATUS_NOT_SUPPORTED;
KeIrqRegister(irq, HdaIsr);
return NTLSTATUS_SUCCESS;
}
// Maps BAR0 and validates the stream-descriptor aperture described by GCAP.
static KSTATUS HdaMapRegisters(HDA_CONTROLLER* controller) {
PCI_BAR bar;
KSTATUS status = PciGetBar(controller->Pdo, 0, &bar);
if (!KSUCCESS(status))
return status;
if (bar.Type != PciBarMemory32 && bar.Type != PciBarMemory64)
return NTLSTATUS_NOT_SUPPORTED;
status = PciMapBar(controller->Pdo,
0,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&controller->Registers,
&controller->RegisterBytes);
if (!KSUCCESS(status))
return status;
if (controller->RegisterBytes < HDA_REG_STREAM_BASE)
return NTLSTATUS_BUFFER_TOO_SMALL;
controller->Gcap = HdaRead16(controller, HDA_REG_GCAP);
controller->InputStreamCount =
(uint8_t)((controller->Gcap & HDA_GCAP_ISS_MASK) >> HDA_GCAP_ISS_SHIFT);
controller->OutputStreamCount =
(uint8_t)((controller->Gcap & HDA_GCAP_OSS_MASK) >> HDA_GCAP_OSS_SHIFT);
controller->BidirectionalStreamCount =
(uint8_t)((controller->Gcap & HDA_GCAP_BSS_MASK) >> HDA_GCAP_BSS_SHIFT);
controller->TotalStreamCount =
(uint8_t)(controller->InputStreamCount +
controller->OutputStreamCount +
controller->BidirectionalStreamCount);
if ((controller->OutputStreamCount == 0 &&
controller->BidirectionalStreamCount == 0) ||
controller->TotalStreamCount == 0 ||
controller->TotalStreamCount > 30u)
return NTLSTATUS_NOT_SUPPORTED;
uint64_t requiredBytes = HDA_REG_STREAM_BASE +
(uint64_t)controller->TotalStreamCount *
HDA_REG_STREAM_STRIDE;
if (controller->RegisterBytes < requiredBytes)
return NTLSTATUS_BUFFER_TOO_SMALL;
// Dedicated output descriptors precede bidirectional descriptors. Use the
// first bidirectional descriptor when no output-only descriptor exists.
uint8_t outputDescriptorBase = controller->InputStreamCount;
uint8_t bidirectionalDescriptorBase =
(uint8_t)(controller->InputStreamCount + controller->OutputStreamCount);
controller->Output.DescriptorIndex = controller->OutputStreamCount
? outputDescriptorBase
: bidirectionalDescriptorBase;
controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG;
return NTLSTATUS_SUCCESS;
}
// Configures MSI first and enables INTx only after MSI setup reports failure.
static KSTATUS HdaConfigureInterrupt(HDA_CONTROLLER* controller) {
KSTATUS status = PciEnableMsiAuto(controller->Pdo, &controller->Irq);
if (KSUCCESS(status)) {
controller->UsingMsi = true;
return NTLSTATUS_SUCCESS;
}
gHdaDdk->Debug->Print(
"HDA: MSI setup failed status=0x%x; attempting explicit INTx fallback\n",
status);
controller->UsingMsi = false;
status = PciGetInterrupt(controller->Pdo, &controller->Irq);
if (!KSUCCESS(status))
return status;
if (controller->Irq == 0xFFu || controller->Irq >= 24u)
return NTLSTATUS_NOT_SUPPORTED;
status = PciEnableDevice(controller->Pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX);
return status;
}
// Waits until queued or active interrupt work no longer references the controller.
static KSTATUS HdaWaitForControllerInterruptWorkIdle(HDA_CONTROLLER* controller) {
if (!controller)
return NTLSTATUS_INVALID_PARAMETER;
for (uint32_t elapsed = 0; elapsed < 100000u; elapsed += 10u) {
if (controller->IsrActive == 0 &&
controller->Dpc.Inserted == 0 && controller->DpcActive == 0)
return NTLSTATUS_SUCCESS;
gHdaDdk->Timer->Stall(10);
}
return NTLSTATUS_TIMEOUT;
}
// Starts one PCI HDA controller and leaves endpoint registration to codec discovery.
static KSTATUS HdaStartDevice(HDA_CONTROLLER* controller) {
if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC ||
!controller->Pdo || !controller->Fdo)
return NTLSTATUS_INVALID_PARAMETER;
if (controller->Started)
return NTLSTATUS_SUCCESS;
KSTATUS status = PciGetDeviceInfo(controller->Pdo, &controller->PciInfo);
if (!KSUCCESS(status))
return status;
if (controller->PciInfo.ClassCode != 0x04u ||
controller->PciInfo.Subclass != 0x03u ||
controller->PciInfo.ProgIf != 0x00u)
return NTLSTATUS_NOT_SUPPORTED;
status = PciEnableDevice(controller->Pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_DISABLE_INTX);
if (!KSUCCESS(status))
return status;
status = HdaMapRegisters(controller);
if (!KSUCCESS(status))
return status;
status = HdaConfigureInterrupt(controller);
if (!KSUCCESS(status))
return status;
status = HdaAllocateDma(controller);
if (!KSUCCESS(status))
return status;
KeInitializeDpc(&controller->Dpc, HdaDpcRoutine, controller);
status = HdaStartRecoveryWorker(controller);
if (!KSUCCESS(status)) {
HdaReleaseDma(controller);
return status;
}
status = HdaInitializeController(controller);
if (!KSUCCESS(status)) {
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
status = HdaRegisterController(controller);
if (!KSUCCESS(status)) {
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
status = HdaRegisterIrq(controller->Irq);
if (!KSUCCESS(status)) {
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
controller->Started = true;
controller->Fdo->PnpState = DevicePnpStarted;
status = HdaCodecInitialize(controller);
if (!KSUCCESS(status) && status != NTLSTATUS_NOT_SUPPORTED &&
status != NTLSTATUS_NOT_FOUND) {
controller->Started = false;
controller->Fdo->PnpState = DevicePnpNotStarted;
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
HdaQuiesceController(controller);
HdaReleaseDma(controller);
return status;
}
if (!KSUCCESS(status)) {
gHdaDdk->Debug->Print(
"HDA: no analog endpoint registered status=0x%x\n",
status);
}
HdaArmInterrupts(controller);
HdaHandleInterrupt(controller);
return NTLSTATUS_SUCCESS;
}
// Handles PnP lifetime transitions for one HDA function device object.
static KSTATUS HdaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return NTLSTATUS_INVALID_PARAMETER;
HDA_CONTROLLER* controller = (HDA_CONTROLLER*)device->DeviceExtension;
if (controller->Magic != HDA_CONTROLLER_MAGIC)
return NTLSTATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* stack = IoGetCurrentIrpStackLocation(irp);
if (!stack || stack->MajorFunction != IRP_MJ_PNP)
return NTLSTATUS_INVALID_PARAMETER;
switch ((IO_PNP_MINOR_FUNCTION)stack->MinorFunction) {
case IRP_MN_START_DEVICE:
return HdaStartDevice(controller);
case IRP_MN_QUERY_REMOVE_DEVICE:
case IRP_MN_CANCEL_REMOVE_DEVICE:
return NTLSTATUS_SUCCESS;
case IRP_MN_SURPRISE_REMOVAL:
controller->Started = false;
controller->SurpriseRemoved = true;
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
KSTATUS surpriseDpcStatus =
HdaWaitForControllerInterruptWorkIdle(controller);
if (!KSUCCESS(surpriseDpcStatus)) {
KePanic(KE_PANIC_CODE_DRIVER,
"HDA: interrupt work did not quiesce during surprise removal status=0x%x\n",
surpriseDpcStatus);
}
HdaCodecStop(controller, IoAudioSurpriseRemoval);
HdaCodecDestroy(controller);
device->PnpState = DevicePnpSurpriseRemoved;
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
if (!controller->SurpriseRemoved && controller->Registers)
HdaWrite32(controller, HDA_REG_INTCTL, 0);
controller->Started = false;
HdaUnregisterController(controller);
HdaStopRecoveryWorker(controller);
KSTATUS dpcStatus = HdaWaitForControllerInterruptWorkIdle(controller);
if (!KSUCCESS(dpcStatus)) {
KePanic(KE_PANIC_CODE_DRIVER,
"HDA: interrupt work did not quiesce during removal status=0x%x\n",
dpcStatus);
}
HdaCodecStop(controller,
controller->SurpriseRemoved
? IoAudioSurpriseRemoval
: IoAudioOrderlyRemoval);
HdaCodecDestroy(controller);
if (!controller->SurpriseRemoved)
HdaQuiesceController(controller);
controller->Removed = true;
HdaReleaseDma(controller);
if (!controller->SurpriseRemoved)
(void)PciEnableDevice(controller->Pdo, PCI_DISABLE_INTX);
device->PnpState = DevicePnpDeleted;
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
default:
return IoForwardIrpToLower(device, irp);
}
}
// Attaches one HDA function device object to a matching PCI PDO.
static KSTATUS HdaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject ||
physicalDeviceObject->Type != DeviceObjectPhysical)
return NTLSTATUS_INVALID_PARAMETER;
DEVICE_OBJECT* fdo = (DEVICE_OBJECT*)0;
KSTATUS status = IoCreateDevice(driver, sizeof(HDA_CONTROLLER), &fdo);
if (!KSUCCESS(status))
return status;
(void)IoSetDeviceName(fdo, SYS_SV_LITERAL("hda"));
HDA_CONTROLLER* controller = (HDA_CONTROLLER*)fdo->DeviceExtension;
memset(controller, 0, sizeof(*controller));
controller->Magic = HDA_CONTROLLER_MAGIC;
controller->Fdo = fdo;
controller->Pdo = physicalDeviceObject;
controller->Irq = 0xFFu;
controller->StateLock = (SPINLOCK)SPINLOCK_INIT;
controller->Output.StreamTag = HDA_OUTPUT_STREAM_TAG;
controller->LowerDevice = IoAttachDeviceToDeviceStack(fdo, physicalDeviceObject);
if (!controller->LowerDevice) {
IoDeleteDevice(fdo);
return NTLSTATUS_FAILURE;
}
return NTLSTATUS_SUCCESS;
}
// Registers the Intel HDA transport driver with the I/O manager.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Debug || !ddk->Debug->Print || !ddk->Dma || !ddk->Timer ||
!ddk->Timer->QueryTicks || !ddk->Timer->Stall || !ddk->Audio ||
!ddk->Scheduler || !ddk->Scheduler->DelayThread)
return NTLSTATUS_INVALID_PARAMETER;
gHdaDdk = ddk;
gHdaControllerCount = 0;
memset(gHdaControllers, 0, sizeof(gHdaControllers));
memset(&gHdaDriver, 0, sizeof(gHdaDriver));
IoInitializeDriverObject(&gHdaDriver, SYS_SV_LITERAL("hda"));
gHdaDriver.IdTable = &gHdaIdTable;
gHdaDriver.DriverAddDevice = HdaAddDevice;
gHdaDriver.MajorFunction[IRP_MJ_PNP] = HdaDispatchPnp;
KSTATUS status = IoRegisterDriverObject(&gHdaDriver);
if (!KSUCCESS(status))
gHdaDdk->Debug->Print("HDA: IoRegisterDriverObject failed status=0x%x\n", status);
return status;
}
code/drivers/audio/hda/hda.h
+389
View File
@@ -0,0 +1,389 @@
// Intel High Definition Audio transport declarations shared by the controller and stream implementation.
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddi/pci.h>
#include <ntlite/ddk/ddk.h>
#define HDA_CONTROLLER_MAGIC 0x4144484Eu
#define HDA_MAX_CONTROLLERS 8u
#define HDA_MAX_CODECS 15u
#define HDA_MAX_NODE_ID 0xFFu
#define HDA_MAX_UNSOLICITED_RESPONSES 32u
#define HDA_OUTPUT_SAMPLE_RATE 48000u
#define HDA_OUTPUT_CHANNELS 2u
#define HDA_OUTPUT_VALID_BITS 16u
#define HDA_OUTPUT_BYTES_PER_SAMPLE 2u
#define HDA_OUTPUT_BYTES_PER_FRAME 4u
#define HDA_OUTPUT_PERIOD_FRAMES 512u
#define HDA_OUTPUT_PERIOD_COUNT 4u
#define HDA_OUTPUT_PERIOD_BYTES (HDA_OUTPUT_PERIOD_FRAMES * HDA_OUTPUT_BYTES_PER_FRAME)
#define HDA_OUTPUT_BUFFER_BYTES (HDA_OUTPUT_PERIOD_COUNT * HDA_OUTPUT_PERIOD_BYTES)
#define HDA_OUTPUT_STREAM_FORMAT 0x0011u
#define HDA_OUTPUT_STREAM_TAG 1u
#define HDA_MAX_LOGICAL_ENDPOINTS 3u
#define HDA_REG_GCAP 0x00u
#define HDA_REG_VMIN 0x02u
#define HDA_REG_VMAJ 0x03u
#define HDA_REG_GCTL 0x08u
#define HDA_REG_STATESTS 0x0Eu
#define HDA_REG_INTCTL 0x20u
#define HDA_REG_INTSTS 0x24u
#define HDA_REG_WALLCLK 0x30u
#define HDA_REG_CORBLBASE 0x40u
#define HDA_REG_CORBUBASE 0x44u
#define HDA_REG_CORBWP 0x48u
#define HDA_REG_CORBRP 0x4Au
#define HDA_REG_CORBCTL 0x4Cu
#define HDA_REG_CORBSTS 0x4Du
#define HDA_REG_CORBSIZE 0x4Eu
#define HDA_REG_RIRBLBASE 0x50u
#define HDA_REG_RIRBUBASE 0x54u
#define HDA_REG_RIRBWP 0x58u
#define HDA_REG_RINTCNT 0x5Au
#define HDA_REG_RIRBCTL 0x5Cu
#define HDA_REG_RIRBSTS 0x5Du
#define HDA_REG_RIRBSIZE 0x5Eu
#define HDA_REG_DPLBASE 0x70u
#define HDA_REG_DPUBASE 0x74u
#define HDA_REG_STREAM_BASE 0x80u
#define HDA_REG_STREAM_STRIDE 0x20u
#define HDA_GCAP_64OK 0x0001u
#define HDA_GCAP_BSS_MASK 0x00F8u
#define HDA_GCAP_BSS_SHIFT 3u
#define HDA_GCAP_ISS_MASK 0x0F00u
#define HDA_GCAP_ISS_SHIFT 8u
#define HDA_GCAP_OSS_MASK 0xF000u
#define HDA_GCAP_OSS_SHIFT 12u
#define HDA_GCTL_RESET 0x00000001u
#define HDA_GCTL_UNSOLICITED_ENABLE 0x00000100u
#define HDA_INT_CONTROLLER_ENABLE 0x40000000u
#define HDA_INT_GLOBAL_ENABLE 0x80000000u
#define HDA_INT_CONTROLLER_STATUS 0x40000000u
#define HDA_INT_GLOBAL_STATUS 0x80000000u
#define HDA_CORBRP_POINTER_MASK 0x00FFu
#define HDA_CORBRP_RESET 0x8000u
#define HDA_CORBCTL_MEMORY_ERROR_INTERRUPT 0x01u
#define HDA_CORBCTL_RUN 0x02u
#define HDA_CORBSTS_MEMORY_ERROR 0x01u
#define HDA_RIRBWP_RESET 0x8000u
#define HDA_RIRBCTL_RESPONSE_INTERRUPT 0x01u
#define HDA_RIRBCTL_RUN 0x02u
#define HDA_RIRBCTL_OVERRUN_INTERRUPT 0x04u
#define HDA_RIRBSTS_RESPONSE 0x01u
#define HDA_RIRBSTS_OVERRUN 0x04u
#define HDA_RIRB_RESPONSE_CODEC_MASK 0x0000000Fu
#define HDA_RIRB_RESPONSE_UNSOLICITED 0x00000010u
#define HDA_RING_SIZE_CAP_2 0x10u
#define HDA_RING_SIZE_CAP_16 0x20u
#define HDA_RING_SIZE_CAP_256 0x40u
#define HDA_RING_SIZE_SELECT_2 0x00u
#define HDA_RING_SIZE_SELECT_16 0x01u
#define HDA_RING_SIZE_SELECT_256 0x02u
#define HDA_DPLBASE_ENABLE 0x00000001u
#define HDA_STREAM_REG_CTL 0x00u
#define HDA_STREAM_REG_STS 0x03u
#define HDA_STREAM_REG_LPIB 0x04u
#define HDA_STREAM_REG_CBL 0x08u
#define HDA_STREAM_REG_LVI 0x0Cu
#define HDA_STREAM_REG_FORMAT 0x12u
#define HDA_STREAM_REG_BDLPL 0x18u
#define HDA_STREAM_REG_BDLPU 0x1Cu
#define HDA_STREAM_CTL_RESET 0x000001u
#define HDA_STREAM_CTL_RUN 0x000002u
#define HDA_STREAM_CTL_INTERRUPT_COMPLETE 0x000004u
#define HDA_STREAM_CTL_INTERRUPT_FIFO_ERROR 0x000008u
#define HDA_STREAM_CTL_INTERRUPT_DESCRIPTOR_ERROR 0x000010u
#define HDA_STREAM_CTL_INTERRUPT_MASK 0x00001Cu
#define HDA_STREAM_CTL_TAG_SHIFT 20u
#define HDA_STREAM_CTL_TAG_MASK 0x00F00000u
#define HDA_STREAM_STS_COMPLETE 0x04u
#define HDA_STREAM_STS_FIFO_ERROR 0x08u
#define HDA_STREAM_STS_DESCRIPTOR_ERROR 0x10u
#define HDA_STREAM_STS_ERROR_MASK 0x18u
#define HDA_STREAM_STS_INTERRUPT_MASK 0x1Cu
#define HDA_BDL_FLAG_INTERRUPT_ON_COMPLETION 0x00000001u
#define HDA_VERB_GET_PARAMETER(parameterId) (0xF0000u | ((uint32_t)(parameterId) & 0xFFu))
#define HDA_PARAMETER_VENDOR_ID 0x00u
#define HDA_CONTROLLER_RESET_TIMEOUT_US 100000u
#define HDA_STREAM_RESET_TIMEOUT_US 10000u
#define HDA_RING_RESET_TIMEOUT_US 10000u
#define HDA_CODEC_COMMAND_TIMEOUT_US 100000u
#define HDA_CODEC_COMMAND_ATTEMPTS 2u
#define HDA_CODEC_RESET_SETTLE_US 1000u
#define HDA_STREAM_ERROR_WINDOW_US 1000000ull
#define HDA_STREAM_ERRORS_BEFORE_CONTROLLER_RESET 3u
#define HDA_RECOVERY_NONE 0u
#define HDA_RECOVERY_STREAM 1u
#define HDA_RECOVERY_CONTROLLER 2u
#pragma pack(push, 1)
typedef struct HDA_BDL_ENTRY {
uint64_t Address;
uint32_t Length;
uint32_t Flags;
} HDA_BDL_ENTRY;
typedef struct HDA_RIRB_ENTRY {
uint32_t Response;
uint32_t ResponseExtra;
} HDA_RIRB_ENTRY;
#pragma pack(pop)
_Static_assert(sizeof(HDA_BDL_ENTRY) == 16u, "HDA BDL entries must be 16 bytes");
_Static_assert(sizeof(HDA_RIRB_ENTRY) == 8u, "HDA RIRB entries must be 8 bytes");
typedef struct HDA_CONTROLLER HDA_CONTROLLER;
typedef struct HDA_CODEC_STATE HDA_CODEC_STATE;
typedef struct HDA_OUTPUT_DESCRIPTION {
void* PeriodBuffer;
size_t PeriodBufferBytes;
uint32_t PeriodCount;
uint32_t FramesPerPeriod;
size_t PeriodStrideBytes;
IO_AUDIO_FORMAT HardwareFormat;
uint8_t StreamTag;
} HDA_OUTPUT_DESCRIPTION;
typedef struct HDA_TRANSPORT_CALLBACKS {
// Reports one BDL-owned period after completion interrupt processing.
void (*PeriodComplete)(void* context,
uint32_t completedPeriodIndex,
uint64_t presentedFrames,
uint64_t finePositionFrames,
uint64_t interruptCount,
uint64_t timestampTicks);
// Delivers one unsolicited codec response outside the controller lock.
void (*UnsolicitedResponse)(void* context,
uint8_t codecAddress,
uint32_t response,
uint32_t responseExtra);
// Restores codec programming in a muted state while transport start and stop are serialized.
KSTATUS (*RestoreAfterControllerReset)(void* context);
// Unmutes the restored route only after controller recovery has restarted DMA.
KSTATUS (*ResumeAfterControllerReset)(void* context);
// Reports a recovery failure that leaves the transport stopped.
void (*TransportFailure)(void* context, KSTATUS status);
} HDA_TRANSPORT_CALLBACKS;
typedef struct HDA_UNSOLICITED_RESPONSE {
uint32_t Response;
uint32_t ResponseExtra;
} HDA_UNSOLICITED_RESPONSE;
typedef struct HDA_OUTPUT_RING {
IO_DMA_BUFFER* Dma;
uint8_t* Buffer;
uint64_t DeviceAddress;
uint8_t NextLogicalPeriod;
uint64_t PresentedFrames;
} HDA_OUTPUT_RING;
typedef struct HDA_OUTPUT_STREAM {
uint8_t DescriptorIndex;
uint8_t StreamTag;
uint8_t BdlPeriodMap[HDA_OUTPUT_PERIOD_COUNT];
uint8_t NextCompletionBdl;
bool Prepared;
bool Running;
volatile long PendingPeriodInterrupts;
volatile long PendingDmaErrors;
HDA_OUTPUT_RING Rings[HDA_MAX_LOGICAL_ENDPOINTS];
uint8_t ActiveRingIndex;
IO_DMA_BUFFER* BdlDma;
HDA_BDL_ENTRY* Bdl;
uint64_t BdlDeviceAddress;
uint64_t PositionRunBaseFrames;
uint64_t LastPositionFrames;
uint64_t PositionWrapFrames;
uint32_t LastHardwarePositionBytes;
bool PositionInitialized;
uint64_t InterruptCount;
uint64_t PeriodCompletionCount;
uint64_t DmaErrorCount;
uint64_t StreamResetCount;
uint64_t PositionBufferReadCount;
uint64_t LpibReadCount;
uint64_t PositionDiscontinuityCount;
} HDA_OUTPUT_STREAM;
struct HDA_CONTROLLER {
uint32_t Magic;
DEVICE_OBJECT* Fdo;
DEVICE_OBJECT* Pdo;
DEVICE_OBJECT* LowerDevice;
PCI_DEVICE_INFO PciInfo;
volatile uint8_t* Registers;
uint64_t RegisterBytes;
uint16_t Gcap;
uint8_t InputStreamCount;
uint8_t OutputStreamCount;
uint8_t BidirectionalStreamCount;
uint8_t TotalStreamCount;
uint16_t CodecMask;
uint8_t Irq;
bool UsingMsi;
bool Started;
bool Removed;
bool SurpriseRemoved;
SPINLOCK StateLock;
KDPC Dpc;
volatile long IsrActive;
volatile long DpcActive;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* CorbDma;
IO_DMA_BUFFER* RirbDma;
IO_DMA_BUFFER* PositionDma;
uint32_t* Corb;
HDA_RIRB_ENTRY* Rirb;
volatile uint64_t* PositionBuffer;
uint64_t CorbDeviceAddress;
uint64_t RirbDeviceAddress;
uint64_t PositionDeviceAddress;
uint16_t CorbEntryCount;
uint16_t RirbEntryCount;
uint16_t CorbWritePointer;
uint16_t RirbReadPointer;
MUTEX* CommandMutex;
MUTEX* StreamMutex;
bool CommandWaiting;
bool CommandResponseReady;
uint8_t CommandCodecAddress;
uint32_t CommandResponse;
uint32_t CommandResponseExtra;
uint64_t CodecCommandCount;
uint64_t CodecCommandRetryCount;
uint64_t CodecCommandFailureCount;
uint64_t RirbResponseCount;
uint64_t RirbUnsolicitedCount;
uint64_t RirbOverrunCount;
uint64_t CorbMemoryErrorCount;
HDA_UNSOLICITED_RESPONSE Unsolicited[HDA_MAX_UNSOLICITED_RESPONSES];
uint8_t UnsolicitedRead;
uint8_t UnsolicitedWrite;
uint8_t UnsolicitedCount;
HDA_OUTPUT_STREAM Output;
HDA_TRANSPORT_CALLBACKS Callbacks;
void* CallbackContext;
SEMAPHORE* RecoverySemaphore;
EVENT* RecoveryFinished;
THREAD* RecoveryThread;
bool RecoveryWorkerRunning;
bool RecoveryInProgress;
uint8_t PendingRecovery;
uint64_t StreamErrorWindowStart;
uint32_t StreamErrorsInWindow;
uint64_t ControllerResetCount;
HDA_CODEC_STATE* Codec;
uint64_t InterruptCount;
uint64_t ControllerInterruptCount;
};
extern const NTL_DDK* gHdaDdk;
// Returns one byte from the controller register aperture.
uint8_t HdaRead8(const HDA_CONTROLLER* controller, uint32_t offset);
// Returns one word from the controller register aperture.
uint16_t HdaRead16(const HDA_CONTROLLER* controller, uint32_t offset);
// Returns one dword from the controller register aperture.
uint32_t HdaRead32(const HDA_CONTROLLER* controller, uint32_t offset);
// Writes one byte to the controller register aperture.
void HdaWrite8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t value);
// Writes one word to the controller register aperture.
void HdaWrite16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t value);
// Writes one dword to the controller register aperture.
void HdaWrite32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t value);
// Returns the register offset of the controller's one output stream descriptor.
uint32_t HdaOutputStreamBase(const HDA_CONTROLLER* controller);
// Allocates the transport and output DMA resources for one controller.
KSTATUS HdaAllocateDma(HDA_CONTROLLER* controller);
// Releases all DMA and synchronization resources owned by one controller.
void HdaReleaseDma(HDA_CONTROLLER* controller);
// Resets and initializes the controller, command rings, position buffer, and output descriptor.
KSTATUS HdaInitializeController(HDA_CONTROLLER* controller);
// Arms controller and output-stream interrupts after ISR registration.
void HdaArmInterrupts(HDA_CONTROLLER* controller);
// Stops hardware access without releasing software resources.
void HdaQuiesceController(HDA_CONTROLLER* controller);
// Sends one 20-bit codec verb payload through CORB/RIRB with one retry.
KSTATUS HdaCodecVerb(HDA_CONTROLLER* controller,
uint8_t codecAddress,
uint8_t nodeId,
uint32_t verbPayload,
uint32_t* responseOut);
// Copies one logical endpoint ring contract for codec and endpoint setup.
KSTATUS HdaDescribeOutput(HDA_CONTROLLER* controller,
uint8_t ringIndex,
HDA_OUTPUT_DESCRIPTION* descriptionOut);
// Selects one logical endpoint ring while the stream is stopped.
KSTATUS HdaSelectOutputRing(HDA_CONTROLLER* controller, uint8_t ringIndex);
// Replaces the transport callbacks while the output stream is stopped.
KSTATUS HdaSetTransportCallbacks(HDA_CONTROLLER* controller,
const HDA_TRANSPORT_CALLBACKS* callbacks,
void* context);
// Starts the prepared fixed-format output stream.
KSTATUS HdaStartOutput(HDA_CONTROLLER* controller);
// Stops the output stream while retaining its DMA ring.
void HdaStopOutput(HDA_CONTROLLER* controller);
// Returns the monotonic fine position derived from the position buffer or LPIB.
uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller);
// Captures one controller interrupt and queues deferred processing.
void HdaHandleInterrupt(HDA_CONTROLLER* controller);
// Runs period delivery, unsolicited callbacks, and recovery scheduling at DPC level.
void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2);
// Starts the passive recovery worker for one controller.
KSTATUS HdaStartRecoveryWorker(HDA_CONTROLLER* controller);
// Stops and joins the passive recovery worker for one controller.
void HdaStopRecoveryWorker(HDA_CONTROLLER* controller);
// Discovers codec topology and registers usable analog render endpoints.
KSTATUS HdaCodecInitialize(HDA_CONTROLLER* controller);
// Stops jack workers and unregisters codec endpoints.
void HdaCodecStop(HDA_CONTROLLER* controller, IO_AUDIO_REMOVAL_MODE mode);
// Releases codec topology storage after interrupt work is quiescent.
void HdaCodecDestroy(HDA_CONTROLLER* controller);
code/drivers/audio/hda/hda_codec.c
+1928
View File
File diff suppressed because it is too large. Load diff
code/drivers/audio/hda/hda_transport.c
+1421
View File
File diff suppressed because it is too large. Load diff
code/drivers/bus/pci/pci.c
+1 -1
View File
@@ -33,7 +33,7 @@ static uint64_t PciAlignUp64(uint64_t value, uint64_t alignment) {
// Implements the default Mmio Flags operation.
static uint64_t PciDefaultMmioFlags(void) {
return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_WRITETHROUGH | NTL_DDK_PAGE_NOCACHE | NTL_DDK_PAGE_NX;
return NTL_DDK_PAGE_PRESENT | NTL_DDK_PAGE_WRITE | NTL_DDK_PAGE_CACHE_UC | NTL_DDK_PAGE_NX;
}
// Creates mmio Va.
code/drivers/bus/usb/usb.c
+1 -1
View File
@@ -37,7 +37,7 @@ static KSTATUS UsbInitializeXhciDriver(void) {
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER;
if (!ddk || !ddk->Dma || !ddk->Timer) return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("usb: driver loading\n");
gUsbDdk = ddk;
code/drivers/bus/usb/usb.h
+83 -40
View File
@@ -124,6 +124,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define XHCI_TRB_SETUP_TRT_IN (3u << 16)
#define XHCI_TRB_STATUS_DIR_IN (1u << 16)
#define XHCI_HCC1_AC64 (1u << 0)
#define XHCI_HCC1_CSZ (1u << 2)
#define XHCI_CONTEXT_ENTRIES(n) ((uint32_t)(n) << 27)
@@ -199,6 +200,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define SCSI_INQUIRY 0x12u
#define SCSI_READ_10 0x28u
#define SCSI_WRITE_10 0x2Au
#define SCSI_SYNCHRONIZE_CACHE_10 0x35u
#define SCSI_READ_CAPACITY_10 0x25u
#define XHCI_RING_TRBS 256u
@@ -393,7 +395,11 @@ typedef struct USB_HCD_OPERATIONS {
KSTATUS (*CancelUrb)(USB_HCD* hcd, USB_URB* urb);
KSTATUS (*ResetRootPort)(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed);
KSTATUS (*ClearRootPortChanges)(USB_HCD* hcd, uint8_t port);
void* (*AllocateDma)(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys);
void (*DestroyPipe)(USB_HCD* hcd, USB_PIPE* pipe);
void* (*AllocateDma)(USB_HCD* hcd,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer);
} USB_HCD_OPERATIONS;
struct USB_HCD {
@@ -454,7 +460,7 @@ struct USB_URB {
USB_PIPE* Pipe;
USB_SETUP_PACKET Setup;
void* Buffer;
PHYSICAL_ADDRESS BufferPhys;
uint64_t BufferDeviceAddress;
uint32_t Length;
uint32_t ActualLength;
uint32_t HcdCompletionCode;
@@ -521,8 +527,9 @@ typedef struct USB_HUB_EXTENSION {
uint32_t PendingHubPorts[1];
SPINLOCK PendingLock;
SEMAPHORE* WorkerSemaphore;
EVENT* WorkerFinished;
THREAD* WorkerThread;
bool WorkerRunning;
volatile bool WorkerRunning;
bool InitialScanPending;
bool InitialScanComplete;
bool InitialScanInProgress;
@@ -533,7 +540,8 @@ typedef struct USB_HUB_EXTENSION {
uint8_t InterruptInterval;
USB_PIPE* InterruptPipe;
uint8_t* ChangeBitmap;
PHYSICAL_ADDRESS ChangeBitmapPhys;
uint64_t ChangeBitmapDeviceAddress;
IO_DMA_BUFFER* ChangeBitmapDma;
USB_URB ChangeUrb;
bool ChangeUrbPending;
uint8_t PowerOnToPowerGoodMs;
@@ -573,7 +581,8 @@ typedef struct USB_HID_EXTENSION {
USB_PIPE* InterruptPipe;
uint8_t* ReportBuffer;
PHYSICAL_ADDRESS ReportBufferPhys;
uint64_t ReportBufferDeviceAddress;
IO_DMA_BUFFER* ReportBufferDma;
uint16_t ReportLength;
USB_URB InterruptUrb;
bool InterruptPending;
@@ -584,6 +593,12 @@ typedef struct USB_HID_EXTENSION {
uint32_t InputDeviceId;
} USB_HID_EXTENSION;
typedef enum USB_STORAGE_FLUSH_CAPABILITY {
UsbStorageFlushUnknown = 0,
UsbStorageFlushSupported,
UsbStorageFlushUnsupported,
} USB_STORAGE_FLUSH_CAPABILITY;
typedef enum USB_STORAGE_EXTENSION_KIND {
UsbStorageExtensionInterface = 0x55534946u,
UsbStorageExtensionDisk = 0x5553444Bu,
@@ -620,6 +635,7 @@ struct USB_STORAGE_EXTENSION {
uint32_t Tag;
uint32_t BlockSize;
uint64_t BlockCount;
USB_STORAGE_FLUSH_CAPABILITY FlushCapability;
volatile long IoBusy;
};
@@ -630,14 +646,15 @@ struct USB_STORAGE_DISK_EXTENSION {
typedef struct XHCI_PIPE_CONTEXT {
XHCI_TRB* Ring;
PHYSICAL_ADDRESS RingPhys;
uint64_t RingDeviceAddress;
IO_DMA_BUFFER* RingDma;
uint32_t Enqueue;
uint8_t Cycle;
uint8_t EndpointId;
USB_PIPE_TYPE Type;
bool Configured;
bool ActiveTransfer;
uint64_t ActiveTrbPhys;
uint64_t ActiveTrbDeviceAddress;
USB_URB* ActiveUrb;
} XHCI_PIPE_CONTEXT;
@@ -683,7 +700,7 @@ typedef struct XHCI_DEVICE {
uint8_t HidLastModifiers;
uint8_t HidLastKeys[6];
uint8_t HidLastButtons;
uint64_t HidLastTrbPhys;
uint64_t HidLastTrbDeviceAddress;
bool MassCandidate;
bool MassActive;
@@ -708,15 +725,25 @@ typedef struct XHCI_DEVICE {
bool MassRemoved;
volatile long MassIoBusy;
PHYSICAL_ADDRESS InputContextPhys;
PHYSICAL_ADDRESS DeviceContextPhys;
PHYSICAL_ADDRESS Ep0RingPhys;
PHYSICAL_ADDRESS HidRingPhys;
PHYSICAL_ADDRESS MassInRingPhys;
PHYSICAL_ADDRESS MassOutRingPhys;
PHYSICAL_ADDRESS DescriptorPhys;
PHYSICAL_ADDRESS HidBufferPhys;
PHYSICAL_ADDRESS MassBufferPhys;
uint64_t InputContextDeviceAddress;
uint64_t DeviceContextDeviceAddress;
uint64_t Ep0RingDeviceAddress;
uint64_t HidRingDeviceAddress;
uint64_t MassInRingDeviceAddress;
uint64_t MassOutRingDeviceAddress;
uint64_t DescriptorDeviceAddress;
uint64_t HidBufferDeviceAddress;
uint64_t MassBufferDeviceAddress;
IO_DMA_BUFFER* InputContextDma;
IO_DMA_BUFFER* DeviceContextDma;
IO_DMA_BUFFER* Ep0RingDma;
IO_DMA_BUFFER* HidRingDma;
IO_DMA_BUFFER* MassInRingDma;
IO_DMA_BUFFER* MassOutRingDma;
IO_DMA_BUFFER* DescriptorDma;
IO_DMA_BUFFER* HidBufferDma;
IO_DMA_BUFFER* MassBufferDma;
uint8_t* InputContext;
uint8_t* DeviceContext;
@@ -746,8 +773,8 @@ typedef struct XHCI_TRANSFER_RESULT {
typedef struct XHCI_WAIT_TARGET {
struct XHCI_WAIT_TARGET* Next;
uint64_t CommandPhys;
uint64_t TransferPhys;
uint64_t CommandDeviceAddress;
uint64_t TransferDeviceAddress;
uint8_t TransferSlotId;
uint8_t TransferEndpointId;
volatile bool CommandMatched;
@@ -793,12 +820,20 @@ typedef struct XHCI_CONTROLLER {
uint8_t PortProtocolMajor[XHCI_MAX_ROOT_PORTS + 1u];
uint8_t PortProtocolMinor[XHCI_MAX_ROOT_PORTS + 1u];
PHYSICAL_ADDRESS DcbaaPhys;
PHYSICAL_ADDRESS ScratchpadArrayPhys;
PHYSICAL_ADDRESS ScratchpadPhys[XHCI_MAX_SCRATCHPADS];
PHYSICAL_ADDRESS CommandRingPhys;
PHYSICAL_ADDRESS EventRingPhys;
PHYSICAL_ADDRESS ErstPhys;
uint64_t DcbaaDeviceAddress;
uint64_t ScratchpadArrayDeviceAddress;
uint64_t ScratchpadDeviceAddress[XHCI_MAX_SCRATCHPADS];
uint64_t CommandRingDeviceAddress;
uint64_t EventRingDeviceAddress;
uint64_t ErstDeviceAddress;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* DcbaaDma;
IO_DMA_BUFFER* ScratchpadArrayDma;
IO_DMA_BUFFER* ScratchpadDma[XHCI_MAX_SCRATCHPADS];
IO_DMA_BUFFER* CommandRingDma;
IO_DMA_BUFFER* EventRingDma;
IO_DMA_BUFFER* ErstDma;
uint64_t* Dcbaa;
uint64_t* ScratchpadArray;
@@ -825,6 +860,7 @@ typedef struct XHCI_CONTROLLER {
XHCI_WAIT_TARGET* WaiterList;
volatile bool Running;
bool SurpriseRemoved;
USB_HCD Hcd;
} XHCI_CONTROLLER;
@@ -867,7 +903,11 @@ void UsbDereferencePipe(USB_PIPE* pipe);
KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb);
KSTATUS UsbCancelUrb(USB_URB* urb);
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys);
void* UsbAllocateDmaBuffer(USB_DEVICE* dev,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer);
void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer);
KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp);
KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp);
@@ -882,7 +922,11 @@ void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value);
void UsbDelayUs(const NTL_DDK* ddk, uint32_t us);
bool UsbWaitSet32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys);
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer);
void XhciReleaseDeviceDma(XHCI_DEVICE* dev);
uint16_t XhciScratchpadCount(uint32_t hcs2);
uint32_t XhciTrbType(const XHCI_TRB* trb);
uint32_t XhciCompletionCode(const XHCI_TRB* trb);
@@ -918,7 +962,7 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci);
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci);
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci);
KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci);
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index);
uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index);
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, XHCI_COMMAND_RESULT* outResult);
KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci);
KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId);
@@ -931,17 +975,17 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed);
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId);
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index);
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length, uint32_t timeoutUs);
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length);
uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index);
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress);
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length, uint32_t timeoutUs);
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, uint64_t bufferDeviceAddress, uint16_t length);
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup);
KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc, uint32_t timeoutUs);
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc);
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength);
uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress);
uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval);
KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outDeviceAddress);
bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key);
INPUT_KEY XhciHidModifierKey(uint8_t bit);
void XhciSubmitHidKey(INPUT_KEY key, bool pressed);
@@ -949,24 +993,23 @@ void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint
void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len);
KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
void XhciArmAllHid(XHCI_CONTROLLER* xhci);
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue);
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue);
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci);
KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject);
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp);
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle);
void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t trbCount, uint8_t cycle);
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe);
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, uint64_t ringDeviceAddress, uint32_t* enqueue, uint8_t* cycle, uint64_t bufferDeviceAddress, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage, uint64_t deviceAddress, uint32_t length, uint32_t* outActual);
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage, const uint8_t* cdb, uint8_t cdbLen, void* data, uint64_t dataDeviceAddress, uint32_t dataLength, bool dataIn);
uint32_t UsbBe32(const uint8_t* p);
void UsbStoreBe32(uint8_t* p, uint32_t v);
void UsbStoreBe16(uint8_t* p, uint16_t v);
KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage);
KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage, bool write, uint64_t lba, uint32_t sectors, void* buffer);
code/drivers/bus/usb/usb_bus.c
+19 -6
View File
@@ -24,9 +24,12 @@ static void UsbDeviceDestroy(OBJECT* object) {
}
}
// Implements the pipe Destroy operation.
// Destroys a pipe after the host controller releases its private context.
static void UsbPipeDestroy(OBJECT* object) {
USB_PIPE* pipe = (USB_PIPE*)object;
if (pipe->Device && pipe->Device->Hcd && pipe->Device->Hcd->Operations &&
pipe->Device->Hcd->Operations->DestroyPipe)
pipe->Device->Hcd->Operations->DestroyPipe(pipe->Device->Hcd, pipe);
if (pipe->Device) {
UsbDereferenceDevice(pipe->Device);
pipe->Device = (USB_DEVICE*)0;
@@ -53,11 +56,21 @@ void UsbDereferencePipe(USB_PIPE* pipe) {
if (pipe) ObDereferenceObject(&pipe->Header);
}
// Creates dma Buffer.
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma)
// Allocates a DMA buffer through the device host-controller adapter.
void* UsbAllocateDmaBuffer(USB_DEVICE* dev,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!dev || !dev->Hcd || !dev->Hcd->Operations ||
!dev->Hcd->Operations->AllocateDma || !outDeviceAddress || !outBuffer)
return (void*)0;
return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys);
return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outDeviceAddress, outBuffer);
}
// Frees a DMA buffer allocated for a USB class driver.
void UsbFreeDmaBuffer(IO_DMA_BUFFER* buffer) {
if (buffer && gUsbDdk && gUsbDdk->Dma)
gUsbDdk->Dma->FreeCommonBuffer(buffer);
}
// Creates device.
@@ -464,7 +477,7 @@ KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) {
if (!parent)
return NTLSTATUS_INVALID_PARAMETER;
for (;;) {
while (true) {
DEVICE_OBJECT* child = UsbReferenceFirstChild(parent);
if (!child)
break;
code/drivers/bus/usb/usb_core.c
+4 -3
View File
@@ -258,6 +258,9 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
XHCI_DEVICE* dev = &xhci->Devices[slotId];
if (dev->Allocated && xhci->Running)
(void)XhciDisableSlot(xhci, slotId);
if (dev->UsbPdo) {
(void)UsbRemovePdoSubtree(dev->UsbPdo, true);
dev->UsbPdo = (DEVICE_OBJECT*)0;
@@ -269,9 +272,7 @@ static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
dev->UsbDevice = (USB_DEVICE*)0;
}
if (dev->Allocated)
(void)XhciDisableSlot(xhci, slotId);
XhciReleaseDeviceDma(dev);
memset(dev, 0, sizeof(*dev));
}
code/drivers/bus/usb/usb_hid.c
+49 -20
View File
@@ -2,6 +2,16 @@
#include "usb.h"
// Releases the HID class DMA buffer after all pending transfers are cancelled.
static void UsbHidReleaseDma(USB_HID_EXTENSION* ext) {
if (!ext)
return;
UsbFreeDmaBuffer(ext->ReportBufferDma);
ext->ReportBufferDma = (IO_DMA_BUFFER*)0;
ext->ReportBuffer = (uint8_t*)0;
ext->ReportBufferDeviceAddress = 0;
}
// Implements the hid Key In Report operation.
bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) {
if (key == 0 || key == 1) return false;
@@ -91,21 +101,21 @@ KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
memset(dev->HidBuffer, 0, dev->HidReportLength);
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciRingAppend(dev->HidRing,
dev->HidRingPhys,
dev->HidRingDeviceAddress,
XHCI_HID_RING_TRBS,
&dev->HidEnqueue,
&dev->HidCycle,
dev->HidBufferPhys,
dev->HidBufferDeviceAddress,
dev->HidReportLength,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
dev->HidLastTrbPhys = trbPhys;
dev->HidLastTrbDeviceAddress = trbDeviceAddress;
dev->HidArmed = true;
XhciBarrier();
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, dev->HidEndpointId);
@@ -125,13 +135,13 @@ void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
}
}
static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return;
XHCI_DEVICE* dev = &xhci->Devices[slotId];
if (!dev->HidActive || endpointId != dev->HidEndpointId || trbPhys != dev->HidLastTrbPhys)
if (!dev->HidActive || endpointId != dev->HidEndpointId || trbDeviceAddress != dev->HidLastTrbDeviceAddress)
return;
dev->HidArmed = false;
@@ -200,17 +210,26 @@ KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->HidRing && dev->HidBuffer) return NTLSTATUS_SUCCESS;
dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingPhys);
dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferPhys);
if (!dev->HidRing || !dev->HidBuffer)
dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingDeviceAddress, &dev->HidRingDma);
dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferDeviceAddress, &dev->HidBufferDma);
if (!dev->HidRing || !dev->HidBuffer) {
if (gUsbDdk && gUsbDdk->Dma) {
gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma);
}
dev->HidBufferDma = (IO_DMA_BUFFER*)0;
dev->HidRingDma = (IO_DMA_BUFFER*)0;
dev->HidBuffer = (uint8_t*)0;
dev->HidRing = (XHCI_TRB*)0;
return NTLSTATUS_NO_MEMORY;
}
dev->HidEnqueue = 0;
dev->HidCycle = 1;
XHCI_TRB* link = &dev->HidRing[XHCI_HID_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->HidRingPhys;
link->ParameterHi = (uint32_t)(dev->HidRingPhys >> 32);
link->ParameterLo = (uint32_t)dev->HidRingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->HidRingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -253,15 +272,15 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
ep[0] = (uint32_t)interval << 16;
ep[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_INT_IN | ((uint32_t)maxPacket << 16);
ep[2] = (uint32_t)(dev->HidRingPhys | XHCI_EP_DCS);
ep[3] = (uint32_t)((dev->HidRingPhys | XHCI_EP_DCS) >> 32);
ep[2] = (uint32_t)(dev->HidRingDeviceAddress | XHCI_EP_DCS);
ep[3] = (uint32_t)((dev->HidRingDeviceAddress | XHCI_EP_DCS) >> 32);
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -499,7 +518,7 @@ static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) {
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed)
return NTLSTATUS_OBJECT_DEAD;
if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferPhys)
if (!ext->LowerDevice || !ext->InterruptPipe || !ext->ReportBuffer || !ext->ReportBufferDeviceAddress)
return NTLSTATUS_INVALID_PARAMETER;
if (ext->InterruptPending)
return NTLSTATUS_SUCCESS;
@@ -511,7 +530,7 @@ static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
ext->InterruptUrb.Device = ext->Device;
ext->InterruptUrb.Pipe = ext->InterruptPipe;
ext->InterruptUrb.Buffer = ext->ReportBuffer;
ext->InterruptUrb.BufferPhys = ext->ReportBufferPhys;
ext->InterruptUrb.BufferDeviceAddress = ext->ReportBufferDeviceAddress;
ext->InterruptUrb.Length = ext->ReportLength;
ext->InterruptUrb.Complete = UsbHidInterruptComplete;
ext->InterruptUrb.CompleteContext = ext;
@@ -565,7 +584,7 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext)
if (reportLength > XHCI_MAX_HID_REPORT) reportLength = XHCI_MAX_HID_REPORT;
ext->ReportLength = reportLength;
ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferPhys);
ext->ReportBuffer = (uint8_t*)UsbAllocateDmaBuffer(ext->Device, 1, &ext->ReportBufferDeviceAddress, &ext->ReportBufferDma);
if (!ext->ReportBuffer)
return NTLSTATUS_NO_MEMORY;
@@ -575,13 +594,22 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext)
ext->MaxPacketSize,
ext->Interval,
&ext->InterruptPipe);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
UsbHidReleaseDma(ext);
return st;
}
if (!ext->AbsoluteMouse) {
st = UsbHidSendBootProtocol(ext);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
if (ext->InterruptPipe) {
ext->InterruptPipe->Open = false;
UsbDereferencePipe(ext->InterruptPipe);
ext->InterruptPipe = (USB_PIPE*)0;
}
UsbHidReleaseDma(ext);
return st;
}
}
memset(ext->LastKeys, 0, sizeof(ext->LastKeys));
@@ -678,6 +706,7 @@ KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
IoDestroyRemoveLock(&ext->RemoveLock);
ext->RemoveLockReady = false;
}
UsbHidReleaseDma(ext);
}
KSTATUS lower = ext && ext->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS;
code/drivers/bus/usb/usb_hub.c
+132 -53
View File
@@ -8,6 +8,16 @@
DRIVER_OBJECT gUsbHubDriver;
// Releases the external-hub change bitmap after transfer rundown.
static void UsbHubReleaseDma(USB_HUB_EXTENSION* hub) {
if (!hub)
return;
UsbFreeDmaBuffer(hub->ChangeBitmapDma);
hub->ChangeBitmapDma = (IO_DMA_BUFFER*)0;
hub->ChangeBitmap = (uint8_t*)0;
hub->ChangeBitmapDeviceAddress = 0;
}
static const IO_DEVICE_ID gUsbHubIds[] = {
{ IoBusUsb, { "USB\\ROOT_HUB", sizeof("USB\\ROOT_HUB") - 1 } },
{ IoBusUsb, { "USB\\CLASS_09", sizeof("USB\\CLASS_09") - 1 } },
@@ -61,8 +71,10 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub,
if (maxAttempts == 0) maxAttempts = 1;
for (uint8_t attempt = 1; attempt <= maxAttempts; attempt++) {
if (!UsbHubRootPortConnected(xhci, port))
return NTLSTATUS_NOT_FOUND;
if (!UsbHubRootPortConnected(xhci, port)) {
last = NTLSTATUS_NOT_FOUND;
break;
}
uint8_t slot = 0;
last = XhciEnumeratePortEx(xhci, port, &slot);
@@ -100,6 +112,40 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub,
return last;
}
// Enumerates one connected root port through the same debounce path at boot and hotplug.
static KSTATUS UsbHubEnumerateConnectedRootPort(USB_HUB_EXTENSION* hub,
XHCI_CONTROLLER* xhci,
uint8_t port,
uint8_t* outSlot) {
if (outSlot)
*outSlot = 0;
if (!hub || !xhci || port == 0 || port > hub->PortCount ||
port > USB_ROOT_HUB_MAX_PORTS)
return NTLSTATUS_INVALID_PARAMETER;
hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING;
hub->RootPortState[port] = UsbHubPortConnectedDebounce;
UsbDelayUs(xhci->Ddk, 30000u);
if (!UsbHubRootPortConnected(xhci, port)) {
UsbHubClearRootPortFailure(hub, port);
hub->RootPortSlot[port] = 0;
hub->RootPortState[port] = UsbHubPortDisconnected;
return NTLSTATUS_NOT_FOUND;
}
hub->RootPortState[port] = UsbHubPortEnumerating;
uint8_t slot = 0;
KSTATUS status = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot);
if (KSUCCESS(status)) {
hub->RootPortSlot[port] = slot;
hub->RootPortState[port] = UsbHubPortChildPresent;
if (outSlot)
*outSlot = slot;
}
return status;
}
// Implements the hub Pop Pending Root Port operation.
static bool UsbHubPopPendingRootPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) {
if (!hub || !outPort) return false;
@@ -184,17 +230,19 @@ static void UsbHubRemoveSlotChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t
DbgPrint("usb-hub: %s port %u removing slot=%u\n",
hub->Kind == UsbHubKindRoot ? "root" : "hub", port, slot);
if (xdev->Allocated && xhci->Running)
(void)XhciDisableSlot(xhci, slot);
if (xdev->UsbPdo)
(void)UsbRemovePdoSubtree(xdev->UsbPdo, true);
if (xdev->Allocated)
(void)XhciDisableSlot(xhci, slot);
if (xdev->UsbDevice) {
xdev->UsbDevice->State = UsbDeviceRemoved;
UsbDereferenceDevice(xdev->UsbDevice);
xdev->UsbDevice = (USB_DEVICE*)0;
}
XhciReleaseDeviceDma(xdev);
memset(xdev, 0, sizeof(*xdev));
if (hub->Kind == UsbHubKindRoot && port <= USB_ROOT_HUB_MAX_PORTS) {
hub->RootPortSlot[port] = 0;
@@ -216,9 +264,9 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub,
uint16_t value,
uint16_t index,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length) {
if (!hub || !hub->LowerDevice || !buffer || !bufferPhys || !length)
if (!hub || !hub->LowerDevice || !buffer || !bufferDeviceAddress || !length)
return NTLSTATUS_INVALID_PARAMETER;
USB_URB urb;
@@ -227,7 +275,7 @@ static KSTATUS UsbHubControlIn(USB_HUB_EXTENSION* hub,
urb.Direction = UsbTransferIn;
urb.Device = hub->Device;
urb.Buffer = buffer;
urb.BufferPhys = bufferPhys;
urb.BufferDeviceAddress = bufferDeviceAddress;
urb.Length = length;
urb.Setup.bmRequestType = requestType;
urb.Setup.bRequest = request;
@@ -260,7 +308,7 @@ static KSTATUS UsbHubControlNoData(USB_HUB_EXTENSION* hub,
// Implements the hub Read Descriptor operation.
static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* outDesc) {
if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapPhys)
if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapDeviceAddress)
return NTLSTATUS_INVALID_PARAMETER;
memset(hub->ChangeBitmap, 0, USB_HUB_CHANGE_BITMAP_BYTES);
@@ -270,7 +318,7 @@ static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR*
(uint16_t)(USB_DESC_HUB << 8),
0,
hub->ChangeBitmap,
hub->ChangeBitmapPhys,
hub->ChangeBitmapDeviceAddress,
USB_HUB_CHANGE_BITMAP_BYTES);
if (!KSUCCESS(st))
return st;
@@ -293,7 +341,7 @@ static KSTATUS UsbHubGetPortStatus(USB_HUB_EXTENSION* hub, uint8_t port, USB_HUB
0,
port,
hub->ChangeBitmap,
hub->ChangeBitmapPhys,
hub->ChangeBitmapDeviceAddress,
sizeof(USB_HUB_PORT_STATUS));
if (!KSUCCESS(st))
return st;
@@ -501,19 +549,8 @@ static void UsbHubHandleRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
return;
}
hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING;
hub->RootPortState[port] = UsbHubPortConnectedDebounce;
UsbDelayUs(xhci->Ddk, 30000);
portsc = XhciRead32(xhci->Op, XhciPortReg(port));
if (!(portsc & XHCI_PORTSC_CCS)) {
UsbHubClearRootPortFailure(hub, port);
return;
}
hub->RootPortState[port] = UsbHubPortEnumerating;
uint8_t slot = 0;
KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot);
KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot);
if (KSUCCESS(st)) {
hub->RootPortSlot[port] = slot;
hub->RootPortState[port] = UsbHubPortChildPresent;
@@ -544,9 +581,11 @@ static bool UsbHubTakeInitialRootScan(USB_HUB_EXTENSION* hub) {
static void UsbHubWorker(void* arg) {
USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)arg;
while (hub && hub->WorkerRunning) {
KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore);
KSTATUS st = KeWaitSemaphore(hub->WorkerSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
continue;
if (!hub->WorkerRunning)
break;
if (hub->Kind == UsbHubKindRoot) {
if (UsbHubTakeInitialRootScan(hub))
@@ -563,6 +602,61 @@ static void UsbHubWorker(void* arg) {
UsbHubArmExternalInterrupt(hub);
}
}
if (hub && hub->WorkerFinished)
KeSignalEvent(hub->WorkerFinished);
}
// Stops the hub worker and releases its synchronization objects.
static void UsbHubStopWorker(USB_HUB_EXTENSION* hub) {
if (!hub)
return;
hub->WorkerRunning = false;
if (hub->WorkerSemaphore)
KeSignalSemaphore(hub->WorkerSemaphore);
if (hub->WorkerThread && hub->WorkerFinished)
(void)KeWaitForEvent(hub->WorkerFinished, 0);
if (hub->WorkerThread) {
ObDereferenceObject((OBJECT*)hub->WorkerThread);
hub->WorkerThread = (THREAD*)0;
}
if (hub->WorkerFinished) {
ObDereferenceObject(&hub->WorkerFinished->Header);
hub->WorkerFinished = (EVENT*)0;
}
if (hub->WorkerSemaphore) {
ObDereferenceObject(&hub->WorkerSemaphore->Header);
hub->WorkerSemaphore = (SEMAPHORE*)0;
}
}
// Creates the synchronization objects and thread for one hub worker.
static KSTATUS UsbHubStartWorker(USB_HUB_EXTENSION* hub) {
if (!hub)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore);
if (!KSUCCESS(status))
return status;
status = KeEventCreate(EventManualReset, 0, &hub->WorkerFinished);
if (!KSUCCESS(status)) {
UsbHubStopWorker(hub);
return status;
}
hub->WorkerRunning = true;
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
status = KeCreateKernelThread((uintptr_t)UsbHubWorker,
(uintptr_t)hub,
&workerParameters,
&hub->WorkerThread);
if (!KSUCCESS(status)) {
hub->WorkerRunning = false;
UsbHubStopWorker(hub);
return status;
}
return NTLSTATUS_SUCCESS;
}
// Implements the hub Do Root Port Scan operation.
@@ -592,10 +686,8 @@ static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd) {
}
attempted++;
hub->RootPortSlot[port] = USB_HUB_PORT_ENUMERATING;
hub->RootPortState[port] = UsbHubPortEnumerating;
uint8_t slot = 0;
KSTATUS st = UsbHubEnumerateRootPortWithRetries(hub, xhci, port, &slot);
KSTATUS st = UsbHubEnumerateConnectedRootPort(hub, xhci, port, &slot);
if (KSUCCESS(st)) {
hub->RootPortSlot[port] = slot;
hub->RootPortState[port] = UsbHubPortChildPresent;
@@ -734,7 +826,7 @@ static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub) {
hub->ChangeUrb.Device = hub->Device;
hub->ChangeUrb.Pipe = hub->InterruptPipe;
hub->ChangeUrb.Buffer = hub->ChangeBitmap;
hub->ChangeUrb.BufferPhys = hub->ChangeBitmapPhys;
hub->ChangeUrb.BufferDeviceAddress = hub->ChangeBitmapDeviceAddress;
hub->ChangeUrb.Length = hub->InterruptMaxPacket;
if (hub->ChangeUrb.Length > USB_HUB_CHANGE_BITMAP_BYTES)
hub->ChangeUrb.Length = USB_HUB_CHANGE_BITMAP_BYTES;
@@ -756,20 +848,10 @@ static KSTATUS UsbHubStartRootDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* h
if (hub->PortCount > USB_ROOT_HUB_MAX_PORTS)
hub->PortCount = USB_ROOT_HUB_MAX_PORTS;
KSTATUS st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore);
KSTATUS st = UsbHubStartWorker(hub);
if (!KSUCCESS(st))
return st;
hub->WorkerRunning = true;
st = KeCreateKernelThread((uintptr_t)UsbHubWorker,
(uintptr_t)hub,
PriorityNormal,
&hub->WorkerThread);
if (!KSUCCESS(st)) {
hub->WorkerRunning = false;
return st;
}
hub->Started = true;
device->PnpState = DevicePnpStarted;
if (hub->HubPdo)
@@ -784,7 +866,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
if (!hub || !hub->Device)
return NTLSTATUS_INVALID_PARAMETER;
hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapPhys);
hub->ChangeBitmap = (uint8_t*)UsbAllocateDmaBuffer(hub->Device, 1, &hub->ChangeBitmapDeviceAddress, &hub->ChangeBitmapDma);
if (!hub->ChangeBitmap)
return NTLSTATUS_NO_MEMORY;
@@ -793,6 +875,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
KSTATUS st = UsbHubReadDescriptor(hub, &desc);
if (!KSUCCESS(st)) {
DbgPrint("usb-hub: slot=%u descriptor read failed st=0x%x\n", hub->Device->SlotId, st);
UsbHubReleaseDma(hub);
return st;
}
@@ -809,17 +892,9 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
UsbDelayUs(((XHCI_CONTROLLER*)hub->Hcd->ControllerContext)->Ddk,
(uint32_t)hub->PowerOnToPowerGoodMs * 1000u);
st = KeSemaphoreCreate(0, 0x7fffffff, &hub->WorkerSemaphore);
if (!KSUCCESS(st))
return st;
hub->WorkerRunning = true;
st = KeCreateKernelThread((uintptr_t)UsbHubWorker,
(uintptr_t)hub,
PriorityNormal,
&hub->WorkerThread);
st = UsbHubStartWorker(hub);
if (!KSUCCESS(st)) {
hub->WorkerRunning = false;
UsbHubReleaseDma(hub);
return st;
}
@@ -829,8 +904,11 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
hub->InterruptMaxPacket,
hub->InterruptInterval,
&hub->InterruptPipe);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
UsbHubStopWorker(hub);
UsbHubReleaseDma(hub);
return st;
}
hub->Started = true;
device->PnpState = DevicePnpStarted;
@@ -897,6 +975,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
(void)UsbCancelUrb(&hub->ChangeUrb);
if (hub->WorkerSemaphore)
KeSignalSemaphore(hub->WorkerSemaphore);
UsbHubStopWorker(hub);
// Remove downstream PDOs owned by this hub FDO. The explicit
// slot arrays are still used as the hardware lookup table; the
@@ -929,8 +1008,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
UsbDereferencePipe(hub->InterruptPipe);
hub->InterruptPipe = (USB_PIPE*)0;
}
if (hub->WorkerSemaphore)
KeSignalSemaphore(hub->WorkerSemaphore);
UsbHubStopWorker(hub);
if (hub->Kind == UsbHubKindRoot && hub->Hcd && hub->Hcd->RootHubContext == hub)
hub->Hcd->RootHubContext = (void*)0;
if (hub->RemoveLockReady) {
@@ -938,6 +1016,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
IoDestroyRemoveLock(&hub->RemoveLock);
hub->RemoveLockReady = false;
}
UsbHubReleaseDma(hub);
}
KSTATUS lower = hub && hub->LowerDevice ? IoForwardIrpToLower(device, irp) : NTLSTATUS_SUCCESS;
code/drivers/bus/usb/usb_storage.c
+261 -50
View File
@@ -56,10 +56,10 @@ static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_E
static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
bool dataIn,
PHYSICAL_ADDRESS phys,
uint64_t deviceAddress,
uint32_t length,
uint32_t* outActual) {
if (!storage || !storage->LowerDevice || !storage->Device || !phys || length == 0)
if (!storage || !storage->LowerDevice || !storage->Device || !deviceAddress || length == 0)
return NTLSTATUS_INVALID_PARAMETER;
USB_PIPE* pipe = dataIn ? storage->BulkInPipe : storage->BulkOutPipe;
@@ -73,7 +73,7 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
urb.Device = storage->Device;
urb.Pipe = pipe;
urb.Buffer = (void*)0;
urb.BufferPhys = phys;
urb.BufferDeviceAddress = deviceAddress;
urb.Length = length;
KSTATUS st = UsbSubmitUrbSync(storage->LowerDevice, &urb);
@@ -83,29 +83,42 @@ static KSTATUS UsbStorageSubmitBulk(USB_STORAGE_EXTENSION* storage,
}
KSTATUS UsbStorageBulkIn(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint64_t deviceAddress,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, true, phys, length, outActual);
return UsbStorageSubmitBulk(storage, true, deviceAddress, length, outActual);
}
KSTATUS UsbStorageBulkOut(USB_STORAGE_EXTENSION* storage,
PHYSICAL_ADDRESS phys,
uint64_t deviceAddress,
uint32_t length,
uint32_t* outActual) {
return UsbStorageSubmitBulk(storage, false, phys, length, outActual);
return UsbStorageSubmitBulk(storage, false, deviceAddress, length, outActual);
}
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
PHYSICAL_ADDRESS dataPhys,
uint32_t dataLength,
bool dataIn) {
typedef struct USB_STORAGE_SENSE_DATA {
bool Valid;
uint8_t ResponseCode;
uint8_t Key;
uint8_t Asc;
uint8_t Ascq;
} USB_STORAGE_SENSE_DATA;
// Executes one BOT command without issuing follow-up commands.
static KSTATUS UsbStorageBotCommandRaw(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
uint64_t dataDeviceAddress,
uint32_t dataLength,
bool dataIn,
uint8_t* outCswStatus,
uint32_t* outResidue) {
if (outCswStatus) *outCswStatus = 0xFFu;
if (outResidue) *outResidue = 0;
if (!storage || !storage->XhciDevice || !cdb || cdbLen == 0 || cdbLen > 16)
return NTLSTATUS_INVALID_PARAMETER;
if (dataLength && (!data || !dataPhys))
if (dataLength && (!data || !dataDeviceAddress))
return NTLSTATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
@@ -120,14 +133,14 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
memcpy(cbw->CommandBlock, cdb, cdbLen);
uint32_t actual = 0;
KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorPhys, sizeof(*cbw), &actual);
KSTATUS st = UsbStorageBulkOut(storage, dev->DescriptorDeviceAddress, sizeof(*cbw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*cbw))
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
if (dataLength) {
actual = 0;
st = dataIn ? UsbStorageBulkIn(storage, dataPhys, dataLength, &actual)
: UsbStorageBulkOut(storage, dataPhys, dataLength, &actual);
st = dataIn ? UsbStorageBulkIn(storage, dataDeviceAddress, dataLength, &actual)
: UsbStorageBulkOut(storage, dataDeviceAddress, dataLength, &actual);
if (!KSUCCESS(st))
return st;
}
@@ -135,7 +148,7 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer;
memset(csw, 0, sizeof(*csw));
actual = 0;
st = UsbStorageBulkIn(storage, dev->DescriptorPhys, sizeof(*csw), &actual);
st = UsbStorageBulkIn(storage, dev->DescriptorDeviceAddress, sizeof(*csw), &actual);
if (!KSUCCESS(st) || actual != sizeof(*csw))
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
@@ -144,15 +157,136 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
dev->SlotId, csw->Signature, csw->Tag, storage->Tag);
return NTLSTATUS_DEVICE_ERROR;
}
if (csw->Status != USB_MASS_BOT_CSW_PASSED) {
DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u\n",
cdb[0], dev->SlotId, csw->Status, csw->DataResidue);
return NTLSTATUS_DEVICE_ERROR;
if (outCswStatus) *outCswStatus = csw->Status;
if (outResidue) *outResidue = csw->DataResidue;
return csw->Status == USB_MASS_BOT_CSW_PASSED
? NTLSTATUS_SUCCESS
: NTLSTATUS_DEVICE_ERROR;
}
// Returns a readable SCSI sense-key name.
static const char* UsbStorageSenseKeyName(uint8_t key) {
switch (key) {
case 0x00u: return "NO SENSE";
case 0x01u: return "RECOVERED ERROR";
case 0x02u: return "NOT READY";
case 0x03u: return "MEDIUM ERROR";
case 0x04u: return "HARDWARE ERROR";
case 0x05u: return "ILLEGAL REQUEST";
case 0x06u: return "UNIT ATTENTION";
case 0x07u: return "DATA PROTECT";
case 0x08u: return "BLANK CHECK";
case 0x0Bu: return "ABORTED COMMAND";
case 0x0Eu: return "MISCOMPARE";
default: return "UNKNOWN";
}
}
// Decodes fixed or descriptor-format REQUEST SENSE data.
static void UsbStorageDecodeSense(const uint8_t* bytes,
uint32_t length,
USB_STORAGE_SENSE_DATA* sense) {
if (!sense) return;
memset(sense, 0, sizeof(*sense));
if (!bytes || length == 0) return;
sense->ResponseCode = bytes[0] & 0x7Fu;
if ((sense->ResponseCode == 0x70u || sense->ResponseCode == 0x71u) && length >= 14u) {
sense->Key = bytes[2] & 0x0Fu;
sense->Asc = bytes[12];
sense->Ascq = bytes[13];
sense->Valid = true;
return;
}
if ((sense->ResponseCode == 0x72u || sense->ResponseCode == 0x73u) && length >= 4u) {
sense->Key = bytes[1] & 0x0Fu;
sense->Asc = bytes[2];
sense->Ascq = bytes[3];
sense->Valid = true;
}
}
// Fetches and decodes sense data after a command failure.
static KSTATUS UsbStorageFetchSense(USB_STORAGE_EXTENSION* storage,
USB_STORAGE_SENSE_DATA* sense) {
if (!storage || !storage->XhciDevice || !sense)
return NTLSTATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t cdb[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
memset(dev->MassBuffer, 0, 18);
uint8_t cswStatus = 0xFFu;
uint32_t residue = 0;
KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferDeviceAddress,
18, true, &cswStatus, &residue);
if (!KSUCCESS(st)) {
DbgPrint("usb-storage: REQUEST SENSE failed slot=%u st=0x%x csw_status=%u residue=%u\n",
dev->SlotId, st, cswStatus, residue);
memset(sense, 0, sizeof(*sense));
return st;
}
uint32_t returned = residue < 18u ? 18u - residue : 0u;
UsbStorageDecodeSense(dev->MassBuffer, returned, sense);
return NTLSTATUS_SUCCESS;
}
// Executes one BOT command and reports SCSI sense data on command failure.
static KSTATUS UsbStorageExecuteCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
uint64_t dataDeviceAddress,
uint32_t dataLength,
bool dataIn,
USB_STORAGE_SENSE_DATA* outSense) {
if (outSense) memset(outSense, 0, sizeof(*outSense));
uint8_t cswStatus = 0xFFu;
uint32_t residue = 0;
KSTATUS st = UsbStorageBotCommandRaw(storage, cdb, cdbLen, data,
dataDeviceAddress, dataLength, dataIn,
&cswStatus, &residue);
if (KSUCCESS(st)) return st;
XHCI_DEVICE* dev = storage ? storage->XhciDevice : (XHCI_DEVICE*)0;
uint32_t slot = dev ? dev->SlotId : 0;
if (cswStatus != USB_MASS_BOT_CSW_FAILED || !cdb || cdb[0] == SCSI_REQUEST_SENSE) {
DbgPrint("usb-storage: command %02x failed slot=%u st=0x%x csw_status=%u residue=%u\n",
cdb ? cdb[0] : 0, slot, st, cswStatus, residue);
return st;
}
USB_STORAGE_SENSE_DATA sense;
KSTATUS senseStatus = UsbStorageFetchSense(storage, &sense);
if (KSUCCESS(senseStatus) && sense.Valid) {
DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=%s key=%02x asc=%02x ascq=%02x\n",
cdb[0], slot, cswStatus, residue,
UsbStorageSenseKeyName(sense.Key), sense.Key, sense.Asc, sense.Ascq);
if (outSense) *outSense = sense;
} else {
DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u sense=unavailable\n",
cdb[0], slot, cswStatus, residue);
}
return st;
}
// Executes one BOT command through the sense-reporting path.
KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
const uint8_t* cdb,
uint8_t cdbLen,
void* data,
uint64_t dataDeviceAddress,
uint32_t dataLength,
bool dataIn) {
return UsbStorageExecuteCommand(storage, cdb, cdbLen, data,
dataDeviceAddress, dataLength, dataIn,
(USB_STORAGE_SENSE_DATA*)0);
}
// Implements the be32 operation.
uint32_t UsbBe32(const uint8_t* p) {
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
@@ -182,7 +316,7 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 };
memset(dev->MassBuffer, 0, 36);
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
dev->MassBuffer, dev->MassBufferDeviceAddress,
36, true);
if (!KSUCCESS(st)) return st;
@@ -195,18 +329,6 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
return NTLSTATUS_SUCCESS;
}
// Implements the storage Request Sense operation.
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return NTLSTATUS_INVALID_PARAMETER;
XHCI_DEVICE* dev = storage->XhciDevice;
uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
memset(dev->MassBuffer, 0, 18);
return UsbStorageBotCommand(storage, rs, sizeof(rs),
dev->MassBuffer, dev->MassBufferPhys, 18, true);
}
// Implements the storage Test Unit Ready operation.
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
@@ -222,7 +344,6 @@ KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
if (KSUCCESS(st)) return NTLSTATUS_SUCCESS;
last = st;
(void)UsbStorageRequestSense(storage);
UsbDelayUs(xhci->Ddk, 10000u);
}
@@ -238,7 +359,7 @@ KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) {
uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
memset(dev->MassBuffer, 0, 8);
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
dev->MassBuffer, dev->MassBufferDeviceAddress,
8, true);
if (!KSUCCESS(st)) return st;
@@ -288,7 +409,7 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
memcpy(dev->MassBuffer, buffer, bytes);
KSTATUS st = UsbStorageBotCommand(storage, cdb, sizeof(cdb),
dev->MassBuffer, dev->MassBufferPhys,
dev->MassBuffer, dev->MassBufferDeviceAddress,
bytes, !write);
if (!KSUCCESS(st)) return st;
@@ -297,6 +418,48 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
return NTLSTATUS_SUCCESS;
}
// Returns whether sense data means SYNCHRONIZE CACHE is unsupported.
static bool UsbStorageSenseRejectsSynchronizeCache(const USB_STORAGE_SENSE_DATA* sense) {
if (!sense || !sense->Valid || sense->Key != 0x05u)
return false;
return sense->Asc == 0x20u || sense->Asc == 0x24u;
}
// Flushes device write caches through SCSI SYNCHRONIZE CACHE(10).
static KSTATUS UsbStorageSynchronizeCache(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return NTLSTATUS_INVALID_PARAMETER;
if (storage->FlushCapability == UsbStorageFlushUnsupported)
return NTLSTATUS_SUCCESS;
uint8_t cdb[10];
memset(cdb, 0, sizeof(cdb));
cdb[0] = SCSI_SYNCHRONIZE_CACHE_10;
for (uint32_t attempt = 0; attempt < 2; attempt++) {
USB_STORAGE_SENSE_DATA sense;
KSTATUS st = UsbStorageExecuteCommand(storage, cdb, sizeof(cdb),
(void*)0, 0, 0, false, &sense);
if (KSUCCESS(st)) {
storage->FlushCapability = UsbStorageFlushSupported;
return NTLSTATUS_SUCCESS;
}
if (UsbStorageSenseRejectsSynchronizeCache(&sense)) {
storage->FlushCapability = UsbStorageFlushUnsupported;
DbgPrint("usb-storage: slot=%u does not support SYNCHRONIZE CACHE; flushes use BOT command-completion semantics only\n",
storage->XhciDevice->SlotId);
return NTLSTATUS_SUCCESS;
}
if (sense.Valid && sense.Key == 0x06u && attempt == 0)
continue;
return st;
}
return NTLSTATUS_DEVICE_ERROR;
}
// Implements the storage Acquire Io operation.
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
if (!storage) return;
@@ -375,24 +538,70 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
return NTLSTATUS_PENDING;
}
// Flushes one USB mass-storage disk synchronously.
static KSTATUS UsbStorageDispatchFlush(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return NTLSTATUS_INVALID_PARAMETER;
USB_STORAGE_EXTENSION* storage = UsbStorageFromDiskDevice(device);
if (!storage || !storage->RemoveLockReady)
return NTLSTATUS_OBJECT_DEAD;
KSTATUS st = IoAcquireRemoveLock(&storage->RemoveLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
return NTLSTATUS_INVALID_PARAMETER;
}
if (!storage->Started || storage->SurpriseRemoved || storage->Removed) {
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, NTLSTATUS_OBJECT_DEAD, 0);
return NTLSTATUS_PENDING;
}
UsbStorageAcquireIo(storage);
st = UsbStorageSynchronizeCache(storage);
UsbStorageReleaseIo(storage);
IoReleaseRemoveLock(&storage->RemoveLock, irp);
IoCompleteRequest(irp, st, 0);
return NTLSTATUS_PENDING;
}
// Creates mass Storage Memory.
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer)
return NTLSTATUS_SUCCESS;
dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingPhys);
dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingPhys);
dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferPhys);
if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer)
dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingDeviceAddress, &dev->MassInRingDma);
dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingDeviceAddress, &dev->MassOutRingDma);
dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferDeviceAddress, &dev->MassBufferDma);
if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer) {
if (gUsbDdk && gUsbDdk->Dma) {
gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma);
}
dev->MassBufferDma = (IO_DMA_BUFFER*)0;
dev->MassOutRingDma = (IO_DMA_BUFFER*)0;
dev->MassInRingDma = (IO_DMA_BUFFER*)0;
dev->MassBuffer = (uint8_t*)0;
dev->MassOutRing = (XHCI_TRB*)0;
dev->MassInRing = (XHCI_TRB*)0;
return NTLSTATUS_NO_MEMORY;
}
dev->MassInEnqueue = 0;
dev->MassInCycle = 1;
dev->MassOutEnqueue = 0;
dev->MassOutCycle = 1;
dev->MassTag = 0x1000u + dev->SlotId;
XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingPhys, XHCI_BULK_RING_TRBS, dev->MassInCycle);
XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingPhys, XHCI_BULK_RING_TRBS, dev->MassOutCycle);
XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassInCycle);
XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingDeviceAddress, XHCI_BULK_RING_TRBS, dev->MassOutCycle);
return NTLSTATUS_SUCCESS;
}
@@ -431,21 +640,21 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de
epIn[0] = 0;
epIn[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_IN | ((uint32_t)inMaxPacket << 16);
epIn[2] = (uint32_t)(dev->MassInRingPhys | XHCI_EP_DCS);
epIn[3] = (uint32_t)((dev->MassInRingPhys | XHCI_EP_DCS) >> 32);
epIn[2] = (uint32_t)(dev->MassInRingDeviceAddress | XHCI_EP_DCS);
epIn[3] = (uint32_t)((dev->MassInRingDeviceAddress | XHCI_EP_DCS) >> 32);
epIn[4] = inMaxPacket;
epOut[0] = 0;
epOut[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_OUT | ((uint32_t)outMaxPacket << 16);
epOut[2] = (uint32_t)(dev->MassOutRingPhys | XHCI_EP_DCS);
epOut[3] = (uint32_t)((dev->MassOutRingPhys | XHCI_EP_DCS) >> 32);
epOut[2] = (uint32_t)(dev->MassOutRingDeviceAddress | XHCI_EP_DCS);
epOut[3] = (uint32_t)((dev->MassOutRingDeviceAddress | XHCI_EP_DCS) >> 32);
epOut[4] = outMaxPacket;
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -549,6 +758,7 @@ static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSIO
storage->BulkInMaxPacket = bulkIn->wMaxPacketSize;
storage->BulkOutMaxPacket = bulkOut->wMaxPacketSize;
storage->Tag = 0x1000u + storage->Device->SlotId;
storage->FlushCapability = UsbStorageFlushUnknown;
xdev->MassCandidate = true;
xdev->MassInterface = storage->InterfaceNumber;
@@ -844,6 +1054,7 @@ KSTATUS UsbInitializeStorageDriver(void) {
gUsbStorageDriver.DriverAddDevice = UsbStorageAddDevice;
gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
gUsbStorageDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = UsbStorageDispatchFlush;
gUsbStorageDriver.MajorFunction[IRP_MJ_PNP] = UsbStorageDispatchPnp;
return IoRegisterDriverObject(&gUsbStorageDriver);
}
code/drivers/bus/usb/usb_xhci.c
+322 -151
View File
@@ -73,32 +73,105 @@ bool UsbWaitClear32(const NTL_DDK* ddk, volatile uint8_t* base, uint32_t reg,
return false;
}
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
PHYSICAL_ADDRESS* outPhys) {
if (!outPhys || pages == 0)
return NULL;
// Allocates page-aligned common-buffer memory from the controller adapter.
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!xhci || !xhci->DmaAdapter || !outDeviceAddress || !outBuffer || pages == 0 ||
pages > ((size_t)-1) / PAGE_SIZE)
return (void*)0;
// Most xHCI runtime objects are one page. Do not route those through the
// physically-contiguous allocator: on real machines that allocator is a
// scarce DMA primitive, while a single frame is already contiguous. QEMU
// hid this because it had simple low memory; laptop firmware does not.
PHYSICAL_ADDRESS phys = 0;
if (pages == 1) {
phys = MmPfnAllocate(MmPfnUseDma);
} else {
phys = MmPfnAllocateContiguous(pages, MmPfnUseDma);
IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
KSTATUS status = xhci->Ddk->Dma->AllocateCommonBuffer(xhci->DmaAdapter,
pages * (size_t)PAGE_SIZE,
PAGE_SIZE,
&buffer);
if (!KSUCCESS(status)) {
DbgPrint("usb: DMA allocation failed pages=%llu st=0x%x\n",
(unsigned long long)pages, status);
return (void*)0;
}
if (!phys) {
DbgPrint("usb: DMA allocation failed pages=%llu\n",
(unsigned long long)pages);
return NULL;
*outDeviceAddress = xhci->Ddk->Dma->GetCommonBufferDeviceAddress(buffer);
*outBuffer = buffer;
return xhci->Ddk->Dma->GetCommonBufferAddress(buffer);
}
// Releases all DMA allocations owned by one xHCI slot.
void XhciReleaseDeviceDma(XHCI_DEVICE* dev) {
if (!dev || !gUsbDdk || !gUsbDdk->Dma)
return;
for (uint32_t endpoint = 1; endpoint < 32; endpoint++) {
USB_PIPE* pipe = dev->EndpointPipes[endpoint];
dev->EndpointPipes[endpoint] = (USB_PIPE*)0;
if (pipe)
UsbDereferencePipe(pipe);
}
void* virt = xhci->Ddk->Memory->PhysToVirt(phys);
memset(virt, 0, pages * (size_t)PAGE_SIZE);
*outPhys = phys;
return virt;
gUsbDdk->Dma->FreeCommonBuffer(dev->MassBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidBufferDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DescriptorDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassOutRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->MassInRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->HidRingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->Ep0RingDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->DeviceContextDma);
gUsbDdk->Dma->FreeCommonBuffer(dev->InputContextDma);
dev->MassBufferDma = (IO_DMA_BUFFER*)0;
dev->HidBufferDma = (IO_DMA_BUFFER*)0;
dev->DescriptorDma = (IO_DMA_BUFFER*)0;
dev->MassOutRingDma = (IO_DMA_BUFFER*)0;
dev->MassInRingDma = (IO_DMA_BUFFER*)0;
dev->HidRingDma = (IO_DMA_BUFFER*)0;
dev->Ep0RingDma = (IO_DMA_BUFFER*)0;
dev->DeviceContextDma = (IO_DMA_BUFFER*)0;
dev->InputContextDma = (IO_DMA_BUFFER*)0;
dev->MassBuffer = (uint8_t*)0;
dev->HidBuffer = (uint8_t*)0;
dev->DescriptorBuffer = (uint8_t*)0;
dev->MassOutRing = (XHCI_TRB*)0;
dev->MassInRing = (XHCI_TRB*)0;
dev->HidRing = (XHCI_TRB*)0;
dev->Ep0Ring = (XHCI_TRB*)0;
dev->DeviceContext = (uint8_t*)0;
dev->InputContext = (uint8_t*)0;
}
// Releases controller-wide xHCI DMA allocations.
static void XhciReleaseControllerDma(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->Ddk || !xhci->Ddk->Dma)
return;
for (uint32_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++)
XhciReleaseDeviceDma(&xhci->Devices[slot]);
for (uint16_t i = 0; i < XHCI_MAX_SCRATCHPADS; i++) {
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadDma[i]);
xhci->ScratchpadDma[i] = (IO_DMA_BUFFER*)0;
xhci->ScratchpadDeviceAddress[i] = 0;
}
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ScratchpadArrayDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->ErstDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->EventRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->CommandRingDma);
xhci->Ddk->Dma->FreeCommonBuffer(xhci->DcbaaDma);
xhci->ScratchpadArrayDma = (IO_DMA_BUFFER*)0;
xhci->ErstDma = (IO_DMA_BUFFER*)0;
xhci->EventRingDma = (IO_DMA_BUFFER*)0;
xhci->CommandRingDma = (IO_DMA_BUFFER*)0;
xhci->DcbaaDma = (IO_DMA_BUFFER*)0;
xhci->ScratchpadArray = (uint64_t*)0;
xhci->Erst = (XHCI_ERST_ENTRY*)0;
xhci->EventRing = (XHCI_TRB*)0;
xhci->CommandRing = (XHCI_TRB*)0;
xhci->Dcbaa = (uint64_t*)0;
xhci->Ddk->Dma->PutAdapter(xhci->DmaAdapter);
xhci->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
// Implements the scratchpad Count operation.
@@ -328,26 +401,26 @@ void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
// Updates erdp.
void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB);
uint64_t erdp = xhci->EventRingDeviceAddress + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB);
XhciWrite64(ir0, 0x18u, erdp | (1ull << 3)); // EHB clear
}
// Returns command Waiter Locked.
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandDeviceAddress) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->CommandPhys && w->CommandPhys == commandPhys)
if (w->CommandDeviceAddress && w->CommandDeviceAddress == commandDeviceAddress)
return w;
}
return (XHCI_WAIT_TARGET*)0;
}
static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint64_t transferDeviceAddress,
uint8_t slotId,
uint8_t endpointId) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->TransferPhys &&
w->TransferPhys == transferPhys &&
if (w->TransferDeviceAddress &&
w->TransferDeviceAddress == transferDeviceAddress &&
w->TransferSlotId == slotId &&
w->TransferEndpointId == endpointId) {
return w;
@@ -357,11 +430,11 @@ static XHCI_WAIT_TARGET* XhciFindTransferWaiterLocked(XHCI_CONTROLLER* xhci,
}
static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
uint64_t commandPhys,
uint64_t commandDeviceAddress,
uint32_t completionCode,
uint8_t slotId) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandPhys);
XHCI_WAIT_TARGET* w = XhciFindCommandWaiterLocked(xhci, commandDeviceAddress);
if (w) {
w->Command.CompletionCode = completionCode;
w->Command.SlotId = slotId;
@@ -373,13 +446,13 @@ static XHCI_WAIT_TARGET* XhciClaimCommandWaiter(XHCI_CONTROLLER* xhci,
}
static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
uint64_t transferPhys,
uint64_t transferDeviceAddress,
uint8_t slotId,
uint8_t endpointId,
uint32_t completionCode,
uint32_t residue) {
uintptr_t flags = KeAcquireSpinlock(&xhci->WaitLock);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferPhys, slotId, endpointId);
XHCI_WAIT_TARGET* w = XhciFindTransferWaiterLocked(xhci, transferDeviceAddress, slotId, endpointId);
if (w) {
w->Transfer.CompletionCode = completionCode;
w->Transfer.SlotId = slotId;
@@ -397,7 +470,7 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
bool processed = false;
for (;;) {
while (true) {
XHCI_TRB* ev = &xhci->EventRing[xhci->EventDequeue];
uint32_t control = ev->Control;
uint8_t cycle = (uint8_t)(control & XHCI_TRB_CYCLE);
@@ -460,8 +533,8 @@ bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
// Waits for target Complete.
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
if (!target) return false;
if (target->CommandPhys && !target->CommandMatched) return false;
if (target->TransferPhys && !target->TransferMatched) return false;
if (target->CommandDeviceAddress && !target->CommandMatched) return false;
if (target->TransferDeviceAddress && !target->TransferMatched) return false;
return true;
}
@@ -568,6 +641,21 @@ static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the interrupt-dispatch registry.
static void XhciUnregisterController(XHCI_CONTROLLER* xhci) {
if (!xhci)
return;
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
if (gXhciControllers[i] != xhci)
continue;
for (uint32_t next = i + 1; next < gXhciControllerCount; next++)
gXhciControllers[next - 1] = gXhciControllers[next];
gXhciControllerCount--;
gXhciControllers[gXhciControllerCount] = (XHCI_CONTROLLER*)0;
return;
}
}
// Implements the isr operation.
void XhciIsr(void) {
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
@@ -663,6 +751,34 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_SUCCESS;
}
// Halts the controller before any device-visible memory can be released.
static void XhciHaltForDmaRelease(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->DmaAdapter)
return;
xhci->InterruptsEnabled = false;
xhci->Running = false;
if (xhci->SurpriseRemoved || !xhci->Op)
return;
if (xhci->Runtime) {
volatile uint8_t* interrupter = xhci->Runtime + 0x20u;
XhciWrite32(interrupter, 0x00u, XHCI_IMAN_IP);
}
uint32_t command = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
command &= ~(XHCI_USBCMD_RS | XHCI_USBCMD_INTE);
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, command);
if (!UsbWaitSet32(xhci->Ddk,
xhci->Op,
XHCI_OP_USBSTS,
XHCI_USBSTS_HCH,
1000000u))
KePanic(KE_PANIC_CODE_DRIVER,
"usb: xHCI failed to halt before DMA release usbsts=%08x\n",
XhciRead32(xhci->Op, XHCI_OP_USBSTS));
}
// Creates scratchpads.
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
if (xhci->ScratchpadCount == 0) return NTLSTATUS_SUCCESS;
@@ -672,7 +788,7 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_NOT_SUPPORTED;
}
xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayPhys);
xhci->ScratchpadArray = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadArrayDeviceAddress, &xhci->ScratchpadArrayDma);
if (!xhci->ScratchpadArray) {
DbgPrint("usb: xHCI scratchpad pointer-array allocation failed count=%u\n",
xhci->ScratchpadCount);
@@ -680,29 +796,29 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
}
for (uint16_t i = 0; i < xhci->ScratchpadCount; i++) {
void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadPhys[i]);
void* page = UsbDmaAllocPages(xhci, 1, &xhci->ScratchpadDeviceAddress[i], &xhci->ScratchpadDma[i]);
if (!page) {
DbgPrint("usb: xHCI scratchpad buffer allocation failed index=%u/%u\n",
i, xhci->ScratchpadCount);
return NTLSTATUS_NO_MEMORY;
}
(void)page;
xhci->ScratchpadArray[i] = xhci->ScratchpadPhys[i];
xhci->ScratchpadArray[i] = xhci->ScratchpadDeviceAddress[i];
}
xhci->Dcbaa[0] = xhci->ScratchpadArrayPhys;
xhci->Dcbaa[0] = xhci->ScratchpadArrayDeviceAddress;
UsbTrace("usb: xHCI allocated %u scratchpad buffer(s) array=0x%016llX\n",
xhci->ScratchpadCount,
(unsigned long long)xhci->ScratchpadArrayPhys);
(unsigned long long)xhci->ScratchpadArrayDeviceAddress);
return NTLSTATUS_SUCCESS;
}
// Creates runtime Memory.
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys);
xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys);
xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingPhys);
xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstPhys);
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaDeviceAddress, &xhci->DcbaaDma);
xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingDeviceAddress, &xhci->CommandRingDma);
xhci->EventRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->EventRingDeviceAddress, &xhci->EventRingDma);
xhci->Erst = (XHCI_ERST_ENTRY*)UsbDmaAllocPages(xhci, 1, &xhci->ErstDeviceAddress, &xhci->ErstDma);
if (!xhci->Dcbaa || !xhci->CommandRing || !xhci->EventRing || !xhci->Erst) {
DbgPrint("usb: xHCI runtime DMA allocation failed dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n",
@@ -716,14 +832,14 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->EventCycle = 1;
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress;
link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
xhci->CommandCycle;
xhci->Erst[0].RingSegmentBase = xhci->EventRingPhys;
xhci->Erst[0].RingSegmentBase = xhci->EventRingDeviceAddress;
xhci->Erst[0].RingSegmentSize = XHCI_EVENT_TRBS;
xhci->Erst[0].Reserved = 0;
@@ -731,10 +847,10 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
if (!KSUCCESS(st)) return st;
UsbTrace("usb: xHCI dma dcbaa=0x%016llX cmd=0x%016llX event=0x%016llX erst=0x%016llX\n",
(unsigned long long)xhci->DcbaaPhys,
(unsigned long long)xhci->CommandRingPhys,
(unsigned long long)xhci->EventRingPhys,
(unsigned long long)xhci->ErstPhys);
(unsigned long long)xhci->DcbaaDeviceAddress,
(unsigned long long)xhci->CommandRingDeviceAddress,
(unsigned long long)xhci->EventRingDeviceAddress,
(unsigned long long)xhci->ErstDeviceAddress);
return NTLSTATUS_SUCCESS;
}
@@ -743,8 +859,8 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
uint8_t slots = xhci->MaxSlots;
if (slots == 0) return NTLSTATUS_DEVICE_ERROR;
XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaPhys);
XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingPhys | xhci->CommandCycle);
XhciWrite64(xhci->Op, XHCI_OP_DCBAAP, xhci->DcbaaDeviceAddress);
XhciWrite64(xhci->Op, XHCI_OP_CRCR, xhci->CommandRingDeviceAddress | xhci->CommandCycle);
XhciWrite32(xhci->Op, XHCI_OP_DNCTRL, 0);
XhciWrite32(xhci->Op, XHCI_OP_CONFIG, slots);
@@ -752,30 +868,30 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
XhciWrite32(ir0, 0x00u, XHCI_IMAN_IP); // clear pending, leave disabled until smoke test passes
XhciWrite32(ir0, 0x04u, 0); // no interrupt moderation yet
XhciWrite32(ir0, 0x08u, 1); // one ERST entry
XhciWrite64(ir0, 0x10u, xhci->ErstPhys);
XhciWrite64(ir0, 0x18u, xhci->EventRingPhys);
XhciWrite64(ir0, 0x10u, xhci->ErstDeviceAddress);
XhciWrite64(ir0, 0x18u, xhci->EventRingDeviceAddress);
XhciBarrier();
return NTLSTATUS_SUCCESS;
}
// Implements the command Trb Phys operation.
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB);
// Returns the device address of one command TRB.
uint64_t XhciCommandTrbDeviceAddress(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
}
static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
uint64_t* outDeviceAddress) {
if (!xhci || !xhci->CommandRing)
return NTLSTATUS_INVALID_PARAMETER;
if (xhci->CommandEnqueue >= XHCI_RING_TRBS - 1u) {
XHCI_TRB* link = &xhci->CommandRing[XHCI_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)xhci->CommandRingPhys;
link->ParameterHi = (uint32_t)(xhci->CommandRingPhys >> 32);
link->ParameterLo = (uint32_t)xhci->CommandRingDeviceAddress;
link->ParameterHi = (uint32_t)(xhci->CommandRingDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -787,7 +903,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
uint32_t index = xhci->CommandEnqueue++;
XHCI_TRB* trb = &xhci->CommandRing[index];
uint64_t trbPhys = XhciCommandTrbPhys(xhci, index);
uint64_t trbDeviceAddress = XhciCommandTrbDeviceAddress(xhci, index);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -795,7 +911,7 @@ static KSTATUS XhciCommandRingAppend(XHCI_CONTROLLER* xhci,
XhciBarrier();
trb->Control = controlWithoutCycle | xhci->CommandCycle;
if (outPhys) *outPhys = trbPhys;
if (outDeviceAddress) *outDeviceAddress = trbDeviceAddress;
return NTLSTATUS_SUCCESS;
}
@@ -811,14 +927,14 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
memset(&target, 0, sizeof(target));
uintptr_t cmdFlags = KeAcquireSpinlock(&xhci->CommandLock);
uint64_t trbPhys = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbPhys);
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciCommandRingAppend(xhci, parameter, status, controlWithoutCycle, &trbDeviceAddress);
if (!KSUCCESS(st)) {
KeReleaseSpinlock(&xhci->CommandLock, cmdFlags);
return st;
}
target.CommandPhys = trbPhys;
target.CommandDeviceAddress = trbDeviceAddress;
XhciRegisterWaitTarget(xhci, &target);
XhciBarrier();
@@ -912,7 +1028,7 @@ KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_ADDRESS_DEVICE << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -1059,20 +1175,22 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
// Creates device Memory.
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys);
dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys);
dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingPhys);
dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorPhys);
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextDeviceAddress, &dev->InputContextDma);
dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextDeviceAddress, &dev->DeviceContextDma);
dev->Ep0Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->Ep0RingDeviceAddress, &dev->Ep0RingDma);
dev->DescriptorBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DescriptorDeviceAddress, &dev->DescriptorDma);
if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer)
if (!dev->InputContext || !dev->DeviceContext || !dev->Ep0Ring || !dev->DescriptorBuffer) {
XhciReleaseDeviceDma(dev);
return NTLSTATUS_NO_MEMORY;
}
dev->Ep0Enqueue = 0;
dev->Ep0Cycle = 1;
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -1116,31 +1234,31 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
ep0[0] = 0;
ep0[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_CONTROL | (maxPacket << 16);
ep0[2] = (uint32_t)(dev->Ep0RingPhys | XHCI_EP_DCS);
ep0[3] = (uint32_t)((dev->Ep0RingPhys | XHCI_EP_DCS) >> 32);
ep0[2] = (uint32_t)(dev->Ep0RingDeviceAddress | XHCI_EP_DCS);
ep0[3] = (uint32_t)((dev->Ep0RingDeviceAddress | XHCI_EP_DCS) >> 32);
ep0[4] = 8;
xhci->Dcbaa[dev->SlotId] = dev->DeviceContextPhys;
xhci->Dcbaa[dev->SlotId] = dev->DeviceContextDeviceAddress;
XhciBarrier();
}
// Implements the ep0 Trb Phys operation.
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB);
// Returns the device address of one endpoint-zero TRB.
uint64_t XhciEp0TrbDeviceAddress(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
}
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint64_t parameter,
uint32_t status,
uint32_t controlWithoutCycle,
uint64_t* outPhys) {
uint64_t* outDeviceAddress) {
if (!dev || !dev->Ep0Ring)
return NTLSTATUS_INVALID_PARAMETER;
if (dev->Ep0Enqueue >= XHCI_EP0_RING_TRBS - 1u) {
XHCI_TRB* link = &dev->Ep0Ring[XHCI_EP0_RING_TRBS - 1u];
link->ParameterLo = (uint32_t)dev->Ep0RingPhys;
link->ParameterHi = (uint32_t)(dev->Ep0RingPhys >> 32);
link->ParameterLo = (uint32_t)dev->Ep0RingDeviceAddress;
link->ParameterHi = (uint32_t)(dev->Ep0RingDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -1152,7 +1270,7 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
uint32_t index = dev->Ep0Enqueue++;
XHCI_TRB* trb = &dev->Ep0Ring[index];
uint64_t phys = XhciEp0TrbPhys(dev, index);
uint64_t deviceAddress = XhciEp0TrbDeviceAddress(dev, index);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -1160,14 +1278,14 @@ KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev,
XhciBarrier();
trb->Control = controlWithoutCycle | dev->Ep0Cycle;
if (outPhys) *outPhys = phys;
if (outDeviceAddress) *outDeviceAddress = deviceAddress;
return NTLSTATUS_SUCCESS;
}
KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length,
uint32_t timeoutUs) {
if (!buffer || length == 0 || length > PAGE_SIZE)
@@ -1188,26 +1306,26 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
st = XhciEp0AppendTrb(
dev,
bufferPhys,
bufferDeviceAddress,
length,
(XHCI_TRB_TYPE_DATA << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_DATA_DIR_IN,
NULL);
if (!KSUCCESS(st)) return st;
uint64_t statusTrbPhys = 0;
uint64_t statusTrbDeviceAddress = 0;
st = XhciEp0AppendTrb(
dev,
0,
0,
(XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_IOC,
&statusTrbPhys);
&statusTrbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferDeviceAddress = statusTrbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
@@ -1235,9 +1353,9 @@ KSTATUS XhciControlInTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
const USB_SETUP_PACKET* setup,
void* buffer,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint16_t length) {
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferPhys, length,
return XhciControlInTimeout(xhci, dev, setup, buffer, bufferDeviceAddress, length,
USB_XHCI_CONTROL_TIMEOUT_US);
}
@@ -1254,7 +1372,7 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
NULL);
if (!KSUCCESS(st)) return st;
uint64_t statusTrbPhys = 0;
uint64_t statusTrbDeviceAddress = 0;
st = XhciEp0AppendTrb(
dev,
0,
@@ -1262,12 +1380,12 @@ KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
(XHCI_TRB_TYPE_STATUS << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_STATUS_DIR_IN |
XHCI_TRB_IOC,
&statusTrbPhys);
&statusTrbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = statusTrbPhys;
target.TransferDeviceAddress = statusTrbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = 1;
XhciRegisterWaitTarget(xhci, &target);
@@ -1304,7 +1422,7 @@ KSTATUS XhciGetDeviceDescriptorTimeout(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
if (timeoutUs == 0) timeoutUs = USB_XHCI_CONTROL_TIMEOUT_US;
KSTATUS st = XhciControlInTimeout(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
sizeof(USB_DEVICE_DESCRIPTOR),
timeoutUs);
if (!KSUCCESS(st)) return st;
@@ -1339,7 +1457,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
KSTATUS st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
sizeof(USB_CONFIGURATION_DESCRIPTOR));
if (!KSUCCESS(st)) return st;
@@ -1366,7 +1484,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
setup.wLength = total;
st = XhciControlIn(xhci, dev, &setup,
dev->DescriptorBuffer,
dev->DescriptorPhys,
dev->DescriptorDeviceAddress,
total);
if (!KSUCCESS(st)) return st;
@@ -1415,17 +1533,17 @@ uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) {
return interval;
}
KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
KSTATUS XhciRingAppend(XHCI_TRB* ring, uint64_t ringDeviceAddress,
uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle,
uint64_t parameter, uint32_t status,
uint32_t controlWithoutCycle, uint64_t* outPhys) {
uint32_t controlWithoutCycle, uint64_t* outDeviceAddress) {
if (!ring || !enqueue || !cycle || trbCount < 2)
return NTLSTATUS_INVALID_PARAMETER;
if (*enqueue >= trbCount - 1u) {
XHCI_TRB* link = &ring[trbCount - 1u];
link->ParameterLo = (uint32_t)ringPhys;
link->ParameterHi = (uint32_t)(ringPhys >> 32);
link->ParameterLo = (uint32_t)ringDeviceAddress;
link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32);
link->Status = 0;
XhciBarrier();
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
@@ -1437,7 +1555,7 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
uint32_t index = (*enqueue)++;
XHCI_TRB* trb = &ring[index];
uint64_t phys = ringPhys + (uint64_t)index * sizeof(XHCI_TRB);
uint64_t deviceAddress = ringDeviceAddress + (uint64_t)index * sizeof(XHCI_TRB);
trb->ParameterLo = (uint32_t)parameter;
trb->ParameterHi = (uint32_t)(parameter >> 32);
@@ -1445,16 +1563,16 @@ KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
XhciBarrier();
trb->Control = controlWithoutCycle | *cycle;
if (outPhys) *outPhys = phys;
if (outDeviceAddress) *outDeviceAddress = deviceAddress;
return NTLSTATUS_SUCCESS;
}
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
void XhciInitialiseTransferRing(XHCI_TRB* ring, uint64_t ringDeviceAddress,
uint32_t trbCount, uint8_t cycle) {
memset(ring, 0, trbCount * sizeof(XHCI_TRB));
XHCI_TRB* link = &ring[trbCount - 1u];
link->ParameterLo = (uint32_t)ringPhys;
link->ParameterHi = (uint32_t)(ringPhys >> 32);
link->ParameterLo = (uint32_t)ringDeviceAddress;
link->ParameterHi = (uint32_t)(ringDeviceAddress >> 32);
link->Status = 0;
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_LINK_TC |
@@ -1464,34 +1582,34 @@ void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys,
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
uint8_t endpointId,
XHCI_TRB* ring,
PHYSICAL_ADDRESS ringPhys,
uint64_t ringDeviceAddress,
uint32_t* enqueue,
uint8_t* cycle,
PHYSICAL_ADDRESS bufferPhys,
uint64_t bufferDeviceAddress,
uint32_t length,
uint32_t* outActual) {
if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferPhys || length == 0)
if (!xhci || !dev || !ring || !enqueue || !cycle || !bufferDeviceAddress || length == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (length > USB_STORAGE_MAX_BYTES)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
KSTATUS st = XhciRingAppend(ring,
ringPhys,
ringDeviceAddress,
XHCI_BULK_RING_TRBS,
enqueue,
cycle,
bufferPhys,
bufferDeviceAddress,
length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
XHCI_WAIT_TARGET target;
memset(&target, 0, sizeof(target));
target.TransferPhys = trbPhys;
target.TransferDeviceAddress = trbDeviceAddress;
target.TransferSlotId = dev->SlotId;
target.TransferEndpointId = endpointId;
@@ -1537,7 +1655,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
if (!ctx) return NTLSTATUS_NO_MEMORY;
memset(ctx, 0, sizeof(*ctx));
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingPhys);
ctx->Ring = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &ctx->RingDeviceAddress, &ctx->RingDma);
if (!ctx->Ring) {
MmFreePool(ctx);
return NTLSTATUS_NO_MEMORY;
@@ -1546,7 +1664,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
ctx->Cycle = 1;
ctx->EndpointId = dci;
ctx->Type = pipe->Type;
XhciInitialiseTransferRing(ctx->Ring, ctx->RingPhys, XHCI_PIPE_RING_TRBS, ctx->Cycle);
XhciInitialiseTransferRing(ctx->Ring, ctx->RingDeviceAddress, XHCI_PIPE_RING_TRBS, ctx->Cycle);
memset(dev->InputContext, 0, PAGE_SIZE);
uint32_t* icc = XhciInputControlContext(xhci, dev);
@@ -1574,21 +1692,22 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
} else if (pipe->Type == UsbPipeBulk) {
endpointType = XHCI_EP_TYPE_BULK_OUT;
} else {
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return NTLSTATUS_NOT_SUPPORTED;
}
ep[0] = (uint32_t)interval << 16;
ep[1] = XHCI_EP_CERR_3 | endpointType | ((uint32_t)maxPacket << 16);
ep[2] = (uint32_t)(ctx->RingPhys | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingPhys | XHCI_EP_DCS) >> 32);
ep[2] = (uint32_t)(ctx->RingDeviceAddress | XHCI_EP_DCS);
ep[3] = (uint32_t)((ctx->RingDeviceAddress | XHCI_EP_DCS) >> 32);
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
KSTATUS st = XhciSubmitCommand(
xhci,
dev->InputContextPhys,
dev->InputContextDeviceAddress,
0,
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
((uint32_t)dev->SlotId << 24),
@@ -1596,6 +1715,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
if (!KSUCCESS(st) || result.CompletionCode != XHCI_CC_SUCCESS) {
DbgPrint("usb: Configure pipe endpoint failed slot=%u ep=%u type=%u st=0x%x cc=%u\n",
dev->SlotId, dci, pipe->Type, st, result.CompletionCode);
xhci->Ddk->Dma->FreeCommonBuffer(ctx->RingDma);
MmFreePool(ctx);
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
@@ -1609,7 +1729,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
// Implements the hcd Submit Interrupt Urb operation.
static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (urb->Pipe->Type != UsbPipeInterrupt || !(urb->Pipe->EndpointAddress & USB_ENDPOINT_DIR_IN))
return NTLSTATUS_NOT_SUPPORTED;
@@ -1623,22 +1743,22 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
if (ctx->ActiveTransfer)
return NTLSTATUS_QUEUE_FULL;
uint64_t trbPhys = 0;
uint64_t trbDeviceAddress = 0;
st = XhciRingAppend(ctx->Ring,
ctx->RingPhys,
ctx->RingDeviceAddress,
XHCI_PIPE_RING_TRBS,
&ctx->Enqueue,
&ctx->Cycle,
urb->BufferPhys,
urb->BufferDeviceAddress,
urb->Length,
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
XHCI_TRB_ISP |
XHCI_TRB_IOC,
&trbPhys);
&trbDeviceAddress);
if (!KSUCCESS(st)) return st;
ctx->ActiveTransfer = true;
ctx->ActiveTrbPhys = trbPhys;
ctx->ActiveTrbDeviceAddress = trbDeviceAddress;
ctx->ActiveUrb = urb;
urb->Pending = true;
urb->Cancelled = false;
@@ -1651,7 +1771,7 @@ static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev
return NTLSTATUS_PENDING;
}
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbDeviceAddress,
uint32_t completionCode, uint8_t slotId,
uint8_t endpointId, uint32_t residue) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS || endpointId >= 32)
@@ -1663,12 +1783,12 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (!ctx->ActiveTransfer || ctx->ActiveTrbPhys != trbPhys)
if (!ctx->ActiveTransfer || ctx->ActiveTrbDeviceAddress != trbDeviceAddress)
return;
USB_URB* urb = ctx->ActiveUrb;
ctx->ActiveTransfer = false;
ctx->ActiveTrbPhys = 0;
ctx->ActiveTrbDeviceAddress = 0;
ctx->ActiveUrb = (USB_URB*)0;
if (!urb)
@@ -1704,23 +1824,23 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
// Implements the hcd Submit Bulk Urb operation.
static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferDeviceAddress || urb->Length == 0)
return NTLSTATUS_INVALID_PARAMETER;
uint8_t endpointId = urb->Pipe->EndpointId;
XHCI_TRB* ring = (XHCI_TRB*)0;
PHYSICAL_ADDRESS ringPhys = 0;
uint64_t ringDeviceAddress = 0;
uint32_t* enqueue = (uint32_t*)0;
uint8_t* cycle = (uint8_t*)0;
if (endpointId == dev->MassBulkInEndpointId) {
ring = dev->MassInRing;
ringPhys = dev->MassInRingPhys;
ringDeviceAddress = dev->MassInRingDeviceAddress;
enqueue = &dev->MassInEnqueue;
cycle = &dev->MassInCycle;
} else if (endpointId == dev->MassBulkOutEndpointId) {
ring = dev->MassOutRing;
ringPhys = dev->MassOutRingPhys;
ringDeviceAddress = dev->MassOutRingDeviceAddress;
enqueue = &dev->MassOutEnqueue;
cycle = &dev->MassOutCycle;
} else {
@@ -1732,10 +1852,10 @@ static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB
dev,
endpointId,
ring,
ringPhys,
ringDeviceAddress,
enqueue,
cycle,
urb->BufferPhys,
urb->BufferDeviceAddress,
urb->Length,
&actual);
urb->ActualLength = KSUCCESS(st) ? actual : 0;
@@ -1759,7 +1879,7 @@ static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
st = XhciControlNoData(xhci, dev, &urb->Setup);
urb->ActualLength = 0;
} else if ((urb->Setup.bmRequestType & USB_DIR_IN) != 0) {
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferPhys, (uint16_t)urb->Length);
st = XhciControlIn(xhci, dev, &urb->Setup, urb->Buffer, urb->BufferDeviceAddress, (uint16_t)urb->Length);
urb->ActualLength = KSUCCESS(st) ? urb->Length : 0;
} else {
st = NTLSTATUS_NOT_SUPPORTED;
@@ -1796,9 +1916,29 @@ static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
}
// Implements the hcd Allocate Dma operation.
static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
static void* XhciHcdAllocateDma(USB_HCD* hcd,
size_t pages,
uint64_t* outDeviceAddress,
IO_DMA_BUFFER** outBuffer) {
if (!hcd) return (void*)0;
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext,
pages,
outDeviceAddress,
outBuffer);
}
// Releases the host-controller context attached to a USB pipe.
static void XhciHcdDestroyPipe(USB_HCD* hcd, USB_PIPE* pipe) {
(void)hcd;
if (!pipe || !pipe->HcdPipeContext)
return;
XHCI_PIPE_CONTEXT* ctx = (XHCI_PIPE_CONTEXT*)pipe->HcdPipeContext;
if (gUsbDdk && gUsbDdk->Dma)
gUsbDdk->Dma->FreeCommonBuffer(ctx->RingDma);
ctx->RingDma = (IO_DMA_BUFFER*)0;
ctx->Ring = (XHCI_TRB*)0;
pipe->HcdPipeContext = (void*)0;
MmFreePool(ctx);
}
// Implements the hcd Reset Root Port operation.
@@ -1819,6 +1959,7 @@ const USB_HCD_OPERATIONS gXhciHcdOperations = {
XhciHcdCancelUrb,
XhciHcdResetRootPort,
XhciHcdClearRootPortChanges,
XhciHcdDestroyPipe,
XhciHcdAllocateDma,
};
@@ -1886,7 +2027,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
uint64_t mappedLength = 0;
st = PciMapBar(xhci->Pdo,
0,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&xhci->Mmio,
&mappedLength);
if (!KSUCCESS(st)) {
@@ -1960,17 +2101,37 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return NTLSTATUS_NOT_SUPPORTED;
}
IO_DMA_REQUIREMENTS dmaRequirements;
memset(&dmaRequirements, 0, sizeof(dmaRequirements));
dmaRequirements.MaximumDeviceAddress =
(xhci->Hcc1 & XHCI_HCC1_AC64) ? UINT64_MAX : UINT32_MAX;
dmaRequirements.Alignment = 1;
dmaRequirements.Boundary = 0x10000u;
dmaRequirements.MaximumSegmentLength = 0x10000u;
dmaRequirements.MaximumSegments = XHCI_PIPE_RING_TRBS - 1u;
st = xhci->Ddk->Dma->GetAdapter(xhci->Fdo,
&dmaRequirements,
&xhci->DmaAdapter);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI DMA adapter creation failed st=0x%x\n", st);
return st;
}
st = XhciAllocateRuntimeMemory(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciProgramRuntime(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciStart(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
st = XhciSubmitNoopCommand(xhci);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st))
goto failDma;
XhciPowerRootPorts(xhci);
XhciLogPortStatus(xhci);
@@ -1978,13 +2139,13 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
st = UsbCreateRootHubPdo(&xhci->Hcd);
if (!KSUCCESS(st) && st != NTLSTATUS_ALREADY_EXISTS) {
DbgPrint("usb: root hub PDO creation failed st=0x%x\n", st);
return st;
goto failDma;
}
st = XhciRegisterController(xhci);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI controller registry full st=0x%x\n", st);
return st;
goto failDma;
}
if (xhci->Irq != 0xFFu) {
@@ -2009,6 +2170,16 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
DbgPrint("usb: xHCI host controller initialised; HID boot input active=%u mass-storage will arrive through hotplug\n",
xhci->HidDeviceCount);
return NTLSTATUS_SUCCESS;
failDma:
XhciHaltForDmaRelease(xhci);
XhciUnregisterController(xhci);
if (xhci->Hcd.RootHubPdo) {
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0;
}
XhciReleaseControllerDma(xhci);
return st;
}
// Dispatches pnp.
@@ -2037,22 +2208,22 @@ KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
xhci->SurpriseRemoved = true;
xhci->Running = false;
xhci->InterruptsEnabled = false;
if (xhci->Op) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
XhciWrite32(xhci->Op, XHCI_OP_USBCMD, cmd & ~XHCI_USBCMD_RS);
}
if (xhci->Hcd.RootHubPdo)
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
xhci->Running = false;
xhci->InterruptsEnabled = false;
if (xhci->Hcd.RootHubPdo)
XhciHaltForDmaRelease(xhci);
if (xhci->Hcd.RootHubPdo) {
(void)UsbRemovePdoSubtree(xhci->Hcd.RootHubPdo, true);
xhci->Hcd.RootHubPdo = (DEVICE_OBJECT*)0;
}
XhciUnregisterController(xhci);
XhciReleaseControllerDma(xhci);
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
code/drivers/display/vmwaresvga/vmwaresvga.c
+166 -14
View File
@@ -748,10 +748,103 @@ static KSTATUS VmwSvgaDisplayGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMo
return NTLSTATUS_SUCCESS;
}
// Implements the display Blit operation.
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
// Streams one pixel row into the write-combined SVGA framebuffer.
static void VmwSvgaCopyToFramebuffer(uint32_t* destination,
const uint32_t* source,
size_t pixelCount) {
if (!destination || !source || pixelCount == 0)
return;
while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) {
*destination++ = *source++;
pixelCount--;
}
while (pixelCount >= 16u) {
__m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
__m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
__m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
__m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
_mm_stream_si128((__m128i*)(destination + 0), v0);
_mm_stream_si128((__m128i*)(destination + 4), v1);
_mm_stream_si128((__m128i*)(destination + 8), v2);
_mm_stream_si128((__m128i*)(destination + 12), v3);
source += 16;
destination += 16;
pixelCount -= 16;
}
while (pixelCount >= 4u) {
__m128i value = _mm_loadu_si128((const __m128i*)source);
_mm_stream_si128((__m128i*)destination, value);
source += 4;
destination += 4;
pixelCount -= 4;
}
while (pixelCount != 0) {
*destination++ = *source++;
pixelCount--;
}
}
// Copies one row into the retained SVGA shadow and framebuffer in one read pass.
static void VmwSvgaCopyToShadowAndFramebuffer(uint32_t* shadow,
uint32_t* framebuffer,
const uint32_t* source,
size_t pixelCount) {
if (!shadow || !framebuffer || !source || pixelCount == 0)
return;
while (pixelCount != 0 && ((uintptr_t)framebuffer & 15u) != 0) {
uint32_t value = *source++;
*shadow++ = value;
*framebuffer++ = value;
pixelCount--;
}
while (pixelCount >= 16u) {
__m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
__m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
__m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
__m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
_mm_storeu_si128((__m128i*)(shadow + 0), v0);
_mm_storeu_si128((__m128i*)(shadow + 4), v1);
_mm_storeu_si128((__m128i*)(shadow + 8), v2);
_mm_storeu_si128((__m128i*)(shadow + 12), v3);
_mm_stream_si128((__m128i*)(framebuffer + 0), v0);
_mm_stream_si128((__m128i*)(framebuffer + 4), v1);
_mm_stream_si128((__m128i*)(framebuffer + 8), v2);
_mm_stream_si128((__m128i*)(framebuffer + 12), v3);
source += 16;
shadow += 16;
framebuffer += 16;
pixelCount -= 16;
}
while (pixelCount >= 4u) {
__m128i value = _mm_loadu_si128((const __m128i*)source);
_mm_storeu_si128((__m128i*)shadow, value);
_mm_stream_si128((__m128i*)framebuffer, value);
source += 4;
shadow += 4;
framebuffer += 4;
pixelCount -= 4;
}
while (pixelCount != 0) {
uint32_t value = *source++;
*shadow++ = value;
*framebuffer++ = value;
pixelCount--;
}
}
// Implements the display pitched blit operation.
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0)
if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
DISPLAY_RECT r = { x, y, w, h };
@@ -766,12 +859,60 @@ static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, i
size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t);
for (int row = 0; row < r.H; row++) {
uint32_t* dst = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX;
const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
memcpy(dst, src, bytesPerRow);
}
return NTLSTATUS_SUCCESS;
}
// Presents one pitched rectangle without a shadow-to-framebuffer second pass.
static KSTATUS VmwSvgaDisplayPresentPitched(DISPLAY_DEVICE* device,
int x,
int y,
int w,
int h,
const uint32_t* pixels,
uint32_t pitchBytes) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Framebuffer || !v->Backbuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
return NTLSTATUS_SUCCESS;
int srcOffsetX = r.X - x;
int srcOffsetY = r.Y - y;
size_t pixelCount = (size_t)r.W;
uint64_t bytes = (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
v->Stats.BlitCount++;
v->Stats.PresentCount++;
v->Stats.PresentRectCount++;
v->Stats.DirectPresentCount++;
v->Stats.ScanoutBytes += bytes;
v->Stats.BytesCopied += bytes * 2u;
for (int row = 0; row < r.H; row++) {
const uint8_t* sourceRow = (const uint8_t*)pixels +
(size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
uint32_t* shadow = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
uint32_t* framebuffer = (uint32_t*)(uintptr_t)(v->Framebuffer +
((size_t)(r.Y + row) * v->BytesPerLine) +
((size_t)r.X * sizeof(uint32_t)));
VmwSvgaCopyToShadowAndFramebuffer(shadow, framebuffer, source, pixelCount);
}
_mm_sfence();
_ReadWriteBarrier();
KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H);
if (!KSUCCESS(st))
return st;
return VmwSvgaSyncDevice(v);
}
// Implements the display Fill Rect operation.
static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
@@ -869,13 +1010,19 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT*
DISPLAY_RECT r = rects[i];
if (!VmwSvgaClipRect(v, &r))
continue;
v->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
size_t rowBytes = (size_t)r.W * sizeof(uint32_t);
uint64_t bytes = (uint64_t)(uint32_t)r.W *
(uint64_t)(uint32_t)r.H * sizeof(uint32_t);
v->Stats.BytesCopied += bytes;
v->Stats.ScanoutBytes += bytes;
size_t pixelCount = (size_t)r.W;
for (int row = 0; row < r.H; row++) {
volatile uint8_t* dstBytes = v->Framebuffer + ((size_t)(r.Y + row) * v->BytesPerLine) + ((size_t)r.X * sizeof(uint32_t));
uint32_t* dst = (uint32_t*)(uintptr_t)(v->Framebuffer +
((size_t)(r.Y + row) * v->BytesPerLine) +
((size_t)r.X * sizeof(uint32_t)));
uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
memcpy((void*)dstBytes, src, rowBytes);
VmwSvgaCopyToFramebuffer(dst, src, pixelCount);
}
_mm_sfence();
_ReadWriteBarrier();
KSTATUS st = VmwSvgaUpdate(v, r.X, r.Y, r.W, r.H);
@@ -1019,6 +1166,7 @@ static KSTATUS VmwSvgaDisplayQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* o
static const DISPLAY_DEVICE_OPS gVmwSvgaDisplayOps = {
.GetMode = VmwSvgaDisplayGetMode,
.Blit = VmwSvgaDisplayBlit,
.PresentPitched = VmwSvgaDisplayPresentPitched,
.FillRect = VmwSvgaDisplayFillRect,
.CopyRect = VmwSvgaDisplayCopyRect,
.ReadRect = VmwSvgaDisplayReadRect,
@@ -1060,15 +1208,19 @@ static KSTATUS VmwSvgaSelectBars(VMWSVGA_DEVICE* v) {
return NTLSTATUS_SUCCESS;
}
// Implements the mmio Page Flags operation.
static uint64_t VmwSvgaMmioPageFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_WRITETHROUGH | PAGE_NOCACHE | PAGE_NX;
// Returns the write-combined mapping flags used for the framebuffer aperture.
static uint64_t VmwSvgaFramebufferPageFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_WC | PAGE_NX;
}
// Returns the uncached mapping flags used for device-control apertures.
static uint64_t VmwSvgaRegisterPageFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC | PAGE_NX;
}
// Maps bars.
static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) {
uint64_t mmioFlags = VmwSvgaMmioPageFlags();
KSTATUS st = PciMapBar(v->Pdo, 1, mmioFlags,
KSTATUS st = PciMapBar(v->Pdo, 1, VmwSvgaFramebufferPageFlags(),
(void**)&v->FramebufferMapping, &v->FramebufferBytes);
if (!KSUCCESS(st) || !v->FramebufferMapping) {
DbgPrint("VMWSVGA: PciMapBar framebuffer failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n",
@@ -1079,7 +1231,7 @@ static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) {
return KSUCCESS(st) ? NTLSTATUS_DEVICE_ERROR : st;
}
st = PciMapBar(v->Pdo, 2, mmioFlags,
st = PciMapBar(v->Pdo, 2, VmwSvgaRegisterPageFlags(),
(void**)&v->Fifo, &v->FifoBytes);
if (!KSUCCESS(st) || !v->Fifo) {
DbgPrint("VMWSVGA: PciMapBar FIFO failed st=0x%x virt=0x%llx base=0x%llx len=0x%llx\n",
code/drivers/drivers.ini
+1
View File
@@ -1,5 +1,6 @@
[boot]
pci.sys
hda.sys
ps2.sys
serial.sys
ahci.sys
code/drivers/fs/fat/fat.c
+37 -12
View File
@@ -80,6 +80,11 @@ static void FatCacheEnsureMutex(void) {
gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
gFatCacheMutex.Owner = (struct THREAD*)0;
LIST_INIT(&gFatCacheMutex.WaitList);
LIST_INIT(&gFatCacheMutex.OwnerLink);
gFatCacheMutex.DonationClass = KeSchedulingNormal;
gFatCacheMutex.DonationPriority = PriorityLow;
gFatCacheMutex.HasDonation = 0;
gFatCacheMutex.Abandoned = 0;
gFatCacheMutexReady = 1;
}
KeReleaseSpinlock(&gFatCacheMutexInitLock, flags);
@@ -89,7 +94,7 @@ static int FatCacheAcquire(void) {
if (!KeCanWait())
return 0;
FatCacheEnsureMutex();
return KSUCCESS(KeAcquireMutex(&gFatCacheMutex));
return KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatCacheMutex, KE_WAIT_INFINITE));
}
static void FatCacheRelease(int held) {
@@ -1913,6 +1918,17 @@ KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
return NTLSTATUS_SUCCESS;
}
// Flushes FAT metadata and the backing device write cache.
KSTATUS FatFlush(IO_FILESYSTEM* fs, void* fileCtx) {
(void)fs;
FAT_FILE* f = (FAT_FILE*)fileCtx;
if (!f || !f->Vol)
return NTLSTATUS_INVALID_PARAMETER;
FlushFatCache(f->Vol);
return IoFlushDisk(f->Vol->DriveIndex);
}
// Releases the requested object.
void FatClose(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs;
@@ -2304,11 +2320,14 @@ KSTATUS FatDelete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (!KSUCCESS(s)) return s;
if (!direntLba) return NTLSTATUS_ACCESS_DENIED;
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
if (!sec) return NTLSTATUS_NO_MEMORY;
s = ReadSectors(v, direntLba, 1, sec);
if (KSUCCESS(s)) { sec[direntOffset] = 0xE5; s = WriteSectors(v, direntLba, 1, sec); }
MmFreePool(sec);
uint32_t parentCluster = 0;
SYS_STRING_VIEW filenameView;
s = FatResolveParentForMutation(v, path, &parentCluster, &filenameView);
if (!KSUCCESS(s)) return s;
// Delete the complete entry run so long-name metadata cannot survive the
// short entry and keep writable vvfat mirrors visibly present.
s = FatDeleteDirentRunV(v, parentCluster, filenameView);
if (!KSUCCESS(s)) return s;
if (cluster >= 2) FreeClusterChain(v, cluster);
@@ -2583,7 +2602,7 @@ KSTATUS FatReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
FAT_DIR_CTX *ctx = (FAT_DIR_CTX *)dirCtx;
FAT_VOL *v = ctx->Vol;
for (;;) {
while (true) {
if (ctx->Cursor != ctx->LoadedCursor) {
if (!KSUCCESS(ReadSectors(v, ctx->LoadedLba, ctx->LoadedSectors, ctx->ClusterBuf)))
return NTLSTATUS_NO_MORE_FILES;
@@ -2918,8 +2937,8 @@ static void FatAsyncWriteSubmitNext(FAT_ASYNC_RW* ctx);
static void FatAsyncWorker(void* arg) {
(void)arg;
for (;;) {
KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore);
while (true) {
KSTATUS s = KeWaitSemaphore(gFatAsyncSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(s)) {
KeYield();
continue;
@@ -2959,8 +2978,9 @@ static KSTATUS FatEnsureAsyncWorkerStarted(void) {
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gFatAsyncSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)FatAsyncWorker, 0,
PriorityNormal, &worker);
&parameters, &worker);
(void)worker;
}
@@ -3355,6 +3375,11 @@ static void FatMountEnsureMutex(void) {
gFatMountMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
gFatMountMutex.Owner = (struct THREAD*)0;
LIST_INIT(&gFatMountMutex.WaitList);
LIST_INIT(&gFatMountMutex.OwnerLink);
gFatMountMutex.DonationClass = KeSchedulingNormal;
gFatMountMutex.DonationPriority = PriorityLow;
gFatMountMutex.HasDonation = 0;
gFatMountMutex.Abandoned = 0;
gFatMountMutexReady = 1;
}
KeReleaseSpinlock(&gFatMountMutexInitLock, flags);
@@ -3375,7 +3400,7 @@ static uint32_t FatTryMountDisk(uint8_t disk) {
return 0;
FatMountEnsureMutex();
if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex)))
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE)))
return 0;
if (gDiskScanned[disk]) {
@@ -3509,7 +3534,7 @@ static void FatUnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) {
uint32_t removed = 0;
FatMountEnsureMutex();
if (!KSUCCESS(KeAcquireMutex(&gFatMountMutex)))
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(&gFatMountMutex, KE_WAIT_INFINITE)))
return;
for (uint8_t i = 0; i < FAT_MAX_VOLUMES; i++) {
code/drivers/fs/fat/fat.h
+1
View File
@@ -22,6 +22,7 @@
typedef struct FAT_VOL_S FAT_VOL;
KSTATUS FatOpen(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
KSTATUS FatFlush(IO_FILESYSTEM *fs, void *fileCtx);
void FatClose(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS FatRead(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead);
KSTATUS FatWrite(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten);
code/drivers/fs/fat/fat_mount.c
+1
View File
@@ -19,6 +19,7 @@ void FatInitVtable(IO_FILESYSTEM* fs, FAT_VOL* vol) {
fs->QueryInfo = FatQueryInfo;
fs->Stat = FatStat;
fs->QueryIdentity = FatQueryIdentity;
fs->Flush = FatFlush;
fs->Close = FatClose;
fs->DriverCtx = vol;
fs->OpenDir = FatOpenDir;
code/drivers/net/lwip/lwip_stack.c
+106 -10
View File
@@ -4,6 +4,7 @@
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddk/ddk.h>
#include <ntlite/status.h>
@@ -74,6 +75,15 @@ typedef struct LWIP_ATTACH_CTX {
err_t Result;
} LWIP_ATTACH_CTX;
typedef struct LWIP_RESOLVE_CONTEXT {
SEMAPHORE* Done;
volatile long References;
err_t Result;
u8_t DnsType;
ip_addr_t Address;
char Name[DNS_MAX_NAME_LENGTH];
} LWIP_RESOLVE_CONTEXT;
typedef struct LWIP_PROVIDER_SOCKET {
SEMAPHORE* Lock;
struct netconn* Conn;
@@ -118,6 +128,9 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how);
static KSTATUS LwipProviderSetOption(void* socket, uint32_t option, const void* value, size_t valueSize);
static KSTATUS LwipProviderGetOption(void* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut);
static KSTATUS LwipProviderPoll(void* socket, uint32_t events, uint32_t* returnedEventsOut);
static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context);
static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument);
static void LwipResolveStartCallback(void* argument);
static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut);
static KSTATUS LwipDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp);
@@ -355,7 +368,7 @@ static KSTATUS LwipLoadNetifConfig(const NET_DEVICE* device, LWIP_NETIF_CONFIG*
static KSTATUS LwipSocketLock(LWIP_PROVIDER_SOCKET* s) {
if (!s || !s->Lock)
return NTLSTATUS_INVALID_PARAMETER;
return KeWaitSemaphore(s->Lock);
return KeWaitSemaphore(s->Lock, KE_WAIT_INFINITE);
}
// Implements the socket Unlock operation.
@@ -464,7 +477,7 @@ static KSTATUS LwipSocketWaitRecvNetbuf(LWIP_PROVIDER_SOCKET* s,
uint32_t originalTimeout = s->ReceiveTimeoutMs;
uint32_t elapsed = 0;
for (;;) {
while (true) {
if (LwipSocketCancelRequested(cancelRequested)) {
netconn_set_recvtimeout(s->Conn, originalTimeout);
return NTLSTATUS_CANCELLED;
@@ -510,7 +523,7 @@ static KSTATUS LwipSocketWaitAccept(LWIP_PROVIDER_SOCKET* s,
uint32_t originalTimeout = s->ReceiveTimeoutMs;
uint32_t elapsed = 0;
for (;;) {
while (true) {
if (LwipSocketCancelRequested(cancelRequested)) {
netconn_set_recvtimeout(s->Conn, originalTimeout);
return NTLSTATUS_CANCELLED;
@@ -778,7 +791,7 @@ static KSTATUS LwipStackInitialize(const NTL_DDK* ddk) {
return st;
tcpip_init(LwipInitDone, done);
st = KeWaitSemaphore(done);
st = KeWaitSemaphore(done, KE_WAIT_INFINITE);
ObDereferenceObject(&done->Header);
if (!KSUCCESS(st))
return st;
@@ -826,7 +839,7 @@ static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device) {
err_t err = tcpip_callback(LwipAttachCallback, &ctx);
if (err == ERR_OK)
st = KeWaitSemaphore(done);
st = KeWaitSemaphore(done, KE_WAIT_INFINITE);
else
st = NTLSTATUS_FAILURE;
@@ -1135,7 +1148,7 @@ static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_K* add
return LwipErrToStatus(err);
}
for (;;) {
while (true) {
if (LwipSocketCancelRequested(cancelRequested)) {
(void)netconn_close(s->Conn);
s->ShutdownRead = 1;
@@ -1595,6 +1608,51 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how) {
return LwipErrToStatus(err);
}
// Releases one resolver owner and destroys the request after the final callback or waiter leaves.
static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context) {
if (!context)
return;
if (_InterlockedDecrement(&context->References) != 0)
return;
if (context->Done)
ObDereferenceObject(&context->Done->Header);
MmFreePool(context);
}
// Completes an asynchronous DNS lookup without retaining the caller's kernel stack.
static void LwipResolveDnsDoneCallback(const char* name, const ip_addr_t* address, void* argument) {
LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument;
(void)name;
if (address) {
context->Address = *address;
context->Result = ERR_OK;
} else {
context->Result = ERR_VAL;
}
KeSignalSemaphore(context->Done);
LwipResolveRelease(context);
}
// Starts one DNS lookup in the tcpip thread and transfers its resolver reference to the callback.
static void LwipResolveStartCallback(void* argument) {
LWIP_RESOLVE_CONTEXT* context = (LWIP_RESOLVE_CONTEXT*)argument;
err_t err = dns_gethostbyname_addrtype(context->Name,
&context->Address,
LwipResolveDnsDoneCallback,
context,
context->DnsType);
context->Result = err;
if (err == ERR_INPROGRESS)
return;
KeSignalSemaphore(context->Done);
LwipResolveRelease(context);
}
// Implements the provider Resolve Host operation.
static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut) {
if (!name || !addressOut)
@@ -1610,10 +1668,48 @@ static KSTATUS LwipProviderResolveHost(const char* name, uint32_t family, SYS_SO
else
return NTLSTATUS_NOT_SUPPORTED;
ip_addr_t resolved;
err_t err = netconn_gethostbyname_addrtype(name, &resolved, dnsType);
if (err != ERR_OK)
return LwipErrToStatus(err);
size_t nameLength = strlen(name);
if (nameLength == 0 || nameLength >= DNS_MAX_NAME_LENGTH)
return NTLSTATUS_INVALID_PARAMETER;
LWIP_RESOLVE_CONTEXT* context =
(LWIP_RESOLVE_CONTEXT*)MmAllocatePool(sizeof(*context));
if (!context)
return NTLSTATUS_NO_MEMORY;
memset(context, 0, sizeof(*context));
memcpy(context->Name, name, nameLength + 1);
context->References = 2;
context->Result = ERR_INPROGRESS;
context->DnsType = dnsType;
KSTATUS status = KeSemaphoreCreate(0, 1, &context->Done);
if (!KSUCCESS(status)) {
MmFreePool(context);
return status;
}
err_t queueError = tcpip_try_callback(LwipResolveStartCallback, context);
if (queueError != ERR_OK) {
LwipResolveRelease(context);
LwipResolveRelease(context);
return LwipErrToStatus(queueError);
}
status = KeWaitSemaphore(context->Done, KE_WAIT_INFINITE);
if (!KSUCCESS(status)) {
DbgPrint("lwIP: resolve waiter left name=%s status=0x%x; DNS request remains active\n",
context->Name,
status);
LwipResolveRelease(context);
return status;
}
err_t result = context->Result;
ip_addr_t resolved = context->Address;
LwipResolveRelease(context);
if (result != ERR_OK)
return LwipErrToStatus(result);
if ((family == SYS_AF_INET && !IP_IS_V4(&resolved)) ||
(family == SYS_AF_INET6 && !IP_IS_V6(&resolved)))
return NTLSTATUS_NOT_FOUND;
code/drivers/net/lwip/lwip_sys_arch.c
+14 -23
View File
@@ -117,19 +117,14 @@ u32_t sys_arch_sem_wait(sys_sem_t* sem, u32_t timeout) {
u32_t start = sys_now();
if (timeout == 0) {
KSTATUS st = KeWaitSemaphore((*sem)->Sem);
KSTATUS st = KeWaitSemaphore((*sem)->Sem, KE_WAIT_INFINITE);
return KSUCCESS(st) ? (u32_t)(sys_now() - start) : SYS_ARCH_TIMEOUT;
}
for (;;) {
if (KSUCCESS(KeTrySemaphore((*sem)->Sem)))
return (u32_t)(sys_now() - start);
if ((u32_t)(sys_now() - start) >= timeout)
return SYS_ARCH_TIMEOUT;
KeYield();
}
KSTATUS status = KeWaitSemaphore((*sem)->Sem, timeout);
return KSUCCESS(status)
? (u32_t)(sys_now() - start)
: SYS_ARCH_TIMEOUT;
}
// Implements the sys sem free operation.
@@ -165,7 +160,7 @@ err_t sys_mutex_new(sys_mutex_t* mutex) {
// Implements the sys mutex lock operation.
void sys_mutex_lock(sys_mutex_t* mutex) {
if (mutex && *mutex && (*mutex)->Sem)
(void)KeWaitSemaphore((*mutex)->Sem);
(void)KeWaitSemaphore((*mutex)->Sem, KE_WAIT_INFINITE);
}
// Implements the sys mutex unlock operation.
@@ -279,28 +274,23 @@ u32_t sys_arch_mbox_fetch(sys_mbox_t* mbox, void** msg, u32_t timeout) {
u32_t start = sys_now();
if (timeout == 0) {
KSTATUS st = KeWaitSemaphore((*mbox)->Items);
KSTATUS st = KeWaitSemaphore((*mbox)->Items, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return SYS_ARCH_TIMEOUT;
return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start));
}
for (;;) {
if (KSUCCESS(KeTrySemaphore((*mbox)->Items)))
return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start));
if ((u32_t)(sys_now() - start) >= timeout)
return SYS_ARCH_TIMEOUT;
KeYield();
}
KSTATUS status = KeWaitSemaphore((*mbox)->Items, timeout);
if (!KSUCCESS(status))
return SYS_ARCH_TIMEOUT;
return LwipMboxPopAfterItem(mbox, msg, (u32_t)(sys_now() - start));
}
u32_t sys_arch_mbox_tryfetch(sys_mbox_t* mbox, void** msg) {
if (!mbox || !*mbox || !(*mbox)->Valid)
return SYS_MBOX_EMPTY;
if (!KSUCCESS(KeTrySemaphore((*mbox)->Items)))
if (!KSUCCESS(KeWaitSemaphore((*mbox)->Items, 0)))
return SYS_MBOX_EMPTY;
return LwipMboxPopAfterItem(mbox, msg, 0);
@@ -347,9 +337,10 @@ sys_thread_t sys_thread_new(const char* name, lwip_thread_fn thread, void* arg,
// function pointer that lwIP supplied, with exactly the argument it supplied.
THREAD* t = 0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(kp);
KSTATUS st = KeCreateKernelThread((uintptr_t)thread,
(uintptr_t)arg,
kp,
&parameters,
&t);
if (!KSUCCESS(st))
KePanic(KE_PANIC_CODE_DRIVER, "lwIP: failed to create kernel thread: 0x%x\n", st);
code/drivers/net/rtl8139/rtl8139.c
+81 -30
View File
@@ -82,29 +82,79 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) {
r->NetDevice.HardwareAddressLength = 6;
}
// Creates dma.
static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma);
r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma);
if (!r->RxPhys || !r->TxPhys)
return NTLSTATUS_NO_MEMORY;
// Stops bus mastering before DMA memory can be released.
static void Rtl8139StopDma(RTL8139_DEVICE* r) {
if (!r || !r->Started)
return;
r->Started = false;
if (r->SurpriseRemoved)
return;
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
Rtl8139Write8(r, RTL8139_REG_CMD, 0);
KSTATUS status = Rtl8139Reset(r);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_DRIVER,
"RTL8139: failed to stop DMA before release status=0x%x\n",
status);
}
// RTL8139 has 32-bit DMA address registers. This is fine for typical QEMU
// and low-memory hobby OS configs, but make failure explicit.
if ((r->RxPhys + RTL8139_RX_ALLOC_SIZE - 1) > 0xFFFFFFFFULL ||
(r->TxPhys + RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE - 1) > 0xFFFFFFFFULL) {
DbgPrint("RTL8139: DMA allocation above 4GiB is not supported by this NIC\n");
return NTLSTATUS_NOT_SUPPORTED;
// Releases the common buffers and adapter owned by one RTL8139 device.
static void Rtl8139ReleaseDma(RTL8139_DEVICE* r) {
if (!r || !gRtl8139Ddk || !gRtl8139Ddk->Dma)
return;
gRtl8139Ddk->Dma->FreeCommonBuffer(r->TxDma);
gRtl8139Ddk->Dma->FreeCommonBuffer(r->RxDma);
r->TxDma = (IO_DMA_BUFFER*)0;
r->RxDma = (IO_DMA_BUFFER*)0;
r->TxBuffers = (uint8_t*)0;
r->RxBuffer = (uint8_t*)0;
r->TxDeviceAddress = 0;
r->RxDeviceAddress = 0;
gRtl8139Ddk->Dma->PutAdapter(r->DmaAdapter);
r->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
// Allocates the receive ring and transmit slots through the device DMA adapter.
static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
IO_DMA_REQUIREMENTS requirements;
memset(&requirements, 0, sizeof(requirements));
requirements.MaximumDeviceAddress = 0xFFFFFFFFULL;
requirements.Alignment = 4;
requirements.MaximumSegmentLength = RTL8139_TX_BUF_SIZE;
requirements.MaximumSegments = 1;
KSTATUS status = gRtl8139Ddk->Dma->GetAdapter(r->Fdo,
&requirements,
&r->DmaAdapter);
if (!KSUCCESS(status))
return status;
status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter,
RTL8139_RX_ALLOC_SIZE,
PAGE_SIZE,
&r->RxDma);
if (!KSUCCESS(status)) {
Rtl8139ReleaseDma(r);
return status;
}
r->RxBuffer = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->RxPhys);
r->TxBuffers = (uint8_t*)gRtl8139Ddk->Memory->PhysToVirt(r->TxPhys);
memset(r->RxBuffer, 0, PAGE_SIZE * RTL8139_RX_PAGES);
memset(r->TxBuffers, 0, PAGE_SIZE * RTL8139_TX_PAGES);
status = gRtl8139Ddk->Dma->AllocateCommonBuffer(r->DmaAdapter,
RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE,
PAGE_SIZE,
&r->TxDma);
if (!KSUCCESS(status)) {
Rtl8139ReleaseDma(r);
return status;
}
r->RxBuffer = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->RxDma);
r->TxBuffers = (uint8_t*)gRtl8139Ddk->Dma->GetCommonBufferAddress(r->TxDma);
r->RxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->RxDma);
r->TxDeviceAddress = gRtl8139Ddk->Dma->GetCommonBufferDeviceAddress(r->TxDma);
DbgPrint("RTL8139: rx=0x%016llX tx=0x%016llX\n",
(unsigned long long)r->RxPhys,
(unsigned long long)r->TxPhys);
(unsigned long long)r->RxDeviceAddress,
(unsigned long long)r->TxDeviceAddress);
return NTLSTATUS_SUCCESS;
}
@@ -136,13 +186,13 @@ static void Rtl8139ProgramHardware(RTL8139_DEVICE* r) {
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
Rtl8139Write16(r, RTL8139_REG_ISR, 0xFFFFu);
Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxPhys);
Rtl8139Write32(r, RTL8139_REG_RBSTART, (uint32_t)r->RxDeviceAddress);
r->RxOffset = 0;
Rtl8139Write16(r, RTL8139_REG_CAPR, 0xFFF0u); // CAPR is rxOffset - 16.
for (uint32_t i = 0; i < RTL8139_TX_SLOTS; i++) {
PHYSICAL_ADDRESS p = r->TxPhys + (PHYSICAL_ADDRESS)i * RTL8139_TX_BUF_SIZE;
Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)p);
uint64_t address = r->TxDeviceAddress + (uint64_t)i * RTL8139_TX_BUF_SIZE;
Rtl8139Write32(r, RTL8139_REG_TSAD(i), (uint32_t)address);
}
Rtl8139Write32(r, RTL8139_REG_TCR,
@@ -360,7 +410,7 @@ static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) {
KSTATUS st = PciMapBar(r->Pdo,
best.Index,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&r->Mmio,
&r->RegisterBytes);
if (!KSUCCESS(st))
@@ -460,8 +510,10 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) {
r->NetDevice.SendFrame = Rtl8139SendFrame;
st = Rtl8139RegisterDevice(r);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
Rtl8139ReleaseDma(r);
return st;
}
KeIrqRegister(r->Irq, Rtl8139Isr);
r->Started = true;
@@ -470,9 +522,9 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) {
st = NetRegisterDevice(&r->NetDevice);
if (!KSUCCESS(st)) {
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
r->Started = false;
Rtl8139StopDma(r);
Rtl8139UnregisterDevice(r);
Rtl8139ReleaseDma(r);
return st;
}
@@ -503,17 +555,16 @@ KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
r->SurpriseRemoved = true;
r->Started = false;
device->PnpState = DevicePnpSurpriseRemoved;
if (r->Started)
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
r->Removed = true;
device->PnpState = DevicePnpDeleted;
if (r->Started)
Rtl8139Write16(r, RTL8139_REG_IMR, 0);
Rtl8139StopDma(r);
Rtl8139UnregisterDevice(r);
Rtl8139ReleaseDma(r);
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
@@ -555,7 +606,7 @@ KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer)
if (!ddk || !ddk->Dma || !ddk->Timer)
return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("RTL8139: driver loading...\n");
code/drivers/net/rtl8139/rtl8139.h
+5 -2
View File
@@ -115,8 +115,11 @@ typedef struct RTL8139_DEVICE {
volatile uint8_t* Mmio;
uint16_t IoBase;
PHYSICAL_ADDRESS RxPhys;
PHYSICAL_ADDRESS TxPhys;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* RxDma;
IO_DMA_BUFFER* TxDma;
uint64_t RxDeviceAddress;
uint64_t TxDeviceAddress;
uint8_t* RxBuffer;
uint8_t* TxBuffers;
uint8_t RxScratch[RTL8139_TX_BUF_SIZE];
code/drivers/net/rtl816x/rtl816x.c
+158 -49
View File
@@ -83,48 +83,117 @@ static void RtlReadMac(RTL816X_DEVICE* r) {
r->NetDevice.HardwareAddressLength = 6;
}
// Implements the rtl Allocate Dma operation.
// Stops bus mastering before DMA memory can be released.
static void RtlStopDma(RTL816X_DEVICE* r) {
if (!r || !r->Started)
return;
r->Started = false;
if (r->SurpriseRemoved)
return;
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlWrite8(r, RTL_REG_CHIP_CMD, 0);
KSTATUS status = RtlReset(r);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_DRIVER,
"RTL816X: failed to stop DMA before release status=0x%x\n",
status);
}
// Releases the common buffers and adapter owned by one RTL816x device.
static void RtlReleaseDma(RTL816X_DEVICE* r) {
if (!r || !gRtl816xDdk || !gRtl816xDdk->Dma)
return;
gRtl816xDdk->Dma->FreeCommonBuffer(r->TxBuffersDma);
gRtl816xDdk->Dma->FreeCommonBuffer(r->RxBuffersDma);
gRtl816xDdk->Dma->FreeCommonBuffer(r->TxRingDma);
gRtl816xDdk->Dma->FreeCommonBuffer(r->RxRingDma);
r->TxBuffersDma = (IO_DMA_BUFFER*)0;
r->RxBuffersDma = (IO_DMA_BUFFER*)0;
r->TxRingDma = (IO_DMA_BUFFER*)0;
r->RxRingDma = (IO_DMA_BUFFER*)0;
r->TxBuffers = (uint8_t*)0;
r->RxBuffers = (uint8_t*)0;
r->TxRing = (RTL_DESC*)0;
r->RxRing = (RTL_DESC*)0;
r->TxBuffersDeviceAddress = 0;
r->RxBuffersDeviceAddress = 0;
r->TxRingDeviceAddress = 0;
r->RxRingDeviceAddress = 0;
gRtl816xDdk->Dma->PutAdapter(r->DmaAdapter);
r->DmaAdapter = (IO_DMA_ADAPTER*)0;
}
// Allocates descriptor rings and packet buffers through the device DMA adapter.
static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma);
r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma);
IO_DMA_REQUIREMENTS requirements;
memset(&requirements, 0, sizeof(requirements));
requirements.MaximumDeviceAddress = (uint64_t)-1;
requirements.Alignment = 1;
requirements.MaximumSegmentLength = RTL_BUF_SIZE;
requirements.MaximumSegments = RTL_TX_COUNT;
if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys)
return NTLSTATUS_NO_MEMORY;
KSTATUS status = gRtl816xDdk->Dma->GetAdapter(r->Fdo,
&requirements,
&r->DmaAdapter);
if (!KSUCCESS(status))
return status;
r->RxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->RxRingPhys);
r->TxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->TxRingPhys);
r->RxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->RxBuffersPhys);
r->TxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->TxBuffersPhys);
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_RING_PAGES,
PAGE_SIZE,
&r->RxRingDma);
if (!KSUCCESS(status)) goto fail;
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_RING_PAGES,
PAGE_SIZE,
&r->TxRingDma);
if (!KSUCCESS(status)) goto fail;
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_RXBUF_PAGES,
PAGE_SIZE,
&r->RxBuffersDma);
if (!KSUCCESS(status)) goto fail;
status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
PAGE_SIZE * RTL_TXBUF_PAGES,
PAGE_SIZE,
&r->TxBuffersDma);
if (!KSUCCESS(status)) goto fail;
memset(r->RxRing, 0, PAGE_SIZE * RTL_RING_PAGES);
memset(r->TxRing, 0, PAGE_SIZE * RTL_RING_PAGES);
memset(r->RxBuffers, 0, PAGE_SIZE * RTL_RXBUF_PAGES);
memset(r->TxBuffers, 0, PAGE_SIZE * RTL_TXBUF_PAGES);
r->RxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxRingDma);
r->TxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxRingDma);
r->RxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxBuffersDma);
r->TxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxBuffersDma);
r->RxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxRingDma);
r->TxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxRingDma);
r->RxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxBuffersDma);
r->TxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxBuffersDma);
DbgPrint("RTL816X: rxRing=0x%016llX txRing=0x%016llX rxBuf=0x%016llX txBuf=0x%016llX\n",
(unsigned long long)r->RxRingPhys,
(unsigned long long)r->TxRingPhys,
(unsigned long long)r->RxBuffersPhys,
(unsigned long long)r->TxBuffersPhys);
(unsigned long long)r->RxRingDeviceAddress,
(unsigned long long)r->TxRingDeviceAddress,
(unsigned long long)r->RxBuffersDeviceAddress,
(unsigned long long)r->TxBuffersDeviceAddress);
return NTLSTATUS_SUCCESS;
fail:
RtlReleaseDma(r);
return status;
}
// Implements the rtl Init Descriptors operation.
static void RtlInitDescriptors(RTL816X_DEVICE* r) {
for (uint32_t i = 0; i < RTL_RX_COUNT; i++) {
PHYSICAL_ADDRESS p = r->RxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE;
r->RxRing[i].addrLo = (uint32_t)p;
r->RxRing[i].addrHi = (uint32_t)(p >> 32);
uint64_t address = r->RxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE;
r->RxRing[i].addrLo = (uint32_t)address;
r->RxRing[i].addrHi = (uint32_t)(address >> 32);
r->RxRing[i].opts2 = 0;
r->RxRing[i].opts1 = RTL_DESC_OWN | RTL_BUF_SIZE | ((i == RTL_RX_COUNT - 1) ? RTL_DESC_EOR : 0);
}
for (uint32_t i = 0; i < RTL_TX_COUNT; i++) {
PHYSICAL_ADDRESS p = r->TxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE;
r->TxRing[i].addrLo = (uint32_t)p;
r->TxRing[i].addrHi = (uint32_t)(p >> 32);
uint64_t address = r->TxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE;
r->TxRing[i].addrLo = (uint32_t)address;
r->TxRing[i].addrHi = (uint32_t)(address >> 32);
r->TxRing[i].opts2 = 0;
r->TxRing[i].opts1 = (i == RTL_TX_COUNT - 1) ? RTL_DESC_EOR : 0;
}
@@ -141,12 +210,12 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) {
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingPhys);
RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingPhys >> 32));
RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingPhys);
RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingPhys >> 32));
RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingPhys);
RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingPhys >> 32));
RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingDeviceAddress);
RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32));
RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingDeviceAddress);
RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32));
RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingDeviceAddress);
RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingDeviceAddress >> 32));
RtlWrite16(r, RTL_REG_CPLUS_CMD, RtlRead16(r, RTL_REG_CPLUS_CMD) & ~(uint16_t)0x00E0u);
RtlWrite16(r, RTL_REG_RX_MAXSIZE, RTL_BUF_SIZE);
@@ -366,7 +435,7 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) {
KSTATUS st = PciMapBar(r->Pdo,
best.Index,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&r->Mmio,
&r->RegisterBytes);
if (!KSUCCESS(st))
@@ -400,6 +469,41 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
return NTLSTATUS_NOT_FOUND;
}
// Configures a dedicated MSI vector and falls back to a routed INTx line.
static KSTATUS RtlConfigureInterrupt(RTL816X_DEVICE* r) {
if (!r || !r->Pdo)
return NTLSTATUS_INVALID_PARAMETER;
r->Irq = 0xFFu;
r->UsingMsi = false;
KSTATUS msiStatus = PciEnableMsiAuto(r->Pdo, &r->Irq);
if (KSUCCESS(msiStatus)) {
r->UsingMsi = true;
DbgPrint("RTL816X: using MSI IRQ slot %u\n", r->Irq);
return NTLSTATUS_SUCCESS;
}
DbgPrint("RTL816X: MSI setup failed status=0x%x; attempting explicit INTx fallback\n",
msiStatus);
KSTATUS intxStatus = PciGetInterrupt(r->Pdo, &r->Irq);
if (!KSUCCESS(intxStatus))
return intxStatus;
if (r->Irq == 0xFFu || r->Irq >= 24u)
return NTLSTATUS_NOT_SUPPORTED;
intxStatus = PciEnableDevice(r->Pdo,
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX |
(r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
if (!KSUCCESS(intxStatus))
return intxStatus;
DbgPrint("RTL816X: using legacy INTx IRQ %u\n", r->Irq);
return NTLSTATUS_SUCCESS;
}
// Implements the rtl Start Device operation.
static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
if (!r || r->Magic != RTL816X_DEVICE_MAGIC || !r->Pdo || !r->Fdo)
@@ -428,18 +532,15 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
st = PciEnableDevice(r->Pdo,
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX |
PCI_DISABLE_INTX |
(r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
if (!KSUCCESS(st)) {
DbgPrint("RTL816X: PciEnableDevice failed: 0x%x\n", st);
return st;
}
st = PciGetInterrupt(r->Pdo, &r->Irq);
if (!KSUCCESS(st) || r->Irq == 0xFFu) {
DbgPrint("RTL816X: no legacy PCI IRQ line wired\n");
return KSUCCESS(st) ? NTLSTATUS_NOT_SUPPORTED : st;
}
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
st = RtlReset(r);
if (!KSUCCESS(st)) {
@@ -447,12 +548,18 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
return st;
}
st = RtlConfigureInterrupt(r);
if (!KSUCCESS(st)) {
DbgPrint("RTL816X: no usable MSI or INTx interrupt status=0x%x\n", st);
return st;
}
RtlReadMac(r);
DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u\n",
DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u mode=%s\n",
r->NetDevice.HardwareAddress[0], r->NetDevice.HardwareAddress[1],
r->NetDevice.HardwareAddress[2], r->NetDevice.HardwareAddress[3],
r->NetDevice.HardwareAddress[4], r->NetDevice.HardwareAddress[5],
r->Irq);
r->Irq, r->UsingMsi ? "MSI" : "INTx");
st = RtlAllocateDma(r);
if (!KSUCCESS(st))
@@ -466,8 +573,10 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
r->NetDevice.SendFrame = RtlSendFrame;
st = RtlRegisterDevice(r);
if (!KSUCCESS(st))
if (!KSUCCESS(st)) {
RtlReleaseDma(r);
return st;
}
KeIrqRegister(r->Irq, RtlIsr);
r->Started = true;
@@ -476,13 +585,14 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
st = NetRegisterDevice(&r->NetDevice);
if (!KSUCCESS(st)) {
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
r->Started = false;
RtlStopDma(r);
RtlUnregisterDevice(r);
RtlReleaseDma(r);
return st;
}
DbgPrint("RTL816X: initialized; RX/TX enabled on IRQ %u\n", r->Irq);
DbgPrint("RTL816X: initialized; RX/TX enabled on %s IRQ %u\n",
r->UsingMsi ? "MSI" : "INTx", r->Irq);
return NTLSTATUS_SUCCESS;
}
@@ -509,17 +619,16 @@ KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
r->SurpriseRemoved = true;
r->Started = false;
device->PnpState = DevicePnpSurpriseRemoved;
if (r->Started)
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
r->Removed = true;
device->PnpState = DevicePnpDeleted;
if (r->Started)
RtlWrite16(r, RTL_REG_INTR_MASK, 0);
RtlStopDma(r);
RtlUnregisterDevice(r);
RtlReleaseDma(r);
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
@@ -561,7 +670,7 @@ KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer)
if (!ddk || !ddk->Dma || !ddk->Timer)
return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("RTL816X: driver loading...\n");
code/drivers/net/rtl816x/rtl816x.h
+10 -4
View File
@@ -117,6 +117,7 @@ typedef struct RTL816X_DEVICE {
PCI_BAR RegisterBar;
uint64_t RegisterBytes;
uint8_t Irq;
bool UsingMsi;
bool Started;
bool Removed;
@@ -126,15 +127,20 @@ typedef struct RTL816X_DEVICE {
volatile uint8_t* Mmio;
uint16_t IoBase;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* RxRingDma;
IO_DMA_BUFFER* TxRingDma;
IO_DMA_BUFFER* RxBuffersDma;
IO_DMA_BUFFER* TxBuffersDma;
RTL_DESC* RxRing;
RTL_DESC* TxRing;
PHYSICAL_ADDRESS RxRingPhys;
PHYSICAL_ADDRESS TxRingPhys;
uint64_t RxRingDeviceAddress;
uint64_t TxRingDeviceAddress;
uint8_t* RxBuffers;
uint8_t* TxBuffers;
PHYSICAL_ADDRESS RxBuffersPhys;
PHYSICAL_ADDRESS TxBuffersPhys;
uint64_t RxBuffersDeviceAddress;
uint64_t TxBuffersDeviceAddress;
uint32_t RxIndex;
uint32_t TxIndex;
code/drivers/storage/ahci/ahci.c
+366 -104
View File
@@ -9,6 +9,11 @@ static AHCI_CONTROLLER* g_controllers[MAX_AHCI_CONTROLLERS];
static int g_controllerCount;
static uint8_t g_nextDiskIndex;
#define AHCI_COMMAND_TIMEOUT_MS 10000ull
#define AHCI_PORT_ERROR_MASK \
(PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define AHCI_PORT_TERMINAL_MASK (PORT_IS_DHRS | AHCI_PORT_ERROR_MASK)
static const IO_DEVICE_ID g_ahciIds[] = {
{ IoBusPci, { "PCI\\CC_010601", sizeof("PCI\\CC_010601") - 1 } },
};
@@ -21,20 +26,21 @@ static const IO_DRIVER_ID_TABLE g_ahciIdTable = {
// Helpers.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl);
static void AhciUnregisterController(AHCI_CONTROLLER* ctl);
// Accessors for port structure memory at known offsets from structVirt/Phys.
// Accessors for port structure memory at known CPU and device-address offsets.
static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) {
return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST);
}
// Implements the cmd list phys operation.
static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_CMDLIST;
// Returns the command-list device address.
static inline uint64_t cmd_list_device_address(AHCI_DRIVE *drv) {
return drv->structDeviceAddress + PORT_OFF_CMDLIST;
}
// Implements the fis phys operation.
static inline uint64_t fis_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_FIS;
// Returns the received-FIS device address.
static inline uint64_t fis_device_address(AHCI_DRIVE *drv) {
return drv->structDeviceAddress + PORT_OFF_FIS;
}
static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
@@ -43,9 +49,9 @@ static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
+ (uint32_t)slot * PORT_CMDTBL_SIZE);
}
// Implements the cmd tbl phys operation.
static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
return drv->structPhys + PORT_OFF_CMDTBLS
// Returns one command-table device address.
static inline uint64_t cmd_tbl_device_address(AHCI_DRIVE *drv, int slot) {
return drv->structDeviceAddress + PORT_OFF_CMDTBLS
+ (uint64_t)slot * PORT_CMDTBL_SIZE;
}
@@ -53,53 +59,108 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
// Port memory initialisation.
// Stops one drive worker before its queue and DMA storage are released.
static void AhciStopWorker(AHCI_DRIVE* drv) {
if (!drv)
return;
drv->workerRunning = false;
if (drv->irpQueue.Semaphore)
KeSignalSemaphore(drv->irpQueue.Semaphore);
if (drv->workerThread && drv->workerFinished)
(void)KeWaitForEvent(drv->workerFinished, 0);
if (drv->workerThread) {
ObDereferenceObject((OBJECT*)drv->workerThread);
drv->workerThread = (THREAD*)0;
}
if (drv->workerFinished) {
ObDereferenceObject(&drv->workerFinished->Header);
drv->workerFinished = (EVENT*)0;
}
}
// Releases the non-DMA synchronization objects owned by one AHCI drive.
static void AhciReleaseDriveRuntime(AHCI_DRIVE* drv) {
if (!drv)
return;
AhciStopWorker(drv);
if (drv->removeLockReady) {
IoDestroyRemoveLock(&drv->removeLock);
drv->removeLockReady = false;
}
if (drv->irpQueue.Semaphore) {
ObDereferenceObject(&drv->irpQueue.Semaphore->Header);
drv->irpQueue.Semaphore = (SEMAPHORE*)0;
}
if (drv->completionEvent) {
ObDereferenceObject(&drv->completionEvent->Header);
drv->completionEvent = (EVENT*)0;
}
}
// Releases the DMA buffers owned by one AHCI drive.
static void AhciReleaseDriveDma(AHCI_DRIVE* drv) {
if (!drv || !gAhciDdk || !gAhciDdk->Dma)
return;
if (drv->port && drv->controller &&
!drv->surpriseRemoved && !drv->controller->surpriseRemoved)
port_stop(drv->port);
gAhciDdk->Dma->FreeCommonBuffer(drv->bounceDma);
gAhciDdk->Dma->FreeCommonBuffer(drv->structDma);
drv->bounceDma = (IO_DMA_BUFFER*)0;
drv->structDma = (IO_DMA_BUFFER*)0;
drv->bounceDeviceAddress = 0;
drv->structDeviceAddress = 0;
drv->bounceVirt = (void*)0;
drv->structVirt = (void*)0;
}
// Allocates and programs one port's command, FIS, and bounce buffers.
static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
if (!drv || !drv->controller || !drv->controller->dmaAdapter)
return NTLSTATUS_INVALID_PARAMETER;
drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma);
if (!drv->structPhys) return NTLSTATUS_NO_MEMORY;
drv->structVirt = gAhciDdk->Memory->PhysToVirt(drv->structPhys);
// Force NOCACHE on DMA descriptor memory.
MmPteProtectRange((uintptr_t)drv->structVirt,
PORT_PAGES_STRUCT * 4096, flags);
memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096);
KSTATUS status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter,
PORT_PAGES_STRUCT * PAGE_SIZE,
PAGE_SIZE,
&drv->structDma);
if (!KSUCCESS(status))
return status;
drv->structVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->structDma);
drv->structDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->structDma);
drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma);
if (!drv->bouncePhys) return NTLSTATUS_NO_MEMORY;
drv->bounceVirt = gAhciDdk->Memory->PhysToVirt(drv->bouncePhys);
// Bounce buffer: NOCACHE to ensure CPU writes reach physical memory
// before the HBA's DMA engine reads them.
MmPteProtectRange((uintptr_t)drv->bounceVirt,
BOUNCE_PAGES * 4096, flags);
memset(drv->bounceVirt, 0, BOUNCE_BYTES);
status = gAhciDdk->Dma->AllocateCommonBuffer(drv->controller->dmaAdapter,
BOUNCE_BYTES,
PAGE_SIZE,
&drv->bounceDma);
if (!KSUCCESS(status)) {
AhciReleaseDriveDma(drv);
return status;
}
drv->bounceVirt = gAhciDdk->Dma->GetCommonBufferAddress(drv->bounceDma);
drv->bounceDeviceAddress = gAhciDdk->Dma->GetCommonBufferDeviceAddress(drv->bounceDma);
HBA_PORT *port = drv->port;
port_stop(port);
// Set command list base.
port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF);
port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32);
port->clb = (uint32_t)(cmd_list_device_address(drv) & 0xFFFFFFFF);
port->clbu = (uint32_t)(cmd_list_device_address(drv) >> 32);
port->fb = (uint32_t)(fis_device_address(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_device_address(drv) >> 32);
// Set FIS base.
port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_phys(drv) >> 32);
// Initialise all 32 command headers, pointing each to its command table.
HBA_CMD_HEADER *hdrs = cmd_list(drv);
for (int i = 0; i < NUM_CMD_SLOTS; i++) {
hdrs[i].prdtl = PORT_PRDT_COUNT;
uint64_t tbl = cmd_tbl_phys(drv, i);
hdrs[i].ctba = (uint32_t)(tbl & 0xFFFFFFFF);
hdrs[i].ctbau = (uint32_t)(tbl >> 32);
uint64_t table = cmd_tbl_device_address(drv, i);
hdrs[i].ctba = (uint32_t)(table & 0xFFFFFFFF);
hdrs[i].ctbau = (uint32_t)(table >> 32);
}
// Clear any stale interrupts then start the port engine.
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
port_start(port);
return NTLSTATUS_SUCCESS;
}
@@ -125,16 +186,22 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
// Issue slot 0.
port->ci = (1u << 0);
// Poll until slot 0 clears.
while (true) {
if (!(port->ci & (1u << 0))) break;
if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) {
// Poll until slot 0 clears, but never let early-init I/O hang forever.
spin = 0;
while ((port->ci & (1u << 0)) && spin < POLL_TIMEOUT) {
if (port->is & AHCI_PORT_ERROR_MASK) {
DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n",
port->is, port->tfd, port->serr);
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
return NTLSTATUS_DEVICE_ERROR;
}
spin++;
}
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: command timed out while polling CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, port->is, port->serr);
return NTLSTATUS_TIMEOUT;
}
if (port->is & PORT_IS_TFES) {
@@ -146,6 +213,30 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
return NTLSTATUS_SUCCESS;
}
// Recovers a port after a command that cannot be trusted to have retired.
static void AhciRecoverPort(AHCI_DRIVE *drv) {
if (!drv || !drv->port)
return;
HBA_PORT *port = drv->port;
uint32_t savedIe = port->ie;
// Keep reset-generated FIS traffic from satisfying a command waiter.
port->ie = 0;
KSTATUS status = port_comreset(drv);
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
drv->lastIs = 0;
if (drv->completionEvent)
KeClearEvent(drv->completionEvent);
port->ie = savedIe;
if (!KSUCCESS(status)) {
DbgPrint("AHCI: port recovery failed st=0x%x CI=0x%x TFD=0x%x SSTS=0x%x\n",
status, port->ci, port->tfd, port->ssts);
}
}
// Interrupt bottom half. The ISR owns hardware acknowledgement and status
// capture; this DPC wakes the worker thread after the hard IRQ has unwound.
@@ -191,7 +282,11 @@ static void AhciIsr(void) {
// Clear the HBA-level bit.
ctl->hba->is = portBit;
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
// DMA Setup, PIO Setup, and other progress interrupts are not
// command completion. Only wake the worker for D2H completion or
// an error that terminates the command.
if (portIs & AHCI_PORT_TERMINAL_MASK)
KeInsertQueueDpc(&drv->completionDpc, (void *)0, (void *)0);
}
}
}
@@ -206,8 +301,19 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
// Clear stale IS and snapshot before issuing so we don't mis-read a
// leftover value from a prior command.
// A serialized slot must be idle before reuse. A stale active bit means a
// prior command escaped normal completion and must be recovered first.
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: slot 0 still active before issue CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, port->is, port->serr);
AhciRecoverPort(drv);
if (port->ci & (1u << 0))
return NTLSTATUS_DEVICE_ERROR;
}
// Clear stale software and hardware completion state before submission.
// The event can otherwise retain a late DPC signal from an older command.
KeClearEvent(drv->completionEvent);
port->is = (uint32_t)-1;
drv->lastIs = 0;
@@ -216,20 +322,54 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT)
spin++;
if (spin == POLL_TIMEOUT) {
DbgPrint("AHCI: port hung (pre-issue)\n");
DbgPrint("AHCI: port hung before issue TFD=0x%x IS=0x%x SERR=0x%x\n",
port->tfd, port->is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_TIMEOUT;
}
// Issue slot 0, then block on the event.
// Issue slot 0, then block with a finite deadline.
port->ci = (1u << 0);
KeWaitForEvent(drv->completionEvent, 0);
EVENT *events[1] = { drv->completionEvent };
KSTATUS waitStatus = KeWaitForMultipleEvents(events, 1, 0,
AHCI_COMMAND_TIMEOUT_MS,
(uint32_t *)0);
// The ISR has cleared IS - check for errors in the snapshot.
uint32_t is = drv->lastIs;
if (is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) {
DbgPrint("AHCI: error IS=0x%x TFD=0x%x SERR=0x%x\n",
is, port->tfd, port->serr);
port->serr = (uint32_t)-1;
// Merge any live status that was not consumed by the ISR. This lets a
// missed IRQ still report the hardware error that accompanied it.
uint32_t liveIs = port->is;
uint32_t is = drv->lastIs | liveIs;
if (liveIs)
port->is = liveIs;
if (is & AHCI_PORT_ERROR_MASK) {
DbgPrint("AHCI: command error IS=0x%x CI=0x%x TFD=0x%x SERR=0x%x\n",
is, port->ci, port->tfd, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_DEVICE_ERROR;
}
if (waitStatus == NTLSTATUS_TIMEOUT) {
if (!(port->ci & (1u << 0))) {
DbgPrint("AHCI: completion IRQ missed; command retired IS=0x%x TFD=0x%x SERR=0x%x\n",
is, port->tfd, port->serr);
return NTLSTATUS_SUCCESS;
}
DbgPrint("AHCI: command timeout CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_TIMEOUT;
}
if (waitStatus != NTLSTATUS_SUCCESS)
return waitStatus;
// An interrupt is only completion if hardware has also retired slot 0.
if (port->ci & (1u << 0)) {
DbgPrint("AHCI: premature completion IRQ CI=0x%x TFD=0x%x IS=0x%x SERR=0x%x\n",
port->ci, port->tfd, is, port->serr);
AhciRecoverPort(drv);
return NTLSTATUS_DEVICE_ERROR;
}
@@ -239,7 +379,7 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
// Build a standard DMA read or write command in slot 0.
static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
uint64_t bufPhys, bool write) {
uint64_t bufferDeviceAddress, bool write) {
HBA_CMD_HEADER *hdr = cmd_list(drv); // slot 0
HBA_CMD_TBL *tbl = cmd_tbl(drv, 0);
@@ -256,7 +396,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
// Fill PRDT entries.
uint32_t remaining = sectors;
uint64_t buf = bufPhys;
uint64_t buf = bufferDeviceAddress;
for (uint32_t i = 0; i < prdtl; i++) {
uint32_t sects = (remaining > 16) ? 16 : remaining;
uint32_t bytes = sects * 512;
@@ -289,6 +429,31 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
fis->counth = (uint8_t)(sectors >> 8);
}
// Builds a non-data ATA FLUSH CACHE EXT command in slot 0.
static void build_flush_cmd(AHCI_DRIVE* drv) {
HBA_CMD_HEADER* hdr = cmd_list(drv);
HBA_CMD_TBL* tbl = cmd_tbl(drv, 0);
memset(tbl, 0, PORT_CMDTBL_SIZE);
hdr->cfl = sizeof(FIS_REG_H2D) / 4;
hdr->w = 0;
hdr->prdtl = 0;
hdr->prdbc = 0;
FIS_REG_H2D* fis = (FIS_REG_H2D*)tbl->cfis;
memset(fis, 0, sizeof(*fis));
fis->fis_type = FIS_TYPE_REG_H2D;
fis->c = 1;
fis->command = ATA_CMD_FLUSH_EXT;
}
// Flushes the drive write cache through the serialized command worker.
static KSTATUS do_flush(AHCI_DRIVE* drv) {
if (!drv) return NTLSTATUS_INVALID_PARAMETER;
build_flush_cmd(drv);
return port_issue_cmd(drv);
}
// IDENTIFY command - fills a 512-byte buffer and returns it.
static KSTATUS port_identify(AHCI_DRIVE *drv) {
@@ -305,8 +470,8 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
hdr->prdtl = 1;
hdr->prdbc = 0;
tbl->prdt_entry[0].dba = (uint32_t)(drv->bouncePhys & 0xFFFFFFFF);
tbl->prdt_entry[0].dbau = (uint32_t)(drv->bouncePhys >> 32);
tbl->prdt_entry[0].dba = (uint32_t)(drv->bounceDeviceAddress & 0xFFFFFFFF);
tbl->prdt_entry[0].dbau = (uint32_t)(drv->bounceDeviceAddress >> 32);
tbl->prdt_entry[0].dbc = 512 - 1;
tbl->prdt_entry[0].i = 1;
@@ -347,7 +512,7 @@ static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba,
while (count > 0) {
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
build_rw_cmd(drv, lba, batch, drv->bouncePhys, false);
build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, false);
KSTATUS st = port_issue_cmd(drv);
if (!KSUCCESS(st)) return st;
@@ -368,7 +533,7 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
uint32_t batch = (count > BOUNCE_SECTORS) ? BOUNCE_SECTORS : count;
memcpy(drv->bounceVirt, in, (size_t)batch * 512);
build_rw_cmd(drv, lba, batch, drv->bouncePhys, true);
build_rw_cmd(drv, lba, batch, drv->bounceDeviceAddress, true);
KSTATUS st = port_issue_cmd(drv);
if (!KSUCCESS(st)) return st;
@@ -463,13 +628,16 @@ static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
drv->surpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
drv->removed = true;
device->PnpState = DevicePnpDeleted;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->removeLockReady)
IoReleaseRemoveLockAndWait(&drv->removeLock, irp);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
IoDeleteDevice(device);
drv->deviceObject = (DEVICE_OBJECT*)0;
return NTLSTATUS_SUCCESS;
@@ -511,7 +679,10 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
for (int i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
drv->surpriseRemoved = true;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->deviceObject)
(void)IoNotifyDiskSurpriseRemoved(drv->deviceObject->u.Disk.DiskIndex);
else
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
}
return NTLSTATUS_SUCCESS;
@@ -520,9 +691,18 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
device->PnpState = DevicePnpDeleted;
for (int i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
drv->removed = true;
IrpQueueCancelAllEx(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
if (drv->deviceObject) {
(void)IoUnregisterDiskDevice(drv->deviceObject->u.Disk.DiskIndex);
} else {
drv->removed = true;
IrpQueueCancelAllWithCallback(&drv->irpQueue, NTLSTATUS_OBJECT_DEAD, AhciCancelReleaseRemoveLock, drv);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
}
}
AhciUnregisterController(ctl);
gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter);
ctl->dmaAdapter = (IO_DMA_ADAPTER*)0;
IoDetachDevice(device);
IoDeleteDevice(device);
return NTLSTATUS_SUCCESS;
@@ -532,6 +712,29 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Queues one write-cache flush on the per-drive worker.
static KSTATUS AhciDispatchFlush(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER;
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
if (!drv || drv->magic != AHCI_DRIVE_MAGIC)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&drv->removeLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) {
IoReleaseRemoveLock(&drv->removeLock, irp);
return NTLSTATUS_INVALID_PARAMETER;
}
irp->Status = NTLSTATUS_FAILURE;
irp->Information = 0;
IrpQueueEnqueue(&drv->irpQueue, irp);
return NTLSTATUS_PENDING;
}
// Dispatches pnp.
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DeviceExtension)
@@ -552,7 +755,7 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
// ever having blocked in a driver call.
// NCQ remains serialized until the driver grows a multi-slot dispatcher:
// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP.
// 1. Wait for a free slot and a pending IRP through the queue semaphore.
// 2. Pick the oldest IRP from the queue.
// 3. Grab a free bit in drv->slotMask (under the slot spinlock).
// 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA),
@@ -561,30 +764,45 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
static void AhciWorker(uintptr_t arg) {
AHCI_DRIVE *drv = (AHCI_DRIVE *)arg;
while (true) {
while (drv && drv->workerRunning) {
// Block until AHCI dispatch signals a new IRP.
KeWaitSemaphore(drv->irpQueue.Semaphore);
KeWaitSemaphore(drv->irpQueue.Semaphore, KE_WAIT_INFINITE);
if (!drv->workerRunning)
break;
IRP *irp = IrpQueueDequeue(&drv->irpQueue);
if (!irp)
continue;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read (drv, lba, sectors, buffer);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
KSTATUS st = NTLSTATUS_INVALID_PARAMETER;
size_t information = 0;
if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) {
st = do_flush(drv);
} else {
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = AhciGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (write)
st = do_write(drv, lba, sectors, buffer);
else
st = do_read(drv, lba, sectors, buffer);
if (KSUCCESS(st))
information = (size_t)sectors * 512u;
}
}
// Publish result then fire completion hooks through the common I/O path.
IoReleaseRemoveLock(&drv->removeLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * 512u) : 0);
IoCompleteRequest(irp, st, information);
}
if (drv && drv->workerFinished)
KeSignalEvent(drv->workerFinished);
}
// AHCI controller initialisation.
@@ -597,18 +815,8 @@ static void AhciWorker(uintptr_t arg) {
// FRE must already be set (port_init_memory does this) so the HBA can
// capture the signature FIS the device sends after the reset.
// PORT_IE bits we care about:
// DHRS (bit 0) - D2H Register FIS received - normal completion
// PSS (bit 1) - PIO Setup FIS received
// DSS (bit 2) - DMA Setup FIS received
// SDBS (bit 3) - Set Device Bits FIS received
// TFES (bit 30) - Task File Error Status - error
// HBFS (bit 29) - Host Bus Fatal error
// HBDS (bit 28) - Host Bus Data error
// IFS (bit 27) - Interface Fatal error.
#define PORT_IE_NORMAL (0x0000000Fu)
#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS)
// Runtime commands are non-NCQ DMA commands. DHRS is their completion signal;
// setup/progress FIS interrupts must not wake the serialized command waiter.
// Registers controller.
static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
@@ -623,6 +831,20 @@ static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
return NTLSTATUS_SUCCESS;
}
// Removes a controller from the interrupt registry.
static void AhciUnregisterController(AHCI_CONTROLLER* ctl) {
if (!ctl)
return;
for (int i = 0; i < g_controllerCount; i++) {
if (g_controllers[i] != ctl)
continue;
g_controllerCount--;
g_controllers[i] = g_controllers[g_controllerCount];
g_controllers[g_controllerCount] = (AHCI_CONTROLLER*)0;
return;
}
}
// Starts controller.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC || !ctl->pdo || !ctl->fdo)
@@ -676,7 +898,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = PciMapBar(ctl->pdo,
AHCI_PCI_BAR_ABAR,
PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
(void**)&ctl->hba,
&ctl->abarLength);
if (!KSUCCESS(st)) {
@@ -712,9 +934,23 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: cap=0x%08x ports=%u slots=%u 64bit=%d\n",
cap, nPorts, nSlots, (int)s64a);
IO_DMA_REQUIREMENTS dmaRequirements;
memset(&dmaRequirements, 0, sizeof(dmaRequirements));
dmaRequirements.MaximumDeviceAddress = s64a ? (uint64_t)-1 : 0xFFFFFFFFULL;
dmaRequirements.Alignment = 2;
dmaRequirements.MaximumSegmentLength = 1u << 22;
dmaRequirements.MaximumSegments = PORT_PRDT_COUNT;
st = gAhciDdk->Dma->GetAdapter(ctl->fdo, &dmaRequirements, &ctl->dmaAdapter);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: DMA adapter creation failed (0x%x)\n", st);
return st;
}
st = AhciRegisterController(ctl);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: controller registry full (0x%x)\n", st);
gAhciDdk->Dma->PutAdapter(ctl->dmaAdapter);
ctl->dmaAdapter = (IO_DMA_ADAPTER*)0;
return st;
}
@@ -751,6 +987,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = port_comreset(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: COMRESET failed\n", portno);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -758,6 +996,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
if (!port_is_sata_disk(port)) {
DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n",
portno, port->sig);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -765,6 +1005,8 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
KeInitializeDpc(&drv->completionDpc, AhciCompletionDpc, drv);
@@ -775,19 +1017,26 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
st = KeSemaphoreCreate(0, 0x7FFFFFFF, &irpSem);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: semaphore create failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
IrpQueueInit(&drv->irpQueue, irpSem);
st = IoInitializeRemoveLock(&drv->removeLock);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: remove-lock init failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
drv->removeLockReady = true;
// Phase 6: IDENTIFY (polled - scheduler not yet running).
st = port_identify(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
@@ -799,12 +1048,14 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
// Phase 7: enable per-port interrupts.
// Clear any stale IS first so we don't get a spurious signal immediately.
port->is = (uint32_t)-1;
port->ie = PORT_IE_ALL;
port->ie = AHCI_PORT_TERMINAL_MASK;
// Phase 8: create and register a disk DEVICE_OBJECT.
st = IoCreateDevice(&g_driver, 0, &drv->deviceObject);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: drive %d: IoCreateDevice failed (0x%x)\n", g_nextDiskIndex, st);
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
drv->deviceObject->DeviceExtension = drv;
@@ -822,19 +1073,29 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: IoRegisterDiskDevice(%u) failed (0x%x)\n", diskIndex, st);
IoDeleteDevice(drv->deviceObject);
drv->deviceObject = (DEVICE_OBJECT*)0;
AhciReleaseDriveRuntime(drv);
AhciReleaseDriveDma(drv);
continue;
}
// Phase 9: spawn the per-drive IRP worker thread.
// The worker owns do_read / do_write exclusively once the scheduler
// is live; it is the only thread that may call port_issue_cmd().
st = KeCreateKernelThread((uintptr_t)AhciWorker, (uintptr_t)drv,
PriorityHigh, &drv->workerThread);
st = KeEventCreate(EventManualReset, 0, &drv->workerFinished);
if (KSUCCESS(st)) {
drv->workerRunning = true;
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh);
st = KeCreateKernelThread((uintptr_t)AhciWorker,
(uintptr_t)drv,
&workerParameters,
&drv->workerThread);
}
if (!KSUCCESS(st)) {
DbgPrint("AHCI: drive %u: worker thread create failed (0x%x)\n",
diskIndex, st);
// Fatal for this device: all disk access is IRP-based now, so
// leave the device registered but clearly fail during testing.
drv->workerRunning = false;
DbgPrint("AHCI: drive %u: worker creation failed (0x%x)\n", diskIndex, st);
(void)IoUnregisterDiskDevice(diskIndex);
ctl->driveCount--;
continue;
}
}
@@ -889,7 +1150,7 @@ static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevic
// Implements the driver Entry operation.
KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
(void)init;
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt)
if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Dma)
return NTLSTATUS_INVALID_PARAMETER;
DbgPrint("AHCI: driver loading...\n");
@@ -903,6 +1164,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
g_driver.DriverAddDevice = AhciAddDevice;
g_driver.MajorFunction[IRP_MJ_READ] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_WRITE] = AhciDispatchReadWrite;
g_driver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AhciDispatchFlush;
g_driver.MajorFunction[IRP_MJ_PNP] = AhciDispatchPnp;
KSTATUS st = IoRegisterDriverObject(&g_driver);
code/drivers/storage/ahci/ahci.h
+11 -4
View File
@@ -250,6 +250,7 @@ typedef volatile struct {
#define PORT_CMD_ICC_ACTIVE (0x1u << 28)
// Port IS register bits (interrupt status).
#define PORT_IS_DHRS (1u << 0) // D2H Register FIS Received.
#define PORT_IS_TFES (1u << 30) // Task File Error Status
#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error
#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error
@@ -321,11 +322,13 @@ typedef struct {
uint8_t portNo; // Physical port index (for ISR dispatch)
// Structures block (command list + FIS + cmd tables).
uint64_t structPhys; // Physical base
void* structVirt; // Virtual base
IO_DMA_BUFFER* structDma;
uint64_t structDeviceAddress; // Device base.
void* structVirt; // CPU base.
// Bounce buffer for DMA data.
uint64_t bouncePhys;
IO_DMA_BUFFER* bounceDma;
uint64_t bounceDeviceAddress;
void* bounceVirt;
// Signaled by the completion DPC when the port raises an interrupt.
@@ -345,13 +348,16 @@ typedef struct {
IO_REMOVE_LOCK removeLock;
bool surpriseRemoved;
bool removed;
bool removeLockReady;
volatile bool workerRunning;
THREAD* workerThread;
EVENT* workerFinished;
DEVICE_OBJECT* deviceObject;
// Multi-slot NCQ needs per-slot ownership state in addition to the serialized queue:
// uint32_t slotMask; // bitmask of in-flight slots
// IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP
// uint64_t slotBouncePhys[NUM_CMD_SLOTS];
// uint64_t slotBounceDeviceAddress[NUM_CMD_SLOTS];
// void* slotBounceVirt[NUM_CMD_SLOTS];
// volatile uint32_t prevSact; // previous port->sact snapshot.
} AHCI_DRIVE;
@@ -361,6 +367,7 @@ struct AHCI_CONTROLLER {
DEVICE_OBJECT* fdo;
DEVICE_OBJECT* pdo;
DEVICE_OBJECT* lowerDevice;
IO_DMA_ADAPTER* dmaAdapter;
PCI_DEVICE_INFO pciInfo;
PCI_BAR abar;
code/drivers/storage/ahci/ahci_port.c
+6 -9
View File
@@ -11,16 +11,13 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
int pageIndex = 0;
for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) {
uintptr_t virt = (uintptr_t)gAhciDdk->Memory->PhysToVirt(phys);
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent);
// Ignore NTLSTATUS_ALREADY_EXISTS - a prior mapping with different flags
// is possible for the first page if the bootloader was overly generous.
// Use MmPteProtectRange to force NOCACHE in that case.
if (st == NTLSTATUS_ALREADY_EXISTS) {
st = MmPteProtectRange(virt, 4096, flags);
}
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC;
KSTATUS st = gAhciDdk->Memory->MapPage(virt,
phys,
flags,
NtlDdkMapReplaceExisting);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
DbgPrint("AHCI: MMIO map failed for page %d (phys=0x%016llX, st=0x%x)\n",
pageIndex, (unsigned long long)phys, st);
return st;
}
code/drivers/storage/ata/ata.c
+74 -20
View File
@@ -92,7 +92,7 @@ static KSTATUS AtaIdentify(ATA_DEVICE* dev) {
uint16_t ident[256];
memset(ident, 0, sizeof(ident));
KSTATUS st = KeWaitSemaphore(ch->Lock);
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
@@ -193,7 +193,7 @@ static KSTATUS AtaReadPio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, void* b
ATA_CHANNEL* ch = dev->Channel;
uint16_t* out = (uint16_t*)buffer;
KSTATUS st = KeWaitSemaphore(ch->Lock);
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
@@ -234,6 +234,22 @@ static KSTATUS AtaFlush(ATA_DEVICE* dev) {
return AtaWaitNotBusy(ch);
}
// Flushes one device while holding the shared channel lock.
static KSTATUS AtaFlushSerialized(ATA_DEVICE* dev) {
if (!dev || !dev->Channel)
return NTLSTATUS_INVALID_PARAMETER;
ATA_CHANNEL* ch = dev->Channel;
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
AtaSelectDrive(dev);
st = AtaFlush(dev);
KeSignalSemaphore(ch->Lock);
return st;
}
// Updates pio.
static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const void* buffer) {
if (!dev || !buffer)
@@ -246,7 +262,7 @@ static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const
ATA_CHANNEL* ch = dev->Channel;
const uint16_t* in = (const uint16_t*)buffer;
KSTATUS st = KeWaitSemaphore(ch->Lock);
KSTATUS st = KeWaitSemaphore(ch->Lock, KE_WAIT_INFINITE);
if (!KSUCCESS(st))
return st;
@@ -308,27 +324,38 @@ static void AtaWorker(uintptr_t arg) {
return;
while (true) {
KeWaitSemaphore(dev->IrpQueue.Semaphore);
KeWaitSemaphore(dev->IrpQueue.Semaphore, KE_WAIT_INFINITE);
IRP* irp = IrpQueueDequeue(&dev->IrpQueue);
if (!irp)
continue;
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
KSTATUS st = AtaGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (lba + sectors > dev->SectorCount)
st = NTLSTATUS_INVALID_PARAMETER;
else
st = write ? AtaWritePio(dev, lba, sectors, buffer)
: AtaReadPio(dev, lba, sectors, buffer);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
KSTATUS st = NTLSTATUS_INVALID_PARAMETER;
size_t information = 0;
if (ios && ios->MajorFunction == IRP_MJ_FLUSH_BUFFERS) {
st = AtaFlushSerialized(dev);
} else {
bool write = false;
uint64_t lba = 0;
uint32_t sectors = 0;
void* buffer = (void*)0;
st = AtaGetIrpReadWrite(irp, &write, &lba, &sectors, &buffer);
if (KSUCCESS(st)) {
if (lba + sectors > dev->SectorCount) {
st = NTLSTATUS_INVALID_PARAMETER;
} else {
st = write ? AtaWritePio(dev, lba, sectors, buffer)
: AtaReadPio(dev, lba, sectors, buffer);
if (KSUCCESS(st))
information = (size_t)sectors * ATA_SECTOR_SIZE;
}
}
}
IoReleaseRemoveLock(&dev->RemoveLock, irp);
IoCompleteRequest(irp, st, KSUCCESS(st) ? ((size_t)sectors * ATA_SECTOR_SIZE) : 0);
IoCompleteRequest(irp, st, information);
}
}
@@ -363,6 +390,31 @@ KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
return NTLSTATUS_PENDING;
}
// Queues one cache flush on the device worker.
static KSTATUS AtaDispatchFlush(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return NTLSTATUS_INVALID_PARAMETER;
ATA_DEVICE* dev = (ATA_DEVICE*)device->DeviceExtension;
if (dev->Magic != ATA_DEVICE_MAGIC || !dev->Present)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = IoAcquireRemoveLock(&dev->RemoveLock, irp);
if (!KSUCCESS(st))
return st;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
if (!ios || ios->MajorFunction != IRP_MJ_FLUSH_BUFFERS) {
IoReleaseRemoveLock(&dev->RemoveLock, irp);
return NTLSTATUS_INVALID_PARAMETER;
}
irp->Status = NTLSTATUS_FAILURE;
irp->Information = 0;
IrpQueueEnqueue(&dev->IrpQueue, irp);
return NTLSTATUS_PENDING;
}
// Returns free Disk Index.
static uint8_t AtaFindFreeDiskIndex(void) {
for (uint8_t i = 0; i < 8; i++) {
@@ -486,8 +538,9 @@ static KSTATUS AtaProbeDrive(ATA_CONTROLLER* ctl, ATA_CHANNEL* ch, uint8_t drive
return st;
}
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityHigh);
st = KeCreateKernelThread((uintptr_t)AtaWorker, (uintptr_t)dev,
PriorityHigh, &dev->WorkerThread);
&workerParameters, &dev->WorkerThread);
if (!KSUCCESS(st))
DbgPrint("ATA: drive %u worker thread create failed (0x%x)\n", dev->DiskIndex, st);
return NTLSTATUS_SUCCESS;
@@ -580,7 +633,7 @@ static void AtaRemoveControllerDevices(ATA_CONTROLLER* ctl, IO_DEVICE_CHANGE cha
if (!dev->Present)
continue;
IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
if (dev->DiskIndex != 0xFFu) {
if (change == IoDeviceSurpriseRemoved)
IoNotifyDiskSurpriseRemoved(dev->DiskIndex);
@@ -647,12 +700,12 @@ KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
device->PnpState = DevicePnpSurpriseRemoved;
dev->Present = false;
IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
return NTLSTATUS_SUCCESS;
case IRP_MN_REMOVE_DEVICE:
device->PnpState = DevicePnpDeleted;
dev->Present = false;
IrpQueueCancelAllEx(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IrpQueueCancelAllWithCallback(&dev->IrpQueue, NTLSTATUS_OBJECT_DEAD, AtaCancelReleaseRemoveLock, dev);
IoReleaseRemoveLockAndWait(&dev->RemoveLock, irp);
IoDeleteDevice(device);
dev->DeviceObject = (DEVICE_OBJECT*)0;
@@ -708,6 +761,7 @@ KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
gAtaDriver.DriverAddDevice = AtaAddDevice;
gAtaDriver.MajorFunction[IRP_MJ_READ] = AtaDispatchReadWrite;
gAtaDriver.MajorFunction[IRP_MJ_WRITE] = AtaDispatchReadWrite;
gAtaDriver.MajorFunction[IRP_MJ_FLUSH_BUFFERS] = AtaDispatchFlush;
gAtaDriver.MajorFunction[IRP_MJ_PNP] = AtaDispatchPnp;
KSTATUS st = IoRegisterDriverObject(&gAtaDriver);
code/kernel/hal/hal_processor.c
+1 -1
View File
@@ -37,6 +37,6 @@ __declspec(dllexport) void HalHalt(void) {
// Masks interrupts and permanently stops the current processor.
__declspec(dllexport) _Noreturn void HalStopProcessor(void) {
_disable();
for (;;)
while (true)
__halt();
}
code/kernel/include/kernel/io_dma.h
+13
View File
@@ -0,0 +1,13 @@
// Kernel DMA declarations shared by the I/O manager and DDK provider.
#pragma once
#include <ntlite/ddk/dma.h>
struct DEVICE_OBJECT;
// Returns the kernel-owned DDK DMA table.
const NTL_DDK_DMA* IoGetDmaDdk(void);
// Detaches the adapter owned by a device before the device object can die.
void IoDmaDetachDevice(struct DEVICE_OBJECT* device);
code/kernel/include/kernel/io_file.h
+10
View File
@@ -12,6 +12,16 @@
struct PROCESS;
// Initializes filesystem namespace change notification before drivers register volumes.
KSTATUS IoInitializeVolumeNamespace(void);
// Returns the generation of the mounted-volume namespace.
uint64_t IoGetVolumeNamespaceGeneration(void);
// Waits until the mounted-volume namespace changes or the timeout expires.
KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration,
uint64_t timeoutMilliseconds);
__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file,
void* buf,
size_t size,
code/kernel/include/kernel/ke_audio.h
+183
View File
@@ -0,0 +1,183 @@
// Kernel audio-core declarations shared by the mixer, syscall layer, and selftests.
#pragma once
#include <stdbool.h>
#include <stdint.h>
#include <ntlite/ddk/audio.h>
#include <kernel/ob_object.h>
#include <kernel/ke_event.h>
#define KE_AUDIO_MAX_ENDPOINTS 32u
#define KE_AUDIO_MAX_STREAMS_PER_ENDPOINT 64u
#define KE_AUDIO_STREAM_RING_FRAMES 4096u
#define KE_AUDIO_OUTPUT_SAMPLE_RATE 48000u
#define KE_AUDIO_OUTPUT_CHANNELS 2u
#define KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD 512u
#define KE_AUDIO_OUTPUT_PERIOD_COUNT 4u
#define KE_AUDIO_OUTPUT_BYTES_PER_FRAME 4u
#define KE_AUDIO_OUTPUT_PERIOD_BYTES 2048u
#define KE_AUDIO_GAIN_RAMP_FRAMES 480u
#define KE_AUDIO_IDLE_STOP_MILLISECONDS 250u
#define KE_AUDIO_MIXER_REALTIME_PRIORITY 16u
#define KE_AUDIO_WAIT_WRITABLE 0x00000001u
#define KE_AUDIO_WAIT_CONSUMED 0x00000002u
#define KE_AUDIO_WAIT_PRESENTED 0x00000004u
#define KE_AUDIO_WAIT_STOPPED 0x00000008u
#define KE_AUDIO_WAIT_DRAINED 0x00000010u
#define KE_AUDIO_WAIT_FAILED 0x00000020u
#define KE_AUDIO_WAIT_DISCONTINUITY 0x00000040u
#define KE_AUDIO_WAIT_VALID_MASK 0x0000007Fu
typedef enum KE_AUDIO_ENDPOINT_STATE {
KeAudioEndpointStarting = 1,
KeAudioEndpointReady,
KeAudioEndpointStopping,
KeAudioEndpointFailed,
KeAudioEndpointRemoved
} KE_AUDIO_ENDPOINT_STATE;
typedef enum KE_AUDIO_STREAM_STATE {
KeAudioStreamCreated = 1,
KeAudioStreamRunning,
KeAudioStreamPaused,
KeAudioStreamDraining,
KeAudioStreamStopped,
KeAudioStreamFailed,
KeAudioStreamClosed
} KE_AUDIO_STREAM_STATE;
typedef struct KE_AUDIO_STREAM KE_AUDIO_STREAM;
// Reports one endpoint snapshot without exposing provider-owned pointers.
typedef struct KE_AUDIO_ENDPOINT_INFO {
uint64_t EndpointId;
IO_AUDIO_ENDPOINT_KIND Kind;
KE_AUDIO_ENDPOINT_STATE State;
uint32_t Capabilities;
IO_AUDIO_FORMAT HardwareFormat;
uint32_t PeriodCount;
uint32_t FramesPerPeriod;
uint8_t Available;
uint8_t IsDefault;
uint8_t HardwareRunning;
uint8_t MasterMuted;
int32_t MasterHardwareGainMilliDb;
float MasterRequestedVolume;
uint32_t StreamCount;
uint32_t ActiveStreamCount;
uint64_t PeriodsMixed;
uint64_t PeriodsCompleted;
uint64_t FramesPresented;
uint64_t HardwarePositionFrames;
uint64_t InterruptCount;
uint64_t LastPresentationTimestampTicks;
uint64_t SilenceFramesInserted;
uint64_t Underruns;
uint64_t DiscontinuityGeneration;
KSTATUS FailureStatus;
} KE_AUDIO_ENDPOINT_INFO;
// Selects the next condition that should signal one stream's wait event.
typedef struct KE_AUDIO_STREAM_WAIT_CONDITION {
uint32_t Mask;
uint32_t WritableFrames;
uint64_t ConsumedFrames;
uint64_t PresentedFrames;
uint64_t DrainedGeneration;
uint64_t DiscontinuityGeneration;
} KE_AUDIO_STREAM_WAIT_CONDITION;
// Reports one logical stream snapshot.
typedef struct KE_AUDIO_STREAM_INFO {
KE_AUDIO_STREAM_STATE State;
IO_AUDIO_FORMAT Format;
uint32_t CapacityFrames;
uint32_t BufferedFrames;
uint32_t WritableFrames;
uint64_t FramesSubmitted;
uint64_t FramesConsumed;
uint64_t FramesPresented;
uint64_t FramesDiscarded;
uint64_t OutputFramesMixed;
uint64_t SilenceFramesInserted;
uint64_t Underruns;
uint64_t DrainedGeneration;
uint64_t DiscontinuityGeneration;
KSTATUS FailureStatus;
float Volume;
float Pan;
uint8_t Muted;
} KE_AUDIO_STREAM_INFO;
// Initializes endpoint registry and mixer ownership before drivers load.
void KeAudioInitialize(void);
// Returns the provider-facing audio DDK table.
const NTL_DDK_AUDIO* KeAudioGetDdk(void);
// Copies current endpoint snapshots and reports the full available count.
KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints,
uint32_t capacity,
uint32_t* countOut);
// Returns the currently selected endpoint identifier, or zero when none is usable.
uint64_t KeAudioGetDefaultEndpointId(void);
// Creates one fixed-format logical render stream and returns a caller-owned reference.
KSTATUS KeAudioCreateStream(uint64_t endpointId,
const IO_AUDIO_FORMAT* format,
KE_AUDIO_STREAM** outStream);
// Records one process handle that owns the stream.
void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream);
// Releases one process-handle owner without blocking the handle-table path.
void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream);
// Returns a borrowed passive event valid while the caller retains the stream.
EVENT* KeAudioStreamWaitEvent(KE_AUDIO_STREAM* stream);
// Arms the stream event for one or more progress conditions without a lost-wakeup window.
KSTATUS KeAudioArmStreamWait(KE_AUDIO_STREAM* stream,
const KE_AUDIO_STREAM_WAIT_CONDITION* condition);
// Copies as many complete source frames as fit without blocking.
KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream,
const void* frames,
uint32_t requestedFrames,
uint32_t* writtenFramesOut);
// Starts one created or stopped stream.
KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream);
// Pauses future mixing while preserving source frames not yet consumed.
KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream);
// Resumes one paused stream.
KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream);
// Rejects new writes and stops after every submitted frame is presented or discarded.
KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream);
// Stops future mixing and discards source frames not yet committed to endpoint periods.
KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream);
// Invalidates queued presentation accounting and returns an empty stream to position zero.
KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream);
// Updates per-stream software gain with a ten-millisecond ramp.
KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream,
float volume,
float pan,
bool muted);
// Updates endpoint master software gain and separate logical mute state.
KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted);
// Copies one coherent stream snapshot.
KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut);
// Runs the final render path selftest behind KE_DEBUG_AUDIO_SELFTEST.
void KiRunAudioFinalPathSelfTest(void);
code/kernel/include/kernel/ke_cpu.h
+3
View File
@@ -7,6 +7,7 @@
// owns CPU-local scheduler, IRQL, GDT, and TSS state.
#include <stdint.h>
#include <stdbool.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_idt.h>
@@ -83,6 +84,8 @@ extern KCPU gCpus[KE_MAX_CPUS];
extern uint32_t gCpuCount;
void KeInitializeBootstrapCpu(void);
void KeInitializeMemoryTypes(void);
bool KeWriteCombiningAvailable(void);
void KeLoadKernelGsBase(KCPU* cpu);
void KeSetUserGsBase(uintptr_t userGsBase);
uintptr_t KeGetUserGsBase(void);
code/kernel/include/kernel/ke_debug.h
-15
View File
@@ -10,21 +10,6 @@
#include <ntlite/ddi/hal_api.h>
#include <ntlite/ddk/debug.h>
#define KE_DEBUG_CONSOLE 0x00000001u
#define KE_DEBUG_SELFTEST 0x00000002u
#define KE_DEBUG_FILE_LOG 0x00000004u
#define KE_DEBUG_POOL_CANARY 0x00000008u
#define KE_DEBUG_POOL_POISON 0x00000010u
#define KE_DEBUG_POOL_QUARANTINE 0x00000020u
#define KE_DEBUG_POOL_CALLERS 0x00000040u
#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u
#define KE_DEBUG_DISPLAY_TRACE 0x00000100u
#define KE_DEBUG_INPUT_TRACE 0x00000200u
#define KE_DEBUG_VM_ASSERTS 0x00000400u
#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u
#define KE_DEBUG_LIST_CHECKS 0x00001000u
void KeInitDebug(const HAL_API* hal, uint32_t flags);
uint32_t KeGetDebugFlags(void);
__declspec(dllexport) void DbgPrint(const char* fmt, ...);
code/kernel/include/kernel/ke_display.h
+3
View File
@@ -28,6 +28,9 @@ typedef struct DISPLAY_CLOCK_OBJECT {
} DISPLAY_CLOCK_OBJECT;
void KeDisplayInitialize(const HAL_API* hal);
// Completes every console glyph queued before the call.
KSTATUS KeDisplayFlushConsoleOutput(void);
DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOCK_OBJECT** outClock);
DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock);
DISPLAY_API void KeDisplayTick(void);
code/kernel/include/kernel/ke_mutex.h
+27 -10
View File
@@ -1,20 +1,37 @@
// Kernel ke mutex declarations shared by implementation files and kernel clients.
// Kernel mutex declarations shared by implementation files and kernel clients.
#pragma once
// ke_mutex.h - binary ownership mutex.
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_list.h>
#include <ntlite/ddk/thread.h>
#include <ntlite/status.h>
struct THREAD;
typedef struct MUTEX {
OBJECT Header;
SPINLOCK Lock;
struct THREAD* Owner; // NULL if unowned
LIST_ENTRY WaitList;
OBJECT Header;
SPINLOCK Lock;
struct THREAD* Owner;
LIST_ENTRY WaitList;
LIST_ENTRY OwnerLink;
KE_SCHEDULING_CLASS DonationClass;
uint32_t DonationPriority;
uint8_t HasDonation;
uint8_t Abandoned;
} MUTEX;
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out);
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex);
__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex);
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex);
// Tests whether a mutex acquisition transferred ownership to the caller.
#define KE_MUTEX_ACQUIRE_SUCCEEDED(status) \
((status) == NTLSTATUS_SUCCESS || (status) == NTLSTATUS_ABANDONED)
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out);
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds);
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex);
// Removes one blocked waiter while preserving priority-inheritance state.
int KiMutexAbortWait(MUTEX* mutex, struct THREAD* thread);
// Releases every mutex owned by a terminating thread as abandoned.
void KiMutexRundownThread(struct THREAD* thread);
code/kernel/include/kernel/ke_process.h
+3 -1
View File
@@ -30,6 +30,7 @@
#define HANDLE_RIGHT_QUERY 0x00000020u
#define HANDLE_RIGHT_TERMINATE 0x00000040u
#define HANDLE_RIGHT_LOCK 0x00000080u
#define HANDLE_RIGHT_RESUME 0x00000100u
#define HANDLE_RIGHT_ALL 0xFFFFFFFFu
struct SYS_EXCEPTION_STATS;
@@ -62,7 +63,8 @@ typedef struct PROCESS {
uint8_t CurrentDrive;
SPINLOCK CwdLock;
// Thread list.
// Live thread set. Dead THREAD objects may outlive this membership while
// retained handles remain waitable and queryable.
LIST_ENTRY ThreadList; // links THREAD.ProcessLink
uint32_t ThreadCount;
code/kernel/include/kernel/ke_sched.h
+57 -2
View File
@@ -1,18 +1,20 @@
// Kernel ke sched declarations shared by implementation files and kernel clients.
#pragma once
// ke_sched.h - Multi-Level Feedback Queue scheduler
// Three run queues (Low / Normal / High). LAPIC timer IRQs drive KeSchedulerTick.
// The scheduler combines dynamic normal queues with static real-time queues.
// LAPIC timer IRQs drive KeSchedulerTick.
#include <kernel/ke_thread.h>
struct KCPU;
#define SCHED_TICK_MS 10u
#define SCHED_REALTIME_QUANTUM_TICKS 1u
#define SCHED_HIGH_QUANTUM_TICKS 2u
#define SCHED_NORMAL_QUANTUM_TICKS 4u
#define SCHED_LOW_QUANTUM_TICKS 8u
#define SCHED_DEMOTE_AFTER_BURST_TICKS 16u
#define SCHED_AGING_INTERVAL_TICKS 100u
#define KE_PRIORITY_INHERITANCE_MAX_DEPTH 64u
typedef struct KE_SCHED_STATS {
uint64_t Ticks;
@@ -27,8 +29,61 @@ typedef struct KE_SCHED_STATS {
uint64_t CpuBurstDemotions;
uint64_t SmtSiblingDeferrals;
uint64_t SmtPrimaryPokes;
uint64_t RealtimeDispatches;
uint64_t RealtimePreemptions;
uint64_t PriorityInheritanceBoosts;
uint64_t PriorityInheritanceDrops;
uint64_t PriorityInheritancePropagations;
uint64_t PriorityInheritanceMaxDepth;
} KE_SCHED_STATS;
struct MUTEX;
typedef struct KE_THREAD_SCHEDULER_SNAPSHOT {
THREAD_STATE State;
KE_SCHEDULING_CLASS BaseSchedulingClass;
uint32_t BasePriority;
KE_SCHEDULING_CLASS EffectiveSchedulingClass;
uint32_t EffectivePriority;
uint32_t RunningCpu;
uint32_t LastCpu;
uint32_t QuantumRemaining;
uint32_t QuantumTicks;
uint32_t CpuBurstTicks;
uint32_t LastRunTicks;
uint32_t WaitObjectType;
KSTATUS WaitStatus;
struct MUTEX* WaitingMutex;
uint8_t Queued;
uint8_t AlertableWait;
} KE_THREAD_SCHEDULER_SNAPSHOT;
// Copies scheduler-owned thread state into a stable diagnostic snapshot.
void KiSchedulerSnapshotThread(THREAD* thread, KE_THREAD_SCHEDULER_SNAPSHOT* snapshot);
// Selects waiters and refreshes donations while the mutex lock is held.
struct THREAD* KiSchedulerSelectMutexWaiter(struct MUTEX* mutex);
int KiSchedulerRefreshMutexDonation(struct MUTEX* mutex);
// Updates one thread's normal-class base priority.
KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority);
// Publishes one mutex dependency atomically with cycle detection.
KSTATUS KiSchedulerPrepareMutexWait(THREAD* thread, struct MUTEX* mutex, THREAD* owner);
void KiSchedulerCancelMutexWait(THREAD* thread, struct MUTEX* mutex);
// Associates mutex ownership with priority-inheritance bookkeeping.
void KiSchedulerAcquireMutexOwnership(THREAD* thread, struct MUTEX* mutex);
void KiSchedulerTransferMutexOwnership(THREAD* oldOwner, THREAD* newOwner,
struct MUTEX* mutex);
void KiSchedulerReleaseMutexOwnership(THREAD* thread, struct MUTEX* mutex);
struct MUTEX* KiSchedulerFirstOwnedMutex(THREAD* thread);
void KiSchedulerNotePriorityPropagation(uint32_t depth);
// Recomputes effective rank and optionally returns a referenced upstream mutex.
int KiSchedulerRefreshEffectivePriority(THREAD* thread, struct MUTEX** waitingMutexOut);
// Called once by KernelMain before any threads are created.
void KeInitializeScheduler(void);
code/kernel/include/kernel/ke_semaphore.h
+1 -2
View File
@@ -17,6 +17,5 @@ typedef struct SEMAPHORE {
} SEMAPHORE;
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out);
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore);
__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* semaphore);
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore, uint64_t timeoutMilliseconds);
__declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* semaphore);
code/kernel/include/kernel/ke_thread.h
+47 -17
View File
@@ -18,6 +18,7 @@
#define KE_USER_EXCEPTION_MAX_DEPTH 8u
typedef struct THREAD THREAD;
struct MUTEX;
#ifndef NTL_THREAD_PRIORITY_DEFINED
typedef enum THREAD_PRIORITY {
@@ -34,6 +35,7 @@ typedef enum THREAD_STATE {
ThreadRunning = 1,
ThreadWaiting = 2,
ThreadDead = 3,
ThreadSuspended = 4,
} THREAD_STATE;
typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread,
@@ -51,17 +53,20 @@ typedef struct KCONTEXT {
struct THREAD {
OBJECT Header; // must be first
uint32_t ThreadId;
THREAD_STATE State;
THREAD_PRIORITY Priority;
uint32_t ThreadId;
THREAD_STATE State;
KE_SCHEDULING_CLASS BaseSchedulingClass;
uint32_t BasePriority;
KE_SCHEDULING_CLASS EffectiveSchedulingClass;
uint32_t EffectivePriority;
// CPU-burst mirror kept here for the assembly-stable THREAD layout.
// Dynamic quantum state lives later in the structure.
uint32_t QuantaConsumed;
uint32_t QuantaConsumed;
PROCESS* Owner;
LIST_ENTRY ProcessLink; // node in Owner->ThreadList
LIST_ENTRY SchedulerLink; // node in run queue or wait list
LIST_ENTRY SchedulerLink; // node in run queue or object wait list
// Kernel stack - never PAGE_USER, pointed to by TssRsp0.
void* KernelStackBase;
@@ -108,6 +113,12 @@ struct THREAD {
uint32_t CpuBurstTicks;
uint32_t LastRunTicks;
uint32_t RunningCpu;
uint32_t SuspendCount;
// Priority inheritance ownership is scheduler-owned. Mutex wait-list
// membership remains protected by the mutex spinlock.
LIST_ENTRY OwnedMutexList;
struct MUTEX* WaitingMutex;
// Scheduler hardening state. Queued means membership in a scheduler ready
// queue only; wait-object membership is tracked by WaitListHead. Idle
@@ -125,7 +136,9 @@ struct THREAD {
KSTATUS WaitStatus;
LIST_ENTRY* WaitListHead;
uint8_t AlertableWait;
uint64_t SleepUntilTick;
LIST_ENTRY TimeoutLink;
uint8_t TimeoutArmed;
uint64_t TimeoutTick;
KSTATUS TimeoutStatus;
// Optional cleanup hook for waits that use auxiliary wait blocks.
@@ -167,6 +180,11 @@ struct THREAD {
uint32_t UserExceptionNoncontinuableMask;
uintptr_t UserExceptionFrameUserAddress;
KSTATUS UserExceptionExitStatus[KE_USER_EXCEPTION_MAX_DEPTH];
// Synchronous kernel work can temporarily retain caller-owned stack or
// process memory. Process termination must wait until that ownership has
// returned instead of freeing the thread underneath an active driver.
volatile long TerminationDeferralCount;
};
// Creates and queues a user thread. On success, *out receives a referenced
@@ -174,24 +192,31 @@ struct THREAD {
// after copying any needed IDs/state. The live scheduler/thread lifetime is
// separate and remains valid until the thread exits and is reaped.
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out);
// Create a kernel-mode-only thread. fn is called directly in ring 0 with
// Cancels a thread that has never left its initial suspended state. This drops
// the scheduler-owned lifetime reference but leaves the caller's reference
// intact.
KSTATUS KeCancelSuspendedThread(THREAD* thread);
// Creates a kernel-mode-only thread. fn is called directly in ring 0 with
// arg; when fn returns the thread exits automatically. No user stack or TEB
// is allocated. The thread is owned by the system process and runs at the
// given priority. On success, non-NULL out receives a referenced THREAD
// pointer owned by the caller; release it with ObDereferenceObject() after
// copying any needed IDs/state or after joining/waiting on caller-owned
// completion state. The scheduler's live thread reference is separate.
// is allocated. The thread is owned by the system process. On success,
// non-NULL out receives a referenced THREAD pointer owned by the caller;
// release it with ObDereferenceObject() after copying any needed IDs/state or
// after joining caller-owned completion state. The scheduler's live thread
// reference is separate.
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, uint64_t affinityMask,
const KE_THREAD_CREATE_PARAMETERS* parameters,
THREAD** out);
// Mark a thread dead and deschedule it permanently.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code);
// Publishes completion, releases per-thread user memory, and removes one dead
// thread from its process live-thread set. Retained handles remain waitable.
void KeFinalizeThreadExit(THREAD* thread);
// Block / unblock primitives used by sync objects.
// Correct wait pattern:
@@ -213,12 +238,17 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status);
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status);
void KeAbortThreadWait(THREAD* thread, KSTATUS status);
int KeAbortThreadWait(THREAD* thread, KSTATUS status);
void KeUnblockThread(THREAD* thread);
KSTATUS KeAlertThread(THREAD* thread);
void KiWakeThreadForKernelApc(THREAD* thread);
void KiWakeThreadForUserApc(THREAD* thread);
// Defers process termination while kernel work owns caller-backed storage.
void KeEnterTerminationDeferral(void);
void KeLeaveTerminationDeferral(void);
int KeThreadTerminationDeferred(const THREAD* thread);
// Assembly stub - implemented in ke_switch.asm.
void KeSwitchContext(THREAD* from, THREAD* to);
__declspec(noreturn) void KeStartInitialThread(THREAD* thread);
code/kernel/include/kernel/ke_time.h
+3
View File
@@ -11,3 +11,6 @@ void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal);
KSTATUS KeTimeReloadSettings(void);
__declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime);
__declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void);
// Returns the monotonic platform counter in microseconds.
uint64_t KeQueryMonotonicTicks(void);
code/kernel/include/kernel/mm_file_cache.h
-17
View File
@@ -43,19 +43,6 @@ typedef struct MM_FILE_CACHE_STATS {
uint64_t DropFailures;
uint64_t StandbyTransitions;
uint64_t StandbyHits;
uint64_t ReadAheadRequests;
uint64_t ReadAheadPagesRequested;
uint64_t ReadAheadPagesQueued;
uint64_t ReadAheadPagesLoaded;
uint64_t ReadAheadPagesSkipped;
uint64_t ReadAheadFailures;
uint64_t ReadAheadPagesThrottled;
uint64_t MaxReadAheadPagesPerRequest;
uint64_t BatchReads;
uint64_t BatchPagesRequested;
uint64_t BatchPagesLoaded;
uint64_t MaxBatchPagesRequested;
uint64_t MaxBatchPagesLoaded;
uint64_t ResidentPages;
uint64_t ActivePages;
uint64_t StandbyPages;
@@ -92,8 +79,4 @@ void MmFileCacheInvalidateAll(FILE_OBJECT* file);
// it never writes back because v1 stores only clean file data.
KSTATUS MmFileCacheDropOnePageForPressure(void);
// Queues best-effort read-ahead for clean file pages. The queue owns a file
// reference until the worker has consumed the request.
KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages);
void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out);
code/kernel/include/kernel/mm_pfn.h
+6
View File
@@ -124,6 +124,12 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min
PHYSICAL_ADDRESS maxExclusive,
MM_PFN_USAGE usage);
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage);
PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count,
PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive,
size_t alignment,
size_t boundary,
MM_PFN_USAGE usage);
__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame);
code/kernel/include/kernel/mm_pte.h
+5 -4
View File
@@ -12,10 +12,11 @@
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
#ifndef PAGE_WRITETHROUGH
#define PAGE_WRITETHROUGH (1ULL << 3)
#endif
#define PAGE_NOCACHE (1ULL << 4)
// Cache policy is selected through the canonical PAT layout.
#define PAGE_CACHE_WB 0ULL
#define PAGE_CACHE_WC (1ULL << 3)
#define PAGE_CACHE_UC (1ULL << 4)
#define PAGE_CACHE_MASK (PAGE_CACHE_WC | PAGE_CACHE_UC)
#define PAGE_ACCESSED (1ULL << 5)
#define PAGE_DIRTY (1ULL << 6)
#define PAGE_NX (1ULL << 63)
code/kernel/include/kernel/mm_virt.h
+11
View File
@@ -66,6 +66,15 @@ typedef struct MM_PINNED_USER_PAGE {
} MM_PINNED_USER_PAGE;
typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
typedef struct MM_KERNEL_VIEW MM_KERNEL_VIEW;
// Maps existing frames into a stable kernel-only virtual range. The view owns
// virtual address space and PTEs, but it does not own the physical frames.
KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames,
size_t pageCount,
MM_KERNEL_VIEW** outView);
void* MmKernelViewBase(const MM_KERNEL_VIEW* view);
void MmDestroyKernelView(MM_KERNEL_VIEW* view);
// Pins one user page and returns a kernel direct-map alias for the requested
// address. Callers may copy through KernelAddress + PageOffset while the page is
@@ -80,6 +89,8 @@ __declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
// unlocked. WriteAccess describes kernel writes to the user buffer.
__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer);
PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index);
// Virtual address descriptors.
// The single shared address space still has per-process VAD ownership so process
code/kernel/include/kernel/syscall_nr.h
+134 -118
View File
@@ -17,138 +17,154 @@
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_GET_THREAD_LIST 9
#define SYS_LOAD_MODULE 10
#define SYS_UNLOAD_MODULE 11
#define SYS_GET_MODULE_INFO 12
#define SYS_GET_MODULE_EXPORT 13
#define SYS_GET_MODULE_LIST 14
#define SYS_GET_CURRENT_TID 15
#define SYS_GET_CURRENT_PID 16
#define SYS_SET_THREAD_PRIORITY 17
#define SYS_YIELD 18
#define SYS_SLEEP 19
#define SYS_WAIT_SINGLE 20
#define SYS_WAIT_MANY 21
#define SYS_SET_USER_APC_DISPATCHER 22
#define SYS_APC_RETURN 23
#define SYS_SET_EXCEPTION_DISPATCHER 24
#define SYS_GET_EXCEPTION_DISPATCHER 25
#define SYS_CONTINUE_EXCEPTION 26
#define SYS_RAISE_EXCEPTION 27
#define SYS_RESUME_THREAD 9
#define SYS_GET_THREAD_LIST 10
#define SYS_LOAD_MODULE 11
#define SYS_UNLOAD_MODULE 12
#define SYS_GET_MODULE_INFO 13
#define SYS_GET_MODULE_EXPORT 14
#define SYS_GET_MODULE_LIST 15
#define SYS_GET_CURRENT_TID 16
#define SYS_GET_CURRENT_PID 17
#define SYS_SET_THREAD_PRIORITY 18
#define SYS_YIELD 19
#define SYS_SLEEP 20
#define SYS_WAIT_SINGLE 21
#define SYS_WAIT_MANY 22
#define SYS_SET_USER_APC_DISPATCHER 23
#define SYS_APC_RETURN 24
#define SYS_SET_EXCEPTION_DISPATCHER 25
#define SYS_GET_EXCEPTION_DISPATCHER 26
#define SYS_CONTINUE_EXCEPTION 27
#define SYS_RAISE_EXCEPTION 28
// Virtual memory, mappings, and handles.
#define SYS_ALLOC_VM 28
#define SYS_FREE_VM 29
#define SYS_PROTECT_VM 30
#define SYS_QUERY_VM 31
#define SYS_CLOSE_HANDLE 32
#define SYS_DUPLICATE_HANDLE 33
#define SYS_CREATE_FILE_MAPPING 34
#define SYS_MAP_FILE 35
#define SYS_ALLOC_VM 29
#define SYS_FREE_VM 30
#define SYS_PROTECT_VM 31
#define SYS_QUERY_VM 32
#define SYS_CLOSE_HANDLE 33
#define SYS_DUPLICATE_HANDLE 34
#define SYS_CREATE_FILE_MAPPING 35
#define SYS_MAP_FILE 36
// Synchronization objects.
#define SYS_CREATE_MUTEX 36
#define SYS_TRY_ACQUIRE_MUTEX 37
#define SYS_RELEASE_MUTEX 38
#define SYS_CREATE_SEM 39
#define SYS_TRY_WAIT_SEM 40
#define SYS_SIGNAL_SEM 41
#define SYS_CREATE_EVENT 42
#define SYS_SIGNAL_EVENT 43
#define SYS_CLEAR_EVENT 44
#define SYS_CREATE_MUTEX 37
#define SYS_TRY_ACQUIRE_MUTEX 38
#define SYS_RELEASE_MUTEX 39
#define SYS_CREATE_SEM 40
#define SYS_TRY_WAIT_SEM 41
#define SYS_SIGNAL_SEM 42
#define SYS_CREATE_EVENT 43
#define SYS_SIGNAL_EVENT 44
#define SYS_CLEAR_EVENT 45
// Queues and named queues.
#define SYS_CREATE_QUEUE 45
#define SYS_SEND_QUEUE 46
#define SYS_RECV_QUEUE 47
#define SYS_PUBLISH_QUEUE 48
#define SYS_OPEN_NAMED_QUEUE 49
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 50
#define SYS_CREATE_QUEUE 46
#define SYS_SEND_QUEUE 47
#define SYS_RECV_QUEUE 48
#define SYS_PUBLISH_QUEUE 49
#define SYS_OPEN_NAMED_QUEUE 50
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 51
// Files, directories, and current directory.
#define SYS_OPEN_FILE 51
#define SYS_READ_FILE 52
#define SYS_WRITE_FILE 53
#define SYS_READ_FILE_ASYNC 54
#define SYS_WRITE_FILE_ASYNC 55
#define SYS_CANCEL_IO 56
#define SYS_SEEK_FILE 57
#define SYS_GET_FILE_POS 58
#define SYS_GET_FILE_SIZE 59
#define SYS_GET_FILE_STAT 60
#define SYS_STAT_FILE 61
#define SYS_TRUNCATE_FILE 62
#define SYS_FLUSH_FILE 63
#define SYS_DELETE_FILE 64
#define SYS_RENAME_FILE 65
#define SYS_LOCK_FILE 66
#define SYS_UNLOCK_FILE 67
#define SYS_OPEN_DIR 68
#define SYS_READ_DIR 69
#define SYS_CREATE_DIR 70
#define SYS_REMOVE_DIR 71
#define SYS_SET_CWD 72
#define SYS_REAL_PATH 73
#define SYS_OPEN_FILE 52
#define SYS_READ_FILE 53
#define SYS_WRITE_FILE 54
#define SYS_READ_FILE_ASYNC 55
#define SYS_WRITE_FILE_ASYNC 56
#define SYS_CANCEL_IO 57
#define SYS_SEEK_FILE 58
#define SYS_GET_FILE_POS 59
#define SYS_GET_FILE_SIZE 60
#define SYS_GET_FILE_STAT 61
#define SYS_STAT_FILE 62
#define SYS_TRUNCATE_FILE 63
#define SYS_FLUSH_FILE 64
#define SYS_DELETE_FILE 65
#define SYS_RENAME_FILE 66
#define SYS_LOCK_FILE 67
#define SYS_UNLOCK_FILE 68
#define SYS_OPEN_DIR 69
#define SYS_READ_DIR 70
#define SYS_CREATE_DIR 71
#define SYS_REMOVE_DIR 72
#define SYS_SET_CWD 73
#define SYS_REAL_PATH 74
// Console, display, and surfaces.
#define SYS_WRITE_CONSOLE 74
#define SYS_GET_CONSOLE_INFO 75
#define SYS_SET_CONSOLE_CURSOR 76
#define SYS_FILL_CONSOLE_CHARACTER 77
#define SYS_FILL_CONSOLE_ATTRIBUTE 78
#define SYS_PUT_SCREEN_CHAR 79
#define SYS_CLEAR_SCREEN 80
#define SYS_GET_SCREEN_SIZE 81
#define SYS_QUERY_DISPLAY_INFO 82
#define SYS_OPEN_DISPLAY_CLOCK 83
#define SYS_CREATE_SURFACE 84
#define SYS_MAP_SURFACE 85
#define SYS_QUERY_SURFACE 86
#define SYS_PRESENT_SURFACE 87
#define SYS_WRITE_CONSOLE 75
#define SYS_GET_CONSOLE_INFO 76
#define SYS_SET_CONSOLE_CURSOR 77
#define SYS_FILL_CONSOLE_CHARACTER 78
#define SYS_FILL_CONSOLE_ATTRIBUTE 79
#define SYS_PUT_SCREEN_CHAR 80
#define SYS_CLEAR_SCREEN 81
#define SYS_GET_SCREEN_SIZE 82
#define SYS_QUERY_DISPLAY_INFO 83
#define SYS_OPEN_DISPLAY_CLOCK 84
#define SYS_CREATE_SURFACE 85
#define SYS_MAP_SURFACE 86
#define SYS_QUERY_SURFACE 87
#define SYS_PRESENT_SURFACE 88
// Native render audio.
#define SYS_AUDIO_ENUMERATE_ENDPOINTS 89
#define SYS_AUDIO_OPEN_STREAM 90
#define SYS_AUDIO_WRITE 91
#define SYS_AUDIO_ARM_WAIT 92
#define SYS_AUDIO_QUERY_STREAM 93
#define SYS_AUDIO_START 94
#define SYS_AUDIO_PAUSE 95
#define SYS_AUDIO_RESUME 96
#define SYS_AUDIO_DRAIN 97
#define SYS_AUDIO_STOP 98
#define SYS_AUDIO_RESET 99
#define SYS_AUDIO_SET_VOLUME 100
#define SYS_AUDIO_SET_MASTER_VOLUME 101
// Input.
#define SYS_POLL_KEYBOARD 88
#define SYS_IS_KEY_DOWN 89
#define SYS_READ_KEYBOARD_CHAR 90
#define SYS_READ_KEY 91
#define SYS_GET_KEY_MODIFIERS 92
#define SYS_QUERY_GLOBAL_INPUT_STATE 93
#define SYS_OPEN_INPUT_QUEUE 94
#define SYS_READ_INPUT_EVENT 95
#define SYS_PEEK_INPUT_EVENT 96
#define SYS_PEEK_MOUSE 97
#define SYS_READ_MOUSE 98
#define SYS_GET_MOUSE_POS 99
#define SYS_POLL_KEYBOARD 102
#define SYS_IS_KEY_DOWN 103
#define SYS_READ_KEYBOARD_CHAR 104
#define SYS_READ_KEY 105
#define SYS_GET_KEY_MODIFIERS 106
#define SYS_QUERY_GLOBAL_INPUT_STATE 107
#define SYS_OPEN_INPUT_QUEUE 108
#define SYS_READ_INPUT_EVENT 109
#define SYS_PEEK_INPUT_EVENT 110
#define SYS_PEEK_MOUSE 111
#define SYS_READ_MOUSE 112
#define SYS_GET_MOUSE_POS 113
// Time, debug, and system information.
#define SYS_CREATE_TIMER 100
#define SYS_SET_TIMER 101
#define SYS_CANCEL_TIMER 102
#define SYS_GET_TICK 103
#define SYS_GET_TICK_FREQUENCY 104
#define SYS_GET_SYSTEM_TIME 105
#define SYS_PRINT_DEBUG 106
#define SYS_QUERY_TRAP_FRAME 107
#define SYS_GET_MEMORY_STATUS 108
#define SYS_QUERY_KERNEL_STATS 109
#define SYS_CREATE_TIMER 114
#define SYS_SET_TIMER 115
#define SYS_CANCEL_TIMER 116
#define SYS_GET_TICK 117
#define SYS_GET_TICK_FREQUENCY 118
#define SYS_GET_SYSTEM_TIME 119
#define SYS_PRINT_DEBUG 120
#define SYS_QUERY_TRAP_FRAME 121
#define SYS_GET_MEMORY_STATUS 122
#define SYS_QUERY_KERNEL_STATS 123
// Native sockets and polling.
#define SYS_CREATE_SOCKET 110
#define SYS_SOCKET_IO_ASYNC 111
#define SYS_BIND_SOCKET 112
#define SYS_LISTEN_SOCKET 113
#define SYS_ACCEPT_SOCKET 114
#define SYS_CONNECT_SOCKET 115
#define SYS_SEND_SOCKET 116
#define SYS_RECV_SOCKET 117
#define SYS_SHUTDOWN_SOCKET 118
#define SYS_SEND_TO_SOCKET 119
#define SYS_RECV_FROM_SOCKET 120
#define SYS_RESOLVE_HOST 121
#define SYS_SET_SOCKET_OPTION 122
#define SYS_GET_SOCKET_OPTION 123
#define SYS_POLL 124
#define SYS_UNPUBLISH_QUEUE 125
#define SYS_CREATE_SOCKET 124
#define SYS_SOCKET_IO_ASYNC 125
#define SYS_BIND_SOCKET 126
#define SYS_LISTEN_SOCKET 127
#define SYS_ACCEPT_SOCKET 128
#define SYS_CONNECT_SOCKET 129
#define SYS_SEND_SOCKET 130
#define SYS_RECV_SOCKET 131
#define SYS_SHUTDOWN_SOCKET 132
#define SYS_SEND_TO_SOCKET 133
#define SYS_RECV_FROM_SOCKET 134
#define SYS_RESOLVE_HOST 135
#define SYS_SET_SOCKET_OPTION 136
#define SYS_GET_SOCKET_OPTION 137
#define SYS_POLL 138
#define SYS_UNPUBLISH_QUEUE 139
#define SYS_MAX 126
#define SYS_MAX 140
code/kernel/io_ddk.c
+9
View File
@@ -5,8 +5,12 @@
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/io_ddk.h>
#include <kernel/io_dma.h>
#include <kernel/ke_audio.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_irq.h>
#include <kernel/ke_sched.h>
#include <kernel/mm_pte.h>
static const HAL_API* gIoDdkHal;
@@ -68,6 +72,7 @@ static KSTATUS IoDdkAllocateMsiVector(uint8_t* outIrq) {
static const NTL_DDK_DEBUG gIoDdkDebug = {
.Print = DbgPrint,
.GetFlags = KeGetDebugFlags,
};
static const NTL_DDK_INTERRUPT gIoDdkInterrupt = {
@@ -99,6 +104,8 @@ static NTL_DDK_TIMER gIoDdkTimer = {
static NTL_DDK gIoDdk = {
.Version = NTL_DDK_VERSION,
.Debug = &gIoDdkDebug,
.Audio = (const NTL_DDK_AUDIO*)0,
.Dma = (const NTL_DDK_DMA*)0,
.Interrupt = &gIoDdkInterrupt,
.Memory = &gIoDdkMemory,
.Platform = &gIoDdkPlatform,
@@ -109,6 +116,8 @@ static NTL_DDK gIoDdk = {
// Initializes the kernel-owned DDK table from platform services.
void IoInitializeDdk(const HAL_API* hal) {
gIoDdkHal = hal;
gIoDdk.Audio = KeAudioGetDdk();
gIoDdk.Dma = IoGetDmaDdk();
gIoDdkTimer.QueryTicks = hal ? hal->Timer.QueryTicks : 0;
gIoDdkTimer.Stall = hal ? hal->Timer.Stall : 0;
}
code/kernel/io_device.c
+51 -240
View File
@@ -4,6 +4,7 @@
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_irp.h>
#include <kernel/io_dma.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
@@ -107,6 +108,7 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
static void IoDeviceDestroy(OBJECT* object) {
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
IoDmaDetachDevice(dev);
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == dev) {
@@ -316,12 +318,19 @@ static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
// Releases device.
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
if (device->DeletePending)
if (!device)
return;
uintptr_t flags = KeAcquireSpinlock(&device->Lock);
if (device->DeletePending) {
KeReleaseSpinlock(&device->Lock, flags);
return;
}
device->DeletePending = true;
device->PnpState = DevicePnpDeleted;
KeReleaseSpinlock(&device->Lock, flags);
IoDmaDetachDevice(device);
IoUnlinkDeviceFromParent(device);
IoRemovePhysicalDeviceRegistryEntry(device);
IoDereferenceDeviceObject(device);
@@ -441,167 +450,50 @@ static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
#define IO_DISK_READAHEAD_MAX_SECTORS 128u
#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u
#define IO_DISK_READAHEAD_MIN_SECTORS 8u
#define IO_DISK_SECTOR_BYTES 512u
typedef struct IO_DISK_READAHEAD_STATE {
typedef struct IO_DISK_STATS_STATE {
SPINLOCK Lock;
uint8_t* Buffer;
uint8_t Valid;
uint8_t Loading;
uint64_t Generation;
uint64_t BaseLba;
uint32_t Count;
uint64_t NextLba;
uint32_t SequentialScore;
IO_DISK_STATS Stats;
} IO_DISK_READAHEAD_STATE;
} IO_DISK_STATS_STATE;
static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES];
static IO_DISK_STATS_STATE gDiskStats[MAX_DISK_DEVICES];
// Updates a maximum statistic while holding the disk read-ahead lock.
static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) {
if (field && *field < value) *field = value;
}
// Prepares the per-disk read-ahead buffer.
static void IoDiskReadAheadPrepare(uint8_t driveIndex) {
// Initializes per-disk I/O statistics.
static void IoDiskStatsPrepare(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
state->Lock = (SPINLOCK)SPINLOCK_INIT;
state->Valid = 0;
state->Loading = 0;
state->Generation = 0;
state->BaseLba = 0;
state->Count = 0;
state->NextLba = 0;
state->SequentialScore = 0;
memset(&state->Stats, 0, sizeof(state->Stats));
if (!state->Buffer) {
state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES,
MM_POOL_TAG('d', 'r', 'a', 'h'));
if (!state->Buffer)
state->Stats.ReadaheadAllocFailures++;
}
}
// Releases the per-disk read-ahead buffer.
static void IoDiskReadAheadRelease(uint8_t driveIndex) {
// Records one disk read request.
static void IoDiskStatsNoteRead(uint8_t driveIndex, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uint8_t* buffer = (uint8_t*)0;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (!state->Loading) {
buffer = state->Buffer;
state->Buffer = (uint8_t*)0;
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->SequentialScore = 0;
KeReleaseSpinlock(&state->Lock, flags);
break;
}
KeReleaseSpinlock(&state->Lock, flags);
KeYield();
}
if (buffer)
MmFreePool(buffer);
}
// Invalidates cached disk read-ahead data for one disk.
static void IoDiskReadAheadInvalidate(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Returns true when one range fully contains another range.
static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount,
uint64_t innerBase, uint32_t innerCount) {
uint64_t outerEnd = outerBase + outerCount;
uint64_t innerEnd = innerBase + innerCount;
if (outerEnd < outerBase || innerEnd < innerBase)
return false;
return innerBase >= outerBase && innerEnd <= outerEnd;
}
// Attempts to satisfy a disk read from the per-disk read-ahead buffer.
static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) {
if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0)
return false;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool hit = state->Buffer && state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, lba, count);
if (hit) {
size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES;
size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES;
memcpy(buffer, state->Buffer + offset, bytes);
state->Stats.ReadaheadHits++;
state->Stats.ReadaheadBytesServed += bytes;
} else {
state->Stats.ReadaheadMisses++;
}
KeReleaseSpinlock(&state->Lock, flags);
return hit;
}
// Records request size and sequential access state for one disk read.
static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES || count == 0)
return 0;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadRequests++;
state->Stats.ReadSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count);
if (lba == state->NextLba) {
if (state->SequentialScore < 32)
state->SequentialScore++;
} else {
state->SequentialScore = 1;
}
state->NextLba = lba + count;
uint32_t score = state->SequentialScore;
if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE)
state->Stats.ReadaheadSequentialDetections++;
if (state->Stats.MaxReadSectors < count)
state->Stats.MaxReadSectors = count;
KeReleaseSpinlock(&state->Lock, flags);
return score;
}
// Records one disk write and invalidates stale read-ahead data.
static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) {
// Records one successful disk write request.
static void IoDiskStatsNoteWrite(uint8_t driveIndex, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.WriteRequests++;
state->Stats.WriteSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
if (state->Stats.MaxWriteSectors < count)
state->Stats.MaxWriteSectors = count;
KeReleaseSpinlock(&state->Lock, flags);
}
@@ -653,7 +545,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
if (pnpStatus == NTLSTATUS_NOT_SUPPORTED)
device->PnpState = DevicePnpStarted;
IoDiskReadAheadPrepare(driveIndex);
IoDiskStatsPrepare(driveIndex);
IoReferenceDeviceObject(device);
gDiskDevices[driveIndex] = device;
@@ -668,8 +560,6 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return NTLSTATUS_NOT_FOUND;
IoDiskReadAheadInvalidate(driveIndex);
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
@@ -698,7 +588,6 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoDiskReadAheadRelease(driveIndex);
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return NTLSTATUS_SUCCESS;
@@ -846,83 +735,6 @@ static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
return IoSubmitIrpSync(dev, &irp);
}
// Chooses a conservative read-ahead span from the current sequential score.
static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) {
uint32_t sectors = currentCount;
if (sectors < IO_DISK_READAHEAD_MIN_SECTORS)
sectors = IO_DISK_READAHEAD_MIN_SECTORS;
if (score >= 8 && sectors < 128u)
sectors = 128u;
else if (score >= 4 && sectors < 64u)
sectors = 64u;
else if (score >= 2 && sectors < 32u)
sectors = 32u;
if (sectors > IO_DISK_READAHEAD_MAX_SECTORS)
sectors = IO_DISK_READAHEAD_MAX_SECTORS;
return sectors;
}
// Loads the next sequential disk range into the read-ahead buffer.
static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev,
uint64_t nextLba, uint32_t currentCount, uint32_t score) {
if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
if (!state->Buffer) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score);
uint64_t diskSectors = dev->u.Disk.SectorCount;
if (nextLba >= diskSectors)
return;
if ((uint64_t)sectors > diskSectors - nextLba)
sectors = (uint32_t)(diskSectors - nextLba);
if (sectors == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool covered = state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors);
if (covered) {
KeReleaseSpinlock(&state->Lock, flags);
return;
}
if (state->Loading) {
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint64_t generation = state->Generation;
state->Loading = 1;
state->Valid = 0;
state->Count = 0;
KeReleaseSpinlock(&state->Lock, flags);
KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer);
flags = KeAcquireSpinlock(&state->Lock);
state->Loading = 0;
if (KSUCCESS(st) && generation == state->Generation) {
state->BaseLba = nextLba;
state->Count = sectors;
state->Valid = 1;
state->Stats.ReadaheadStores++;
state->Stats.ReadaheadSectorsLoaded += sectors;
IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors);
} else {
state->Valid = 0;
state->Count = 0;
if (!KSUCCESS(st))
state->Stats.ReadaheadReadFailures++;
}
KeReleaseSpinlock(&state->Lock, flags);
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
@@ -938,17 +750,8 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
return rangeStatus;
}
if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) {
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return NTLSTATUS_SUCCESS;
}
KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
if (KSUCCESS(st))
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDiskStatsNoteRead(driveIndex, count);
IoDereferenceDeviceObject(dev);
return st;
@@ -971,19 +774,39 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
if (KSUCCESS(st))
IoDiskReadAheadNoteWrite(driveIndex, count);
IoDiskStatsNoteWrite(driveIndex, count);
IoDereferenceDeviceObject(dev);
return st;
}
// Flushes volatile write caches for one disk device.
__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return NTLSTATUS_NOT_FOUND;
IRP irp;
IoInitializeIrp(&irp, 1);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(&irp);
if (!ios) {
IoDereferenceDeviceObject(dev);
return NTLSTATUS_FAILURE;
}
ios->MajorFunction = IRP_MJ_FLUSH_BUFFERS;
ios->MinorFunction = 0;
KSTATUS st = IoSubmitIrpSync(dev, &irp);
IoDereferenceDeviceObject(dev);
return st;
}
// Queries aggregate disk I/O statistics.
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i];
IO_DISK_STATS_STATE* state = &gDiskStats[i];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (gDiskDevices[i])
out->RegisteredDisks++;
@@ -995,18 +818,6 @@ __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
out->MaxReadSectors = state->Stats.MaxReadSectors;
if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
out->MaxWriteSectors = state->Stats.MaxWriteSectors;
out->ReadaheadHits += state->Stats.ReadaheadHits;
out->ReadaheadMisses += state->Stats.ReadaheadMisses;
out->ReadaheadStores += state->Stats.ReadaheadStores;
out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations;
out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed;
out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded;
out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections;
out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed;
out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures;
out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures;
if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors)
out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors;
KeReleaseSpinlock(&state->Lock, flags);
}
}
code/kernel/io_dma.c
+1056
View File
File diff suppressed because it is too large. Load diff
code/kernel/io_file.c
+64 -72
View File
@@ -26,6 +26,47 @@ typedef struct {
static VOLUME_ENTRY gVolumes[MAX_DRIVES];
static EVENT* gVolumeChangeEvent;
static SPINLOCK gVolumeNamespaceLock = SPINLOCK_INIT;
static uint64_t gVolumeNamespaceGeneration;
// Publishes a mounted-volume namespace change to boot discovery and other waiters.
static void KiIoNotifyVolumeNamespaceChange(void) {
uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock);
gVolumeNamespaceGeneration++;
KeReleaseSpinlock(&gVolumeNamespaceLock, flags);
if (gVolumeChangeEvent)
KeSignalEvent(gVolumeChangeEvent);
}
// Initializes the waitable mounted-volume namespace notification.
KSTATUS IoInitializeVolumeNamespace(void) {
if (gVolumeChangeEvent)
return NTLSTATUS_SUCCESS;
return KeEventCreate(EventAutoReset, 0, &gVolumeChangeEvent);
}
// Returns the current mounted-volume namespace generation.
uint64_t IoGetVolumeNamespaceGeneration(void) {
uintptr_t flags = KeAcquireSpinlock(&gVolumeNamespaceLock);
uint64_t generation = gVolumeNamespaceGeneration;
KeReleaseSpinlock(&gVolumeNamespaceLock, flags);
return generation;
}
// Waits for a mounted-volume namespace change without polling driver discovery.
KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration,
uint64_t timeoutMilliseconds) {
if (IoGetVolumeNamespaceGeneration() != observedGeneration)
return NTLSTATUS_SUCCESS;
if (!gVolumeChangeEvent)
return NTLSTATUS_OBJECT_DEAD;
EVENT* events[1] = { gVolumeChangeEvent };
uint32_t selected = 0;
return KeWaitForMultipleEvents(events, 1, 0, timeoutMilliseconds, &selected);
}
// Async file I/O worker.
typedef enum IO_ASYNC_FILE_OP {
IoAsyncFileRead = 0,
@@ -275,8 +316,8 @@ static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
static void KiIoAsyncFileWorker(void* arg) {
(void)arg;
for (;;) {
KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore);
while (true) {
KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(s)) {
KeYield();
continue;
@@ -335,9 +376,10 @@ static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker,
0,
PriorityNormal,
&parameters,
&worker);
// Keep the creator reference: this worker is intentionally persistent
// for the lifetime of the kernel.
@@ -751,6 +793,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
return NTLSTATUS_ALREADY_EXISTS;
gVolumes[volumeIndex].Fs = fs;
gVolumes[volumeIndex].Vol = *vol;
KiIoNotifyVolumeNamespaceChange();
return NTLSTATUS_SUCCESS;
}
@@ -762,18 +805,23 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
return NTLSTATUS_NOT_FOUND;
memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex]));
KiIoNotifyVolumeNamespaceChange();
return NTLSTATUS_SUCCESS;
}
// Implements the unregister Filesystems For Disk operation.
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
KSTATUS last = NTLSTATUS_NOT_FOUND;
bool changed = false;
for (uint8_t i = 0; i < MAX_DRIVES; i++) {
if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) {
memset(&gVolumes[i], 0, sizeof(gVolumes[i]));
last = NTLSTATUS_SUCCESS;
changed = true;
}
}
if (changed)
KiIoNotifyVolumeNamespaceChange();
return last;
}
@@ -819,7 +867,11 @@ __declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
static void KiFileDestroy(OBJECT* obj) {
FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
if (f->Fs && f->FileCtx) {
(void)IoFlushFile(f);
// Close drains kernel-owned dirty pages, but stable-media semantics
// remain explicit through IoFlushFile so every close does not issue a
// whole-device cache flush.
(void)MmVadSyncFileMappings(f);
(void)MmFileCacheFlushFile(f);
MmFileCacheInvalidateAll(f);
f->Fs->Close(f->Fs, f->FileCtx);
}
@@ -853,71 +905,11 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
f->Size = fs->GetSize(fs, fileCtx);
f->Offset = 0;
f->Flags = 0;
f->ReadaheadScore = 0;
f->ReadaheadNextOffset = 0;
f->ReadaheadIssuedUntil = 0;
f->ReadaheadSuppressedUntil = 0;
*out = f;
return NTLSTATUS_SUCCESS;
}
static uint64_t KiAlignUpPage64(uint64_t value) {
return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
}
static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) {
if (!file || bytesRead < 512)
return;
if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0)
return;
uint64_t end = offset + bytesRead;
uint64_t start = 0;
uint32_t pages = 0;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
if (offset == file->ReadaheadNextOffset) {
if (file->ReadaheadScore < 32)
file->ReadaheadScore++;
} else {
file->ReadaheadScore = 1;
file->ReadaheadIssuedUntil = end;
}
file->ReadaheadNextOffset = end;
if (end < file->ReadaheadSuppressedUntil) {
pages = 0;
} else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) {
pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u);
start = KiAlignUpPage64(end);
if (start < file->ReadaheadIssuedUntil)
start = KiAlignUpPage64(file->ReadaheadIssuedUntil);
if (start >= fileSize) {
pages = 0;
} else {
uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE);
if (maxEnd > fileSize)
pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE);
file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE);
}
}
KeReleaseSpinlock(&file->Lock, flags);
if (pages) {
KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages);
if (ras == NTLSTATUS_QUEUE_FULL) {
flags = KeAcquireSpinlock(&file->Lock);
uint64_t suppressUntil = end + (1024u * 1024u);
if (suppressUntil < end || suppressUntil > fileSize)
suppressUntil = fileSize;
if (file->ReadaheadSuppressedUntil < suppressUntil)
file->ReadaheadSuppressedUntil = suppressUntil;
KeReleaseSpinlock(&file->Lock, flags);
}
}
}
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -979,7 +971,6 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
}
*bytesRead = total;
KiIoMaybeScheduleReadAhead(file, offset, total, fileSize);
return NTLSTATUS_SUCCESS;
}
@@ -1022,8 +1013,13 @@ __declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) {
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
KSTATUS sync = MmVadSyncFileMappings(file);
KSTATUS flush = MmFileCacheFlushFile(file);
return KSUCCESS(sync) ? flush : sync;
KSTATUS cache = MmFileCacheFlushFile(file);
KSTATUS media = NTLSTATUS_SUCCESS;
if (KSUCCESS(sync) && KSUCCESS(cache) && file->Fs && file->Fs->Flush)
media = file->Fs->Flush(file->Fs, file->FileCtx);
if (!KSUCCESS(sync)) return sync;
if (!KSUCCESS(cache)) return cache;
return media;
}
// IoReadFile - sequential read at current offset.
@@ -1091,10 +1087,6 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o
f->Size = 0;
f->Offset = 0;
f->Flags = 0;
f->ReadaheadScore = 0;
f->ReadaheadNextOffset = 0;
f->ReadaheadIssuedUntil = 0;
f->ReadaheadSuppressedUntil = 0;
*out = f;
return NTLSTATUS_SUCCESS;
@@ -1278,7 +1270,7 @@ __declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint
IO_FILE_LOCK_WAITER waiter;
memset(&waiter, 0, sizeof(waiter));
for (;;) {
while (true) {
IO_FILE_LOCK_ENTRY* entry = (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k'));
if (!entry) {
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
code/kernel/ke_apc.c
+3 -3
View File
@@ -200,7 +200,7 @@ void KiDeliverKernelApcs(void) {
thread->ApcDeliveryActive = 1;
for (;;) {
while (true) {
if (KeGetCurrentIrql() != PASSIVE_LEVEL || thread->KernelApcDisable != 0)
break;
@@ -375,7 +375,7 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
void KiRundownApcState(THREAD* thread) {
if (!thread) return;
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
LIST_ENTRY* node = ListPopHead(&thread->ApcQueue);
if (LIST_IS_EMPTY(&thread->ApcQueue))
@@ -389,7 +389,7 @@ void KiRundownApcState(THREAD* thread) {
apc->RundownRoutine(apc);
}
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue);
if (LIST_IS_EMPTY(&thread->UserApcQueue))
code/kernel/ke_audio.c
+2460
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_cpu.c
+130
View File
@@ -4,19 +4,148 @@
#include <string.h>
#include <intrin.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <ntlite/ddi/hal_api.h>
#define BOOTSTRAP_CPU_STACK_SIZE 0x4000u
#define IA32_GS_BASE 0xC0000101u
#define IA32_KERNEL_GS_BASE 0xC0000102u
#define IA32_MTRRCAP 0x000000FEu
#define IA32_MTRR_PHYSBASE0 0x00000200u
#define IA32_MTRR_PHYSMASK0 0x00000201u
#define IA32_PAT 0x00000277u
#define IA32_MTRR_DEF_TYPE 0x000002FFu
#define CPUID_FEATURE_MTRR (1u << 12)
#define CPUID_FEATURE_PAT (1u << 16)
#define MTRR_CAP_VARIABLE_COUNT 0x000000FFull
#define MTRR_MASK_VALID (1ull << 11)
#define MTRR_DEF_TYPE_MASK 0x000000FFull
#define MTRR_DEF_FIXED_ENABLE (1ull << 10)
#define MTRR_DEF_ENABLE (1ull << 11)
#define MEMORY_TYPE_UC 0x00ull
#define MEMORY_TYPE_WC 0x01ull
#define MEMORY_TYPE_WB 0x06ull
#define CR0_CACHE_DISABLE (1ull << 30)
#define CR0_NOT_WRITE_THROUGH (1ull << 29)
#define CR4_PAGE_GLOBAL_ENABLE (1ull << 7)
#define CR4_PCID_ENABLE (1ull << 17)
#define PAT_ENTRY(type, index) ((uint64_t)(type) << ((index) * 8u))
#define CANONICAL_PAT (PAT_ENTRY(MEMORY_TYPE_WB, 0) | \
PAT_ENTRY(MEMORY_TYPE_WC, 1) | \
PAT_ENTRY(MEMORY_TYPE_UC, 2) | \
PAT_ENTRY(MEMORY_TYPE_UC, 3) | \
PAT_ENTRY(MEMORY_TYPE_WB, 4) | \
PAT_ENTRY(MEMORY_TYPE_WC, 5) | \
PAT_ENTRY(MEMORY_TYPE_UC, 6) | \
PAT_ENTRY(MEMORY_TYPE_UC, 7))
KCPU gCpus[KE_MAX_CPUS];
uint32_t gCpuCount = 0;
static uint8_t gBootstrapCpuStack[BOOTSTRAP_CPU_STACK_SIZE];
static volatile long gKernelGsReady = 0;
static volatile long gWriteCombiningAvailable = 0;
extern KCPU* KeReadCurrentCpuFromGs(void);
// Returns whether this CPU exposes the memory-type mechanisms required by the kernel policy.
static bool KiMemoryTypeFeaturesAvailable(void) {
int regs[4];
__cpuid(regs, 1);
uint32_t features = (uint32_t)regs[3];
return (features & (CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT)) ==
(CPUID_FEATURE_MTRR | CPUID_FEATURE_PAT);
}
// Flushes all translations before memory-type state changes and leaves global pages disabled.
static uintptr_t KiDisableGlobalTranslations(void) {
uintptr_t cr4 = __readcr4();
// nt-lite does not enable PCID. Refuse inherited PCID state rather than
// pretending that a CR3 reload flushed every tagged translation.
if (cr4 & CR4_PCID_ENABLE)
HalStopProcessor();
if (cr4 & CR4_PAGE_GLOBAL_ENABLE)
__writecr4(cr4 & ~CR4_PAGE_GLOBAL_ENABLE);
else
__writecr3(__readcr3());
return cr4;
}
// Returns whether this CPU accepted the complete canonical PAT and MTRR policy.
static bool KiMemoryTypePolicyInstalled(uint32_t variableCount) {
if (__readmsr(IA32_PAT) != CANONICAL_PAT)
return false;
uint64_t expectedDefault = MTRR_DEF_ENABLE | MEMORY_TYPE_WB;
uint64_t defaultType = __readmsr(IA32_MTRR_DEF_TYPE);
uint64_t relevantDefaultBits = MTRR_DEF_TYPE_MASK | MTRR_DEF_FIXED_ENABLE | MTRR_DEF_ENABLE;
if ((defaultType & relevantDefaultBits) != expectedDefault)
return false;
for (uint32_t i = 0; i < variableCount; i++) {
uint64_t base = __readmsr(IA32_MTRR_PHYSBASE0 + i * 2u);
uint64_t mask = __readmsr(IA32_MTRR_PHYSMASK0 + i * 2u);
if (base != 0 || (mask & MTRR_MASK_VALID) != 0)
return false;
}
return true;
}
// Programs the kernel-owned PAT policy over a uniform write-back MTRR baseline.
void KeInitializeMemoryTypes(void) {
if (!KiMemoryTypeFeaturesAvailable())
HalStopProcessor();
uint32_t variableCount = (uint32_t)(__readmsr(IA32_MTRRCAP) & MTRR_CAP_VARIABLE_COUNT);
if (KiMemoryTypePolicyInstalled(variableCount)) {
_InterlockedExchange(&gWriteCombiningAvailable, 1);
return;
}
uintptr_t interruptFlags = HalDisableInterrupts();
uintptr_t cr0 = __readcr0();
// Every CPU runs this before entering normal kernel execution. The BSP
// installs the policy while it is alone, and each AP installs it before
// publishing its started state.
__writecr0((cr0 | CR0_CACHE_DISABLE) & ~CR0_NOT_WRITE_THROUGH);
__wbinvd();
uintptr_t cr4 = KiDisableGlobalTranslations();
// MTRRs provide only a uniform WB baseline. PAT is the sole owner of the
// page-granular WB, WC, and UC policy selected by kernel mappings.
__writemsr(IA32_MTRR_DEF_TYPE, 0);
for (uint32_t i = 0; i < variableCount; i++) {
__writemsr(IA32_MTRR_PHYSBASE0 + i * 2u, 0);
__writemsr(IA32_MTRR_PHYSMASK0 + i * 2u, 0);
}
__writemsr(IA32_PAT, CANONICAL_PAT);
__writemsr(IA32_MTRR_DEF_TYPE, MTRR_DEF_ENABLE | MEMORY_TYPE_WB);
__wbinvd();
__writecr3(__readcr3());
__writecr0(cr0);
if (cr4 & CR4_PAGE_GLOBAL_ENABLE)
__writecr4(cr4);
bool installed = KiMemoryTypePolicyInstalled(variableCount);
HalRestoreInterrupts(interruptFlags);
if (!installed)
HalStopProcessor();
_InterlockedExchange(&gWriteCombiningAvailable, 1);
}
// Reports whether PAT write-combining is available on the current platform.
bool KeWriteCombiningAvailable(void) {
return _InterlockedCompareExchange(&gWriteCombiningAvailable, 0, 0) != 0;
}
// Loads kernel Gs Base.
void KeLoadKernelGsBase(KCPU* cpu) {
if (!cpu)
@@ -69,6 +198,7 @@ void KeInitializeBootstrapCpu(void) {
cpu->Tss.RSP0 = cpu->KernelStackTop;
gCpuCount = 1;
KeInitializeMemoryTypes();
KeLoadKernelGsBase(cpu);
KeSetUserGsBase(0);
}
code/kernel/ke_debug.c
+56 -18
View File
@@ -220,7 +220,29 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
line, length);
}
// Flushes debug Log To File.
// Writes one debug-log segment and rejects successful zero-progress writes.
static KSTATUS KiDebugLogWriteAll(FILE_OBJECT* file,
const char* data,
size_t length,
size_t* totalWritten) {
if (!file || (!data && length) || !totalWritten)
return NTLSTATUS_INVALID_PARAMETER;
while (length != 0) {
size_t written = 0;
KSTATUS st = IoWriteFile(file, data, length, &written);
if (!KSUCCESS(st))
return st;
if (written == 0)
return NTLSTATUS_DEVICE_ERROR;
data += written;
length -= written;
*totalWritten += written;
}
return NTLSTATUS_SUCCESS;
}
// Persists the in-memory debug ring and flushes it through the storage stack.
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -237,11 +259,6 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (used == 0) {
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return NTLSTATUS_SUCCESS;
}
(void)IoDeleteFile(path);
FILE_OBJECT* file = (FILE_OBJECT*)0;
@@ -251,23 +268,32 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
return st;
}
size_t written = 0;
size_t totalWritten = 0;
size_t first = gDebugLogSize - start;
if (first > used) first = used;
st = IoWriteFile(file, gDebugLogBuffer + start, first, &written);
totalWritten += written;
// Shutdown logging must not depend on deferred file-cache writeback. FAT
// still updates the directory entry and cluster chain through its normal
// write path, while IoFlushFile pushes metadata and the device cache.
file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;
if (KSUCCESS(st) && used > first) {
written = 0;
st = IoWriteFile(file, gDebugLogBuffer, used - first, &written);
totalWritten += written;
size_t totalWritten = 0;
if (used != 0) {
size_t first = gDebugLogSize - start;
if (first > used)
first = used;
st = KiDebugLogWriteAll(file, gDebugLogBuffer + start, first, &totalWritten);
if (KSUCCESS(st) && used > first) {
st = KiDebugLogWriteAll(file, gDebugLogBuffer,
used - first, &totalWritten);
}
}
IoCloseFile(file);
if (KSUCCESS(st) && totalWritten != used)
st = NTLSTATUS_DEVICE_ERROR;
if (KSUCCESS(st))
st = IoFlushFile(file);
IoCloseFile(file);
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return (KSUCCESS(st) && totalWritten == used) ? NTLSTATUS_SUCCESS : st;
return st;
}
// Attempts to persist the debug log during a panic when the filesystem is usable.
@@ -863,6 +889,18 @@ __declspec(dllexport) KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) {
if (!KeFaultFrameFromUser(frame))
KiPanicKernelFault(proc, thread, frame);
DbgPrint("USERFAULT: pid=%u tid=%u vector=%llu rip=0x%016llX rsp=0x%016llX cr2=0x%016llX error=0x%016llX rcx=0x%016llX rbx=0x%016llX rax=0x%016llX\n",
proc ? proc->ProcessId : 0,
thread ? thread->ThreadId : 0,
frame->Vector,
(uint64_t)frame->Rip,
(uint64_t)frame->Rsp,
(uint64_t)frame->Cr2,
(uint64_t)frame->ErrorCode,
(uint64_t)frame->Rcx,
(uint64_t)frame->Rbx,
(uint64_t)frame->Rax);
KSTATUS delivered = KeDeliverUserException(frame);
if (KSUCCESS(delivered))
return NTLSTATUS_SUCCESS;
code/kernel/ke_display.c
+463 -116
View File
@@ -19,6 +19,7 @@
#include <kernel/ke_list.h>
#include <kernel/ob_object.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_cpu.h>
#define DISPLAY_DEFAULT_FG 0x00CCCCCCu
#define DISPLAY_DEFAULT_BG 0x00012456u
@@ -44,6 +45,15 @@ typedef struct DISPLAY_OUTPUT_GLYPH {
uint32_t Background;
} DISPLAY_OUTPUT_GLYPH;
typedef struct DISPLAY_OUTPUT_DRAW {
char Character;
uint32_t Foreground;
uint32_t Background;
int Column;
int Row;
size_t ScrollCount;
} DISPLAY_OUTPUT_DRAW;
typedef struct BOOTFB_CONTEXT {
const HAL_API* Hal;
uint32_t* Framebuffer;
@@ -59,6 +69,7 @@ static DISPLAY_DEVICE* gDisplayList;
static DISPLAY_DEVICE* gPrimaryDisplay;
static SPINLOCK gDisplayLock = SPINLOCK_INIT;
static MUTEX* gDisplayProviderMutex;
static MUTEX* gDisplayOutputMutex;
static KWORK_ITEM gDisplayCursorWorkItem;
static volatile long gDisplayCursorWorkInitialized;
static bool gDisplayTransitionInProgress;
@@ -104,8 +115,10 @@ static KWORK_ITEM gDisplayOutputWorkItem;
static volatile long gDisplayOutputWorkInitialized;
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg);
static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count);
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels);
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color);
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h);
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride);
@@ -119,6 +132,7 @@ static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats)
static const DISPLAY_DEVICE_OPS gBootFbOps = {
.GetMode = BootFbGetMode,
.Blit = BootFbBlit,
.PresentPitched = BootFbPresentPitched,
.FillRect = BootFbFillRect,
.CopyRect = BootFbCopyRect,
.ReadRect = BootFbReadRect,
@@ -192,10 +206,15 @@ static KSTATUS KiDisplayAcquireProviderMutex(bool* acquired) {
if (KeGetDebugFlags() & KE_DEBUG_DISPLAY_ASSERTS)
KeAssertIrql(PASSIVE_LEVEL);
KSTATUS s = KeAcquireMutex(gDisplayProviderMutex);
if (KSUCCESS(s) && acquired)
KSTATUS s = KeAcquireMutex(gDisplayProviderMutex, KE_WAIT_INFINITE);
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s))
return s;
if (acquired)
*acquired = true;
return s;
// Provider operations still own the mutex after an abandoned acquisition.
return NTLSTATUS_SUCCESS;
}
static void KiDisplayReleaseProviderMutex(bool acquired) {
@@ -449,6 +468,34 @@ static bool KiDisplayConsoleGridFromMode(const DISPLAY_MODE* mode, int* outCols,
return true;
}
// Returns whether two dirty rectangles overlap or share an edge.
static bool KiDisplayDirtyRectsMergeable(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
if (!a || !b)
return false;
int64_t aRight = (int64_t)a->X + a->W;
int64_t aBottom = (int64_t)a->Y + a->H;
int64_t bRight = (int64_t)b->X + b->W;
int64_t bBottom = (int64_t)b->Y + b->H;
return (int64_t)a->X <= bRight && (int64_t)b->X <= aRight &&
(int64_t)a->Y <= bBottom && (int64_t)b->Y <= aBottom;
}
// Unions two clipped dirty rectangles.
static DISPLAY_RECT KiDisplayUnionDirtyRects(const DISPLAY_RECT* a, const DISPLAY_RECT* b) {
int64_t left = a->X < b->X ? a->X : b->X;
int64_t top = a->Y < b->Y ? a->Y : b->Y;
int64_t aRight = (int64_t)a->X + a->W;
int64_t bRight = (int64_t)b->X + b->W;
int64_t aBottom = (int64_t)a->Y + a->H;
int64_t bBottom = (int64_t)b->Y + b->H;
int64_t right = aRight > bRight ? aRight : bRight;
int64_t bottom = aBottom > bBottom ? aBottom : bBottom;
DISPLAY_RECT result = { (int32_t)left, (int32_t)top,
(int32_t)(right - left), (int32_t)(bottom - top) };
return result;
}
static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
if (!mode || gDirtyAll || w <= 0 || h <= 0)
return;
@@ -457,6 +504,19 @@ static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int
if (!KiDisplayClipRect(mode, &rect))
return;
for (size_t i = 0; i < gDirtyRectCount;) {
if (!KiDisplayDirtyRectsMergeable(&rect, &gDirtyRects[i])) {
i++;
continue;
}
rect = KiDisplayUnionDirtyRects(&rect, &gDirtyRects[i]);
gDirtyRectCount--;
if (i != gDirtyRectCount)
gDirtyRects[i] = gDirtyRects[gDirtyRectCount];
i = 0;
}
if (rect.X == 0 && rect.Y == 0 &&
rect.W == (int32_t)mode->Width && rect.H == (int32_t)mode->Height) {
KiDisplayDirtyAll();
@@ -478,6 +538,24 @@ static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, in
KiDisplayAddDirtyForModeLocked(mode, x, y, w, h);
}
static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
if (!mode || w <= 0 || h <= 0)
return;
DISPLAY_RECT rect = { x, y, w, h };
if (!KiDisplayClipRect(mode, &rect))
return;
uint64_t bytes = (uint64_t)(uint32_t)rect.W *
(uint64_t)(uint32_t)rect.H * sizeof(uint32_t);
gDisplayStats.BlitCount++;
gDisplayStats.PresentCount++;
gDisplayStats.PresentRectCount++;
gDisplayStats.DirectPresentCount++;
gDisplayStats.ScanoutBytes += bytes;
gDisplayStats.BytesCopied += bytes;
}
static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
gDisplayStats.FillRectCount++;
gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
@@ -660,7 +738,7 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
uint32_t glyphPixels[FONT_W * FONT_H];
KiDisplayBuildGlyphPixels(c, fg, bg, glyphPixels);
KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels);
KSTATUS s = ops->Blit(device, x, y, FONT_W, FONT_H, glyphPixels, FONT_W * sizeof(uint32_t));
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
if (device == gPrimaryDisplay)
@@ -670,20 +748,26 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
return s;
}
static KSTATUS KiDisplayDrawBlitSnapshot(DISPLAY_DEVICE* device,
const DISPLAY_DEVICE_OPS* ops,
const DISPLAY_MODE* mode,
int x,
int y,
int w,
int h,
const uint32_t* pixels) {
if (!pixels || w <= 0 || h <= 0)
// Draws a pitched source rectangle and records it as one display operation.
static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device,
const DISPLAY_DEVICE_OPS* ops,
const DISPLAY_MODE* mode,
int x,
int y,
int w,
int h,
const uint32_t* pixels,
uint32_t pitchBytes) {
if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
if (!device || !ops || !ops->Blit || !mode)
if (!device || !ops || !mode)
return NTLSTATUS_NOT_SUPPORTED;
KSTATUS s = ops->Blit(device, x, y, w, h, pixels);
if (!ops->Blit)
return NTLSTATUS_NOT_SUPPORTED;
KSTATUS s = ops->Blit(device, x, y, w, h, pixels, pitchBytes);
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
if (device == gPrimaryDisplay)
@@ -867,7 +951,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Blit)
return NTLSTATUS_NOT_SUPPORTED;
KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels);
KSTATUS s = gPrimaryDisplay->Ops->Blit(gPrimaryDisplay, x, y, w, h, pixels, (uint32_t)w * sizeof(uint32_t));
if (KSUCCESS(s)) {
gDisplayStats.BlitCount++;
gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
@@ -1043,11 +1127,18 @@ static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uin
}
// Drains buffered output through the active display provider.
static void KiDisplayOutputWorker(void* context) {
(void)context;
static KSTATUS KiDisplayDrainOutput(void) {
DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT];
bool mutexHeld = false;
for (;;) {
if (gDisplayOutputMutex) {
KSTATUS acquireStatus = KeAcquireMutex(gDisplayOutputMutex, KE_WAIT_INFINITE);
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(acquireStatus))
return acquireStatus;
mutexHeld = true;
}
while (true) {
size_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) {
@@ -1059,12 +1150,28 @@ static void KiDisplayOutputWorker(void* context) {
if (count == 0)
break;
for (size_t i = 0; i < count; i++) {
(void)KiDisplayPrintCharInternal(glyphs[i].Character,
glyphs[i].Foreground,
glyphs[i].Background);
KSTATUS drawStatus = KiDisplayPrintGlyphsInternal(glyphs, count);
if (!KSUCCESS(drawStatus)) {
if (mutexHeld)
KeReleaseMutex(gDisplayOutputMutex);
return drawStatus;
}
}
if (mutexHeld)
KeReleaseMutex(gDisplayOutputMutex);
return NTLSTATUS_SUCCESS;
}
// Drains buffered output from the passive work queue.
static void KiDisplayOutputWorker(void* context) {
(void)context;
(void)KiDisplayDrainOutput();
}
// Completes buffered console output before a caller overlays console pixels.
KSTATUS KeDisplayFlushConsoleOutput(void) {
return KiDisplayDrainOutput();
}
// Queues the runtime display output worker when passive work is available.
@@ -1157,6 +1264,8 @@ void KeDisplayInitialize(const HAL_API* hal) {
_InterlockedExchange(&gDisplayClockInitialized, 1);
if (!gDisplayProviderMutex)
(void)KeMutexCreate(&gDisplayProviderMutex);
if (!gDisplayOutputMutex)
(void)KeMutexCreate(&gDisplayOutputMutex);
KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
_InterlockedExchange(&gDisplayCursorWorkInitialized, 1);
KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0);
@@ -1212,8 +1321,9 @@ void KeDisplayInitialize(const HAL_API* hal) {
(void)KeDisplaySetPrimary(&gBootFbDevice);
DISPLAY_MODE mode;
if (KSUCCESS(KeDisplayGetMode(&mode))) {
DbgPrint("Display: bootfb primary %ux%u stride=%u\n",
mode.Width, mode.Height, mode.Stride);
DbgPrint("Display: bootfb primary %ux%u stride=%u cache=%s\n",
mode.Width, mode.Height, mode.Stride,
KeWriteCombiningAvailable() ? "WC" : "WT");
}
// The registration messages above were routed through the HAL boot
@@ -1277,7 +1387,7 @@ DISPLAY_API EVENT* KeDisplayClockWaitEvent(DISPLAY_CLOCK_OBJECT* clock) {
// Signals every display clock once for a provider-owned frame generation.
static void KiDisplaySignalFrameGeneration(uint64_t generation) {
for (;;) {
while (true) {
EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
uint32_t count = 0;
@@ -1340,7 +1450,7 @@ DISPLAY_API void KeDisplayTick(void) {
return;
uint64_t now = KiDisplayClockReadSchedulerTick();
for (;;) {
while (true) {
EVENT* events[DISPLAY_CLOCK_SIGNAL_BATCH];
uint32_t count = 0;
@@ -1424,7 +1534,8 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE
s = oldDevice->Ops->ReadRect(oldDevice, 0, (int)y, (int)newMode.Width, 1, row, newMode.Width);
if (!KSUCCESS(s))
break;
s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row);
s = newDevice->Ops->Blit(newDevice, 0, (int)y, (int)newMode.Width, 1, row,
newMode.Width * sizeof(uint32_t));
if (!KSUCCESS(s))
break;
}
@@ -1610,6 +1721,8 @@ DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) {
for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next)
count++;
info.DeviceCount = count;
if (KeWriteCombiningAvailable())
info.Flags |= DISPLAY_INFO_FLAG_PAT_WC_AVAILABLE;
info.ManagerStats = gDisplayStats;
DISPLAY_DEVICE* device = KiDisplayHasDevice() ? gPrimaryDisplay : (DISPLAY_DEVICE*)0;
@@ -1660,7 +1773,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi
s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
if (KSUCCESS(s)) {
(void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y, w, h, pixels);
s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels,
(uint32_t)w * sizeof(uint32_t));
s = KiDisplayFinishProviderDraw(device, ops, caps, s);
}
@@ -1668,8 +1782,8 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi
return s;
}
DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch) {
if (!source || !scratch || w <= 0 || h <= 0)
DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
bool providerHeld = false;
@@ -1684,14 +1798,7 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW
s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
if (KSUCCESS(s)) {
(void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
for (int row = 0; row < h; ++row) {
s = source(context, row, scratch, (size_t)w);
if (!KSUCCESS(s))
break;
s = KiDisplayDrawBlitSnapshot(device, ops, &mode, x, y + row, w, 1, scratch);
if (!KSUCCESS(s))
break;
}
s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode, x, y, w, h, pixels, pitchBytes);
s = KiDisplayFinishProviderDraw(device, ops, caps, s);
}
@@ -1699,6 +1806,47 @@ DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW
return s;
}
DISPLAY_API KSTATUS KeDisplayPresentPitched(int x,
int y,
int w,
int h,
const uint32_t* pixels,
uint32_t pitchBytes) {
if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
if (!KSUCCESS(s))
return s;
DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
uint32_t caps = 0;
DISPLAY_MODE mode;
s = KiDisplayGetProviderSnapshot(&device, &ops, &caps, &mode);
if (KSUCCESS(s)) {
(void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
if (ops->PresentPitched) {
s = ops->PresentPitched(device, x, y, w, h, pixels, pitchBytes);
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
if (device == gPrimaryDisplay)
KiDisplayRecordDirectPresentLocked(&mode, x, y, w, h);
KeReleaseSpinlock(&gDisplayLock, flags);
}
KiDisplayRestoreCursorAfterDrawSnapshot(device, ops, caps);
} else {
s = KiDisplayDrawBlitPitchedSnapshot(device, ops, &mode,
x, y, w, h, pixels, pitchBytes);
s = KiDisplayFinishProviderDraw(device, ops, caps, s);
}
}
KiDisplayReleaseProviderMutex(providerHeld);
return s;
}
DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) {
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
@@ -1835,18 +1983,28 @@ DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t
return s;
}
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
// Renders one or more console glyphs as one provider transaction and one present.
static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count) {
if (!glyphs && count != 0)
return NTLSTATUS_INVALID_PARAMETER;
if (count == 0)
return NTLSTATUS_SUCCESS;
if (count > DISPLAY_OUTPUT_DRAIN_COUNT)
return NTLSTATUS_INVALID_PARAMETER;
if (KiDisplayOutputSuspended()) {
if (!KeCanWait())
return KiDisplayFallbackPrintChar(c, fg, bg);
if (!KeCanWait()) {
for (size_t i = 0; i < count; i++)
(void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
}
return NTLSTATUS_SUCCESS;
}
// Once the scheduler/work-queue era begins, display provider callbacks are
// PASSIVE_LEVEL-only. The HAL output endpoint buffers normal text so this
// fallback is reserved for explicit display calls made at elevated IRQL.
if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait())
return KiDisplayFallbackPrintChar(c, fg, bg);
if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait()) {
for (size_t i = 0; i < count; i++)
(void)KiDisplayFallbackPrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
return NTLSTATUS_SUCCESS;
}
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
@@ -1861,8 +2019,10 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
if (!KSUCCESS(s)) {
KiDisplayReleaseProviderMutex(providerHeld);
const HAL_API* hal = gDisplayHal;
if (hal && hal->Screen.PrintChar)
hal->Screen.PrintChar(c, fg, bg);
if (hal && hal->Screen.PrintChar) {
for (size_t i = 0; i < count; i++)
hal->Screen.PrintChar(glyphs[i].Character, glyphs[i].Foreground, glyphs[i].Background);
}
return s;
}
@@ -1875,71 +2035,94 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
(void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
bool drawChar = false;
bool needScroll = false;
int drawCol = 0;
int drawRow = 0;
DISPLAY_OUTPUT_DRAW draws[DISPLAY_OUTPUT_DRAIN_COUNT];
size_t drawCount = 0;
size_t scrollCount = 0;
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
if (c == '\n') {
gConsoleX = 0;
gConsoleY++;
} else if (c == '\r') {
gConsoleX = 0;
} else if (c == '\b') {
if (gConsoleX > 0) {
gConsoleX--;
} else if (gConsoleY > 0) {
gConsoleY--;
gConsoleX = cols - 1;
}
} else {
drawChar = true;
drawCol = gConsoleX;
drawRow = gConsoleY;
if (++gConsoleX >= cols) {
for (size_t glyphIndex = 0; glyphIndex < count; glyphIndex++) {
char c = glyphs[glyphIndex].Character;
int drawCol = 0;
int drawRow = 0;
bool drawChar = false;
bool needScroll = false;
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
if (c == '\n') {
gConsoleX = 0;
gConsoleY++;
} else if (c == '\r') {
gConsoleX = 0;
} else if (c == '\b') {
if (gConsoleX > 0) {
gConsoleX--;
} else if (gConsoleY > 0) {
gConsoleY--;
gConsoleX = cols - 1;
}
} else {
drawChar = true;
drawCol = gConsoleX;
drawRow = gConsoleY;
if (++gConsoleX >= cols) {
gConsoleX = 0;
gConsoleY++;
}
if (drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) {
size_t cell = KiDisplayConsoleIndex(drawCol, drawRow);
gConsoleCells[cell] = c;
gConsoleAttrs[cell] = (uint8_t)gConsoleAttribute;
}
}
if (gConsoleY >= rows) {
KiDisplayConsoleScrollCellsLocked(cols, rows, 1);
gConsoleY = rows - 1;
if (gConsoleX < 0) gConsoleX = 0;
if (gConsoleX >= cols) gConsoleX = cols - 1;
needScroll = true;
}
KeReleaseSpinlock(&gDisplayLock, flags);
if (drawChar) {
DISPLAY_OUTPUT_DRAW* draw = &draws[drawCount++];
draw->Character = c;
draw->Foreground = glyphs[glyphIndex].Foreground;
draw->Background = glyphs[glyphIndex].Background;
draw->Column = drawCol;
draw->Row = drawRow;
draw->ScrollCount = scrollCount;
}
if (needScroll)
scrollCount++;
}
if (scrollCount != 0) {
uint64_t scrollPixels64 = (uint64_t)scrollCount * FONT_H;
int scrollPixels = scrollPixels64 >= mode.Height ? (int)mode.Height : (int)scrollPixels64;
if ((uint32_t)scrollPixels >= mode.Height) {
s = KiDisplayFillRectSnapshot(device, ops, &mode, 0, 0,
(int)mode.Width, (int)mode.Height, DISPLAY_DEFAULT_BG);
} else {
s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, scrollPixels,
(int)mode.Width, (int)(mode.Height - (uint32_t)scrollPixels));
if (KSUCCESS(s)) {
s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
(int)(mode.Height - (uint32_t)scrollPixels),
(int)mode.Width, scrollPixels, DISPLAY_DEFAULT_BG);
}
}
}
if (gConsoleY >= rows)
needScroll = true;
KeReleaseSpinlock(&gDisplayLock, flags);
if (drawChar) {
s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, c, drawCol * FONT_W, drawRow * FONT_H, fg, bg);
if (KSUCCESS(s)) {
flags = KeAcquireSpinlock(&gDisplayLock);
if (device == gPrimaryDisplay && drawCol >= 0 && drawCol < cols && drawRow >= 0 && drawRow < rows) {
size_t i = KiDisplayConsoleIndex(drawCol, drawRow);
gConsoleCells[i] = c;
gConsoleAttrs[i] = (uint8_t)gConsoleAttribute;
}
KeReleaseSpinlock(&gDisplayLock, flags);
}
} else {
s = NTLSTATUS_SUCCESS;
}
if (KSUCCESS(s) && needScroll) {
s = KiDisplayCopyRectSnapshot(device, ops, &mode, 0, 0, 0, FONT_H,
(int)mode.Width, (int)(mode.Height - FONT_H));
if (KSUCCESS(s)) {
s = KiDisplayFillRectSnapshot(device, ops, &mode, 0,
(int)(mode.Height - FONT_H),
(int)mode.Width, FONT_H, DISPLAY_DEFAULT_BG);
}
if (KSUCCESS(s)) {
flags = KeAcquireSpinlock(&gDisplayLock);
if (device == gPrimaryDisplay) {
KiDisplayConsoleScrollCellsLocked(cols, rows, 1);
gConsoleY = rows - 1;
if (gConsoleX < 0) gConsoleX = 0;
if (gConsoleX >= cols) gConsoleX = cols - 1;
}
KeReleaseSpinlock(&gDisplayLock, flags);
}
for (size_t i = 0; i < drawCount && KSUCCESS(s); i++) {
DISPLAY_OUTPUT_DRAW* draw = &draws[i];
size_t laterScrolls = scrollCount - draw->ScrollCount;
int finalRow = draw->Row - (int)laterScrolls;
if (finalRow < 0 || finalRow >= rows)
continue;
s = KiDisplayBlitGlyphSnapshot(device, ops, &mode, draw->Character,
draw->Column * FONT_W, finalRow * FONT_H,
draw->Foreground, draw->Background);
}
s = KiDisplayFinishProviderDraw(device, ops, caps, s);
@@ -1947,6 +2130,12 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
return s;
}
// Renders one console character through the shared batched console path.
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
DISPLAY_OUTPUT_GLYPH glyph = { c, fg, bg };
return KiDisplayPrintGlyphsInternal(&glyph, 1);
}
DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
return KiDisplayPrintCharInternal(c, fg, bg);
}
@@ -2285,10 +2474,114 @@ static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Blit operation.
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
// Copies one pixel row into the write-combined scanout aperture.
static void BootFbCopyToScanout(uint32_t* destination,
const uint32_t* source,
size_t pixelCount) {
if (!destination || !source || pixelCount == 0)
return;
if (!KeWriteCombiningAvailable()) {
memcpy(destination, source, pixelCount * sizeof(uint32_t));
return;
}
while (pixelCount != 0 && ((uintptr_t)destination & 15u) != 0) {
*destination++ = *source++;
pixelCount--;
}
while (pixelCount >= 16u) {
__m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
__m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
__m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
__m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
_mm_stream_si128((__m128i*)(destination + 0), v0);
_mm_stream_si128((__m128i*)(destination + 4), v1);
_mm_stream_si128((__m128i*)(destination + 8), v2);
_mm_stream_si128((__m128i*)(destination + 12), v3);
source += 16;
destination += 16;
pixelCount -= 16;
}
while (pixelCount >= 4u) {
__m128i value = _mm_loadu_si128((const __m128i*)source);
_mm_stream_si128((__m128i*)destination, value);
source += 4;
destination += 4;
pixelCount -= 4;
}
while (pixelCount != 0) {
*destination++ = *source++;
pixelCount--;
}
}
// Copies one source row into the retained shadow and scanout in one read pass.
static void BootFbCopyToShadowAndScanout(uint32_t* shadow,
uint32_t* scanout,
const uint32_t* source,
size_t pixelCount) {
if (!shadow || !scanout || !source || pixelCount == 0)
return;
if (!KeWriteCombiningAvailable()) {
memcpy(shadow, source, pixelCount * sizeof(uint32_t));
memcpy(scanout, source, pixelCount * sizeof(uint32_t));
return;
}
while (pixelCount != 0 && ((uintptr_t)scanout & 15u) != 0) {
uint32_t value = *source++;
*shadow++ = value;
*scanout++ = value;
pixelCount--;
}
while (pixelCount >= 16u) {
__m128i v0 = _mm_loadu_si128((const __m128i*)(source + 0));
__m128i v1 = _mm_loadu_si128((const __m128i*)(source + 4));
__m128i v2 = _mm_loadu_si128((const __m128i*)(source + 8));
__m128i v3 = _mm_loadu_si128((const __m128i*)(source + 12));
_mm_storeu_si128((__m128i*)(shadow + 0), v0);
_mm_storeu_si128((__m128i*)(shadow + 4), v1);
_mm_storeu_si128((__m128i*)(shadow + 8), v2);
_mm_storeu_si128((__m128i*)(shadow + 12), v3);
_mm_stream_si128((__m128i*)(scanout + 0), v0);
_mm_stream_si128((__m128i*)(scanout + 4), v1);
_mm_stream_si128((__m128i*)(scanout + 8), v2);
_mm_stream_si128((__m128i*)(scanout + 12), v3);
source += 16;
shadow += 16;
scanout += 16;
pixelCount -= 16;
}
while (pixelCount >= 4u) {
__m128i value = _mm_loadu_si128((const __m128i*)source);
_mm_storeu_si128((__m128i*)shadow, value);
_mm_stream_si128((__m128i*)scanout, value);
source += 4;
shadow += 4;
scanout += 4;
pixelCount -= 4;
}
while (pixelCount != 0) {
uint32_t value = *source++;
*shadow++ = value;
*scanout++ = value;
pixelCount--;
}
}
// Implements the boot framebuffer pitched blit operation.
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0)
if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
uint32_t* dstBase = fb->Backbuffer ? fb->Backbuffer : fb->Framebuffer;
@@ -2309,12 +2602,61 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co
for (int row = 0; row < endY - startY; row++) {
uint32_t* dst = dstBase + ((size_t)(startY + row) * fb->Stride) + startX;
const uint32_t* src = pixels + ((size_t)(srcOffsetY + row) * (size_t)w) + srcOffsetX;
const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
memcpy(dst, src, bytesPerRow);
}
return NTLSTATUS_SUCCESS;
}
// Presents one pitched source rectangle without a shadow-to-scanout second pass.
static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
int x,
int y,
int w,
int h,
const uint32_t* pixels,
uint32_t pitchBytes) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
int startX = x < 0 ? 0 : x;
int startY = y < 0 ? 0 : y;
int endX = x + w > (int)fb->Width ? (int)fb->Width : x + w;
int endY = y + h > (int)fb->Height ? (int)fb->Height : y + h;
if (startX >= endX || startY >= endY)
return NTLSTATUS_SUCCESS;
int srcOffsetX = startX - x;
int srcOffsetY = startY - y;
size_t pixelCount = (size_t)(endX - startX);
uint64_t bytes = (uint64_t)pixelCount * (uint64_t)(endY - startY) * sizeof(uint32_t);
fb->Stats.BlitCount++;
fb->Stats.PresentCount++;
fb->Stats.PresentRectCount++;
fb->Stats.DirectPresentCount++;
fb->Stats.ScanoutBytes += bytes;
fb->Stats.BytesCopied += fb->Backbuffer ? bytes * 2u : bytes;
for (int row = 0; row < endY - startY; row++) {
const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
uint32_t* scanout = fb->Framebuffer + ((size_t)(startY + row) * fb->Stride) + startX;
if (fb->Backbuffer) {
uint32_t* shadow = fb->Backbuffer + ((size_t)(startY + row) * fb->Stride) + startX;
BootFbCopyToShadowAndScanout(shadow, scanout, source, pixelCount);
} else {
BootFbCopyToScanout(scanout, source, pixelCount);
}
}
_mm_sfence();
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Fill Rect operation.
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
@@ -2425,14 +2767,18 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects,
DISPLAY_RECT r = rects[i];
if (!KiDisplayClipRect(&mode, &r))
continue;
fb->Stats.BytesCopied += (uint64_t)(uint32_t)r.W * (uint64_t)(uint32_t)r.H * sizeof(uint32_t);
size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t);
uint64_t bytes = (uint64_t)(uint32_t)r.W *
(uint64_t)(uint32_t)r.H * sizeof(uint32_t);
fb->Stats.BytesCopied += bytes;
fb->Stats.ScanoutBytes += bytes;
size_t pixelCount = (size_t)r.W;
for (int row = 0; row < r.H; row++) {
uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
memcpy(dst, src, bytesPerRow);
BootFbCopyToScanout(dst, src, pixelCount);
}
}
_mm_sfence();
return NTLSTATUS_SUCCESS;
}
@@ -2540,6 +2886,7 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
fb->Stats.CursorHideCount++;
BootFbRestoreCursor(fb);
}
_mm_sfence();
return NTLSTATUS_SUCCESS;
}
code/kernel/ke_dpc.c
+1 -1
View File
@@ -81,7 +81,7 @@ void KiDrainDpcQueue(void) {
if (oldIrql < DISPATCH_LEVEL)
KeRaiseIrql(DISPATCH_LEVEL);
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
LIST_ENTRY* node = KeListPopHeadChecked(&gDpcQueue);
if (!node) {
code/kernel/ke_exception.c
+1 -1
View File
@@ -60,7 +60,7 @@ static _Noreturn void KiTerminateCurrentExceptionProcess(KSTATUS exitStatus) {
if (thread)
KeExitThread(thread, exitCode);
for (;;) {
while (true) {
KeYield();
}
}
code/kernel/ke_input.c
+1 -1
View File
@@ -390,7 +390,7 @@ static KSTATUS KiInputWaitKey(KE_INPUT_CONTEXT* context, uint8_t alertable, uint
if (!context->WaitEvent)
return NTLSTATUS_FAILURE;
for (;;) {
while (true) {
EVENT* event = context->WaitEvent;
uint32_t selected = 0;
KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected);
code/kernel/ke_irp.c
+40 -13
View File
@@ -5,6 +5,7 @@
#include <kernel/ke_irp.h>
#include <kernel/ke_event.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_apc.h>
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
@@ -38,12 +39,13 @@ __declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
return CONTAINING_RECORD(node, IRP, Link);
}
__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context) {
// Cancels every queued IRP and invokes an optional ownership callback.
__declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context) {
if (!q) return;
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY* node = ListPopHead(&q->Head);
KeReleaseSpinlock(&q->Lock, flags);
@@ -60,7 +62,7 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
// Implements the irp Queue Cancel All operation.
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
IrpQueueCancelAllWithCallback(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
}
// Remove locks.
@@ -284,6 +286,14 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t
irp->Status = status;
irp->Information = information;
// Completion callbacks may recycle an embedded IRP, while a synchronous
// waiter may release stack storage as soon as its event is signaled.
// Snapshot every completion target before either ownership handoff.
KAPC* completionApc = (KAPC*)irp->CompletionApc;
EVENT* completionEvent = (EVENT*)irp->CompletionEvent;
IRP_COMPLETION_FN completionFn = irp->CompletionFn;
void* completionCtx = irp->CompletionCtx;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
if (dev) {
@@ -292,12 +302,18 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t
ios->DeviceObject = (DEVICE_OBJECT*)0;
}
if (irp->CompletionApc)
KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status);
if (irp->CompletionEvent)
KeSignalEvent((EVENT*)irp->CompletionEvent);
if (irp->CompletionFn)
irp->CompletionFn(irp, irp->CompletionCtx);
if (completionApc)
KeInsertQueueApc(completionApc, irp, (void*)(uintptr_t)status);
// The callback owns the final callback-based handoff. Do not dereference
// the IRP after it returns because embedded requests may be freed there.
if (completionFn)
completionFn(irp, completionCtx);
// The event is the final synchronous ownership handoff. A waiter may
// invalidate a stack IRP immediately after this signal becomes visible.
if (completionEvent)
KeSignalEvent(completionEvent);
return NTLSTATUS_SUCCESS;
}
@@ -314,6 +330,7 @@ __declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
// Submits irp Sync.
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
EVENT* ev = (EVENT*)0;
KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev);
@@ -321,12 +338,22 @@ __declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
irp->CompletionEvent = ev;
// Drivers may retain this IRP, its transfer buffer, and the caller's
// kernel stack until completion. Defer process termination across that
// ownership interval rather than letting teardown free live storage.
KeEnterTerminationDeferral();
st = IoSubmitIrp(device, irp);
if (st == NTLSTATUS_PENDING) {
KeWaitForEvent(ev, 0);
do {
st = KeWaitForEvent(ev, 0);
} while (st != NTLSTATUS_SUCCESS);
st = irp->Status;
}
// Completion events are signaled only after IoCompleteRequest has finished
// every operation that can dereference a synchronous caller's stack IRP.
irp->CompletionEvent = (EVENT*)0;
ObDereferenceObject(&ev->Header);
KeLeaveTerminationDeferral();
return st;
}
code/kernel/ke_irq.c
+3 -1
View File
@@ -126,7 +126,9 @@ void KiExitInterrupt(void) {
// high-IRQL region.
if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
THREAD* current = cpu->CurrentThread;
if (current && !current->IsIdleThread && current->Owner && KeIsProcessTerminating(current->Owner)) {
if (current && !current->IsIdleThread && current->Owner &&
KeIsProcessTerminating(current->Owner) &&
!KeThreadTerminationDeferred(current)) {
KeExitThread(current, current->Owner->ExitCode);
while (true) ;
}
code/kernel/ke_mutex.c
+253 -44
View File
@@ -1,4 +1,4 @@
// Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups.
// Kernel mutex primitive. This file owns exclusive ownership, abandonment, wait queues, and priority inheritance.
#include <stdbool.h>
#include <kernel/ke_list.h>
@@ -6,76 +6,285 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_debug.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
// Implements the mutex Destroy operation.
static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
// Reports an invalid mutex ownership transition.
static void KiMutexBug(const char* where, MUTEX* mutex, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"mutex invariant failed: %s: mutex=0x%016llX thread=0x%016llX tid=%u owner=0x%016llX refs=%ld waitEmpty=%u ownerLink=0x%016llX/0x%016llX threadWaiting=0x%016llX threadOwnedEmpty=%u: %s\n",
where,
(uint64_t)mutex,
(uint64_t)thread,
thread ? thread->ThreadId : 0,
mutex ? (uint64_t)mutex->Owner : 0,
mutex ? mutex->Header.RefCount : 0,
mutex ? (LIST_IS_EMPTY(&mutex->WaitList) ? 1u : 0u) : 1u,
mutex ? (uint64_t)mutex->OwnerLink.Fwd : 0,
mutex ? (uint64_t)mutex->OwnerLink.Bwd : 0,
thread ? (uint64_t)thread->WaitingMutex : 0,
thread ? (LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u) : 1u,
why ? why : "?");
}
// Implements the mutex Create operation.
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) {
// Stabilizes an owner pointer before it crosses the mutex lock boundary.
static void KiMutexReferenceOwnerLocked(MUTEX* mutex, THREAD* owner) {
if (!owner || !ObTryReferenceObject(&owner->Header))
KiMutexBug("KiMutexReferenceOwnerLocked", mutex, owner,
"live mutex owner could not be referenced");
}
// Tests whether one scheduler link is currently attached to a list.
static int KiMutexWaiterIsLinked(const THREAD* thread) {
if (!thread) return 0;
return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
thread->SchedulerLink.Bwd != &thread->SchedulerLink;
}
// Propagates one effective-rank change without carrying raw owners across locks.
static void KiMutexPropagatePriority(THREAD* thread) {
int releaseThread = 0;
for (uint32_t depth = 1; thread && depth <= KE_PRIORITY_INHERITANCE_MAX_DEPTH; depth++) {
MUTEX* waitingMutex = (MUTEX*)0;
int changed = KiSchedulerRefreshEffectivePriority(thread, &waitingMutex);
if (!changed) {
if (releaseThread) ObDereferenceObject(&thread->Header);
return;
}
KiSchedulerNotePriorityPropagation(depth);
if (!waitingMutex) {
if (releaseThread) ObDereferenceObject(&thread->Header);
return;
}
uintptr_t flags = KeAcquireSpinlock(&waitingMutex->Lock);
if (thread->WaitingMutex != waitingMutex ||
thread->WaitObject != &waitingMutex->Header ||
thread->WaitListHead != &waitingMutex->WaitList ||
!KiMutexWaiterIsLinked(thread)) {
KeReleaseSpinlock(&waitingMutex->Lock, flags);
ObDereferenceObject(&waitingMutex->Header);
if (releaseThread) ObDereferenceObject(&thread->Header);
return;
}
THREAD* owner = waitingMutex->Owner;
if (owner) KiMutexReferenceOwnerLocked(waitingMutex, owner);
int donationChanged = KiSchedulerRefreshMutexDonation(waitingMutex);
KeReleaseSpinlock(&waitingMutex->Lock, flags);
ObDereferenceObject(&waitingMutex->Header);
if (releaseThread) ObDereferenceObject(&thread->Header);
if (!donationChanged || !owner) {
if (owner) ObDereferenceObject(&owner->Header);
return;
}
thread = owner;
releaseThread = 1;
}
if (thread)
KiMutexBug("KiMutexPropagatePriority", thread->WaitingMutex, thread,
"priority inheritance chain exceeded the supported depth");
}
// Initializes one mutex's ownership bookkeeping.
static void KiMutexInitialize(MUTEX* mutex) {
mutex->Lock = (SPINLOCK)SPINLOCK_INIT;
mutex->Owner = (THREAD*)0;
LIST_INIT(&mutex->WaitList);
LIST_INIT(&mutex->OwnerLink);
mutex->DonationClass = KeSchedulingNormal;
mutex->DonationPriority = PriorityLow;
mutex->HasDonation = 0;
mutex->Abandoned = 0;
}
// Destroys one unowned mutex object.
static void KiMutexDestroy(OBJECT* object) {
MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
if (mutex->Owner || !LIST_IS_EMPTY(&mutex->WaitList) ||
mutex->OwnerLink.Fwd != &mutex->OwnerLink ||
mutex->OwnerLink.Bwd != &mutex->OwnerLink) {
KiMutexBug("KiMutexDestroy", mutex, mutex->Owner,
"owned or waited mutex reached object destruction");
}
}
// Assigns an unowned mutex to one thread and consumes abandonment state.
static KSTATUS KiMutexAcquireUnownedLocked(MUTEX* mutex, THREAD* thread) {
KSTATUS status = mutex->Abandoned ? NTLSTATUS_ABANDONED : NTLSTATUS_SUCCESS;
mutex->Abandoned = 0;
ObReferenceObject(&mutex->Header);
KiSchedulerAcquireMutexOwnership(thread, mutex);
return status;
}
// Creates a mutex object.
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out) {
if (!out) return NTLSTATUS_INVALID_PARAMETER;
MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
if (!m) return NTLSTATUS_NO_MEMORY;
m->Header.Destroy = KiMutexDestroy;
m->Lock = (SPINLOCK)SPINLOCK_INIT;
m->Owner = (struct THREAD *)0;
LIST_INIT(&m->WaitList);
*out = m;
MUTEX* mutex = (MUTEX*)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
if (!mutex) return NTLSTATUS_NO_MEMORY;
mutex->Header.Destroy = KiMutexDestroy;
KiMutexInitialize(mutex);
*out = mutex;
return NTLSTATUS_SUCCESS;
}
// Implements the acquire Mutex operation.
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) {
// Acquires a mutex within the requested timeout and donates priority while blocked.
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds) {
if (!mutex) return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (!self) return NTLSTATUS_FAILURE;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KSTATUS status = KiMutexAcquireUnownedLocked(mutex, self);
KeReleaseSpinlock(&mutex->Lock, flags);
return NTLSTATUS_SUCCESS;
return status;
}
if (mutex->Owner == self) {
KeReleaseSpinlock(&mutex->Lock, flags);
return NTLSTATUS_DEADLOCK;
}
if (timeoutMilliseconds == 0) {
KeReleaseSpinlock(&mutex->Lock, flags);
return NTLSTATUS_TIMEOUT;
}
THREAD* owner = mutex->Owner;
KiMutexReferenceOwnerLocked(mutex, owner);
KSTATUS waitStatus = KiSchedulerPrepareMutexWait(self, mutex, owner);
if (!KSUCCESS(waitStatus)) {
KeReleaseSpinlock(&mutex->Lock, flags);
ObDereferenceObject(&owner->Header);
return waitStatus;
}
KePrepareBlockThread(self, &mutex->Header, 0);
KeListInsertTailChecked(&mutex->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &mutex->WaitList);
if (timeoutMilliseconds != KE_WAIT_INFINITE) {
KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT);
if (!KSUCCESS(timeoutStatus)) {
KeListRemoveChecked(&self->SchedulerLink);
KiSchedulerCancelMutexWait(self, mutex);
(void)KiSchedulerRefreshMutexDonation(mutex);
KeUnblockThreadWithStatus(self, timeoutStatus);
KeReleaseSpinlock(&mutex->Lock, flags);
KiMutexPropagatePriority(owner);
ObDereferenceObject(&owner->Header);
return timeoutStatus;
}
}
(void)KiSchedulerRefreshMutexDonation(mutex);
KeReleaseSpinlock(&mutex->Lock, flags);
KiMutexPropagatePriority(owner);
ObDereferenceObject(&owner->Header);
return KeCommitBlockThread(self, 0);
}
// Implements the try Acquire Mutex operation.
__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return NTLSTATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KeReleaseSpinlock(&mutex->Lock, flags);
return NTLSTATUS_SUCCESS;
}
KeReleaseSpinlock(&mutex->Lock, flags);
return NTLSTATUS_FAILURE;
}
// Releases mutex.
// Releases a mutex and hands ownership to its highest-ranked waiter.
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
LIST_ENTRY* node = KeListPopHeadChecked(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
// Direct handoff: the selected waiter owns the mutex when its wait
// returns. Do not drop Owner to NULL, or another thread can steal it.
mutex->Owner = waiter;
KeReleaseSpinlock(&mutex->Lock, flags);
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!self || mutex->Owner != self)
KiMutexBug("KeReleaseMutex", mutex, self, "release by non-owner");
THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex);
if (waiter) {
KeListRemoveChecked(&waiter->SchedulerLink);
KiSchedulerTransferMutexOwnership(self, waiter, mutex);
(void)KiSchedulerRefreshMutexDonation(mutex);
(void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0);
KeUnblockThreadWithStatus(waiter, NTLSTATUS_SUCCESS);
KeReleaseSpinlock(&mutex->Lock, flags);
KiMutexPropagatePriority(self);
return;
}
mutex->Owner = (struct THREAD *)0;
KiSchedulerReleaseMutexOwnership(self, mutex);
(void)KiSchedulerRefreshMutexDonation(mutex);
KeReleaseSpinlock(&mutex->Lock, flags);
KiMutexPropagatePriority(self);
ObDereferenceObject(&mutex->Header);
}
// Removes one blocked mutex waiter during thread-wait rundown.
int KiMutexAbortWait(MUTEX* mutex, THREAD* thread) {
if (!mutex || !thread) return 0;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (thread->WaitingMutex != mutex ||
thread->WaitObject != &mutex->Header ||
thread->WaitListHead != &mutex->WaitList ||
!KiMutexWaiterIsLinked(thread)) {
KeReleaseSpinlock(&mutex->Lock, flags);
return 0;
}
KeListRemoveChecked(&thread->SchedulerLink);
KiSchedulerCancelMutexWait(thread, mutex);
THREAD* owner = mutex->Owner;
if (owner) KiMutexReferenceOwnerLocked(mutex, owner);
(void)KiSchedulerRefreshMutexDonation(mutex);
KeReleaseSpinlock(&mutex->Lock, flags);
KiMutexPropagatePriority(owner);
if (owner) ObDereferenceObject(&owner->Header);
KiMutexPropagatePriority(thread);
return 1;
}
// Abandons one mutex owned by a terminating thread.
static void KiMutexAbandonOwned(MUTEX* mutex, THREAD* owner) {
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (mutex->Owner != owner)
KiMutexBug("KiMutexAbandonOwned", mutex, owner, "ownership list mismatch");
THREAD* waiter = KiSchedulerSelectMutexWaiter(mutex);
if (waiter) {
KeListRemoveChecked(&waiter->SchedulerLink);
KiSchedulerTransferMutexOwnership(owner, waiter, mutex);
mutex->Abandoned = 0;
(void)KiSchedulerRefreshMutexDonation(mutex);
(void)KiSchedulerRefreshEffectivePriority(waiter, (MUTEX**)0);
KeUnblockThreadWithStatus(waiter, NTLSTATUS_ABANDONED);
KeReleaseSpinlock(&mutex->Lock, flags);
KiMutexPropagatePriority(owner);
return;
}
KiSchedulerReleaseMutexOwnership(owner, mutex);
mutex->Abandoned = 1;
(void)KiSchedulerRefreshMutexDonation(mutex);
KeReleaseSpinlock(&mutex->Lock, flags);
KiMutexPropagatePriority(owner);
ObDereferenceObject(&mutex->Header);
}
// Releases every mutex still owned by a terminating thread.
void KiMutexRundownThread(THREAD* thread) {
if (!thread) return;
while (true) {
MUTEX* mutex = KiSchedulerFirstOwnedMutex(thread);
if (!mutex)
return;
KiMutexAbandonOwned(mutex, thread);
}
}
code/kernel/ke_process.c
+55 -25
View File
@@ -7,6 +7,7 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_event.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_system_root.h>
#include <kernel/ke_userenv.h>
#include <kernel/io_file.h>
@@ -14,6 +15,7 @@
#include <kernel/ob_object.h>
#include <kernel/ke_named_queue.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_audio.h>
#include <kernel/hal/hal_apic.h>
#include <ntlite/syscall.h>
@@ -217,6 +219,7 @@ static uint32_t KiThreadStateToSys(THREAD_STATE state) {
case ThreadRunning: return SYS_THREAD_STATE_RUNNING;
case ThreadWaiting: return SYS_THREAD_STATE_WAITING;
case ThreadDead: return SYS_THREAD_STATE_DEAD;
case ThreadSuspended: return SYS_THREAD_STATE_SUSPENDED;
default: return SYS_THREAD_STATE_DEAD;
}
}
@@ -265,25 +268,33 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
SYS_THREAD_INFO* info = &infoOut[count];
memset(info, 0, sizeof(*info));
info->Size = sizeof(*info);
KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
KiSchedulerSnapshotThread(thread, &snapshot);
info->ProcessId = process->ProcessId;
info->ThreadId = thread->ThreadId;
info->State = KiThreadStateToSys(thread->State);
info->Priority = (uint32_t)thread->Priority;
info->RunningCpu = thread->RunningCpu;
info->LastCpu = thread->LastCpu;
info->QuantumRemaining = thread->QuantumRemaining;
info->QuantumTicks = thread->QuantumTicks;
info->CpuBurstTicks = thread->CpuBurstTicks;
info->LastRunTicks = thread->LastRunTicks;
info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE;
info->WaitStatus = (uint32_t)thread->WaitStatus;
info->State = KiThreadStateToSys(snapshot.State);
info->BaseSchedulingClass = (uint32_t)snapshot.BaseSchedulingClass;
info->BasePriority = snapshot.BasePriority;
info->EffectiveSchedulingClass = (uint32_t)snapshot.EffectiveSchedulingClass;
info->EffectivePriority = snapshot.EffectivePriority;
info->RunningCpu = snapshot.RunningCpu;
info->LastCpu = snapshot.LastCpu;
info->QuantumRemaining = snapshot.QuantumRemaining;
info->QuantumTicks = snapshot.QuantumTicks;
info->CpuBurstTicks = snapshot.CpuBurstTicks;
info->LastRunTicks = snapshot.LastRunTicks;
info->WaitObjectType = snapshot.WaitObjectType != 0
? KiWaitObjectTypeToSys((OBJECT_TYPE)snapshot.WaitObjectType)
: SYS_THREAD_WAIT_NONE;
info->WaitStatus = (uint32_t)snapshot.WaitStatus;
info->KernelApcDisable = thread->KernelApcDisable;
if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT;
if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE;
if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL;
if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED;
if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT;
if (snapshot.Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED;
if (snapshot.AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT;
if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING;
if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING;
}
@@ -296,10 +307,26 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
return NTLSTATUS_SUCCESS;
}
// Releases all kernel objects still reachable from a process handle table.
// This is part of process exit, not merely object destruction: a parent may keep
// a handle to an exited process for wait/query, but the dead process must not
// keep its own files, mappings, queues, sockets, or synchronization objects alive.
// Records type-specific ownership after a handle slot becomes visible.
static void KiProcessHandleOpened(OBJECT* object) {
if (!object)
return;
if (object->Type == ObjectTypeAudioStream)
KeAudioStreamHandleOpened((KE_AUDIO_STREAM*)object);
}
// Releases type-specific ownership before the handle-table reference drops.
static void KiProcessReleaseHandleObject(OBJECT* object) {
if (!object)
return;
if (object->Type == ObjectTypeAudioStream)
KeAudioStreamHandleClosed((KE_AUDIO_STREAM*)object);
ObDereferenceObject(object);
}
// Releases every handle-table reference owned by one process.
static void KiProcessReleaseAllHandles(PROCESS* proc) {
if (!proc) return;
@@ -317,7 +344,7 @@ static void KiProcessReleaseAllHandles(PROCESS* proc) {
if (obj) {
KeNamedQueueUnpublishHandle(proc, (uint32_t)i, obj);
ObDereferenceObject(obj);
KiProcessReleaseHandleObject(obj);
}
}
}
@@ -671,14 +698,16 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
if (t == self)
continue;
// A terminating process must not be held open by worker threads that
// are blocked in non-alertable waits. User-mode DLLs such as Bink can
// leave IO/submit workers parked on semaphores; after the foreground
// thread crashes, those waiters would otherwise keep ThreadCount > 0
// and prevent the process ExitEvent from waking the shell.
if (t->State == ThreadWaiting)
// Most user waits must be interrupted immediately. A thread can defer
// termination only while synchronous kernel work still owns its stack
// or process memory; that thread is allowed to finish the ownership
// interval before the normal termination gates reap it.
int deferred = KeThreadTerminationDeferred(t);
if (t->State == ThreadSuspended && !deferred)
(void)KeResumeThread(t, (uint32_t*)0);
else if (t->State == ThreadWaiting && !deferred)
KeAbortThreadWait(t, NTLSTATUS_ALERTED);
else
else if (!deferred)
(void)KeAlertThread(t);
if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
@@ -727,6 +756,7 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri
process->Handles[i].Object = object;
process->Handles[i].Rights = rights;
process->Handles[i].Flags = 0;
KiProcessHandleOpened(object);
*handleOut = (uint32_t)i;
KeReleaseSpinlock(&process->HandleLock, flags);
return NTLSTATUS_SUCCESS;
@@ -756,7 +786,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) {
if (obj->Type == ObjectTypeFile)
(void)IoFlushFile((FILE_OBJECT*)obj);
KeNamedQueueUnpublishHandle(process, handle, obj);
ObDereferenceObject(obj);
KiProcessReleaseHandleObject(obj);
return;
}
KeReleaseSpinlock(&process->HandleLock, flags);
code/kernel/ke_queue.c
+2 -2
View File
@@ -133,7 +133,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMi
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count < QUEUE_DEPTH) {
uint32_t previousCount = queue->Count;
@@ -165,7 +165,7 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilli
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count > 0) {
uint32_t previousCount = queue->Count;
code/kernel/ke_sched.c
+726 -98
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_selftest.c
+591 -2
View File
@@ -11,11 +11,14 @@
#include <kernel/ke_event.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_sched.h>
#include <ntlite/status.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/io_dma.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
@@ -240,9 +243,10 @@ static void KiRunKernelApcSelfTest(void) {
s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: FinishedEvent create failed %d\n", s);
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker,
(uintptr_t)&st,
PriorityNormal,
&workerParameters,
&st.WorkerThread);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker create failed %d\n", s);
@@ -310,9 +314,10 @@ static void KiRunKernelApcSelfTest(void) {
s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert FinishedEvent create failed %d\n", s);
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker,
(uintptr_t)&st,
PriorityNormal,
&workerParameters,
&st.WorkerThread);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable worker create failed %d\n", s);
@@ -584,6 +589,213 @@ static void KiRunPfnDatabaseSelfTest(void) {
DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
}
// Verifies common-buffer, direct mapping, bounce mapping, and teardown accounting.
static void KiRunDmaSelfTest(void) {
const NTL_DDK_DMA* dma = IoGetDmaDdk();
if (!dma)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: DDK table missing\n");
IO_DMA_STATS before;
IO_DMA_STATS after;
dma->QueryStats(&before);
DRIVER_OBJECT driver;
memset(&driver, 0, sizeof(driver));
IoInitializeDriverObject(&driver, SYS_SV_LITERAL("dma-selftest"));
DEVICE_OBJECT* device = (DEVICE_OBJECT*)0;
KSTATUS status = IoCreateDevice(&driver, 0, &device);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: device creation failed status=%u\n", status);
IO_DMA_REQUIREMENTS requirements;
memset(&requirements, 0, sizeof(requirements));
requirements.MaximumDeviceAddress = UINT64_MAX;
requirements.Alignment = PAGE_SIZE;
requirements.MaximumSegmentLength = PAGE_SIZE;
requirements.MaximumSegments = 4;
IO_DMA_ADAPTER* adapter = (IO_DMA_ADAPTER*)0;
IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
MM_PINNED_MEMORY* pinned = (MM_PINNED_MEMORY*)0;
IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)0;
status = dma->GetAdapter(device, &requirements, &adapter);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: adapter creation failed status=%u\n", status);
status = dma->AllocateCommonBuffer(adapter, PAGE_SIZE, PAGE_SIZE, &buffer);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: common buffer allocation failed status=%u\n", status);
uint8_t* bytes = (uint8_t*)dma->GetCommonBufferAddress(buffer);
uint64_t deviceAddress = dma->GetCommonBufferDeviceAddress(buffer);
if (!bytes || !deviceAddress || (deviceAddress & (PAGE_SIZE - 1)) != 0 ||
dma->GetCommonBufferSize(buffer) != PAGE_SIZE)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: common buffer metadata invalid address=0x%016llX\n",
(unsigned long long)deviceAddress);
status = dma->PinKernelMemory(bytes, PAGE_SIZE, IoDmaBidirectional, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: kernel pin failed status=%u\n", status);
status = dma->MapPinnedMemory(adapter,
pinned,
0,
PAGE_SIZE,
IoDmaBidirectional,
IO_DMA_MAP_NO_BOUNCE,
&mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
dma->GetSegments(mapping)[0].DeviceAddress != deviceAddress)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: direct mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
for (size_t i = 0; i < 128; i++)
bytes[i + 1] = (uint8_t)(i ^ 0x5Au);
status = dma->PinKernelMemory(bytes + 1, 128, IoDmaToDevice, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: unaligned kernel pin failed status=%u\n", status);
status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaToDevice, 0, &mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
(dma->GetSegments(mapping)[0].DeviceAddress & (PAGE_SIZE - 1)) != 0 ||
dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: bounce mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
status = dma->SynchronizeForDevice(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: bounce synchronization failed status=%u\n", status);
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
memset(bytes + 1, 0, 128);
status = dma->PinKernelMemory(bytes + 1, 128, IoDmaFromDevice, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive pin failed status=%u\n", status);
status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaFromDevice, 0, &mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: receive bounce mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
status = dma->SynchronizeForDevice(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive handoff failed status=%u\n", status);
uint64_t receiveDeviceAddress = dma->GetSegments(mapping)[0].DeviceAddress;
uint8_t* receiveDeviceAlias = (uint8_t*)MmPfnToVirtual(receiveDeviceAddress & PAGE_MASK);
if (!receiveDeviceAlias)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive device alias missing\n");
receiveDeviceAlias += receiveDeviceAddress & (PAGE_SIZE - 1);
for (size_t i = 0; i < 128; i++)
receiveDeviceAlias[i] = (uint8_t)(i ^ 0xA5u);
status = dma->SynchronizeForCpu(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive synchronization failed status=%u\n", status);
for (size_t i = 0; i < 128; i++) {
if (bytes[i + 1] != (uint8_t)(i ^ 0xA5u))
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: receive bounce copy mismatch offset=%llu\n",
(unsigned long long)i);
}
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: no process for user pinning\n");
void* userBase = (void*)0;
status = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&userBase);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD creation failed status=%u\n", status);
status = MmVadEnsureResident(process, userBase, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD materialization failed status=%u\n", status);
((uint8_t*)userBase)[0] = 0xD5u;
status = dma->PinUserMemory(process, userBase, PAGE_SIZE, IoDmaToDevice, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user pin failed status=%u\n", status);
status = MmVadDestroy(process, userBase);
if (status != NTLSTATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: pinned user VAD was destroyable status=%u\n",
status);
status = dma->MapPinnedMemory(adapter,
pinned,
0,
PAGE_SIZE,
IoDmaToDevice,
IO_DMA_MAP_NO_BOUNCE,
&mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: user mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
status = dma->SynchronizeForDevice(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user synchronization failed status=%u\n", status);
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
status = MmVadDestroy(process, userBase);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD destroy failed status=%u\n", status);
dma->FreeCommonBuffer(buffer);
dma->PutAdapter(adapter);
IoDeleteDevice(device);
dma->QueryStats(&after);
if (after.AdaptersCreated != before.AdaptersCreated + 1 ||
after.AdaptersDestroyed != before.AdaptersDestroyed + 1 ||
after.AdaptersInUse != before.AdaptersInUse ||
after.CommonBuffersAllocated < before.CommonBuffersAllocated + 3 ||
after.CommonBuffersFreed < before.CommonBuffersFreed + 3 ||
after.CommonBuffersInUse != before.CommonBuffersInUse ||
after.CommonBufferBytesInUse != before.CommonBufferBytesInUse ||
after.PinnedRangesInUse != before.PinnedRangesInUse ||
after.PinnedPagesInUse != before.PinnedPagesInUse ||
after.PinnedKernelRanges != before.PinnedKernelRanges + 3 ||
after.PinnedUserRanges != before.PinnedUserRanges + 1 ||
after.PinnedRangesReleased != before.PinnedRangesReleased + 4 ||
after.StreamingMappings != before.StreamingMappings + 4 ||
after.StreamingMappingsUnmapped != before.StreamingMappingsUnmapped + 4 ||
after.StreamingMappingsInUse != before.StreamingMappingsInUse ||
after.DirectMappings != before.DirectMappings + 2 ||
after.BounceMappings != before.BounceMappings + 2 ||
after.BounceBytesCopiedToDevice < before.BounceBytesCopiedToDevice + 128 ||
after.BounceBytesCopiedToCpu < before.BounceBytesCopiedToCpu + 128)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: accounting mismatch\n");
DbgPrint("DMA selftest: common/kernel-pin/user-pin/direct/send-bounce/receive-bounce/teardown PASS\n");
}
// Verifies the page-table manager invariants that the higher VMM will depend on.
static void KiRunMemoryMappingSelfTest(void) {
PHYSICAL_ADDRESS phys = 0;
@@ -1155,6 +1367,380 @@ static void KiRunAsyncFileIoSelfTest(void) {
DbgPrint("FILE/ASYNC selftest: PASS\n");
}
typedef struct KI_SCHEDULER_SELFTEST_STATE {
MUTEX* GateMutex;
MUTEX* LinkMutex;
EVENT* ChainReadyEvent;
EVENT* RealtimeReadyEvent;
EVENT* ChainFinishedEvent;
EVENT* RealtimeFinishedEvent;
volatile KSTATUS ChainAcquireStatus;
volatile KSTATUS RealtimeAcquireStatus;
} KI_SCHEDULER_SELFTEST_STATE;
// Waits until one worker has entered the expected mutex wait state.
static KE_THREAD_SCHEDULER_SNAPSHOT KiSchedulerAwaitMutexWait(THREAD* thread, MUTEX* mutex) {
KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
memset(&snapshot, 0, sizeof(snapshot));
for (uint32_t attempt = 0; attempt < 100; attempt++) {
KiSchedulerSnapshotThread(thread, &snapshot);
if (snapshot.State == ThreadWaiting && snapshot.WaitingMutex == mutex)
return snapshot;
KSTATUS status = KeDelayThread(SCHED_TICK_MS);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: state wait failed status=%u\n", status);
}
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: thread %u did not block on mutex=0x%016llX state=%u waiting=0x%016llX\n",
thread ? thread->ThreadId : 0,
(uint64_t)mutex,
(unsigned)snapshot.State,
(uint64_t)snapshot.WaitingMutex);
}
typedef struct KI_SYNC_TIMEOUT_SELFTEST_STATE {
MUTEX* Mutex;
EVENT* ReadyEvent;
EVENT* FinishedEvent;
volatile KSTATUS MutexStatus;
} KI_SYNC_TIMEOUT_SELFTEST_STATE;
// Waits on the held mutex long enough for the timeout engine to remove donation state.
static void KiSynchronizationTimeoutWorker(void* argument) {
KI_SYNC_TIMEOUT_SELFTEST_STATE* state = (KI_SYNC_TIMEOUT_SELFTEST_STATE*)argument;
KeSignalEvent(state->ReadyEvent);
state->MutexStatus = KeAcquireMutex(state->Mutex, SCHED_TICK_MS * 20ULL);
if (KE_MUTEX_ACQUIRE_SUCCEEDED(state->MutexStatus))
KeReleaseMutex(state->Mutex);
KeSignalEvent(state->FinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
// Verifies finite mutex and semaphore waits consume no object state after timeout.
static void KiRunSynchronizationTimeoutSelfTest(void) {
DbgPrint("SYNC/TIMEOUT selftest: begin\n");
KI_SYNC_TIMEOUT_SELFTEST_STATE state;
memset(&state, 0, sizeof(state));
state.MutexStatus = NTLSTATUS_PENDING;
KSTATUS status = KeMutexCreate(&state.Mutex);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: mutex create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.ReadyEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: ready event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.FinishedEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finished event create failed status=%u\n", status);
status = KeAcquireMutex(state.Mutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: owner acquire failed status=%u\n", status);
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_REALTIME(12);
status = KeCreateKernelThread((uintptr_t)KiSynchronizationTimeoutWorker,
(uintptr_t)&state,
&parameters,
&worker);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: worker create failed status=%u\n", status);
status = KeWaitForEvent(state.ReadyEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: ready wait failed status=%u\n", status);
(void)KiSchedulerAwaitMutexWait(worker, state.Mutex);
THREAD* self = KeGetCurrentThread();
KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
KiSchedulerSnapshotThread(self, &snapshot);
if (snapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
snapshot.EffectivePriority != 12) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finite waiter donation missing class=%u priority=%u\n",
snapshot.EffectiveSchedulingClass,
snapshot.EffectivePriority);
}
status = KeWaitForEvent(state.FinishedEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finished wait failed status=%u\n", status);
if (state.MutexStatus != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: mutex wait returned status=%u\n",
state.MutexStatus);
KiSchedulerSnapshotThread(self, &snapshot);
if (snapshot.EffectiveSchedulingClass != snapshot.BaseSchedulingClass ||
snapshot.EffectivePriority != snapshot.BasePriority) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: timed-out donation remained base=%u/%u effective=%u/%u\n",
snapshot.BaseSchedulingClass,
snapshot.BasePriority,
snapshot.EffectiveSchedulingClass,
snapshot.EffectivePriority);
}
KeReleaseMutex(state.Mutex);
status = KeAcquireMutex(state.Mutex, 0);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: zero-timeout mutex acquire failed status=%u\n", status);
KeReleaseMutex(state.Mutex);
SEMAPHORE* semaphore = (SEMAPHORE*)0;
status = KeSemaphoreCreate(0, 1, &semaphore);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: semaphore create failed status=%u\n", status);
status = KeWaitSemaphore(semaphore, 0);
if (status != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: zero-timeout semaphore wait returned status=%u\n", status);
status = KeWaitSemaphore(semaphore, SCHED_TICK_MS * 3ULL);
if (status != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finite semaphore wait returned status=%u\n", status);
KeSignalSemaphore(semaphore);
status = KeWaitSemaphore(semaphore, 0);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: signaled semaphore probe failed status=%u\n", status);
status = KeWaitSemaphore(semaphore, 0);
if (status != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: semaphore count was consumed twice status=%u\n", status);
ObDereferenceObject(&worker->Header);
ObDereferenceObject(&semaphore->Header);
ObDereferenceObject(&state.Mutex->Header);
ObDereferenceObject(&state.ReadyEvent->Header);
ObDereferenceObject(&state.FinishedEvent->Header);
DbgPrint("SYNC/TIMEOUT selftest: PASS\n");
}
// Owns the link mutex and then blocks behind the self-test thread.
static void KiSchedulerChainWorker(void* argument) {
KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;
KSTATUS status = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain link acquire failed status=%u\n", status);
KeSignalEvent(state->ChainReadyEvent);
state->ChainAcquireStatus = KeAcquireMutex(state->GateMutex, KE_WAIT_INFINITE);
if (state->ChainAcquireStatus != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain gate acquire failed status=%u\n",
state->ChainAcquireStatus);
KeReleaseMutex(state->GateMutex);
KeReleaseMutex(state->LinkMutex);
KeSignalEvent(state->ChainFinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
// Blocks a real-time thread behind the normal chain worker.
static void KiSchedulerRealtimeWorker(void* argument) {
KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;
KeSignalEvent(state->RealtimeReadyEvent);
state->RealtimeAcquireStatus = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
if (state->RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time acquire failed status=%u\n",
state->RealtimeAcquireStatus);
KeReleaseMutex(state->LinkMutex);
KeSignalEvent(state->RealtimeFinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
// Verifies real-time dispatch and transitive mutex priority inheritance.
static void KiRunRealtimeSchedulerSelfTest(void) {
DbgPrint("SCHED selftest: begin\n");
THREAD* self = KeGetCurrentThread();
if (!self || self->BaseSchedulingClass != KeSchedulingNormal)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: caller is not a normal thread\n");
KE_SCHED_STATS before;
KE_SCHED_STATS after;
KeGetSchedulerStats(&before);
KI_SCHEDULER_SELFTEST_STATE state;
memset(&state, 0, sizeof(state));
state.ChainAcquireStatus = NTLSTATUS_PENDING;
state.RealtimeAcquireStatus = NTLSTATUS_PENDING;
KSTATUS status = KeMutexCreate(&state.GateMutex);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: gate mutex create failed status=%u\n", status);
status = KeMutexCreate(&state.LinkMutex);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: link mutex create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.ChainReadyEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain ready event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.RealtimeReadyEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time ready event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.ChainFinishedEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain finished event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.RealtimeFinishedEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time finished event create failed status=%u\n", status);
status = KeAcquireMutex(state.GateMutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: owner acquire failed status=%u\n", status);
THREAD* chainWorker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS chainParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
status = KeCreateKernelThread((uintptr_t)KiSchedulerChainWorker,
(uintptr_t)&state,
&chainParameters,
&chainWorker);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain worker create failed status=%u\n", status);
status = KeWaitForEvent(state.ChainReadyEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain ready wait failed status=%u\n", status);
(void)KiSchedulerAwaitMutexWait(chainWorker, state.GateMutex);
status = KeAcquireMutex(state.LinkMutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_DEADLOCK)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: mutex cycle was not rejected status=%u\n", status);
THREAD* realtimeWorker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS realtimeParameters = KE_THREAD_CREATE_REALTIME(16);
status = KeCreateKernelThread((uintptr_t)KiSchedulerRealtimeWorker,
(uintptr_t)&state,
&realtimeParameters,
&realtimeWorker);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time worker create failed status=%u\n", status);
status = KeWaitForEvent(state.RealtimeReadyEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time ready wait failed status=%u\n", status);
KE_THREAD_SCHEDULER_SNAPSHOT realtimeSnapshot =
KiSchedulerAwaitMutexWait(realtimeWorker, state.LinkMutex);
(void)realtimeSnapshot;
KE_THREAD_SCHEDULER_SNAPSHOT chainSnapshot;
KE_THREAD_SCHEDULER_SNAPSHOT selfSnapshot;
KiSchedulerSnapshotThread(chainWorker, &chainSnapshot);
KiSchedulerSnapshotThread(self, &selfSnapshot);
if (chainSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
chainSnapshot.EffectivePriority != 16)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: intermediate donation missing class=%u priority=%u\n",
chainSnapshot.EffectiveSchedulingClass,
chainSnapshot.EffectivePriority);
if (selfSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
selfSnapshot.EffectivePriority != 16)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: transitive donation missing class=%u priority=%u\n",
selfSnapshot.EffectiveSchedulingClass,
selfSnapshot.EffectivePriority);
KeReleaseMutex(state.GateMutex);
KiSchedulerSnapshotThread(self, &selfSnapshot);
if (selfSnapshot.EffectiveSchedulingClass != selfSnapshot.BaseSchedulingClass ||
selfSnapshot.EffectivePriority != selfSnapshot.BasePriority)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: inherited priority was not restored base=%u/%u effective=%u/%u\n",
selfSnapshot.BaseSchedulingClass,
selfSnapshot.BasePriority,
selfSnapshot.EffectiveSchedulingClass,
selfSnapshot.EffectivePriority);
status = KeWaitForEvent(state.ChainFinishedEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain finished wait failed status=%u\n", status);
status = KeWaitForEvent(state.RealtimeFinishedEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time finished wait failed status=%u\n", status);
if (state.ChainAcquireStatus != NTLSTATUS_SUCCESS ||
state.RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: handoff statuses chain=%u realtime=%u\n",
state.ChainAcquireStatus,
state.RealtimeAcquireStatus);
KeGetSchedulerStats(&after);
if (after.RealtimeDispatches <= before.RealtimeDispatches ||
after.PriorityInheritanceBoosts < before.PriorityInheritanceBoosts + 2 ||
after.PriorityInheritanceDrops < before.PriorityInheritanceDrops + 2 ||
after.PriorityInheritancePropagations < before.PriorityInheritancePropagations + 2 ||
after.PriorityInheritanceMaxDepth < 2)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: scheduler counters did not prove transitive inheritance\n");
ObDereferenceObject(&chainWorker->Header);
ObDereferenceObject(&realtimeWorker->Header);
ObDereferenceObject(&state.GateMutex->Header);
ObDereferenceObject(&state.LinkMutex->Header);
ObDereferenceObject(&state.ChainReadyEvent->Header);
ObDereferenceObject(&state.RealtimeReadyEvent->Header);
ObDereferenceObject(&state.ChainFinishedEvent->Header);
ObDereferenceObject(&state.RealtimeFinishedEvent->Header);
DbgPrint("SCHED selftest: PASS\n");
}
// Runs trap Frame Self Test.
static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: begin\n");
@@ -1251,8 +1837,11 @@ void KiRunEarlyKernelSelfTests(void) {
// Runs kernel Self Tests.
void KiRunKernelSelfTests(const HAL_API* hal) {
DbgPrint("Kernel selftests: begin\n");
KiRunSynchronizationTimeoutSelfTest();
KiRunRealtimeSchedulerSelfTest();
KiRunKernelApcSelfTest();
KiRunPfnDatabaseSelfTest();
KiRunDmaSelfTest();
KiRunMemoryMappingSelfTest();
KiRunVadSparseSelfTest();
KiRunVadCopyOnWriteSelfTest();
code/kernel/ke_semaphore.c
+18 -16
View File
@@ -26,10 +26,10 @@ __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SE
return NTLSTATUS_SUCCESS;
}
// Waits for semaphore.
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
// Waits for one semaphore count within the requested timeout.
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem, uint64_t timeoutMilliseconds) {
if (!sem) return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
@@ -39,24 +39,26 @@ __declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
return NTLSTATUS_SUCCESS;
}
if (timeoutMilliseconds == 0) {
KeReleaseSpinlock(&sem->Lock, flags);
return NTLSTATUS_TIMEOUT;
}
KePrepareBlockThread(self, &sem->Header, 0);
KeListInsertTailChecked(&sem->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &sem->WaitList);
KeReleaseSpinlock(&sem->Lock, flags);
return KeCommitBlockThread(self, 0);
}
// Implements the try Semaphore operation.
__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
if (!sem) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (sem->Count > 0) {
sem->Count--;
KeReleaseSpinlock(&sem->Lock, flags);
return NTLSTATUS_SUCCESS;
if (timeoutMilliseconds != KE_WAIT_INFINITE) {
KSTATUS timeoutStatus = KeArmCurrentThreadTimeout(timeoutMilliseconds, NTLSTATUS_TIMEOUT);
if (!KSUCCESS(timeoutStatus)) {
KeListRemoveChecked(&self->SchedulerLink);
KeSetThreadWaitList(self, (LIST_ENTRY*)0);
KeUnblockThreadWithStatus(self, timeoutStatus);
KeReleaseSpinlock(&sem->Lock, flags);
return timeoutStatus;
}
}
KeReleaseSpinlock(&sem->Lock, flags);
return NTLSTATUS_FAILURE;
return KeCommitBlockThread(self, 0);
}
// Signals semaphore.
code/kernel/ke_smp.c
+9 -6
View File
@@ -289,6 +289,8 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
if ((cr4 & AP_CR4_REQUIRED) != AP_CR4_REQUIRED)
__writecr4(cr4 | AP_CR4_REQUIRED);
KeInitializeMemoryTypes();
if (gApBootBlock) {
gApBootBlock->Stage = AP_STAGE_ENTRY;
gApBootBlock->State = AP_STATE_KERNEL_ENTRY;
@@ -320,7 +322,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
gApBootBlock->State = AP_STATE_KERNEL_PARKED;
}
for (;;) {
while (true) {
cpu->SchedulerTicks++;
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
@@ -980,11 +982,12 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
cpuIndex, st);
}
st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker,
(uintptr_t)arg,
PriorityNormal,
(1ULL << cpuIndex),
(THREAD**)0);
KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
workerParameters.AffinityMask = 1ULL << cpuIndex;
st = KeCreateKernelThread((uintptr_t)KiSmpApControlledWorker,
(uintptr_t)arg,
&workerParameters,
(THREAD**)0);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u worker create failed status=0x%x\n",
code/kernel/ke_switch.asm
+10 -8
View File
@@ -2,16 +2,16 @@
;
; Struct offsets:
; sizeof(OBJECT) = 24 (type:4, refcount:4, state:4, flags:4, destroy:8)
; THREAD.SavedContext offset = 96
; THREAD.TssRsp0 offset = 112
; THREAD.Teb offset = 120
; THREAD.FpuContext offset = 176
; THREAD.SavedContext offset = 112
; THREAD.TssRsp0 offset = 128
; THREAD.Teb offset = 136
; THREAD.FpuContext offset = 192
; TSS64.RSP0 offset = 4
;
THREAD_SavedContext EQU 96
THREAD_TssRsp0 EQU 112
THREAD_Teb EQU 120
THREAD_FpuContext EQU 176
THREAD_SavedContext EQU 112
THREAD_TssRsp0 EQU 128
THREAD_Teb EQU 136
THREAD_FpuContext EQU 192
TSS_RSP0_OFFSET EQU 4
EXTERN KeSetCurrentTssRsp0 : PROC
@@ -131,12 +131,14 @@ KeStartInitialThread ENDP
; We load RCX from the stack (first arg per MS x64), then IRETQ.
; -------------------------------------------------------------------------
EXTERN KeSchedulerPostSwitch : PROC
EXTERN KePrepareUserThreadStart : PROC
EXTERN KeGetCurrentThread : PROC
EXTERN KeExitThread : PROC
KeThreadEntry PROC
sub rsp, 40 ; Reserve shadow space and restore MS x64 call alignment.
call KeSchedulerPostSwitch
call KePrepareUserThreadStart
add rsp, 40
pop rcx ; load user arg
swapgs ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU
code/kernel/ke_syscall.c
+596 -122
View File
@@ -71,7 +71,9 @@
#include <kernel/ke_trap.h>
#include <kernel/ke_input.h>
#include <kernel/ke_display.h>
#include <kernel/ke_audio.h>
#include <kernel/ke_time.h>
#include <kernel/mm_pool.h>
#include <ntlite/syscall.h>
// HAL pointer - set once at boot by KeSyscallSetHal().
@@ -182,7 +184,8 @@ typedef struct SURFACE_OBJECT {
uint64_t PresentBytesCopied;
PHYSICAL_ADDRESS* Frames;
void* UserBase;
uint32_t* RowScratch;
MM_KERNEL_VIEW* KernelView;
void* KernelBase;
MUTEX* Lock;
} SURFACE_OBJECT;
@@ -274,6 +277,12 @@ static void KiSurfaceDestroy(OBJECT* obj) {
surface->UserBase = (void*)0;
}
if (surface->KernelView) {
MmDestroyKernelView(surface->KernelView);
surface->KernelView = (MM_KERNEL_VIEW*)0;
surface->KernelBase = (void*)0;
}
if (surface->Lock) {
ObDereferenceObject(&surface->Lock->Header);
surface->Lock = (MUTEX*)0;
@@ -288,52 +297,19 @@ static void KiSurfaceDestroy(OBJECT* obj) {
surface->Frames = (PHYSICAL_ADDRESS*)0;
}
if (surface->RowScratch) {
MmFreePool(surface->RowScratch);
surface->RowScratch = (uint32_t*)0;
}
}
typedef struct SURFACE_ROW_CONTEXT {
SURFACE_OBJECT* Surface;
uint32_t SourceX;
uint32_t SourceY;
} SURFACE_ROW_CONTEXT;
static KSTATUS KiSurfaceCopyRow(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount) {
SURFACE_ROW_CONTEXT* ctx = (SURFACE_ROW_CONTEXT*)context;
if (!ctx || !ctx->Surface || !scratch || rowIndex < 0)
// Creates the process mapping used for user drawing.
static KSTATUS KiSurfaceEnsureViewLocked(SURFACE_OBJECT* surface) {
if (!surface || !surface->Owner || !surface->Frames || !surface->PageCount)
return NTLSTATUS_INVALID_PARAMETER;
if (surface->UserBase)
return NTLSTATUS_SUCCESS;
SURFACE_OBJECT* surface = ctx->Surface;
uint64_t row = (uint64_t)ctx->SourceY + (uint64_t)(uint32_t)rowIndex;
uint64_t offset = row * (uint64_t)surface->PitchBytes + (uint64_t)ctx->SourceX * sizeof(uint32_t);
uint64_t bytes = (uint64_t)pixelCount * sizeof(uint32_t);
if (offset > (uint64_t)surface->PayloadSize || bytes > (uint64_t)surface->PayloadSize - offset)
return NTLSTATUS_INVALID_PARAMETER;
uint8_t* dst = (uint8_t*)scratch;
while (bytes) {
size_t frameIndex = (size_t)(offset / PAGE_SIZE);
size_t pageOffset = (size_t)(offset & (PAGE_SIZE - 1));
if (frameIndex >= surface->PageCount || !surface->Frames[frameIndex])
return NTLSTATUS_INVALID_PARAMETER;
size_t chunk = PAGE_SIZE - pageOffset;
if (chunk > bytes)
chunk = (size_t)bytes;
const uint8_t* src = (const uint8_t*)MmPfnToVirtual(surface->Frames[frameIndex]);
if (!src)
return NTLSTATUS_INVALID_PARAMETER;
memcpy(dst, src + pageOffset, chunk);
dst += chunk;
offset += chunk;
bytes -= chunk;
}
return NTLSTATUS_SUCCESS;
return MmVadCreateSharedViewFromFrames(surface->Owner, surface->Frames, surface->PageCount,
surface->PayloadSize,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO, MmVadShared, &surface->UserBase);
}
// Implements the syscall Init operation.
@@ -513,8 +489,9 @@ static uint64_t SyscallExitThread(uint64_t a0, uint64_t a1, uint64_t a2,
}
// SYS_CREATE_THREAD a0=*SYS_CREATE_THREAD_REQUEST.
// Creates a runnable user thread and optionally returns both its waitable
// handle and diagnostic TID. The handle is the synchronization identity.
// Creates a user thread and optionally returns both its waitable handle and
// diagnostic TID. Suspended creation leaves the thread off every ready queue
// until SYS_RESUME_THREAD decrements its initial suspend count.
static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
@@ -527,8 +504,11 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
if (!proc) return NTLSTATUS_FAILURE;
if (!req.Entry) return NTLSTATUS_INVALID_PARAMETER;
if (req.Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
if (req.Flags) return NTLSTATUS_INVALID_PARAMETER;
if (req.Flags & ~SYS_CREATE_THREAD_FLAG_SUSPENDED)
return NTLSTATUS_INVALID_PARAMETER;
if (!req.ThreadHandleOut && !req.ThreadIdOut) return NTLSTATUS_INVALID_PARAMETER;
if ((req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED) && !req.ThreadHandleOut)
return NTLSTATUS_INVALID_PARAMETER;
s = MmProbeUserRead((const void*)req.Entry, 1);
if (!KSUCCESS(s)) return s;
@@ -542,16 +522,22 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
}
THREAD* newt = NULL;
KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(req.Priority);
if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
threadParameters.Flags |= KE_THREAD_CREATE_FLAG_SUSPENDED;
s = KeCreateThread(proc, req.Entry, req.Argument,
(THREAD_PRIORITY)req.Priority, &newt);
&threadParameters, &newt);
if (!KSUCCESS(s)) return s;
uint32_t threadHandle = 0;
if (req.ThreadHandleOut) {
s = KeAllocateHandleWithRights(proc, &newt->Header,
HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY,
HANDLE_RIGHT_WAIT | HANDLE_RIGHT_QUERY |
HANDLE_RIGHT_RESUME,
&threadHandle);
if (!KSUCCESS(s)) {
if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
(void)KeCancelSuspendedThread(newt);
ObDereferenceObject(&newt->Header);
return s;
}
@@ -561,6 +547,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadHandleOut, threadHandle);
if (!KSUCCESS(s)) {
KeCloseHandle(proc, threadHandle);
if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
(void)KeCancelSuspendedThread(newt);
ObDereferenceObject(&newt->Header);
return s;
}
@@ -570,6 +558,8 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
s = KiCopyU32ToUser((uint64_t)(uintptr_t)req.ThreadIdOut, newt->ThreadId);
if (!KSUCCESS(s)) {
if (threadHandle) KeCloseHandle(proc, threadHandle);
if (req.Flags & SYS_CREATE_THREAD_FLAG_SUSPENDED)
(void)KeCancelSuspendedThread(newt);
ObDereferenceObject(&newt->Header);
return s;
}
@@ -579,6 +569,35 @@ static uint64_t SyscallCreateThread(uint64_t a0, uint64_t a1, uint64_t a2,
return NTLSTATUS_SUCCESS;
}
// SYS_RESUME_THREAD a0=threadHandle, a1=previousSuspendCountOut.
static uint64_t SyscallResumeThread(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(uint32_t));
if (!KSUCCESS(status)) return status;
PROCESS* process = KeGetCurrentProcess();
if (!process) return NTLSTATUS_FAILURE;
OBJECT* object = KeLookupHandleWithRights(process, (uint32_t)a0,
HANDLE_RIGHT_RESUME);
if (!object) return NTLSTATUS_INVALID_HANDLE;
if (object->Type != ObjectTypeThread) {
ObDereferenceObject(object);
return NTLSTATUS_INVALID_HANDLE;
}
THREAD* thread = CONTAINING_RECORD(object, THREAD, Header);
uint32_t previousSuspendCount = 0;
status = KeResumeThread(thread, &previousSuspendCount);
ObDereferenceObject(object);
if (!KSUCCESS(status)) return status;
return KiCopyU32ToUser(a1, previousSuspendCount);
}
static void KiFinishAlertableUserWait(uint8_t alertable) {
if (!alertable) return;
@@ -619,6 +638,9 @@ static KSTATUS KiHandleToWaitEvent(PROCESS* proc, uint32_t handle, EVENT** event
case ObjectTypeDisplayClock:
event = KeDisplayClockWaitEvent((DISPLAY_CLOCK_OBJECT*)obj);
break;
case ObjectTypeAudioStream:
event = KeAudioStreamWaitEvent((KE_AUDIO_STREAM*)obj);
break;
default:
ObDereferenceObject(obj);
return NTLSTATUS_NOT_SUPPORTED;
@@ -656,28 +678,14 @@ static uint64_t SyscallWaitSingle(uint64_t a0, uint64_t a1, uint64_t a2,
if (obj->Type == ObjectTypeMutex) {
if (alertable) s = NTLSTATUS_NOT_SUPPORTED;
else if (timeoutMs == 0) {
s = KeTryAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header));
if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT;
} else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) {
s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header));
} else {
s = NTLSTATUS_NOT_SUPPORTED;
}
else s = KeAcquireMutex(CONTAINING_RECORD(obj, MUTEX, Header), timeoutMs);
ObDereferenceObject(obj);
return s;
}
if (obj->Type == ObjectTypeSemaphore) {
if (alertable) s = NTLSTATUS_NOT_SUPPORTED;
else if (timeoutMs == 0) {
s = KeTrySemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header));
if (s == NTLSTATUS_FAILURE) s = NTLSTATUS_TIMEOUT;
} else if (timeoutMs == SYS_WAIT_TIMEOUT_INFINITE) {
s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header));
} else {
s = NTLSTATUS_NOT_SUPPORTED;
}
else s = KeWaitSemaphore(CONTAINING_RECORD(obj, SEMAPHORE, Header), timeoutMs);
ObDereferenceObject(obj);
return s;
}
@@ -992,8 +1000,7 @@ static uint64_t SyscallSetThreadPriority(uint64_t a0, uint64_t a1, uint64_t a2,
if (a0 >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
THREAD* t = KeGetCurrentThread();
if (!t) return NTLSTATUS_FAILURE;
t->Priority = (THREAD_PRIORITY)a0;
return NTLSTATUS_SUCCESS;
return KiSchedulerSetNormalPriority(t, (THREAD_PRIORITY)a0);
}
// SYS_VM_* page-granular user virtual memory.
@@ -1163,7 +1170,8 @@ static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2,
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex, HANDLE_RIGHT_WAIT);
if (!mx) return NTLSTATUS_NOT_FOUND;
KSTATUS s = KeTryAcquireMutex(mx);
KSTATUS s = KeAcquireMutex(mx, 0);
if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE;
ObDereferenceObject(&mx->Header);
return s;
}
@@ -1204,7 +1212,8 @@ static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2,
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore, HANDLE_RIGHT_WAIT);
if (!sem) return NTLSTATUS_NOT_FOUND;
KSTATUS s = KeTrySemaphore(sem);
KSTATUS s = KeWaitSemaphore(sem, 0);
if (s == NTLSTATUS_TIMEOUT) s = NTLSTATUS_FAILURE;
ObDereferenceObject(&sem->Header);
return s;
}
@@ -1672,7 +1681,7 @@ static void* KiAllocateSyscallIoBuffer(size_t requested, size_t* actualOut) {
if (!requested) return NULL;
size_t chunk = requested < SYSCALL_FILE_IO_CHUNK_BYTES ? requested : SYSCALL_FILE_IO_CHUNK_BYTES;
for (;;) {
while (true) {
void* buf = MmAllocatePoolTag(chunk, MM_POOL_TAG('s', 'i', 'o', ' '));
if (buf) {
if (actualOut) *actualOut = chunk;
@@ -2275,9 +2284,8 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2,
surface->PageCount = pageCount;
surface->Frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS), MM_POOL_TAG('s', 'f', 'r', 'm'));
surface->RowScratch = (uint32_t*)MmAllocatePoolTag(rowBytes, MM_POOL_TAG('s', 'r', 'o', 'w'));
s = KeMutexCreate(&surface->Lock);
if (!surface->Frames || !surface->RowScratch || !KSUCCESS(s)) {
if (!surface->Frames || !KSUCCESS(s)) {
ObDereferenceObject(&surface->Header);
return NTLSTATUS_NO_MEMORY;
}
@@ -2295,6 +2303,14 @@ static uint64_t SyscallCreateSurface(uint64_t a0, uint64_t a1, uint64_t a2,
memset(kva, 0, PAGE_SIZE);
}
s = MmCreateKernelViewFromFrames(surface->Frames, surface->PageCount,
&surface->KernelView);
if (!KSUCCESS(s)) {
ObDereferenceObject(&surface->Header);
return s;
}
surface->KernelBase = MmKernelViewBase(surface->KernelView);
uint32_t handle = 0;
s = KiAllocHandleForCurrentWithRights(&surface->Header, HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE | HANDLE_RIGHT_QUERY, &handle);
if (!KSUCCESS(s))
@@ -2327,18 +2343,13 @@ static uint64_t SyscallMapSurface(uint64_t a0, uint64_t a1, uint64_t a2,
return NTLSTATUS_ACCESS_DENIED;
}
KSTATUS s = KeAcquireMutex(surface->Lock);
if (!KSUCCESS(s)) {
KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE);
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(s)) {
ObDereferenceObject(&surface->Header);
return s;
}
if (!surface->UserBase) {
s = MmVadCreateSharedViewFromFrames(current, surface->Frames, surface->PageCount,
surface->PayloadSize,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO, MmVadShared, &surface->UserBase);
}
s = KiSurfaceEnsureViewLocked(surface);
void* userBase = surface->UserBase;
uint32_t pitchBytes = surface->PitchBytes;
@@ -2374,8 +2385,8 @@ static uint64_t SyscallQuerySurface(uint64_t a0, uint64_t a1, uint64_t a2,
SYS_SURFACE_DESC desc;
memset(&desc, 0, sizeof(desc));
KSTATUS s = KeAcquireMutex(surface->Lock);
if (KSUCCESS(s)) {
KSTATUS s = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE);
if (KE_MUTEX_ACQUIRE_SUCCEEDED(s)) {
desc.Width = surface->Width;
desc.Height = surface->Height;
desc.PitchBytes = surface->PitchBytes;
@@ -2475,20 +2486,21 @@ static uint64_t SyscallPresentSurface(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t rowsCopied = 0;
uint64_t bytesCopied = 0;
KSTATUS s = NTLSTATUS_SUCCESS;
for (uint32_t i = 0; i < rectCount; ++i) {
KSTATUS s = surface->KernelBase ? NTLSTATUS_SUCCESS : NTLSTATUS_DEVICE_ERROR;
const uint8_t* sourceBase = (const uint8_t*)surface->KernelBase;
for (uint32_t i = 0; i < rectCount && KSUCCESS(s); ++i) {
DISPLAY_RECT clipped;
if (!KiClipSurfaceRect(surface, &rects[i], &clipped)) {
emptyClips++;
continue;
}
SURFACE_ROW_CONTEXT ctx;
ctx.Surface = surface;
ctx.SourceX = (uint32_t)clipped.X;
ctx.SourceY = (uint32_t)clipped.Y;
s = KeDisplayBlitRows(clipped.X, clipped.Y, clipped.W, clipped.H,
KiSurfaceCopyRow, &ctx, surface->RowScratch);
const uint32_t* source = (const uint32_t*)(sourceBase +
(size_t)(uint32_t)clipped.Y * surface->PitchBytes +
(size_t)(uint32_t)clipped.X * sizeof(uint32_t));
s = KeDisplayPresentPitched(clipped.X, clipped.Y, clipped.W, clipped.H,
source, surface->PitchBytes);
if (!KSUCCESS(s))
break;
@@ -2657,7 +2669,7 @@ static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2,
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!a0) return NTLSTATUS_INVALID_PARAMETER;
for (;;) {
while (true) {
char ch = KeInputReadChar();
if (ch)
return CopyToUser((void*)(uintptr_t)a0, &ch, sizeof(ch));
@@ -2674,7 +2686,7 @@ static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2,
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!a0) return NTLSTATUS_INVALID_PARAMETER;
for (;;) {
while (true) {
INPUT_KEY key = KeInputReadKey();
if (key != INPUT_KEY_NONE)
return CopyToUser((void*)(uintptr_t)a0, &key, sizeof(key));
@@ -2722,7 +2734,7 @@ static uint64_t SyscallKbReadInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
if (!KSUCCESS(s)) return s;
INPUT_KEY_RECORD record;
for (;;) {
while (true) {
bool available = queue ? KeInputQueueReadEvent(queue, &record)
: KeInputReadEvent(&record);
if (available) {
@@ -2759,7 +2771,7 @@ static uint64_t SyscallKbPeekInputEvent(uint64_t a0, uint64_t a1, uint64_t a2,
if (!KSUCCESS(s)) return s;
INPUT_KEY_RECORD record;
for (;;) {
while (true) {
bool available = queue ? KeInputQueuePeekEvent(queue, &record)
: KeInputPeekEvent(&record);
if (available) {
@@ -2995,7 +3007,8 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
THREAD* thread;
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread);
const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateThread(proc, entryPoint, 0, &threadParameters, &thread);
if (!KSUCCESS(s)) {
ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint);
MmVadDebugDumpProcess(proc);
@@ -3070,7 +3083,7 @@ static uint64_t SyscallGetProcessExitCode(uint64_t a0, uint64_t a1, uint64_t a2,
}
// SYS_WRITE_CONSOLE (60) a0=*buf a1=len
// Writes user console text in white; distinct from green SYS_PRINT_DEBUG.
// Writes user console text and completes display delivery before returning.
static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
@@ -3092,7 +3105,7 @@ static uint64_t SyscallWriteConsole(uint64_t a0, uint64_t a1, uint64_t a2,
KePrintConsole(chunk, todo);
offset += todo;
}
return NTLSTATUS_SUCCESS;
return KeDisplayFlushConsoleOutput();
}
// SYS_OPEN_DIR (54) a0=*dirPath a1=*handleOut.
@@ -3309,7 +3322,7 @@ void KeNamedQueueUnpublishHandle(PROCESS* process, uint32_t handle, OBJECT* obje
if (!process || !object || object->Type != ObjectTypeQueue)
return;
for (;;) {
while (true) {
QUEUE* queue = (QUEUE*)0;
uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
@@ -4473,7 +4486,7 @@ static uint64_t SyscallPoll(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t start = gHal ? gHal->Timer.QueryTicks() : 0;
size_t ready = 0;
for (;;) {
while (true) {
ready = 0;
for (size_t i = 0; i < count; i++) {
fds[i].ReturnedEvents = 0;
@@ -4982,6 +4995,7 @@ static uint64_t SyscallQueryDisplayInfo(uint64_t a0, uint64_t a1, uint64_t a2,
out.Mode = ki.Mode;
out.Caps = ki.Caps;
out.DeviceCount = ki.DeviceCount;
out.Flags = ki.Flags;
out.ManagerStats = ki.ManagerStats;
out.DeviceStats = ki.DeviceStats;
@@ -5012,6 +5026,12 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
us.CpuBurstDemotions = ks.CpuBurstDemotions;
us.SmtSiblingDeferrals = ks.SmtSiblingDeferrals;
us.SmtPrimaryPokes = ks.SmtPrimaryPokes;
us.RealtimeDispatches = ks.RealtimeDispatches;
us.RealtimePreemptions = ks.RealtimePreemptions;
us.PriorityInheritanceBoosts = ks.PriorityInheritanceBoosts;
us.PriorityInheritanceDrops = ks.PriorityInheritanceDrops;
us.PriorityInheritancePropagations = ks.PriorityInheritancePropagations;
us.PriorityInheritanceMaxDepth = ks.PriorityInheritanceMaxDepth;
return KiReturnKernelStats(a1, (size_t)a2, a3, &us, sizeof(us));
}
@@ -5199,19 +5219,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
um.DropFailures = km.DropFailures;
um.StandbyTransitions = km.StandbyTransitions;
um.StandbyHits = km.StandbyHits;
um.ReadAheadRequests = km.ReadAheadRequests;
um.ReadAheadPagesRequested = km.ReadAheadPagesRequested;
um.ReadAheadPagesQueued = km.ReadAheadPagesQueued;
um.ReadAheadPagesLoaded = km.ReadAheadPagesLoaded;
um.ReadAheadPagesSkipped = km.ReadAheadPagesSkipped;
um.ReadAheadFailures = km.ReadAheadFailures;
um.ReadAheadPagesThrottled = km.ReadAheadPagesThrottled;
um.MaxReadAheadPagesPerRequest = km.MaxReadAheadPagesPerRequest;
um.BatchReads = km.BatchReads;
um.BatchPagesRequested = km.BatchPagesRequested;
um.BatchPagesLoaded = km.BatchPagesLoaded;
um.MaxBatchPagesRequested = km.MaxBatchPagesRequested;
um.MaxBatchPagesLoaded = km.MaxBatchPagesLoaded;
um.ResidentPages = km.ResidentPages;
um.ActivePages = km.ActivePages;
um.StandbyPages = km.StandbyPages;
@@ -5285,17 +5292,6 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
um.WriteSectors = km.WriteSectors;
um.MaxReadSectors = km.MaxReadSectors;
um.MaxWriteSectors = km.MaxWriteSectors;
um.ReadaheadHits = km.ReadaheadHits;
um.ReadaheadMisses = km.ReadaheadMisses;
um.ReadaheadStores = km.ReadaheadStores;
um.ReadaheadInvalidations = km.ReadaheadInvalidations;
um.ReadaheadBytesServed = km.ReadaheadBytesServed;
um.ReadaheadSectorsLoaded = km.ReadaheadSectorsLoaded;
um.ReadaheadSequentialDetections = km.ReadaheadSequentialDetections;
um.ReadaheadSuppressed = km.ReadaheadSuppressed;
um.ReadaheadAllocFailures = km.ReadaheadAllocFailures;
um.ReadaheadReadFailures = km.ReadaheadReadFailures;
um.MaxReadaheadSectors = km.MaxReadaheadSectors;
return KiReturnKernelStats(a1, (size_t)a2, a3, &um, sizeof(um));
}
@@ -5374,6 +5370,466 @@ static uint64_t SyscallQueryKernelStats(uint64_t a0, uint64_t a1, uint64_t a2,
}
}
// Converts one public audio format into the kernel mixer format.
static KSTATUS KiImportAudioFormat(const SYS_AUDIO_FORMAT* source, IO_AUDIO_FORMAT* destination) {
if (!source || !destination)
return NTLSTATUS_INVALID_PARAMETER;
if (source->SampleFormat != SysAudioSampleS16 ||
(source->ChannelCount != 1 && source->ChannelCount != 2) ||
(source->SampleRate != 11025u && source->SampleRate != 22050u &&
source->SampleRate != 44100u && source->SampleRate != 48000u))
return NTLSTATUS_NOT_SUPPORTED;
destination->SampleRate = source->SampleRate;
destination->ChannelCount = source->ChannelCount;
destination->SampleFormat = IoAudioSampleS16;
return NTLSTATUS_SUCCESS;
}
// Converts one kernel audio format into the public syscall representation.
static SYS_AUDIO_FORMAT KiExportAudioFormat(const IO_AUDIO_FORMAT* source) {
SYS_AUDIO_FORMAT destination;
destination.SampleRate = source->SampleRate;
destination.ChannelCount = source->ChannelCount;
destination.SampleFormat = source->SampleFormat == IoAudioSampleS16
? SysAudioSampleS16
: 0;
return destination;
}
// Converts one normalized kernel gain into unsigned Q16.
static uint32_t KiExportAudioGainQ16(float gain) {
if (gain <= 0.0f) return 0;
if (gain >= 1.0f) return SYS_AUDIO_GAIN_Q16_UNITY;
return (uint32_t)(gain * (float)SYS_AUDIO_GAIN_Q16_UNITY + 0.5f);
}
// Converts one normalized kernel pan into signed Q16.
static int32_t KiExportAudioPanQ16(float pan) {
if (pan <= -1.0f) return SYS_AUDIO_PAN_Q16_LEFT;
if (pan >= 1.0f) return SYS_AUDIO_PAN_Q16_RIGHT;
float scaled = pan * (float)SYS_AUDIO_GAIN_Q16_UNITY;
return (int32_t)(scaled + (scaled >= 0.0f ? 0.5f : -0.5f));
}
// Converts one public unsigned Q16 gain into the kernel representation.
static float KiImportAudioGainQ16(uint32_t gainQ16) {
return (float)gainQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY;
}
// Converts one public signed Q16 pan into the kernel representation.
static float KiImportAudioPanQ16(int32_t panQ16) {
return (float)panQ16 / (float)SYS_AUDIO_GAIN_Q16_UNITY;
}
// Converts one kernel endpoint kind into its public ABI value.
static uint32_t KiExportAudioEndpointKind(IO_AUDIO_ENDPOINT_KIND kind) {
switch (kind) {
case IoAudioEndpointLineOut: return SysAudioEndpointLineOut;
case IoAudioEndpointSpeakerHeadphone: return SysAudioEndpointSpeakerHeadphone;
case IoAudioEndpointOtherAnalog: return SysAudioEndpointOtherAnalog;
default: return 0;
}
}
// Converts provider endpoint capabilities into public capability bits.
static uint32_t KiExportAudioEndpointCapabilities(uint32_t capabilities) {
uint32_t result = 0;
if (capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE)
result |= SYS_AUDIO_ENDPOINT_CAP_JACK_SENSE;
if (capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN)
result |= SYS_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN;
return result;
}
// Converts one endpoint lifecycle value into its public ABI value.
static uint32_t KiExportAudioEndpointState(KE_AUDIO_ENDPOINT_STATE state) {
switch (state) {
case KeAudioEndpointStarting: return SysAudioEndpointStarting;
case KeAudioEndpointReady: return SysAudioEndpointReady;
case KeAudioEndpointStopping: return SysAudioEndpointStopping;
case KeAudioEndpointFailed: return SysAudioEndpointFailed;
case KeAudioEndpointRemoved: return SysAudioEndpointRemoved;
default: return 0;
}
}
// Converts one stream lifecycle value into its public ABI value.
static uint32_t KiExportAudioStreamState(KE_AUDIO_STREAM_STATE state) {
switch (state) {
case KeAudioStreamCreated: return SysAudioStreamCreated;
case KeAudioStreamRunning: return SysAudioStreamRunning;
case KeAudioStreamPaused: return SysAudioStreamPaused;
case KeAudioStreamDraining: return SysAudioStreamDraining;
case KeAudioStreamStopped: return SysAudioStreamStopped;
case KeAudioStreamFailed: return SysAudioStreamFailed;
case KeAudioStreamClosed:
default:
return 0;
}
}
// Converts public wait flags into the kernel stream condition mask.
static uint32_t KiImportAudioWaitMask(uint32_t mask) {
uint32_t result = 0;
if (mask & SYS_AUDIO_WAIT_WRITABLE) result |= KE_AUDIO_WAIT_WRITABLE;
if (mask & SYS_AUDIO_WAIT_CONSUMED) result |= KE_AUDIO_WAIT_CONSUMED;
if (mask & SYS_AUDIO_WAIT_PRESENTED) result |= KE_AUDIO_WAIT_PRESENTED;
if (mask & SYS_AUDIO_WAIT_STOPPED) result |= KE_AUDIO_WAIT_STOPPED;
if (mask & SYS_AUDIO_WAIT_DRAINED) result |= KE_AUDIO_WAIT_DRAINED;
if (mask & SYS_AUDIO_WAIT_FAILED) result |= KE_AUDIO_WAIT_FAILED;
if (mask & SYS_AUDIO_WAIT_DISCONTINUITY) result |= KE_AUDIO_WAIT_DISCONTINUITY;
return result;
}
// Looks up one audio stream handle with the requested operation rights.
static KE_AUDIO_STREAM* KiLookupAudioStream(uint32_t handle, uint32_t rights) {
OBJECT* object = KiHandleToObject(handle, ObjectTypeAudioStream, rights);
return object ? (KE_AUDIO_STREAM*)object : (KE_AUDIO_STREAM*)0;
}
// SYS_AUDIO_ENUMERATE_ENDPOINTS a0=*SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST.
static uint64_t SyscallAudioEnumerateEndpoints(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST request;
KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 ||
!request.CountOut || (request.Capacity && !request.Endpoints))
return NTLSTATUS_INVALID_PARAMETER;
status = MmProbeUserWrite(request.CountOut, sizeof(*request.CountOut));
if (!KSUCCESS(status)) return status;
uint32_t copyCapacity = request.Capacity < SYS_AUDIO_MAX_ENDPOINTS
? request.Capacity
: SYS_AUDIO_MAX_ENDPOINTS;
if (copyCapacity) {
status = MmProbeUserWrite(request.Endpoints,
(size_t)copyCapacity * sizeof(*request.Endpoints));
if (!KSUCCESS(status)) return status;
}
KE_AUDIO_ENDPOINT_INFO kernelInfo[KE_AUDIO_MAX_ENDPOINTS];
uint32_t count = 0;
status = KeAudioQueryEndpoints(kernelInfo, KE_AUDIO_MAX_ENDPOINTS, &count);
if (KFAILED(status)) return status;
SYS_AUDIO_ENDPOINT_INFO publicInfo[SYS_AUDIO_MAX_ENDPOINTS];
uint32_t copyCount = copyCapacity < count ? copyCapacity : count;
for (uint32_t index = 0; index < copyCount; index++) {
const KE_AUDIO_ENDPOINT_INFO* source = &kernelInfo[index];
SYS_AUDIO_ENDPOINT_INFO* destination = &publicInfo[index];
memset(destination, 0, sizeof(*destination));
destination->Size = sizeof(*destination);
destination->Version = SYS_AUDIO_ABI_VERSION;
destination->EndpointId = source->EndpointId;
destination->Kind = KiExportAudioEndpointKind(source->Kind);
if (source->Available) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE;
if (source->IsDefault) destination->Flags |= SYS_AUDIO_ENDPOINT_FLAG_DEFAULT;
destination->Capabilities = KiExportAudioEndpointCapabilities(source->Capabilities);
destination->State = KiExportAudioEndpointState(source->State);
destination->HardwareFormat = KiExportAudioFormat(&source->HardwareFormat);
destination->PeriodCount = source->PeriodCount;
destination->FramesPerPeriod = source->FramesPerPeriod;
destination->MasterVolumeQ16 = KiExportAudioGainQ16(source->MasterRequestedVolume);
destination->MasterMuted = source->MasterMuted ? 1u : 0u;
destination->FailureStatus = source->FailureStatus;
}
if (copyCount) {
status = CopyToUser(request.Endpoints,
publicInfo,
(size_t)copyCount * sizeof(publicInfo[0]));
if (!KSUCCESS(status)) return status;
}
status = CopyToUser(request.CountOut, &count, sizeof(count));
if (!KSUCCESS(status)) return status;
return request.Capacity < count ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS;
}
// SYS_AUDIO_OPEN_STREAM a0=*SYS_AUDIO_OPEN_STREAM_REQUEST.
static uint64_t SyscallAudioOpenStream(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_AUDIO_OPEN_STREAM_REQUEST request;
KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 ||
!request.EndpointId || !request.StreamHandleOut)
return NTLSTATUS_INVALID_PARAMETER;
status = MmProbeUserWrite(request.StreamHandleOut, sizeof(*request.StreamHandleOut));
if (!KSUCCESS(status)) return status;
IO_AUDIO_FORMAT format;
status = KiImportAudioFormat(&request.Format, &format);
if (!KSUCCESS(status)) return status;
KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)0;
status = KeAudioCreateStream(request.EndpointId, &format, &stream);
if (!KSUCCESS(status)) return status;
uint32_t handle = 0;
status = KiAllocHandleForCurrentWithRights((OBJECT*)stream,
HANDLE_RIGHT_WRITE |
HANDLE_RIGHT_WAIT |
HANDLE_RIGHT_QUERY,
&handle);
if (!KSUCCESS(status)) return status;
status = CopyToUser(request.StreamHandleOut, &handle, sizeof(handle));
if (!KSUCCESS(status)) {
PROCESS* process = KeGetCurrentProcess();
if (process) KeCloseHandle(process, handle);
}
return status;
}
// SYS_AUDIO_WRITE a0=*SYS_AUDIO_WRITE_REQUEST.
static uint64_t SyscallAudioWrite(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_AUDIO_WRITE_REQUEST request;
KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle ||
!request.WrittenFramesOut || (request.FrameCount && !request.Frames))
return NTLSTATUS_INVALID_PARAMETER;
status = MmProbeUserWrite(request.WrittenFramesOut, sizeof(*request.WrittenFramesOut));
if (!KSUCCESS(status)) return status;
uint32_t written = 0;
KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE);
if (!stream) return NTLSTATUS_INVALID_HANDLE;
if (!request.FrameCount) {
ObDereferenceObject((OBJECT*)stream);
return CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
}
KE_AUDIO_STREAM_INFO info;
status = KeAudioQueryStream(stream, &info);
if (!KSUCCESS(status)) {
ObDereferenceObject((OBJECT*)stream);
return status;
}
if (info.State == KeAudioStreamFailed) {
status = KFAILED(info.FailureStatus) ? info.FailureStatus : NTLSTATUS_DEVICE_ERROR;
ObDereferenceObject((OBJECT*)stream);
(void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
return status;
}
if (info.State == KeAudioStreamDraining) {
ObDereferenceObject((OBJECT*)stream);
(void)CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
return NTLSTATUS_INVALID_PARAMETER;
}
if (!info.WritableFrames) {
ObDereferenceObject((OBJECT*)stream);
status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
return KSUCCESS(status) ? NTLSTATUS_WOULD_BLOCK : status;
}
uint32_t frameCount = request.FrameCount < info.WritableFrames
? request.FrameCount
: info.WritableFrames;
uint32_t frameBytes = (uint32_t)info.Format.ChannelCount * sizeof(int16_t);
size_t byteCount = (size_t)frameCount * frameBytes;
status = MmProbeUserRead(request.Frames, byteCount);
if (!KSUCCESS(status)) {
ObDereferenceObject((OBJECT*)stream);
return status;
}
void* buffer = MmAllocatePoolTag(byteCount, MM_POOL_TAG('a', 'u', 'w', 'r'));
if (!buffer) {
ObDereferenceObject((OBJECT*)stream);
return NTLSTATUS_NO_MEMORY;
}
status = CopyFromUser(buffer, request.Frames, byteCount);
if (KSUCCESS(status))
status = CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
if (KSUCCESS(status))
status = KeAudioWriteStream(stream, buffer, frameCount, &written);
MmFreePool(buffer);
ObDereferenceObject((OBJECT*)stream);
if (!KSUCCESS(status)) return status;
return CopyToUser(request.WrittenFramesOut, &written, sizeof(written));
}
// SYS_AUDIO_ARM_WAIT a0=*SYS_AUDIO_ARM_WAIT_REQUEST.
static uint64_t SyscallAudioArmWait(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_AUDIO_ARM_WAIT_REQUEST request;
KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Size != sizeof(request) || request.Flags != 0 || request.Reserved != 0 ||
!request.StreamHandle || !request.Mask ||
(request.Mask & ~SYS_AUDIO_WAIT_VALID_MASK) != 0)
return NTLSTATUS_INVALID_PARAMETER;
KE_AUDIO_STREAM_WAIT_CONDITION condition;
condition.Mask = KiImportAudioWaitMask(request.Mask);
condition.WritableFrames = request.WritableFrames;
condition.ConsumedFrames = request.ConsumedFrames;
condition.PresentedFrames = request.PresentedFrames;
condition.DrainedGeneration = request.DrainedGeneration;
condition.DiscontinuityGeneration = request.DiscontinuityGeneration;
KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WAIT);
if (!stream) return NTLSTATUS_INVALID_HANDLE;
status = KeAudioArmStreamWait(stream, &condition);
ObDereferenceObject((OBJECT*)stream);
return status;
}
// SYS_AUDIO_QUERY_STREAM a0=streamHandle a1=*SYS_AUDIO_STREAM_INFO.
static uint64_t SyscallAudioQueryStream(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
if (!a0 || !a1) return NTLSTATUS_INVALID_PARAMETER;
uint32_t userSize = 0;
KSTATUS status = CopyFromUser(&userSize, (const void*)(uintptr_t)a1, sizeof(userSize));
if (!KSUCCESS(status)) return status;
if (userSize != sizeof(SYS_AUDIO_STREAM_INFO))
return NTLSTATUS_INVALID_PARAMETER;
status = MmProbeUserWrite((void*)(uintptr_t)a1, sizeof(SYS_AUDIO_STREAM_INFO));
if (!KSUCCESS(status)) return status;
KE_AUDIO_STREAM* stream = KiLookupAudioStream((uint32_t)a0, HANDLE_RIGHT_QUERY);
if (!stream) return NTLSTATUS_INVALID_HANDLE;
KE_AUDIO_STREAM_INFO kernelInfo;
status = KeAudioQueryStream(stream, &kernelInfo);
ObDereferenceObject((OBJECT*)stream);
if (!KSUCCESS(status)) return status;
SYS_AUDIO_STREAM_INFO publicInfo;
memset(&publicInfo, 0, sizeof(publicInfo));
publicInfo.Size = sizeof(publicInfo);
publicInfo.Version = SYS_AUDIO_ABI_VERSION;
publicInfo.State = KiExportAudioStreamState(kernelInfo.State);
publicInfo.Format = KiExportAudioFormat(&kernelInfo.Format);
publicInfo.CapacityFrames = kernelInfo.CapacityFrames;
publicInfo.BufferedFrames = kernelInfo.BufferedFrames;
publicInfo.WritableFrames = kernelInfo.WritableFrames;
publicInfo.FailureStatus = kernelInfo.FailureStatus;
publicInfo.FramesSubmitted = kernelInfo.FramesSubmitted;
publicInfo.FramesConsumed = kernelInfo.FramesConsumed;
publicInfo.FramesPresented = kernelInfo.FramesPresented;
publicInfo.FramesDiscarded = kernelInfo.FramesDiscarded;
publicInfo.OutputFramesMixed = kernelInfo.OutputFramesMixed;
publicInfo.SilenceFramesInserted = kernelInfo.SilenceFramesInserted;
publicInfo.Underruns = kernelInfo.Underruns;
publicInfo.DrainedGeneration = kernelInfo.DrainedGeneration;
publicInfo.DiscontinuityGeneration = kernelInfo.DiscontinuityGeneration;
publicInfo.VolumeQ16 = KiExportAudioGainQ16(kernelInfo.Volume);
publicInfo.PanQ16 = KiExportAudioPanQ16(kernelInfo.Pan);
publicInfo.Muted = kernelInfo.Muted;
return CopyToUser((void*)(uintptr_t)a1, &publicInfo, sizeof(publicInfo));
}
typedef KSTATUS (*KI_AUDIO_STREAM_OPERATION)(KE_AUDIO_STREAM* stream);
// Invokes one existing kernel stream operation through a write-capable handle.
static KSTATUS KiCallAudioStreamOperation(uint32_t handle, KI_AUDIO_STREAM_OPERATION operation) {
if (!handle || !operation) return NTLSTATUS_INVALID_PARAMETER;
KE_AUDIO_STREAM* stream = KiLookupAudioStream(handle, HANDLE_RIGHT_WRITE);
if (!stream) return NTLSTATUS_INVALID_HANDLE;
KSTATUS status = operation(stream);
ObDereferenceObject((OBJECT*)stream);
return status;
}
// SYS_AUDIO_START a0=streamHandle.
static uint64_t SyscallAudioStart(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStartStream);
}
// SYS_AUDIO_PAUSE a0=streamHandle.
static uint64_t SyscallAudioPause(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KiCallAudioStreamOperation((uint32_t)a0, KeAudioPauseStream);
}
// SYS_AUDIO_RESUME a0=streamHandle.
static uint64_t SyscallAudioResume(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResumeStream);
}
// SYS_AUDIO_DRAIN a0=streamHandle.
static uint64_t SyscallAudioDrain(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KiCallAudioStreamOperation((uint32_t)a0, KeAudioDrainStream);
}
// SYS_AUDIO_STOP a0=streamHandle.
static uint64_t SyscallAudioStop(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KiCallAudioStreamOperation((uint32_t)a0, KeAudioStopStream);
}
// SYS_AUDIO_RESET a0=streamHandle.
static uint64_t SyscallAudioReset(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KiCallAudioStreamOperation((uint32_t)a0, KeAudioResetStream);
}
// SYS_AUDIO_SET_VOLUME a0=*SYS_AUDIO_SET_VOLUME_REQUEST.
static uint64_t SyscallAudioSetVolume(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_AUDIO_SET_VOLUME_REQUEST request;
KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Size != sizeof(request) || request.Flags != 0 || !request.StreamHandle ||
request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY ||
request.PanQ16 < SYS_AUDIO_PAN_Q16_LEFT ||
request.PanQ16 > SYS_AUDIO_PAN_Q16_RIGHT)
return NTLSTATUS_INVALID_PARAMETER;
KE_AUDIO_STREAM* stream = KiLookupAudioStream(request.StreamHandle, HANDLE_RIGHT_WRITE);
if (!stream) return NTLSTATUS_INVALID_HANDLE;
status = KeAudioSetStreamVolume(stream,
KiImportAudioGainQ16(request.VolumeQ16),
KiImportAudioPanQ16(request.PanQ16),
request.Muted != 0);
ObDereferenceObject((OBJECT*)stream);
return status;
}
// SYS_AUDIO_SET_MASTER_VOLUME a0=*SYS_AUDIO_SET_MASTER_VOLUME_REQUEST.
static uint64_t SyscallAudioSetMasterVolume(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SYS_AUDIO_SET_MASTER_VOLUME_REQUEST request;
KSTATUS status = KiCopyUserRequest(a0, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Size != sizeof(request) || request.Flags != 0 || !request.EndpointId ||
request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY)
return NTLSTATUS_INVALID_PARAMETER;
return KeAudioSetMasterVolume(request.EndpointId,
KiImportAudioGainQ16(request.VolumeQ16),
request.Muted != 0);
}
// Dispatch table keyed by SYS_* constants.
typedef uint64_t (*SYSCALL_HANDLER)(uint64_t, uint64_t, uint64_t,
uint64_t, uint64_t, uint64_t);
@@ -5389,6 +5845,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
[SYS_GET_PROCESS_LIST] = SyscallGetProcessList,
[SYS_GET_PROCESS_PID] = SyscallGetProcessPid,
[SYS_CREATE_THREAD] = SyscallCreateThread,
[SYS_RESUME_THREAD] = SyscallResumeThread,
[SYS_GET_THREAD_LIST] = SyscallGetThreadList,
[SYS_LOAD_MODULE] = SyscallLoadModule,
[SYS_UNLOAD_MODULE] = SyscallUnloadModule,
@@ -5480,6 +5937,21 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
[SYS_QUERY_SURFACE] = SyscallQuerySurface,
[SYS_PRESENT_SURFACE] = SyscallPresentSurface,
// Native render audio.
[SYS_AUDIO_ENUMERATE_ENDPOINTS] = SyscallAudioEnumerateEndpoints,
[SYS_AUDIO_OPEN_STREAM] = SyscallAudioOpenStream,
[SYS_AUDIO_WRITE] = SyscallAudioWrite,
[SYS_AUDIO_ARM_WAIT] = SyscallAudioArmWait,
[SYS_AUDIO_QUERY_STREAM] = SyscallAudioQueryStream,
[SYS_AUDIO_START] = SyscallAudioStart,
[SYS_AUDIO_PAUSE] = SyscallAudioPause,
[SYS_AUDIO_RESUME] = SyscallAudioResume,
[SYS_AUDIO_DRAIN] = SyscallAudioDrain,
[SYS_AUDIO_STOP] = SyscallAudioStop,
[SYS_AUDIO_RESET] = SyscallAudioReset,
[SYS_AUDIO_SET_VOLUME] = SyscallAudioSetVolume,
[SYS_AUDIO_SET_MASTER_VOLUME] = SyscallAudioSetMasterVolume,
// Input.
[SYS_POLL_KEYBOARD] = SyscallKbPoll,
[SYS_IS_KEY_DOWN] = SyscallKbIsKeyDown,
@@ -5532,6 +6004,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
THREAD* self = KeGetCurrentThread();
PROCESS* owner = self ? self->Owner : (PROCESS*)0;
if (owner && KeIsProcessTerminating(owner) &&
!KeThreadTerminationDeferred(self) &&
nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) {
int32_t exitCode = owner->ExitCode;
KeExitThread(self, exitCode);
@@ -5547,6 +6020,7 @@ uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
// has been marked terminating.
owner = self ? self->Owner : (PROCESS*)0;
if (owner && KeIsProcessTerminating(owner) &&
!KeThreadTerminationDeferred(self) &&
nr != SYS_EXIT_PROCESS && nr != SYS_EXIT_THREAD) {
int32_t exitCode = owner->ExitCode;
KeExitThread(self, exitCode);
code/kernel/ke_system_root.c
+60 -2
View File
@@ -7,10 +7,15 @@
#include <kernel/ke_system_root.h>
#include <kernel/ke_debug.h>
#include <kernel/io_file.h>
#include <kernel/ke_sched.h>
static uint8_t gSystemRootDrive;
static bool gSystemRootReady;
#define KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS 30000ull
#define KE_SYSTEM_ROOT_WAIT_SLICE_MS 1000ull
#define KE_SYSTEM_ROOT_LOG_INTERVAL_MS 5000ull
// Builds a drive-qualified path without depending on any process CWD.
static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) {
if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES)
@@ -50,23 +55,76 @@ static bool KiDriveLooksLikeSystemRoot(uint8_t drive) {
return true;
}
// Selects the mounted volume that contains the kernel/system tree.
KSTATUS KeInitializeSystemRootVolume(void) {
// Returns the current scheduler time in milliseconds.
static uint64_t KiSystemRootNowMilliseconds(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks * SCHED_TICK_MS;
}
// Scans the currently mounted volumes for the system tree.
static KSTATUS KiTrySelectSystemRootVolume(uint32_t* mountedOut) {
uint32_t mounted = 0;
for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) {
if (!IoFilesystemPresent(drive))
continue;
mounted++;
if (!KiDriveLooksLikeSystemRoot(drive))
continue;
gSystemRootDrive = drive;
gSystemRootReady = true;
if (mountedOut)
*mountedOut = mounted;
DbgPrint("System root: drive=%u\n", drive);
return NTLSTATUS_SUCCESS;
}
if (mountedOut)
*mountedOut = mounted;
return NTLSTATUS_NOT_FOUND;
}
// Waits for the mounted boot volume instead of racing asynchronous bus discovery.
KSTATUS KeInitializeSystemRootVolume(void) {
uint64_t start = KiSystemRootNowMilliseconds();
uint64_t deadline = start + KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS;
uint64_t nextLog = start;
for (;;) {
uint64_t generation = IoGetVolumeNamespaceGeneration();
uint32_t mounted = 0;
KSTATUS status = KiTrySelectSystemRootVolume(&mounted);
if (KSUCCESS(status))
return status;
uint64_t now = KiSystemRootNowMilliseconds();
if (now >= deadline) {
DbgPrint("System root: discovery timed out after %llu ms; mounted=%u generation=%llu\n",
(unsigned long long)KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS,
mounted,
(unsigned long long)generation);
return NTLSTATUS_TIMEOUT;
}
if (now >= nextLog) {
DbgPrint("System root: waiting for boot volume; mounted=%u generation=%llu remaining=%llu ms\n",
mounted,
(unsigned long long)generation,
(unsigned long long)(deadline - now));
nextLog = now + KE_SYSTEM_ROOT_LOG_INTERVAL_MS;
}
uint64_t remaining = deadline - now;
uint64_t wait = remaining < KE_SYSTEM_ROOT_WAIT_SLICE_MS
? remaining
: KE_SYSTEM_ROOT_WAIT_SLICE_MS;
KSTATUS waitStatus = IoWaitForVolumeNamespaceChange(generation, wait);
if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT)
return waitStatus;
}
}
// Returns whether the system root volume has been selected.
bool KeSystemRootReady(void) {
return gSystemRootReady;
code/kernel/ke_thread.c
+255 -105
View File
@@ -27,14 +27,105 @@
_Static_assert(offsetof(type, field) == (expected), \
#type "." #field " offset must match assembly")
STATIC_ASSERT_OFFSET(THREAD, SavedContext, 96);
STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 112);
STATIC_ASSERT_OFFSET(THREAD, Teb, 120);
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 176);
STATIC_ASSERT_OFFSET(THREAD, SavedContext, 112);
STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 128);
STATIC_ASSERT_OFFSET(THREAD, Teb, 136);
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 192);
static uint32_t gNextThreadId = 1;
extern void KeThreadEntry(void);
// Prevents a newly scheduled user thread from entering a terminating process.
void KePrepareUserThreadStart(void) {
THREAD* thread = KeGetCurrentThread();
PROCESS* owner = thread ? thread->Owner : (PROCESS*)0;
if (!thread || !owner || !KeIsProcessTerminating(owner))
return;
KeExitThread(thread, owner->ExitCode);
while (true) {
KeYield();
}
}
// Finalizes one dead thread's completion, user memory, and process membership.
void KeFinalizeThreadExit(THREAD* thread) {
if (!thread)
return;
if (thread->State != ThreadDead) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread finalized before death: thread=0x%016llX tid=%u state=%u\n",
(uint64_t)thread,
thread->ThreadId,
(unsigned)thread->State);
}
// Thread completion is manual-reset and idempotent. Publish it before
// process-owned handles can be released by the final live-thread exit.
if (thread->ExitEvent)
KeSignalEvent(thread->ExitEvent);
PROCESS* owner = thread->Owner;
if (!owner)
return;
// Every exiting thread releases its own user allocations before leaving
// the live-thread set. Therefore the final detach can destroy the remaining
// process VADs without racing a later THREAD destructor.
if (!owner->ResourcesReleased) {
if (thread->Teb && thread->TebSize)
MmFreeUserMemoryForProcess(owner, thread->Teb);
if (thread->UserStackBase)
MmFreeUserMemoryForProcess(owner, thread->UserStackBase);
}
thread->Teb = (void*)0;
thread->TebSize = 0;
thread->UserStackBase = (void*)0;
uint32_t remaining = 0;
int detached = 0;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
int fwdDetached = thread->ProcessLink.Fwd == &thread->ProcessLink;
int bwdDetached = thread->ProcessLink.Bwd == &thread->ProcessLink;
if (fwdDetached != bwdDetached) {
KeReleaseSpinlock(&owner->HandleLock, flags);
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread process linkage corrupt: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
(uint64_t)thread,
thread->ThreadId,
(uint64_t)thread->ProcessLink.Fwd,
(uint64_t)thread->ProcessLink.Bwd);
}
if (!fwdDetached) {
if (owner->ThreadCount == 0) {
KeReleaseSpinlock(&owner->HandleLock, flags);
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread process count underflow: thread=0x%016llX tid=%u pid=%u\n",
(uint64_t)thread,
thread->ThreadId,
owner->ProcessId);
}
KeListRemoveChecked(&thread->ProcessLink);
owner->ThreadCount--;
detached = 1;
}
remaining = owner->ThreadCount;
KeReleaseSpinlock(&owner->HandleLock, flags);
if (!detached)
return;
if (remaining == 0) {
// ThreadCount represents live process membership, not retained thread
// handles. Dead THREAD objects may remain waitable after process exit.
_InterlockedExchange(&owner->Terminating, 1);
KeCompleteProcessExitIfLastThread(owner);
}
}
// Implements the thread Destroy operation.
static void KiThreadDestroy(OBJECT* obj) {
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
@@ -44,6 +135,26 @@ static void KiThreadDestroy(OBJECT* obj) {
// wait syscall to run normal cleanup.
KeRundownThreadWait(thread, NTLSTATUS_ALERTED);
if (thread->WaitingMutex || !LIST_IS_EMPTY(&thread->OwnedMutexList)) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread destroyed with mutex state: thread=0x%016llX tid=%u waiting=0x%016llX ownedEmpty=%u\n",
(uint64_t)thread,
thread->ThreadId,
(uint64_t)thread->WaitingMutex,
LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u);
}
if (thread->TimeoutArmed ||
thread->TimeoutLink.Fwd != &thread->TimeoutLink ||
thread->TimeoutLink.Bwd != &thread->TimeoutLink) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread destroyed with timeout state: thread=0x%016llX tid=%u armed=%u link=0x%016llX/0x%016llX\n",
(uint64_t)thread,
thread->ThreadId,
(unsigned)thread->TimeoutArmed,
(uint64_t)thread->TimeoutLink.Fwd,
(uint64_t)thread->TimeoutLink.Bwd);
}
KiRundownApcState(thread);
if (thread->ExitEvent) {
@@ -51,39 +162,28 @@ static void KiThreadDestroy(OBJECT* obj) {
thread->ExitEvent = (EVENT*)0;
}
PROCESS* ownerForVadList = thread->Owner;
if (ownerForVadList) {
if (thread->Teb && thread->TebSize) {
MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
}
if (thread->Teb || thread->TebSize || thread->UserStackBase) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread destroyed with user memory: thread=0x%016llX tid=%u teb=0x%016llX tebSize=%llu stack=0x%016llX\n",
(uint64_t)thread,
thread->ThreadId,
(uint64_t)thread->Teb,
(uint64_t)thread->TebSize,
(uint64_t)thread->UserStackBase);
}
if (thread->UserStackBase) {
MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase);
thread->UserStackBase = (void *)0;
}
if (thread->ProcessLink.Fwd != &thread->ProcessLink ||
thread->ProcessLink.Bwd != &thread->ProcessLink) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread destroyed while still process-linked: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
(uint64_t)thread,
thread->ThreadId,
(uint64_t)thread->ProcessLink.Fwd,
(uint64_t)thread->ProcessLink.Bwd);
}
if (thread->Owner) {
PROCESS* owner = thread->Owner;
uint32_t remaining = 0;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
KeListRemoveChecked(&thread->ProcessLink);
if (owner->ThreadCount) owner->ThreadCount--;
remaining = owner->ThreadCount;
KeReleaseSpinlock(&owner->HandleLock, flags);
if (remaining == 0) {
// The last thread leaving a process is process exit even if user
// code called ExitThread instead of ExitProcess. KeExitProcess /
// KeTerminateProcess set Terminating earlier; this covers the last
// thread voluntarily exiting without an explicit process-exit call.
_InterlockedExchange(&owner->Terminating, 1);
KeCompleteProcessExitIfLastThread(owner);
}
ObDereferenceObject(&owner->Header);
ObDereferenceObject(&thread->Owner->Header);
thread->Owner = (PROCESS *)0;
}
@@ -94,35 +194,51 @@ static void KiThreadDestroy(OBJECT* obj) {
}
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out) {
if (!owner || !out) return NTLSTATUS_INVALID_PARAMETER;
if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) {
if (!owner || !parameters || !out) return NTLSTATUS_INVALID_PARAMETER;
if (parameters->SchedulingClass != KeSchedulingNormal ||
parameters->Priority >= PriorityCount ||
(parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED))
return NTLSTATUS_INVALID_PARAMETER;
uint8_t initiallySuspended =
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!thread) return NTLSTATUS_NO_MEMORY;
thread->Header.Destroy = KiThreadDestroy;
thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
thread->State = ThreadReady;
thread->Priority = priority;
thread->QuantaConsumed = 0;
thread->State = initiallySuspended ? ThreadSuspended : ThreadReady;
thread->BaseSchedulingClass = parameters->SchedulingClass;
thread->BasePriority = parameters->Priority;
thread->EffectiveSchedulingClass = parameters->SchedulingClass;
thread->EffectivePriority = parameters->Priority;
thread->QuantaConsumed = 0;
thread->QuantumTicks = 0;
thread->QuantumRemaining = 0;
thread->CpuBurstTicks = 0;
thread->LastRunTicks = 0;
thread->RunningCpu = KE_INVALID_CPU;
thread->SuspendCount = initiallySuspended ? 1u : 0u;
LIST_INIT(&thread->OwnedMutexList);
thread->WaitingMutex = (MUTEX*)0;
thread->Queued = 0;
thread->IsIdleThread = 0;
thread->SchedulerFlags = 0;
thread->LastCpu = KE_INVALID_CPU;
thread->AffinityMask = KeGetDefaultThreadAffinityMask();
thread->AffinityMask = parameters->AffinityMask
? parameters->AffinityMask
: KeGetDefaultThreadAffinityMask();
LIST_INIT(&thread->ProcessLink);
LIST_INIT(&thread->SchedulerLink);
thread->WaitObject = (OBJECT *)0;
thread->WaitStatus = NTLSTATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
LIST_INIT(&thread->TimeoutLink);
thread->TimeoutArmed = 0;
thread->TimeoutTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
@@ -131,6 +247,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->UserExceptionNoncontinuableMask = 0;
thread->UserExceptionFrameUserAddress = 0;
memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
thread->TerminationDeferralCount = 0;
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
@@ -245,24 +362,58 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
ObReferenceObject(&thread->Header);
*out = thread;
KeReadyThread(thread);
if (!initiallySuspended)
KeReadyThread(thread);
return NTLSTATUS_SUCCESS;
}
// Enters a region that must retain the current thread during process termination.
void KeEnterTerminationDeferral(void) {
KeAssertIrql(PASSIVE_LEVEL);
THREAD* thread = KeGetCurrentThread();
if (!thread || thread->IsIdleThread)
return;
_InterlockedIncrement(&thread->TerminationDeferralCount);
}
// Leaves a process-termination deferral region.
void KeLeaveTerminationDeferral(void) {
KeAssertIrql(PASSIVE_LEVEL);
THREAD* thread = KeGetCurrentThread();
if (!thread || thread->IsIdleThread)
return;
long remaining = _InterlockedDecrement(&thread->TerminationDeferralCount);
if (remaining < 0) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread termination deferral underflow: thread=0x%016llX tid=%u\n",
(uint64_t)thread,
thread->ThreadId);
}
if (remaining == 0 && thread->Owner && KeIsProcessTerminating(thread->Owner))
KiRequestReschedule();
}
// Returns whether one thread still owns termination-sensitive kernel work.
int KeThreadTerminationDeferred(const THREAD* thread) {
return thread && _InterlockedCompareExchange((volatile long*)&thread->TerminationDeferralCount,
0,
0) > 0;
}
// Implements the exit Thread operation.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
(void)code;
if (!thread) return;
KiRundownApcState(thread);
// Wake any pthread_join waiters before the thread is marked dead and
// potentially freed. Manual-reset means all concurrent waiters unblock.
if (thread->ExitEvent)
KeSignalEvent(thread->ExitEvent);
KiMutexRundownThread(thread);
thread->State = ThreadDead;
KeFinalizeThreadExit(thread);
KeYield();
}
@@ -277,10 +428,9 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
// [ksp - 16 - sizeof(KCONTEXT)] KCONTEXT { .Rip = KeKernelThreadEntry }.
extern void KeKernelThreadEntry(void);
static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority,
uint64_t affinityMask,
THREAD** out) {
static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
const KE_THREAD_CREATE_PARAMETERS* parameters,
THREAD** out) {
// Kernel worker threads are owned by the system process. The scheduler's
// hand-built CPU idle thread stays attached to PID 0 (idle), while driver,
// async I/O, and init worker threads are grouped under PID 1 (system).
@@ -289,34 +439,57 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD* currentThread = KeGetCurrentThread();
owner = currentThread ? currentThread->Owner : (PROCESS*)0;
}
if (!owner) return NTLSTATUS_INVALID_PARAMETER;
if (priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
if (!owner || !parameters) return NTLSTATUS_INVALID_PARAMETER;
if (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED)
return NTLSTATUS_INVALID_PARAMETER;
if ((parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) && !out)
return NTLSTATUS_INVALID_PARAMETER;
if (parameters->SchedulingClass == KeSchedulingNormal) {
if (parameters->Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
} else if (parameters->SchedulingClass == KeSchedulingRealtime) {
if (parameters->Priority > KE_REALTIME_PRIORITY_MAX) return NTLSTATUS_INVALID_PARAMETER;
} else {
return NTLSTATUS_INVALID_PARAMETER;
}
uint8_t initiallySuspended =
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!thread) return NTLSTATUS_NO_MEMORY;
thread->Header.Destroy = KiThreadDestroy;
thread->ThreadId = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1);
thread->State = ThreadReady;
thread->Priority = priority;
thread->QuantaConsumed = 0;
thread->State = initiallySuspended ? ThreadSuspended : ThreadReady;
thread->BaseSchedulingClass = parameters->SchedulingClass;
thread->BasePriority = parameters->Priority;
thread->EffectiveSchedulingClass = parameters->SchedulingClass;
thread->EffectivePriority = parameters->Priority;
thread->QuantaConsumed = 0;
thread->QuantumTicks = 0;
thread->QuantumRemaining = 0;
thread->CpuBurstTicks = 0;
thread->LastRunTicks = 0;
thread->RunningCpu = KE_INVALID_CPU;
thread->SuspendCount = initiallySuspended ? 1u : 0u;
LIST_INIT(&thread->OwnedMutexList);
thread->WaitingMutex = (MUTEX*)0;
thread->Queued = 0;
thread->IsIdleThread = 0;
thread->SchedulerFlags = 0;
thread->LastCpu = KE_INVALID_CPU;
thread->AffinityMask = affinityMask ? affinityMask : KeGetDefaultThreadAffinityMask();
thread->AffinityMask = parameters->AffinityMask
? parameters->AffinityMask
: KeGetDefaultThreadAffinityMask();
LIST_INIT(&thread->ProcessLink);
LIST_INIT(&thread->SchedulerLink);
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = NTLSTATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
LIST_INIT(&thread->TimeoutLink);
thread->TimeoutArmed = 0;
thread->TimeoutTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
@@ -325,6 +498,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
thread->UserExceptionNoncontinuableMask = 0;
thread->UserExceptionFrameUserAddress = 0;
memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
thread->TerminationDeferralCount = 0;
thread->UserStackBase = (void*)0; // no user stack
thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx
thread->TebSize = 0;
@@ -371,20 +545,15 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
}
KeInitThreadFpuContext(thread);
KeReadyThread(thread);
if (!initiallySuspended)
KeReadyThread(thread);
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out) {
return KiCreateKernelThreadWithAffinity(fn, arg, priority,
KeGetDefaultThreadAffinityMask(), out);
}
__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, uint64_t affinityMask,
const KE_THREAD_CREATE_PARAMETERS* parameters,
THREAD** out) {
return KiCreateKernelThreadWithAffinity(fn, arg, priority, affinityMask, out);
return KiCreateKernelThread(fn, arg, parameters, out);
}
// Prepares block Thread.
@@ -438,16 +607,6 @@ void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
routine(thread, context, status);
}
// Implements the finish Wait operation.
static void KiFinishWait(THREAD* thread, KSTATUS status) {
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = status;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
}
// Implements the commit Block Thread operation.
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
KeAssertIrql(PASSIVE_LEVEL);
@@ -463,19 +622,14 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
if (thread->State != ThreadWaiting) {
KSTATUS status = thread->WaitStatus;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
return status;
}
// If a user APC was already pending before the thread fully blocked,
// consume the alertable wait immediately instead of going to sleep.
if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
if (thread->WaitListHead)
KeListRemoveChecked(&thread->SchedulerLink);
KiFinishWait(thread, NTLSTATUS_USER_APC);
thread->State = ThreadRunning;
return NTLSTATUS_USER_APC;
if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC))
return NTLSTATUS_USER_APC;
}
KiSchedulerNoteVoluntaryBlock(thread);
@@ -486,8 +640,6 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
KiDeliverKernelApcs();
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
return thread->WaitStatus;
}
@@ -525,8 +677,6 @@ static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* wa
KeListRemoveChecked(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
return 1;
}
@@ -546,10 +696,7 @@ static int KiAbortObjectWait(THREAD* thread) {
}
case ObjectTypeMutex: {
MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList);
KeReleaseSpinlock(&mutex->Lock, flags);
return removed;
return KiMutexAbortWait(mutex, thread);
}
case ObjectTypeSemaphore: {
SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
@@ -569,24 +716,29 @@ static int KiAbortObjectWait(THREAD* thread) {
// unsignaled after the foreground thread has faulted. Object wait-list removal
// must use the object's own lock; otherwise signal/abort races can leave stale
// scheduler links in an event, mutex, or semaphore wait queue.
void KeAbortThreadWait(THREAD* thread, KSTATUS status) {
if (!thread) return;
if (thread->State != ThreadWaiting) return;
int KeAbortThreadWait(THREAD* thread, KSTATUS status) {
if (!thread) return 0;
if (thread->State != ThreadWaiting) return 0;
if (thread->WaitRundownRoutine) {
KeRundownThreadWait(thread, status);
return;
return thread->State != ThreadWaiting;
}
if (KiAbortObjectWait(thread)) {
KeUnblockThreadWithStatus(thread, status);
return;
return 1;
}
// Sleep/timeouts and already-dequeued handoffs are owned by the scheduler.
// This path removes from the scheduler sleep list when present and otherwise
// completes the wait in place.
// A signaler that already dequeued an object waiter owns completion. Let
// that handoff finish rather than returning timeout after ownership or a
// semaphore count has already transferred to this thread.
if (thread->WaitObject)
return 0;
// Delays and auxiliary scheduler waits have no object-owned queue.
KeUnblockThreadFromWaitListWithStatus(thread, status);
return 1;
}
// Implements the unblock Thread operation.
@@ -601,11 +753,9 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
if (thread->State != ThreadWaiting || !thread->AlertableWait)
return NTLSTATUS_PENDING;
if (thread->WaitListHead)
KeListRemoveChecked(&thread->SchedulerLink);
KeUnblockThreadWithStatus(thread, status);
return NTLSTATUS_SUCCESS;
return KeAbortThreadWait(thread, status)
? NTLSTATUS_SUCCESS
: NTLSTATUS_PENDING;
}
// Implements the alert Thread operation.
code/kernel/ke_time.c
+9 -2
View File
@@ -99,7 +99,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out
uint32_t sod = (uint32_t)(epochSeconds % 86400u);
uint32_t year = 1970u;
for (;;) {
while (true) {
uint32_t diy = KiIsLeapYear(year) ? 366u : 365u;
if (days < diy) break;
days -= diy;
@@ -107,7 +107,7 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out
}
uint32_t month = 1u;
for (;;) {
while (true) {
uint32_t dim = KiDaysInMonth(year, month);
if (days < dim) break;
days -= dim;
@@ -519,6 +519,13 @@ __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime) {
return NTLSTATUS_SUCCESS;
}
// Returns the monotonic platform counter in microseconds.
uint64_t KeQueryMonotonicTicks(void) {
return (gTimeHal && gTimeHal->Timer.QueryTicks)
? gTimeHal->Timer.QueryTicks()
: 0;
}
// Returns unix Time Seconds.
__declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void) {
SYS_SYSTEM_TIME t;
code/kernel/ke_work.c
+28 -5
View File
@@ -8,6 +8,7 @@
#include <kernel/ke_work.h>
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
@@ -17,17 +18,31 @@
static SPINLOCK gWorkQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gWorkQueueList;
static EVENT* gWorkQueueEvent;
static KDPC gWorkQueueWakeDpc;
static volatile long gWorkQueueListReady;
static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started.
// Signals the passive worker after an enqueue from hardware interrupt context.
static void KiWorkQueueWakeDpc(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
KeAssertIrql(DISPATCH_LEVEL);
EVENT* event = gWorkQueueEvent;
if (event)
KeSignalEvent(event);
}
static void KiWorkQueueWorker(void* arg) {
(void)arg;
KeAssertIrql(PASSIVE_LEVEL);
for (;;) {
while (true) {
(void)KeWaitForEvent(gWorkQueueEvent, 0);
for (;;) {
while (true) {
KWORK_ITEM* item = (KWORK_ITEM*)0;
uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
if (!LIST_IS_EMPTY(&gWorkQueueList)) {
@@ -79,8 +94,14 @@ bool KeQueueWorkItem(KWORK_ITEM* item) {
EVENT* event = gWorkQueueEvent;
KeReleaseSpinlock(&gWorkQueueLock, flags);
if (event)
KeSignalEvent(event);
if (event) {
// Event signaling can detach waiters, so callers above passive level
// defer the wake until no caller-owned spinlock remains held.
if (KeGetCurrentIrql() == PASSIVE_LEVEL)
KeSignalEvent(event);
else
(void)KeInsertQueueDpc(&gWorkQueueWakeDpc, (void*)0, (void*)0);
}
return true;
}
@@ -95,14 +116,16 @@ KSTATUS KeStartWorkQueue(void) {
}
LIST_INIT(&gWorkQueueList);
KeInitializeDpc(&gWorkQueueWakeDpc, KiWorkQueueWakeDpc, (void*)0);
_InterlockedExchange(&gWorkQueueListReady, 1);
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker,
0,
PriorityNormal,
&parameters,
&worker);
// Persistent system worker; keep the creator reference for kernel life.
(void)worker;
code/kernel/kmain.c
+18 -2
View File
@@ -29,6 +29,7 @@
#include <kernel/ke_input.h>
#include <kernel/ke_time.h>
#include <kernel/ke_display.h>
#include <kernel/ke_audio.h>
#include <kernel/ke_settings.h>
#include <kernel/ke_work.h>
#include <kernel/ke_system_root.h>
@@ -422,7 +423,8 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
// Create the main thread last. KeCreateThread queues it before returning.
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread);
const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, &threadParameters, &thread);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
proc->ImageBase = 0;
@@ -447,6 +449,9 @@ void SystemInitThread(void* arg) {
KiRunEarlyKernelSelfTests();
KiInitializeDrivers(args->bootInfo, args->hal);
if (args->bootInfo->Debug & KE_DEBUG_AUDIO_SELFTEST)
KiRunAudioFinalPathSelfTest();
KeTimeReloadSettings();
KiInitializeSwapFromSettings();
@@ -532,6 +537,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
gKmainHal = hal;
KeInitDebug(hal, bootInfo->Debug);
DbgPrint("CPU: PAT owns WB/WC/UC policy; MTRR default=WB fixed=off variable=off.\n");
KeTimeInitialize(bootInfo, hal);
KeDisplayInitialize(hal);
@@ -566,6 +572,15 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeInitializeScheduler();
KSTATUS volumeNamespaceStatus = IoInitializeVolumeNamespace();
if (!KSUCCESS(volumeNamespaceStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"I/O: failed to initialize volume namespace notifications, status=0x%x\n",
volumeNamespaceStatus);
}
KeAudioInitialize();
KSTATUS workStatus = KeStartWorkQueue();
if (!KSUCCESS(workStatus)) {
KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus);
@@ -585,9 +600,10 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Spawn the Initialization Thread. No creator reference is needed; the
// scheduler owns the live thread lifetime.
const KE_THREAD_CREATE_PARAMETERS initThreadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
(uintptr_t)initArgs,
PriorityNormal, (THREAD**)0);
&initThreadParameters, (THREAD**)0);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st);
code/kernel/ldr_section.c
+11 -1
View File
@@ -327,8 +327,18 @@ static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section,
if (!size)
return NTLSTATUS_SUCCESS;
// Image prototype pages are the canonical cache for executable image bytes.
// Read the backing filesystem directly so the fault path has one cache owner.
if (!section->BackingFile->Fs || !section->BackingFile->Fs->Read)
return NTLSTATUS_NOT_SUPPORTED;
size_t got = 0;
KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got);
KSTATUS s = section->BackingFile->Fs->Read(section->BackingFile->Fs,
section->BackingFile->FileCtx,
fileOffset,
dst,
size,
&got);
if (!KSUCCESS(s))
return s;
if (got != size)
code/kernel/mm_file_cache.c
+50 -377
View File
@@ -16,19 +16,12 @@
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_list.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#define MM_FILE_CACHE_BUCKETS 64u
#define MM_FILE_CACHE_SOFT_MAX_PAGES 512u
#define MM_FILE_CACHE_READAHEAD_MAX_REQUESTS 64u
#define MM_FILE_CACHE_READAHEAD_MAX_PAGES 32u
#define MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET 128u
#define MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES 16u
#define MM_FILE_CACHE_BATCH_MAX_PAGES 16u
#define MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES 64u
#define MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES 128u
#define MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES 192u
#define MM_FILE_CACHE_WRITEBACK_BATCH_PAGES 16u
@@ -48,13 +41,6 @@ typedef struct MM_FILE_CACHE_ENTRY {
uint64_t DirtySequence;
} MM_FILE_CACHE_ENTRY;
typedef struct MM_FILE_CACHE_READAHEAD_REQUEST {
LIST_ENTRY Link;
FILE_OBJECT* File;
uint64_t Offset;
uint32_t Pages;
} MM_FILE_CACHE_READAHEAD_REQUEST;
static SPINLOCK gFileCacheLock = SPINLOCK_INIT;
static MM_FILE_CACHE_ENTRY* gFileCacheBuckets[MM_FILE_CACHE_BUCKETS];
static uint64_t gFileCacheResidentPages;
@@ -65,13 +51,6 @@ static uint64_t gFileCacheMaxDirtyPages;
static uint64_t gFileCacheMaxResidentPages;
static MM_FILE_CACHE_STATS gFileCacheStats;
static SPINLOCK gReadAheadLock = SPINLOCK_INIT;
static LIST_ENTRY gReadAheadQueue;
static SEMAPHORE* gReadAheadSemaphore;
static uint32_t gReadAheadQueuedRequests;
static uint32_t gReadAheadQueuedPages;
static int gReadAheadQueueReady;
static int gReadAheadStartState;
static SPINLOCK gWritebackLock = SPINLOCK_INIT;
static SEMAPHORE* gWritebackSemaphore;
@@ -253,7 +232,8 @@ static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOff
return NTLSTATUS_NOT_FOUND;
}
static void KiFileCacheAddReadAheadStat(uint64_t* field, uint64_t value) {
// Adds to one file-cache statistic under the cache lock.
static void KiFileCacheAddStat(uint64_t* field, uint64_t value) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
*field += value;
KeReleaseSpinlock(&gFileCacheLock, flags);
@@ -265,17 +245,6 @@ static void KiFileCacheSignalWriteback(void) {
KeSignalSemaphore(gWritebackSemaphore);
}
// Updates a maximum counter while the file-cache stats lock is held.
static void KiFileCacheMaxStatLocked(uint64_t* field, uint64_t value) {
if (*field < value)
*field = value;
}
static int KiFileCachePageExistsLocked(FILE_OBJECT* file, uint64_t offset) {
uint32_t bucket = KiFileCacheHash(file, offset);
return KiFileCacheFindLocked(file, offset, bucket) != (MM_FILE_CACHE_ENTRY*)0;
}
// Finds the lowest dirty page for a file, or globally when file is NULL.
static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) {
MM_FILE_CACHE_ENTRY* best = (MM_FILE_CACHE_ENTRY*)0;
@@ -292,115 +261,6 @@ static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file)
return best;
}
static MM_FILE_CACHE_READAHEAD_REQUEST* KiFileCacheReadAheadDequeue(void) {
uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock);
LIST_ENTRY* node = ListPopHead(&gReadAheadQueue);
if (node) {
MM_FILE_CACHE_READAHEAD_REQUEST* req = CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link);
if (gReadAheadQueuedRequests)
gReadAheadQueuedRequests--;
if (gReadAheadQueuedPages >= req->Pages)
gReadAheadQueuedPages -= req->Pages;
else
gReadAheadQueuedPages = 0;
}
KeReleaseSpinlock(&gReadAheadLock, flags);
if (!node)
return (MM_FILE_CACHE_READAHEAD_REQUEST*)0;
return CONTAINING_RECORD(node, MM_FILE_CACHE_READAHEAD_REQUEST, Link);
}
static void KiFileCacheReadAheadWorker(void* arg) {
(void)arg;
KeAssertIrql(PASSIVE_LEVEL);
for (;;) {
KSTATUS ws = KeWaitSemaphore(gReadAheadSemaphore);
if (!KSUCCESS(ws)) {
KeYield();
continue;
}
MM_FILE_CACHE_READAHEAD_REQUEST* req = KiFileCacheReadAheadDequeue();
if (!req)
continue;
for (uint32_t i = 0; i < req->Pages; ++i) {
uint64_t offset = req->Offset + ((uint64_t)i * PAGE_SIZE);
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
if (gFileCacheResidentPages + MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES >=
MM_FILE_CACHE_SOFT_MAX_PAGES) {
gFileCacheStats.ReadAheadPagesThrottled += (uint64_t)(req->Pages - i);
KeReleaseSpinlock(&gFileCacheLock, flags);
break;
}
int exists = KiFileCachePageExistsLocked(req->File, offset);
if (exists)
gFileCacheStats.ReadAheadPagesSkipped++;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (exists)
continue;
MM_FILE_CACHE_PAGE page;
KSTATUS s = MmFileCacheGetPage(req->File, offset, &page);
flags = KeAcquireSpinlock(&gFileCacheLock);
if (KSUCCESS(s))
gFileCacheStats.ReadAheadPagesLoaded++;
else
gFileCacheStats.ReadAheadFailures++;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (!KSUCCESS(s))
break;
MmFileCacheReleasePage(&page);
}
ObDereferenceObject(&req->File->Header);
MmFreePool(req);
}
}
static KSTATUS KiFileCacheEnsureReadAheadStarted(void) {
uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock);
if (gReadAheadStartState == 2) {
KeReleaseSpinlock(&gReadAheadLock, flags);
return NTLSTATUS_SUCCESS;
}
if (gReadAheadStartState == 1) {
KeReleaseSpinlock(&gReadAheadLock, flags);
while (gReadAheadStartState == 1)
KeYield();
return gReadAheadStartState == 2 ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
}
gReadAheadStartState = 1;
KeReleaseSpinlock(&gReadAheadLock, flags);
LIST_INIT(&gReadAheadQueue);
gReadAheadQueuedRequests = 0;
gReadAheadQueuedPages = 0;
gReadAheadQueueReady = 1;
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gReadAheadSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
s = KeCreateKernelThread((uintptr_t)KiFileCacheReadAheadWorker,
0,
PriorityNormal,
&worker);
(void)worker;
}
flags = KeAcquireSpinlock(&gReadAheadLock);
gReadAheadStartState = KSUCCESS(s) ? 2 : 0;
if (!KSUCCESS(s))
gReadAheadQueueReady = 0;
KeReleaseSpinlock(&gReadAheadLock, flags);
return s;
}
// Flushes the next dirty cache page for file, or globally when file is NULL.
static KSTATUS KiFileCacheFlushOneDirtyPage(FILE_OBJECT* file, int* didWork) {
if (didWork) *didWork = 0;
@@ -462,8 +322,8 @@ static void KiFileCacheWritebackWorker(void* arg) {
(void)arg;
KeAssertIrql(PASSIVE_LEVEL);
for (;;) {
KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore);
while (true) {
KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(ws)) {
KeYield();
continue;
@@ -504,9 +364,10 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) {
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gWritebackSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)KiFileCacheWritebackWorker,
0,
PriorityNormal,
&parameters,
&worker);
(void)worker;
}
@@ -517,15 +378,13 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) {
return s;
}
static size_t KiMinSize(size_t a, size_t b) { return a < b ? a : b; }
static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_PAGE* outPage, int waitForLoading) {
if (outPage)
memset(outPage, 0, sizeof(*outPage));
uint32_t bucket = KiFileCacheHash(file, aligned);
int counted = 0;
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
if (!counted) {
gFileCacheStats.Lookups++;
@@ -589,7 +448,7 @@ static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CA
}
static void KiFileCacheKeepBounded(FILE_OBJECT* skipFile, uint64_t skipOffset) {
for (;;) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
int over = gFileCacheResidentPages > MM_FILE_CACHE_SOFT_MAX_PAGES;
KeReleaseSpinlock(&gFileCacheLock, flags);
@@ -718,42 +577,12 @@ static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry,
return NTLSTATUS_SUCCESS;
}
static uint32_t KiFileCacheChooseForegroundBatchPages(void) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
uint64_t resident = gFileCacheResidentPages;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (resident >= MM_FILE_CACHE_SOFT_MAX_PAGES)
return 1;
uint64_t headroom = MM_FILE_CACHE_SOFT_MAX_PAGES - resident;
if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES)
return 1;
if (headroom <= MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES)
return 2;
if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 2u))
return 4;
if (headroom <= (MM_FILE_CACHE_PRESSURE_BATCH_HEADROOM_PAGES * 4u))
return 8;
return MM_FILE_CACHE_BATCH_MAX_PAGES;
}
static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file,
// Loads exactly one demanded file page into the coherent cache.
static KSTATUS KiFileCacheLoadPage(FILE_OBJECT* file,
uint64_t aligned,
uint32_t maxPages,
MM_FILE_CACHE_PAGE* requestedPage,
uint32_t* loadedPagesOut) {
if (loadedPagesOut)
*loadedPagesOut = 0;
if (!file || maxPages == 0)
MM_FILE_CACHE_PAGE* outPage) {
if (!file || !outPage)
return NTLSTATUS_INVALID_PARAMETER;
if (maxPages > MM_FILE_CACHE_BATCH_MAX_PAGES)
maxPages = MM_FILE_CACHE_BATCH_MAX_PAGES;
uintptr_t statFlags = KeAcquireSpinlock(&gFileCacheLock);
gFileCacheStats.BatchPagesRequested += maxPages;
KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesRequested, maxPages);
KeReleaseSpinlock(&gFileCacheLock, statFlags);
uintptr_t fileFlags = KeAcquireSpinlock(&file->Lock);
uint64_t fileSize = file->Size;
@@ -761,134 +590,55 @@ static KSTATUS KiFileCacheLoadSpan(FILE_OBJECT* file,
if (aligned >= fileSize)
return NTLSTATUS_SUCCESS;
MM_FILE_CACHE_ENTRY* entries[MM_FILE_CACHE_BATCH_MAX_PAGES];
PHYSICAL_ADDRESS frames[MM_FILE_CACHE_BATCH_MAX_PAGES];
memset(entries, 0, sizeof(entries));
memset(frames, 0, sizeof(frames));
MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)0;
KSTATUS s = KiFileCacheReserveLoadingPage(file, aligned, &entry);
if (s == NTLSTATUS_ALREADY_EXISTS)
return KiFileCacheTryHit(file, aligned, outPage, 1);
if (!KSUCCESS(s))
return s;
uint32_t pages = 0;
KSTATUS reserveStatus = NTLSTATUS_SUCCESS;
while (pages < maxPages) {
uint64_t pageOffset = aligned + ((uint64_t)pages * PAGE_SIZE);
if (pageOffset >= fileSize)
break;
reserveStatus = KiFileCacheReserveLoadingPage(file, pageOffset, &entries[pages]);
if (!KSUCCESS(reserveStatus))
break;
pages++;
}
if (pages == 0) {
if (requestedPage)
return KiFileCacheTryHit(file, aligned, requestedPage, 1);
return reserveStatus == NTLSTATUS_ALREADY_EXISTS ? NTLSTATUS_ALREADY_EXISTS : reserveStatus;
}
uint8_t* readBuffer = (uint8_t*)MmAllocatePoolTag((size_t)pages * PAGE_SIZE, MM_POOL_TAG('f', 'c', 'b', 'l'));
if (!readBuffer) {
for (uint32_t i = 0; i < pages; ++i)
KiFileCacheCancelLoadingPage(entries[i]);
KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1);
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseCleanFileCache);
if (!frame) {
KiFileCacheCancelLoadingPage(entry);
KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
return NTLSTATUS_NO_MEMORY;
}
memset(readBuffer, 0, (size_t)pages * PAGE_SIZE);
for (uint32_t i = 0; i < pages; ++i) {
frames[i] = MmPfnAllocate(MmPfnUseCleanFileCache);
if (!frames[i]) {
for (uint32_t j = 0; j < i; ++j)
(void)MmPfnRelease(frames[j]);
for (uint32_t j = 0; j < pages; ++j)
KiFileCacheCancelLoadingPage(entries[j]);
MmFreePool(readBuffer);
KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1);
return NTLSTATUS_NO_MEMORY;
}
void* page = MmPfnToVirtual(frame);
if (!page) {
(void)MmPfnRelease(frame);
KiFileCacheCancelLoadingPage(entry);
KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
return NTLSTATUS_FAILURE;
}
memset(page, 0, PAGE_SIZE);
size_t toRead = PAGE_SIZE;
uint64_t remaining = fileSize - aligned;
if (remaining < PAGE_SIZE)
toRead = (size_t)remaining;
size_t toRead = KiMinSize((size_t)pages * PAGE_SIZE, (size_t)(fileSize - aligned));
size_t got = 0;
KSTATUS s = NTLSTATUS_NOT_SUPPORTED;
s = NTLSTATUS_NOT_SUPPORTED;
if (file->Fs && file->Fs->Read)
s = file->Fs->Read(file->Fs, file->FileCtx, aligned, readBuffer, toRead, &got);
s = file->Fs->Read(file->Fs, file->FileCtx, aligned, page, toRead, &got);
if (!KSUCCESS(s)) {
for (uint32_t i = 0; i < pages; ++i) {
if (frames[i])
(void)MmPfnRelease(frames[i]);
KiFileCacheCancelLoadingPage(entries[i]);
}
MmFreePool(readBuffer);
KiFileCacheAddReadAheadStat(&gFileCacheStats.LoadFailures, 1);
(void)MmPfnRelease(frame);
KiFileCacheCancelLoadingPage(entry);
KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
return s;
}
if (got > toRead)
got = toRead;
if (got < PAGE_SIZE)
memset((uint8_t*)page + got, 0, PAGE_SIZE - got);
uint64_t published = 0;
KSTATUS requestedStatus = NTLSTATUS_NOT_FOUND;
for (uint32_t i = 0; i < pages; ++i) {
uint64_t pageOffset = aligned + ((uint64_t)i * PAGE_SIZE);
size_t pageBase = (size_t)i * PAGE_SIZE;
if (pageBase >= got && pageOffset >= fileSize) {
(void)MmPfnRelease(frames[i]);
frames[i] = 0;
KiFileCacheCancelLoadingPage(entries[i]);
continue;
}
uint32_t validBytes = 0;
if (pageBase < got) {
size_t avail = got - pageBase;
if (avail > PAGE_SIZE) avail = PAGE_SIZE;
validBytes = (uint32_t)avail;
}
void* page = MmPfnToVirtual(frames[i]);
if (!page) {
(void)MmPfnRelease(frames[i]);
frames[i] = 0;
KiFileCacheCancelLoadingPage(entries[i]);
continue;
}
memcpy(page, readBuffer + pageBase, validBytes);
if (validBytes < PAGE_SIZE)
memset((uint8_t*)page + validBytes, 0, PAGE_SIZE - validBytes);
MM_FILE_CACHE_PAGE* out = (pageOffset == aligned) ? requestedPage : (MM_FILE_CACHE_PAGE*)0;
KSTATUS ps = KiFileCacheCompleteLoadingPage(entries[i], frames[i], validBytes, out, &published);
if (KSUCCESS(ps)) {
if (pageOffset == aligned)
requestedStatus = NTLSTATUS_SUCCESS;
frames[i] = 0;
} else {
if (pageOffset == aligned)
requestedStatus = ps;
frames[i] = 0;
}
}
MmFreePool(readBuffer);
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
if (published) {
gFileCacheStats.BatchReads++;
gFileCacheStats.BatchPagesLoaded += published;
KiFileCacheMaxStatLocked(&gFileCacheStats.MaxBatchPagesLoaded, published);
}
KeReleaseSpinlock(&gFileCacheLock, flags);
if (loadedPagesOut)
*loadedPagesOut = (uint32_t)published;
s = KiFileCacheCompleteLoadingPage(entry, frame, (uint32_t)got, outPage, (uint64_t*)0);
if (!KSUCCESS(s))
return s;
KiFileCacheKeepBounded(file, aligned);
if (requestedPage) {
if (KSUCCESS(requestedStatus))
return NTLSTATUS_SUCCESS;
return KiFileCacheTryHit(file, aligned, requestedPage, 1);
}
return published ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND;
return NTLSTATUS_SUCCESS;
}
// Returns a retained cache page suitable for in-place buffered writes.
@@ -906,7 +656,7 @@ static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file,
return NTLSTATUS_SUCCESS;
if (needExistingBytes && aligned < oldFileSize) {
s = KiFileCacheLoadSpan(file, aligned, 1, outPage, (uint32_t*)0);
s = KiFileCacheLoadPage(file, aligned, outPage);
if (KSUCCESS(s))
return NTLSTATUS_SUCCESS;
if (s != NTLSTATUS_NOT_FOUND)
@@ -1063,7 +813,7 @@ KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file) {
KeReleaseSpinlock(&gFileCacheLock, flags);
KSTATUS firstError = NTLSTATUS_SUCCESS;
for (;;) {
while (true) {
int didWork = 0;
KSTATUS s = KiFileCacheFlushOneDirtyPage(file, &didWork);
if (!didWork)
@@ -1090,7 +840,7 @@ KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAG
KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1);
if (KSUCCESS(s))
return NTLSTATUS_SUCCESS;
return KiFileCacheLoadSpan(file, aligned, KiFileCacheChooseForegroundBatchPages(), outPage, (uint32_t*)0);
return KiFileCacheLoadPage(file, aligned, outPage);
}
void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page) {
@@ -1111,7 +861,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) {
}
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
for (;;) {
while (true) {
MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
@@ -1145,7 +895,7 @@ void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) {
void MmFileCacheInvalidateAll(FILE_OBJECT* file) {
if (!file) return;
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
for (;;) {
while (true) {
MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
@@ -1179,83 +929,6 @@ KSTATUS MmFileCacheDropOnePageForPressure(void) {
return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0);
}
KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages) {
if (!file || pages == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait())
return NTLSTATUS_INVALID_IRQL;
uint32_t requestedPages = pages;
if (pages > MM_FILE_CACHE_READAHEAD_MAX_PAGES)
pages = MM_FILE_CACHE_READAHEAD_MAX_PAGES;
uintptr_t cacheFlags = KeAcquireSpinlock(&gFileCacheLock);
gFileCacheStats.ReadAheadPagesRequested += requestedPages;
KiFileCacheMaxStatLocked(&gFileCacheStats.MaxReadAheadPagesPerRequest, requestedPages);
uint64_t resident = gFileCacheResidentPages;
uint64_t headroom = resident < MM_FILE_CACHE_SOFT_MAX_PAGES
? MM_FILE_CACHE_SOFT_MAX_PAGES - resident
: 0;
if (headroom <= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES) {
gFileCacheStats.ReadAheadPagesThrottled += pages;
KeReleaseSpinlock(&gFileCacheLock, cacheFlags);
return NTLSTATUS_QUEUE_FULL;
}
headroom -= MM_FILE_CACHE_READAHEAD_MIN_HEADROOM_PAGES;
if ((uint64_t)pages > headroom)
pages = (uint32_t)headroom;
KeReleaseSpinlock(&gFileCacheLock, cacheFlags);
if (pages == 0)
return NTLSTATUS_QUEUE_FULL;
KSTATUS s = KiFileCacheEnsureReadAheadStarted();
if (!KSUCCESS(s)) {
KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1);
return s;
}
MM_FILE_CACHE_READAHEAD_REQUEST* req =
(MM_FILE_CACHE_READAHEAD_REQUEST*)MmAllocatePoolTag(sizeof(*req), MM_POOL_TAG('r', 'd', 'a', 'h'));
if (!req) {
KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadFailures, 1);
return NTLSTATUS_NO_MEMORY;
}
LIST_INIT(&req->Link);
req->File = file;
req->Offset = offset & ~(uint64_t)(PAGE_SIZE - 1);
req->Pages = pages;
ObReferenceObject(&file->Header);
uintptr_t flags = KeAcquireSpinlock(&gReadAheadLock);
if (!gReadAheadQueueReady ||
gReadAheadQueuedRequests >= MM_FILE_CACHE_READAHEAD_MAX_REQUESTS ||
gReadAheadQueuedPages >= MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET) {
KeReleaseSpinlock(&gReadAheadLock, flags);
ObDereferenceObject(&file->Header);
MmFreePool(req);
KiFileCacheAddReadAheadStat(&gFileCacheStats.ReadAheadPagesThrottled, pages);
return NTLSTATUS_QUEUE_FULL;
}
uint32_t queueHeadroom = MM_FILE_CACHE_READAHEAD_QUEUE_PAGE_BUDGET - gReadAheadQueuedPages;
if (pages > queueHeadroom)
pages = queueHeadroom;
req->Pages = pages;
ListInsertTail(&gReadAheadQueue, &req->Link);
gReadAheadQueuedRequests++;
gReadAheadQueuedPages += pages;
KeReleaseSpinlock(&gReadAheadLock, flags);
flags = KeAcquireSpinlock(&gFileCacheLock);
gFileCacheStats.ReadAheadRequests++;
gFileCacheStats.ReadAheadPagesQueued += pages;
KeReleaseSpinlock(&gFileCacheLock, flags);
KeSignalSemaphore(gReadAheadSemaphore);
return NTLSTATUS_SUCCESS;
}
void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
code/kernel/mm_phys.c
+85 -18
View File
@@ -472,8 +472,13 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS min
return (PHYSICAL_ADDRESS)0;
}
// Allocates a physically contiguous run for DMA or hardware structures.
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) {
// Allocates a physically contiguous run inside an address and alignment window.
PHYSICAL_ADDRESS MmPfnAllocateContiguousInRange(size_t count,
PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive,
size_t alignment,
size_t boundary,
MM_PFN_USAGE usage) {
if (!count || count > gMaxPfn) {
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
gPfnFailedContiguousAllocations++;
@@ -482,6 +487,50 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_
return (PHYSICAL_ADDRESS)0;
}
if (!alignment)
alignment = PAGE_SIZE;
if (alignment < PAGE_SIZE)
alignment = PAGE_SIZE;
if ((alignment & (alignment - 1)) != 0 ||
(boundary && (boundary & (boundary - 1)) != 0)) {
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
gPfnFailedContiguousAllocations++;
gPfnFailedContiguousInvalidRequest++;
KeReleaseSpinlock(&gPfnLock, flags);
return (PHYSICAL_ADDRESS)0;
}
if (count > ((size_t)-1) / PAGE_SIZE) {
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
gPfnFailedContiguousAllocations++;
gPfnFailedContiguousInvalidRequest++;
KeReleaseSpinlock(&gPfnLock, flags);
return (PHYSICAL_ADDRESS)0;
}
size_t byteCount = count * PAGE_SIZE;
if (boundary && byteCount > boundary) {
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
gPfnFailedContiguousAllocations++;
gPfnFailedContiguousInvalidRequest++;
KeReleaseSpinlock(&gPfnLock, flags);
return (PHYSICAL_ADDRESS)0;
}
minInclusive = KiAlignUp(minInclusive);
maxExclusive = KiAlignDown(maxExclusive);
PHYSICAL_ADDRESS managedLimit = (PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE;
if (maxExclusive > managedLimit)
maxExclusive = managedLimit;
if (maxExclusive <= minInclusive ||
(PHYSICAL_ADDRESS)byteCount > maxExclusive - minInclusive) {
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
gPfnFailedContiguousAllocations++;
gPfnFailedContiguousInvalidRequest++;
KeReleaseSpinlock(&gPfnLock, flags);
return (PHYSICAL_ADDRESS)0;
}
if (!KiPfnUsageMayUseReserve(usage))
KiPfnReclaimForPressure();
@@ -493,23 +542,31 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_
return (PHYSICAL_ADDRESS)0;
}
size_t runStart = 0;
size_t runLength = 0;
for (size_t pfn = 1; pfn < gMaxPfn; pfn++) {
if (KiPfnAvailableLocked(pfn)) {
if (!runLength)
runStart = pfn;
runLength++;
if (runLength == count) {
for (size_t i = 0; i < count; i++)
KiClaimPfnLocked(runStart + i, usage);
KeReleaseSpinlock(&gPfnLock, flags);
return MmPfnAddress(runStart);
}
} else {
runLength = 0;
size_t firstPfn = (size_t)(minInclusive / PAGE_SIZE);
size_t limitPfn = (size_t)(maxExclusive / PAGE_SIZE);
for (size_t runStart = firstPfn; runStart + count <= limitPfn; ++runStart) {
PHYSICAL_ADDRESS runAddress = MmPfnAddress(runStart);
if ((runAddress & (alignment - 1)) != 0)
continue;
if (boundary) {
PHYSICAL_ADDRESS runEnd = runAddress + (PHYSICAL_ADDRESS)byteCount - 1;
if ((runAddress & ~((PHYSICAL_ADDRESS)boundary - 1)) !=
(runEnd & ~((PHYSICAL_ADDRESS)boundary - 1)))
continue;
}
size_t available = 0;
while (available < count && KiPfnAvailableLocked(runStart + available))
available++;
if (available != count) {
runStart += available;
continue;
}
for (size_t i = 0; i < count; ++i)
KiClaimPfnLocked(runStart + i, usage);
KeReleaseSpinlock(&gPfnLock, flags);
return runAddress;
}
gPfnFailedContiguousAllocations++;
@@ -518,6 +575,16 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_
return (PHYSICAL_ADDRESS)0;
}
// Allocates a physically contiguous run for DMA or hardware structures.
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage) {
return MmPfnAllocateContiguousInRange(count,
PAGE_SIZE,
(PHYSICAL_ADDRESS)gMaxPfn * PAGE_SIZE,
PAGE_SIZE,
0,
usage);
}
// Increments the reference count on an allocated frame.
__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame) {
size_t pfn = MmPfnIndex(frame);
code/kernel/mm_pte.c
+16
View File
@@ -60,6 +60,14 @@ static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose
return purpose;
}
// Returns whether flags select one canonical PAT cache policy.
static bool KiPteCacheFlagsValid(uint64_t flags) {
uint64_t cacheFlags = flags & PAGE_CACHE_MASK;
return cacheFlags == PAGE_CACHE_WB ||
cacheFlags == PAGE_CACHE_WC ||
cacheFlags == PAGE_CACHE_UC;
}
// Ensures that one parent entry points to a child page table.
// New child tables are zeroed before publication so no stale mappings become visible.
static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
@@ -200,6 +208,10 @@ int MmPteHandleShootdownNmi(void) {
// Maps or replaces one 4 KiB leaf PTE in the shared page tables.
__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) {
KiPteStatInc(&gPteStats.MapCalls);
if (!KiPteCacheFlagsValid(flags)) {
KiPteStatInc(&gPteStats.MapFailures);
return NTLSTATUS_INVALID_PARAMETER;
}
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
@@ -286,6 +298,10 @@ __declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
// Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) {
KiPteStatInc(&gPteStats.ProtectCalls);
if (!KiPteCacheFlagsValid(flags)) {
KiPteStatInc(&gPteStats.ProtectFailures);
return NTLSTATUS_INVALID_PARAMETER;
}
if (!size) return NTLSTATUS_SUCCESS;
// The protection operation is page-granular even when callers pass an interior address.
code/kernel/mm_virt.c
+132 -5
View File
@@ -7,6 +7,7 @@
// FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings.
// FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map.
// FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys.
// FFFFA00000000000-FFFFAFFFFFFFFFFF is the kernel shared-view arena.
// FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool.
//
// The current VM model uses one shared PML4, so user pointers are globally
@@ -112,6 +113,12 @@ typedef struct MM_VAD_PAGE {
uint32_t SwapSlot;
} MM_VAD_PAGE;
struct MM_KERNEL_VIEW {
LIST_ENTRY Link;
uintptr_t Base;
size_t PageCount;
};
typedef struct MM_VAD {
LIST_ENTRY ProcessLink;
LIST_ENTRY GlobalLink;
@@ -143,13 +150,19 @@ static LIST_ENTRY gVadList;
static SPINLOCK gVadLock = SPINLOCK_INIT;
static int gVadListReady = 0;
static MM_VAD_STATS gVadStats;
static LIST_ENTRY gKernelViewList;
static SPINLOCK gKernelViewLock = SPINLOCK_INIT;
#define MM_VAD_ARENA_BASE 0x0000100000000000ULL
#define MM_VAD_ARENA_BASE 0x0000100000000000ULL
#define MM_KERNEL_VIEW_BASE 0xFFFFA00000000000ULL
#define MM_KERNEL_VIEW_LIMIT 0xFFFFB00000000000ULL
// Initializes the VAD list after the shared page table and kernel pool are ready.
static void KiVadInitialize(void) {
LIST_INIT(&gVadList);
LIST_INIT(&gKernelViewList);
gVadLock = (SPINLOCK)SPINLOCK_INIT;
gKernelViewLock = (SPINLOCK)SPINLOCK_INIT;
memset(&gVadStats, 0, sizeof(gVadStats));
gVadListReady = 1;
}
@@ -170,6 +183,108 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
(void)MmPfnPin(gVadZeroPageFrame);
}
// Inserts a kernel view before one list node while preserving address order.
static void KiKernelViewInsertBefore(LIST_ENTRY* before, MM_KERNEL_VIEW* view) {
view->Link.Fwd = before;
view->Link.Bwd = before->Bwd;
before->Bwd->Fwd = &view->Link;
before->Bwd = &view->Link;
}
// Maps existing frames into the kernel shared-view arena.
KSTATUS MmCreateKernelViewFromFrames(const PHYSICAL_ADDRESS* frames,
size_t pageCount,
MM_KERNEL_VIEW** outView) {
if (!outView)
return NTLSTATUS_INVALID_PARAMETER;
*outView = (MM_KERNEL_VIEW*)0;
if (!frames || pageCount == 0 || pageCount > (SIZE_MAX / PAGE_SIZE))
return NTLSTATUS_INVALID_PARAMETER;
size_t viewBytes = pageCount * PAGE_SIZE;
MM_KERNEL_VIEW* view = (MM_KERNEL_VIEW*)MmAllocatePoolTag(sizeof(MM_KERNEL_VIEW),
MM_POOL_TAG('k', 'v', 'i', 'w'));
if (!view)
return NTLSTATUS_NO_MEMORY;
memset(view, 0, sizeof(*view));
LIST_INIT(&view->Link);
uintptr_t candidate = MM_KERNEL_VIEW_BASE;
LIST_ENTRY* before = &gKernelViewList;
uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock);
for (LIST_ENTRY* it = gKernelViewList.Fwd; it != &gKernelViewList; it = it->Fwd) {
MM_KERNEL_VIEW* existing = CONTAINING_RECORD(it, MM_KERNEL_VIEW, Link);
if (candidate <= existing->Base &&
viewBytes <= (size_t)(existing->Base - candidate)) {
before = it;
break;
}
uintptr_t existingEnd = existing->Base + existing->PageCount * PAGE_SIZE;
if (existingEnd > candidate)
candidate = existingEnd;
}
if (candidate > MM_KERNEL_VIEW_LIMIT || viewBytes > MM_KERNEL_VIEW_LIMIT - candidate) {
KeReleaseSpinlock(&gKernelViewLock, lockFlags);
MmFreePool(view);
return NTLSTATUS_NO_MEMORY;
}
view->Base = candidate;
view->PageCount = pageCount;
KiKernelViewInsertBefore(before, view);
KeReleaseSpinlock(&gKernelViewLock, lockFlags);
const uint64_t hardwareFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_NX;
size_t mappedPages = 0;
KSTATUS s = NTLSTATUS_SUCCESS;
for (; mappedPages < pageCount; mappedPages++) {
if (!frames[mappedPages]) {
s = NTLSTATUS_INVALID_PARAMETER;
break;
}
s = MmPteMapPage(candidate + mappedPages * PAGE_SIZE, frames[mappedPages],
hardwareFlags, MmPteMapFailIfPresent);
if (!KSUCCESS(s))
break;
}
if (!KSUCCESS(s)) {
while (mappedPages != 0) {
mappedPages--;
(void)MmPteUnmapPage(candidate + mappedPages * PAGE_SIZE);
}
lockFlags = KeAcquireSpinlock(&gKernelViewLock);
ListRemove(&view->Link);
KeReleaseSpinlock(&gKernelViewLock, lockFlags);
MmFreePool(view);
return s;
}
*outView = view;
return NTLSTATUS_SUCCESS;
}
// Returns the base address of one kernel shared view.
void* MmKernelViewBase(const MM_KERNEL_VIEW* view) {
return view ? (void*)view->Base : (void*)0;
}
// Unmaps and releases one kernel shared view without releasing its frames.
void MmDestroyKernelView(MM_KERNEL_VIEW* view) {
if (!view)
return;
for (size_t i = 0; i < view->PageCount; i++)
(void)MmPteUnmapPage(view->Base + i * PAGE_SIZE);
uintptr_t lockFlags = KeAcquireSpinlock(&gKernelViewLock);
ListRemove(&view->Link);
KeReleaseSpinlock(&gKernelViewLock, lockFlags);
MmFreePool(view);
}
// VAD utility helpers.
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
@@ -199,7 +314,7 @@ static KSTATUS KiFindVadRange(size_t reservationSize, size_t alignment, uintptr_
uintptr_t candidate = KiAlignUpPtr(MM_VAD_ARENA_BASE, alignment);
if (!candidate) return NTLSTATUS_INVALID_PARAMETER;
for (;;) {
while (true) {
if (candidate < MM_VAD_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate))
return NTLSTATUS_NO_MEMORY;
@@ -438,7 +553,7 @@ static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) {
static int KiVadTryBeginPaging(MM_VAD_PAGE* page) {
if (!page) return 0;
for (;;) {
while (true) {
uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0);
if (oldFlags & MM_VAD_PAGE_PAGING)
return 0;
@@ -454,7 +569,7 @@ static int KiVadTryBeginPaging(MM_VAD_PAGE* page) {
static void KiVadEndPaging(MM_VAD_PAGE* page) {
if (!page) return;
for (;;) {
while (true) {
uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0);
uint32_t newFlags = oldFlags & ~MM_VAD_PAGE_PAGING;
if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags,
@@ -1157,6 +1272,18 @@ __declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process,
return NTLSTATUS_SUCCESS;
}
// Returns the page count held by a locked user buffer.
size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer) {
return buffer ? buffer->PageCount : 0;
}
// Returns one pinned frame held by a locked user buffer.
PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index) {
if (!buffer || index >= buffer->PageCount)
return (PHYSICAL_ADDRESS)0;
return buffer->Pages[index].Frame;
}
// Unlocks a user buffer locked for asynchronous kernel access.
__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer) {
if (!buffer)
@@ -2256,7 +2383,7 @@ void MmVadDestroyAllForProcess(PROCESS* process) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process) return;
for (;;) {
while (true) {
uintptr_t procFlags = KeAcquireSpinlock(&process->VadLock);
LIST_ENTRY* node = (LIST_ENTRY*)0;
int sawLocked = 0;
code/kernel/net_socket.c
+4 -3
View File
@@ -379,8 +379,8 @@ static void NetAsyncSocketExecute(NET_ASYNC_SOCKET_REQUEST_OBJECT* req,
static void NetAsyncSocketWorker(void* arg) {
(void)arg;
for (;;) {
KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore);
while (true) {
KSTATUS waitStatus = KeWaitSemaphore(gAsyncSocketSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(waitStatus)) {
KeYield();
continue;
@@ -426,9 +426,10 @@ static KSTATUS NetEnsureAsyncSocketWorkerStarted(void) {
KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncSocketSemaphore);
if (KSUCCESS(status)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
status = KeCreateKernelThread((uintptr_t)NetAsyncSocketWorker,
0,
PriorityNormal,
&parameters,
&worker);
(void)worker;
}
code/kernel/ob_object.c
+3 -1
View File
@@ -31,6 +31,8 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' ');
case ObjectTypeModule: return MM_POOL_TAG('m', 'o', 'd', 'u');
case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k');
case ObjectTypeAudioEndpoint: return MM_POOL_TAG('a', 'e', 'p', 't');
case ObjectTypeAudioStream: return MM_POOL_TAG('a', 's', 't', 'r');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}
@@ -64,7 +66,7 @@ __declspec(dllexport) bool ObTryReferenceObject(OBJECT* object) {
if (!object)
return false;
for (;;) {
while (true) {
if (_InterlockedCompareExchange(&object->LifetimeState, 0, 0) != ObjectAlive)
return false;
if ((_InterlockedCompareExchange(&object->Flags, 0, 0) & OBJECT_FLAG_INSERTED) == 0)
code/public/ntlite/audio.h
+220
View File
@@ -0,0 +1,220 @@
// Public native render-stream syscall ABI declarations.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
#define SYS_AUDIO_ABI_VERSION 3u
#define SYS_AUDIO_MAX_ENDPOINTS 32u
#define SYS_AUDIO_STREAM_CAPACITY_FRAMES 4096u
#define SYS_AUDIO_GAIN_Q16_UNITY 65536u
#define SYS_AUDIO_PAN_Q16_LEFT (-65536)
#define SYS_AUDIO_PAN_Q16_CENTER 0
#define SYS_AUDIO_PAN_Q16_RIGHT 65536
// Identifies one logical render destination.
typedef enum SYS_AUDIO_ENDPOINT_KIND {
SysAudioEndpointLineOut = 1,
SysAudioEndpointSpeakerHeadphone,
SysAudioEndpointOtherAnalog
} SYS_AUDIO_ENDPOINT_KIND;
// Identifies the interleaved PCM sample representation accepted by native streams.
typedef enum SYS_AUDIO_SAMPLE_FORMAT {
SysAudioSampleS16 = 1
} SYS_AUDIO_SAMPLE_FORMAT;
// Describes one interleaved PCM stream format.
typedef struct SYS_AUDIO_FORMAT {
uint32_t SampleRate;
uint16_t ChannelCount;
uint16_t SampleFormat;
} SYS_AUDIO_FORMAT;
#define SYS_AUDIO_ENDPOINT_FLAG_AVAILABLE 0x00000001u
#define SYS_AUDIO_ENDPOINT_FLAG_DEFAULT 0x00000002u
#define SYS_AUDIO_ENDPOINT_FLAG_VALID_MASK 0x00000003u
#define SYS_AUDIO_ENDPOINT_CAP_JACK_SENSE 0x00000001u
#define SYS_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN 0x00000002u
#define SYS_AUDIO_ENDPOINT_CAP_VALID_MASK 0x00000003u
// Identifies the lifecycle state reported for one render endpoint.
typedef enum SYS_AUDIO_ENDPOINT_STATE {
SysAudioEndpointStarting = 1,
SysAudioEndpointReady,
SysAudioEndpointStopping,
SysAudioEndpointFailed,
SysAudioEndpointRemoved
} SYS_AUDIO_ENDPOINT_STATE;
// Reports one endpoint snapshot keyed by its stable kernel identifier.
typedef struct SYS_AUDIO_ENDPOINT_INFO {
uint32_t Size;
uint32_t Version;
uint64_t EndpointId;
uint32_t Kind;
uint32_t Flags;
uint32_t Capabilities;
uint32_t State;
SYS_AUDIO_FORMAT HardwareFormat;
uint32_t PeriodCount;
uint32_t FramesPerPeriod;
uint32_t MasterVolumeQ16;
uint32_t MasterMuted;
KSTATUS FailureStatus;
uint32_t Reserved;
} SYS_AUDIO_ENDPOINT_INFO;
// Selects the endpoint array used by one enumeration call.
typedef struct SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST {
uint32_t Size;
uint32_t Flags;
SYS_AUDIO_ENDPOINT_INFO* Endpoints;
uint32_t Capacity;
uint32_t Reserved;
uint32_t* CountOut;
} SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST;
// Selects one endpoint and source format for a new render stream handle.
typedef struct SYS_AUDIO_OPEN_STREAM_REQUEST {
uint32_t Size;
uint32_t Flags;
uint64_t EndpointId;
SYS_AUDIO_FORMAT Format;
uint32_t Reserved;
uint32_t* StreamHandleOut;
} SYS_AUDIO_OPEN_STREAM_REQUEST;
// Describes one bounded nonblocking stream write.
typedef struct SYS_AUDIO_WRITE_REQUEST {
uint32_t Size;
uint32_t Flags;
uint32_t StreamHandle;
uint32_t FrameCount;
const void* Frames;
uint32_t* WrittenFramesOut;
} SYS_AUDIO_WRITE_REQUEST;
#define SYS_AUDIO_WAIT_WRITABLE 0x00000001u
#define SYS_AUDIO_WAIT_CONSUMED 0x00000002u
#define SYS_AUDIO_WAIT_PRESENTED 0x00000004u
#define SYS_AUDIO_WAIT_STOPPED 0x00000008u
#define SYS_AUDIO_WAIT_DRAINED 0x00000010u
#define SYS_AUDIO_WAIT_FAILED 0x00000020u
#define SYS_AUDIO_WAIT_DISCONTINUITY 0x00000040u
#define SYS_AUDIO_WAIT_VALID_MASK 0x0000007Fu
// Arms the stream handle event for one or more progress conditions.
typedef struct SYS_AUDIO_ARM_WAIT_REQUEST {
uint32_t Size;
uint32_t Flags;
uint32_t StreamHandle;
uint32_t Mask;
// Frame counters are inclusive absolute targets and generations are change baselines.
uint32_t WritableFrames;
uint32_t Reserved;
uint64_t ConsumedFrames;
uint64_t PresentedFrames;
uint64_t DrainedGeneration;
uint64_t DiscontinuityGeneration;
} SYS_AUDIO_ARM_WAIT_REQUEST;
// Identifies the public lifecycle state of one native render stream.
typedef enum SYS_AUDIO_STREAM_STATE {
SysAudioStreamCreated = 1,
SysAudioStreamRunning,
SysAudioStreamPaused,
SysAudioStreamDraining,
SysAudioStreamStopped,
SysAudioStreamFailed
} SYS_AUDIO_STREAM_STATE;
// Reports one coherent stream snapshot.
typedef struct SYS_AUDIO_STREAM_INFO {
uint32_t Size;
uint32_t Version;
uint32_t State;
uint32_t Reserved;
SYS_AUDIO_FORMAT Format;
uint32_t CapacityFrames;
uint32_t BufferedFrames;
uint32_t WritableFrames;
KSTATUS FailureStatus;
uint64_t FramesSubmitted;
uint64_t FramesConsumed;
uint64_t FramesPresented;
uint64_t FramesDiscarded;
uint64_t OutputFramesMixed;
uint64_t SilenceFramesInserted;
// Underruns count recoverable source starvation periods.
uint64_t Underruns;
uint64_t DrainedGeneration;
// Discontinuities report explicit control or hardware timeline changes.
uint64_t DiscontinuityGeneration;
uint32_t VolumeQ16;
int32_t PanQ16;
uint8_t Muted;
uint8_t ReservedBytes[7];
} SYS_AUDIO_STREAM_INFO;
// Updates one stream's software gain, balance, and mute targets.
// Volume uses unsigned Q16 in [0, 1] and pan uses signed Q16 in [-1, 1].
typedef struct SYS_AUDIO_SET_VOLUME_REQUEST {
uint32_t Size;
uint32_t Flags;
uint32_t StreamHandle;
uint32_t Muted;
uint32_t VolumeQ16;
int32_t PanQ16;
} SYS_AUDIO_SET_VOLUME_REQUEST;
// Updates one endpoint's master software gain and mute state.
typedef struct SYS_AUDIO_SET_MASTER_VOLUME_REQUEST {
uint32_t Size;
uint32_t Flags;
uint64_t EndpointId;
uint32_t Muted;
uint32_t VolumeQ16;
} SYS_AUDIO_SET_MASTER_VOLUME_REQUEST;
// Enumerates render endpoints and returns the complete count even when capacity is insufficient.
KSTATUS SysAudioEnumerateEndpoints(const SYS_AUDIO_ENUMERATE_ENDPOINTS_REQUEST* request);
// Opens one waitable, process-owned native render stream handle.
KSTATUS SysAudioOpenStream(const SYS_AUDIO_OPEN_STREAM_REQUEST* request);
// Copies as many complete frames as currently fit and returns WOULD_BLOCK when none fit.
KSTATUS SysAudioWrite(const SYS_AUDIO_WRITE_REQUEST* request);
// Replaces the any-condition progress targets that drive one stream handle's wait state.
KSTATUS SysAudioArmWait(const SYS_AUDIO_ARM_WAIT_REQUEST* request);
// Copies one coherent stream snapshot after the caller initializes infoOut->Size.
KSTATUS SysAudioQueryStream(uint32_t streamHandle, SYS_AUDIO_STREAM_INFO* infoOut);
// Starts one created or stopped stream.
KSTATUS SysAudioStart(uint32_t streamHandle);
// Pauses future mixing while preserving unconsumed source frames.
KSTATUS SysAudioPause(uint32_t streamHandle);
// Resumes one paused stream.
KSTATUS SysAudioResume(uint32_t streamHandle);
// Rejects new writes and stops after all submitted frames reach a terminal disposition.
KSTATUS SysAudioDrain(uint32_t streamHandle);
// Stops future mixing and discards source frames not yet committed to hardware periods.
KSTATUS SysAudioStop(uint32_t streamHandle);
// Returns one stopped stream to an empty position-zero state.
KSTATUS SysAudioReset(uint32_t streamHandle);
// Updates one stream's software volume, pan, and mute state.
KSTATUS SysAudioSetVolume(const SYS_AUDIO_SET_VOLUME_REQUEST* request);
// Updates one endpoint's master software volume and mute state.
KSTATUS SysAudioSetMasterVolume(const SYS_AUDIO_SET_MASTER_VOLUME_REQUEST* request);
code/public/ntlite/ddi/display.h
+7 -1
View File
@@ -25,7 +25,13 @@ typedef struct DISPLAY_DEVICE_OPS {
KSTATUS (*Blit)(DISPLAY_DEVICE* device,
int x, int y, int w, int h,
const uint32_t* pixels);
const uint32_t* pixels,
uint32_t pitchBytes);
KSTATUS (*PresentPitched)(DISPLAY_DEVICE* device,
int x, int y, int w, int h,
const uint32_t* pixels,
uint32_t pitchBytes);
KSTATUS (*FillRect)(DISPLAY_DEVICE* device,
int x, int y, int w, int h,
code/public/ntlite/ddi/io_driver.h
+3 -11
View File
@@ -20,6 +20,7 @@ typedef void (*DRIVER_UNLOAD_FN)(void);
typedef struct DRIVER_OBJECT DRIVER_OBJECT;
typedef struct DEVICE_OBJECT DEVICE_OBJECT;
typedef struct IRP IRP;
struct IO_DMA_ADAPTER;
typedef KSTATUS (*PDRIVER_DISPATCH)(DEVICE_OBJECT* device, IRP* irp);
typedef KSTATUS (*PDRIVER_ADD_DEVICE)(DRIVER_OBJECT* driver,
@@ -89,6 +90,7 @@ struct DEVICE_OBJECT {
DEVICE_OBJECT* LowerDevice; // device directly below this FDO/filter
void* DeviceExtension;
struct IO_DMA_ADAPTER* DmaAdapter;
uint32_t Flags;
uint32_t Characteristics;
@@ -183,17 +185,6 @@ typedef struct IO_DISK_STATS {
uint64_t WriteSectors;
uint64_t MaxReadSectors;
uint64_t MaxWriteSectors;
uint64_t ReadaheadHits;
uint64_t ReadaheadMisses;
uint64_t ReadaheadStores;
uint64_t ReadaheadInvalidations;
uint64_t ReadaheadBytesServed;
uint64_t ReadaheadSectorsLoaded;
uint64_t ReadaheadSequentialDetections;
uint64_t ReadaheadSuppressed;
uint64_t ReadaheadAllocFailures;
uint64_t ReadaheadReadFailures;
uint64_t MaxReadaheadSectors;
} IO_DISK_STATS;
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
@@ -220,4 +211,5 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
const void* buffer);
__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex);
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out);
code/public/ntlite/ddk/audio.h
+105
View File
@@ -0,0 +1,105 @@
// Public nt-lite kernel-audio provider declarations.
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <ntlite/status.h>
struct DEVICE_OBJECT;
typedef struct IO_AUDIO_ENDPOINT IO_AUDIO_ENDPOINT;
// Identifies one logical render destination rather than one codec pin.
typedef enum IO_AUDIO_ENDPOINT_KIND {
IoAudioEndpointLineOut = 1,
IoAudioEndpointSpeakerHeadphone,
IoAudioEndpointOtherAnalog
} IO_AUDIO_ENDPOINT_KIND;
// Identifies the canonical interleaved PCM sample representation.
typedef enum IO_AUDIO_SAMPLE_FORMAT {
IoAudioSampleS16 = 1
} IO_AUDIO_SAMPLE_FORMAT;
// Describes interleaved PCM frames.
typedef struct IO_AUDIO_FORMAT {
uint32_t SampleRate;
uint16_t ChannelCount;
uint16_t SampleFormat;
} IO_AUDIO_FORMAT;
#define IO_AUDIO_ENDPOINT_CAP_JACK_SENSE 0x00000001u
#define IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN 0x00000002u
#define IO_AUDIO_ENDPOINT_CAP_VALID_MASK 0x00000003u
// Bounds provider-advertised hardware gain ranges in millidecibels.
#define IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB 200000
// Distinguishes orderly teardown from surprise removal where registers may no longer be accessible.
typedef enum IO_AUDIO_REMOVAL_MODE {
IoAudioOrderlyRemoval = 1,
IoAudioSurpriseRemoval
} IO_AUDIO_REMOVAL_MODE;
#define IO_AUDIO_FAILURE_HARDWARE_INACCESSIBLE 0x00000001u
#define IO_AUDIO_FAILURE_VALID_MASK 0x00000001u
// Provider operations are invoked only at PASSIVE_LEVEL by the endpoint mixer thread.
typedef struct IO_AUDIO_PROVIDER_OPERATIONS {
// Starts DMA only after the ring contains at least two prepared periods.
// Failure must leave the provider stopped; Stop may still be called for partial-start cleanup.
KSTATUS (*Start)(void* providerContext);
// Stops DMA before unregister permits provider storage to be released.
void (*Stop)(void* providerContext);
// Applies endpoint gain and mute only with IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN.
KSTATUS (*SetGain)(void* providerContext, int32_t gainMilliDb, uint8_t muted);
} IO_AUDIO_PROVIDER_OPERATIONS;
// Registers one fixed-format render ring owned by a hardware provider.
// The provider must keep DMA stopped and the physical output muted until Start is called.
typedef struct IO_AUDIO_ENDPOINT_REGISTRATION {
struct DEVICE_OBJECT* Device;
IO_AUDIO_ENDPOINT_KIND Kind;
uint32_t Capabilities;
uint8_t InitiallyAvailable;
void* ProviderContext;
IO_AUDIO_PROVIDER_OPERATIONS Operations;
void* PeriodBuffer;
size_t PeriodBufferBytes;
uint32_t PeriodCount;
uint32_t FramesPerPeriod;
size_t PeriodStrideBytes;
IO_AUDIO_FORMAT HardwareFormat;
// Gain fields must be zero unless IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN is present.
int32_t HardwareGainMinimumMilliDb;
int32_t HardwareGainMaximumMilliDb;
uint32_t HardwareGainStepMilliDb;
} IO_AUDIO_ENDPOINT_REGISTRATION;
// Driver-visible services for registering and advancing kernel audio endpoints.
typedef struct NTL_DDK_AUDIO {
// Registers one endpoint and returns a provider-owned reference valid until unregister returns.
KSTATUS (*RegisterEndpoint)(const IO_AUDIO_ENDPOINT_REGISTRATION* registration,
IO_AUDIO_ENDPOINT** outEndpoint);
// Stops the endpoint, removes it from enumeration, and consumes the provider-owned reference.
void (*UnregisterEndpoint)(IO_AUDIO_ENDPOINT* endpoint, IO_AUDIO_REMOVAL_MODE mode);
// Reports one completed hardware period from ISR/DPC completion processing.
// presentedFrames, hardwarePositionFrames, and interruptCount must be monotonic for the
// endpoint lifetime. timestampTicks uses the DDK monotonic microsecond source.
void (*NotifyPeriod)(IO_AUDIO_ENDPOINT* endpoint,
uint32_t completedPeriodIndex,
uint64_t presentedFrames,
uint64_t hardwarePositionFrames,
uint64_t interruptCount,
uint64_t timestampTicks);
// Updates jack-derived endpoint availability without changing endpoint identity.
void (*SetEndpointAvailable)(IO_AUDIO_ENDPOINT* endpoint, uint8_t available);
// Fails an endpoint and wakes every stream owner.
void (*ReportEndpointFailure)(IO_AUDIO_ENDPOINT* endpoint,
KSTATUS status,
uint32_t flags);
} NTL_DDK_AUDIO;
code/public/ntlite/ddk/ddk.h
+5 -1
View File
@@ -4,7 +4,9 @@
#include <stdint.h>
#include <ntlite/ddk/async.h>
#include <ntlite/ddk/audio.h>
#include <ntlite/ddk/debug.h>
#include <ntlite/ddk/dma.h>
#include <ntlite/ddk/display.h>
#include <ntlite/ddk/dpc.h>
#include <ntlite/ddk/driver.h>
@@ -25,12 +27,14 @@
#include <ntlite/ddk/time.h>
#include <ntlite/ddk/timer.h>
#define NTL_DDK_VERSION 2u
#define NTL_DDK_VERSION 1u
// Root DDK table passed to kernel drivers by the driver loader.
typedef struct NTL_DDK {
uint32_t Version;
const NTL_DDK_DEBUG* Debug;
const NTL_DDK_AUDIO* Audio;
const NTL_DDK_DMA* Dma;
const NTL_DDK_INTERRUPT* Interrupt;
const NTL_DDK_MEMORY* Memory;
const NTL_DDK_PLATFORM* Platform;
code/public/ntlite/ddk/debug.h
+18
View File
@@ -2,11 +2,29 @@
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
#define KE_DEBUG_CONSOLE 0x00000001u
#define KE_DEBUG_SELFTEST 0x00000002u
#define KE_DEBUG_FILE_LOG 0x00000004u
#define KE_DEBUG_POOL_CANARY 0x00000008u
#define KE_DEBUG_POOL_POISON 0x00000010u
#define KE_DEBUG_POOL_QUARANTINE 0x00000020u
#define KE_DEBUG_POOL_CALLERS 0x00000040u
#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u
#define KE_DEBUG_DISPLAY_TRACE 0x00000100u
#define KE_DEBUG_INPUT_TRACE 0x00000200u
#define KE_DEBUG_VM_ASSERTS 0x00000400u
#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u
#define KE_DEBUG_LIST_CHECKS 0x00001000u
#define KE_DEBUG_AUDIO 0x00002000u
#define KE_DEBUG_AUDIO_SELFTEST 0x00004000u
// Driver-visible debug output service table passed through NTL_DDK.
typedef struct NTL_DDK_DEBUG {
void (*Print)(const char* fmt, ...);
uint32_t (*GetFlags)(void);
} NTL_DDK_DEBUG;
// Current fatal categories used by kernel-mode driver diagnostics.
code/public/ntlite/ddk/display.h
+4 -3
View File
@@ -21,6 +21,8 @@ typedef struct KE_DISPLAY_INFO {
DISPLAY_MODE Mode;
uint32_t Caps;
uint32_t DeviceCount;
uint32_t Flags;
uint32_t Reserved;
DISPLAY_STATS ManagerStats;
DISPLAY_STATS DeviceStats;
} KE_DISPLAY_INFO;
@@ -40,10 +42,9 @@ DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode);
// Queries the current display size.
DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight);
typedef KSTATUS (*KE_DISPLAY_ROW_SOURCE)(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount);
DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels);
DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch);
DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
DISPLAY_API KSTATUS KeDisplayPresentPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color);
DISPLAY_API KSTATUS KeDisplayClear(uint32_t color);
DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h);
code/public/ntlite/ddk/dma.h
+146
View File
@@ -0,0 +1,146 @@
// Public nt-lite DMA declarations.
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <ntlite/status.h>
struct DEVICE_OBJECT;
struct PROCESS;
typedef struct IO_DMA_ADAPTER IO_DMA_ADAPTER;
typedef struct IO_DMA_BUFFER IO_DMA_BUFFER;
typedef struct IO_DMA_MAPPING IO_DMA_MAPPING;
typedef struct MM_PINNED_MEMORY MM_PINNED_MEMORY;
typedef enum IO_DMA_DIRECTION {
IoDmaToDevice = 1,
IoDmaFromDevice,
IoDmaBidirectional
} IO_DMA_DIRECTION;
// Fails a streaming mapping instead of allocating a bounce buffer.
#define IO_DMA_MAP_NO_BOUNCE 0x00000001u
// Describes immutable constraints shared by one device adapter. Address,
// alignment, and boundary limits apply to common buffers. Segment limits apply
// to streaming mappings. Zero selects the unconstrained default for each field.
typedef struct IO_DMA_REQUIREMENTS {
uint64_t MaximumDeviceAddress;
size_t Alignment;
size_t Boundary;
size_t MaximumSegmentLength;
uint32_t MaximumSegments;
} IO_DMA_REQUIREMENTS;
// Describes one device-visible span in a streaming mapping.
typedef struct IO_DMA_SEGMENT {
uint64_t DeviceAddress;
size_t Length;
} IO_DMA_SEGMENT;
// Reports cumulative activity and live ownership for validation.
typedef struct IO_DMA_STATS {
uint64_t AdaptersCreated;
uint64_t AdaptersDestroyed;
uint64_t AdaptersInUse;
uint64_t CommonBuffersAllocated;
uint64_t CommonBuffersFreed;
uint64_t CommonBuffersInUse;
uint64_t CommonBufferBytesInUse;
uint64_t PinnedKernelRanges;
uint64_t PinnedUserRanges;
uint64_t PinnedRangesReleased;
uint64_t PinnedRangesInUse;
uint64_t PinnedPagesInUse;
uint64_t StreamingMappings;
uint64_t StreamingMappingsUnmapped;
uint64_t StreamingMappingsInUse;
uint64_t DirectMappings;
uint64_t BounceMappings;
uint64_t BounceBytesCopiedToDevice;
uint64_t BounceBytesCopiedToCpu;
uint64_t FailedAllocations;
uint64_t FailedMappings;
} IO_DMA_STATS;
// Driver-visible DMA services. Device addresses are intentionally distinct
// from physical addresses even while the first implementation maps them one to
// one on x86-64.
typedef struct NTL_DDK_DMA {
// Returns the one adapter and requirement set owned by a device object.
KSTATUS (*GetAdapter)(struct DEVICE_OBJECT* device,
const IO_DMA_REQUIREMENTS* requirements,
IO_DMA_ADAPTER** outAdapter);
// Releases one caller-owned adapter reference.
void (*PutAdapter)(IO_DMA_ADAPTER* adapter);
// Allocates coherent memory with stable CPU and base device addresses. The
// caller must quiesce every device reference before releasing the buffer.
KSTATUS (*AllocateCommonBuffer)(IO_DMA_ADAPTER* adapter,
size_t size,
size_t alignment,
IO_DMA_BUFFER** outBuffer);
// Releases a coherent common buffer.
void (*FreeCommonBuffer)(IO_DMA_BUFFER* buffer);
// Returns the stable CPU address of a common buffer.
void* (*GetCommonBufferAddress)(const IO_DMA_BUFFER* buffer);
// Returns the stable device address of a common buffer.
uint64_t (*GetCommonBufferDeviceAddress)(const IO_DMA_BUFFER* buffer);
// Returns the caller-requested size of a common buffer.
size_t (*GetCommonBufferSize)(const IO_DMA_BUFFER* buffer);
// Stabilizes a kernel range for one or more streaming mappings. Pinning
// preserves the pages, but does not transfer ownership to a device.
KSTATUS (*PinKernelMemory)(void* address,
size_t size,
IO_DMA_DIRECTION direction,
MM_PINNED_MEMORY** outMemory);
// Stabilizes a process-owned user range for one or more mappings. The
// descriptor retains the process and range until the final unpin.
KSTATUS (*PinUserMemory)(struct PROCESS* process,
void* address,
size_t size,
IO_DMA_DIRECTION direction,
MM_PINNED_MEMORY** outMemory);
// Releases one pinned-memory reference after all mappings leave.
void (*UnpinMemory)(MM_PINNED_MEMORY* memory);
// Maps a pinned subrange and creates a constrained segment list. A new
// mapping begins CPU-owned and requires external operation serialization.
KSTATUS (*MapPinnedMemory)(IO_DMA_ADAPTER* adapter,
MM_PINNED_MEMORY* memory,
size_t offset,
size_t length,
IO_DMA_DIRECTION direction,
uint32_t flags,
IO_DMA_MAPPING** outMapping);
// Releases a mapping and performs a final receive-side bounce copy only
// when ownership was transferred to the device and not returned to the CPU.
void (*UnmapMemory)(IO_DMA_MAPPING* mapping);
// Transfers mapping ownership from the CPU to the device. Streaming DMA
// must not begin before this operation succeeds.
KSTATUS (*SynchronizeForDevice)(IO_DMA_MAPPING* mapping);
// Transfers mapping ownership from the device to the CPU. CPU readers must
// not consume receive data before this operation succeeds.
KSTATUS (*SynchronizeForCpu)(IO_DMA_MAPPING* mapping);
// Returns the immutable segment count owned by a mapping.
uint32_t (*GetSegmentCount)(const IO_DMA_MAPPING* mapping);
// Returns the immutable segment array owned by a mapping.
const IO_DMA_SEGMENT* (*GetSegments)(const IO_DMA_MAPPING* mapping);
// Copies current DMA counters into caller-owned storage.
void (*QueryStats)(IO_DMA_STATS* outStats);
} NTL_DDK_DMA;
code/public/ntlite/ddk/file.h
+1 -4
View File
@@ -59,6 +59,7 @@ typedef struct IO_FILESYSTEM {
KSTATUS (*QueryInfo)(struct IO_FILESYSTEM* fs, void* fileCtx, SYS_FILE_STAT* out);
KSTATUS (*Stat)(struct IO_FILESYSTEM* fs, SYS_STRING_VIEW path, SYS_FILE_STAT* out);
KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx, IO_FILE_IDENTITY* out);
KSTATUS (*Flush)(struct IO_FILESYSTEM* fs, void* fileCtx);
void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx);
void (*QueryStats)(struct IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out);
void* DriverCtx;
@@ -84,10 +85,6 @@ typedef struct FILE_OBJECT {
uint64_t Size;
uint64_t Offset;
uint32_t Flags;
uint32_t ReadaheadScore;
uint64_t ReadaheadNextOffset;
uint64_t ReadaheadIssuedUntil;
uint64_t ReadaheadSuppressedUntil;
} FILE_OBJECT;
typedef struct DIR_OBJECT {
code/public/ntlite/ddk/irp.h
+10 -9
View File
@@ -23,11 +23,12 @@ typedef enum IO_MAJOR_FUNCTION {
IRP_MJ_CLOSE = 1,
IRP_MJ_READ = 2,
IRP_MJ_WRITE = 3,
IRP_MJ_DEVICE_CONTROL = 4,
IRP_MJ_INTERNAL_DEVICE_CONTROL = 5,
IRP_MJ_PNP = 6,
IRP_MJ_POWER = 7,
IRP_MJ_MAXIMUM_FUNCTION = 8,
IRP_MJ_FLUSH_BUFFERS = 4,
IRP_MJ_DEVICE_CONTROL = 5,
IRP_MJ_INTERNAL_DEVICE_CONTROL = 6,
IRP_MJ_PNP = 7,
IRP_MJ_POWER = 8,
IRP_MJ_MAXIMUM_FUNCTION = 9,
} IO_MAJOR_FUNCTION;
// PnP minor functions.
@@ -159,10 +160,10 @@ __declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp);
__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q);
// Cancels every IRP in a queue and invokes a callback for each one.
__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context);
__declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q,
KSTATUS status,
IRP_QUEUE_CANCEL_CALLBACK callback,
void* context);
// Cancels every IRP in a queue.
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status);
code/public/ntlite/ddk/memory.h
+18 -112
View File
@@ -14,12 +14,14 @@
#define PAGE_MASK (~(PAGE_SIZE - 1ULL))
#endif
#define NTL_DDK_PAGE_SIZE PAGE_SIZE
#define NTL_DDK_PAGE_PRESENT (1ULL << 0)
#define NTL_DDK_PAGE_WRITE (1ULL << 1)
#define NTL_DDK_PAGE_WRITETHROUGH (1ULL << 3)
#define NTL_DDK_PAGE_NOCACHE (1ULL << 4)
#define NTL_DDK_PAGE_NX (1ULL << 63)
#define NTL_DDK_PAGE_SIZE PAGE_SIZE
#define NTL_DDK_PAGE_PRESENT (1ULL << 0)
#define NTL_DDK_PAGE_WRITE (1ULL << 1)
#define NTL_DDK_PAGE_CACHE_WB 0ULL
#define NTL_DDK_PAGE_CACHE_WC (1ULL << 3)
#define NTL_DDK_PAGE_CACHE_UC (1ULL << 4)
#define NTL_DDK_PAGE_CACHE_MASK (NTL_DDK_PAGE_CACHE_WC | NTL_DDK_PAGE_CACHE_UC)
#define NTL_DDK_PAGE_NX (1ULL << 63)
#ifndef PAGE_PRESENT
#define PAGE_PRESENT (1ULL << 0)
@@ -30,11 +32,17 @@
#ifndef PAGE_USER
#define PAGE_USER (1ULL << 2)
#endif
#ifndef PAGE_WRITETHROUGH
#define PAGE_WRITETHROUGH (1ULL << 3)
#ifndef PAGE_CACHE_WB
#define PAGE_CACHE_WB 0ULL
#endif
#ifndef PAGE_NOCACHE
#define PAGE_NOCACHE (1ULL << 4)
#ifndef PAGE_CACHE_WC
#define PAGE_CACHE_WC (1ULL << 3)
#endif
#ifndef PAGE_CACHE_UC
#define PAGE_CACHE_UC (1ULL << 4)
#endif
#ifndef PAGE_CACHE_MASK
#define PAGE_CACHE_MASK (PAGE_CACHE_WC | PAGE_CACHE_UC)
#endif
#ifndef PAGE_ACCESSED
#define PAGE_ACCESSED (1ULL << 5)
@@ -56,55 +64,6 @@ typedef enum NTL_DDK_MAP_MODE {
NtlDdkMapReplaceExisting = 1,
} NTL_DDK_MAP_MODE;
#ifndef NTL_MM_PTE_MAP_MODE_DEFINED
typedef enum MM_PTE_MAP_MODE {
MmPteMapFailIfPresent = 0,
MmPteMapReplaceExisting = 1,
} MM_PTE_MAP_MODE;
#define NTL_MM_PTE_MAP_MODE_DEFINED 1
#endif
#ifndef NTL_MM_PFN_TYPES_DEFINED
typedef enum MM_PFN_STATE {
MmPfnStateInvalid = 0,
MmPfnStateReserved,
MmPfnStateFree,
MmPfnStateZeroed,
MmPfnStateActive,
MmPfnStateStandby,
MmPfnStateModified,
} MM_PFN_STATE;
typedef enum MM_PFN_USAGE {
MmPfnUseNone = 0,
MmPfnUseKernel,
MmPfnUsePageTable,
MmPfnUseUser,
MmPfnUseCleanFileCache,
MmPfnUseImage,
MmPfnUseMappedFile,
MmPfnUseAnonymous,
MmPfnUseCow,
MmPfnUseSurface,
MmPfnUseDriver,
MmPfnUseDma,
MmPfnUseBootloader,
MmPfnUsePfnDatabase,
MmPfnUseMmio,
} MM_PFN_USAGE;
typedef struct MM_PFN {
uint16_t State;
uint16_t Usage;
uint16_t RefCount;
uint16_t PinCount;
uint32_t Flags;
uint32_t MappedCount;
void* Owner;
uint64_t BackingPage;
} MM_PFN;
#define NTL_MM_PFN_TYPES_DEFINED 1
#endif
#define MM_POOL_TAG(a, b, c, d) \
((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
@@ -131,56 +90,3 @@ __declspec(dllexport) void MmFreePool(void* ptr);
// Reallocates kernel pool memory.
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// Allocates one physical frame for a specific driver-visible usage.
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage);
// Allocates one physical frame below maxExclusive.
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage);
// Allocates one physical frame in a range.
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive,
MM_PFN_USAGE usage);
// Allocates contiguous physical frames.
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage);
// Releases a frame reference.
__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame);
// Retains a physical frame.
__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame);
// Releases a retained physical frame.
__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame);
// Pins a physical frame.
__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame);
// Unpins a physical frame.
__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame);
// Returns the PFN database entry for a frame.
__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame);
// Returns the direct-map alias for a physical frame.
__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame);
// Maps one page into the shared kernel address space.
__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt,
PHYSICAL_ADDRESS phys,
uint64_t flags,
MM_PTE_MAP_MODE mode);
// Removes one page mapping from the shared kernel address space.
__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt);
// Replaces flags on a mapped range.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags);
// Translates a virtual address through the shared kernel page tables.
__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt);
// Returns the physical address and flags for a mapped page.
__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
code/public/ntlite/ddk/mutex.h
+12 -5
View File
@@ -13,16 +13,23 @@ typedef struct MUTEX {
SPINLOCK Lock;
THREAD* Owner;
LIST_ENTRY WaitList;
LIST_ENTRY OwnerLink;
KE_SCHEDULING_CLASS DonationClass;
uint32_t DonationPriority;
uint8_t HasDonation;
uint8_t Abandoned;
} MUTEX;
// Tests whether a mutex acquisition transferred ownership to the caller.
#define KE_MUTEX_ACQUIRE_SUCCEEDED(status) \
((status) == NTLSTATUS_SUCCESS || (status) == NTLSTATUS_ABANDONED)
// Creates a mutex object.
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX** out);
// Acquires a mutex, blocking when necessary.
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex);
// Attempts to acquire a mutex without blocking.
__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex);
// Acquires a mutex within the requested timeout and reports abandonment on transfer.
// Zero performs a nonblocking probe; KE_WAIT_INFINITE waits without a timeout.
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilliseconds);
// Releases a mutex.
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex);
Loaded 100 of 177 files, more files were not shown because too many files have changed in this diff. Show more