move keyboard and mouse out of HAL into a standalone ps2.sys driver, implement usb stack with xhci controller discovery, hid keyboard/mouse, and bulk-only mass storage, add pci msi support with capability chain walking and software irq vector allocation (slots 16-23), rewrite user virtual memory as a USER_VA_REGION descriptor arena with per-process ownership tracking and guard page support, replacing the old bump allocator and top-down image region, add MmCreateUserRegion/MmDestroyAllUserRegions so process teardown reclaims all user va in one pass, allocate a pool-backed shadow framebuffer in hal_screen to fix cursor tearing and scroll corruption, reserve ap trampoline page below 1mb in uefi bootloader before boot services exit, rewrite memset/memcpy/memmove/memcmp with __stosq/__movsq intrinsics, promote driver flag literals to macros and add multi-tu STANDALONE_DRIVERS table to build.c, split rtl8139/rtl816x into header+source, rename KbReadScancode to KbReadKey, improve page fault output with MmDebugDescribeAddress and user region dump, fix %p to %016llX throughout.
This commit is contained in:
maksw committed
1 parent
b970d7f59c
commit
b838f8d83d
130 files changed
+8137
-2932
No files matched your search
+1
-1
@@ -2,6 +2,6 @@ build.obj
|
||||
bin/
|
||||
obj/
|
||||
qemu.log
|
||||
disk.vhd
|
||||
*.vhd
|
||||
code/build/build.exe
|
||||
code/build/build.exe.old
|
||||
Binary file not shown.
+1
-1
@@ -1,5 +1,5 @@
|
||||
[kernel]
|
||||
; bitmask: 1 = debug prints, 2 = self check
|
||||
debug = 0
|
||||
debug = 1
|
||||
[init]
|
||||
commandline = apps/shell_native.exe
|
||||
+19
-1
@@ -20,6 +20,7 @@
|
||||
#include <ntlite/boot_info.h>
|
||||
#include <ntlite/memory_map.h>
|
||||
#include <common/ini_parser.h>
|
||||
#pragma intrinsic(_disable)
|
||||
|
||||
int _fltused = 0;
|
||||
|
||||
@@ -287,6 +288,21 @@ static void DriversIniCb(const char* section, const char* key,
|
||||
// ---------------------------------------------------------------------------
|
||||
// EFI Memory Map -> MEMORY_MAP conversion
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static EFI_PHYSICAL_ADDRESS AllocateApTrampolinePageBelow1M(void) {
|
||||
// AP startup via SIPI requires the first instruction page to live below
|
||||
// 1 MiB. Some real UEFI firmware exposes no ConventionalMemory below
|
||||
// 640 KiB after ExitBootServices, so reserve a loader-data page here while
|
||||
// boot services can still satisfy AllocateMaxAddress.
|
||||
EFI_PHYSICAL_ADDRESS phys = 0x0009F000ULL;
|
||||
EFI_STATUS st = gBS->AllocatePages(AllocateMaxAddress, EfiLoaderData, 1, &phys);
|
||||
if (EFI_ERROR(st) || phys == 0 || phys >= 0x000A0000ULL || (phys & 0xFFFULL)) {
|
||||
return 0;
|
||||
}
|
||||
gBS->SetMem((VOID*)phys, PAGE_SIZE, 0);
|
||||
return phys;
|
||||
}
|
||||
|
||||
static void ConvertMemoryMap(MEMORY_MAP* out) {
|
||||
UINTN mapSize = 0, descSz = 0, mapKey; UINT32 descVer;
|
||||
gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer);
|
||||
@@ -738,6 +754,8 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
|
||||
|
||||
//Print(L"7\n");
|
||||
|
||||
bi->ApTrampolinePhys = (uintptr_t)AllocateApTrampolinePageBelow1M();
|
||||
|
||||
// --- Map physical RAM as direct window AND identity map ---
|
||||
UINTN mapSize = 0, descSz = 0, mapKey; UINT32 descVer;
|
||||
gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer);
|
||||
@@ -880,7 +898,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
|
||||
}
|
||||
|
||||
// Print it out to verify the bootloader actually found it before launching the kernel
|
||||
//Print(L"Found RSDP at: 0x%llx\n", bi->RsdpPhysical);
|
||||
//Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);
|
||||
|
||||
bi->PhysicalMapBase = PHYS_MAP_BASE;
|
||||
bi->Debug = (UINT32)ps.debug;
|
||||
|
||||
+45
-26
@@ -1,20 +1,48 @@
|
||||
#define NOB_IMPLEMENTATION
|
||||
#include "nob.h"
|
||||
|
||||
// Replicate your compiler and linker flags
|
||||
const char *CFLAGS = "/nologo /W3 /WX /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include /Icode\\kernel\\include";
|
||||
const char *LWIP_CFLAGS = "/nologo /W3 /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include /Icode\\kernel\\include /Icode\\drivers\\net\\lwip\\include /Icode\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706";
|
||||
const char *LFLAGS = "/nologo /NODEFAULTLIB /DEBUG";
|
||||
// Replicate your compiler and linker flags. Keep the literals in macros so
|
||||
// they can be used in file-scope aggregate initializers. MSVC C rejects
|
||||
// initializing a static object with the value of another object, even if that
|
||||
// object is a const char*.
|
||||
#define CFLAGS_LITERAL "/nologo /W3 /WX /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi- /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include /Icode\\kernel\\include"
|
||||
#define LWIP_CFLAGS_LITERAL CFLAGS_LITERAL " /Icode\\drivers\\net\\lwip\\include /Icode\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706"
|
||||
#define LFLAGS_LITERAL "/nologo /NODEFAULTLIB /DEBUG"
|
||||
|
||||
const char *CFLAGS = CFLAGS_LITERAL;
|
||||
const char *LWIP_CFLAGS = LWIP_CFLAGS_LITERAL;
|
||||
const char *LFLAGS = LFLAGS_LITERAL;
|
||||
|
||||
typedef struct Driver_Target {
|
||||
const char *name;
|
||||
const char *obj_dir;
|
||||
const char *sources;
|
||||
const char *cflags;
|
||||
} Driver_Target;
|
||||
|
||||
#define DRIVER_TARGET(name_, obj_dir_, sources_) { (name_), (obj_dir_), (sources_), CFLAGS_LITERAL }
|
||||
#define DRIVER_TARGET_FLAGS(name_, obj_dir_, sources_, cflags_) { (name_), (obj_dir_), (sources_), (cflags_) }
|
||||
|
||||
static const Driver_Target STANDALONE_DRIVERS[] = {
|
||||
DRIVER_TARGET("ps2", "obj\\drivers\\ps2", "code\\drivers\\input\\ps2\\ps2.c code\\drivers\\input\\ps2\\ps2_i8042.c"),
|
||||
DRIVER_TARGET("ide", "obj\\drivers\\ide", "code\\drivers\\storage\\ide\\ide.c code\\drivers\\storage\\ide\\ide_bus.c"),
|
||||
DRIVER_TARGET("ahci", "obj\\drivers\\ahci", "code\\drivers\\storage\\ahci\\ahci.c code\\drivers\\storage\\ahci\\ahci_port.c"),
|
||||
DRIVER_TARGET("fat32", "obj\\drivers\\fat32", "code\\drivers\\fs\\fat32\\fat32.c code\\drivers\\fs\\fat32\\fat32_mount.c"),
|
||||
DRIVER_TARGET("usb", "obj\\drivers\\usb", "code\\drivers\\bus\\usb\\usb.c code\\drivers\\bus\\usb\\usb_core.c code\\drivers\\bus\\usb\\usb_xhci.c code\\drivers\\bus\\usb\\usb_hid.c code\\drivers\\bus\\usb\\usb_storage.c"),
|
||||
DRIVER_TARGET("rtl816x", "obj\\drivers\\rtl816x", "code\\drivers\\net\\rtl816x\\rtl816x.c code\\drivers\\net\\rtl816x\\rtl816x_pci.c"),
|
||||
DRIVER_TARGET("rtl8139", "obj\\drivers\\rtl8139", "code\\drivers\\net\\rtl8139\\rtl8139.c code\\drivers\\net\\rtl8139\\rtl8139_pci.c"),
|
||||
};
|
||||
|
||||
void create_layout() {
|
||||
const char *dirs[] = {
|
||||
"obj", "bin", "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
|
||||
"obj\\syscall", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init",
|
||||
"obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
|
||||
"obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
|
||||
"obj", "bin", "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
|
||||
"obj\\syscall", "obj\\win32", "obj\\posix", "obj\\drivers", "obj\\init",
|
||||
"obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
|
||||
"obj\\async_apc", "obj\\async_cancel", "obj\\edit", "obj\\httpget",
|
||||
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video"
|
||||
};
|
||||
for (size_t i = 0; i < NOB_ARRAY_LEN(dirs); i++) nob_mkdir_if_not_exists(dirs[i]);
|
||||
for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) nob_mkdir_if_not_exists(STANDALONE_DRIVERS[i].obj_dir);
|
||||
}
|
||||
|
||||
// Helper to run a shell command string and wait for it
|
||||
@@ -60,7 +88,7 @@ int main(int argc, char **argv) {
|
||||
"cl /nologo /c /Foobj\\libc.obj code\\common\\libc.c",
|
||||
|
||||
// Bootloader
|
||||
nob_temp_sprintf("cl %s /MP /c /Icode\\boot\\uefi\\include /Foobj\\boot\\ code\\boot\\uefi\\*.c && link %s /SUBSYSTEM:EFI_APPLICATION /ENTRY:UefiMain /OUT:bin\\bootx64.efi code\\boot\\uefi\\efilib.lib obj\\ini_parser.obj obj\\libc.obj obj\\boot\\*.obj", CFLAGS, LFLAGS),
|
||||
nob_temp_sprintf("cl %s /MP /c /Icode\\boot\\uefi\\include /Foobj\\boot\\ code\\boot\\uefi\\*.c && link %s /FORCE:MULTIPLE /SUBSYSTEM:EFI_APPLICATION /ENTRY:UefiMain /OUT:bin\\bootx64.efi obj\\libc.obj obj\\ini_parser.obj code\\boot\\uefi\\efilib.lib obj\\boot\\*.obj", CFLAGS, LFLAGS),
|
||||
|
||||
// HAL
|
||||
nob_temp_sprintf("cl %s /MP /c /Foobj\\hal\\ code\\kernel\\hal\\*.c && link %s /DLL /OUT:bin\\hal.dll /ENTRY:HalInitialize /IGNORE:4216 obj\\printf.obj obj\\libc.obj obj\\hal\\*.obj", CFLAGS, LFLAGS),
|
||||
@@ -82,23 +110,13 @@ int main(int argc, char **argv) {
|
||||
nob_log(NOB_INFO, "--- Compiling Drivers and Apps in Parallel ---");
|
||||
Nob_Procs procs = {0};
|
||||
|
||||
// Queue standalone Drivers
|
||||
const char *drivers[][2] = {
|
||||
{"ide", "storage\\ide"},
|
||||
{"ahci", "storage\\ahci"},
|
||||
{"fat32", "fs\\fat32"},
|
||||
{"usb", "bus\\usb"},
|
||||
{"rtl816x", "net\\rtl816x"},
|
||||
{"rtl8139", "net\\rtl8139"}
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < NOB_ARRAY_LEN(drivers); i++) {
|
||||
const char *name = drivers[i][0];
|
||||
const char *path = drivers[i][1];
|
||||
|
||||
// Queue standalone drivers. Each driver is now a target with one or more
|
||||
// translation units, but still links to one .sys image with DriverEntry.
|
||||
for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) {
|
||||
const Driver_Target *driver = &STANDALONE_DRIVERS[i];
|
||||
const char *cmd = nob_temp_sprintf(
|
||||
"cl %s /c /Foobj\\drivers\\%s.obj code\\drivers\\%s\\%s.c && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj obj\\drivers\\%s.obj",
|
||||
CFLAGS, name, path, name, name, name
|
||||
"cl %s /MP /c /Fo%s\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj %s\\*.obj",
|
||||
driver->cflags, driver->obj_dir, driver->sources, driver->name, driver->obj_dir
|
||||
);
|
||||
nob_da_append(&procs, run_async(cmd));
|
||||
}
|
||||
@@ -179,7 +197,8 @@ int main(int argc, char **argv) {
|
||||
{"bin\\win32.dll", "Z:\\system\\win32.dll"}, {"bin\\posix.dll", "Z:\\system\\posix.dll"},
|
||||
{"code\\settings.ini", "Z:\\system\\settings.ini"}, {"bin\\init.exe", "Z:\\system\\init.exe"},
|
||||
|
||||
{"bin\\ide.sys", "Z:\\drivers\\ide.sys"}, {"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"},
|
||||
{"bin\\ps2.sys", "Z:\\drivers\\ps2.sys"}, {"bin\\ide.sys", "Z:\\drivers\\ide.sys"},
|
||||
{"bin\\ahci.sys", "Z:\\drivers\\ahci.sys"},
|
||||
{"bin\\fat32.sys", "Z:\\drivers\\fat32.sys"}, {"bin\\lwip.sys", "Z:\\drivers\\lwip.sys"},
|
||||
{"bin\\usb.sys", "Z:\\drivers\\usb.sys"}, {"bin\\rtl816x.sys", "Z:\\drivers\\rtl816x.sys"},
|
||||
{"bin\\rtl8139.sys", "Z:\\drivers\\rtl8139.sys"}, {"code\\drivers\\drivers.ini", "Z:\\drivers\\drivers.ini"},
|
||||
|
||||
+97
-8
@@ -1,6 +1,7 @@
|
||||
// everyone needs a little libc
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <intrin.h>
|
||||
|
||||
char* strcpy(char* dest, const char* src) {
|
||||
char* d = dest;
|
||||
@@ -100,22 +101,110 @@ char* strrchr(const char* s, int c) {
|
||||
return (char*)last;
|
||||
}
|
||||
|
||||
void* memset(void* dest, int c, size_t n) {
|
||||
unsigned char* d = (unsigned char*)dest;
|
||||
size_t qwords = n / 8;
|
||||
size_t tail = n % 8;
|
||||
|
||||
if (qwords > 0) {
|
||||
uint64_t value = (unsigned char)c;
|
||||
value |= value << 8;
|
||||
value |= value << 16;
|
||||
value |= value << 32;
|
||||
__stosq((unsigned __int64*)d, value, qwords);
|
||||
d += qwords * 8;
|
||||
}
|
||||
|
||||
if (tail > 0) {
|
||||
__stosb(d, (unsigned char)c, tail);
|
||||
}
|
||||
|
||||
return dest;
|
||||
}
|
||||
|
||||
void* memcpy(void* dest, const void* src, size_t n) {
|
||||
unsigned char* d = (unsigned char*)dest;
|
||||
const unsigned char* s = (const unsigned char*)src;
|
||||
size_t qwords = n / 8;
|
||||
size_t tail = n % 8;
|
||||
|
||||
if (qwords > 0) {
|
||||
__movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords);
|
||||
d += qwords * 8;
|
||||
s += qwords * 8;
|
||||
}
|
||||
|
||||
if (tail > 0) {
|
||||
__movsb(d, s, tail);
|
||||
}
|
||||
|
||||
return dest;
|
||||
}
|
||||
|
||||
void* memmove(void* dest, const void* src, size_t n) {
|
||||
uint8_t* d = (uint8_t *)dest;
|
||||
const uint8_t* s = (const uint8_t *)src;
|
||||
unsigned char* d = (unsigned char*)dest;
|
||||
const unsigned char* s = (const unsigned char*)src;
|
||||
|
||||
if (d < s) {
|
||||
for (size_t i = 0; i < n; i++) d[i] = s[i];
|
||||
// Forward copy: safe to use chunked __movsq + tail
|
||||
size_t qwords = n / 8;
|
||||
size_t tail = n % 8;
|
||||
if (qwords > 0) {
|
||||
__movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords);
|
||||
d += qwords * 8;
|
||||
s += qwords * 8;
|
||||
}
|
||||
if (tail > 0) {
|
||||
__movsb(d, s, tail);
|
||||
}
|
||||
} else if (d > s) {
|
||||
for (size_t i = n; i > 0; i--) d[i - 1] = s[i - 1];
|
||||
// Backward copy: must process from end to avoid corruption
|
||||
size_t qwords = n / 8;
|
||||
size_t tail = n % 8;
|
||||
|
||||
// Copy tail bytes first from the very end
|
||||
if (tail > 0) {
|
||||
for (size_t i = tail; i > 0; i--) {
|
||||
d[qwords * 8 + i - 1] = s[qwords * 8 + i - 1];
|
||||
}
|
||||
}
|
||||
|
||||
// Copy 64-bit chunks backward
|
||||
if (qwords > 0) {
|
||||
unsigned __int64* dq = (unsigned __int64*)d;
|
||||
const unsigned __int64* sq = (const unsigned __int64*)s;
|
||||
for (size_t i = qwords; i > 0; i--) {
|
||||
dq[i - 1] = sq[i - 1];
|
||||
}
|
||||
}
|
||||
}
|
||||
return dest;
|
||||
}
|
||||
|
||||
int memcmp(const void* s1, const void* s2, size_t n) {
|
||||
const uint8_t* a = (const uint8_t *)s1;
|
||||
const uint8_t* b = (const uint8_t *)s2;
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
if (a[i] != b[i]) return a[i] - b[i];
|
||||
const unsigned __int64* a64 = (const unsigned __int64*)s1;
|
||||
const unsigned __int64* b64 = (const unsigned __int64*)s2;
|
||||
size_t qwords = n / 8;
|
||||
size_t tail = n % 8;
|
||||
|
||||
// Compare 8-byte blocks
|
||||
for (size_t i = 0; i < qwords; i++) {
|
||||
if (a64[i] != b64[i]) {
|
||||
// Locate the exact mismatching byte within the quadword
|
||||
const unsigned char* a8 = (const unsigned char*)&a64[i];
|
||||
const unsigned char* b8 = (const unsigned char*)&b64[i];
|
||||
for (size_t j = 0; j < 8; j++) {
|
||||
if (a8[j] != b8[j]) return (int)a8[j] - (int)b8[j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compare remaining tail bytes
|
||||
const unsigned char* a8 = (const unsigned char*)(a64 + qwords);
|
||||
const unsigned char* b8 = (const unsigned char*)(b64 + qwords);
|
||||
for (size_t i = 0; i < tail; i++) {
|
||||
if (a8[i] != b8[i]) return (int)a8[i] - (int)b8[i];
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -1,9 +1,86 @@
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
// usb.c - USB bus driver entry point and PCI controller discovery.
|
||||
#include "usb.h"
|
||||
|
||||
XHCI_CONTROLLER gXhci;
|
||||
bool gHaveXhci = false;
|
||||
|
||||
const char* UsbControllerName(uint8_t progIf) {
|
||||
switch (progIf) {
|
||||
case 0x00: return "UHCI";
|
||||
case 0x10: return "OHCI";
|
||||
case 0x20: return "EHCI";
|
||||
case 0x30: return "xHCI";
|
||||
default: return "USB";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void UsbDiscoverControllers(const HAL_API* hal) {
|
||||
uint32_t found = 0;
|
||||
uint32_t xhciOk = 0;
|
||||
|
||||
for (uint16_t bus = 0; bus < 256; bus++) {
|
||||
for (uint8_t dev = 0; dev < 32; dev++) {
|
||||
for (uint8_t fn = 0; fn < 8; fn++) {
|
||||
uint32_t vd = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_VENDOR_DEVICE);
|
||||
if ((vd & 0xFFFFu) == PCI_VENDOR_INVALID)
|
||||
continue;
|
||||
|
||||
uint32_t classReg = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_CLASS);
|
||||
uint8_t classCode = (uint8_t)(classReg >> 24);
|
||||
uint8_t subclass = (uint8_t)(classReg >> 16);
|
||||
uint8_t progIf = (uint8_t)(classReg >> 8);
|
||||
if (classCode != PCI_CLASS_SERIAL_BUS || subclass != PCI_SUBCLASS_USB)
|
||||
continue;
|
||||
|
||||
USB_BAR bar0;
|
||||
bool hasBar0 = UsbReadBar(hal, (uint8_t)bus, dev, fn, 0, &bar0);
|
||||
uint32_t rawBar0 = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_BAR0);
|
||||
uint32_t rawBar1 = hal->Pci.Read32((uint8_t)bus, dev, fn, PCI_REG_BAR0 + 4u);
|
||||
uint8_t irq = hal->Pci.GetIrqLine((uint8_t)bus, dev, fn);
|
||||
|
||||
DbgPrint("usb: found %s controller at %u:%u.%u vendor=%04x device=%04x irq=%u bar0raw=%08x bar1raw=%08x",
|
||||
UsbControllerName(progIf),
|
||||
bus, dev, fn,
|
||||
vd & 0xFFFFu, vd >> 16,
|
||||
irq, rawBar0, rawBar1);
|
||||
if (hasBar0 && bar0.Kind == UsbBarMmio) {
|
||||
DbgPrint(" mmio=0x%016llX\n", (unsigned long long)bar0.Base);
|
||||
} else if (hasBar0 && bar0.Kind == UsbBarIo) {
|
||||
DbgPrint(" io=0x%016llX\n", (unsigned long long)bar0.Base);
|
||||
} else {
|
||||
DbgPrint(" no-bar0\n");
|
||||
}
|
||||
|
||||
if (progIf == PCI_PROGIF_XHCI) {
|
||||
KSTATUS st = XhciInitialise(hal, (uint8_t)bus, dev, fn,
|
||||
(uint16_t)(vd & 0xFFFFu),
|
||||
(uint16_t)(vd >> 16));
|
||||
if (KSUCCESS(st) || st == STATUS_ALREADY_EXISTS) xhciOk++;
|
||||
else DbgPrint("usb: xHCI init failed st=0x%x\n", st);
|
||||
}
|
||||
found++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!found)
|
||||
DbgPrint("usb: no PCI USB controllers discovered\n");
|
||||
else
|
||||
DbgPrint("usb: discovery complete; controllers=%u xhci_initialised=%u\n", found, xhciOk);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
(void)hal;
|
||||
DbgPrint("Test runtime driver\n");
|
||||
if (!hal) return STATUS_INVALID_PARAMETER;
|
||||
DbgPrint("usb: discovery starting\n");
|
||||
|
||||
KSTATUS st = UsbRegisterClassMatchers();
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: class matcher registration failed st=0x%x\n", st);
|
||||
return st;
|
||||
}
|
||||
|
||||
UsbDiscoverControllers(hal);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
// usb.h - internal USB/xHCI driver interfaces shared across usb.sys translation units.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/ddi/input_device.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
|
||||
#ifndef USB_DEBUG_VERBOSE
|
||||
#define USB_DEBUG_VERBOSE 0
|
||||
#endif
|
||||
|
||||
#if USB_DEBUG_VERBOSE
|
||||
#define UsbTrace DbgPrint
|
||||
#else
|
||||
static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
|
||||
#endif
|
||||
|
||||
|
||||
#define PCI_VENDOR_INVALID 0xFFFFu
|
||||
#define PCI_CLASS_SERIAL_BUS 0x0Cu
|
||||
#define PCI_SUBCLASS_USB 0x03u
|
||||
#define PCI_PROGIF_XHCI 0x30u
|
||||
#define PCI_REG_VENDOR_DEVICE 0x00u
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_CLASS 0x08u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
|
||||
#define XHCI_CAP_CAPLENGTH 0x00u
|
||||
#define XHCI_CAP_HCSPARAMS1 0x04u
|
||||
#define XHCI_CAP_HCSPARAMS2 0x08u
|
||||
#define XHCI_CAP_HCCPARAMS1 0x10u
|
||||
#define XHCI_CAP_DBOFF 0x14u
|
||||
#define XHCI_CAP_RTSOFF 0x18u
|
||||
|
||||
#define XHCI_OP_USBCMD 0x00u
|
||||
#define XHCI_OP_USBSTS 0x04u
|
||||
#define XHCI_OP_PAGESIZE 0x08u
|
||||
#define XHCI_OP_DNCTRL 0x14u
|
||||
#define XHCI_OP_CRCR 0x18u
|
||||
#define XHCI_OP_DCBAAP 0x30u
|
||||
#define XHCI_OP_CONFIG 0x38u
|
||||
#define XHCI_OP_PORT_BASE 0x400u
|
||||
#define XHCI_OP_PORT_STRIDE 0x10u
|
||||
|
||||
#define XHCI_USBCMD_RS (1u << 0)
|
||||
#define XHCI_USBCMD_HCRST (1u << 1)
|
||||
#define XHCI_USBCMD_INTE (1u << 2)
|
||||
#define XHCI_USBCMD_HSEE (1u << 3)
|
||||
|
||||
#define XHCI_USBSTS_HCH (1u << 0)
|
||||
#define XHCI_USBSTS_HSE (1u << 2)
|
||||
#define XHCI_USBSTS_EINT (1u << 3)
|
||||
#define XHCI_USBSTS_PCD (1u << 4)
|
||||
#define XHCI_USBSTS_CNR (1u << 11)
|
||||
#define XHCI_USBSTS_HCE (1u << 12)
|
||||
#define XHCI_USBSTS_CLEAR_MASK (XHCI_USBSTS_HSE | XHCI_USBSTS_EINT | \
|
||||
XHCI_USBSTS_PCD | XHCI_USBSTS_HCE)
|
||||
|
||||
#define XHCI_IMAN_IP (1u << 0)
|
||||
#define XHCI_IMAN_IE (1u << 1)
|
||||
|
||||
#define XHCI_PORTSC_CCS (1u << 0)
|
||||
#define XHCI_PORTSC_PED (1u << 1)
|
||||
#define XHCI_PORTSC_OCA (1u << 3)
|
||||
#define XHCI_PORTSC_PR (1u << 4)
|
||||
#define XHCI_PORTSC_PP (1u << 9)
|
||||
#define XHCI_PORTSC_SPEED_SHIFT 10u
|
||||
#define XHCI_PORTSC_SPEED_MASK (0xFu << XHCI_PORTSC_SPEED_SHIFT)
|
||||
#define XHCI_PORTSC_CSC (1u << 17)
|
||||
#define XHCI_PORTSC_PEC (1u << 18)
|
||||
#define XHCI_PORTSC_WRC (1u << 19)
|
||||
#define XHCI_PORTSC_OCC (1u << 20)
|
||||
#define XHCI_PORTSC_PRC (1u << 21)
|
||||
#define XHCI_PORTSC_PLC (1u << 22)
|
||||
#define XHCI_PORTSC_CEC (1u << 23)
|
||||
#define XHCI_PORTSC_CHANGE_MASK (XHCI_PORTSC_CSC | XHCI_PORTSC_PEC | \
|
||||
XHCI_PORTSC_WRC | XHCI_PORTSC_OCC | \
|
||||
XHCI_PORTSC_PRC | XHCI_PORTSC_PLC | \
|
||||
XHCI_PORTSC_CEC)
|
||||
|
||||
#define XHCI_EXT_CAP_LEGACY 0x01u
|
||||
#define XHCI_EXT_CAP_PROTOCOL 0x02u
|
||||
#define XHCI_LEGSUP_BIOS_OWNED (1u << 16)
|
||||
#define XHCI_LEGSUP_OS_OWNED (1u << 24)
|
||||
|
||||
#define XHCI_TRB_TYPE_SHIFT 10u
|
||||
#define XHCI_TRB_TYPE_MASK 0x3Fu
|
||||
#define XHCI_TRB_CYCLE (1u << 0)
|
||||
#define XHCI_TRB_LINK_TC (1u << 1)
|
||||
#define XHCI_TRB_TYPE_LINK 6u
|
||||
#define XHCI_TRB_TYPE_NOOP_CMD 23u
|
||||
#define XHCI_TRB_TYPE_CMD_DONE 33u
|
||||
#define XHCI_TRB_TYPE_PORT_SC 34u
|
||||
#define XHCI_TRB_TYPE_TRANSFER 32u
|
||||
#define XHCI_TRB_TYPE_NORMAL 1u
|
||||
#define XHCI_TRB_TYPE_SETUP 2u
|
||||
#define XHCI_TRB_TYPE_DATA 3u
|
||||
#define XHCI_TRB_TYPE_STATUS 4u
|
||||
#define XHCI_TRB_TYPE_ENABLE_SLOT 9u
|
||||
#define XHCI_TRB_TYPE_ADDRESS_DEVICE 11u
|
||||
#define XHCI_TRB_TYPE_CONFIGURE_ENDPOINT 12u
|
||||
|
||||
|
||||
#define XHCI_CC_SUCCESS 1u
|
||||
|
||||
#define XHCI_TRB_ISP (1u << 2)
|
||||
#define XHCI_TRB_IOC (1u << 5)
|
||||
#define XHCI_TRB_IDT (1u << 6)
|
||||
#define XHCI_TRB_DATA_DIR_IN (1u << 16)
|
||||
#define XHCI_TRB_SETUP_TRT_NONE (0u << 16)
|
||||
#define XHCI_TRB_SETUP_TRT_IN (3u << 16)
|
||||
#define XHCI_TRB_STATUS_DIR_IN (1u << 16)
|
||||
|
||||
#define XHCI_HCC1_CSZ (1u << 2)
|
||||
|
||||
#define XHCI_CONTEXT_ENTRIES(n) ((uint32_t)(n) << 27)
|
||||
#define XHCI_SLOT_SPEED(s) ((uint32_t)(s) << 20)
|
||||
#define XHCI_SLOT_ROOT_PORT(p) ((uint32_t)(p) << 16)
|
||||
#define XHCI_EP_TYPE_CONTROL (4u << 3)
|
||||
#define XHCI_EP_TYPE_BULK_OUT (2u << 3)
|
||||
#define XHCI_EP_TYPE_BULK_IN (6u << 3)
|
||||
#define XHCI_EP_TYPE_INT_IN (7u << 3)
|
||||
#define XHCI_EP_CERR_3 (3u << 1)
|
||||
#define XHCI_EP_DCS 1ull
|
||||
|
||||
#define USB_REQ_GET_DESCRIPTOR 0x06u
|
||||
#define USB_REQ_SET_CONFIGURATION 0x09u
|
||||
#define USB_REQ_SET_IDLE 0x0Au
|
||||
#define USB_REQ_SET_PROTOCOL 0x0Bu
|
||||
#define USB_DESC_DEVICE 0x01u
|
||||
#define USB_DESC_CONFIGURATION 0x02u
|
||||
#define USB_DESC_INTERFACE 0x04u
|
||||
#define USB_DESC_ENDPOINT 0x05u
|
||||
#define USB_DESC_HID 0x21u
|
||||
#define USB_DESC_CS_INTERFACE 0x24u
|
||||
#define USB_DIR_OUT 0x00u
|
||||
#define USB_DIR_IN 0x80u
|
||||
#define USB_TYPE_STANDARD 0x00u
|
||||
#define USB_TYPE_CLASS 0x20u
|
||||
#define USB_RECIP_DEVICE 0x00u
|
||||
#define USB_RECIP_INTERFACE 0x01u
|
||||
#define USB_ENDPOINT_DIR_IN 0x80u
|
||||
#define USB_ENDPOINT_NUMBER_MASK 0x0Fu
|
||||
#define USB_ENDPOINT_ATTR_TYPE_MASK 0x03u
|
||||
#define USB_ENDPOINT_TYPE_CONTROL 0x00u
|
||||
#define USB_ENDPOINT_TYPE_ISOCH 0x01u
|
||||
#define USB_ENDPOINT_TYPE_BULK 0x02u
|
||||
#define USB_ENDPOINT_TYPE_INTERRUPT 0x03u
|
||||
#define USB_CLASS_HID 0x03u
|
||||
#define USB_CLASS_MASS_STORAGE 0x08u
|
||||
#define USB_MASS_SUBCLASS_SCSI 0x06u
|
||||
#define USB_MASS_PROTOCOL_BOT 0x50u
|
||||
#define USB_HID_SUBCLASS_BOOT 0x01u
|
||||
#define USB_HID_PROTOCOL_KEYBOARD 0x01u
|
||||
#define USB_HID_PROTOCOL_MOUSE 0x02u
|
||||
#define USB_MASS_BOT_CBW_SIGNATURE 0x43425355u
|
||||
#define USB_MASS_BOT_CSW_SIGNATURE 0x53425355u
|
||||
#define USB_MASS_BOT_CSW_PASSED 0u
|
||||
#define USB_MASS_BOT_CSW_FAILED 1u
|
||||
|
||||
#define SCSI_TEST_UNIT_READY 0x00u
|
||||
#define SCSI_REQUEST_SENSE 0x03u
|
||||
#define SCSI_INQUIRY 0x12u
|
||||
#define SCSI_READ_10 0x28u
|
||||
#define SCSI_WRITE_10 0x2Au
|
||||
#define SCSI_READ_CAPACITY_10 0x25u
|
||||
|
||||
|
||||
#define XHCI_RING_TRBS 256u
|
||||
#define XHCI_EVENT_TRBS 256u
|
||||
#define XHCI_DCBAA_ENTRIES 256u
|
||||
#define XHCI_MMIO_FLOOR_BYTES 0x10000u
|
||||
#define XHCI_MAX_SCRATCHPADS 1024u
|
||||
#define XHCI_MAX_TRACKED_SLOTS 64u
|
||||
#define XHCI_MAX_ROOT_PORTS 255u
|
||||
#define XHCI_EP0_RING_TRBS 256u
|
||||
#define XHCI_HID_RING_TRBS 256u
|
||||
#define XHCI_BULK_RING_TRBS 256u
|
||||
#define XHCI_MAX_HID_REPORT 16u
|
||||
#define USB_STORAGE_DMA_PAGES 1u
|
||||
#define USB_STORAGE_MAX_BYTES (USB_STORAGE_DMA_PAGES * PAGE_SIZE)
|
||||
#define USB_STORAGE_SECTOR_SIZE 512u
|
||||
#define USB_MAX_TRACKED_INTERFACES 8u
|
||||
#define USB_MAX_TRACKED_ENDPOINTS 8u
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef struct XHCI_TRB {
|
||||
uint32_t ParameterLo;
|
||||
uint32_t ParameterHi;
|
||||
uint32_t Status;
|
||||
uint32_t Control;
|
||||
} XHCI_TRB;
|
||||
|
||||
typedef struct XHCI_ERST_ENTRY {
|
||||
uint64_t RingSegmentBase;
|
||||
uint32_t RingSegmentSize;
|
||||
uint32_t Reserved;
|
||||
} XHCI_ERST_ENTRY;
|
||||
|
||||
typedef struct USB_SETUP_PACKET {
|
||||
uint8_t bmRequestType;
|
||||
uint8_t bRequest;
|
||||
uint16_t wValue;
|
||||
uint16_t wIndex;
|
||||
uint16_t wLength;
|
||||
} USB_SETUP_PACKET;
|
||||
|
||||
typedef struct USB_DEVICE_DESCRIPTOR {
|
||||
uint8_t bLength;
|
||||
uint8_t bDescriptorType;
|
||||
uint16_t bcdUSB;
|
||||
uint8_t bDeviceClass;
|
||||
uint8_t bDeviceSubClass;
|
||||
uint8_t bDeviceProtocol;
|
||||
uint8_t bMaxPacketSize0;
|
||||
uint16_t idVendor;
|
||||
uint16_t idProduct;
|
||||
uint16_t bcdDevice;
|
||||
uint8_t iManufacturer;
|
||||
uint8_t iProduct;
|
||||
uint8_t iSerialNumber;
|
||||
uint8_t bNumConfigurations;
|
||||
} USB_DEVICE_DESCRIPTOR;
|
||||
|
||||
typedef struct USB_CONFIGURATION_DESCRIPTOR {
|
||||
uint8_t bLength;
|
||||
uint8_t bDescriptorType;
|
||||
uint16_t wTotalLength;
|
||||
uint8_t bNumInterfaces;
|
||||
uint8_t bConfigurationValue;
|
||||
uint8_t iConfiguration;
|
||||
uint8_t bmAttributes;
|
||||
uint8_t bMaxPower;
|
||||
} USB_CONFIGURATION_DESCRIPTOR;
|
||||
|
||||
typedef struct USB_INTERFACE_DESCRIPTOR {
|
||||
uint8_t bLength;
|
||||
uint8_t bDescriptorType;
|
||||
uint8_t bInterfaceNumber;
|
||||
uint8_t bAlternateSetting;
|
||||
uint8_t bNumEndpoints;
|
||||
uint8_t bInterfaceClass;
|
||||
uint8_t bInterfaceSubClass;
|
||||
uint8_t bInterfaceProtocol;
|
||||
uint8_t iInterface;
|
||||
} USB_INTERFACE_DESCRIPTOR;
|
||||
|
||||
typedef struct USB_ENDPOINT_DESCRIPTOR {
|
||||
uint8_t bLength;
|
||||
uint8_t bDescriptorType;
|
||||
uint8_t bEndpointAddress;
|
||||
uint8_t bmAttributes;
|
||||
uint16_t wMaxPacketSize;
|
||||
uint8_t bInterval;
|
||||
} USB_ENDPOINT_DESCRIPTOR;
|
||||
|
||||
typedef struct USB_MASS_CBW {
|
||||
uint32_t Signature;
|
||||
uint32_t Tag;
|
||||
uint32_t DataTransferLength;
|
||||
uint8_t Flags;
|
||||
uint8_t Lun;
|
||||
uint8_t CommandBlockLength;
|
||||
uint8_t CommandBlock[16];
|
||||
} USB_MASS_CBW;
|
||||
|
||||
typedef struct USB_MASS_CSW {
|
||||
uint32_t Signature;
|
||||
uint32_t Tag;
|
||||
uint32_t DataResidue;
|
||||
uint8_t Status;
|
||||
} USB_MASS_CSW;
|
||||
#pragma pack(pop)
|
||||
|
||||
typedef struct USB_TRACKED_INTERFACE {
|
||||
bool Present;
|
||||
USB_INTERFACE_DESCRIPTOR Descriptor;
|
||||
uint8_t EndpointCount;
|
||||
USB_ENDPOINT_DESCRIPTOR Endpoints[USB_MAX_TRACKED_ENDPOINTS];
|
||||
} USB_TRACKED_INTERFACE;
|
||||
|
||||
typedef enum USB_BAR_KIND {
|
||||
UsbBarNone,
|
||||
UsbBarIo,
|
||||
UsbBarMmio
|
||||
} USB_BAR_KIND;
|
||||
|
||||
typedef struct USB_BAR {
|
||||
USB_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} USB_BAR;
|
||||
|
||||
typedef struct XHCI_DEVICE {
|
||||
bool Allocated;
|
||||
bool Addressed;
|
||||
uint8_t SlotId;
|
||||
uint8_t Port;
|
||||
uint8_t Speed;
|
||||
uint16_t MaxPacketSize0;
|
||||
uint16_t VendorId;
|
||||
uint16_t ProductId;
|
||||
uint16_t BcdUsb;
|
||||
uint16_t BcdDevice;
|
||||
uint8_t DeviceClass;
|
||||
uint8_t DeviceSubclass;
|
||||
uint8_t DeviceProtocol;
|
||||
uint8_t ConfigurationValue;
|
||||
bool Configured;
|
||||
|
||||
uint8_t InterfaceCount;
|
||||
USB_TRACKED_INTERFACE Interfaces[USB_MAX_TRACKED_INTERFACES];
|
||||
|
||||
bool HidCandidate;
|
||||
bool HidActive;
|
||||
bool HidKeyboard;
|
||||
bool HidMouse;
|
||||
bool HidArmed;
|
||||
uint8_t HidInterface;
|
||||
uint8_t HidProtocol;
|
||||
uint8_t HidEndpointAddress;
|
||||
uint8_t HidEndpointId;
|
||||
uint8_t HidInterval;
|
||||
uint16_t HidMaxPacket;
|
||||
uint16_t HidReportLength;
|
||||
uint32_t HidEnqueue;
|
||||
uint8_t HidCycle;
|
||||
uint8_t HidLastModifiers;
|
||||
uint8_t HidLastKeys[6];
|
||||
uint8_t HidLastButtons;
|
||||
uint64_t HidLastTrbPhys;
|
||||
|
||||
bool MassCandidate;
|
||||
bool MassActive;
|
||||
uint8_t MassInterface;
|
||||
uint8_t MassBulkInAddress;
|
||||
uint8_t MassBulkOutAddress;
|
||||
uint8_t MassBulkInEndpointId;
|
||||
uint8_t MassBulkOutEndpointId;
|
||||
uint16_t MassBulkInMaxPacket;
|
||||
uint16_t MassBulkOutMaxPacket;
|
||||
uint32_t MassInEnqueue;
|
||||
uint8_t MassInCycle;
|
||||
uint32_t MassOutEnqueue;
|
||||
uint8_t MassOutCycle;
|
||||
uint32_t MassTag;
|
||||
uint32_t MassBlockSize;
|
||||
uint64_t MassBlockCount;
|
||||
DEVICE_OBJECT* MassDeviceObject;
|
||||
volatile long MassIoBusy;
|
||||
|
||||
PHYSICAL_ADDRESS InputContextPhys;
|
||||
PHYSICAL_ADDRESS DeviceContextPhys;
|
||||
PHYSICAL_ADDRESS Ep0RingPhys;
|
||||
PHYSICAL_ADDRESS HidRingPhys;
|
||||
PHYSICAL_ADDRESS MassInRingPhys;
|
||||
PHYSICAL_ADDRESS MassOutRingPhys;
|
||||
PHYSICAL_ADDRESS DescriptorPhys;
|
||||
PHYSICAL_ADDRESS HidBufferPhys;
|
||||
PHYSICAL_ADDRESS MassBufferPhys;
|
||||
|
||||
uint8_t* InputContext;
|
||||
uint8_t* DeviceContext;
|
||||
XHCI_TRB* Ep0Ring;
|
||||
XHCI_TRB* HidRing;
|
||||
XHCI_TRB* MassInRing;
|
||||
XHCI_TRB* MassOutRing;
|
||||
uint8_t* DescriptorBuffer;
|
||||
uint8_t* HidBuffer;
|
||||
uint8_t* MassBuffer;
|
||||
|
||||
uint32_t Ep0Enqueue;
|
||||
uint8_t Ep0Cycle;
|
||||
} XHCI_DEVICE;
|
||||
|
||||
typedef struct XHCI_COMMAND_RESULT {
|
||||
uint32_t CompletionCode;
|
||||
uint8_t SlotId;
|
||||
} XHCI_COMMAND_RESULT;
|
||||
|
||||
typedef struct XHCI_TRANSFER_RESULT {
|
||||
uint32_t CompletionCode;
|
||||
uint8_t SlotId;
|
||||
uint8_t EndpointId;
|
||||
uint32_t Residue;
|
||||
} XHCI_TRANSFER_RESULT;
|
||||
|
||||
typedef struct XHCI_WAIT_TARGET {
|
||||
uint64_t CommandPhys;
|
||||
uint64_t TransferPhys;
|
||||
uint8_t TransferSlotId;
|
||||
uint8_t TransferEndpointId;
|
||||
bool CommandMatched;
|
||||
bool TransferMatched;
|
||||
XHCI_COMMAND_RESULT Command;
|
||||
XHCI_TRANSFER_RESULT Transfer;
|
||||
} XHCI_WAIT_TARGET;
|
||||
|
||||
typedef struct XHCI_CONTROLLER {
|
||||
const HAL_API* Hal;
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
uint8_t Irq;
|
||||
bool UsingMsi;
|
||||
|
||||
uint64_t MmioPhys;
|
||||
uint32_t MmioBytes;
|
||||
volatile uint8_t* Mmio;
|
||||
volatile uint8_t* Cap;
|
||||
volatile uint8_t* Op;
|
||||
volatile uint8_t* Runtime;
|
||||
volatile uint8_t* Doorbells;
|
||||
|
||||
uint8_t CapLength;
|
||||
uint16_t Version;
|
||||
uint32_t Hcs1;
|
||||
uint32_t Hcs2;
|
||||
uint32_t Hcc1;
|
||||
uint32_t DbOff;
|
||||
uint32_t RtsOff;
|
||||
uint8_t MaxSlots;
|
||||
uint16_t MaxIntrs;
|
||||
uint8_t MaxPorts;
|
||||
uint16_t ScratchpadCount;
|
||||
uint8_t PortProtocolMajor[XHCI_MAX_ROOT_PORTS + 1u];
|
||||
uint8_t PortProtocolMinor[XHCI_MAX_ROOT_PORTS + 1u];
|
||||
|
||||
PHYSICAL_ADDRESS DcbaaPhys;
|
||||
PHYSICAL_ADDRESS ScratchpadArrayPhys;
|
||||
PHYSICAL_ADDRESS ScratchpadPhys[XHCI_MAX_SCRATCHPADS];
|
||||
PHYSICAL_ADDRESS CommandRingPhys;
|
||||
PHYSICAL_ADDRESS EventRingPhys;
|
||||
PHYSICAL_ADDRESS ErstPhys;
|
||||
|
||||
uint64_t* Dcbaa;
|
||||
uint64_t* ScratchpadArray;
|
||||
XHCI_TRB* CommandRing;
|
||||
XHCI_TRB* EventRing;
|
||||
XHCI_ERST_ENTRY* Erst;
|
||||
|
||||
uint32_t CommandEnqueue;
|
||||
uint8_t CommandCycle;
|
||||
uint32_t EventDequeue;
|
||||
uint8_t EventCycle;
|
||||
|
||||
uint8_t ContextBytes;
|
||||
XHCI_DEVICE Devices[XHCI_MAX_TRACKED_SLOTS + 1u];
|
||||
uint32_t HidDeviceCount;
|
||||
|
||||
KDPC EventDpc;
|
||||
volatile bool InterruptsEnabled;
|
||||
volatile uint32_t InterruptCount;
|
||||
volatile uint32_t EventDpcCount;
|
||||
SPINLOCK EventLock;
|
||||
|
||||
volatile XHCI_WAIT_TARGET* ActiveWait;
|
||||
volatile bool Running;
|
||||
} XHCI_CONTROLLER;
|
||||
|
||||
|
||||
extern XHCI_CONTROLLER gXhci;
|
||||
extern bool gHaveXhci;
|
||||
extern DRIVER_OBJECT gUsbStorageDriver;
|
||||
|
||||
uint32_t UsbAlignUp32(uint32_t value, uint32_t align);
|
||||
void XhciBarrier(void);
|
||||
uint8_t XhciRead8(volatile uint8_t* base, uint32_t reg);
|
||||
uint16_t XhciRead16(volatile uint8_t* base, uint32_t reg);
|
||||
uint32_t XhciRead32(volatile uint8_t* base, uint32_t reg);
|
||||
void XhciWrite32(volatile uint8_t* base, uint32_t reg, uint32_t value);
|
||||
uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg);
|
||||
void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value);
|
||||
void UsbDelayUs(const HAL_API* hal, uint32_t us);
|
||||
bool UsbWaitSet32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
|
||||
bool UsbWaitClear32(const HAL_API* hal, volatile uint8_t* base, uint32_t reg, uint32_t mask, uint32_t timeoutUs);
|
||||
bool UsbReadBar(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, USB_BAR* out);
|
||||
KSTATUS UsbMapMmio(const HAL_API* hal, uint64_t physBase, uint32_t bytes, volatile uint8_t** outMmio);
|
||||
void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages, PHYSICAL_ADDRESS* outPhys);
|
||||
uint16_t XhciScratchpadCount(uint32_t hcs2);
|
||||
uint32_t XhciTrbType(const XHCI_TRB* trb);
|
||||
uint32_t XhciCompletionCode(const XHCI_TRB* trb);
|
||||
uint8_t XhciEventSlotId(const XHCI_TRB* trb);
|
||||
uint8_t XhciEventEndpointId(const XHCI_TRB* trb);
|
||||
uint32_t XhciTransferResidue(const XHCI_TRB* trb);
|
||||
uint32_t* XhciInputControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
uint32_t* XhciInputSlotContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
uint32_t* XhciInputEndpointContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId);
|
||||
uint32_t XhciEpMaxPacket(uint8_t speed);
|
||||
uint64_t UsbSetupPacketToU64(const USB_SETUP_PACKET* setup);
|
||||
const char* XhciPortSpeedName(uint32_t speed);
|
||||
void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci);
|
||||
uint8_t XhciPortProtocolMajor(XHCI_CONTROLLER* xhci, uint8_t port, uint32_t portsc);
|
||||
const char* UsbEndpointTypeName(uint8_t type);
|
||||
const char* UsbDirectionName(uint8_t endpointAddress);
|
||||
const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto);
|
||||
void XhciLogPortStatus(XHCI_CONTROLLER* xhci);
|
||||
void XhciAckInterrupter0(XHCI_CONTROLLER* xhci);
|
||||
void XhciUpdateErdp(XHCI_CONTROLLER* xhci);
|
||||
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
|
||||
bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target);
|
||||
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target);
|
||||
KSTATUS XhciWaitForTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target, uint32_t timeoutUs);
|
||||
void XhciEventDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2);
|
||||
void XhciIsr(void);
|
||||
KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci);
|
||||
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index);
|
||||
KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, XHCI_COMMAND_RESULT* outResult);
|
||||
KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId);
|
||||
KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS XhciStart(XHCI_CONTROLLER* xhci);
|
||||
void XhciEnableInterrupts(XHCI_CONTROLLER* xhci);
|
||||
uint32_t XhciPortReg(uint8_t port);
|
||||
void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port);
|
||||
KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed);
|
||||
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index);
|
||||
KSTATUS XhciEp0AppendTrb(XHCI_DEVICE* dev, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
|
||||
KSTATUS XhciControlIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup, void* buffer, PHYSICAL_ADDRESS bufferPhys, uint16_t length);
|
||||
KSTATUS XhciControlNoData(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_SETUP_PACKET* setup);
|
||||
KSTATUS XhciGetDeviceDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_DEVICE_DESCRIPTOR* outDesc);
|
||||
KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationIndex, USB_CONFIGURATION_DESCRIPTOR* outConfig, uint16_t* outTotalLength);
|
||||
uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress);
|
||||
uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval);
|
||||
KSTATUS XhciRingAppend(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint32_t* enqueue, uint8_t* cycle, uint64_t parameter, uint32_t status, uint32_t controlWithoutCycle, uint64_t* outPhys);
|
||||
bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key);
|
||||
INPUT_KEY XhciHidModifierKey(uint8_t bit);
|
||||
void XhciSubmitHidKey(INPUT_KEY key, bool pressed);
|
||||
void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len);
|
||||
void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len);
|
||||
KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
void XhciArmAllHid(XHCI_CONTROLLER* xhci);
|
||||
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys, uint32_t completionCode, uint8_t slotId, uint8_t endpointId, uint32_t residue);
|
||||
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub, uint8_t endpointType, bool wantIn);
|
||||
bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
|
||||
KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context);
|
||||
void XhciInitialiseTransferRing(XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t trbCount, uint8_t cycle);
|
||||
KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t endpointId, XHCI_TRB* ring, PHYSICAL_ADDRESS ringPhys, uint32_t* enqueue, uint8_t* cycle, PHYSICAL_ADDRESS bufferPhys, uint32_t length, uint32_t* outActual);
|
||||
KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
|
||||
KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, PHYSICAL_ADDRESS phys, uint32_t length, uint32_t* outActual);
|
||||
KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const uint8_t* cdb, uint8_t cdbLen, void* data, PHYSICAL_ADDRESS dataPhys, uint32_t dataLength, bool dataIn);
|
||||
uint32_t UsbBe32(const uint8_t* p);
|
||||
void UsbStoreBe32(uint8_t* p, uint32_t v);
|
||||
void UsbStoreBe16(uint8_t* p, uint16_t v);
|
||||
KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, bool write, uint64_t lba, uint32_t sectors, void* buffer);
|
||||
void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
void UsbStorageReleaseIo(XHCI_DEVICE* dev);
|
||||
KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp);
|
||||
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
uint8_t UsbFindFreeDiskIndex(void);
|
||||
KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev);
|
||||
bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context);
|
||||
KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context);
|
||||
KSTATUS UsbRegisterClassMatchers(void);
|
||||
KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, uint8_t configurationValue);
|
||||
void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_CONFIGURATION_DESCRIPTOR* config, uint16_t totalLength);
|
||||
void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev);
|
||||
KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, const USB_DEVICE_DESCRIPTOR* devDesc);
|
||||
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port);
|
||||
void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci);
|
||||
KSTATUS XhciInitialise(const HAL_API* hal, uint8_t bus, uint8_t dev, uint8_t fn, uint16_t vendor, uint16_t device);
|
||||
@@ -0,0 +1,361 @@
|
||||
// usb_core.c - USB descriptor/configuration parsing, interface publishing, and root-port enumeration.
|
||||
#include "usb.h"
|
||||
|
||||
KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
uint8_t configurationValue) {
|
||||
USB_SETUP_PACKET setup;
|
||||
setup.bmRequestType = USB_DIR_OUT;
|
||||
setup.bRequest = USB_REQ_SET_CONFIGURATION;
|
||||
setup.wValue = configurationValue;
|
||||
setup.wIndex = 0;
|
||||
setup.wLength = 0;
|
||||
|
||||
KSTATUS st = XhciControlNoData(xhci, dev, &setup);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: SET_CONFIGURATION slot=%u value=%u failed st=0x%x\n",
|
||||
dev->SlotId, configurationValue, st);
|
||||
return st;
|
||||
}
|
||||
|
||||
dev->ConfigurationValue = configurationValue;
|
||||
dev->Configured = true;
|
||||
UsbTrace("usb: SET_CONFIGURATION slot=%u value=%u complete\n",
|
||||
dev->SlotId, configurationValue);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
const USB_CONFIGURATION_DESCRIPTOR* config,
|
||||
uint16_t totalLength) {
|
||||
(void)xhci;
|
||||
|
||||
memset(dev->Interfaces, 0, sizeof(dev->Interfaces));
|
||||
dev->InterfaceCount = 0;
|
||||
|
||||
UsbTrace("usb: config slot=%u value=%u total=%u interfaces=%u attrs=%02x maxpower=%umA\n",
|
||||
dev->SlotId,
|
||||
config->bConfigurationValue,
|
||||
totalLength,
|
||||
config->bNumInterfaces,
|
||||
config->bmAttributes,
|
||||
(uint32_t)config->bMaxPower * 2u);
|
||||
|
||||
uint16_t off = config->bLength;
|
||||
uint8_t currentInterface = 0xFFu;
|
||||
uint8_t currentClass = 0;
|
||||
uint8_t currentSubClass = 0;
|
||||
uint8_t currentProtocol = 0;
|
||||
bool hidCandidate = false;
|
||||
bool massCandidate = false;
|
||||
uint8_t hidInterruptIn = 0;
|
||||
uint8_t massBulkIn = 0;
|
||||
uint8_t massBulkOut = 0;
|
||||
USB_TRACKED_INTERFACE* tracked = (USB_TRACKED_INTERFACE*)0;
|
||||
|
||||
while (off + 2u <= totalLength) {
|
||||
const uint8_t* desc = dev->DescriptorBuffer + off;
|
||||
uint8_t len = desc[0];
|
||||
uint8_t type = desc[1];
|
||||
|
||||
if (len < 2 || off + len > totalLength) {
|
||||
DbgPrint("usb: descriptor parse stopped slot=%u off=%u len=%u type=%u total=%u\n",
|
||||
dev->SlotId, off, len, type, totalLength);
|
||||
break;
|
||||
}
|
||||
|
||||
if (type == USB_DESC_INTERFACE && len >= sizeof(USB_INTERFACE_DESCRIPTOR)) {
|
||||
const USB_INTERFACE_DESCRIPTOR* iface = (const USB_INTERFACE_DESCRIPTOR*)desc;
|
||||
currentInterface = iface->bInterfaceNumber;
|
||||
currentClass = iface->bInterfaceClass;
|
||||
currentSubClass = iface->bInterfaceSubClass;
|
||||
currentProtocol = iface->bInterfaceProtocol;
|
||||
hidCandidate = (currentClass == USB_CLASS_HID);
|
||||
massCandidate = (currentClass == USB_CLASS_MASS_STORAGE &&
|
||||
currentSubClass == USB_MASS_SUBCLASS_SCSI &&
|
||||
currentProtocol == USB_MASS_PROTOCOL_BOT);
|
||||
hidInterruptIn = 0;
|
||||
massBulkIn = 0;
|
||||
massBulkOut = 0;
|
||||
tracked = (USB_TRACKED_INTERFACE*)0;
|
||||
|
||||
if (dev->InterfaceCount < USB_MAX_TRACKED_INTERFACES) {
|
||||
tracked = &dev->Interfaces[dev->InterfaceCount++];
|
||||
memset(tracked, 0, sizeof(*tracked));
|
||||
tracked->Present = true;
|
||||
memcpy(&tracked->Descriptor, iface, sizeof(*iface));
|
||||
} else {
|
||||
DbgPrint("usb: slot=%u too many interfaces, interface %u not published\n",
|
||||
dev->SlotId, iface->bInterfaceNumber);
|
||||
}
|
||||
|
||||
UsbTrace("usb: interface slot=%u if=%u alt=%u eps=%u class=%02x subclass=%02x proto=%02x (%s)\n",
|
||||
dev->SlotId,
|
||||
iface->bInterfaceNumber,
|
||||
iface->bAlternateSetting,
|
||||
iface->bNumEndpoints,
|
||||
currentClass,
|
||||
currentSubClass,
|
||||
currentProtocol,
|
||||
UsbInterfaceClassName(currentClass, currentSubClass, currentProtocol));
|
||||
} else if (type == USB_DESC_ENDPOINT && len >= sizeof(USB_ENDPOINT_DESCRIPTOR)) {
|
||||
const USB_ENDPOINT_DESCRIPTOR* ep = (const USB_ENDPOINT_DESCRIPTOR*)desc;
|
||||
uint8_t epType = ep->bmAttributes & USB_ENDPOINT_ATTR_TYPE_MASK;
|
||||
uint8_t epNumber = ep->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK;
|
||||
bool in = (ep->bEndpointAddress & USB_ENDPOINT_DIR_IN) != 0;
|
||||
|
||||
if (tracked && tracked->EndpointCount < USB_MAX_TRACKED_ENDPOINTS) {
|
||||
memcpy(&tracked->Endpoints[tracked->EndpointCount++], ep, sizeof(*ep));
|
||||
} else if (tracked) {
|
||||
DbgPrint("usb: slot=%u if=%u too many endpoints, endpoint %02x not published\n",
|
||||
dev->SlotId, currentInterface, ep->bEndpointAddress);
|
||||
}
|
||||
|
||||
UsbTrace("usb: endpoint slot=%u if=%u addr=%02x ep=%u dir=%s type=%s maxpkt=%u interval=%u\n",
|
||||
dev->SlotId,
|
||||
currentInterface,
|
||||
ep->bEndpointAddress,
|
||||
epNumber,
|
||||
UsbDirectionName(ep->bEndpointAddress),
|
||||
UsbEndpointTypeName(epType),
|
||||
ep->wMaxPacketSize,
|
||||
ep->bInterval);
|
||||
|
||||
if (hidCandidate && in && epType == USB_ENDPOINT_TYPE_INTERRUPT && hidInterruptIn == 0) {
|
||||
hidInterruptIn = ep->bEndpointAddress;
|
||||
}
|
||||
if (massCandidate && epType == USB_ENDPOINT_TYPE_BULK) {
|
||||
if (in && massBulkIn == 0) massBulkIn = ep->bEndpointAddress;
|
||||
if (!in && massBulkOut == 0) massBulkOut = ep->bEndpointAddress;
|
||||
}
|
||||
} else if (type == USB_DESC_HID) {
|
||||
UsbTrace("usb: hid descriptor slot=%u if=%u len=%u\n",
|
||||
dev->SlotId, currentInterface, len);
|
||||
} else if (type == USB_DESC_CS_INTERFACE) {
|
||||
UsbTrace("usb: class-specific interface descriptor slot=%u if=%u len=%u subtype=%u\n",
|
||||
dev->SlotId, currentInterface, len, len >= 3 ? desc[2] : 0);
|
||||
}
|
||||
|
||||
uint16_t next = (uint16_t)(off + len);
|
||||
if (hidCandidate && hidInterruptIn &&
|
||||
(next >= totalLength || (next + 1u < totalLength && dev->DescriptorBuffer[next + 1u] == USB_DESC_INTERFACE))) {
|
||||
UsbTrace("usb: class candidate slot=%u if=%u hid interrupt-in=%02x subclass=%02x proto=%02x\n",
|
||||
dev->SlotId,
|
||||
currentInterface,
|
||||
hidInterruptIn,
|
||||
currentSubClass,
|
||||
currentProtocol);
|
||||
hidCandidate = false;
|
||||
}
|
||||
if (massCandidate && massBulkIn && massBulkOut &&
|
||||
(next >= totalLength || (next + 1u < totalLength && dev->DescriptorBuffer[next + 1u] == USB_DESC_INTERFACE))) {
|
||||
UsbTrace("usb: class candidate slot=%u if=%u mass-storage bulk-in=%02x bulk-out=%02x\n",
|
||||
dev->SlotId,
|
||||
currentInterface,
|
||||
massBulkIn,
|
||||
massBulkOut);
|
||||
massCandidate = false;
|
||||
}
|
||||
|
||||
off = next;
|
||||
}
|
||||
|
||||
if (hidCandidate && hidInterruptIn) {
|
||||
UsbTrace("usb: class candidate slot=%u if=%u hid interrupt-in=%02x subclass=%02x proto=%02x\n",
|
||||
dev->SlotId,
|
||||
currentInterface,
|
||||
hidInterruptIn,
|
||||
currentSubClass,
|
||||
currentProtocol);
|
||||
}
|
||||
if (massCandidate && massBulkIn && massBulkOut) {
|
||||
UsbTrace("usb: class candidate slot=%u if=%u mass-storage bulk-in=%02x bulk-out=%02x\n",
|
||||
dev->SlotId,
|
||||
currentInterface,
|
||||
massBulkIn,
|
||||
massBulkOut);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void XhciPublishUsbInterfaces(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
for (uint8_t i = 0; i < dev->InterfaceCount; i++) {
|
||||
USB_TRACKED_INTERFACE* iface = &dev->Interfaces[i];
|
||||
if (!iface->Present)
|
||||
continue;
|
||||
|
||||
IO_PUBLISHED_DEVICE pub;
|
||||
memset(&pub, 0, sizeof(pub));
|
||||
pub.BusType = IoBusUsb;
|
||||
pub.Name = "usb-interface";
|
||||
pub.BusContext = xhci;
|
||||
pub.DeviceContext = dev;
|
||||
|
||||
pub.u.Usb.VendorId = dev->VendorId;
|
||||
pub.u.Usb.ProductId = dev->ProductId;
|
||||
pub.u.Usb.BcdUsb = dev->BcdUsb;
|
||||
pub.u.Usb.BcdDevice = dev->BcdDevice;
|
||||
pub.u.Usb.DeviceClass = dev->DeviceClass;
|
||||
pub.u.Usb.DeviceSubclass = dev->DeviceSubclass;
|
||||
pub.u.Usb.DeviceProtocol = dev->DeviceProtocol;
|
||||
pub.u.Usb.SlotId = dev->SlotId;
|
||||
pub.u.Usb.Port = dev->Port;
|
||||
pub.u.Usb.Speed = dev->Speed;
|
||||
pub.u.Usb.ConfigurationValue = dev->ConfigurationValue;
|
||||
pub.u.Usb.InterfaceNumber = iface->Descriptor.bInterfaceNumber;
|
||||
pub.u.Usb.AlternateSetting = iface->Descriptor.bAlternateSetting;
|
||||
pub.u.Usb.InterfaceClass = iface->Descriptor.bInterfaceClass;
|
||||
pub.u.Usb.InterfaceSubclass = iface->Descriptor.bInterfaceSubClass;
|
||||
pub.u.Usb.InterfaceProtocol = iface->Descriptor.bInterfaceProtocol;
|
||||
pub.u.Usb.EndpointCount = iface->EndpointCount;
|
||||
if (pub.u.Usb.EndpointCount > IO_PUBLISHED_USB_MAX_ENDPOINTS)
|
||||
pub.u.Usb.EndpointCount = IO_PUBLISHED_USB_MAX_ENDPOINTS;
|
||||
|
||||
for (uint8_t ep = 0; ep < pub.u.Usb.EndpointCount; ep++) {
|
||||
pub.u.Usb.Endpoints[ep].Address = iface->Endpoints[ep].bEndpointAddress;
|
||||
pub.u.Usb.Endpoints[ep].Attributes = iface->Endpoints[ep].bmAttributes;
|
||||
pub.u.Usb.Endpoints[ep].MaxPacketSize = iface->Endpoints[ep].wMaxPacketSize;
|
||||
pub.u.Usb.Endpoints[ep].Interval = iface->Endpoints[ep].bInterval;
|
||||
}
|
||||
|
||||
KSTATUS st = IoPublishDevice(&pub);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: publish interface slot=%u if=%u failed st=0x%x\n",
|
||||
dev->SlotId, iface->Descriptor.bInterfaceNumber, st);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
const USB_DEVICE_DESCRIPTOR* devDesc) {
|
||||
if (devDesc->bNumConfigurations == 0) {
|
||||
DbgPrint("usb: slot=%u has no configurations\n", dev->SlotId);
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
USB_CONFIGURATION_DESCRIPTOR config;
|
||||
memset(&config, 0, sizeof(config));
|
||||
uint16_t totalLength = 0;
|
||||
KSTATUS st = XhciGetConfigurationDescriptor(xhci, dev, 0, &config, &totalLength);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: slot=%u configuration descriptor read failed st=0x%x\n",
|
||||
dev->SlotId, st);
|
||||
return st;
|
||||
}
|
||||
|
||||
XhciLogConfigurationTree(xhci, dev, &config, totalLength);
|
||||
|
||||
if (config.bConfigurationValue == 0) {
|
||||
DbgPrint("usb: slot=%u configuration value 0 is invalid for SET_CONFIGURATION\n",
|
||||
dev->SlotId);
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
|
||||
st = XhciSetConfiguration(xhci, dev, config.bConfigurationValue);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
XhciPublishUsbInterfaces(xhci, dev);
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
|
||||
uint8_t speed = 0;
|
||||
KSTATUS st = XhciResetPort(xhci, port, &speed);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
uint8_t slotId = 0;
|
||||
st = XhciEnableSlot(xhci, &slotId);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
XHCI_DEVICE* dev = &xhci->Devices[slotId];
|
||||
memset(dev, 0, sizeof(*dev));
|
||||
dev->Allocated = true;
|
||||
dev->SlotId = slotId;
|
||||
dev->Port = port;
|
||||
dev->Speed = speed;
|
||||
|
||||
st = XhciAllocateDeviceMemory(xhci, dev);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
XhciBuildDefaultControlContext(xhci, dev);
|
||||
|
||||
st = XhciAddressDevice(xhci, dev);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
dev->Addressed = true;
|
||||
|
||||
USB_DEVICE_DESCRIPTOR desc;
|
||||
memset(&desc, 0, sizeof(desc));
|
||||
st = XhciGetDeviceDescriptor(xhci, dev, &desc);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: port %u slot %u descriptor read failed st=0x%x\n",
|
||||
port, slotId, st);
|
||||
return st;
|
||||
}
|
||||
|
||||
dev->MaxPacketSize0 = desc.bMaxPacketSize0;
|
||||
dev->VendorId = desc.idVendor;
|
||||
dev->ProductId = desc.idProduct;
|
||||
dev->BcdUsb = desc.bcdUSB;
|
||||
dev->BcdDevice = desc.bcdDevice;
|
||||
dev->DeviceClass = desc.bDeviceClass;
|
||||
dev->DeviceSubclass = desc.bDeviceSubClass;
|
||||
dev->DeviceProtocol = desc.bDeviceProtocol;
|
||||
|
||||
DbgPrint("usb: device port=%u slot=%u speed=%s(%u) vid=%04x pid=%04x class=%02x subclass=%02x proto=%02x mps0=%u configs=%u usb=%04x\n",
|
||||
port,
|
||||
slotId,
|
||||
XhciPortSpeedName(speed),
|
||||
speed,
|
||||
desc.idVendor,
|
||||
desc.idProduct,
|
||||
desc.bDeviceClass,
|
||||
desc.bDeviceSubClass,
|
||||
desc.bDeviceProtocol,
|
||||
desc.bMaxPacketSize0,
|
||||
desc.bNumConfigurations,
|
||||
desc.bcdUSB);
|
||||
|
||||
st = XhciDiscoverAndSetConfiguration(xhci, dev, &desc);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
|
||||
uint32_t attempted = 0;
|
||||
uint32_t configured = 0;
|
||||
|
||||
for (uint8_t port = 1; port <= xhci->MaxPorts; port++) {
|
||||
uint32_t portsc = XhciRead32(xhci->Op, XhciPortReg(port));
|
||||
if (!(portsc & XHCI_PORTSC_CCS))
|
||||
continue;
|
||||
|
||||
attempted++;
|
||||
KSTATUS st = XhciEnumeratePort(xhci, port);
|
||||
if (KSUCCESS(st)) configured++;
|
||||
else DbgPrint("usb: root-port %u enumeration failed st=0x%x\n", port, st);
|
||||
}
|
||||
|
||||
DbgPrint("usb: xHCI root-port enumeration complete; attempted=%u configured=%u\n",
|
||||
attempted, configured);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbRegisterClassMatchers(void) {
|
||||
memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
|
||||
gUsbStorageDriver.Name = "usbstor";
|
||||
gUsbStorageDriver.MajorFunction[IRP_MJ_READ] = UsbStorageDispatchReadWrite;
|
||||
gUsbStorageDriver.MajorFunction[IRP_MJ_WRITE] = UsbStorageDispatchReadWrite;
|
||||
|
||||
KSTATUS st = IoRegisterDriverMatcher("usbhid", IoBusUsb, UsbHidMatch, UsbHidBind, NULL);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
st = IoRegisterDriverMatcher("usbstor", IoBusUsb, UsbStorageMatch, UsbStorageBind, NULL);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
// usb_hid.c - USB HID boot keyboard/mouse class binding and report handling.
|
||||
#include "usb.h"
|
||||
|
||||
bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) {
|
||||
if (key == 0 || key == 1) return false;
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
if (keys[i] == key) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
INPUT_KEY XhciHidModifierKey(uint8_t bit) {
|
||||
switch (bit) {
|
||||
case 0: return INPUT_KEY_LEFT_CTRL;
|
||||
case 1: return INPUT_KEY_LEFT_SHIFT;
|
||||
case 2: return INPUT_KEY_LEFT_ALT;
|
||||
case 3: return INPUT_KEY_LEFT_GUI;
|
||||
case 4: return INPUT_KEY_RIGHT_CTRL;
|
||||
case 5: return INPUT_KEY_RIGHT_SHIFT;
|
||||
case 6: return INPUT_KEY_RIGHT_ALT;
|
||||
case 7: return INPUT_KEY_RIGHT_GUI;
|
||||
default: return INPUT_KEY_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void XhciSubmitHidKey(INPUT_KEY key, bool pressed) {
|
||||
if (key == INPUT_KEY_NONE) return;
|
||||
INPUT_KEY_EVENT ev;
|
||||
ev.Key = key;
|
||||
ev.Pressed = pressed;
|
||||
KeInputSubmitKey(&ev);
|
||||
}
|
||||
|
||||
|
||||
void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) {
|
||||
if (len < 8) return;
|
||||
|
||||
uint8_t modifiers = report[0];
|
||||
for (uint8_t bit = 0; bit < 8; bit++) {
|
||||
uint8_t mask = (uint8_t)(1u << bit);
|
||||
bool wasDown = (dev->HidLastModifiers & mask) != 0;
|
||||
bool isDown = (modifiers & mask) != 0;
|
||||
if (wasDown != isDown)
|
||||
XhciSubmitHidKey(XhciHidModifierKey(bit), isDown);
|
||||
}
|
||||
|
||||
const uint8_t* keys = &report[2];
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
uint8_t oldKey = dev->HidLastKeys[i];
|
||||
if (oldKey > 1 && !XhciHidKeyInReport(keys, oldKey))
|
||||
XhciSubmitHidKey((INPUT_KEY)oldKey, false);
|
||||
}
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
uint8_t newKey = keys[i];
|
||||
if (newKey > 1 && !XhciHidKeyInReport(dev->HidLastKeys, newKey))
|
||||
XhciSubmitHidKey((INPUT_KEY)newKey, true);
|
||||
}
|
||||
|
||||
dev->HidLastModifiers = modifiers;
|
||||
memcpy(dev->HidLastKeys, keys, sizeof(dev->HidLastKeys));
|
||||
}
|
||||
|
||||
|
||||
void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) {
|
||||
if (len < 3) return;
|
||||
|
||||
int dx = (int)(int8_t)report[1];
|
||||
int dy = (int)(int8_t)report[2];
|
||||
int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0;
|
||||
uint8_t buttons = report[0] & 0x07u;
|
||||
|
||||
if (dx || dy || wheel || buttons != dev->HidLastButtons) {
|
||||
INPUT_MOUSE_EVENT ev;
|
||||
ev.dx = dx;
|
||||
ev.dy = dy;
|
||||
ev.wheel = wheel;
|
||||
ev.buttons = buttons;
|
||||
KeInputSubmitMouse(&ev);
|
||||
}
|
||||
|
||||
dev->HidLastButtons = buttons;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
if (!dev->HidActive || dev->HidArmed || !dev->HidRing || !dev->HidBuffer)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
memset(dev->HidBuffer, 0, dev->HidReportLength);
|
||||
|
||||
uint64_t trbPhys = 0;
|
||||
KSTATUS st = XhciRingAppend(dev->HidRing,
|
||||
dev->HidRingPhys,
|
||||
XHCI_HID_RING_TRBS,
|
||||
&dev->HidEnqueue,
|
||||
&dev->HidCycle,
|
||||
dev->HidBufferPhys,
|
||||
dev->HidReportLength,
|
||||
(XHCI_TRB_TYPE_NORMAL << XHCI_TRB_TYPE_SHIFT) |
|
||||
XHCI_TRB_ISP |
|
||||
XHCI_TRB_IOC,
|
||||
&trbPhys);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
dev->HidLastTrbPhys = trbPhys;
|
||||
dev->HidArmed = true;
|
||||
XhciBarrier();
|
||||
XhciWrite32(xhci->Doorbells, (uint32_t)dev->SlotId * 4u, dev->HidEndpointId);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
|
||||
for (uint8_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++) {
|
||||
XHCI_DEVICE* dev = &xhci->Devices[slot];
|
||||
if (!dev->HidActive) continue;
|
||||
KSTATUS st = XhciArmHidInterruptIn(xhci, dev);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: HID arm failed slot=%u ep=%u st=0x%x\n",
|
||||
dev->SlotId, dev->HidEndpointId, st);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
|
||||
uint32_t completionCode, uint8_t slotId,
|
||||
uint8_t endpointId, uint32_t residue) {
|
||||
if (slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS) return;
|
||||
|
||||
XHCI_DEVICE* dev = &xhci->Devices[slotId];
|
||||
if (!dev->HidActive || endpointId != dev->HidEndpointId || trbPhys != dev->HidLastTrbPhys)
|
||||
return;
|
||||
|
||||
dev->HidArmed = false;
|
||||
if (completionCode != XHCI_CC_SUCCESS) {
|
||||
DbgPrint("usb: HID transfer failed slot=%u ep=%u cc=%u residue=%u\n",
|
||||
slotId, endpointId, completionCode, residue);
|
||||
return;
|
||||
}
|
||||
|
||||
uint16_t length = dev->HidReportLength;
|
||||
if (residue < length) length = (uint16_t)(length - residue);
|
||||
|
||||
if (dev->HidKeyboard)
|
||||
XhciProcessBootKeyboardReport(dev, dev->HidBuffer, length);
|
||||
else if (dev->HidMouse)
|
||||
XhciProcessBootMouseReport(dev, dev->HidBuffer, length);
|
||||
|
||||
KSTATUS st = XhciArmHidInterruptIn(xhci, dev);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: HID re-arm failed slot=%u ep=%u st=0x%x\n",
|
||||
slotId, endpointId, st);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
|
||||
if (xhci->HidDeviceCount == 0)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
XhciArmAllHid(xhci);
|
||||
DbgPrint("usb: HID interrupt input armed devices=%u\n", xhci->HidDeviceCount);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
USB_SETUP_PACKET setup;
|
||||
setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
|
||||
setup.bRequest = USB_REQ_SET_PROTOCOL;
|
||||
setup.wValue = 0; // boot protocol
|
||||
setup.wIndex = dev->HidInterface;
|
||||
setup.wLength = 0;
|
||||
|
||||
KSTATUS st = XhciControlNoData(xhci, dev, &setup);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: HID SET_PROTOCOL boot failed slot=%u if=%u st=0x%x\n",
|
||||
dev->SlotId, dev->HidInterface, st);
|
||||
return st;
|
||||
}
|
||||
|
||||
setup.bRequest = USB_REQ_SET_IDLE;
|
||||
setup.wValue = 0;
|
||||
st = XhciControlNoData(xhci, dev, &setup);
|
||||
if (!KSUCCESS(st)) {
|
||||
// SET_IDLE is useful for keyboards but not worth rejecting an otherwise
|
||||
// usable boot device in the early stack.
|
||||
DbgPrint("usb: HID SET_IDLE failed slot=%u if=%u st=0x%x; continuing\n",
|
||||
dev->SlotId, dev->HidInterface, st);
|
||||
}
|
||||
|
||||
DbgPrint("usb: HID boot protocol selected slot=%u if=%u proto=%u\n",
|
||||
dev->SlotId, dev->HidInterface, dev->HidProtocol);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
if (dev->HidRing && dev->HidBuffer) return STATUS_SUCCESS;
|
||||
|
||||
dev->HidRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->HidRingPhys);
|
||||
dev->HidBuffer = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->HidBufferPhys);
|
||||
if (!dev->HidRing || !dev->HidBuffer)
|
||||
return STATUS_NO_MEMORY;
|
||||
|
||||
dev->HidEnqueue = 0;
|
||||
dev->HidCycle = 1;
|
||||
|
||||
XHCI_TRB* link = &dev->HidRing[XHCI_HID_RING_TRBS - 1u];
|
||||
link->ParameterLo = (uint32_t)dev->HidRingPhys;
|
||||
link->ParameterHi = (uint32_t)(dev->HidRingPhys >> 32);
|
||||
link->Status = 0;
|
||||
link->Control = (XHCI_TRB_TYPE_LINK << XHCI_TRB_TYPE_SHIFT) |
|
||||
XHCI_TRB_LINK_TC |
|
||||
dev->HidCycle;
|
||||
|
||||
memset(dev->HidLastKeys, 0, sizeof(dev->HidLastKeys));
|
||||
dev->HidLastModifiers = 0;
|
||||
dev->HidLastButtons = 0;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
if (!dev->HidCandidate)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
KSTATUS st = XhciAllocateHidMemory(xhci, dev);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
uint8_t dci = dev->HidEndpointId;
|
||||
if (dci < 2 || dci > 31)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
memset(dev->InputContext, 0, PAGE_SIZE);
|
||||
uint32_t* icc = XhciInputControlContext(xhci, dev);
|
||||
uint32_t* slot = XhciInputSlotContext(xhci, dev);
|
||||
uint32_t* ep = XhciInputEndpointContext(xhci, dev, dci);
|
||||
|
||||
icc[0] = 0;
|
||||
icc[1] = (1u << 0) | (1u << dci);
|
||||
|
||||
slot[0] = XHCI_CONTEXT_ENTRIES(dci) | XHCI_SLOT_SPEED(dev->Speed);
|
||||
slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
|
||||
slot[2] = 0;
|
||||
slot[3] = 0;
|
||||
|
||||
uint16_t maxPacket = dev->HidMaxPacket & 0x07FFu;
|
||||
if (maxPacket == 0) maxPacket = dev->HidKeyboard ? 8u : 4u;
|
||||
uint8_t interval = XhciInterruptInterval(dev->Speed, dev->HidInterval);
|
||||
|
||||
ep[0] = (uint32_t)interval << 16;
|
||||
ep[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_INT_IN | ((uint32_t)maxPacket << 16);
|
||||
ep[2] = (uint32_t)(dev->HidRingPhys | XHCI_EP_DCS);
|
||||
ep[3] = (uint32_t)((dev->HidRingPhys | XHCI_EP_DCS) >> 32);
|
||||
ep[4] = (uint32_t)maxPacket | ((uint32_t)maxPacket << 16);
|
||||
|
||||
XHCI_COMMAND_RESULT result;
|
||||
memset(&result, 0, sizeof(result));
|
||||
st = XhciSubmitCommand(
|
||||
xhci,
|
||||
dev->InputContextPhys,
|
||||
0,
|
||||
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
|
||||
((uint32_t)dev->SlotId << 24),
|
||||
&result);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb: Configure Endpoint failed slot=%u ep=%u st=0x%x cc=%u\n",
|
||||
dev->SlotId, dci, st, result.CompletionCode);
|
||||
return st;
|
||||
}
|
||||
|
||||
dev->HidReportLength = maxPacket;
|
||||
if (dev->HidKeyboard && dev->HidReportLength < 8) dev->HidReportLength = 8;
|
||||
if (dev->HidMouse && dev->HidReportLength < 3) dev->HidReportLength = 3;
|
||||
if (dev->HidReportLength > XHCI_MAX_HID_REPORT) dev->HidReportLength = XHCI_MAX_HID_REPORT;
|
||||
|
||||
st = XhciSetHidBootProtocol(xhci, dev);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
dev->HidActive = true;
|
||||
xhci->HidDeviceCount++;
|
||||
uint32_t inputId = 0;
|
||||
KeInputRegisterDevice(dev->HidKeyboard ? "usb-hid-keyboard" : "usb-hid-mouse", 0, &inputId);
|
||||
|
||||
DbgPrint("usb: HID boot %s bound slot=%u if=%u ep=%02x dci=%u maxpkt=%u interval=%u input=%u\n",
|
||||
dev->HidKeyboard ? "keyboard" : "mouse",
|
||||
dev->SlotId,
|
||||
dev->HidInterface,
|
||||
dev->HidEndpointAddress,
|
||||
dci,
|
||||
maxPacket,
|
||||
dev->HidInterval,
|
||||
inputId);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
const IO_PUBLISHED_USB_ENDPOINT* UsbFindEndpoint(const IO_PUBLISHED_DEVICE* pub,
|
||||
uint8_t endpointType,
|
||||
bool wantIn) {
|
||||
if (!pub || pub->BusType != IoBusUsb) return (const IO_PUBLISHED_USB_ENDPOINT*)0;
|
||||
|
||||
for (uint8_t i = 0; i < pub->u.Usb.EndpointCount; i++) {
|
||||
const IO_PUBLISHED_USB_ENDPOINT* ep = &pub->u.Usb.Endpoints[i];
|
||||
uint8_t type = ep->Attributes & USB_ENDPOINT_ATTR_TYPE_MASK;
|
||||
bool in = (ep->Address & USB_ENDPOINT_DIR_IN) != 0;
|
||||
if (type == endpointType && in == wantIn)
|
||||
return ep;
|
||||
}
|
||||
|
||||
return (const IO_PUBLISHED_USB_ENDPOINT*)0;
|
||||
}
|
||||
|
||||
|
||||
bool UsbHidMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
|
||||
(void)context;
|
||||
if (!pub || pub->BusType != IoBusUsb)
|
||||
return false;
|
||||
if (pub->u.Usb.InterfaceClass != USB_CLASS_HID)
|
||||
return false;
|
||||
if (pub->u.Usb.InterfaceSubclass != USB_HID_SUBCLASS_BOOT)
|
||||
return false;
|
||||
if (pub->u.Usb.InterfaceProtocol != USB_HID_PROTOCOL_KEYBOARD &&
|
||||
pub->u.Usb.InterfaceProtocol != USB_HID_PROTOCOL_MOUSE)
|
||||
return false;
|
||||
|
||||
return UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_INTERRUPT, true) != (const IO_PUBLISHED_USB_ENDPOINT*)0;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbHidBind(IO_PUBLISHED_DEVICE* pub, void* context) {
|
||||
(void)context;
|
||||
if (!pub || !pub->BusContext || !pub->DeviceContext)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext;
|
||||
XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext;
|
||||
|
||||
if (dev->HidActive || dev->HidCandidate)
|
||||
return STATUS_ALREADY_EXISTS;
|
||||
|
||||
const IO_PUBLISHED_USB_ENDPOINT* ep = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_INTERRUPT, true);
|
||||
if (!ep)
|
||||
return STATUS_NOT_FOUND;
|
||||
|
||||
dev->HidCandidate = true;
|
||||
dev->HidKeyboard = (pub->u.Usb.InterfaceProtocol == USB_HID_PROTOCOL_KEYBOARD);
|
||||
dev->HidMouse = (pub->u.Usb.InterfaceProtocol == USB_HID_PROTOCOL_MOUSE);
|
||||
dev->HidInterface = pub->u.Usb.InterfaceNumber;
|
||||
dev->HidProtocol = pub->u.Usb.InterfaceProtocol;
|
||||
dev->HidEndpointAddress = ep->Address;
|
||||
dev->HidEndpointId = XhciEndpointIdFromAddress(ep->Address);
|
||||
dev->HidInterval = ep->Interval;
|
||||
dev->HidMaxPacket = ep->MaxPacketSize;
|
||||
|
||||
KSTATUS st = XhciConfigureHidEndpoint(xhci, dev);
|
||||
if (!KSUCCESS(st)) {
|
||||
dev->HidCandidate = false;
|
||||
dev->HidKeyboard = false;
|
||||
dev->HidMouse = false;
|
||||
}
|
||||
return st;
|
||||
}
|
||||
@@ -0,0 +1,472 @@
|
||||
// usb_storage.c - USB mass-storage BOT/SCSI class binding and disk bridge.
|
||||
#include "usb.h"
|
||||
|
||||
DRIVER_OBJECT gUsbStorageDriver;
|
||||
|
||||
KSTATUS UsbStorageBulkIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
PHYSICAL_ADDRESS phys, uint32_t length,
|
||||
uint32_t* outActual) {
|
||||
return XhciBulkTransfer(xhci,
|
||||
dev,
|
||||
dev->MassBulkInEndpointId,
|
||||
dev->MassInRing,
|
||||
dev->MassInRingPhys,
|
||||
&dev->MassInEnqueue,
|
||||
&dev->MassInCycle,
|
||||
phys,
|
||||
length,
|
||||
outActual);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageBulkOut(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
PHYSICAL_ADDRESS phys, uint32_t length,
|
||||
uint32_t* outActual) {
|
||||
return XhciBulkTransfer(xhci,
|
||||
dev,
|
||||
dev->MassBulkOutEndpointId,
|
||||
dev->MassOutRing,
|
||||
dev->MassOutRingPhys,
|
||||
&dev->MassOutEnqueue,
|
||||
&dev->MassOutCycle,
|
||||
phys,
|
||||
length,
|
||||
outActual);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageBotCommand(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
const uint8_t* cdb, uint8_t cdbLen,
|
||||
void* data, PHYSICAL_ADDRESS dataPhys,
|
||||
uint32_t dataLength, bool dataIn) {
|
||||
if (!xhci || !dev || !cdb || cdbLen == 0 || cdbLen > 16)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (dataLength && (!data || !dataPhys))
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
USB_MASS_CBW* cbw = (USB_MASS_CBW*)dev->DescriptorBuffer;
|
||||
memset(cbw, 0, sizeof(*cbw));
|
||||
cbw->Signature = USB_MASS_BOT_CBW_SIGNATURE;
|
||||
cbw->Tag = ++dev->MassTag;
|
||||
cbw->DataTransferLength = dataLength;
|
||||
cbw->Flags = dataIn ? USB_DIR_IN : USB_DIR_OUT;
|
||||
cbw->Lun = 0;
|
||||
cbw->CommandBlockLength = cdbLen;
|
||||
memcpy(cbw->CommandBlock, cdb, cdbLen);
|
||||
|
||||
uint32_t actual = 0;
|
||||
KSTATUS st = UsbStorageBulkOut(xhci, dev, dev->DescriptorPhys, sizeof(*cbw), &actual);
|
||||
if (!KSUCCESS(st) || actual != sizeof(*cbw))
|
||||
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
|
||||
|
||||
if (dataLength) {
|
||||
actual = 0;
|
||||
if (dataIn)
|
||||
st = UsbStorageBulkIn(xhci, dev, dataPhys, dataLength, &actual);
|
||||
else
|
||||
st = UsbStorageBulkOut(xhci, dev, dataPhys, dataLength, &actual);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
USB_MASS_CSW* csw = (USB_MASS_CSW*)dev->DescriptorBuffer;
|
||||
memset(csw, 0, sizeof(*csw));
|
||||
actual = 0;
|
||||
st = UsbStorageBulkIn(xhci, dev, dev->DescriptorPhys, sizeof(*csw), &actual);
|
||||
if (!KSUCCESS(st) || actual != sizeof(*csw))
|
||||
return KSUCCESS(st) ? STATUS_DEVICE_ERROR : st;
|
||||
|
||||
if (csw->Signature != USB_MASS_BOT_CSW_SIGNATURE || csw->Tag != dev->MassTag) {
|
||||
DbgPrint("usb-storage: invalid CSW slot=%u sig=%08x tag=%08x expected=%08x\n",
|
||||
dev->SlotId, csw->Signature, csw->Tag, dev->MassTag);
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
if (csw->Status != USB_MASS_BOT_CSW_PASSED) {
|
||||
DbgPrint("usb-storage: command %02x failed slot=%u csw_status=%u residue=%u\n",
|
||||
cdb[0], dev->SlotId, csw->Status, csw->DataResidue);
|
||||
return STATUS_DEVICE_ERROR;
|
||||
}
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
uint32_t UsbBe32(const uint8_t* p) {
|
||||
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
|
||||
((uint32_t)p[2] << 8) | (uint32_t)p[3];
|
||||
}
|
||||
|
||||
|
||||
void UsbStoreBe32(uint8_t* p, uint32_t v) {
|
||||
p[0] = (uint8_t)(v >> 24);
|
||||
p[1] = (uint8_t)(v >> 16);
|
||||
p[2] = (uint8_t)(v >> 8);
|
||||
p[3] = (uint8_t)v;
|
||||
}
|
||||
|
||||
|
||||
void UsbStoreBe16(uint8_t* p, uint16_t v) {
|
||||
p[0] = (uint8_t)(v >> 8);
|
||||
p[1] = (uint8_t)v;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageInquiry(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
uint8_t cdb[6] = { SCSI_INQUIRY, 0, 0, 0, 36, 0 };
|
||||
memset(dev->MassBuffer, 0, 36);
|
||||
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
|
||||
dev->MassBuffer, dev->MassBufferPhys,
|
||||
36, true);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
char vendor[9];
|
||||
char product[17];
|
||||
memcpy(vendor, dev->MassBuffer + 8, 8); vendor[8] = 0;
|
||||
memcpy(product, dev->MassBuffer + 16, 16); product[16] = 0;
|
||||
DbgPrint("usb-storage: INQUIRY slot=%u vendor='%s' product='%s'\n",
|
||||
dev->SlotId, vendor, product);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageRequestSense(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
uint8_t rs[6] = { SCSI_REQUEST_SENSE, 0, 0, 0, 18, 0 };
|
||||
memset(dev->MassBuffer, 0, 18);
|
||||
return UsbStorageBotCommand(xhci, dev, rs, sizeof(rs),
|
||||
dev->MassBuffer, dev->MassBufferPhys, 18, true);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageTestUnitReady(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
uint8_t cdb[6] = { SCSI_TEST_UNIT_READY, 0, 0, 0, 0, 0 };
|
||||
|
||||
KSTATUS last = STATUS_DEVICE_ERROR;
|
||||
for (uint32_t attempt = 0; attempt < 10; attempt++) {
|
||||
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
|
||||
(void*)0, 0, 0, false);
|
||||
if (KSUCCESS(st)) return STATUS_SUCCESS;
|
||||
|
||||
last = st;
|
||||
(void)UsbStorageRequestSense(xhci, dev);
|
||||
UsbDelayUs(xhci->Hal, 10000u);
|
||||
}
|
||||
|
||||
return last;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageReadCapacity(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
uint8_t cdb[10] = { SCSI_READ_CAPACITY_10, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
|
||||
memset(dev->MassBuffer, 0, 8);
|
||||
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
|
||||
dev->MassBuffer, dev->MassBufferPhys,
|
||||
8, true);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
uint32_t lastLba = UsbBe32(dev->MassBuffer + 0);
|
||||
uint32_t blockSize = UsbBe32(dev->MassBuffer + 4);
|
||||
if (blockSize == 0)
|
||||
return STATUS_DEVICE_ERROR;
|
||||
|
||||
dev->MassBlockSize = blockSize;
|
||||
dev->MassBlockCount = (uint64_t)lastLba + 1ull;
|
||||
DbgPrint("usb-storage: capacity slot=%u blocks=%llu block_size=%u (%llu MB)\n",
|
||||
dev->SlotId,
|
||||
(unsigned long long)dev->MassBlockCount,
|
||||
dev->MassBlockSize,
|
||||
(unsigned long long)((dev->MassBlockCount * (uint64_t)dev->MassBlockSize) / (1024ull * 1024ull)));
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageReadWriteSectors(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
|
||||
bool write, uint64_t lba,
|
||||
uint32_t sectors, void* buffer) {
|
||||
if (!xhci || !dev || !buffer || sectors == 0)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
if (lba > 0xFFFFFFFFull || sectors > 0xFFFFu)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uint32_t bytes = sectors * USB_STORAGE_SECTOR_SIZE;
|
||||
if (bytes > USB_STORAGE_MAX_BYTES)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uint8_t cdb[10];
|
||||
memset(cdb, 0, sizeof(cdb));
|
||||
cdb[0] = write ? SCSI_WRITE_10 : SCSI_READ_10;
|
||||
UsbStoreBe32(&cdb[2], (uint32_t)lba);
|
||||
UsbStoreBe16(&cdb[7], (uint16_t)sectors);
|
||||
|
||||
if (write)
|
||||
memcpy(dev->MassBuffer, buffer, bytes);
|
||||
|
||||
KSTATUS st = UsbStorageBotCommand(xhci, dev, cdb, sizeof(cdb),
|
||||
dev->MassBuffer, dev->MassBufferPhys,
|
||||
bytes, !write);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
if (!write)
|
||||
memcpy(buffer, dev->MassBuffer, bytes);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void UsbStorageAcquireIo(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
while (_InterlockedCompareExchange(&dev->MassIoBusy, 1, 0) != 0) {
|
||||
UsbDelayUs(xhci->Hal, 10);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void UsbStorageReleaseIo(XHCI_DEVICE* dev) {
|
||||
_InterlockedExchange(&dev->MassIoBusy, 0);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp || !device->DeviceExtension)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
XHCI_DEVICE* dev = (XHCI_DEVICE*)device->DeviceExtension;
|
||||
if (!dev->MassActive || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
|
||||
IoCompleteRequest(irp, STATUS_DEVICE_ERROR, 0);
|
||||
return STATUS_PENDING;
|
||||
}
|
||||
|
||||
uint8_t* buffer = (uint8_t*)irp->Buffer;
|
||||
uint64_t lba = irp->Lba;
|
||||
uint32_t remaining = irp->SectorCount;
|
||||
uint32_t done = 0;
|
||||
bool write = (irp->Flags & IRP_FLAG_WRITE) != 0;
|
||||
uint32_t maxSectors = USB_STORAGE_MAX_BYTES / USB_STORAGE_SECTOR_SIZE;
|
||||
|
||||
KSTATUS st = STATUS_SUCCESS;
|
||||
UsbStorageAcquireIo(&gXhci, dev);
|
||||
while (remaining) {
|
||||
uint32_t chunk = remaining;
|
||||
if (chunk > maxSectors) chunk = maxSectors;
|
||||
st = UsbStorageReadWriteSectors(&gXhci, dev, write, lba, chunk,
|
||||
buffer + (done * USB_STORAGE_SECTOR_SIZE));
|
||||
if (!KSUCCESS(st))
|
||||
break;
|
||||
lba += chunk;
|
||||
done += chunk;
|
||||
remaining -= chunk;
|
||||
}
|
||||
UsbStorageReleaseIo(dev);
|
||||
|
||||
IoCompleteRequest(irp, st, KSUCCESS(st) ? (size_t)done * USB_STORAGE_SECTOR_SIZE : 0);
|
||||
return STATUS_PENDING;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
dev->MassInRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassInRingPhys);
|
||||
dev->MassOutRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &dev->MassOutRingPhys);
|
||||
dev->MassBuffer = (uint8_t*)UsbDmaAllocPages(xhci, USB_STORAGE_DMA_PAGES, &dev->MassBufferPhys);
|
||||
if (!dev->MassInRing || !dev->MassOutRing || !dev->MassBuffer)
|
||||
return STATUS_NO_MEMORY;
|
||||
|
||||
dev->MassInEnqueue = 0;
|
||||
dev->MassInCycle = 1;
|
||||
dev->MassOutEnqueue = 0;
|
||||
dev->MassOutCycle = 1;
|
||||
dev->MassTag = 0x1000u + dev->SlotId;
|
||||
XhciInitialiseTransferRing(dev->MassInRing, dev->MassInRingPhys, XHCI_BULK_RING_TRBS, dev->MassInCycle);
|
||||
XhciInitialiseTransferRing(dev->MassOutRing, dev->MassOutRingPhys, XHCI_BULK_RING_TRBS, dev->MassOutCycle);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
|
||||
if (!dev->MassCandidate)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
KSTATUS st = XhciAllocateMassStorageMemory(xhci, dev);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
uint8_t inDci = dev->MassBulkInEndpointId;
|
||||
uint8_t outDci = dev->MassBulkOutEndpointId;
|
||||
if (inDci < 2 || inDci > 31 || outDci < 2 || outDci > 31)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
memset(dev->InputContext, 0, PAGE_SIZE);
|
||||
uint32_t* icc = XhciInputControlContext(xhci, dev);
|
||||
uint32_t* slot = XhciInputSlotContext(xhci, dev);
|
||||
uint32_t* epIn = XhciInputEndpointContext(xhci, dev, inDci);
|
||||
uint32_t* epOut = XhciInputEndpointContext(xhci, dev, outDci);
|
||||
|
||||
icc[0] = 0;
|
||||
icc[1] = (1u << 0) | (1u << inDci) | (1u << outDci);
|
||||
|
||||
uint8_t maxDci = inDci > outDci ? inDci : outDci;
|
||||
slot[0] = XHCI_CONTEXT_ENTRIES(maxDci) | XHCI_SLOT_SPEED(dev->Speed);
|
||||
slot[1] = XHCI_SLOT_ROOT_PORT(dev->Port);
|
||||
slot[2] = 0;
|
||||
slot[3] = 0;
|
||||
|
||||
uint16_t inMaxPacket = dev->MassBulkInMaxPacket & 0x07FFu;
|
||||
uint16_t outMaxPacket = dev->MassBulkOutMaxPacket & 0x07FFu;
|
||||
if (inMaxPacket == 0) inMaxPacket = 512;
|
||||
if (outMaxPacket == 0) outMaxPacket = 512;
|
||||
|
||||
epIn[0] = 0;
|
||||
epIn[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_IN | ((uint32_t)inMaxPacket << 16);
|
||||
epIn[2] = (uint32_t)(dev->MassInRingPhys | XHCI_EP_DCS);
|
||||
epIn[3] = (uint32_t)((dev->MassInRingPhys | XHCI_EP_DCS) >> 32);
|
||||
epIn[4] = inMaxPacket;
|
||||
|
||||
epOut[0] = 0;
|
||||
epOut[1] = XHCI_EP_CERR_3 | XHCI_EP_TYPE_BULK_OUT | ((uint32_t)outMaxPacket << 16);
|
||||
epOut[2] = (uint32_t)(dev->MassOutRingPhys | XHCI_EP_DCS);
|
||||
epOut[3] = (uint32_t)((dev->MassOutRingPhys | XHCI_EP_DCS) >> 32);
|
||||
epOut[4] = outMaxPacket;
|
||||
|
||||
XHCI_COMMAND_RESULT result;
|
||||
memset(&result, 0, sizeof(result));
|
||||
st = XhciSubmitCommand(
|
||||
xhci,
|
||||
dev->InputContextPhys,
|
||||
0,
|
||||
(XHCI_TRB_TYPE_CONFIGURE_ENDPOINT << XHCI_TRB_TYPE_SHIFT) |
|
||||
((uint32_t)dev->SlotId << 24),
|
||||
&result);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb-storage: Configure Endpoint failed slot=%u in=%u out=%u st=0x%x cc=%u\n",
|
||||
dev->SlotId, inDci, outDci, st, result.CompletionCode);
|
||||
return st;
|
||||
}
|
||||
|
||||
UsbTrace("usb-storage: bulk endpoints configured slot=%u if=%u in=%02x/dci=%u out=%02x/dci=%u\n",
|
||||
dev->SlotId,
|
||||
dev->MassInterface,
|
||||
dev->MassBulkInAddress,
|
||||
inDci,
|
||||
dev->MassBulkOutAddress,
|
||||
outDci);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
uint8_t UsbFindFreeDiskIndex(void) {
|
||||
for (uint8_t i = 0; i < 8; i++) {
|
||||
if (!IoDiskDevicePresent(i))
|
||||
return i;
|
||||
}
|
||||
return 0xFFu;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageRegisterDisk(XHCI_DEVICE* dev) {
|
||||
if (!dev || dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE || dev->MassBlockCount == 0)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
|
||||
DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0;
|
||||
KSTATUS st = IoCreateDevice(&gUsbStorageDriver, 0, &disk);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
disk->DeviceExtension = dev;
|
||||
disk->SectorCount = dev->MassBlockCount;
|
||||
disk->AlignmentRequirement = USB_STORAGE_SECTOR_SIZE;
|
||||
disk->Flags |= DEVICE_OBJECT_REMOVABLE;
|
||||
|
||||
uint8_t index = UsbFindFreeDiskIndex();
|
||||
if (index == 0xFFu) {
|
||||
IoDeleteDevice(disk);
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
st = IoRegisterDiskDevice(index, disk);
|
||||
if (!KSUCCESS(st)) {
|
||||
IoDeleteDevice(disk);
|
||||
return st;
|
||||
}
|
||||
|
||||
dev->MassDeviceObject = disk;
|
||||
DbgPrint("usb-storage: registered disk %u slot=%u sectors=%llu\n",
|
||||
index, dev->SlotId, (unsigned long long)dev->MassBlockCount);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
bool UsbStorageMatch(const IO_PUBLISHED_DEVICE* pub, void* context) {
|
||||
(void)context;
|
||||
if (!pub || pub->BusType != IoBusUsb)
|
||||
return false;
|
||||
if (pub->u.Usb.InterfaceClass != USB_CLASS_MASS_STORAGE)
|
||||
return false;
|
||||
if (pub->u.Usb.InterfaceSubclass != USB_MASS_SUBCLASS_SCSI)
|
||||
return false;
|
||||
if (pub->u.Usb.InterfaceProtocol != USB_MASS_PROTOCOL_BOT)
|
||||
return false;
|
||||
|
||||
return UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true) &&
|
||||
UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS UsbStorageBind(IO_PUBLISHED_DEVICE* pub, void* context) {
|
||||
(void)context;
|
||||
if (!pub || !pub->BusContext || !pub->DeviceContext)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
XHCI_CONTROLLER* xhci = (XHCI_CONTROLLER*)pub->BusContext;
|
||||
XHCI_DEVICE* dev = (XHCI_DEVICE*)pub->DeviceContext;
|
||||
if (dev->MassActive || dev->MassCandidate)
|
||||
return STATUS_ALREADY_EXISTS;
|
||||
|
||||
const IO_PUBLISHED_USB_ENDPOINT* bulkIn = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, true);
|
||||
const IO_PUBLISHED_USB_ENDPOINT* bulkOut = UsbFindEndpoint(pub, USB_ENDPOINT_TYPE_BULK, false);
|
||||
if (!bulkIn || !bulkOut)
|
||||
return STATUS_NOT_FOUND;
|
||||
|
||||
dev->MassCandidate = true;
|
||||
dev->MassInterface = pub->u.Usb.InterfaceNumber;
|
||||
dev->MassBulkInAddress = bulkIn->Address;
|
||||
dev->MassBulkOutAddress = bulkOut->Address;
|
||||
dev->MassBulkInEndpointId = XhciEndpointIdFromAddress(bulkIn->Address);
|
||||
dev->MassBulkOutEndpointId = XhciEndpointIdFromAddress(bulkOut->Address);
|
||||
dev->MassBulkInMaxPacket = bulkIn->MaxPacketSize;
|
||||
dev->MassBulkOutMaxPacket = bulkOut->MaxPacketSize;
|
||||
|
||||
KSTATUS st = XhciConfigureMassStorageEndpoints(xhci, dev);
|
||||
if (!KSUCCESS(st)) goto fail;
|
||||
|
||||
st = UsbStorageInquiry(xhci, dev);
|
||||
if (!KSUCCESS(st)) goto fail;
|
||||
|
||||
st = UsbStorageTestUnitReady(xhci, dev);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("usb-storage: TEST UNIT READY failed slot=%u st=0x%x\n", dev->SlotId, st);
|
||||
goto fail;
|
||||
}
|
||||
|
||||
st = UsbStorageReadCapacity(xhci, dev);
|
||||
if (!KSUCCESS(st)) goto fail;
|
||||
if (dev->MassBlockSize != USB_STORAGE_SECTOR_SIZE) {
|
||||
DbgPrint("usb-storage: unsupported block size slot=%u block_size=%u\n",
|
||||
dev->SlotId, dev->MassBlockSize);
|
||||
st = STATUS_NOT_SUPPORTED;
|
||||
goto fail;
|
||||
}
|
||||
|
||||
dev->MassActive = true;
|
||||
st = UsbStorageRegisterDisk(dev);
|
||||
if (!KSUCCESS(st)) goto fail;
|
||||
|
||||
DbgPrint("usb-storage: bound slot=%u if=%u\n", dev->SlotId, dev->MassInterface);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
fail:
|
||||
dev->MassCandidate = false;
|
||||
dev->MassActive = false;
|
||||
DbgPrint("usb-storage: bind failed slot=%u if=%u st=0x%x\n",
|
||||
dev->SlotId, pub->u.Usb.InterfaceNumber, st);
|
||||
return st;
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,9 +1,10 @@
|
||||
[boot]
|
||||
ps2.sys
|
||||
ide.sys
|
||||
ahci.sys
|
||||
usb.sys
|
||||
fat32.sys
|
||||
|
||||
[runtime]
|
||||
lwip.sys
|
||||
usb.sys
|
||||
rtl8139.sys
|
||||
+119
-154
@@ -1,54 +1,5 @@
|
||||
// fat32.c - FAT32 filesystem driver
|
||||
// Read + write support: file data, cluster allocation, directory entries.
|
||||
//
|
||||
// Performance improvements over the original:
|
||||
//
|
||||
// 1. FAT sector cache (FAT_CACHE_SLOTS write-back entries).
|
||||
// FatEntry() and WriteFatEntry() go through the cache; the hot FAT
|
||||
// region stays in RAM instead of being read from disk on every chain
|
||||
// traversal step. A 32-slot cache covers 32 x (512/4) = 4096
|
||||
// consecutive cluster entries — enough to keep the entire FAT of a
|
||||
// small partition in memory, and to avoid repeated reads even on large
|
||||
// ones.
|
||||
//
|
||||
// 2. Fat32ReadDir() cluster buffer cached in FAT32_DIR_CTX.
|
||||
// The original re-read the same cluster from disk on every call
|
||||
// (allocate -> read -> return one entry -> free). Now the buffer is
|
||||
// allocated once in Fat32OpenDir, freed in Fat32CloseDir, and only
|
||||
// re-read when the iteration moves to a new cluster.
|
||||
//
|
||||
// 3. AllocCluster() next-free hint.
|
||||
// The search now starts from the cluster after the last allocation
|
||||
// (volume NextFreeHint) and wraps around once, so repeated allocations
|
||||
// on a fragmented volume don't rescan from sector 0 every time.
|
||||
//
|
||||
// 4. Contiguous-cluster run coalescing in Fat32Read() and Fat32Write().
|
||||
// After fetching a cluster's FAT entry the code peeks ahead: if the
|
||||
// next cluster is physically adjacent (cluster+1) it extends the run
|
||||
// rather than issuing a separate ReadSectors/WriteSectors call. A
|
||||
// freshly-written unfragmented file now transfers in one or a handful
|
||||
// of large DMA bursts instead of SectorsPerCluster-sized slices.
|
||||
//
|
||||
// 5. FAT write batching via a dirty-sector cache.
|
||||
// WriteFatEntry() marks modified FAT cache slots dirty instead of
|
||||
// flushing immediately. FlushFatCache() (called from Fat32Close,
|
||||
// Fat32Truncate, and Fat32Delete) writes each dirty slot once for
|
||||
// every FAT copy, collapsing the 2x(read+write) per-entry cost of the
|
||||
// original into a single write per dirty sector per FAT copy.
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/io_file.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
// fat32.c - FAT32 filesystem core, directory, cache, and I/O implementation.
|
||||
#include "fat32.h"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// On-disk structures
|
||||
@@ -1042,7 +993,7 @@ static int FindFreeDirentRun(FAT32_VOL *v, uint32_t dirCluster, uint32_t needed,
|
||||
// ---------------------------------------------------------------------------
|
||||
// Open / Close
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
|
||||
KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
|
||||
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
@@ -1059,7 +1010,7 @@ static KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
|
||||
void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
|
||||
(void)fs;
|
||||
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
|
||||
FlushFatCache(f->Vol);
|
||||
@@ -1079,7 +1030,7 @@ static void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
|
||||
|
||||
#define MAX_RUN_SECTORS 128u
|
||||
|
||||
static KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
uint64_t offset, void *buf,
|
||||
size_t size, size_t *bytesRead) {
|
||||
(void)fs;
|
||||
@@ -1174,7 +1125,7 @@ static KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
// Same strategy as read. Whole-cluster-aligned runs skip the read-modify-
|
||||
// write; partial head/tail clusters still require it.
|
||||
|
||||
static KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
uint64_t offset, const void *buf,
|
||||
size_t size, size_t *bytesWritten) {
|
||||
(void)fs;
|
||||
@@ -1279,9 +1230,9 @@ static KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
// ---------------------------------------------------------------------------
|
||||
// Create
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
|
||||
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
|
||||
|
||||
static KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
|
||||
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
|
||||
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
@@ -1385,7 +1336,7 @@ static KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileC
|
||||
// ---------------------------------------------------------------------------
|
||||
// Truncate
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
|
||||
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
|
||||
(void)fs;
|
||||
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
|
||||
FAT32_VOL *v = f->Vol;
|
||||
@@ -1445,7 +1396,7 @@ update_size:
|
||||
// ---------------------------------------------------------------------------
|
||||
// Delete
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
|
||||
KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
|
||||
if (!fs || !SysSvIsValid(path) || path.Length == 0)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
@@ -1467,7 +1418,7 @@ static KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
|
||||
uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
|
||||
(void)fs; return ((FAT32_FILE *)fileCtx)->FileSize;
|
||||
}
|
||||
|
||||
@@ -1489,7 +1440,7 @@ typedef struct {
|
||||
uint8_t *ClusterBuf; // allocated once in OpenDir, freed in CloseDir
|
||||
} FAT32_DIR_CTX;
|
||||
|
||||
static KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx) {
|
||||
KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx) {
|
||||
if (!fs || !dirCtx || !SysSvIsValid(dirPath))
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
@@ -1520,7 +1471,7 @@ static KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **d
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
|
||||
KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
|
||||
(void)fs;
|
||||
FAT32_DIR_CTX *ctx = (FAT32_DIR_CTX *)dirCtx;
|
||||
FAT32_VOL *v = ctx->Vol;
|
||||
@@ -1575,7 +1526,7 @@ static KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
|
||||
return STATUS_NO_MORE_FILES;
|
||||
}
|
||||
|
||||
static void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx) {
|
||||
void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx) {
|
||||
(void)fs;
|
||||
FAT32_DIR_CTX *ctx = (FAT32_DIR_CTX *)dirCtx;
|
||||
MmFreePool(ctx->ClusterBuf);
|
||||
@@ -1934,7 +1885,7 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
|
||||
Fat32AsyncFinish(ctx, s, ctx->Done);
|
||||
}
|
||||
|
||||
static KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
uint64_t offset, void *buf, size_t size,
|
||||
IO_FS_ASYNC_COMPLETE complete,
|
||||
void *completeContext) {
|
||||
@@ -2087,7 +2038,7 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
|
||||
Fat32AsyncFinish(ctx, s, ctx->Done);
|
||||
}
|
||||
|
||||
static KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
uint64_t offset, const void *buf, size_t size,
|
||||
IO_FS_ASYNC_COMPLETE complete,
|
||||
void *completeContext) {
|
||||
@@ -2174,22 +2125,101 @@ static KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
|
||||
static FAT32_VOL gVols[FAT32_MAX_VOLUMES];
|
||||
static IO_FILESYSTEM gFs[FAT32_MAX_VOLUMES];
|
||||
|
||||
static void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
|
||||
memset(fs, 0, sizeof(*fs));
|
||||
fs->Open = Fat32Open;
|
||||
fs->Read = Fat32Read;
|
||||
fs->Write = Fat32Write;
|
||||
fs->ReadAsync = Fat32ReadAsync;
|
||||
fs->WriteAsync = Fat32WriteAsync;
|
||||
fs->Create = Fat32Create;
|
||||
fs->Delete = Fat32Delete;
|
||||
fs->Truncate = Fat32Truncate;
|
||||
fs->GetSize = Fat32GetSize;
|
||||
fs->Close = Fat32Close;
|
||||
fs->DriverCtx = vol;
|
||||
fs->OpenDir = Fat32OpenDir;
|
||||
fs->ReadDir = Fat32ReadDir;
|
||||
fs->CloseDir = Fat32CloseDir;
|
||||
|
||||
static uint32_t gMountedCount;
|
||||
static bool gDiskScanned[FAT32_MAX_VOLUMES];
|
||||
|
||||
static uint32_t Fat32TryMountDisk(uint8_t disk) {
|
||||
if (disk >= FAT32_MAX_VOLUMES)
|
||||
return 0;
|
||||
if (gDiskScanned[disk])
|
||||
return 0;
|
||||
gDiskScanned[disk] = true;
|
||||
|
||||
if (!IoDiskDevicePresent(disk))
|
||||
return 0;
|
||||
|
||||
uint32_t mountedHere = 0;
|
||||
FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK];
|
||||
uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK);
|
||||
if (!partCount) {
|
||||
DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk);
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (uint32_t part = 0; part < partCount && gMountedCount < FAT32_MAX_VOLUMES; part++) {
|
||||
uint8_t volumeIndex = (uint8_t)gMountedCount;
|
||||
FAT32_VOL* v = &gVols[volumeIndex];
|
||||
memset(v, 0, sizeof(*v));
|
||||
v->DriveIndex = disk;
|
||||
v->PartitionLba = parts[part].StartLba;
|
||||
v->PartitionSize = parts[part].SectorCount;
|
||||
|
||||
uint8_t sector[512];
|
||||
if (!KSUCCESS(ReadSectors(v, v->PartitionLba, 1, sector))) {
|
||||
DbgPrint("FAT32: disk %u part %u: failed to read VBR\n", disk, part);
|
||||
continue;
|
||||
}
|
||||
|
||||
BPB *bpb = (BPB *)sector;
|
||||
if (!IsFat32Vbr(v, v->PartitionLba)) {
|
||||
DbgPrint("FAT32: disk %u part %u: unsupported FAT variant\n", disk, part);
|
||||
continue;
|
||||
}
|
||||
|
||||
v->BytesPerSector = bpb->BytesPerSector;
|
||||
v->SectorsPerCluster = bpb->SectorsPerCluster;
|
||||
v->BytesPerCluster = bpb->BytesPerSector * bpb->SectorsPerCluster;
|
||||
v->ReservedSectors = bpb->ReservedSectors;
|
||||
v->NumFats = bpb->NumFats;
|
||||
v->FatSize = bpb->FatSize32;
|
||||
v->RootCluster = bpb->RootCluster;
|
||||
v->DataStart = v->PartitionLba + bpb->ReservedSectors + (uint64_t)bpb->NumFats * bpb->FatSize32;
|
||||
v->NextFreeHint = 2;
|
||||
|
||||
uint32_t totalSectors = bpb->TotalSectors16 ? bpb->TotalSectors16 : bpb->TotalSectors32;
|
||||
uint32_t dataSectors = totalSectors - bpb->ReservedSectors - bpb->NumFats * bpb->FatSize32;
|
||||
v->TotalClusters = dataSectors / bpb->SectorsPerCluster;
|
||||
if (v->PartitionSize == 0 || v->PartitionSize > totalSectors)
|
||||
v->PartitionSize = totalSectors;
|
||||
|
||||
Fat32InitVtable(&gFs[volumeIndex], v);
|
||||
|
||||
IO_VOLUME_PARAMS volParams;
|
||||
volParams.DiskIndex = v->DriveIndex;
|
||||
volParams.PartitionLba = v->PartitionLba;
|
||||
volParams.PartitionSize = v->PartitionSize;
|
||||
|
||||
KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams);
|
||||
if (KSUCCESS(status)) {
|
||||
gMountedCount++;
|
||||
mountedHere++;
|
||||
DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n",
|
||||
disk, part, volumeIndex, (unsigned long long)v->PartitionLba, v->TotalClusters);
|
||||
} else {
|
||||
DbgPrint("FAT32: disk %u part %u mount as drive %u failed status=%d.\n",
|
||||
disk, part, volumeIndex, status);
|
||||
}
|
||||
}
|
||||
|
||||
if (partCount && !mountedHere && gMountedCount >= FAT32_MAX_VOLUMES) {
|
||||
DbgPrint("FAT32: disk %u: volume table full\n", disk);
|
||||
}
|
||||
|
||||
return mountedHere;
|
||||
}
|
||||
|
||||
static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* context) {
|
||||
(void)device;
|
||||
(void)context;
|
||||
uint32_t before = gMountedCount;
|
||||
uint32_t mounted = Fat32TryMountDisk(driveIndex);
|
||||
if (mounted) {
|
||||
DbgPrint("FAT32: hotplug scan mounted %u new volume(s); total=%u\n",
|
||||
mounted, gMountedCount);
|
||||
} else if (gMountedCount != before) {
|
||||
DbgPrint("FAT32: hotplug scan total=%u\n", gMountedCount);
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API *hal) {
|
||||
@@ -2198,84 +2228,19 @@ KSTATUS DriverEntry(const HAL_API *hal) {
|
||||
|
||||
FatCacheInvalidate();
|
||||
(void)hal;
|
||||
gMountedCount = 0;
|
||||
memset(gDiskScanned, 0, sizeof(gDiskScanned));
|
||||
|
||||
uint32_t mounted = 0;
|
||||
KSTATUS lastStatus = STATUS_NOT_SUPPORTED;
|
||||
|
||||
for (uint8_t disk = 0; disk < FAT32_MAX_VOLUMES && mounted < FAT32_MAX_VOLUMES; disk++) {
|
||||
if (!IoDiskDevicePresent(disk))
|
||||
continue;
|
||||
|
||||
FAT32_PARTITION_CANDIDATE parts[FAT32_MAX_PARTITIONS_PER_DISK];
|
||||
uint32_t partCount = Fat32EnumerateDiskVolumes(disk, parts, FAT32_MAX_PARTITIONS_PER_DISK);
|
||||
if (!partCount) {
|
||||
DbgPrint("FAT32: disk %u: no FAT32 volumes\n", disk);
|
||||
continue;
|
||||
}
|
||||
|
||||
for (uint32_t part = 0; part < partCount && mounted < FAT32_MAX_VOLUMES; part++) {
|
||||
uint8_t volumeIndex = (uint8_t)mounted;
|
||||
FAT32_VOL* v = &gVols[volumeIndex];
|
||||
memset(v, 0, sizeof(*v));
|
||||
v->DriveIndex = disk;
|
||||
v->PartitionLba = parts[part].StartLba;
|
||||
v->PartitionSize = parts[part].SectorCount;
|
||||
|
||||
uint8_t sector[512];
|
||||
if (!KSUCCESS(ReadSectors(v, v->PartitionLba, 1, sector))) {
|
||||
lastStatus = STATUS_DEVICE_ERROR;
|
||||
DbgPrint("FAT32: disk %u part %u: failed to read VBR\n", disk, part);
|
||||
continue;
|
||||
}
|
||||
|
||||
BPB *bpb = (BPB *)sector;
|
||||
if (!IsFat32Vbr(v, v->PartitionLba)) {
|
||||
lastStatus = STATUS_NOT_SUPPORTED;
|
||||
DbgPrint("FAT32: disk %u part %u: unsupported FAT variant\n", disk, part);
|
||||
continue;
|
||||
}
|
||||
|
||||
v->BytesPerSector = bpb->BytesPerSector;
|
||||
v->SectorsPerCluster = bpb->SectorsPerCluster;
|
||||
v->BytesPerCluster = bpb->BytesPerSector * bpb->SectorsPerCluster;
|
||||
v->ReservedSectors = bpb->ReservedSectors;
|
||||
v->NumFats = bpb->NumFats;
|
||||
v->FatSize = bpb->FatSize32;
|
||||
v->RootCluster = bpb->RootCluster;
|
||||
v->DataStart = v->PartitionLba + bpb->ReservedSectors + (uint64_t)bpb->NumFats * bpb->FatSize32;
|
||||
v->NextFreeHint = 2;
|
||||
|
||||
uint32_t totalSectors = bpb->TotalSectors16 ? bpb->TotalSectors16 : bpb->TotalSectors32;
|
||||
uint32_t dataSectors = totalSectors - bpb->ReservedSectors - bpb->NumFats * bpb->FatSize32;
|
||||
v->TotalClusters = dataSectors / bpb->SectorsPerCluster;
|
||||
if (v->PartitionSize == 0 || v->PartitionSize > totalSectors)
|
||||
v->PartitionSize = totalSectors;
|
||||
|
||||
Fat32InitVtable(&gFs[volumeIndex], v);
|
||||
|
||||
IO_VOLUME_PARAMS volParams;
|
||||
volParams.DiskIndex = v->DriveIndex;
|
||||
volParams.PartitionLba = v->PartitionLba;
|
||||
volParams.PartitionSize = v->PartitionSize;
|
||||
|
||||
KSTATUS status = IoRegisterFilesystem(volumeIndex, &gFs[volumeIndex], &volParams);
|
||||
if (KSUCCESS(status)) {
|
||||
mounted++;
|
||||
DbgPrint("FAT32: disk %u part %u mounted as drive %u (lba=%llu clusters=%u).\n",
|
||||
disk, part, volumeIndex, (unsigned long long)v->PartitionLba, v->TotalClusters);
|
||||
} else {
|
||||
lastStatus = status;
|
||||
DbgPrint("FAT32: disk %u part %u mount as drive %u failed status=%d.\n",
|
||||
disk, part, volumeIndex, status);
|
||||
}
|
||||
}
|
||||
KSTATUS cbStatus = IoRegisterDiskArrivalCallback(Fat32DiskArrival, (void*)0);
|
||||
if (!KSUCCESS(cbStatus)) {
|
||||
DbgPrint("FAT32: disk-arrival callback registration failed st=0x%x\n", cbStatus);
|
||||
}
|
||||
|
||||
if (!mounted) {
|
||||
if (!gMountedCount) {
|
||||
DbgPrint("FAT32: no mountable FAT32 volumes found.\n");
|
||||
return lastStatus;
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
DbgPrint("FAT32: Init complete; mounted %u volume(s).\n", mounted);
|
||||
DbgPrint("FAT32: Init complete; mounted %u volume(s).\n", gMountedCount);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// fat32.h - shared FAT32 driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/io_file.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
|
||||
typedef struct FAT32_VOL_S FAT32_VOL;
|
||||
|
||||
KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
|
||||
void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx);
|
||||
KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, size_t *bytesRead);
|
||||
KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, size_t *bytesWritten);
|
||||
KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
|
||||
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
|
||||
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
|
||||
KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
|
||||
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
|
||||
uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx);
|
||||
KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx);
|
||||
KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out);
|
||||
void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx);
|
||||
|
||||
void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol);
|
||||
@@ -0,0 +1,20 @@
|
||||
// fat32_mount.c - filesystem vtable/mount wiring.
|
||||
#include "fat32.h"
|
||||
|
||||
void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
|
||||
memset(fs, 0, sizeof(*fs));
|
||||
fs->Open = Fat32Open;
|
||||
fs->Read = Fat32Read;
|
||||
fs->Write = Fat32Write;
|
||||
fs->ReadAsync = Fat32ReadAsync;
|
||||
fs->WriteAsync = Fat32WriteAsync;
|
||||
fs->Create = Fat32Create;
|
||||
fs->Delete = Fat32Delete;
|
||||
fs->Truncate = Fat32Truncate;
|
||||
fs->GetSize = Fat32GetSize;
|
||||
fs->Close = Fat32Close;
|
||||
fs->DriverCtx = vol;
|
||||
fs->OpenDir = Fat32OpenDir;
|
||||
fs->ReadDir = Fat32ReadDir;
|
||||
fs->CloseDir = Fat32CloseDir;
|
||||
}
|
||||
@@ -0,0 +1,263 @@
|
||||
// ps2.c - PS/2 keyboard/mouse event decoding and kernel input binding.
|
||||
#include "ps2.h"
|
||||
|
||||
static KDPC gMouseDpc;
|
||||
static volatile bool gKeyboardNextExtended;
|
||||
static volatile uint8_t gMousePacket[3];
|
||||
static volatile int gMousePacketIndex;
|
||||
static PS2_MOUSE_PACKET gMouseDpcQueue[MOUSE_DPC_QUEUE_SIZE];
|
||||
static volatile int gMouseDpcHead;
|
||||
static volatile int gMouseDpcTail;
|
||||
|
||||
static INPUT_KEY Ps2Set1ToKey(uint8_t make, bool extended) {
|
||||
if (extended) {
|
||||
switch (make) {
|
||||
case 0x1C: return INPUT_KEY_KP_ENTER;
|
||||
case 0x1D: return INPUT_KEY_RIGHT_CTRL;
|
||||
case 0x35: return INPUT_KEY_KP_DIVIDE;
|
||||
case 0x38: return INPUT_KEY_RIGHT_ALT;
|
||||
case 0x47: return INPUT_KEY_HOME;
|
||||
case 0x48: return INPUT_KEY_UP;
|
||||
case 0x49: return INPUT_KEY_PAGE_UP;
|
||||
case 0x4B: return INPUT_KEY_LEFT;
|
||||
case 0x4D: return INPUT_KEY_RIGHT;
|
||||
case 0x4F: return INPUT_KEY_END;
|
||||
case 0x50: return INPUT_KEY_DOWN;
|
||||
case 0x51: return INPUT_KEY_PAGE_DOWN;
|
||||
case 0x52: return INPUT_KEY_INSERT;
|
||||
case 0x53: return INPUT_KEY_DELETE;
|
||||
default: return INPUT_KEY_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
switch (make) {
|
||||
case 0x01: return INPUT_KEY_ESCAPE;
|
||||
case 0x02: return INPUT_KEY_1;
|
||||
case 0x03: return INPUT_KEY_2;
|
||||
case 0x04: return INPUT_KEY_3;
|
||||
case 0x05: return INPUT_KEY_4;
|
||||
case 0x06: return INPUT_KEY_5;
|
||||
case 0x07: return INPUT_KEY_6;
|
||||
case 0x08: return INPUT_KEY_7;
|
||||
case 0x09: return INPUT_KEY_8;
|
||||
case 0x0A: return INPUT_KEY_9;
|
||||
case 0x0B: return INPUT_KEY_0;
|
||||
case 0x0C: return INPUT_KEY_MINUS;
|
||||
case 0x0D: return INPUT_KEY_EQUAL;
|
||||
case 0x0E: return INPUT_KEY_BACKSPACE;
|
||||
case 0x0F: return INPUT_KEY_TAB;
|
||||
case 0x10: return INPUT_KEY_Q;
|
||||
case 0x11: return INPUT_KEY_W;
|
||||
case 0x12: return INPUT_KEY_E;
|
||||
case 0x13: return INPUT_KEY_R;
|
||||
case 0x14: return INPUT_KEY_T;
|
||||
case 0x15: return INPUT_KEY_Y;
|
||||
case 0x16: return INPUT_KEY_U;
|
||||
case 0x17: return INPUT_KEY_I;
|
||||
case 0x18: return INPUT_KEY_O;
|
||||
case 0x19: return INPUT_KEY_P;
|
||||
case 0x1A: return INPUT_KEY_LEFT_BRACKET;
|
||||
case 0x1B: return INPUT_KEY_RIGHT_BRACKET;
|
||||
case 0x1C: return INPUT_KEY_ENTER;
|
||||
case 0x1D: return INPUT_KEY_LEFT_CTRL;
|
||||
case 0x1E: return INPUT_KEY_A;
|
||||
case 0x1F: return INPUT_KEY_S;
|
||||
case 0x20: return INPUT_KEY_D;
|
||||
case 0x21: return INPUT_KEY_F;
|
||||
case 0x22: return INPUT_KEY_G;
|
||||
case 0x23: return INPUT_KEY_H;
|
||||
case 0x24: return INPUT_KEY_J;
|
||||
case 0x25: return INPUT_KEY_K;
|
||||
case 0x26: return INPUT_KEY_L;
|
||||
case 0x27: return INPUT_KEY_SEMICOLON;
|
||||
case 0x28: return INPUT_KEY_APOSTROPHE;
|
||||
case 0x29: return INPUT_KEY_GRAVE;
|
||||
case 0x2A: return INPUT_KEY_LEFT_SHIFT;
|
||||
case 0x2B: return INPUT_KEY_BACKSLASH;
|
||||
case 0x2C: return INPUT_KEY_Z;
|
||||
case 0x2D: return INPUT_KEY_X;
|
||||
case 0x2E: return INPUT_KEY_C;
|
||||
case 0x2F: return INPUT_KEY_V;
|
||||
case 0x30: return INPUT_KEY_B;
|
||||
case 0x31: return INPUT_KEY_N;
|
||||
case 0x32: return INPUT_KEY_M;
|
||||
case 0x33: return INPUT_KEY_COMMA;
|
||||
case 0x34: return INPUT_KEY_PERIOD;
|
||||
case 0x35: return INPUT_KEY_SLASH;
|
||||
case 0x36: return INPUT_KEY_RIGHT_SHIFT;
|
||||
case 0x37: return INPUT_KEY_KP_MULTIPLY;
|
||||
case 0x38: return INPUT_KEY_LEFT_ALT;
|
||||
case 0x39: return INPUT_KEY_SPACE;
|
||||
case 0x3A: return INPUT_KEY_CAPS_LOCK;
|
||||
case 0x3B: return INPUT_KEY_F1;
|
||||
case 0x3C: return INPUT_KEY_F2;
|
||||
case 0x3D: return INPUT_KEY_F3;
|
||||
case 0x3E: return INPUT_KEY_F4;
|
||||
case 0x3F: return INPUT_KEY_F5;
|
||||
case 0x40: return INPUT_KEY_F6;
|
||||
case 0x41: return INPUT_KEY_F7;
|
||||
case 0x42: return INPUT_KEY_F8;
|
||||
case 0x43: return INPUT_KEY_F9;
|
||||
case 0x44: return INPUT_KEY_F10;
|
||||
case 0x45: return INPUT_KEY_NUM_LOCK;
|
||||
case 0x47: return INPUT_KEY_KP_7;
|
||||
case 0x48: return INPUT_KEY_KP_8;
|
||||
case 0x49: return INPUT_KEY_KP_9;
|
||||
case 0x4A: return INPUT_KEY_KP_MINUS;
|
||||
case 0x4B: return INPUT_KEY_KP_4;
|
||||
case 0x4C: return INPUT_KEY_KP_5;
|
||||
case 0x4D: return INPUT_KEY_KP_6;
|
||||
case 0x4E: return INPUT_KEY_KP_PLUS;
|
||||
case 0x4F: return INPUT_KEY_KP_1;
|
||||
case 0x50: return INPUT_KEY_KP_2;
|
||||
case 0x51: return INPUT_KEY_KP_3;
|
||||
case 0x52: return INPUT_KEY_KP_0;
|
||||
case 0x53: return INPUT_KEY_KP_PERIOD;
|
||||
case 0x57: return INPUT_KEY_F11;
|
||||
case 0x58: return INPUT_KEY_F12;
|
||||
default: return INPUT_KEY_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
static void Ps2KeyboardIsr(void) {
|
||||
uint8_t sc = __inbyte(PS2_DATA);
|
||||
if (sc == SC1_EXT_PREFIX) {
|
||||
gKeyboardNextExtended = true;
|
||||
return;
|
||||
}
|
||||
|
||||
bool extended = gKeyboardNextExtended;
|
||||
bool released = (sc & SC1_BREAK_BIT) != 0;
|
||||
uint8_t make = sc & SC1_MAKE_MASK;
|
||||
gKeyboardNextExtended = false;
|
||||
|
||||
INPUT_KEY key = Ps2Set1ToKey(make, extended);
|
||||
if (key == INPUT_KEY_NONE) return;
|
||||
|
||||
INPUT_KEY_EVENT event = {
|
||||
.Key = key,
|
||||
.Pressed = !released,
|
||||
};
|
||||
KeInputSubmitKey(&event);
|
||||
}
|
||||
|
||||
static void Ps2MouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
|
||||
(void)dpc;
|
||||
(void)context;
|
||||
(void)systemArgument1;
|
||||
(void)systemArgument2;
|
||||
|
||||
while (gMouseDpcHead != gMouseDpcTail) {
|
||||
PS2_MOUSE_PACKET packet = gMouseDpcQueue[gMouseDpcHead];
|
||||
gMouseDpcHead = (gMouseDpcHead + 1) & (int)MOUSE_DPC_QUEUE_MASK;
|
||||
|
||||
INPUT_MOUSE_EVENT event = {
|
||||
.dx = packet.dx,
|
||||
.dy = packet.dy,
|
||||
.wheel = 0,
|
||||
.buttons = packet.buttons,
|
||||
};
|
||||
KeInputSubmitMouse(&event);
|
||||
}
|
||||
}
|
||||
|
||||
static void Ps2MouseIsr(void) {
|
||||
uint8_t data = __inbyte(PS2_DATA);
|
||||
|
||||
if (gMousePacketIndex == 0 && !(data & 0x08u))
|
||||
return;
|
||||
|
||||
gMousePacket[gMousePacketIndex++] = data;
|
||||
if (gMousePacketIndex < 3)
|
||||
return;
|
||||
gMousePacketIndex = 0;
|
||||
|
||||
uint8_t flags = gMousePacket[0];
|
||||
if (flags & 0xC0u)
|
||||
return;
|
||||
|
||||
int dx = (int)gMousePacket[1] - ((flags & 0x10u) ? 256 : 0);
|
||||
int dy = (int)gMousePacket[2] - ((flags & 0x20u) ? 256 : 0);
|
||||
dy = -dy;
|
||||
|
||||
int next = (gMouseDpcTail + 1) & (int)MOUSE_DPC_QUEUE_MASK;
|
||||
if (next != gMouseDpcHead) {
|
||||
gMouseDpcQueue[gMouseDpcTail].dx = dx;
|
||||
gMouseDpcQueue[gMouseDpcTail].dy = dy;
|
||||
gMouseDpcQueue[gMouseDpcTail].buttons = flags & 0x07u;
|
||||
gMouseDpcTail = next;
|
||||
KeInsertQueueDpc(&gMouseDpc, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
static void Ps2ControllerPrepare(void) {
|
||||
Ps2WriteCommand(0xADu); // disable keyboard port
|
||||
Ps2WriteCommand(0xA7u); // disable aux port
|
||||
Ps2FlushOutput();
|
||||
|
||||
uint8_t commandByte = Ps2ReadCommandByte();
|
||||
commandByte &= (uint8_t)~0x01u; // keep keyboard IRQ disabled during polled init
|
||||
commandByte &= (uint8_t)~0x02u; // keep mouse IRQ disabled during polled init
|
||||
commandByte |= 0x40u; // translate keyboard stream to set 1
|
||||
commandByte &= (uint8_t)~0x10u; // keyboard clock enabled
|
||||
commandByte &= (uint8_t)~0x20u; // mouse clock enabled
|
||||
Ps2WriteCommandByte(commandByte);
|
||||
|
||||
Ps2WriteCommand(0xAEu); // enable keyboard port
|
||||
Ps2WriteCommand(0xA8u); // enable aux port
|
||||
Ps2FlushOutput();
|
||||
}
|
||||
|
||||
static void Ps2ControllerEnableInterrupts(void) {
|
||||
uint8_t commandByte = Ps2ReadCommandByte();
|
||||
commandByte |= 0x01u;
|
||||
commandByte |= 0x02u;
|
||||
Ps2WriteCommandByte(commandByte);
|
||||
}
|
||||
|
||||
static void Ps2KeyboardInitialize(void) {
|
||||
// Enable scanning. If the keyboard is absent, the timeout/ACK failure is
|
||||
// non-fatal; the IRQ path simply never produces events.
|
||||
if (!Ps2SendKeyboard(0xF4u))
|
||||
DbgPrint("ps2: keyboard enable did not ACK\n");
|
||||
}
|
||||
|
||||
static void Ps2MouseInitialize(void) {
|
||||
if (!Ps2SendMouse(0xFFu)) {
|
||||
DbgPrint("ps2: mouse reset did not ACK\n");
|
||||
return;
|
||||
}
|
||||
|
||||
(void)Ps2ReadData(); // BAT completion, usually 0xAA
|
||||
(void)Ps2ReadData(); // device id, usually 0x00
|
||||
|
||||
if (!Ps2SendMouse(0xF6u))
|
||||
DbgPrint("ps2: mouse set-defaults did not ACK\n");
|
||||
if (!Ps2SendMouse(0xF4u))
|
||||
DbgPrint("ps2: mouse enable did not ACK\n");
|
||||
}
|
||||
|
||||
KSTATUS DriverEntry(const HAL_API* hal) {
|
||||
(void)hal;
|
||||
|
||||
uint32_t keyboardId = 0, mouseId = 0;
|
||||
KeInputRegisterDevice("ps2-keyboard", 0, &keyboardId);
|
||||
KeInputRegisterDevice("ps2-mouse", 0, &mouseId);
|
||||
|
||||
KeInitializeDpc(&gMouseDpc, Ps2MouseDpcRoutine, 0);
|
||||
|
||||
// Keep 8042 IRQ generation disabled while doing polled controller/device
|
||||
// setup. The old ordering routed IRQ1/IRQ12 first, which let ACK/BAT bytes
|
||||
// race the ISR and get consumed as fake key/mouse input before the polled
|
||||
// init path could read them.
|
||||
Ps2ControllerPrepare();
|
||||
Ps2KeyboardInitialize();
|
||||
Ps2MouseInitialize();
|
||||
|
||||
KeIrqRegister(PS2_IRQ_KEYBOARD, Ps2KeyboardIsr);
|
||||
KeIrqRegister(PS2_IRQ_MOUSE, Ps2MouseIsr);
|
||||
Ps2ControllerEnableInterrupts();
|
||||
|
||||
DbgPrint("ps2: initialized keyboard=%u mouse=%u\n", keyboardId, mouseId);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// ps2.h - shared i8042/PS/2 driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/status.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/ddi/input_device.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
|
||||
#define PS2_DATA 0x60u
|
||||
#define PS2_STAT 0x64u
|
||||
#define PS2_CMD 0x64u
|
||||
|
||||
#define PS2_STAT_OUT_FULL 0x01u
|
||||
#define PS2_STAT_IN_FULL 0x02u
|
||||
|
||||
#define PS2_IRQ_KEYBOARD 1u
|
||||
#define PS2_IRQ_MOUSE 12u
|
||||
|
||||
#define SC1_BREAK_BIT 0x80u
|
||||
#define SC1_MAKE_MASK 0x7Fu
|
||||
#define SC1_EXT_PREFIX 0xE0u
|
||||
|
||||
#define MOUSE_DPC_QUEUE_SIZE 64u
|
||||
#define MOUSE_DPC_QUEUE_MASK (MOUSE_DPC_QUEUE_SIZE - 1u)
|
||||
|
||||
typedef struct PS2_MOUSE_PACKET {
|
||||
int dx;
|
||||
int dy;
|
||||
uint8_t buttons;
|
||||
} PS2_MOUSE_PACKET;
|
||||
|
||||
void Ps2Pause(void);
|
||||
void Ps2WaitInput(void);
|
||||
bool Ps2WaitOutput(void);
|
||||
void Ps2FlushOutput(void);
|
||||
uint8_t Ps2ReadData(void);
|
||||
void Ps2WriteCommand(uint8_t command);
|
||||
void Ps2WriteData(uint8_t data);
|
||||
uint8_t Ps2ReadCommandByte(void);
|
||||
void Ps2WriteCommandByte(uint8_t commandByte);
|
||||
bool Ps2SendKeyboard(uint8_t byte);
|
||||
bool Ps2SendMouse(uint8_t byte);
|
||||
@@ -0,0 +1,64 @@
|
||||
// ps2_i8042.c - 8042 controller and device command helpers.
|
||||
#include "ps2.h"
|
||||
|
||||
void Ps2Pause(void) {
|
||||
_mm_pause();
|
||||
}
|
||||
|
||||
void Ps2WaitInput(void) {
|
||||
int timeout = 100000;
|
||||
while ((__inbyte(PS2_STAT) & PS2_STAT_IN_FULL) && --timeout) {
|
||||
Ps2Pause();
|
||||
}
|
||||
}
|
||||
|
||||
bool Ps2WaitOutput(void) {
|
||||
int timeout = 5000000;
|
||||
while (!(__inbyte(PS2_STAT) & PS2_STAT_OUT_FULL) && --timeout) {
|
||||
Ps2Pause();
|
||||
}
|
||||
return timeout != 0;
|
||||
}
|
||||
|
||||
void Ps2FlushOutput(void) {
|
||||
int timeout = 100000;
|
||||
while ((__inbyte(PS2_STAT) & PS2_STAT_OUT_FULL) && --timeout) {
|
||||
(void)__inbyte(PS2_DATA);
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t Ps2ReadData(void) {
|
||||
if (!Ps2WaitOutput()) return 0xFFu;
|
||||
return __inbyte(PS2_DATA);
|
||||
}
|
||||
|
||||
void Ps2WriteCommand(uint8_t command) {
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, command);
|
||||
}
|
||||
|
||||
void Ps2WriteData(uint8_t data) {
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_DATA, data);
|
||||
}
|
||||
|
||||
uint8_t Ps2ReadCommandByte(void) {
|
||||
Ps2WriteCommand(0x20u);
|
||||
return Ps2ReadData();
|
||||
}
|
||||
|
||||
void Ps2WriteCommandByte(uint8_t commandByte) {
|
||||
Ps2WriteCommand(0x60u);
|
||||
Ps2WriteData(commandByte);
|
||||
}
|
||||
|
||||
bool Ps2SendKeyboard(uint8_t byte) {
|
||||
Ps2WriteData(byte);
|
||||
return Ps2ReadData() == 0xFAu;
|
||||
}
|
||||
|
||||
bool Ps2SendMouse(uint8_t byte) {
|
||||
Ps2WriteCommand(0xD4u);
|
||||
Ps2WriteData(byte);
|
||||
return Ps2ReadData() == 0xFAu;
|
||||
}
|
||||
@@ -4,7 +4,8 @@
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#pragma intrinsic(_disable)
|
||||
#pragma intrinsic(_enable)
|
||||
#include <lwip/opt.h>
|
||||
#include <lwip/sys.h>
|
||||
#include <lwip/err.h>
|
||||
|
||||
@@ -1,150 +1,9 @@
|
||||
// rtl8139.c - Realtek RTL8139 / QEMU rtl8139 PCI fast Ethernet driver.
|
||||
//
|
||||
// Scope for this first version:
|
||||
// * PCI ID 10ec:8139, legacy INTx interrupts.
|
||||
// * MMIO preferred, I/O BAR fallback.
|
||||
// * Classic RTL8139 register interface: one RX ring buffer and four TX slots.
|
||||
// * No checksum offload, C+ mode, VLAN, multicast hash programming, or power mgmt.
|
||||
//
|
||||
// This is intentionally a separate mini-driver from rtl816x.sys. RTL8139 and
|
||||
// RTL8111/8168 share a vendor but not a useful packet-DMA programming model.
|
||||
// rtl8139.c - packet I/O, interrupt, and DriverEntry path.
|
||||
#include "rtl8139.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <ntlite/ddi/net_device.h>
|
||||
|
||||
#define RTL8139_VENDOR_REALTEK 0x10ECu
|
||||
#define RTL8139_DEVICE_ID 0x8139u
|
||||
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_COMMAND_IO (1u << 0)
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
|
||||
#define RTL8139_REG_IDR0 0x00u
|
||||
#define RTL8139_REG_MAR0 0x08u
|
||||
#define RTL8139_REG_TSD0 0x10u
|
||||
#define RTL8139_REG_TSAD0 0x20u
|
||||
#define RTL8139_REG_RBSTART 0x30u
|
||||
#define RTL8139_REG_CMD 0x37u
|
||||
#define RTL8139_REG_CAPR 0x38u
|
||||
#define RTL8139_REG_CBR 0x3Au
|
||||
#define RTL8139_REG_IMR 0x3Cu
|
||||
#define RTL8139_REG_ISR 0x3Eu
|
||||
#define RTL8139_REG_TCR 0x40u
|
||||
#define RTL8139_REG_RCR 0x44u
|
||||
#define RTL8139_REG_CONFIG1 0x52u
|
||||
|
||||
#define RTL8139_CMD_BUFE 0x01u
|
||||
#define RTL8139_CMD_TE 0x04u
|
||||
#define RTL8139_CMD_RE 0x08u
|
||||
#define RTL8139_CMD_RST 0x10u
|
||||
|
||||
#define RTL8139_ISR_ROK 0x0001u
|
||||
#define RTL8139_ISR_RER 0x0002u
|
||||
#define RTL8139_ISR_TOK 0x0004u
|
||||
#define RTL8139_ISR_TER 0x0008u
|
||||
#define RTL8139_ISR_RXOVW 0x0010u
|
||||
#define RTL8139_ISR_PUN 0x0020u
|
||||
#define RTL8139_ISR_FOVW 0x0040u
|
||||
#define RTL8139_ISR_LENCHG 0x2000u
|
||||
#define RTL8139_ISR_TIMEOUT 0x4000u
|
||||
#define RTL8139_ISR_SYSERR 0x8000u
|
||||
|
||||
#define RTL8139_INTR_ENABLES (RTL8139_ISR_ROK | RTL8139_ISR_RER | RTL8139_ISR_TOK | \
|
||||
RTL8139_ISR_TER | RTL8139_ISR_RXOVW | RTL8139_ISR_PUN | \
|
||||
RTL8139_ISR_FOVW | RTL8139_ISR_LENCHG | RTL8139_ISR_SYSERR)
|
||||
|
||||
#define RTL8139_RCR_AAP (1u << 0)
|
||||
#define RTL8139_RCR_APM (1u << 1)
|
||||
#define RTL8139_RCR_AM (1u << 2)
|
||||
#define RTL8139_RCR_AB (1u << 3)
|
||||
#define RTL8139_RCR_AR (1u << 4)
|
||||
#define RTL8139_RCR_AER (1u << 5)
|
||||
#define RTL8139_RCR_WRAP (1u << 7)
|
||||
#define RTL8139_RCR_MXDMA_UNLIMITED (7u << 8)
|
||||
#define RTL8139_RCR_RBLEN_8K (0u << 11)
|
||||
|
||||
#define RTL8139_TCR_MXDMA_UNLIMITED (7u << 8)
|
||||
#define RTL8139_TCR_IFG_NORMAL (3u << 24)
|
||||
|
||||
#define RTL8139_TSD_SIZE_MASK 0x00001FFFu
|
||||
#define RTL8139_TSD_OWN (1u << 13)
|
||||
#define RTL8139_TSD_TUN (1u << 14)
|
||||
#define RTL8139_TSD_TOK (1u << 15)
|
||||
#define RTL8139_TSD_TABT (1u << 30)
|
||||
|
||||
#define RTL8139_RX_OK 0x0001u
|
||||
#define RTL8139_RX_BAD_BITS 0x003Eu // FAE | CRC | LONG | RUNT | ISE; MAR/PAM/BAR are valid match flags.
|
||||
|
||||
#define RTL8139_TX_SLOTS 4u
|
||||
#define RTL8139_TX_BUF_SIZE 2048u
|
||||
#define RTL8139_RX_RING_SIZE 8192u
|
||||
#define RTL8139_RX_PAD 16u
|
||||
#define RTL8139_RX_GUARD 1500u
|
||||
#define RTL8139_RX_ALLOC_SIZE (RTL8139_RX_RING_SIZE + RTL8139_RX_PAD + RTL8139_RX_GUARD)
|
||||
#define RTL8139_RX_PAGES ((RTL8139_RX_ALLOC_SIZE + 4095u) / 4096u)
|
||||
#define RTL8139_TX_PAGES (((RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE) + 4095u) / 4096u)
|
||||
#define RTL8139_MMIO_PAGES 1u
|
||||
#define RTL8139_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u)
|
||||
|
||||
#define RTL8139_REG_TSD(slot) (RTL8139_REG_TSD0 + ((slot) * 4u))
|
||||
#define RTL8139_REG_TSAD(slot) (RTL8139_REG_TSAD0 + ((slot) * 4u))
|
||||
|
||||
typedef enum RTL8139_BAR_KIND {
|
||||
Rtl8139BarNone = 0,
|
||||
Rtl8139BarIo,
|
||||
Rtl8139BarMmio,
|
||||
} RTL8139_BAR_KIND;
|
||||
|
||||
typedef struct RTL8139_BAR {
|
||||
RTL8139_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} RTL8139_BAR;
|
||||
|
||||
typedef struct RTL8139_DEVICE {
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
uint8_t Irq;
|
||||
|
||||
bool UseMmio;
|
||||
volatile uint8_t* Mmio;
|
||||
uint16_t IoBase;
|
||||
|
||||
PHYSICAL_ADDRESS RxPhys;
|
||||
PHYSICAL_ADDRESS TxPhys;
|
||||
uint8_t* RxBuffer;
|
||||
uint8_t* TxBuffers;
|
||||
uint8_t RxScratch[RTL8139_TX_BUF_SIZE];
|
||||
|
||||
uint32_t RxOffset;
|
||||
uint32_t TxSlot;
|
||||
SPINLOCK TxLock;
|
||||
KDPC Dpc;
|
||||
volatile uint16_t PendingIsr;
|
||||
|
||||
NET_DEVICE NetDevice;
|
||||
} RTL8139_DEVICE;
|
||||
|
||||
static const HAL_API* gHal;
|
||||
static RTL8139_DEVICE gDevice;
|
||||
static bool gDevicePresent = false;
|
||||
const HAL_API* gHal;
|
||||
RTL8139_DEVICE gDevice;
|
||||
bool gDevicePresent = false;
|
||||
|
||||
static __forceinline void Rtl8139MemoryBarrier(void) {
|
||||
_ReadWriteBarrier();
|
||||
@@ -185,112 +44,9 @@ static void Rtl8139Write32(RTL8139_DEVICE* r, uint32_t reg, uint32_t value) {
|
||||
else __outdword((unsigned short)(r->IoBase + reg), value);
|
||||
}
|
||||
|
||||
static bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out) {
|
||||
uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u);
|
||||
uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg);
|
||||
|
||||
out->Kind = Rtl8139BarNone;
|
||||
out->Base = 0;
|
||||
out->Index = index;
|
||||
|
||||
if (lo == 0 || lo == 0xFFFFFFFFu)
|
||||
return false;
|
||||
|
||||
if (lo & 1u) {
|
||||
uint32_t base = lo & ~0x3u;
|
||||
if (!base) return false;
|
||||
out->Kind = Rtl8139BarIo;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t base = (uint64_t)(lo & ~0xFu);
|
||||
uint32_t type = (lo >> 1) & 0x3u;
|
||||
if (type == 0x2u && index < 5) {
|
||||
uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u));
|
||||
base |= ((uint64_t)hi << 32);
|
||||
}
|
||||
|
||||
if (!base) return false;
|
||||
out->Kind = Rtl8139BarMmio;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
static KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio) {
|
||||
uint64_t pageBase = physBase & PAGE_MASK;
|
||||
uint64_t pageOff = physBase - pageBase;
|
||||
|
||||
for (uint32_t i = 0; i < RTL8139_MMIO_PAGES; i++) {
|
||||
PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE;
|
||||
uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys);
|
||||
KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
*outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
|
||||
RTL8139_BAR best = { Rtl8139BarNone, 0, 0 };
|
||||
RTL8139_BAR fallback = { Rtl8139BarNone, 0, 0 };
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
RTL8139_BAR bar;
|
||||
if (!Rtl8139ReadBar(r->Bus, r->Dev, r->Fn, i, &bar))
|
||||
continue;
|
||||
|
||||
if (bar.Kind == Rtl8139BarMmio) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Kind == Rtl8139BarIo && fallback.Kind == Rtl8139BarNone)
|
||||
fallback = bar;
|
||||
|
||||
if (bar.Kind == Rtl8139BarMmio) {
|
||||
uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u));
|
||||
if (((lo >> 1) & 0x3u) == 0x2u)
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (best.Kind == Rtl8139BarNone)
|
||||
best = fallback;
|
||||
|
||||
if (best.Kind == Rtl8139BarMmio) {
|
||||
KSTATUS st = Rtl8139MapMmio(best.Base, &r->Mmio);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL8139: BAR%u MMIO base=0x%llx virt=%p\n",
|
||||
best.Index, (unsigned long long)best.Base, r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (best.Kind == Rtl8139BarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL8139: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
static void Rtl8139EnablePci(RTL8139_DEVICE* r) {
|
||||
uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND);
|
||||
cmd |= PCI_COMMAND_MASTER;
|
||||
if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY;
|
||||
else cmd |= PCI_COMMAND_IO;
|
||||
cmd &= ~PCI_COMMAND_INTX_DISABLE;
|
||||
gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd);
|
||||
}
|
||||
|
||||
static KSTATUS Rtl8139Reset(RTL8139_DEVICE* r) {
|
||||
Rtl8139Write8(r, RTL8139_REG_CMD, RTL8139_CMD_RST);
|
||||
@@ -328,7 +84,7 @@ static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
|
||||
memset(r->RxBuffer, 0, PAGE_SIZE * RTL8139_RX_PAGES);
|
||||
memset(r->TxBuffers, 0, PAGE_SIZE * RTL8139_TX_PAGES);
|
||||
|
||||
DbgPrint("RTL8139: rx=0x%llx tx=0x%llx\n",
|
||||
DbgPrint("RTL8139: rx=0x%016llX tx=0x%016llX\n",
|
||||
(unsigned long long)r->RxPhys,
|
||||
(unsigned long long)r->TxPhys);
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
// rtl8139.h - shared rtl8139 driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <ntlite/ddi/net_device.h>
|
||||
|
||||
#define RTL8139_VENDOR_REALTEK 0x10ECu
|
||||
#define RTL8139_DEVICE_ID 0x8139u
|
||||
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_COMMAND_IO (1u << 0)
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
|
||||
#define RTL8139_REG_IDR0 0x00u
|
||||
#define RTL8139_REG_MAR0 0x08u
|
||||
#define RTL8139_REG_TSD0 0x10u
|
||||
#define RTL8139_REG_TSAD0 0x20u
|
||||
#define RTL8139_REG_RBSTART 0x30u
|
||||
#define RTL8139_REG_CMD 0x37u
|
||||
#define RTL8139_REG_CAPR 0x38u
|
||||
#define RTL8139_REG_CBR 0x3Au
|
||||
#define RTL8139_REG_IMR 0x3Cu
|
||||
#define RTL8139_REG_ISR 0x3Eu
|
||||
#define RTL8139_REG_TCR 0x40u
|
||||
#define RTL8139_REG_RCR 0x44u
|
||||
#define RTL8139_REG_CONFIG1 0x52u
|
||||
|
||||
#define RTL8139_CMD_BUFE 0x01u
|
||||
#define RTL8139_CMD_TE 0x04u
|
||||
#define RTL8139_CMD_RE 0x08u
|
||||
#define RTL8139_CMD_RST 0x10u
|
||||
|
||||
#define RTL8139_ISR_ROK 0x0001u
|
||||
#define RTL8139_ISR_RER 0x0002u
|
||||
#define RTL8139_ISR_TOK 0x0004u
|
||||
#define RTL8139_ISR_TER 0x0008u
|
||||
#define RTL8139_ISR_RXOVW 0x0010u
|
||||
#define RTL8139_ISR_PUN 0x0020u
|
||||
#define RTL8139_ISR_FOVW 0x0040u
|
||||
#define RTL8139_ISR_LENCHG 0x2000u
|
||||
#define RTL8139_ISR_TIMEOUT 0x4000u
|
||||
#define RTL8139_ISR_SYSERR 0x8000u
|
||||
|
||||
#define RTL8139_INTR_ENABLES (RTL8139_ISR_ROK | RTL8139_ISR_RER | RTL8139_ISR_TOK | \
|
||||
RTL8139_ISR_TER | RTL8139_ISR_RXOVW | RTL8139_ISR_PUN | \
|
||||
RTL8139_ISR_FOVW | RTL8139_ISR_LENCHG | RTL8139_ISR_SYSERR)
|
||||
|
||||
#define RTL8139_RCR_AAP (1u << 0)
|
||||
#define RTL8139_RCR_APM (1u << 1)
|
||||
#define RTL8139_RCR_AM (1u << 2)
|
||||
#define RTL8139_RCR_AB (1u << 3)
|
||||
#define RTL8139_RCR_AR (1u << 4)
|
||||
#define RTL8139_RCR_AER (1u << 5)
|
||||
#define RTL8139_RCR_WRAP (1u << 7)
|
||||
#define RTL8139_RCR_MXDMA_UNLIMITED (7u << 8)
|
||||
#define RTL8139_RCR_RBLEN_8K (0u << 11)
|
||||
|
||||
#define RTL8139_TCR_MXDMA_UNLIMITED (7u << 8)
|
||||
#define RTL8139_TCR_IFG_NORMAL (3u << 24)
|
||||
|
||||
#define RTL8139_TSD_SIZE_MASK 0x00001FFFu
|
||||
#define RTL8139_TSD_OWN (1u << 13)
|
||||
#define RTL8139_TSD_TUN (1u << 14)
|
||||
#define RTL8139_TSD_TOK (1u << 15)
|
||||
#define RTL8139_TSD_TABT (1u << 30)
|
||||
|
||||
#define RTL8139_RX_OK 0x0001u
|
||||
#define RTL8139_RX_BAD_BITS 0x003Eu // FAE | CRC | LONG | RUNT | ISE; MAR/PAM/BAR are valid match flags.
|
||||
|
||||
#define RTL8139_TX_SLOTS 4u
|
||||
#define RTL8139_TX_BUF_SIZE 2048u
|
||||
#define RTL8139_RX_RING_SIZE 8192u
|
||||
#define RTL8139_RX_PAD 16u
|
||||
#define RTL8139_RX_GUARD 1500u
|
||||
#define RTL8139_RX_ALLOC_SIZE (RTL8139_RX_RING_SIZE + RTL8139_RX_PAD + RTL8139_RX_GUARD)
|
||||
#define RTL8139_RX_PAGES ((RTL8139_RX_ALLOC_SIZE + 4095u) / 4096u)
|
||||
#define RTL8139_TX_PAGES (((RTL8139_TX_SLOTS * RTL8139_TX_BUF_SIZE) + 4095u) / 4096u)
|
||||
#define RTL8139_MMIO_PAGES 1u
|
||||
#define RTL8139_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u)
|
||||
|
||||
#define RTL8139_REG_TSD(slot) (RTL8139_REG_TSD0 + ((slot) * 4u))
|
||||
#define RTL8139_REG_TSAD(slot) (RTL8139_REG_TSAD0 + ((slot) * 4u))
|
||||
|
||||
typedef enum RTL8139_BAR_KIND {
|
||||
Rtl8139BarNone = 0,
|
||||
Rtl8139BarIo,
|
||||
Rtl8139BarMmio,
|
||||
} RTL8139_BAR_KIND;
|
||||
|
||||
typedef struct RTL8139_BAR {
|
||||
RTL8139_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} RTL8139_BAR;
|
||||
|
||||
typedef struct RTL8139_DEVICE {
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
uint8_t Irq;
|
||||
|
||||
bool UseMmio;
|
||||
volatile uint8_t* Mmio;
|
||||
uint16_t IoBase;
|
||||
|
||||
PHYSICAL_ADDRESS RxPhys;
|
||||
PHYSICAL_ADDRESS TxPhys;
|
||||
uint8_t* RxBuffer;
|
||||
uint8_t* TxBuffers;
|
||||
uint8_t RxScratch[RTL8139_TX_BUF_SIZE];
|
||||
|
||||
uint32_t RxOffset;
|
||||
uint32_t TxSlot;
|
||||
SPINLOCK TxLock;
|
||||
KDPC Dpc;
|
||||
volatile uint16_t PendingIsr;
|
||||
|
||||
NET_DEVICE NetDevice;
|
||||
} RTL8139_DEVICE;
|
||||
|
||||
|
||||
extern const HAL_API* gHal;
|
||||
extern RTL8139_DEVICE gDevice;
|
||||
extern bool gDevicePresent;
|
||||
|
||||
bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out);
|
||||
KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio);
|
||||
KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r);
|
||||
void Rtl8139EnablePci(RTL8139_DEVICE* r);
|
||||
@@ -0,0 +1,112 @@
|
||||
// rtl8139_pci.c - PCI BAR/MMIO front-end for rtl8139.
|
||||
#include "rtl8139.h"
|
||||
|
||||
bool Rtl8139ReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL8139_BAR* out) {
|
||||
uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u);
|
||||
uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg);
|
||||
|
||||
out->Kind = Rtl8139BarNone;
|
||||
out->Base = 0;
|
||||
out->Index = index;
|
||||
|
||||
if (lo == 0 || lo == 0xFFFFFFFFu)
|
||||
return false;
|
||||
|
||||
if (lo & 1u) {
|
||||
uint32_t base = lo & ~0x3u;
|
||||
if (!base) return false;
|
||||
out->Kind = Rtl8139BarIo;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t base = (uint64_t)(lo & ~0xFu);
|
||||
uint32_t type = (lo >> 1) & 0x3u;
|
||||
if (type == 0x2u && index < 5) {
|
||||
uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u));
|
||||
base |= ((uint64_t)hi << 32);
|
||||
}
|
||||
|
||||
if (!base) return false;
|
||||
out->Kind = Rtl8139BarMmio;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS Rtl8139MapMmio(uint64_t physBase, volatile uint8_t** outMmio) {
|
||||
uint64_t pageBase = physBase & PAGE_MASK;
|
||||
uint64_t pageOff = physBase - pageBase;
|
||||
|
||||
for (uint32_t i = 0; i < RTL8139_MMIO_PAGES; i++) {
|
||||
PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE;
|
||||
uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys);
|
||||
KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
*outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
|
||||
RTL8139_BAR best = { Rtl8139BarNone, 0, 0 };
|
||||
RTL8139_BAR fallback = { Rtl8139BarNone, 0, 0 };
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
RTL8139_BAR bar;
|
||||
if (!Rtl8139ReadBar(r->Bus, r->Dev, r->Fn, i, &bar))
|
||||
continue;
|
||||
|
||||
if (bar.Kind == Rtl8139BarMmio) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Kind == Rtl8139BarIo && fallback.Kind == Rtl8139BarNone)
|
||||
fallback = bar;
|
||||
|
||||
if (bar.Kind == Rtl8139BarMmio) {
|
||||
uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u));
|
||||
if (((lo >> 1) & 0x3u) == 0x2u)
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (best.Kind == Rtl8139BarNone)
|
||||
best = fallback;
|
||||
|
||||
if (best.Kind == Rtl8139BarMmio) {
|
||||
KSTATUS st = Rtl8139MapMmio(best.Base, &r->Mmio);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL8139: BAR%u MMIO base=0x%016llX virt=0x%016llX\n",
|
||||
best.Index, (unsigned long long)best.Base, r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (best.Kind == Rtl8139BarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL8139: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
|
||||
void Rtl8139EnablePci(RTL8139_DEVICE* r) {
|
||||
uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND);
|
||||
cmd |= PCI_COMMAND_MASTER;
|
||||
if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY;
|
||||
else cmd |= PCI_COMMAND_IO;
|
||||
cmd &= ~PCI_COMMAND_INTX_DISABLE;
|
||||
gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd);
|
||||
}
|
||||
@@ -1,168 +1,9 @@
|
||||
// rtl816x.c - first-pass Realtek RTL8111/8168/8411 PCIe Gigabit NIC driver.
|
||||
//
|
||||
// Scope for this version:
|
||||
// * RTL8168/RTL8169-compatible PCI IDs, legacy INTx interrupt delivery.
|
||||
// * MMIO preferred, I/O BAR fallback.
|
||||
// * Descriptor-mode RX/TX with plain Ethernet frames.
|
||||
// * No checksum offload, TSO, VLAN, jumbo frames, MSI/MSI-X, or EEE.
|
||||
//
|
||||
// The driver registers a NET_DEVICE so a later ARP/IPv4/ICMP stack can attach
|
||||
// without knowing Realtek register details.
|
||||
// rtl816x.c - packet I/O, interrupt, and DriverEntry path.
|
||||
#include "rtl816x.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <ntlite/ddi/net_device.h>
|
||||
|
||||
#define RTL_VENDOR_REALTEK 0x10ECu
|
||||
#define RTL_DEVICE_8168 0x8168u
|
||||
#define RTL_DEVICE_8169 0x8169u
|
||||
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_REG_IRQ 0x3Cu
|
||||
#define PCI_COMMAND_IO (1u << 0)
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
|
||||
#define RTL_REG_MAC0 0x00u
|
||||
#define RTL_REG_MAR0 0x08u
|
||||
#define RTL_REG_TNPDS_LO 0x20u
|
||||
#define RTL_REG_TNPDS_HI 0x24u
|
||||
#define RTL_REG_THPDS_LO 0x28u
|
||||
#define RTL_REG_THPDS_HI 0x2Cu
|
||||
#define RTL_REG_CHIP_CMD 0x37u
|
||||
#define RTL_REG_TX_POLL 0x38u
|
||||
#define RTL_REG_INTR_MASK 0x3Cu
|
||||
#define RTL_REG_INTR_STAT 0x3Eu
|
||||
#define RTL_REG_TX_CONFIG 0x40u
|
||||
#define RTL_REG_RX_CONFIG 0x44u
|
||||
#define RTL_REG_CFG9346 0x50u
|
||||
#define RTL_REG_CONFIG1 0x52u
|
||||
#define RTL_REG_PHY_STATUS 0x6Cu
|
||||
#define RTL_REG_RX_MAXSIZE 0xDAu
|
||||
#define RTL_REG_CPLUS_CMD 0xE0u
|
||||
#define RTL_REG_RDSAR_LO 0xE4u
|
||||
#define RTL_REG_RDSAR_HI 0xE8u
|
||||
|
||||
#define RTL_CMD_RESET 0x10u
|
||||
#define RTL_CMD_RX_ENABLE 0x08u
|
||||
#define RTL_CMD_TX_ENABLE 0x04u
|
||||
|
||||
#define RTL_TXPOLL_NPQ 0x40u
|
||||
|
||||
#define RTL_ISR_ROK 0x0001u
|
||||
#define RTL_ISR_RER 0x0002u
|
||||
#define RTL_ISR_TOK 0x0004u
|
||||
#define RTL_ISR_TER 0x0008u
|
||||
#define RTL_ISR_RXOVW 0x0010u
|
||||
#define RTL_ISR_LINKCHG 0x0020u
|
||||
#define RTL_ISR_FOVW 0x0040u
|
||||
#define RTL_ISR_TDU 0x0080u
|
||||
#define RTL_ISR_SWINT 0x0100u
|
||||
#define RTL_ISR_TIMEOUT 0x4000u
|
||||
#define RTL_ISR_SYSERR 0x8000u
|
||||
|
||||
#define RTL_INTR_ENABLES (RTL_ISR_ROK | RTL_ISR_RER | RTL_ISR_TOK | RTL_ISR_TER | \
|
||||
RTL_ISR_RXOVW | RTL_ISR_LINKCHG | RTL_ISR_FOVW | \
|
||||
RTL_ISR_TDU | RTL_ISR_SYSERR)
|
||||
|
||||
#define RTL_RXCFG_AAP (1u << 0) // accept all physical packets/promiscuous
|
||||
#define RTL_RXCFG_APM (1u << 1) // accept physical match
|
||||
#define RTL_RXCFG_AM (1u << 2) // accept multicast
|
||||
#define RTL_RXCFG_AB (1u << 3) // accept broadcast
|
||||
#define RTL_RXCFG_AR (1u << 4) // accept runt
|
||||
#define RTL_RXCFG_AER (1u << 5) // accept errored
|
||||
#define RTL_RXCFG_DMA_UNLIMITED (7u << 8)
|
||||
|
||||
#define RTL_TXCFG_DMA_UNLIMITED (7u << 8)
|
||||
#define RTL_TXCFG_IFG (3u << 24)
|
||||
|
||||
#define RTL_DESC_OWN (1u << 31)
|
||||
#define RTL_DESC_EOR (1u << 30)
|
||||
#define RTL_DESC_FS (1u << 29)
|
||||
#define RTL_DESC_LS (1u << 28)
|
||||
#define RTL_RXDESC_RES (1u << 21)
|
||||
#define RTL_RXDESC_RWT (1u << 22)
|
||||
#define RTL_RXDESC_CRC (1u << 19)
|
||||
#define RTL_RXDESC_RUNT (1u << 20)
|
||||
#define RTL_DESC_LENGTH_MASK 0x00003FFFu
|
||||
|
||||
#define RTL_RX_COUNT 64u
|
||||
#define RTL_TX_COUNT 64u
|
||||
#define RTL_BUF_SIZE 2048u
|
||||
#define RTL_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u)
|
||||
#define RTL_RING_PAGES 1u
|
||||
#define RTL_RXBUF_PAGES ((RTL_RX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u)
|
||||
#define RTL_TXBUF_PAGES ((RTL_TX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u)
|
||||
#define RTL_MMIO_PAGES 1u
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef struct RTL_DESC {
|
||||
volatile uint32_t opts1;
|
||||
volatile uint32_t opts2;
|
||||
volatile uint32_t addrLo;
|
||||
volatile uint32_t addrHi;
|
||||
} RTL_DESC;
|
||||
#pragma pack(pop)
|
||||
|
||||
typedef enum RTL_BAR_KIND {
|
||||
RtlBarNone = 0,
|
||||
RtlBarIo,
|
||||
RtlBarMmio,
|
||||
} RTL_BAR_KIND;
|
||||
|
||||
typedef struct RTL_BAR {
|
||||
RTL_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} RTL_BAR;
|
||||
|
||||
typedef struct RTL816X_DEVICE {
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
uint8_t Irq;
|
||||
uint16_t DeviceId;
|
||||
|
||||
bool UseMmio;
|
||||
volatile uint8_t* Mmio;
|
||||
uint16_t IoBase;
|
||||
|
||||
RTL_DESC* RxRing;
|
||||
RTL_DESC* TxRing;
|
||||
PHYSICAL_ADDRESS RxRingPhys;
|
||||
PHYSICAL_ADDRESS TxRingPhys;
|
||||
|
||||
uint8_t* RxBuffers;
|
||||
uint8_t* TxBuffers;
|
||||
PHYSICAL_ADDRESS RxBuffersPhys;
|
||||
PHYSICAL_ADDRESS TxBuffersPhys;
|
||||
|
||||
uint32_t RxIndex;
|
||||
uint32_t TxIndex;
|
||||
SPINLOCK TxLock;
|
||||
KDPC Dpc;
|
||||
volatile uint16_t PendingIsr;
|
||||
|
||||
NET_DEVICE NetDevice;
|
||||
} RTL816X_DEVICE;
|
||||
|
||||
static const HAL_API* gHal;
|
||||
static RTL816X_DEVICE gDevice;
|
||||
static bool gDevicePresent = false;
|
||||
const HAL_API* gHal;
|
||||
RTL816X_DEVICE gDevice;
|
||||
bool gDevicePresent = false;
|
||||
|
||||
static __forceinline void RtlMemoryBarrier(void) {
|
||||
_ReadWriteBarrier();
|
||||
@@ -203,125 +44,10 @@ static void RtlWrite32(RTL816X_DEVICE* r, uint32_t reg, uint32_t value) {
|
||||
else __outdword((unsigned short)(r->IoBase + reg), value);
|
||||
}
|
||||
|
||||
static bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId) {
|
||||
if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8168, outBus, outDev, outFn)) {
|
||||
*outDeviceId = RTL_DEVICE_8168;
|
||||
return true;
|
||||
}
|
||||
if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8169, outBus, outDev, outFn)) {
|
||||
*outDeviceId = RTL_DEVICE_8169;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out) {
|
||||
uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u);
|
||||
uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg);
|
||||
|
||||
out->Kind = RtlBarNone;
|
||||
out->Base = 0;
|
||||
out->Index = index;
|
||||
|
||||
if (lo == 0 || lo == 0xFFFFFFFFu)
|
||||
return false;
|
||||
|
||||
if (lo & 1u) {
|
||||
uint32_t base = lo & ~0x3u;
|
||||
if (!base) return false;
|
||||
out->Kind = RtlBarIo;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t base = (uint64_t)(lo & ~0xFu);
|
||||
uint32_t type = (lo >> 1) & 0x3u;
|
||||
if (type == 0x2u && index < 5) {
|
||||
uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u));
|
||||
base |= ((uint64_t)hi << 32);
|
||||
}
|
||||
|
||||
if (!base) return false;
|
||||
out->Kind = RtlBarMmio;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
static KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio) {
|
||||
uint64_t pageBase = physBase & PAGE_MASK;
|
||||
uint64_t pageOff = physBase - pageBase;
|
||||
|
||||
for (uint32_t i = 0; i < RTL_MMIO_PAGES; i++) {
|
||||
PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE;
|
||||
uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys);
|
||||
KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
*outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
|
||||
RTL_BAR best = { RtlBarNone, 0, 0 };
|
||||
RTL_BAR fallback = { RtlBarNone, 0, 0 };
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
RTL_BAR bar;
|
||||
if (!RtlReadBar(r->Bus, r->Dev, r->Fn, i, &bar))
|
||||
continue;
|
||||
|
||||
if (bar.Kind == RtlBarMmio) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Kind == RtlBarIo && fallback.Kind == RtlBarNone)
|
||||
fallback = bar;
|
||||
|
||||
// Skip the high dword of 64-bit memory BARs.
|
||||
if (bar.Kind == RtlBarMmio) {
|
||||
uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u));
|
||||
if (((lo >> 1) & 0x3u) == 0x2u)
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (best.Kind == RtlBarNone)
|
||||
best = fallback;
|
||||
|
||||
if (best.Kind == RtlBarMmio) {
|
||||
KSTATUS st = RtlMapMmio(best.Base, &r->Mmio);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL816X: BAR%u MMIO base=0x%llx virt=%p\n",
|
||||
best.Index, (unsigned long long)best.Base, r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (best.Kind == RtlBarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL816X: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
static void RtlEnablePci(RTL816X_DEVICE* r) {
|
||||
uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND);
|
||||
cmd |= PCI_COMMAND_MASTER;
|
||||
if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY;
|
||||
else cmd |= PCI_COMMAND_IO;
|
||||
cmd &= ~PCI_COMMAND_INTX_DISABLE;
|
||||
gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd);
|
||||
}
|
||||
|
||||
static KSTATUS RtlReset(RTL816X_DEVICE* r) {
|
||||
RtlWrite8(r, RTL_REG_CHIP_CMD, RTL_CMD_RESET);
|
||||
@@ -359,7 +85,7 @@ static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
|
||||
memset(r->RxBuffers, 0, PAGE_SIZE * RTL_RXBUF_PAGES);
|
||||
memset(r->TxBuffers, 0, PAGE_SIZE * RTL_TXBUF_PAGES);
|
||||
|
||||
DbgPrint("RTL816X: rxRing=0x%llx txRing=0x%llx rxBuf=0x%llx txBuf=0x%llx\n",
|
||||
DbgPrint("RTL816X: rxRing=0x%016llX txRing=0x%016llX rxBuf=0x%016llX txBuf=0x%016llX\n",
|
||||
(unsigned long long)r->RxRingPhys,
|
||||
(unsigned long long)r->TxRingPhys,
|
||||
(unsigned long long)r->RxBuffersPhys,
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
// rtl816x.h - shared rtl816x driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <ntlite/ddi/net_device.h>
|
||||
|
||||
#define RTL_VENDOR_REALTEK 0x10ECu
|
||||
#define RTL_DEVICE_8168 0x8168u
|
||||
#define RTL_DEVICE_8169 0x8169u
|
||||
|
||||
#define PCI_REG_COMMAND 0x04u
|
||||
#define PCI_REG_BAR0 0x10u
|
||||
#define PCI_REG_IRQ 0x3Cu
|
||||
#define PCI_COMMAND_IO (1u << 0)
|
||||
#define PCI_COMMAND_MEMORY (1u << 1)
|
||||
#define PCI_COMMAND_MASTER (1u << 2)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
|
||||
#define RTL_REG_MAC0 0x00u
|
||||
#define RTL_REG_MAR0 0x08u
|
||||
#define RTL_REG_TNPDS_LO 0x20u
|
||||
#define RTL_REG_TNPDS_HI 0x24u
|
||||
#define RTL_REG_THPDS_LO 0x28u
|
||||
#define RTL_REG_THPDS_HI 0x2Cu
|
||||
#define RTL_REG_CHIP_CMD 0x37u
|
||||
#define RTL_REG_TX_POLL 0x38u
|
||||
#define RTL_REG_INTR_MASK 0x3Cu
|
||||
#define RTL_REG_INTR_STAT 0x3Eu
|
||||
#define RTL_REG_TX_CONFIG 0x40u
|
||||
#define RTL_REG_RX_CONFIG 0x44u
|
||||
#define RTL_REG_CFG9346 0x50u
|
||||
#define RTL_REG_CONFIG1 0x52u
|
||||
#define RTL_REG_PHY_STATUS 0x6Cu
|
||||
#define RTL_REG_RX_MAXSIZE 0xDAu
|
||||
#define RTL_REG_CPLUS_CMD 0xE0u
|
||||
#define RTL_REG_RDSAR_LO 0xE4u
|
||||
#define RTL_REG_RDSAR_HI 0xE8u
|
||||
|
||||
#define RTL_CMD_RESET 0x10u
|
||||
#define RTL_CMD_RX_ENABLE 0x08u
|
||||
#define RTL_CMD_TX_ENABLE 0x04u
|
||||
|
||||
#define RTL_TXPOLL_NPQ 0x40u
|
||||
|
||||
#define RTL_ISR_ROK 0x0001u
|
||||
#define RTL_ISR_RER 0x0002u
|
||||
#define RTL_ISR_TOK 0x0004u
|
||||
#define RTL_ISR_TER 0x0008u
|
||||
#define RTL_ISR_RXOVW 0x0010u
|
||||
#define RTL_ISR_LINKCHG 0x0020u
|
||||
#define RTL_ISR_FOVW 0x0040u
|
||||
#define RTL_ISR_TDU 0x0080u
|
||||
#define RTL_ISR_SWINT 0x0100u
|
||||
#define RTL_ISR_TIMEOUT 0x4000u
|
||||
#define RTL_ISR_SYSERR 0x8000u
|
||||
|
||||
#define RTL_INTR_ENABLES (RTL_ISR_ROK | RTL_ISR_RER | RTL_ISR_TOK | RTL_ISR_TER | \
|
||||
RTL_ISR_RXOVW | RTL_ISR_LINKCHG | RTL_ISR_FOVW | \
|
||||
RTL_ISR_TDU | RTL_ISR_SYSERR)
|
||||
|
||||
#define RTL_RXCFG_AAP (1u << 0) // accept all physical packets/promiscuous
|
||||
#define RTL_RXCFG_APM (1u << 1) // accept physical match
|
||||
#define RTL_RXCFG_AM (1u << 2) // accept multicast
|
||||
#define RTL_RXCFG_AB (1u << 3) // accept broadcast
|
||||
#define RTL_RXCFG_AR (1u << 4) // accept runt
|
||||
#define RTL_RXCFG_AER (1u << 5) // accept errored
|
||||
#define RTL_RXCFG_DMA_UNLIMITED (7u << 8)
|
||||
|
||||
#define RTL_TXCFG_DMA_UNLIMITED (7u << 8)
|
||||
#define RTL_TXCFG_IFG (3u << 24)
|
||||
|
||||
#define RTL_DESC_OWN (1u << 31)
|
||||
#define RTL_DESC_EOR (1u << 30)
|
||||
#define RTL_DESC_FS (1u << 29)
|
||||
#define RTL_DESC_LS (1u << 28)
|
||||
#define RTL_RXDESC_RES (1u << 21)
|
||||
#define RTL_RXDESC_RWT (1u << 22)
|
||||
#define RTL_RXDESC_CRC (1u << 19)
|
||||
#define RTL_RXDESC_RUNT (1u << 20)
|
||||
#define RTL_DESC_LENGTH_MASK 0x00003FFFu
|
||||
|
||||
#define RTL_RX_COUNT 64u
|
||||
#define RTL_TX_COUNT 64u
|
||||
#define RTL_BUF_SIZE 2048u
|
||||
#define RTL_RX_WIRE_MAX (NET_ETHERNET_MAX_FRAME + 4u)
|
||||
#define RTL_RING_PAGES 1u
|
||||
#define RTL_RXBUF_PAGES ((RTL_RX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u)
|
||||
#define RTL_TXBUF_PAGES ((RTL_TX_COUNT * RTL_BUF_SIZE + 4095u) / 4096u)
|
||||
#define RTL_MMIO_PAGES 1u
|
||||
|
||||
#pragma pack(push, 1)
|
||||
typedef struct RTL_DESC {
|
||||
volatile uint32_t opts1;
|
||||
volatile uint32_t opts2;
|
||||
volatile uint32_t addrLo;
|
||||
volatile uint32_t addrHi;
|
||||
} RTL_DESC;
|
||||
#pragma pack(pop)
|
||||
|
||||
typedef enum RTL_BAR_KIND {
|
||||
RtlBarNone = 0,
|
||||
RtlBarIo,
|
||||
RtlBarMmio,
|
||||
} RTL_BAR_KIND;
|
||||
|
||||
typedef struct RTL_BAR {
|
||||
RTL_BAR_KIND Kind;
|
||||
uint64_t Base;
|
||||
uint8_t Index;
|
||||
} RTL_BAR;
|
||||
|
||||
typedef struct RTL816X_DEVICE {
|
||||
uint8_t Bus;
|
||||
uint8_t Dev;
|
||||
uint8_t Fn;
|
||||
uint8_t Irq;
|
||||
uint16_t DeviceId;
|
||||
|
||||
bool UseMmio;
|
||||
volatile uint8_t* Mmio;
|
||||
uint16_t IoBase;
|
||||
|
||||
RTL_DESC* RxRing;
|
||||
RTL_DESC* TxRing;
|
||||
PHYSICAL_ADDRESS RxRingPhys;
|
||||
PHYSICAL_ADDRESS TxRingPhys;
|
||||
|
||||
uint8_t* RxBuffers;
|
||||
uint8_t* TxBuffers;
|
||||
PHYSICAL_ADDRESS RxBuffersPhys;
|
||||
PHYSICAL_ADDRESS TxBuffersPhys;
|
||||
|
||||
uint32_t RxIndex;
|
||||
uint32_t TxIndex;
|
||||
SPINLOCK TxLock;
|
||||
KDPC Dpc;
|
||||
volatile uint16_t PendingIsr;
|
||||
|
||||
NET_DEVICE NetDevice;
|
||||
} RTL816X_DEVICE;
|
||||
|
||||
|
||||
extern const HAL_API* gHal;
|
||||
extern RTL816X_DEVICE gDevice;
|
||||
extern bool gDevicePresent;
|
||||
|
||||
bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId);
|
||||
bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out);
|
||||
KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio);
|
||||
KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r);
|
||||
void RtlEnablePci(RTL816X_DEVICE* r);
|
||||
@@ -0,0 +1,126 @@
|
||||
// rtl816x_pci.c - PCI BAR/MMIO front-end for rtl816x.
|
||||
#include "rtl816x.h"
|
||||
|
||||
bool RtlFindDevice(uint8_t* outBus, uint8_t* outDev, uint8_t* outFn, uint16_t* outDeviceId) {
|
||||
if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8168, outBus, outDev, outFn)) {
|
||||
*outDeviceId = RTL_DEVICE_8168;
|
||||
return true;
|
||||
}
|
||||
if (gHal->Pci.FindDevice(RTL_VENDOR_REALTEK, RTL_DEVICE_8169, outBus, outDev, outFn)) {
|
||||
*outDeviceId = RTL_DEVICE_8169;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
bool RtlReadBar(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t index, RTL_BAR* out) {
|
||||
uint8_t reg = (uint8_t)(PCI_REG_BAR0 + index * 4u);
|
||||
uint32_t lo = gHal->Pci.Read32(bus, dev, fn, reg);
|
||||
|
||||
out->Kind = RtlBarNone;
|
||||
out->Base = 0;
|
||||
out->Index = index;
|
||||
|
||||
if (lo == 0 || lo == 0xFFFFFFFFu)
|
||||
return false;
|
||||
|
||||
if (lo & 1u) {
|
||||
uint32_t base = lo & ~0x3u;
|
||||
if (!base) return false;
|
||||
out->Kind = RtlBarIo;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
uint64_t base = (uint64_t)(lo & ~0xFu);
|
||||
uint32_t type = (lo >> 1) & 0x3u;
|
||||
if (type == 0x2u && index < 5) {
|
||||
uint32_t hi = gHal->Pci.Read32(bus, dev, fn, (uint8_t)(reg + 4u));
|
||||
base |= ((uint64_t)hi << 32);
|
||||
}
|
||||
|
||||
if (!base) return false;
|
||||
out->Kind = RtlBarMmio;
|
||||
out->Base = base;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS RtlMapMmio(uint64_t physBase, volatile uint8_t** outMmio) {
|
||||
uint64_t pageBase = physBase & PAGE_MASK;
|
||||
uint64_t pageOff = physBase - pageBase;
|
||||
|
||||
for (uint32_t i = 0; i < RTL_MMIO_PAGES; i++) {
|
||||
PHYSICAL_ADDRESS phys = pageBase + (uint64_t)i * PAGE_SIZE;
|
||||
uintptr_t virt = (uintptr_t)gHal->PhysToVirt(phys);
|
||||
KSTATUS st = MmMapPage(virt, phys, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (st == STATUS_ALREADY_EXISTS)
|
||||
st = MmSetPageProtect(virt, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
}
|
||||
|
||||
*outMmio = (volatile uint8_t*)((uintptr_t)gHal->PhysToVirt(pageBase) + pageOff);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
|
||||
RTL_BAR best = { RtlBarNone, 0, 0 };
|
||||
RTL_BAR fallback = { RtlBarNone, 0, 0 };
|
||||
|
||||
for (uint8_t i = 0; i < 6; i++) {
|
||||
RTL_BAR bar;
|
||||
if (!RtlReadBar(r->Bus, r->Dev, r->Fn, i, &bar))
|
||||
continue;
|
||||
|
||||
if (bar.Kind == RtlBarMmio) {
|
||||
best = bar;
|
||||
break;
|
||||
}
|
||||
if (bar.Kind == RtlBarIo && fallback.Kind == RtlBarNone)
|
||||
fallback = bar;
|
||||
|
||||
// Skip the high dword of 64-bit memory BARs.
|
||||
if (bar.Kind == RtlBarMmio) {
|
||||
uint32_t lo = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, (uint8_t)(PCI_REG_BAR0 + i * 4u));
|
||||
if (((lo >> 1) & 0x3u) == 0x2u)
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (best.Kind == RtlBarNone)
|
||||
best = fallback;
|
||||
|
||||
if (best.Kind == RtlBarMmio) {
|
||||
KSTATUS st = RtlMapMmio(best.Base, &r->Mmio);
|
||||
if (!KSUCCESS(st))
|
||||
return st;
|
||||
r->UseMmio = true;
|
||||
DbgPrint("RTL816X: BAR%u MMIO base=0x%016llX virt=0x%016llX\n",
|
||||
best.Index, (unsigned long long)best.Base, r->Mmio);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (best.Kind == RtlBarIo) {
|
||||
if (best.Base > 0xFFFFu)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
r->IoBase = (uint16_t)best.Base;
|
||||
r->UseMmio = false;
|
||||
DbgPrint("RTL816X: BAR%u I/O base=0x%04x\n", best.Index, r->IoBase);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
|
||||
void RtlEnablePci(RTL816X_DEVICE* r) {
|
||||
uint32_t cmd = gHal->Pci.Read32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND);
|
||||
cmd |= PCI_COMMAND_MASTER;
|
||||
if (r->UseMmio) cmd |= PCI_COMMAND_MEMORY;
|
||||
else cmd |= PCI_COMMAND_IO;
|
||||
cmd &= ~PCI_COMMAND_INTX_DISABLE;
|
||||
gHal->Pci.Write32(r->Bus, r->Dev, r->Fn, PCI_REG_COMMAND, cmd);
|
||||
}
|
||||
@@ -1,436 +1,18 @@
|
||||
// ahci.c - AHCI driver, interrupt-driven with per-drive async IRP queue.
|
||||
// Supports SATA drives only (no ATAPI). Registers each found drive with the
|
||||
// kernel disk device registry via IoRegisterDiskDevice().
|
||||
//
|
||||
// Memory model:
|
||||
// MmAllocateContiguous(n) returns a physical address for n*4K pages.
|
||||
// hal->PhysToVirt(phys) converts to kernel virtual address.
|
||||
// DMA buffers are bounce-buffered (allocated via MmAllocateContiguous) to
|
||||
// avoid needing a VirtToPhys conversion for arbitrary kernel pointers.
|
||||
//
|
||||
// Async I/O model (IRP path):
|
||||
// Each drive has a dedicated kernel worker thread (AhciWorker) that blocks
|
||||
// on a per-drive counting semaphore. Callers submit IRPs through IoCallDriver,
|
||||
// which dispatches to AHCI and enqueues the IRP. The worker dequeues
|
||||
// one IRP at a time, runs the synchronous hardware transfer (blocking the
|
||||
// worker, NOT the submitting thread), then writes irp->Status and fires
|
||||
// irp->CompletionEvent / irp->CompletionFn.
|
||||
//
|
||||
// This gives true async I/O from the caller's perspective: IoSubmitIrp()
|
||||
// returns STATUS_PENDING immediately. Multiple callers can have outstanding
|
||||
// requests against different drives concurrently without stalling each other.
|
||||
//
|
||||
// NCQ extension (future work - see "NCQ TODO" comments):
|
||||
// To saturate modern SSDs, replace the serialised single-slot model with
|
||||
// Native Command Queuing:
|
||||
// • Allocate per-slot bounce buffers (NUM_NCQ_SLOTS × SLOT_BOUNCE_BYTES).
|
||||
// • Use ATA_CMD_READ_FPDMA_QUEUED / WRITE_FPDMA_QUEUED.
|
||||
// • Encode the slot tag in FIS.countl bits 7:3.
|
||||
// • Issue via port->sact (NCQ active) rather than port->ci.
|
||||
// • In AhciIsr: completed slots = prev_sact ^ port->sact; look up
|
||||
// slot_irp[slot] and fire each completion from within the ISR.
|
||||
// • The worker becomes a pure dispatcher: grab a free slot, build the
|
||||
// command, issue it, return immediately. No per-slot KeWaitForEvent.
|
||||
//
|
||||
// Interrupt model (unchanged):
|
||||
// GHC.IE and per-port PORT_IE are enabled after each port is initialised.
|
||||
// AhciIsr fires on the PCI IRQ line, captures/clears port status, and queues
|
||||
// a per-drive DPC. The DPC signals completionEvent outside hard IRQ context.
|
||||
// port_issue_cmd blocks the worker thread on KeWaitForEvent instead of
|
||||
// spinning on CI. port_issue_cmd_poll is kept for early init (IDENTIFY)
|
||||
// before the scheduler is live.
|
||||
// ahci.c - AHCI disk command path and DriverEntry.
|
||||
#include "ahci.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
typedef enum {
|
||||
FIS_TYPE_REG_H2D = 0x27,
|
||||
FIS_TYPE_REG_D2H = 0x34,
|
||||
FIS_TYPE_DMA_ACT = 0x39,
|
||||
FIS_TYPE_DMA_SETUP = 0x41,
|
||||
FIS_TYPE_DATA = 0x46,
|
||||
FIS_TYPE_BIST = 0x58,
|
||||
FIS_TYPE_PIO_SETUP = 0x5F,
|
||||
FIS_TYPE_DEV_BITS = 0xA1,
|
||||
} FIS_TYPE;
|
||||
|
||||
#pragma pack(push, 1)
|
||||
// Register FIS - Host to Device (H2D)
|
||||
typedef struct {
|
||||
uint8_t fis_type; // FIS_TYPE_REG_H2D
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 3;
|
||||
uint8_t c : 1; // 1 = Command, 0 = Control
|
||||
uint8_t command;
|
||||
uint8_t featurel;
|
||||
// DW1
|
||||
uint8_t lba0, lba1, lba2;
|
||||
uint8_t device;
|
||||
// DW2
|
||||
uint8_t lba3, lba4, lba5;
|
||||
uint8_t featureh;
|
||||
// DW3
|
||||
uint8_t countl, counth;
|
||||
uint8_t icc;
|
||||
uint8_t control;
|
||||
// DW4
|
||||
uint8_t rsv1[4];
|
||||
} FIS_REG_H2D;
|
||||
|
||||
// Register FIS - Device to Host (D2H)
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 2;
|
||||
uint8_t i : 1;
|
||||
uint8_t rsv1 : 1;
|
||||
uint8_t status;
|
||||
uint8_t error;
|
||||
uint8_t lba0, lba1, lba2;
|
||||
uint8_t device;
|
||||
uint8_t lba3, lba4, lba5;
|
||||
uint8_t rsv2;
|
||||
uint8_t countl, counth;
|
||||
uint8_t rsv3[2];
|
||||
uint8_t rsv4[4];
|
||||
} FIS_REG_D2H;
|
||||
|
||||
// DMA Setup FIS
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 1;
|
||||
uint8_t d : 1;
|
||||
uint8_t i : 1;
|
||||
uint8_t a : 1;
|
||||
uint8_t rsved[2];
|
||||
uint64_t DMAbufferID;
|
||||
uint32_t rsvd;
|
||||
uint32_t DMAbufOffset;
|
||||
uint32_t TransferCount;
|
||||
uint32_t resvd;
|
||||
} FIS_DMA_SETUP;
|
||||
|
||||
// PIO Setup FIS
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 1;
|
||||
uint8_t d : 1;
|
||||
uint8_t i : 1;
|
||||
uint8_t rsv1 : 1;
|
||||
uint8_t status;
|
||||
uint8_t error;
|
||||
uint8_t lba0, lba1, lba2;
|
||||
uint8_t device;
|
||||
uint8_t lba3, lba4, lba5;
|
||||
uint8_t rsv2;
|
||||
uint8_t countl, counth;
|
||||
uint8_t rsv3;
|
||||
uint8_t e_status;
|
||||
uint16_t tc;
|
||||
uint8_t rsv4[2];
|
||||
} FIS_PIO_SETUP;
|
||||
|
||||
// Set Device Bits FIS
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 2;
|
||||
uint8_t i : 1;
|
||||
uint8_t n : 1;
|
||||
uint8_t statusl : 4;
|
||||
uint8_t rsv1 : 4;
|
||||
uint8_t statush : 4;
|
||||
uint8_t rsv2 : 4;
|
||||
uint8_t error;
|
||||
uint32_t protocol;
|
||||
} FIS_DEV_BITS;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HBA structures (volatile - MMIO)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Received FIS structure in host memory
|
||||
typedef volatile struct {
|
||||
FIS_DMA_SETUP dsfis; // 0x00 DMA Setup FIS
|
||||
uint8_t pad0[4];
|
||||
FIS_PIO_SETUP psfis; // 0x20 PIO Setup FIS
|
||||
uint8_t pad1[12];
|
||||
FIS_REG_D2H rfis; // 0x40 Register D2H FIS
|
||||
uint8_t pad2[4];
|
||||
FIS_DEV_BITS sdbfis; // 0x58 Set Device Bits FIS
|
||||
uint8_t ufis[64]; // 0x60 Unknown FIS
|
||||
uint8_t rsv[0x100-0xA0];
|
||||
} HBA_FIS;
|
||||
|
||||
// Command List header (one entry = one slot)
|
||||
typedef struct {
|
||||
// DW0
|
||||
uint8_t cfl : 5; // Command FIS length in DWORDs (2..16)
|
||||
uint8_t a : 1; // ATAPI
|
||||
uint8_t w : 1; // Write (H2D=1, D2H=0)
|
||||
uint8_t p : 1; // Prefetchable
|
||||
uint8_t r : 1; // Reset
|
||||
uint8_t b : 1; // BIST
|
||||
uint8_t c : 1; // Clear busy on R_OK
|
||||
uint8_t rsv0 : 1;
|
||||
uint8_t pmp : 4; // Port multiplier port
|
||||
uint16_t prdtl; // PRDT length in entries
|
||||
// DW1
|
||||
volatile uint32_t prdbc; // PRDT byte count transferred
|
||||
// DW2,3
|
||||
uint32_t ctba; // Command table base address (128-byte aligned)
|
||||
uint32_t ctbau; // Upper 32 bits
|
||||
// DW4-7
|
||||
uint32_t rsv1[4];
|
||||
} HBA_CMD_HEADER;
|
||||
|
||||
// Physical Region Descriptor Table entry
|
||||
typedef struct {
|
||||
uint32_t dba; // Data base address
|
||||
uint32_t dbau; // Upper 32 bits
|
||||
uint32_t rsv0;
|
||||
uint32_t dbc : 22; // Byte count (0-based, so actual = dbc+1)
|
||||
uint32_t rsv1 : 9;
|
||||
uint32_t i : 1; // Interrupt on completion
|
||||
} HBA_PRDT_ENTRY;
|
||||
|
||||
// Command Table
|
||||
typedef struct {
|
||||
uint8_t cfis[64]; // Command FIS (up to 64 bytes)
|
||||
uint8_t acmd[16]; // ATAPI command (12 or 16 bytes)
|
||||
uint8_t rsv[48]; // Reserved
|
||||
HBA_PRDT_ENTRY prdt_entry[1]; // PRDT entries (variable length)
|
||||
} HBA_CMD_TBL;
|
||||
#pragma pack(pop)
|
||||
|
||||
// Port registers
|
||||
typedef volatile struct {
|
||||
uint32_t clb; // 0x00 Command list base (1K-aligned)
|
||||
uint32_t clbu; // 0x04 Command list base upper
|
||||
uint32_t fb; // 0x08 FIS base (256-byte aligned)
|
||||
uint32_t fbu; // 0x0C FIS base upper
|
||||
uint32_t is; // 0x10 Interrupt status
|
||||
uint32_t ie; // 0x14 Interrupt enable
|
||||
uint32_t cmd; // 0x18 Command and status
|
||||
uint32_t rsv0; // 0x1C Reserved
|
||||
uint32_t tfd; // 0x20 Task file data
|
||||
uint32_t sig; // 0x24 Signature
|
||||
uint32_t ssts; // 0x28 SATA status
|
||||
uint32_t sctl; // 0x2C SATA control
|
||||
uint32_t serr; // 0x30 SATA error
|
||||
uint32_t sact; // 0x34 SATA active
|
||||
uint32_t ci; // 0x38 Command issue
|
||||
uint32_t sntf; // 0x3C SATA notification
|
||||
uint32_t fbs; // 0x40 FIS-based switch control
|
||||
uint32_t rsv1[11];
|
||||
uint32_t vendor[4];
|
||||
} HBA_PORT;
|
||||
|
||||
// Generic Host Control registers (starts at ABAR)
|
||||
typedef volatile struct {
|
||||
uint32_t cap; // 0x00 Host capability
|
||||
uint32_t ghc; // 0x04 Global host control
|
||||
uint32_t is; // 0x08 Interrupt status
|
||||
uint32_t pi; // 0x0C Ports implemented
|
||||
uint32_t vs; // 0x10 Version
|
||||
uint32_t ccc_ctl; // 0x14 CCC control
|
||||
uint32_t ccc_pts; // 0x18 CCC ports
|
||||
uint32_t em_loc; // 0x1C Enclosure mgmt location
|
||||
uint32_t em_ctl; // 0x20 Enclosure mgmt control
|
||||
uint32_t cap2; // 0x24 Extended capabilities
|
||||
uint32_t bohc; // 0x28 BIOS/OS handoff control
|
||||
uint8_t rsv[0xA0-0x2C];
|
||||
uint8_t vendor[0x100-0xA0];
|
||||
HBA_PORT ports[32]; // 0x100 Port control registers
|
||||
} HBA_MEM;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// PCI
|
||||
#define PCI_AHCI_CLASS 0x01
|
||||
#define PCI_AHCI_SUBCLASS 0x06
|
||||
#define PCI_CMD_REG 0x04
|
||||
#define PCI_BAR5_REG 0x24
|
||||
#define PCI_CMD_BUSMASTER (1 << 2)
|
||||
#define PCI_CMD_MEMSPACE (1 << 1)
|
||||
|
||||
// GHC register bits
|
||||
#define GHC_AE (1u << 31) // AHCI Enable
|
||||
#define GHC_IE (1u << 1) // Interrupt Enable
|
||||
#define GHC_HR (1u << 0) // HBA Reset
|
||||
|
||||
// BOHC register bits
|
||||
#define BOHC_BOS (1u << 0) // BIOS-owned semaphore
|
||||
#define BOHC_OOS (1u << 1) // OS-owned semaphore
|
||||
#define BOHC_BB (1u << 4) // BIOS Busy
|
||||
|
||||
// CAP2 bits
|
||||
#define CAP2_BOH (1u << 0) // BIOS/OS handoff capable
|
||||
|
||||
// Port CMD register bits
|
||||
#define PORT_CMD_ST (1u << 0) // Start
|
||||
#define PORT_CMD_FRE (1u << 4) // FIS Receive Enable
|
||||
#define PORT_CMD_FR (1u << 14) // FIS Receive Running
|
||||
#define PORT_CMD_CR (1u << 15) // Command List Running
|
||||
#define PORT_CMD_POD (1u << 2) // Power On Device
|
||||
#define PORT_CMD_SUD (1u << 1) // Spin-Up Device
|
||||
#define PORT_CMD_ICC_ACTIVE (0x1u << 28)
|
||||
|
||||
// Port IS register bits (interrupt status)
|
||||
#define PORT_IS_TFES (1u << 30) // Task File Error Status
|
||||
#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error
|
||||
#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error
|
||||
#define PORT_IS_IFS (1u << 27) // Interface Fatal Error
|
||||
|
||||
// Port SSTS register
|
||||
#define PORT_DET_PRESENT 0x3
|
||||
#define PORT_IPM_ACTIVE 0x1
|
||||
|
||||
// Port signatures
|
||||
#define SATA_SIG_ATA 0x00000101
|
||||
#define SATA_SIG_ATAPI 0xEB140101
|
||||
#define SATA_SIG_SEMB 0xC33C0101
|
||||
#define SATA_SIG_PM 0x96690101
|
||||
|
||||
// ATA commands
|
||||
#define ATA_CMD_READ_DMA_EX 0x25
|
||||
#define ATA_CMD_WRITE_DMA_EX 0x35
|
||||
#define ATA_CMD_IDENTIFY 0xEC
|
||||
#define ATA_CMD_FLUSH_EXT 0xEA
|
||||
|
||||
// ATA status bits
|
||||
#define ATA_SR_BUSY 0x80
|
||||
#define ATA_SR_DRQ 0x08
|
||||
#define ATA_SR_ERR 0x01
|
||||
|
||||
// IDENTIFY word offsets
|
||||
#define ATA_IDENT_W83 83 // bit 10 = 48-bit LBA supported
|
||||
#define ATA_IDENT_W100 100 // 48-bit LBA sector count (4 words)
|
||||
#define ATA_IDENT_W60 60 // 28-bit LBA sector count (2 words)
|
||||
|
||||
// Layout constants (per port, single contiguous physical allocation)
|
||||
// Offset 0: Command List 1024 bytes (32 × 32-byte headers)
|
||||
// Offset 1024: Received FIS 256 bytes
|
||||
// Offset 2048: Command Tables 32 × 256 bytes = 8192 bytes
|
||||
// Total: 10240 bytes → 3 pages (12288 bytes)
|
||||
#define PORT_PAGES_STRUCT 3
|
||||
#define PORT_OFF_CMDLIST 0
|
||||
#define PORT_OFF_FIS 1024
|
||||
#define PORT_OFF_CMDTBLS 2048
|
||||
#define PORT_CMDTBL_SIZE 256 // 64 CFIS + 16 ACMD + 48 rsv + 8×16 PRDT
|
||||
#define PORT_PRDT_COUNT 8 // PRDT entries per command table
|
||||
|
||||
// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages
|
||||
#define BOUNCE_SECTORS 128
|
||||
#define BOUNCE_BYTES (BOUNCE_SECTORS * 512)
|
||||
#define BOUNCE_PAGES (BOUNCE_BYTES / 4096)
|
||||
|
||||
// Polling timeout (spin iterations)
|
||||
#define POLL_TIMEOUT 1000000
|
||||
|
||||
#define MAX_AHCI_DRIVES 8
|
||||
#define NUM_CMD_SLOTS 32
|
||||
|
||||
// Map MMIO physical range into the physical window as uncacheable.
|
||||
// Must be called before any pointer dereference via PhysToVirt().
|
||||
// AHCI MMIO layout: 0x100 GHC + 32*0x80 ports = 0x1100 bytes → 2 pages minimum.
|
||||
// We map 8 pages (32KB) without probing the BAR size.
|
||||
#define AHCI_MMIO_PAGES 8
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Driver-global state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static const HAL_API *g_hal;
|
||||
static HBA_MEM *g_hba; // HBA MMIO base (set during init)
|
||||
static uint8_t g_irq; // PCI IRQ line for this HBA
|
||||
static DRIVER_OBJECT g_driver; // AHCI disk driver object
|
||||
|
||||
// Per-drive context
|
||||
typedef struct {
|
||||
HBA_PORT *port; // MMIO port registers (virtual)
|
||||
uint64_t sectorCount; // Total LBA sectors on device
|
||||
uint8_t portNo; // Physical port index (for ISR dispatch)
|
||||
|
||||
// Structures block (command list + FIS + cmd tables)
|
||||
uint64_t structPhys; // Physical base
|
||||
void *structVirt; // Virtual base
|
||||
|
||||
// Bounce buffer for DMA data
|
||||
uint64_t bouncePhys;
|
||||
void *bounceVirt;
|
||||
|
||||
// Signaled by the completion DPC when the port raises an interrupt.
|
||||
// Auto-reset: one Wait consumes one completion.
|
||||
EVENT *completionEvent;
|
||||
|
||||
// Last IS snapshot captured by the ISR (for error checking post-wait).
|
||||
volatile uint32_t lastIs;
|
||||
|
||||
// Bottom-half used to wake the worker outside hard IRQ/DIRQL context.
|
||||
KDPC completionDpc;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IRP queue - async dispatch
|
||||
// -----------------------------------------------------------------------
|
||||
// AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP
|
||||
// and signals the semaphore to wake the worker.
|
||||
IRP_QUEUE irpQueue;
|
||||
THREAD *workerThread;
|
||||
DEVICE_OBJECT *deviceObject;
|
||||
|
||||
// NCQ TODO: for multi-slot NCQ, add:
|
||||
// uint32_t slotMask; // bitmask of in-flight slots
|
||||
// IRP *slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP
|
||||
// uint64_t slotBouncePhys[NUM_CMD_SLOTS];
|
||||
// void *slotBounceVirt[NUM_CMD_SLOTS];
|
||||
// volatile uint32_t prevSact; // previous port->sact snapshot
|
||||
} AHCI_DRIVE;
|
||||
|
||||
static AHCI_DRIVE g_drives[MAX_AHCI_DRIVES];
|
||||
static int g_driveCount;
|
||||
const HAL_API *g_hal;
|
||||
HBA_MEM *g_hba;
|
||||
uint8_t g_irq;
|
||||
DRIVER_OBJECT g_driver;
|
||||
AHCI_DRIVE g_drives[MAX_AHCI_DRIVES];
|
||||
int g_driveCount;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
static KSTATUS ahci_map_mmio(uint64_t physBase) {
|
||||
for (int i = 0; i < AHCI_MMIO_PAGES; i++) {
|
||||
uint64_t phys = physBase + (uint64_t)i * 4096;
|
||||
uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys);
|
||||
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
|
||||
KSTATUS st = MmMapPage(virt, phys, flags);
|
||||
// Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags
|
||||
// is possible for the first page if the bootloader was overly generous.
|
||||
// Use MmSetPageProtect to force NOCACHE in that case.
|
||||
if (st == STATUS_ALREADY_EXISTS) {
|
||||
st = MmSetPageProtect(virt, 4096, flags);
|
||||
}
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%llx, st=0x%x)\n",
|
||||
i, (unsigned long long)phys, st);
|
||||
return st;
|
||||
}
|
||||
}
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Accessors for port structure memory at known offsets from structVirt/Phys
|
||||
static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) {
|
||||
@@ -460,29 +42,7 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
|
||||
// Port engine start / stop (polling safe)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static void port_stop(HBA_PORT *port) {
|
||||
// Clear ST (bit 0)
|
||||
port->cmd &= ~PORT_CMD_ST;
|
||||
// Wait for CR (bit 15) to clear (command list DMA stops)
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(port->cmd & PORT_CMD_CR)) break;
|
||||
}
|
||||
// Clear FRE (bit 4)
|
||||
port->cmd &= ~PORT_CMD_FRE;
|
||||
// Wait for FR (bit 14) to clear
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(port->cmd & PORT_CMD_FR)) break;
|
||||
}
|
||||
}
|
||||
|
||||
static void port_start(HBA_PORT *port) {
|
||||
// Wait until CR is clear before setting ST
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(port->cmd & PORT_CMD_CR)) break;
|
||||
}
|
||||
port->cmd |= PORT_CMD_FRE;
|
||||
port->cmd |= PORT_CMD_ST;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Port memory initialisation
|
||||
@@ -878,100 +438,16 @@ static void AhciWorker(uintptr_t arg) {
|
||||
// AHCI controller initialisation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
static KSTATUS hba_reset(HBA_MEM *hba) {
|
||||
// BIOS/OS handoff (spec §10.6.3)
|
||||
if (hba->cap2 & CAP2_BOH) {
|
||||
hba->bohc |= BOHC_OOS;
|
||||
// Wait for BIOS to clear BOS
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(hba->bohc & BOHC_BOS)) break;
|
||||
}
|
||||
// Give the BIOS another moment to clear Busy
|
||||
for (int i = 0; i < 800000; i++) {
|
||||
if (!(hba->bohc & BOHC_BB)) break;
|
||||
}
|
||||
}
|
||||
|
||||
// Enable AHCI mode before reset so the reset actually takes effect
|
||||
hba->ghc |= GHC_AE;
|
||||
|
||||
// HBA reset
|
||||
hba->ghc |= GHC_HR;
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(hba->ghc & GHC_HR)) break;
|
||||
}
|
||||
if (hba->ghc & GHC_HR) {
|
||||
DbgPrint("AHCI: HBA reset timed out\n");
|
||||
return STATUS_TIMEOUT;
|
||||
}
|
||||
|
||||
// Re-enable AHCI mode (reset cleared it), then enable global interrupts.
|
||||
// GHC.IE lets the HBA assert its PCI interrupt line when a port fires.
|
||||
hba->ghc |= GHC_AE | GHC_IE;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// Before memory init: only check physical link. Signature is not yet valid.
|
||||
static bool port_link_up(HBA_PORT *port) {
|
||||
uint32_t det = port->ssts & 0x0F;
|
||||
uint32_t ipm = (port->ssts >> 8) & 0x0F;
|
||||
return (det == PORT_DET_PRESENT && ipm == PORT_IPM_ACTIVE);
|
||||
}
|
||||
|
||||
// After COMRESET + FIS receive: check signature for SATA disk.
|
||||
static bool port_is_sata_disk(HBA_PORT *port) {
|
||||
return port->sig == SATA_SIG_ATA;
|
||||
}
|
||||
|
||||
static void ahci_delay_ms(uint32_t ms) {
|
||||
for (uint32_t i = 0; i < ms * 1000; i++)
|
||||
__outbyte(0x80, 0);
|
||||
}
|
||||
|
||||
// Issue a COMRESET on the port, wait for link re-establishment and D2H FIS.
|
||||
// FRE must already be set (port_init_memory does this) so the HBA can
|
||||
// capture the signature FIS the device sends after the reset.
|
||||
static KSTATUS port_comreset(AHCI_DRIVE *drv) {
|
||||
HBA_PORT *port = drv->port;
|
||||
|
||||
port->cmd &= ~PORT_CMD_ST;
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++)
|
||||
if (!(port->cmd & PORT_CMD_CR)) break;
|
||||
|
||||
// Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms
|
||||
port->sctl = (port->sctl & ~0x0Fu) | 0x1u;
|
||||
ahci_delay_ms(1);
|
||||
|
||||
// Deassert: DET = 0
|
||||
port->sctl = port->sctl & ~0x0Fu;
|
||||
|
||||
// Poll for DET = 3, up to 1 second
|
||||
bool link_ok = false;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
if ((port->ssts & 0x0Fu) == PORT_DET_PRESENT) { link_ok = true; break; }
|
||||
ahci_delay_ms(1);
|
||||
}
|
||||
if (!link_ok) {
|
||||
DbgPrint("AHCI: COMRESET: link did not come back (ssts=0x%x)\n", port->ssts);
|
||||
return STATUS_TIMEOUT;
|
||||
}
|
||||
|
||||
port->serr = (uint32_t)-1;
|
||||
port->is = (uint32_t)-1;
|
||||
port->cmd |= PORT_CMD_ST;
|
||||
|
||||
// Wait up to 500ms for a valid signature (D2H FIS from device)
|
||||
for (int i = 0; i < 500; i++) {
|
||||
uint32_t sig = port->sig;
|
||||
if (sig != 0xFFFFFFFFu && sig != 0x00000000u) break;
|
||||
ahci_delay_ms(1);
|
||||
}
|
||||
|
||||
DbgPrint("AHCI: port COMRESET done: sig=0x%08x ssts=0x%08x\n",
|
||||
port->sig, port->ssts);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// PORT_IE bits we care about:
|
||||
// DHRS (bit 0) - D2H Register FIS received - normal completion
|
||||
@@ -1032,7 +508,7 @@ KSTATUS AhciInitialize(const HAL_API *hal) {
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
g_hba = (HBA_MEM *)hal->PhysToVirt(abarPhys);
|
||||
DbgPrint("AHCI: ABAR phys=0x%llx version=0x%08x\n",
|
||||
DbgPrint("AHCI: ABAR phys=0x%016llX version=0x%08x\n",
|
||||
(unsigned long long)abarPhys, g_hba->vs);
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
// ahci.h - shared AHCI driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
typedef enum {
|
||||
FIS_TYPE_REG_H2D = 0x27,
|
||||
FIS_TYPE_REG_D2H = 0x34,
|
||||
FIS_TYPE_DMA_ACT = 0x39,
|
||||
FIS_TYPE_DMA_SETUP = 0x41,
|
||||
FIS_TYPE_DATA = 0x46,
|
||||
FIS_TYPE_BIST = 0x58,
|
||||
FIS_TYPE_PIO_SETUP = 0x5F,
|
||||
FIS_TYPE_DEV_BITS = 0xA1,
|
||||
} FIS_TYPE;
|
||||
|
||||
#pragma pack(push, 1)
|
||||
// Register FIS - Host to Device (H2D)
|
||||
typedef struct {
|
||||
uint8_t fis_type; // FIS_TYPE_REG_H2D
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 3;
|
||||
uint8_t c : 1; // 1 = Command, 0 = Control
|
||||
uint8_t command;
|
||||
uint8_t featurel;
|
||||
// DW1
|
||||
uint8_t lba0, lba1, lba2;
|
||||
uint8_t device;
|
||||
// DW2
|
||||
uint8_t lba3, lba4, lba5;
|
||||
uint8_t featureh;
|
||||
// DW3
|
||||
uint8_t countl, counth;
|
||||
uint8_t icc;
|
||||
uint8_t control;
|
||||
// DW4
|
||||
uint8_t rsv1[4];
|
||||
} FIS_REG_H2D;
|
||||
|
||||
// Register FIS - Device to Host (D2H)
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 2;
|
||||
uint8_t i : 1;
|
||||
uint8_t rsv1 : 1;
|
||||
uint8_t status;
|
||||
uint8_t error;
|
||||
uint8_t lba0, lba1, lba2;
|
||||
uint8_t device;
|
||||
uint8_t lba3, lba4, lba5;
|
||||
uint8_t rsv2;
|
||||
uint8_t countl, counth;
|
||||
uint8_t rsv3[2];
|
||||
uint8_t rsv4[4];
|
||||
} FIS_REG_D2H;
|
||||
|
||||
// DMA Setup FIS
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 1;
|
||||
uint8_t d : 1;
|
||||
uint8_t i : 1;
|
||||
uint8_t a : 1;
|
||||
uint8_t rsved[2];
|
||||
uint64_t DMAbufferID;
|
||||
uint32_t rsvd;
|
||||
uint32_t DMAbufOffset;
|
||||
uint32_t TransferCount;
|
||||
uint32_t resvd;
|
||||
} FIS_DMA_SETUP;
|
||||
|
||||
// PIO Setup FIS
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 1;
|
||||
uint8_t d : 1;
|
||||
uint8_t i : 1;
|
||||
uint8_t rsv1 : 1;
|
||||
uint8_t status;
|
||||
uint8_t error;
|
||||
uint8_t lba0, lba1, lba2;
|
||||
uint8_t device;
|
||||
uint8_t lba3, lba4, lba5;
|
||||
uint8_t rsv2;
|
||||
uint8_t countl, counth;
|
||||
uint8_t rsv3;
|
||||
uint8_t e_status;
|
||||
uint16_t tc;
|
||||
uint8_t rsv4[2];
|
||||
} FIS_PIO_SETUP;
|
||||
|
||||
// Set Device Bits FIS
|
||||
typedef struct {
|
||||
uint8_t fis_type;
|
||||
uint8_t pmport : 4;
|
||||
uint8_t rsv0 : 2;
|
||||
uint8_t i : 1;
|
||||
uint8_t n : 1;
|
||||
uint8_t statusl : 4;
|
||||
uint8_t rsv1 : 4;
|
||||
uint8_t statush : 4;
|
||||
uint8_t rsv2 : 4;
|
||||
uint8_t error;
|
||||
uint32_t protocol;
|
||||
} FIS_DEV_BITS;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HBA structures (volatile - MMIO)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Received FIS structure in host memory
|
||||
typedef volatile struct {
|
||||
FIS_DMA_SETUP dsfis; // 0x00 DMA Setup FIS
|
||||
uint8_t pad0[4];
|
||||
FIS_PIO_SETUP psfis; // 0x20 PIO Setup FIS
|
||||
uint8_t pad1[12];
|
||||
FIS_REG_D2H rfis; // 0x40 Register D2H FIS
|
||||
uint8_t pad2[4];
|
||||
FIS_DEV_BITS sdbfis; // 0x58 Set Device Bits FIS
|
||||
uint8_t ufis[64]; // 0x60 Unknown FIS
|
||||
uint8_t rsv[0x100-0xA0];
|
||||
} HBA_FIS;
|
||||
|
||||
// Command List header (one entry = one slot)
|
||||
typedef struct {
|
||||
// DW0
|
||||
uint8_t cfl : 5; // Command FIS length in DWORDs (2..16)
|
||||
uint8_t a : 1; // ATAPI
|
||||
uint8_t w : 1; // Write (H2D=1, D2H=0)
|
||||
uint8_t p : 1; // Prefetchable
|
||||
uint8_t r : 1; // Reset
|
||||
uint8_t b : 1; // BIST
|
||||
uint8_t c : 1; // Clear busy on R_OK
|
||||
uint8_t rsv0 : 1;
|
||||
uint8_t pmp : 4; // Port multiplier port
|
||||
uint16_t prdtl; // PRDT length in entries
|
||||
// DW1
|
||||
volatile uint32_t prdbc; // PRDT byte count transferred
|
||||
// DW2,3
|
||||
uint32_t ctba; // Command table base address (128-byte aligned)
|
||||
uint32_t ctbau; // Upper 32 bits
|
||||
// DW4-7
|
||||
uint32_t rsv1[4];
|
||||
} HBA_CMD_HEADER;
|
||||
|
||||
// Physical Region Descriptor Table entry
|
||||
typedef struct {
|
||||
uint32_t dba; // Data base address
|
||||
uint32_t dbau; // Upper 32 bits
|
||||
uint32_t rsv0;
|
||||
uint32_t dbc : 22; // Byte count (0-based, so actual = dbc+1)
|
||||
uint32_t rsv1 : 9;
|
||||
uint32_t i : 1; // Interrupt on completion
|
||||
} HBA_PRDT_ENTRY;
|
||||
|
||||
// Command Table
|
||||
typedef struct {
|
||||
uint8_t cfis[64]; // Command FIS (up to 64 bytes)
|
||||
uint8_t acmd[16]; // ATAPI command (12 or 16 bytes)
|
||||
uint8_t rsv[48]; // Reserved
|
||||
HBA_PRDT_ENTRY prdt_entry[1]; // PRDT entries (variable length)
|
||||
} HBA_CMD_TBL;
|
||||
#pragma pack(pop)
|
||||
|
||||
// Port registers
|
||||
typedef volatile struct {
|
||||
uint32_t clb; // 0x00 Command list base (1K-aligned)
|
||||
uint32_t clbu; // 0x04 Command list base upper
|
||||
uint32_t fb; // 0x08 FIS base (256-byte aligned)
|
||||
uint32_t fbu; // 0x0C FIS base upper
|
||||
uint32_t is; // 0x10 Interrupt status
|
||||
uint32_t ie; // 0x14 Interrupt enable
|
||||
uint32_t cmd; // 0x18 Command and status
|
||||
uint32_t rsv0; // 0x1C Reserved
|
||||
uint32_t tfd; // 0x20 Task file data
|
||||
uint32_t sig; // 0x24 Signature
|
||||
uint32_t ssts; // 0x28 SATA status
|
||||
uint32_t sctl; // 0x2C SATA control
|
||||
uint32_t serr; // 0x30 SATA error
|
||||
uint32_t sact; // 0x34 SATA active
|
||||
uint32_t ci; // 0x38 Command issue
|
||||
uint32_t sntf; // 0x3C SATA notification
|
||||
uint32_t fbs; // 0x40 FIS-based switch control
|
||||
uint32_t rsv1[11];
|
||||
uint32_t vendor[4];
|
||||
} HBA_PORT;
|
||||
|
||||
// Generic Host Control registers (starts at ABAR)
|
||||
typedef volatile struct {
|
||||
uint32_t cap; // 0x00 Host capability
|
||||
uint32_t ghc; // 0x04 Global host control
|
||||
uint32_t is; // 0x08 Interrupt status
|
||||
uint32_t pi; // 0x0C Ports implemented
|
||||
uint32_t vs; // 0x10 Version
|
||||
uint32_t ccc_ctl; // 0x14 CCC control
|
||||
uint32_t ccc_pts; // 0x18 CCC ports
|
||||
uint32_t em_loc; // 0x1C Enclosure mgmt location
|
||||
uint32_t em_ctl; // 0x20 Enclosure mgmt control
|
||||
uint32_t cap2; // 0x24 Extended capabilities
|
||||
uint32_t bohc; // 0x28 BIOS/OS handoff control
|
||||
uint8_t rsv[0xA0-0x2C];
|
||||
uint8_t vendor[0x100-0xA0];
|
||||
HBA_PORT ports[32]; // 0x100 Port control registers
|
||||
} HBA_MEM;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constants
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// PCI
|
||||
#define PCI_AHCI_CLASS 0x01
|
||||
#define PCI_AHCI_SUBCLASS 0x06
|
||||
#define PCI_CMD_REG 0x04
|
||||
#define PCI_BAR5_REG 0x24
|
||||
#define PCI_CMD_BUSMASTER (1 << 2)
|
||||
#define PCI_CMD_MEMSPACE (1 << 1)
|
||||
|
||||
// GHC register bits
|
||||
#define GHC_AE (1u << 31) // AHCI Enable
|
||||
#define GHC_IE (1u << 1) // Interrupt Enable
|
||||
#define GHC_HR (1u << 0) // HBA Reset
|
||||
|
||||
// BOHC register bits
|
||||
#define BOHC_BOS (1u << 0) // BIOS-owned semaphore
|
||||
#define BOHC_OOS (1u << 1) // OS-owned semaphore
|
||||
#define BOHC_BB (1u << 4) // BIOS Busy
|
||||
|
||||
// CAP2 bits
|
||||
#define CAP2_BOH (1u << 0) // BIOS/OS handoff capable
|
||||
|
||||
// Port CMD register bits
|
||||
#define PORT_CMD_ST (1u << 0) // Start
|
||||
#define PORT_CMD_FRE (1u << 4) // FIS Receive Enable
|
||||
#define PORT_CMD_FR (1u << 14) // FIS Receive Running
|
||||
#define PORT_CMD_CR (1u << 15) // Command List Running
|
||||
#define PORT_CMD_POD (1u << 2) // Power On Device
|
||||
#define PORT_CMD_SUD (1u << 1) // Spin-Up Device
|
||||
#define PORT_CMD_ICC_ACTIVE (0x1u << 28)
|
||||
|
||||
// Port IS register bits (interrupt status)
|
||||
#define PORT_IS_TFES (1u << 30) // Task File Error Status
|
||||
#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error
|
||||
#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error
|
||||
#define PORT_IS_IFS (1u << 27) // Interface Fatal Error
|
||||
|
||||
// Port SSTS register
|
||||
#define PORT_DET_PRESENT 0x3
|
||||
#define PORT_IPM_ACTIVE 0x1
|
||||
|
||||
// Port signatures
|
||||
#define SATA_SIG_ATA 0x00000101
|
||||
#define SATA_SIG_ATAPI 0xEB140101
|
||||
#define SATA_SIG_SEMB 0xC33C0101
|
||||
#define SATA_SIG_PM 0x96690101
|
||||
|
||||
// ATA commands
|
||||
#define ATA_CMD_READ_DMA_EX 0x25
|
||||
#define ATA_CMD_WRITE_DMA_EX 0x35
|
||||
#define ATA_CMD_IDENTIFY 0xEC
|
||||
#define ATA_CMD_FLUSH_EXT 0xEA
|
||||
|
||||
// ATA status bits
|
||||
#define ATA_SR_BUSY 0x80
|
||||
#define ATA_SR_DRQ 0x08
|
||||
#define ATA_SR_ERR 0x01
|
||||
|
||||
// IDENTIFY word offsets
|
||||
#define ATA_IDENT_W83 83 // bit 10 = 48-bit LBA supported
|
||||
#define ATA_IDENT_W100 100 // 48-bit LBA sector count (4 words)
|
||||
#define ATA_IDENT_W60 60 // 28-bit LBA sector count (2 words)
|
||||
|
||||
// Layout constants (per port, single contiguous physical allocation)
|
||||
// Offset 0: Command List 1024 bytes (32 × 32-byte headers)
|
||||
// Offset 1024: Received FIS 256 bytes
|
||||
// Offset 2048: Command Tables 32 × 256 bytes = 8192 bytes
|
||||
// Total: 10240 bytes → 3 pages (12288 bytes)
|
||||
#define PORT_PAGES_STRUCT 3
|
||||
#define PORT_OFF_CMDLIST 0
|
||||
#define PORT_OFF_FIS 1024
|
||||
#define PORT_OFF_CMDTBLS 2048
|
||||
#define PORT_CMDTBL_SIZE 256 // 64 CFIS + 16 ACMD + 48 rsv + 8×16 PRDT
|
||||
#define PORT_PRDT_COUNT 8 // PRDT entries per command table
|
||||
|
||||
// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages
|
||||
#define BOUNCE_SECTORS 128
|
||||
#define BOUNCE_BYTES (BOUNCE_SECTORS * 512)
|
||||
#define BOUNCE_PAGES (BOUNCE_BYTES / 4096)
|
||||
|
||||
// Polling timeout (spin iterations)
|
||||
#define POLL_TIMEOUT 1000000
|
||||
|
||||
#define MAX_AHCI_DRIVES 8
|
||||
#define NUM_CMD_SLOTS 32
|
||||
|
||||
// Map MMIO physical range into the physical window as uncacheable.
|
||||
// Must be called before any pointer dereference via PhysToVirt().
|
||||
// AHCI MMIO layout: 0x100 GHC + 32*0x80 ports = 0x1100 bytes → 2 pages minimum.
|
||||
// We map 8 pages (32KB) without probing the BAR size.
|
||||
#define AHCI_MMIO_PAGES 8
|
||||
|
||||
// Per-drive context
|
||||
typedef struct {
|
||||
HBA_PORT *port; // MMIO port registers (virtual)
|
||||
uint64_t sectorCount; // Total LBA sectors on device
|
||||
uint8_t portNo; // Physical port index (for ISR dispatch)
|
||||
|
||||
// Structures block (command list + FIS + cmd tables)
|
||||
uint64_t structPhys; // Physical base
|
||||
void* structVirt; // Virtual base
|
||||
|
||||
// Bounce buffer for DMA data
|
||||
uint64_t bouncePhys;
|
||||
void* bounceVirt;
|
||||
|
||||
// Signaled by the completion DPC when the port raises an interrupt.
|
||||
// Auto-reset: one Wait consumes one completion.
|
||||
EVENT *completionEvent;
|
||||
|
||||
// Last IS snapshot captured by the ISR (for error checking post-wait).
|
||||
volatile uint32_t lastIs;
|
||||
|
||||
// Bottom-half used to wake the worker outside hard IRQ/DIRQL context.
|
||||
KDPC completionDpc;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// IRP queue - async dispatch
|
||||
// -----------------------------------------------------------------------
|
||||
// AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP
|
||||
// and signals the semaphore to wake the worker.
|
||||
IRP_QUEUE irpQueue;
|
||||
THREAD* workerThread;
|
||||
DEVICE_OBJECT* deviceObject;
|
||||
|
||||
// NCQ TODO: for multi-slot NCQ, add:
|
||||
// uint32_t slotMask; // bitmask of in-flight slots
|
||||
// IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP
|
||||
// uint64_t slotBouncePhys[NUM_CMD_SLOTS];
|
||||
// void* slotBounceVirt[NUM_CMD_SLOTS];
|
||||
// volatile uint32_t prevSact; // previous port->sact snapshot
|
||||
} AHCI_DRIVE;
|
||||
|
||||
|
||||
extern const HAL_API *g_hal;
|
||||
extern HBA_MEM *g_hba;
|
||||
extern uint8_t g_irq;
|
||||
extern DRIVER_OBJECT g_driver;
|
||||
extern AHCI_DRIVE g_drives[MAX_AHCI_DRIVES];
|
||||
extern int g_driveCount;
|
||||
|
||||
KSTATUS ahci_map_mmio(uint64_t physBase);
|
||||
void port_stop(HBA_PORT *port);
|
||||
void port_start(HBA_PORT *port);
|
||||
KSTATUS hba_reset(HBA_MEM *hba);
|
||||
bool port_link_up(HBA_PORT *port);
|
||||
bool port_is_sata_disk(HBA_PORT *port);
|
||||
void ahci_delay_ms(uint32_t ms);
|
||||
KSTATUS port_comreset(AHCI_DRIVE *drv);
|
||||
@@ -0,0 +1,143 @@
|
||||
// ahci_port.c - AHCI MMIO mapping, HBA reset, and port link bring-up helpers.
|
||||
#include "ahci.h"
|
||||
|
||||
KSTATUS ahci_map_mmio(uint64_t physBase) {
|
||||
for (int i = 0; i < AHCI_MMIO_PAGES; i++) {
|
||||
uint64_t phys = physBase + (uint64_t)i * 4096;
|
||||
uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys);
|
||||
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
|
||||
KSTATUS st = MmMapPage(virt, phys, flags);
|
||||
// Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags
|
||||
// is possible for the first page if the bootloader was overly generous.
|
||||
// Use MmSetPageProtect to force NOCACHE in that case.
|
||||
if (st == STATUS_ALREADY_EXISTS) {
|
||||
st = MmSetPageProtect(virt, 4096, flags);
|
||||
}
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
|
||||
i, (unsigned long long)phys, st);
|
||||
return st;
|
||||
}
|
||||
}
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
void port_stop(HBA_PORT *port) {
|
||||
// Clear ST (bit 0)
|
||||
port->cmd &= ~PORT_CMD_ST;
|
||||
// Wait for CR (bit 15) to clear (command list DMA stops)
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(port->cmd & PORT_CMD_CR)) break;
|
||||
}
|
||||
// Clear FRE (bit 4)
|
||||
port->cmd &= ~PORT_CMD_FRE;
|
||||
// Wait for FR (bit 14) to clear
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(port->cmd & PORT_CMD_FR)) break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void port_start(HBA_PORT *port) {
|
||||
// Wait until CR is clear before setting ST
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(port->cmd & PORT_CMD_CR)) break;
|
||||
}
|
||||
port->cmd |= PORT_CMD_FRE;
|
||||
port->cmd |= PORT_CMD_ST;
|
||||
}
|
||||
|
||||
|
||||
KSTATUS hba_reset(HBA_MEM *hba) {
|
||||
// BIOS/OS handoff (spec §10.6.3)
|
||||
if (hba->cap2 & CAP2_BOH) {
|
||||
hba->bohc |= BOHC_OOS;
|
||||
// Wait for BIOS to clear BOS
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(hba->bohc & BOHC_BOS)) break;
|
||||
}
|
||||
// Give the BIOS another moment to clear Busy
|
||||
for (int i = 0; i < 800000; i++) {
|
||||
if (!(hba->bohc & BOHC_BB)) break;
|
||||
}
|
||||
}
|
||||
|
||||
// Enable AHCI mode before reset so the reset actually takes effect
|
||||
hba->ghc |= GHC_AE;
|
||||
|
||||
// HBA reset
|
||||
hba->ghc |= GHC_HR;
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++) {
|
||||
if (!(hba->ghc & GHC_HR)) break;
|
||||
}
|
||||
if (hba->ghc & GHC_HR) {
|
||||
DbgPrint("AHCI: HBA reset timed out\n");
|
||||
return STATUS_TIMEOUT;
|
||||
}
|
||||
|
||||
// Re-enable AHCI mode (reset cleared it), then enable global interrupts.
|
||||
// GHC.IE lets the HBA assert its PCI interrupt line when a port fires.
|
||||
hba->ghc |= GHC_AE | GHC_IE;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
bool port_link_up(HBA_PORT *port) {
|
||||
uint32_t det = port->ssts & 0x0F;
|
||||
uint32_t ipm = (port->ssts >> 8) & 0x0F;
|
||||
return (det == PORT_DET_PRESENT && ipm == PORT_IPM_ACTIVE);
|
||||
}
|
||||
|
||||
|
||||
bool port_is_sata_disk(HBA_PORT *port) {
|
||||
return port->sig == SATA_SIG_ATA;
|
||||
}
|
||||
|
||||
|
||||
void ahci_delay_ms(uint32_t ms) {
|
||||
for (uint32_t i = 0; i < ms * 1000; i++)
|
||||
__outbyte(0x80, 0);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS port_comreset(AHCI_DRIVE *drv) {
|
||||
HBA_PORT *port = drv->port;
|
||||
|
||||
port->cmd &= ~PORT_CMD_ST;
|
||||
for (int i = 0; i < POLL_TIMEOUT; i++)
|
||||
if (!(port->cmd & PORT_CMD_CR)) break;
|
||||
|
||||
// Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms
|
||||
port->sctl = (port->sctl & ~0x0Fu) | 0x1u;
|
||||
ahci_delay_ms(1);
|
||||
|
||||
// Deassert: DET = 0
|
||||
port->sctl = port->sctl & ~0x0Fu;
|
||||
|
||||
// Poll for DET = 3, up to 1 second
|
||||
bool link_ok = false;
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
if ((port->ssts & 0x0Fu) == PORT_DET_PRESENT) { link_ok = true; break; }
|
||||
ahci_delay_ms(1);
|
||||
}
|
||||
if (!link_ok) {
|
||||
DbgPrint("AHCI: COMRESET: link did not come back (ssts=0x%x)\n", port->ssts);
|
||||
return STATUS_TIMEOUT;
|
||||
}
|
||||
|
||||
port->serr = (uint32_t)-1;
|
||||
port->is = (uint32_t)-1;
|
||||
port->cmd |= PORT_CMD_ST;
|
||||
|
||||
// Wait up to 500ms for a valid signature (D2H FIS from device)
|
||||
for (int i = 0; i < 500; i++) {
|
||||
uint32_t sig = port->sig;
|
||||
if (sig != 0xFFFFFFFFu && sig != 0x00000000u) break;
|
||||
ahci_delay_ms(1);
|
||||
}
|
||||
|
||||
DbgPrint("AHCI: port COMRESET done: sig=0x%08x ssts=0x%08x\n",
|
||||
port->sig, port->ssts);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
+19
-207
@@ -1,130 +1,26 @@
|
||||
// ide.c — IDE/ATA driver: PIO (sync fallback) + Bus Master DMA (async)
|
||||
//
|
||||
// Architecture
|
||||
// ------------
|
||||
// The dispatch path mirrors AHCI:
|
||||
// IoCallDriver → IdeDispatchReadWrite → IRP queue or inline PIO
|
||||
// IdeWorker → dequeues IRP, calls IdeDmaRw(), completes the IRP
|
||||
// IdeIsr → clears BM status, queues a DPC
|
||||
// IdeDpc → signals gDmaEvent to unblock the worker
|
||||
//
|
||||
// Graceful degradation: if Bus Master init fails (no PCI IDE controller,
|
||||
// BAR4 not assigned, PRDT allocation above 4 GB, etc.) the same device
|
||||
// object remains registered and dispatch completes IRPs inline through PIO.
|
||||
//
|
||||
// PRDT layout
|
||||
// -----------
|
||||
// One page allocated via MmAllocateContiguous. We build up to PRDT_MAX_ENTRIES
|
||||
// entries by walking the transfer buffer in page-sized chunks and looking up
|
||||
// each page's physical address via MmGetPhysicalAddress. Adjacent physical
|
||||
// pages are coalesced into a single PRD entry where possible.
|
||||
//
|
||||
// LBA48 vs LBA28
|
||||
// --------------
|
||||
// DMA commands exist in both modes: CMD_READ/WRITE_DMA_EXT (0x25/0x35) for
|
||||
// LBA48 and CMD_READ/WRITE_DMA (0xC8/0xCA) for LBA28. The existing
|
||||
// gSupportsLba48 flag picks the right variant.
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IDE register map (primary channel, base 0x1F0)
|
||||
// ---------------------------------------------------------------------------
|
||||
#define IDE_DATA 0x1F0
|
||||
#define IDE_ERR 0x1F1
|
||||
#define IDE_SECCNT 0x1F2
|
||||
#define IDE_LBA0 0x1F3
|
||||
#define IDE_LBA1 0x1F4
|
||||
#define IDE_LBA2 0x1F5
|
||||
#define IDE_DRIVE 0x1F6
|
||||
#define IDE_STATUS 0x1F7
|
||||
#define IDE_CMD 0x1F7
|
||||
#define IDE_ALT 0x3F6
|
||||
|
||||
#define IDE_BUSY 0x80
|
||||
#define IDE_READY 0x40
|
||||
#define IDE_ERR_F 0x01
|
||||
#define IDE_DRQ 0x08
|
||||
|
||||
// ATA commands
|
||||
#define CMD_READ_EXT 0x24 // PIO 48-bit read
|
||||
#define CMD_WRITE_EXT 0x34 // PIO 48-bit write
|
||||
#define CMD_READ_DMA_EXT 0x25 // DMA 48-bit read
|
||||
#define CMD_WRITE_DMA_EXT 0x35 // DMA 48-bit write
|
||||
#define CMD_READ_DMA 0xC8 // DMA 28-bit read
|
||||
#define CMD_WRITE_DMA 0xCA // DMA 28-bit write
|
||||
#define CMD_IDENT 0xEC
|
||||
|
||||
// IDE primary IRQ
|
||||
#define IDE_IRQ_PRIMARY 14
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bus Master IDE register offsets (relative to gBmBase, primary channel)
|
||||
// ---------------------------------------------------------------------------
|
||||
#define BM_CMD_REG 0x00 // command
|
||||
#define BM_STATUS_REG 0x02 // status
|
||||
#define BM_PRDT_REG 0x04 // PRDT physical address (32-bit)
|
||||
|
||||
// BM_CMD_REG bits
|
||||
#define BM_CMD_START 0x01 // start DMA transfer
|
||||
#define BM_CMD_READ 0x08 // 1 = disk→memory (read), 0 = memory→disk (write)
|
||||
|
||||
// BM_STATUS_REG bits
|
||||
#define BM_STATUS_ACTIVE 0x01 // DMA active
|
||||
#define BM_STATUS_ERROR 0x02 // DMA error (write 1 to clear)
|
||||
#define BM_STATUS_IRQ 0x04 // interrupt pending (write 1 to clear)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Physical Region Descriptor (PRDT entry, 8 bytes)
|
||||
// The PRDT itself must not cross a 64 KB boundary and must be 4-byte aligned.
|
||||
// We allocate a full page so both constraints are trivially satisfied.
|
||||
// ---------------------------------------------------------------------------
|
||||
#pragma pack(push, 1)
|
||||
typedef struct {
|
||||
uint32_t physAddr; // physical base of this memory region
|
||||
uint16_t byteCount; // byte count; 0 means 64 KB
|
||||
uint16_t flags; // bit 15: End Of Table
|
||||
} PRD;
|
||||
#pragma pack(pop)
|
||||
|
||||
#define PRD_EOT 0x8000u
|
||||
// 512 entries × 8 bytes = 4096 bytes = exactly one page; supports 512×64 KB = 32 MB
|
||||
#define PRDT_MAX_ENTRIES 512
|
||||
// ide.c - IDE/ATA disk I/O path and DriverEntry.
|
||||
#include "ide.h"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Module state
|
||||
// ---------------------------------------------------------------------------
|
||||
static const HAL_API *gHal;
|
||||
static DRIVER_OBJECT gDriver;
|
||||
static DEVICE_OBJECT *gDevice = NULL;
|
||||
static bool gDmaEnabled = false;
|
||||
static uint64_t gSectorCount = 0;
|
||||
static bool gSupportsLba48 = false;
|
||||
const HAL_API *gHal;
|
||||
DRIVER_OBJECT gDriver;
|
||||
DEVICE_OBJECT *gDevice = NULL;
|
||||
bool gDmaEnabled = false;
|
||||
uint64_t gSectorCount = 0;
|
||||
bool gSupportsLba48 = false;
|
||||
|
||||
// DMA hardware state
|
||||
static uint16_t gBmBase = 0; // Bus Master I/O base port (BAR4 & ~3)
|
||||
static PRD *gPrdtVirt = NULL; // kernel VA of PRDT page
|
||||
static uint32_t gPrdtPhys = 0; // 32-bit physical address of PRDT
|
||||
uint16_t gBmBase = 0; // Bus Master I/O base port (BAR4 & ~3)
|
||||
PRD *gPrdtVirt = NULL; // kernel VA of PRDT page
|
||||
uint32_t gPrdtPhys = 0; // 32-bit physical address of PRDT
|
||||
|
||||
// Async worker synchronisation
|
||||
static IRP_QUEUE gIrpQueue;
|
||||
static SEMAPHORE *gIrpSemaphore = NULL;
|
||||
static EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer
|
||||
static KDPC gDmaDpc; // bottom half for DMA completion
|
||||
IRP_QUEUE gIrpQueue;
|
||||
SEMAPHORE *gIrpSemaphore = NULL;
|
||||
EVENT *gDmaEvent = NULL; // auto-reset; DPC signals after each xfer
|
||||
KDPC gDmaDpc; // bottom half for DMA completion
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bus Master I/O helpers
|
||||
@@ -255,7 +151,7 @@ static uint32_t BuildPrdt(void *virtBuf, uint32_t byteLen) {
|
||||
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va);
|
||||
if (phys >> 32) {
|
||||
// Physical address above 4 GB — BM IDE's 32-bit PRDT can't reach it.
|
||||
DbgPrint("IDE DMA: buffer page at VA 0x%llx has phys 0x%llx > 4 GB\n",
|
||||
DbgPrint("IDE DMA: buffer page at VA 0x%016llX has phys 0x%016llX > 4 GB\n",
|
||||
(uint64_t)va, (uint64_t)phys);
|
||||
return 0;
|
||||
}
|
||||
@@ -294,7 +190,7 @@ static uint32_t BuildPrdt(void *virtBuf, uint32_t byteLen) {
|
||||
// IDE DMA completion DPC. The ISR owns hardware acknowledgement; this bottom
|
||||
// half wakes the worker outside hard IRQ/DIRQL context.
|
||||
// ---------------------------------------------------------------------------
|
||||
static void IdeDmaCompletionDpc(KDPC *dpc, void *context,
|
||||
void IdeDmaCompletionDpc(KDPC *dpc, void *context,
|
||||
void *systemArgument1, void *systemArgument2) {
|
||||
(void)dpc;
|
||||
(void)context;
|
||||
@@ -312,7 +208,7 @@ static void IdeDmaCompletionDpc(KDPC *dpc, void *context,
|
||||
// interrupt bits here and queue a DPC to wake the worker thread.
|
||||
// The worker reads the final status and fires IRP completion hooks.
|
||||
// ---------------------------------------------------------------------------
|
||||
static void IdeIsr(void) {
|
||||
void IdeIsr(void) {
|
||||
uint8_t bmSt = BmRead8(BM_STATUS_REG);
|
||||
if (!(bmSt & BM_STATUS_IRQ))
|
||||
return; // not our interrupt (spurious or secondary channel)
|
||||
@@ -417,7 +313,7 @@ static KSTATUS IdeDmaRw(uint64_t lba, uint32_t count, void *buf, bool write) {
|
||||
// Blocks on gIrpSemaphore. Each IrpQueueEnqueue increments it by one, so
|
||||
// the worker wakes once per submitted IRP. Mirrors AhciWorker exactly.
|
||||
// ---------------------------------------------------------------------------
|
||||
static void IdeWorker(uintptr_t arg) {
|
||||
void IdeWorker(uintptr_t arg) {
|
||||
(void)arg;
|
||||
while (true) {
|
||||
KeWaitSemaphore(gIrpSemaphore);
|
||||
@@ -473,90 +369,6 @@ static KSTATUS IdeDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
|
||||
// Returns true on success. On any failure, logs a message and returns false;
|
||||
// dispatch will complete IRPs inline through PIO.
|
||||
// ---------------------------------------------------------------------------
|
||||
static bool IdeDmaInit(void) {
|
||||
// ---- Find PCI IDE controller (class 0x01, subclass 0x01) --------------
|
||||
uint8_t bus, dev, fn;
|
||||
if (!gHal->Pci.FindClass(0x01, 0x01, &bus, &dev, &fn)) {
|
||||
DbgPrint("IDE DMA: No PCI IDE controller found — using PIO\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Read BAR4 (Bus Master base, I/O space) ---------------------------
|
||||
// PCI config offset 0x20. Bit 0 must be 1 for I/O BAR.
|
||||
uint32_t bar4 = gHal->Pci.Read32(bus, dev, fn, 0x20);
|
||||
if (!(bar4 & 0x1)) {
|
||||
DbgPrint("IDE DMA: BAR4 is a memory BAR (0x%08x) — unexpected, using PIO\n", bar4);
|
||||
return false;
|
||||
}
|
||||
uint16_t bmBase = (uint16_t)(bar4 & 0xFFFC);
|
||||
if (bmBase == 0) {
|
||||
DbgPrint("IDE DMA: BAR4 is zero (firmware didn't assign it) — using PIO\n");
|
||||
return false;
|
||||
}
|
||||
DbgPrint("IDE DMA: Bus Master base = 0x%04X (bus %u, dev %u, fn %u)\n",
|
||||
bmBase, bus, dev, fn);
|
||||
|
||||
// ---- Enable Bus Mastering in PCI Command register (bit 2) ------------
|
||||
uint32_t pciCmd = gHal->Pci.Read32(bus, dev, fn, 0x04);
|
||||
gHal->Pci.Write32(bus, dev, fn, 0x04, pciCmd | (1u << 2));
|
||||
|
||||
// ---- Allocate PRDT page (must be 32-bit physical, 4-byte aligned) ----
|
||||
PHYSICAL_ADDRESS prdtPhys = MmAllocateContiguous(1);
|
||||
if (!prdtPhys) {
|
||||
DbgPrint("IDE DMA: Failed to allocate PRDT page\n");
|
||||
return false;
|
||||
}
|
||||
if (prdtPhys >> 32) {
|
||||
// MmAllocateContiguous returned memory above 4 GB — BM IDE can't
|
||||
// address it. This shouldn't happen on normal x86 systems with early
|
||||
// boot allocations but we check defensively.
|
||||
DbgPrint("IDE DMA: PRDT page is above 4 GB (0x%llx) — using PIO\n",
|
||||
(uint64_t)prdtPhys);
|
||||
return false;
|
||||
}
|
||||
void *prdtVirt = gHal->PhysToVirt(prdtPhys);
|
||||
DbgPrint("IDE DMA: PRDT at phys 0x%08x, virt %p\n", (uint32_t)prdtPhys, prdtVirt);
|
||||
|
||||
// ---- Create synchronisation objects ------------------------------------
|
||||
SEMAPHORE *sem = NULL;
|
||||
if (!KSUCCESS(KeSemaphoreCreate(0, 0x7FFFFFFF, &sem))) {
|
||||
DbgPrint("IDE DMA: Failed to create IRP semaphore\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
EVENT *dmaEv = NULL;
|
||||
if (!KSUCCESS(KeEventCreate(EventAutoReset, 0, &dmaEv))) {
|
||||
DbgPrint("IDE DMA: Failed to create DMA completion event\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Commit module state -----------------------------------------------
|
||||
gBmBase = bmBase;
|
||||
gPrdtPhys = (uint32_t)prdtPhys;
|
||||
gPrdtVirt = (PRD *)prdtVirt;
|
||||
gIrpSemaphore = sem;
|
||||
gDmaEvent = dmaEv;
|
||||
KeInitializeDpc(&gDmaDpc, IdeDmaCompletionDpc, (void *)0);
|
||||
|
||||
IrpQueueInit(&gIrpQueue, sem);
|
||||
|
||||
// ---- Register IRQ 14 --------------------------------------------------
|
||||
KeIrqRegister(IDE_IRQ_PRIMARY, IdeIsr);
|
||||
DbgPrint("IDE DMA: IRQ %u registered\n", IDE_IRQ_PRIMARY);
|
||||
|
||||
// ---- Spawn worker thread -----------------------------------------------
|
||||
KSTATUS st = KeCreateKernelThread((uintptr_t)IdeWorker, 0,
|
||||
PriorityHigh, (THREAD **)NULL);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("IDE DMA: Worker thread creation failed (0x%x)\n", st);
|
||||
// Not fatal for init — the PRDT/IRQ are live but async callers will
|
||||
// never get completions. Better to surface the failure clearly.
|
||||
return false;
|
||||
}
|
||||
|
||||
DbgPrint("IDE DMA: Init complete\n");
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DriverEntry
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
// ide.h - shared IDE/ATA driver internals.
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// IDE register map (primary channel, base 0x1F0)
|
||||
// ---------------------------------------------------------------------------
|
||||
#define IDE_DATA 0x1F0
|
||||
#define IDE_ERR 0x1F1
|
||||
#define IDE_SECCNT 0x1F2
|
||||
#define IDE_LBA0 0x1F3
|
||||
#define IDE_LBA1 0x1F4
|
||||
#define IDE_LBA2 0x1F5
|
||||
#define IDE_DRIVE 0x1F6
|
||||
#define IDE_STATUS 0x1F7
|
||||
#define IDE_CMD 0x1F7
|
||||
#define IDE_ALT 0x3F6
|
||||
|
||||
#define IDE_BUSY 0x80
|
||||
#define IDE_READY 0x40
|
||||
#define IDE_ERR_F 0x01
|
||||
#define IDE_DRQ 0x08
|
||||
|
||||
// ATA commands
|
||||
#define CMD_READ_EXT 0x24 // PIO 48-bit read
|
||||
#define CMD_WRITE_EXT 0x34 // PIO 48-bit write
|
||||
#define CMD_READ_DMA_EXT 0x25 // DMA 48-bit read
|
||||
#define CMD_WRITE_DMA_EXT 0x35 // DMA 48-bit write
|
||||
#define CMD_READ_DMA 0xC8 // DMA 28-bit read
|
||||
#define CMD_WRITE_DMA 0xCA // DMA 28-bit write
|
||||
#define CMD_IDENT 0xEC
|
||||
|
||||
// IDE primary IRQ
|
||||
#define IDE_IRQ_PRIMARY 14
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Bus Master IDE register offsets (relative to gBmBase, primary channel)
|
||||
// ---------------------------------------------------------------------------
|
||||
#define BM_CMD_REG 0x00 // command
|
||||
#define BM_STATUS_REG 0x02 // status
|
||||
#define BM_PRDT_REG 0x04 // PRDT physical address (32-bit)
|
||||
|
||||
// BM_CMD_REG bits
|
||||
#define BM_CMD_START 0x01 // start DMA transfer
|
||||
#define BM_CMD_READ 0x08 // 1 = disk→memory (read), 0 = memory→disk (write)
|
||||
|
||||
// BM_STATUS_REG bits
|
||||
#define BM_STATUS_ACTIVE 0x01 // DMA active
|
||||
#define BM_STATUS_ERROR 0x02 // DMA error (write 1 to clear)
|
||||
#define BM_STATUS_IRQ 0x04 // interrupt pending (write 1 to clear)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Physical Region Descriptor (PRDT entry, 8 bytes)
|
||||
// The PRDT itself must not cross a 64 KB boundary and must be 4-byte aligned.
|
||||
// We allocate a full page so both constraints are trivially satisfied.
|
||||
// ---------------------------------------------------------------------------
|
||||
#pragma pack(push, 1)
|
||||
typedef struct {
|
||||
uint32_t physAddr; // physical base of this memory region
|
||||
uint16_t byteCount; // byte count; 0 means 64 KB
|
||||
uint16_t flags; // bit 15: End Of Table
|
||||
} PRD;
|
||||
#pragma pack(pop)
|
||||
|
||||
#define PRD_EOT 0x8000u
|
||||
// 512 entries × 8 bytes = 4096 bytes = exactly one page; supports 512×64 KB = 32 MB
|
||||
#define PRDT_MAX_ENTRIES 512
|
||||
|
||||
|
||||
extern const HAL_API *gHal;
|
||||
extern DRIVER_OBJECT gDriver;
|
||||
extern DEVICE_OBJECT *gDevice;
|
||||
extern bool gDmaEnabled;
|
||||
extern uint64_t gSectorCount;
|
||||
extern bool gSupportsLba48;
|
||||
extern uint16_t gBmBase;
|
||||
extern PRD *gPrdtVirt;
|
||||
extern uint32_t gPrdtPhys;
|
||||
extern IRP_QUEUE gIrpQueue;
|
||||
extern SEMAPHORE *gIrpSemaphore;
|
||||
extern EVENT *gDmaEvent;
|
||||
extern KDPC gDmaDpc;
|
||||
|
||||
void IdeDmaCompletionDpc(KDPC *dpc, void *context, void *systemArgument1, void *systemArgument2);
|
||||
void IdeIsr(void);
|
||||
void IdeWorker(uintptr_t arg);
|
||||
bool IdeDmaInit(void);
|
||||
@@ -0,0 +1,87 @@
|
||||
// ide_bus.c - Bus-master IDE PCI/DMA initialisation.
|
||||
#include "ide.h"
|
||||
|
||||
bool IdeDmaInit(void) {
|
||||
// ---- Find PCI IDE controller (class 0x01, subclass 0x01) --------------
|
||||
uint8_t bus, dev, fn;
|
||||
if (!gHal->Pci.FindClass(0x01, 0x01, &bus, &dev, &fn)) {
|
||||
DbgPrint("IDE DMA: No PCI IDE controller found — using PIO\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Read BAR4 (Bus Master base, I/O space) ---------------------------
|
||||
// PCI config offset 0x20. Bit 0 must be 1 for I/O BAR.
|
||||
uint32_t bar4 = gHal->Pci.Read32(bus, dev, fn, 0x20);
|
||||
if (!(bar4 & 0x1)) {
|
||||
DbgPrint("IDE DMA: BAR4 is a memory BAR (0x%08x) — unexpected, using PIO\n", bar4);
|
||||
return false;
|
||||
}
|
||||
uint16_t bmBase = (uint16_t)(bar4 & 0xFFFC);
|
||||
if (bmBase == 0) {
|
||||
DbgPrint("IDE DMA: BAR4 is zero (firmware didn't assign it) — using PIO\n");
|
||||
return false;
|
||||
}
|
||||
DbgPrint("IDE DMA: Bus Master base = 0x%04X (bus %u, dev %u, fn %u)\n",
|
||||
bmBase, bus, dev, fn);
|
||||
|
||||
// ---- Enable Bus Mastering in PCI Command register (bit 2) ------------
|
||||
uint32_t pciCmd = gHal->Pci.Read32(bus, dev, fn, 0x04);
|
||||
gHal->Pci.Write32(bus, dev, fn, 0x04, pciCmd | (1u << 2));
|
||||
|
||||
// ---- Allocate PRDT page (must be 32-bit physical, 4-byte aligned) ----
|
||||
PHYSICAL_ADDRESS prdtPhys = MmAllocateContiguous(1);
|
||||
if (!prdtPhys) {
|
||||
DbgPrint("IDE DMA: Failed to allocate PRDT page\n");
|
||||
return false;
|
||||
}
|
||||
if (prdtPhys >> 32) {
|
||||
// MmAllocateContiguous returned memory above 4 GB — BM IDE can't
|
||||
// address it. This shouldn't happen on normal x86 systems with early
|
||||
// boot allocations but we check defensively.
|
||||
DbgPrint("IDE DMA: PRDT page is above 4 GB (0x%016llX) — using PIO\n",
|
||||
(uint64_t)prdtPhys);
|
||||
return false;
|
||||
}
|
||||
void *prdtVirt = gHal->PhysToVirt(prdtPhys);
|
||||
DbgPrint("IDE DMA: PRDT at phys 0x%08x, virt 0x%016llX\n", (uint32_t)prdtPhys, prdtVirt);
|
||||
|
||||
// ---- Create synchronisation objects ------------------------------------
|
||||
SEMAPHORE *sem = NULL;
|
||||
if (!KSUCCESS(KeSemaphoreCreate(0, 0x7FFFFFFF, &sem))) {
|
||||
DbgPrint("IDE DMA: Failed to create IRP semaphore\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
EVENT *dmaEv = NULL;
|
||||
if (!KSUCCESS(KeEventCreate(EventAutoReset, 0, &dmaEv))) {
|
||||
DbgPrint("IDE DMA: Failed to create DMA completion event\n");
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---- Commit module state -----------------------------------------------
|
||||
gBmBase = bmBase;
|
||||
gPrdtPhys = (uint32_t)prdtPhys;
|
||||
gPrdtVirt = (PRD *)prdtVirt;
|
||||
gIrpSemaphore = sem;
|
||||
gDmaEvent = dmaEv;
|
||||
KeInitializeDpc(&gDmaDpc, IdeDmaCompletionDpc, (void *)0);
|
||||
|
||||
IrpQueueInit(&gIrpQueue, sem);
|
||||
|
||||
// ---- Register IRQ 14 --------------------------------------------------
|
||||
KeIrqRegister(IDE_IRQ_PRIMARY, IdeIsr);
|
||||
DbgPrint("IDE DMA: IRQ %u registered\n", IDE_IRQ_PRIMARY);
|
||||
|
||||
// ---- Spawn worker thread -----------------------------------------------
|
||||
KSTATUS st = KeCreateKernelThread((uintptr_t)IdeWorker, 0,
|
||||
PriorityHigh, (THREAD **)NULL);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("IDE DMA: Worker thread creation failed (0x%x)\n", st);
|
||||
// Not fatal for init — the PRDT/IRQ are live but async callers will
|
||||
// never get completions. Better to surface the failure clearly.
|
||||
return false;
|
||||
}
|
||||
|
||||
DbgPrint("IDE DMA: Init complete\n");
|
||||
return true;
|
||||
}
|
||||
+76
-22
@@ -16,11 +16,23 @@ static volatile uint32_t* gLapic = (volatile uint32_t *)0;
|
||||
#define LAPIC_ICR_HI 0x310
|
||||
#define LAPIC_ENABLE 0x100 // bit in spurious vector register
|
||||
|
||||
#define IA32_APIC_BASE 0x0000001Bu
|
||||
#define IA32_APIC_BASE_ENABLE (1ULL << 11)
|
||||
#define IA32_APIC_BASE_X2APIC (1ULL << 10)
|
||||
#define IA32_X2APIC_ID 0x00000802u
|
||||
#define IA32_X2APIC_EOI 0x0000080Bu
|
||||
#define IA32_X2APIC_SVR 0x0000080Fu
|
||||
#define IA32_X2APIC_ICR 0x00000830u
|
||||
|
||||
#define CPUID_FEATURE_ECX_X2APIC (1u << 21)
|
||||
|
||||
#define APIC_VECTOR_RESCHEDULE 0xF0u
|
||||
|
||||
// ---- I/O APIC ----
|
||||
static volatile uint32_t* gIoapic = (volatile uint32_t *)0;
|
||||
static uint32_t gBspLapicId = 0;
|
||||
static uint8_t gX2ApicSupported = 0;
|
||||
static uint8_t gX2ApicEnabled = 0;
|
||||
#define IOAPIC_IDX 0x00
|
||||
#define IOAPIC_DATA 0x10
|
||||
|
||||
@@ -33,6 +45,14 @@ static void IoapicWrite(uint8_t reg, uint32_t val) {
|
||||
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val;
|
||||
}
|
||||
|
||||
static void HalApicDetectMode(void) {
|
||||
int regs[4] = {0, 0, 0, 0};
|
||||
__cpuid(regs, 1);
|
||||
gX2ApicSupported = (regs[2] & CPUID_FEATURE_ECX_X2APIC) ? 1u : 0u;
|
||||
|
||||
uint64_t apicBase = __readmsr(IA32_APIC_BASE);
|
||||
gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u;
|
||||
}
|
||||
|
||||
static void HalApicStallFallback(uint32_t loops) {
|
||||
for (volatile uint32_t i = 0; i < loops; i++) {
|
||||
@@ -46,6 +66,9 @@ static void HalApicStallUs(uint32_t us) {
|
||||
}
|
||||
|
||||
static void HalApicWaitIcrIdle(void) {
|
||||
if (gX2ApicEnabled)
|
||||
return;
|
||||
|
||||
for (uint32_t i = 0; i < 1000000u; i++) {
|
||||
if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12)))
|
||||
return;
|
||||
@@ -54,6 +77,11 @@ static void HalApicWaitIcrIdle(void) {
|
||||
}
|
||||
|
||||
static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
|
||||
if (gX2ApicEnabled) {
|
||||
__writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low);
|
||||
return;
|
||||
}
|
||||
|
||||
HalApicWaitIcrIdle();
|
||||
LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24;
|
||||
LAPIC_REG(LAPIC_ICR_LO) = low;
|
||||
@@ -63,52 +91,70 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
|
||||
// IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high).
|
||||
#define IOREDTBL_LO(n) (0x10 + 2*(n))
|
||||
#define IOREDTBL_HI(n) (0x11 + 2*(n))
|
||||
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> 0x21, ...
|
||||
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
|
||||
#define IRQ_VECTOR_BASE 0x20
|
||||
|
||||
__declspec(dllexport) uint32_t HalGetBspLapicId(void) {
|
||||
return gBspLapicId;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
|
||||
if (!gLapic) return;
|
||||
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
|
||||
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void) {
|
||||
return gX2ApicEnabled;
|
||||
}
|
||||
|
||||
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId) {
|
||||
if (gX2ApicEnabled)
|
||||
return 1u;
|
||||
return (lapicId < 256u) ? 1u : 0u;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
|
||||
uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE;
|
||||
|
||||
if (gX2ApicEnabled) {
|
||||
apicBase |= IA32_APIC_BASE_X2APIC;
|
||||
__writemsr(IA32_APIC_BASE, apicBase);
|
||||
__writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu);
|
||||
return;
|
||||
}
|
||||
|
||||
__writemsr(IA32_APIC_BASE, apicBase);
|
||||
if (gLapic)
|
||||
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void HalApicSendPanicIpi(void) {
|
||||
if (!gLapic) return;
|
||||
if (!gLapic && !gX2ApicEnabled) return;
|
||||
|
||||
// Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed
|
||||
// Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed
|
||||
// vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still
|
||||
// enter the panic path. The NMI vector is already installed in the IDT on
|
||||
// enter the panic path. The NMI vector is already installed in the IDT on
|
||||
// every CPU that has reached kernel-owned state.
|
||||
HalApicWaitIcrIdle();
|
||||
LAPIC_REG(LAPIC_ICR_HI) = 0;
|
||||
LAPIC_REG(LAPIC_ICR_LO) = (3u << 18) | (4u << 8); // all-excluding-self, NMI
|
||||
HalApicWaitIcrIdle();
|
||||
HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI
|
||||
}
|
||||
|
||||
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) {
|
||||
if (!gLapic) return;
|
||||
if (!gLapic && !gX2ApicEnabled) return;
|
||||
HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination
|
||||
}
|
||||
|
||||
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
|
||||
if (!gLapic) return;
|
||||
if (!gLapic && !gX2ApicEnabled) return;
|
||||
|
||||
// TLB shootdown must reach CPUs that may be spinning with IF=0 inside
|
||||
// high-IRQL/spinlock regions, so use NMI delivery rather than a fixed
|
||||
// vector. The NMI stub handles active shootdowns and returns; unexpected
|
||||
// vector. The NMI stub handles active shootdowns and returns; unexpected
|
||||
// NMIs still go through the normal fault/panic path.
|
||||
HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) {
|
||||
if (!gLapic || !startupVector || lapicId == gBspLapicId)
|
||||
if ((!gLapic && !gX2ApicEnabled) || !startupVector || lapicId == gBspLapicId)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (!HalApicCanAddressId(lapicId))
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
|
||||
// INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12.
|
||||
// INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12.
|
||||
HalApicSendIpi(lapicId, 0x0000C500u); // INIT, level assert, level-triggered
|
||||
HalApicStallUs(10000);
|
||||
HalApicSendIpi(lapicId, 0x00008500u); // INIT, level deassert
|
||||
@@ -124,7 +170,7 @@ void HalApicRouteIrq(uint8_t irq) {
|
||||
// Full initial setup of the redirection entry.
|
||||
// Sets the vector and clears the mask bit in one write.
|
||||
uint32_t vector = IRQ_VECTOR_BASE + irq; // e.g. IRQ 1 -> vector 0x21
|
||||
IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24); // external IRQs target the BSP
|
||||
IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24); // external IRQs target the BSP
|
||||
IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed
|
||||
}
|
||||
|
||||
@@ -138,29 +184,37 @@ void HalApicUnmaskIrq(uint8_t irq) {
|
||||
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
|
||||
lo &= ~(1 << 16); // clear mask bit
|
||||
lo = (lo & ~0xFF) | (IRQ_VECTOR_BASE + irq);
|
||||
IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24);
|
||||
IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24);
|
||||
IoapicWrite(IOREDTBL_LO(irq), lo);
|
||||
}
|
||||
|
||||
void HalApicSendEoi(void) {
|
||||
if (gX2ApicEnabled) {
|
||||
__writemsr(IA32_X2APIC_EOI, 0);
|
||||
return;
|
||||
}
|
||||
LAPIC_REG(LAPIC_EOI) = 0;
|
||||
}
|
||||
|
||||
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
|
||||
HalApicDetectMode();
|
||||
|
||||
gLapic = (volatile uint32_t *)(gApi.PhysToVirt(lapicPhys));
|
||||
gIoapic = (volatile uint32_t *)(gApi.PhysToVirt(ioapicPhys));
|
||||
|
||||
// The BSP is the CPU executing HAL init. Keep all external/device IRQs
|
||||
// routed to this LAPIC; CPU-local IPIs use the local APIC directly.
|
||||
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
|
||||
if (gX2ApicEnabled)
|
||||
gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID);
|
||||
else
|
||||
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
|
||||
|
||||
// Enable Local APIC, set spurious vector to 0xFF.
|
||||
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
|
||||
HalApicEnableCurrentCpu();
|
||||
|
||||
// Mask all IOAPIC lines initially.
|
||||
uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
|
||||
for (uint8_t i = 0; i <= maxRedir; i++) {
|
||||
IoapicWrite(IOREDTBL_LO(i), (1 << 16) | (IRQ_VECTOR_BASE + i));
|
||||
IoapicWrite(IOREDTBL_HI(i), gBspLapicId << 24);
|
||||
IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24);
|
||||
}
|
||||
}
|
||||
@@ -1,204 +1 @@
|
||||
// hal_keyboard.c - PS/2 keyboard, interrupt-driven, with live state tracking
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <intrin.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
|
||||
// -- Port addresses ------------------------------------------------------------
|
||||
#define PS2_DATA 0x60u
|
||||
#define PS2_STAT 0x64u
|
||||
#define PS2_CMD 0x64u
|
||||
|
||||
// -- Ring buffer (for ASCII/typing) -------------------------------------------
|
||||
#define KBD_BUF_SIZE 256u
|
||||
#define KBD_BUF_MASK (KBD_BUF_SIZE - 1u)
|
||||
|
||||
static uint8_t gBuf[KBD_BUF_SIZE];
|
||||
static volatile int gHead = 0;
|
||||
static volatile int gTail = 0;
|
||||
|
||||
// -- Deferred callback queue ---------------------------------------------------
|
||||
// The hard IRQ path updates the key state and stores completed make codes.
|
||||
// Optional client callbacks are delivered later from the kernel-queued DPC,
|
||||
// never directly from DIRQL/ISR context.
|
||||
#define KBD_DPC_BUF_SIZE 128u
|
||||
#define KBD_DPC_BUF_MASK (KBD_DPC_BUF_SIZE - 1u)
|
||||
|
||||
static uint8_t gDpcBuf[KBD_DPC_BUF_SIZE];
|
||||
static volatile int gDpcHead = 0;
|
||||
static volatile int gDpcTail = 0;
|
||||
|
||||
// -- Live Key State (for GetAsyncKeyState) ------------------------------------
|
||||
// These arrays track whether a key is currently held down.
|
||||
static volatile uint8_t gKeyState[128]; // Standard scan codes
|
||||
static volatile uint8_t gKeyStateExt[128]; // Extended (E0) scan codes
|
||||
|
||||
// -- Scan-code set 1 constants -------------------------------------------------
|
||||
#define SC1_BREAK_BIT 0x80u
|
||||
#define SC1_MAKE_MASK 0x7Fu
|
||||
#define SC1_EXT_PREFIX 0xE0u
|
||||
|
||||
// -- Modifier flags ------------------------------------------------------------
|
||||
#define MOD_LSHIFT (1u << 0)
|
||||
#define MOD_RSHIFT (1u << 1)
|
||||
#define MOD_LCTRL (1u << 2)
|
||||
#define MOD_RCTRL (1u << 3)
|
||||
#define MOD_LALT (1u << 4)
|
||||
#define MOD_RALT (1u << 5)
|
||||
#define MOD_CAPS (1u << 6)
|
||||
#define MOD_NUMLOCK (1u << 7)
|
||||
|
||||
#define MOD_SHIFT (MOD_LSHIFT | MOD_RSHIFT)
|
||||
#define MOD_CTRL (MOD_LCTRL | MOD_RCTRL)
|
||||
#define MOD_ALT (MOD_LALT | MOD_RALT)
|
||||
|
||||
static volatile uint32_t gModifiers = 0;
|
||||
static volatile bool gNextExtended = false;
|
||||
static void (* volatile gCallback)(uint8_t) = NULL;
|
||||
|
||||
// -- ASCII Translation Tables --------------------------------------------------
|
||||
static const char gUnshifted[128] = {
|
||||
0, 0, '1','2','3','4','5','6','7','8','9','0','-','=','\b','\t',
|
||||
'q','w','e','r','t','y','u','i','o','p','[',']','\n', 0, 'a','s',
|
||||
'd','f','g','h','j','k','l',';','\'','`', 0, '\\','z','x','c','v',
|
||||
'b','n','m',',','.','/', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.'
|
||||
};
|
||||
|
||||
static const char gShifted[128] = {
|
||||
0, 0, '!','@','#','$','%','^','&','*','(',')','_','+','\b','\t',
|
||||
'Q','W','E','R','T','Y','U','I','O','P','{','}','\n', 0, 'A','S',
|
||||
'D','F','G','H','J','K','L',':','"','~', 0, '|','Z','X','C','V',
|
||||
'B','N','M','<','>','?', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.'
|
||||
};
|
||||
|
||||
// -- Deferred callback delivery ------------------------------------------------
|
||||
static void KbdDpc(void) {
|
||||
for (;;) {
|
||||
if (gDpcHead == gDpcTail)
|
||||
break;
|
||||
|
||||
uint8_t make = gDpcBuf[gDpcHead];
|
||||
gDpcHead = (gDpcHead + 1) & (int)KBD_DPC_BUF_MASK;
|
||||
|
||||
void (*cb)(uint8_t) = (void (*)(uint8_t))gCallback;
|
||||
if (cb)
|
||||
cb(make);
|
||||
}
|
||||
}
|
||||
|
||||
// -- ISR: The Engine -----------------------------------------------------------
|
||||
void KbdIsr(void) {
|
||||
uint8_t sc = __inbyte(PS2_DATA);
|
||||
|
||||
if (sc == SC1_EXT_PREFIX) {
|
||||
gNextExtended = true;
|
||||
return;
|
||||
}
|
||||
|
||||
bool extended = gNextExtended;
|
||||
bool is_break = (sc & SC1_BREAK_BIT) != 0;
|
||||
uint8_t make = sc & SC1_MAKE_MASK;
|
||||
gNextExtended = false;
|
||||
|
||||
// Update the live state arrays
|
||||
if (extended) {
|
||||
gKeyStateExt[make] = is_break ? 0 : 1;
|
||||
} else {
|
||||
gKeyState[make] = is_break ? 0 : 1;
|
||||
}
|
||||
|
||||
// Update modifiers
|
||||
if (!extended) {
|
||||
switch (make) {
|
||||
case 0x2A: gModifiers = is_break ? (gModifiers & ~MOD_LSHIFT) : (gModifiers | MOD_LSHIFT); break;
|
||||
case 0x36: gModifiers = is_break ? (gModifiers & ~MOD_RSHIFT) : (gModifiers | MOD_RSHIFT); break;
|
||||
case 0x1D: gModifiers = is_break ? (gModifiers & ~MOD_LCTRL) : (gModifiers | MOD_LCTRL); break;
|
||||
case 0x38: gModifiers = is_break ? (gModifiers & ~MOD_LALT) : (gModifiers | MOD_LALT); break;
|
||||
case 0x3A: if (!is_break) gModifiers ^= MOD_CAPS; break;
|
||||
case 0x45: if (!is_break) gModifiers ^= MOD_NUMLOCK; break;
|
||||
}
|
||||
} else {
|
||||
switch (make) {
|
||||
case 0x1D: gModifiers = is_break ? (gModifiers & ~MOD_RCTRL) : (gModifiers | MOD_RCTRL); break;
|
||||
case 0x38: gModifiers = is_break ? (gModifiers & ~MOD_RALT) : (gModifiers | MOD_RALT); break;
|
||||
}
|
||||
}
|
||||
|
||||
// Buffer management (only for 'make' codes)
|
||||
if (!is_break) {
|
||||
int next = (gTail + 1) & (int)KBD_BUF_MASK;
|
||||
if (next != gHead) {
|
||||
gBuf[gTail] = make;
|
||||
gTail = next;
|
||||
}
|
||||
|
||||
// Queue optional callback delivery to the DPC path. Keep a separate
|
||||
// callback queue so the public ReadScancode ring remains independent.
|
||||
if (gCallback) {
|
||||
int dpcNext = (gDpcTail + 1) & (int)KBD_DPC_BUF_MASK;
|
||||
if (dpcNext != gDpcHead) {
|
||||
gDpcBuf[gDpcTail] = make;
|
||||
gDpcTail = dpcNext;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -- Public API ----------------------------------------------------------------
|
||||
|
||||
// Use this inside your GetAsyncKeyState implementation!
|
||||
bool KbdIsKeyDown(uint8_t scancode, bool extended) {
|
||||
if (extended) return gKeyStateExt[scancode & 0x7F] != 0;
|
||||
return gKeyState[scancode & 0x7F] != 0;
|
||||
}
|
||||
|
||||
static uint8_t KbdPoll(void) {
|
||||
return (gHead == gTail) ? 0 : gBuf[gHead];
|
||||
}
|
||||
|
||||
static uint8_t KbdReadScancode(void) {
|
||||
while (gHead == gTail) { _mm_pause(); }
|
||||
uint8_t sc = gBuf[gHead];
|
||||
gHead = (gHead + 1) & (int)KBD_BUF_MASK;
|
||||
return sc;
|
||||
}
|
||||
|
||||
static char KbdReadChar(void) {
|
||||
uint8_t make = KbdReadScancode();
|
||||
if (make >= 128) return 0;
|
||||
|
||||
uint32_t mods = gModifiers;
|
||||
bool shifted = (mods & MOD_SHIFT) != 0;
|
||||
|
||||
if (mods & MOD_CAPS) {
|
||||
char base = gUnshifted[make];
|
||||
if (base >= 'a' && base <= 'z') shifted = !shifted;
|
||||
}
|
||||
return shifted ? gShifted[make] : gUnshifted[make];
|
||||
}
|
||||
|
||||
static uint32_t KbdGetModifiers(void) {
|
||||
return gModifiers;
|
||||
}
|
||||
|
||||
static void KbdSetCallback(void (*cb)(uint8_t scancode)) {
|
||||
gCallback = cb;
|
||||
}
|
||||
|
||||
void HalKeyboardInitialize(HAL_KEYBOARD *out) {
|
||||
while (__inbyte(PS2_STAT) & 2);
|
||||
__outbyte(PS2_CMD, 0xA8);
|
||||
while (__inbyte(PS2_STAT) & 1) { __inbyte(PS2_DATA); }
|
||||
|
||||
out->Poll = KbdPoll;
|
||||
out->ReadScancode = KbdReadScancode;
|
||||
out->ReadChar = KbdReadChar;
|
||||
out->GetModifiers = KbdGetModifiers;
|
||||
out->SetCallback = KbdSetCallback;
|
||||
out->IsKeyDown = KbdIsKeyDown;
|
||||
out->Isr = KbdIsr;
|
||||
out->Dpc = KbdDpc;
|
||||
out->IrqNumber = 1;
|
||||
}
|
||||
// hal_keyboard.c - input moved out of HAL into kernel input core and ps2.sys.
|
||||
@@ -5,12 +5,11 @@
|
||||
#include <stdint.h>
|
||||
#include <stdarg.h>
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_disable)
|
||||
#include <common/printf.h>
|
||||
|
||||
// Declared in other hal_*.c files:
|
||||
void HalScreenInitialize(uintptr_t fb, int w, int h, int stride, HAL_SCREEN* out);
|
||||
void HalKeyboardInitialize(HAL_KEYBOARD* out);
|
||||
void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH);
|
||||
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
|
||||
HAL_SCREEN* out);
|
||||
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out);
|
||||
void HalPciInitialize(HAL_PCI* out);
|
||||
extern void HalSetDebugEnabled(int v);
|
||||
@@ -242,7 +241,7 @@ static void ParseMadt(MADT* madt) {
|
||||
uint32_t lapicId = *(uint32_t *)(p + 4);
|
||||
uint32_t flags = *(uint32_t *)(p + 8);
|
||||
uint32_t uid = *(uint32_t *)(p + 12);
|
||||
if ((flags & 1u) && lapicId < 256u)
|
||||
if (flags & 1u)
|
||||
HalAddCpuRecord(uid, lapicId, flags);
|
||||
}
|
||||
|
||||
@@ -395,7 +394,7 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
|
||||
debugEnabled = initParams->Debug;
|
||||
|
||||
// Screen first so DbgPrint works immediately.
|
||||
HalScreenInitialize(initParams->FramebufferBase + initParams->PhysicalWindowBase,
|
||||
HalScreenInitialize(initParams->AllocatePool, initParams->FramebufferBase + initParams->PhysicalWindowBase,
|
||||
(int)initParams->FramebufferWidth,
|
||||
(int)initParams->FramebufferHeight,
|
||||
(int)initParams->FramebufferStride,
|
||||
@@ -412,10 +411,9 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
|
||||
gLapicPhys, gIoapicPhys, gHpetPhys);
|
||||
|
||||
HalApicInitialize(gLapicPhys, gIoapicPhys);
|
||||
HalKeyboardInitialize(&gApi.Keyboard);
|
||||
HalMouseInitialize(&gApi.Mouse,
|
||||
(int)initParams->FramebufferWidth,
|
||||
(int)initParams->FramebufferHeight);
|
||||
HalDbgPrint("HAL: local APIC mode=%s BSP_ID=%u\n",
|
||||
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
|
||||
HalGetBspLapicId());
|
||||
|
||||
KSTATUS ts = HalTimerInitialize(gLapicPhys, gHpetPhys, &gApi.Timer);
|
||||
if (!KSUCCESS(ts)) HalDbgPrint("HAL: HPET unavailable, using fallback\n");
|
||||
|
||||
+1
-305
@@ -1,305 +1 @@
|
||||
// hal_mouse.c - PS/2 auxiliary mouse driver
|
||||
//
|
||||
// Sits on IRQ 12, sharing the 8042 controller with the keyboard (IRQ 1).
|
||||
// hal_keyboard.c has already issued command 0xA8 (enable aux port) by the
|
||||
// time HalMouseInitialize is called, so we only need to:
|
||||
// 1. Enable aux interrupts in the 8042 command byte (bit 1, clears bit 5).
|
||||
// 2. Send the standard mouse init sequence (reset → defaults → enable).
|
||||
// 3. Wire up the ISR and expose the HAL_MOUSE vtable.
|
||||
//
|
||||
// Packet format - standard 3-byte PS/2 mouse (no scroll wheel):
|
||||
// Byte 0: [YOV][XOV][YS][XS][1][M][R][L]
|
||||
// Byte 1: delta-X unsigned magnitude (sign in byte-0 bit 4)
|
||||
// Byte 2: delta-Y unsigned magnitude (sign in byte-0 bit 5); PS/2 Y+ = up
|
||||
//
|
||||
// The ISR accumulates the 3-byte packet, converts it to a screen-space delta
|
||||
// (Y-axis flipped), and queues the completed packet to the HAL's kernel-queued
|
||||
// bottom half. Cursor drawing, public event enqueue, and optional callbacks
|
||||
// run from that bottom half instead of hard IRQ context.
|
||||
//
|
||||
// Threading model
|
||||
// ---------------
|
||||
// Public events are produced by the mouse DPC and consumed by callers.
|
||||
// Raw completed packets are produced by the ISR and consumed by the same DPC.
|
||||
// On SMP, add acquire/release barriers around the ring head/tail pairs.
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <intrin.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/hal/hal_debug.h>
|
||||
|
||||
// -- PS/2 port addresses -------------------------------------------------------
|
||||
#define PS2_DATA 0x60u
|
||||
#define PS2_STAT 0x64u
|
||||
#define PS2_CMD 0x64u
|
||||
|
||||
// -- Ring buffer ---------------------------------------------------------------
|
||||
#define MOUSE_BUF_SIZE 64u
|
||||
#define MOUSE_BUF_MASK (MOUSE_BUF_SIZE - 1u)
|
||||
|
||||
typedef struct {
|
||||
int dx, dy;
|
||||
uint8_t buttons; // bit0=left, bit1=right, bit2=middle
|
||||
} MOUSE_EVENT;
|
||||
|
||||
static MOUSE_EVENT gBuf[MOUSE_BUF_SIZE];
|
||||
static volatile int gHead = 0; // public consumer read pointer
|
||||
static volatile int gTail = 0; // public producer write pointer (updated by DPC)
|
||||
|
||||
// Completed hardware packets waiting for the DPC bottom half.
|
||||
static MOUSE_EVENT gDpcBuf[MOUSE_BUF_SIZE];
|
||||
static volatile int gDpcHead = 0;
|
||||
static volatile int gDpcTail = 0;
|
||||
|
||||
// -- Absolute cursor position & screen bounds ----------------------------------
|
||||
static volatile int gMouseX = 0;
|
||||
static volatile int gMouseY = 0;
|
||||
static volatile int gBoundsW = 1920;
|
||||
static volatile int gBoundsH = 1080;
|
||||
|
||||
// -- 3-byte packet assembly ----------------------------------------------------
|
||||
static volatile uint8_t gPacket[3];
|
||||
static volatile int gPacketIdx = 0; // 0, 1, or 2
|
||||
|
||||
// -- Optional callback (called from mouse DPC) ---------------------------------
|
||||
static void (* volatile gCallback)(int dx, int dy, uint8_t buttons) = NULL;
|
||||
|
||||
// -- Cursor helpers provided by hal_screen.c -----------------------------------
|
||||
// Declared here; defined in hal_screen.c.
|
||||
extern void HalScreenDrawCursor(int x, int y);
|
||||
extern void HalScreenEraseCursor(void);
|
||||
|
||||
// -- 8042 low-level helpers ----------------------------------------------------
|
||||
// Wait until the 8042's input buffer is empty (bit 1 of status = 0) so we
|
||||
// can safely write to PS2_DATA or PS2_CMD.
|
||||
static void Ps2WaitInput(void) {
|
||||
int timeout = 100000;
|
||||
while ((__inbyte(PS2_STAT) & 0x02u) && --timeout) { /* spin */ }
|
||||
}
|
||||
|
||||
// Wait until the 8042's output buffer has data (bit 0 of status = 1).
|
||||
static void Ps2WaitOutput(void) {
|
||||
int timeout = 5000000;
|
||||
while (!(__inbyte(PS2_STAT) & 0x01u) && --timeout) { /* spin */ }
|
||||
}
|
||||
|
||||
// Reads from PS2_DATA until the output buffer is empty
|
||||
static void Ps2FlushOutput(void) {
|
||||
int timeout = 100000;
|
||||
while ((__inbyte(PS2_STAT) & 0x01u) && --timeout) {
|
||||
__inbyte(PS2_DATA);
|
||||
}
|
||||
}
|
||||
|
||||
// Route the next write to the auxiliary (mouse) device rather than the keyboard.
|
||||
static void MouseSendByte(uint8_t byte) {
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, 0xD4u); // "write to mouse" prefix
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_DATA, byte);
|
||||
}
|
||||
|
||||
// Read one byte from the PS/2 output buffer (shared by keyboard and mouse).
|
||||
static uint8_t MouseRecvByte(void) {
|
||||
Ps2WaitOutput();
|
||||
return __inbyte(PS2_DATA);
|
||||
}
|
||||
|
||||
// -- Deferred mouse bottom half -------------------------------------------------
|
||||
static void MouseDpc(void) {
|
||||
for (;;) {
|
||||
if (gDpcHead == gDpcTail)
|
||||
break;
|
||||
|
||||
MOUSE_EVENT ev = gDpcBuf[gDpcHead];
|
||||
gDpcHead = (gDpcHead + 1) & (int)MOUSE_BUF_MASK;
|
||||
|
||||
// Update absolute position and redraw the cursor outside DIRQL.
|
||||
HalScreenEraseCursor();
|
||||
|
||||
int nx = gMouseX + ev.dx;
|
||||
int ny = gMouseY + ev.dy;
|
||||
if (nx < 0) nx = 0;
|
||||
if (nx >= gBoundsW) nx = gBoundsW - 1;
|
||||
if (ny < 0) ny = 0;
|
||||
if (ny >= gBoundsH) ny = gBoundsH - 1;
|
||||
gMouseX = nx;
|
||||
gMouseY = ny;
|
||||
|
||||
HalScreenDrawCursor(nx, ny);
|
||||
|
||||
// Publish event to the normal consumer ring.
|
||||
int next = (gTail + 1) & (int)MOUSE_BUF_MASK;
|
||||
if (next != gHead) {
|
||||
gBuf[gTail] = ev;
|
||||
gTail = next;
|
||||
}
|
||||
|
||||
void (*cb)(int, int, uint8_t) = (void (*)(int, int, uint8_t))gCallback;
|
||||
if (cb)
|
||||
cb(ev.dx, ev.dy, ev.buttons);
|
||||
}
|
||||
}
|
||||
|
||||
// -- Interrupt service routine --------------------------------------------------
|
||||
// Called with interrupts disabled by KeIrqDispatch (IRQ 12).
|
||||
// The dispatcher sends the APIC EOI after this function returns.
|
||||
static void MouseIsr(void) {
|
||||
uint8_t data = __inbyte(PS2_DATA);
|
||||
|
||||
// Synchronisation guard: the first byte of every packet has bit 3 set.
|
||||
// If we lose sync, wait for the next valid first byte before assembling.
|
||||
if (gPacketIdx == 0) {
|
||||
if (!(data & 0x08u)) return; // not a valid first byte - discard
|
||||
}
|
||||
|
||||
gPacket[gPacketIdx++] = data;
|
||||
if (gPacketIdx < 3) return; // packet not yet complete
|
||||
gPacketIdx = 0;
|
||||
|
||||
// -- Decode flags ----------------------------------------------------------
|
||||
uint8_t flags = gPacket[0];
|
||||
|
||||
// Overflow bits: if either is set the movement data is meaningless.
|
||||
if (flags & 0xC0u) return;
|
||||
|
||||
uint8_t buttons = flags & 0x07u;
|
||||
|
||||
// Convert unsigned magnitudes + sign bits to signed 9-bit values.
|
||||
int dx = (int)gPacket[1] - ((flags & 0x10u) ? 256 : 0);
|
||||
int dy = (int)gPacket[2] - ((flags & 0x20u) ? 256 : 0);
|
||||
dy = -dy; // PS/2 Y+ = up; screen Y+ = down, so invert.
|
||||
|
||||
// -- Queue completed packet to the DPC bottom half -------------------------
|
||||
int next = (gDpcTail + 1) & (int)MOUSE_BUF_MASK;
|
||||
if (next != gDpcHead) {
|
||||
gDpcBuf[gDpcTail].dx = dx;
|
||||
gDpcBuf[gDpcTail].dy = dy;
|
||||
gDpcBuf[gDpcTail].buttons = buttons;
|
||||
gDpcTail = next;
|
||||
}
|
||||
}
|
||||
|
||||
// -- Consumer API --------------------------------------------------------------
|
||||
static bool MousePoll(void) {
|
||||
return gHead != gTail;
|
||||
}
|
||||
|
||||
static bool MouseRead(int* dx, int* dy, uint8_t* buttons) {
|
||||
if (gHead == gTail) return false;
|
||||
*dx = gBuf[gHead].dx;
|
||||
*dy = gBuf[gHead].dy;
|
||||
*buttons = gBuf[gHead].buttons;
|
||||
gHead = (gHead + 1) & (int)MOUSE_BUF_MASK;
|
||||
return true;
|
||||
}
|
||||
|
||||
static void MouseGetPosition(int* x, int* y) {
|
||||
*x = gMouseX;
|
||||
*y = gMouseY;
|
||||
}
|
||||
|
||||
static void MouseSetPosition(int x, int y) {
|
||||
gMouseX = x;
|
||||
gMouseY = y;
|
||||
}
|
||||
|
||||
static void MouseSetBounds(int w, int h) {
|
||||
gBoundsW = w;
|
||||
gBoundsH = h;
|
||||
}
|
||||
|
||||
static void MouseSetCallback(void (*cb)(int dx, int dy, uint8_t buttons)) {
|
||||
gCallback = cb;
|
||||
}
|
||||
|
||||
// -- Initializer ---------------------------------------------------------------
|
||||
// Called from HalInitialize in hal_main.c after the screen is ready.
|
||||
// hal_keyboard.c has already sent 0xA8 (enable aux port) to port 0x64.
|
||||
void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH) {
|
||||
gBoundsW = screenW;
|
||||
gBoundsH = screenH;
|
||||
gMouseX = screenW / 2;
|
||||
gMouseY = screenH / 2;
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 0: Deafen the controller and flush
|
||||
// -----------------------------------------------------------------
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, 0xA7u); // Disable Aux port (ignores mouse)
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, 0xADu); // Disable Main port (ignores keyboard)
|
||||
|
||||
// Now flush the buffer. Because the ports are disabled, we guarantee
|
||||
// no new data can sneak in behind our backs.
|
||||
Ps2FlushOutput();
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 1: Safely configure the 8042 Command Byte
|
||||
// -----------------------------------------------------------------
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, 0x20u); // read current command byte
|
||||
Ps2WaitOutput();
|
||||
uint8_t cmd = __inbyte(PS2_DATA);
|
||||
|
||||
cmd |= 0x02u; // enable aux interrupt
|
||||
cmd &= ~0x20u; // clear "disable aux clock"
|
||||
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, 0x60u); // write command byte
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_DATA, cmd);
|
||||
|
||||
// Re-enable both ports now that config is safely written
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, 0xA8u); // Enable Aux port
|
||||
Ps2WaitInput();
|
||||
__outbyte(PS2_CMD, 0xAEu); // Enable Main port
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 2: Reset mouse (0xFF).
|
||||
// -----------------------------------------------------------------
|
||||
MouseSendByte(0xFFu); // reset
|
||||
|
||||
// The BIOS or an earlier phase might have left a movement packet
|
||||
// partially in flight right as we re-enabled the port.
|
||||
// We loop and discard anything that isn't the 0xFA ACK.
|
||||
uint8_t ack;
|
||||
int retries = 10;
|
||||
do {
|
||||
ack = MouseRecvByte();
|
||||
} while (ack != 0xFAu && --retries > 0);
|
||||
|
||||
// The output timeout covers the physical hardware self-test latency.
|
||||
MouseRecvByte(); // 0xAA - BAT passed
|
||||
MouseRecvByte(); // 0x00 - device ID
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 3: Set default parameters (sampling rate 100, 4 counts/mm).
|
||||
// Mouse replies: ACK.
|
||||
// -----------------------------------------------------------------
|
||||
MouseSendByte(0xF6u); // set defaults
|
||||
MouseRecvByte(); // ACK
|
||||
|
||||
// -----------------------------------------------------------------
|
||||
// Step 4: Enable data reporting.
|
||||
// Mouse will now send 3-byte packets on every movement/button event.
|
||||
// -----------------------------------------------------------------
|
||||
MouseSendByte(0xF4u); // enable data reporting
|
||||
MouseRecvByte(); // ACK
|
||||
|
||||
// Draw the initial cursor at the centre of the screen.
|
||||
HalScreenDrawCursor(gMouseX, gMouseY);
|
||||
|
||||
// Populate vtable.
|
||||
out->Poll = MousePoll;
|
||||
out->Read = MouseRead;
|
||||
out->SetCallback = MouseSetCallback;
|
||||
out->GetPosition = MouseGetPosition;
|
||||
out->SetPosition = MouseSetPosition;
|
||||
out->SetBounds = MouseSetBounds;
|
||||
out->Isr = MouseIsr;
|
||||
out->Dpc = MouseDpc;
|
||||
out->IrqNumber = 12;
|
||||
}
|
||||
// hal_mouse.c - input moved out of HAL into kernel input core and ps2.sys.
|
||||
@@ -1,10 +1,27 @@
|
||||
// hal_pci.c - PCI configuration space via Mechanism 1 port I/O
|
||||
#include <stdint.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/hal/hal_apic.h>
|
||||
#include <kernel/hal/hal_debug.h>
|
||||
|
||||
#define PCI_CONFIG_ADDR 0xCF8
|
||||
#define PCI_CONFIG_DATA 0xCFC
|
||||
|
||||
#define PCI_REG_COMMAND_STATUS 0x04u
|
||||
#define PCI_REG_CAP_PTR 0x34u
|
||||
#define PCI_STATUS_CAP_LIST (1u << 20)
|
||||
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
|
||||
|
||||
#define PCI_CAP_ID_MSI 0x05u
|
||||
#define PCI_CAP_ID_MSIX 0x11u
|
||||
#define PCI_MSI_CTRL_ENABLE (1u << 0)
|
||||
#define PCI_MSI_CTRL_MME_MASK (7u << 4)
|
||||
#define PCI_MSI_CTRL_64BIT (1u << 7)
|
||||
#define PCI_MSI_CTRL_PVM (1u << 8)
|
||||
|
||||
#define PCI_MSI_ADDRESS_BASE 0xFEE00000u
|
||||
#define PCI_IRQ_VECTOR_BASE 0x20u
|
||||
|
||||
static uint32_t MakeAddr(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) {
|
||||
return 0x80000000UL
|
||||
| ((uint32_t)bus << 16)
|
||||
@@ -24,6 +41,25 @@ static void PciWrite32(uint8_t bus, uint8_t dev, uint8_t fn,
|
||||
__outdword(PCI_CONFIG_DATA, val);
|
||||
}
|
||||
|
||||
static uint8_t PciRead8(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) {
|
||||
uint32_t v = PciRead32(bus, dev, fn, reg);
|
||||
return (uint8_t)(v >> ((reg & 3u) * 8u));
|
||||
}
|
||||
|
||||
static uint16_t PciRead16(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) {
|
||||
uint32_t v = PciRead32(bus, dev, fn, reg);
|
||||
return (uint16_t)(v >> ((reg & 2u) * 8u));
|
||||
}
|
||||
|
||||
static void PciWrite16(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg, uint16_t value) {
|
||||
uint8_t aligned = reg & ~3u;
|
||||
uint32_t shift = (reg & 2u) * 8u;
|
||||
uint32_t old = PciRead32(bus, dev, fn, aligned);
|
||||
old &= ~(0xFFFFu << shift);
|
||||
old |= ((uint32_t)value << shift);
|
||||
PciWrite32(bus, dev, fn, aligned, old);
|
||||
}
|
||||
|
||||
static bool PciFindDevice(uint16_t vendor, uint16_t device,
|
||||
uint8_t* outBus, uint8_t* outDev, uint8_t* outFn) {
|
||||
for (int bus = 0; bus < 256; bus++)
|
||||
@@ -61,10 +97,82 @@ static uint8_t PciGetIrqLine(uint8_t bus, uint8_t dev, uint8_t fn) {
|
||||
return (uint8_t)(val & 0xFF); // 0xFF = not connected
|
||||
}
|
||||
|
||||
static uint8_t PciFindCapability(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t capId) {
|
||||
uint32_t status = PciRead32(bus, dev, fn, PCI_REG_COMMAND_STATUS);
|
||||
if ((status & PCI_STATUS_CAP_LIST) == 0)
|
||||
return 0;
|
||||
|
||||
uint8_t cap = PciRead8(bus, dev, fn, PCI_REG_CAP_PTR) & 0xFCu;
|
||||
for (uint32_t guard = 0; cap && guard < 48u; guard++) {
|
||||
uint32_t hdr = PciRead32(bus, dev, fn, cap);
|
||||
uint8_t id = (uint8_t)(hdr & 0xFFu);
|
||||
uint8_t next = (uint8_t)((hdr >> 8) & 0xFCu);
|
||||
if (id == capId)
|
||||
return cap;
|
||||
cap = next;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static KSTATUS PciEnableMsi(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t irqSlot) {
|
||||
if (irqSlot >= 24u)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uint8_t cap = PciFindCapability(bus, dev, fn, PCI_CAP_ID_MSI);
|
||||
if (!cap) {
|
||||
HalDbgPrint("PCI %u:%u.%u: MSI capability not present\n", bus, dev, fn);
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
uint32_t bspLapic = HalGetBspLapicId();
|
||||
if (!HalApicUsingX2Apic() && bspLapic >= 256u)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
if (HalApicUsingX2Apic() && bspLapic >= 256u) {
|
||||
// Extended x2APIC MSI destinations need interrupt-remapping awareness.
|
||||
// Keep the first MSI implementation xAPIC-compatible and explicit.
|
||||
HalDbgPrint("PCI %u:%u.%u: MSI x2APIC destination %u not supported yet\n",
|
||||
bus, dev, fn, bspLapic);
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
uint16_t control = PciRead16(bus, dev, fn, (uint8_t)(cap + 2u));
|
||||
bool is64 = (control & PCI_MSI_CTRL_64BIT) != 0;
|
||||
bool pvm = (control & PCI_MSI_CTRL_PVM) != 0;
|
||||
uint8_t vector = (uint8_t)(PCI_IRQ_VECTOR_BASE + irqSlot);
|
||||
uint32_t address = PCI_MSI_ADDRESS_BASE | ((bspLapic & 0xFFu) << 12);
|
||||
uint16_t data = vector;
|
||||
|
||||
PciWrite32(bus, dev, fn, (uint8_t)(cap + 4u), address);
|
||||
if (is64) {
|
||||
PciWrite32(bus, dev, fn, (uint8_t)(cap + 8u), 0);
|
||||
PciWrite16(bus, dev, fn, (uint8_t)(cap + 12u), data);
|
||||
if (pvm)
|
||||
PciWrite32(bus, dev, fn, (uint8_t)(cap + 16u), 0);
|
||||
} else {
|
||||
PciWrite16(bus, dev, fn, (uint8_t)(cap + 8u), data);
|
||||
if (pvm)
|
||||
PciWrite32(bus, dev, fn, (uint8_t)(cap + 12u), 0);
|
||||
}
|
||||
|
||||
control &= (uint16_t)~PCI_MSI_CTRL_MME_MASK; // enable exactly one message
|
||||
control |= PCI_MSI_CTRL_ENABLE;
|
||||
PciWrite16(bus, dev, fn, (uint8_t)(cap + 2u), control);
|
||||
|
||||
uint32_t cmd = PciRead32(bus, dev, fn, PCI_REG_COMMAND_STATUS);
|
||||
cmd |= PCI_COMMAND_INTX_DISABLE;
|
||||
PciWrite32(bus, dev, fn, PCI_REG_COMMAND_STATUS, cmd);
|
||||
|
||||
HalDbgPrint("PCI %u:%u.%u: MSI enabled cap=0x%02x vector=0x%02x irqslot=%u dest=%u %s\n",
|
||||
bus, dev, fn, cap, vector, irqSlot, bspLapic, is64 ? "64-bit" : "32-bit");
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void HalPciInitialize(HAL_PCI* out) {
|
||||
out->Read32 = PciRead32;
|
||||
out->Write32 = PciWrite32;
|
||||
out->FindDevice = PciFindDevice;
|
||||
out->FindClass = PciFindClass;
|
||||
out->GetIrqLine = PciGetIrqLine;
|
||||
out->EnableMsi = PciEnableMsi;
|
||||
}
|
||||
@@ -7,8 +7,10 @@
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
|
||||
static uint32_t* gFb = (uint32_t *)0;
|
||||
static uint32_t* gBackbuffer = (uint32_t *)0;
|
||||
static int gWidth = 0;
|
||||
static int gHeight = 0;
|
||||
static int gStride = 0; // pixels per row
|
||||
@@ -20,14 +22,39 @@ static int gCursorX = 0, gCursorY = 0;
|
||||
// ---------------------------------------------------------------------------
|
||||
// 8x16 VGA font - printable ASCII only (glyph 0 = space = 0x20)
|
||||
// ---------------------------------------------------------------------------
|
||||
#include <kernel/hal/font8x16.h> // see font8x16.h below
|
||||
#include <kernel/hal/font8x16.h> // see font8x16.h
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internals
|
||||
// ---------------------------------------------------------------------------
|
||||
static inline uint32_t* ScreenBackbufferPixel(int x, int y) {
|
||||
return gBackbuffer ? (gBackbuffer + y * gStride + x) : (gFb + y * gStride + x);
|
||||
}
|
||||
|
||||
static inline void PutPixelRaw(int x, int y, uint32_t color) {
|
||||
if ((unsigned)x < (unsigned)gWidth && (unsigned)y < (unsigned)gHeight)
|
||||
gFb[y * gStride + x] = color;
|
||||
if ((unsigned)x >= (unsigned)gWidth || (unsigned)y >= (unsigned)gHeight)
|
||||
return;
|
||||
|
||||
ScreenBackbufferPixel(x, y)[0] = color;
|
||||
gFb[y * gStride + x] = color;
|
||||
}
|
||||
|
||||
static void ScreenFlushRows(int startY, int rows) {
|
||||
if (!gBackbuffer || rows <= 0)
|
||||
return;
|
||||
|
||||
if (startY < 0) {
|
||||
rows += startY;
|
||||
startY = 0;
|
||||
}
|
||||
if (startY >= gHeight || rows <= 0)
|
||||
return;
|
||||
if (startY + rows > gHeight)
|
||||
rows = gHeight - startY;
|
||||
|
||||
size_t rowBytes = (size_t)gStride * sizeof(uint32_t);
|
||||
memcpy(gFb + startY * gStride, gBackbuffer + startY * gStride,
|
||||
(size_t)rows * rowBytes);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -36,7 +63,14 @@ static inline void PutPixelRaw(int x, int y, uint32_t color) {
|
||||
static void ScreenClear(uint32_t color) {
|
||||
for (int y = 0; y < gHeight; y++)
|
||||
for (int x = 0; x < gWidth; x++)
|
||||
gFb[y * gStride + x] = color;
|
||||
ScreenBackbufferPixel(x, y)[0] = color;
|
||||
ScreenFlushRows(0, gHeight);
|
||||
if (!gBackbuffer) {
|
||||
// ScreenFlushRows is a no-op without a backbuffer.
|
||||
for (int y = 0; y < gHeight; y++)
|
||||
for (int x = 0; x < gWidth; x++)
|
||||
gFb[y * gStride + x] = color;
|
||||
}
|
||||
gCursorX = 0; gCursorY = 0;
|
||||
}
|
||||
|
||||
@@ -66,25 +100,13 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
|
||||
// Calculate bytes to copy per row
|
||||
size_t bytesPerRow = (size_t)drawW * sizeof(uint32_t);
|
||||
|
||||
// If the width we are drawing exactly matches the physical screen stride,
|
||||
// AND it matches the original source image width, memory is perfectly contiguous.
|
||||
if (drawW == gStride && drawW == w) {
|
||||
uint32_t* dst = gFb + (startY * gStride) + startX;
|
||||
const uint32_t* src = pixels + (srcOffsetY * w) + srcOffsetX;
|
||||
|
||||
memcpy(dst, src, bytesPerRow * drawH);
|
||||
return;
|
||||
}
|
||||
|
||||
// Copy row-by-row
|
||||
for (int row = 0; row < drawH; row++) {
|
||||
// Destination in the actual framebuffer (accounting for stride)
|
||||
uint32_t* dst = gFb + ((startY + row) * gStride) + startX;
|
||||
|
||||
// Source from the provided pixel buffer
|
||||
uint32_t* shadow = ScreenBackbufferPixel(startX, startY + row);
|
||||
uint32_t* fb = gFb + ((startY + row) * gStride) + startX;
|
||||
const uint32_t* src = pixels + ((srcOffsetY + row) * w) + srcOffsetX;
|
||||
|
||||
memcpy(dst, src, bytesPerRow);
|
||||
|
||||
memcpy(shadow, src, bytesPerRow);
|
||||
memcpy(fb, src, bytesPerRow);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -92,12 +114,17 @@ static void ScreenGetSize(int* w, int* h) { *w = gWidth; *h = gHeight; }
|
||||
|
||||
static void ScreenScroll(int lines) {
|
||||
if (lines <= 0 || lines >= gHeight) { ScreenClear(0); return; }
|
||||
int rowBytes = gStride * sizeof(uint32_t);
|
||||
memmove(gFb, gFb + lines * gStride,
|
||||
|
||||
size_t rowBytes = (size_t)gStride * sizeof(uint32_t);
|
||||
uint32_t* shadow = gBackbuffer ? gBackbuffer : gFb;
|
||||
|
||||
memmove(shadow, shadow + lines * gStride,
|
||||
(size_t)(gHeight - lines) * rowBytes);
|
||||
// Clear the revealed rows at the bottom.
|
||||
memset(gFb + (gHeight - lines) * gStride, 0,
|
||||
memset(shadow + (gHeight - lines) * gStride, 0,
|
||||
(size_t)lines * rowBytes);
|
||||
|
||||
ScreenFlushRows(0, gHeight);
|
||||
}
|
||||
|
||||
static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
|
||||
@@ -195,7 +222,7 @@ static void CursorSaveBg(int x, int y) {
|
||||
int fx = x + col;
|
||||
uint32_t px = 0;
|
||||
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight)
|
||||
px = gFb[fy * gStride + fx];
|
||||
px = ScreenBackbufferPixel(fx, fy)[0];
|
||||
gCursorSavedBg[row * CURSOR_W + col] = px;
|
||||
}
|
||||
}
|
||||
@@ -207,8 +234,12 @@ static void CursorRestoreBg(int x, int y) {
|
||||
int fy = y + row;
|
||||
for (int col = 0; col < CURSOR_W; col++) {
|
||||
int fx = x + col;
|
||||
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight)
|
||||
gFb[fy * gStride + fx] = gCursorSavedBg[row * CURSOR_W + col];
|
||||
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) {
|
||||
uint32_t px = gBackbuffer
|
||||
? ScreenBackbufferPixel(fx, fy)[0]
|
||||
: gCursorSavedBg[row * CURSOR_W + col];
|
||||
gFb[fy * gStride + fx] = px;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -248,8 +279,8 @@ void HalScreenDrawCursor(int x, int y) {
|
||||
if ((src >> 24) == 0) continue; // fully transparent, skip
|
||||
int fx = x + col;
|
||||
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) {
|
||||
uint32_t* dst = &gFb[fy * gStride + fx];
|
||||
*dst = BlendPixel(*dst, src);
|
||||
uint32_t dst = ScreenBackbufferPixel(fx, fy)[0];
|
||||
gFb[fy * gStride + fx] = BlendPixel(dst, src);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -266,12 +297,21 @@ void HalScreenEraseCursor(void) {
|
||||
// ---------------------------------------------------------------------------
|
||||
// Populate the HAL_SCREEN vtable
|
||||
// ---------------------------------------------------------------------------
|
||||
void HalScreenInitialize(uintptr_t fb, int w, int h, int stride,
|
||||
HAL_SCREEN* out) {
|
||||
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
|
||||
HAL_SCREEN* out) {
|
||||
gFb = (uint32_t *)fb;
|
||||
gWidth = w;
|
||||
gHeight = h;
|
||||
gStride = stride;
|
||||
|
||||
size_t backbufferPixels = (size_t)gStride * (size_t)gHeight;
|
||||
size_t backbufferBytes = backbufferPixels * sizeof(uint32_t);
|
||||
if (gStride > 0 && gHeight > 0 &&
|
||||
backbufferPixels / (size_t)gStride == (size_t)gHeight &&
|
||||
backbufferBytes / sizeof(uint32_t) == backbufferPixels) {
|
||||
gBackbuffer = (uint32_t*)AllocatePool(backbufferBytes);
|
||||
}
|
||||
|
||||
out->Clear = ScreenClear;
|
||||
out->PutPixel = ScreenPutPixel;
|
||||
out->Blit = ScreenBlit;
|
||||
|
||||
@@ -51,7 +51,7 @@ static uint64_t QueryTicks(void) {
|
||||
}
|
||||
|
||||
// Fallback: combine the coarse periodic tick with the current LAPIC
|
||||
// countdown. Read the coarse tick twice so an interrupt between gTicks and
|
||||
// countdown. Read the coarse tick twice so an interrupt between gTicks and
|
||||
// LAPIC_TIMER_CUR cannot make time jump backwards by almost one quantum.
|
||||
uint64_t baseTicks1;
|
||||
uint64_t baseTicks2;
|
||||
@@ -177,7 +177,7 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
|
||||
|
||||
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out) {
|
||||
gLapic = (volatile uint32_t *)gApi.PhysToVirt(lapicPhys);
|
||||
HalDbgPrint("HAL: lapicPhys=0x%llX, lapicVirt=0x%llX, hpetPhys=0x%llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
|
||||
HalDbgPrint("HAL: lapicPhys=0x%016llX, lapicVirt=0x%016llX, hpetPhys=0x%016llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
|
||||
|
||||
uint32_t ticksPerMs = 0;
|
||||
if (hpetPhys) {
|
||||
|
||||
@@ -28,6 +28,8 @@ typedef struct HAL_CPU_TOPOLOGY {
|
||||
|
||||
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out);
|
||||
__declspec(dllexport) uint32_t HalGetBspLapicId(void);
|
||||
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void);
|
||||
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId);
|
||||
__declspec(dllexport) void HalApicEnableCurrentCpu(void);
|
||||
__declspec(dllexport) void HalApicSendPanicIpi(void);
|
||||
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId);
|
||||
|
||||
@@ -14,9 +14,9 @@
|
||||
typedef SYS_DIR_ENTRY KDIR_ENTRY;
|
||||
#define KDIR_NAME_MAX SYS_DIR_NAME_MAX
|
||||
|
||||
// Native asynchronous filesystem completion. Filesystems that provide
|
||||
// Native asynchronous filesystem completion. Filesystems that provide
|
||||
// ReadAsync/WriteAsync accept the operation, return STATUS_PENDING, and later
|
||||
// call Complete(CompleteContext, status, information). Filesystems without
|
||||
// call Complete(CompleteContext, status, information). Filesystems without
|
||||
// these callbacks fall back to the generic VFS async worker.
|
||||
typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
|
||||
KSTATUS status,
|
||||
|
||||
@@ -84,7 +84,7 @@ __declspec(dllexport) void KeEnterCriticalRegion(void);
|
||||
__declspec(dllexport) void KeLeaveCriticalRegion(void);
|
||||
__declspec(dllexport) bool KeAreKernelApcsDisabled(void);
|
||||
|
||||
// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL,
|
||||
// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL,
|
||||
// after context switches, and after waits complete.
|
||||
void KiDeliverKernelApcs(void);
|
||||
|
||||
|
||||
@@ -26,8 +26,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void);
|
||||
// Fault dispatch - called from the IDT fault stubs (ke_syscall.asm).
|
||||
//
|
||||
// The old scalar entry only reported RIP/error/CR2, which is not enough for
|
||||
// debugging null indirect calls. The frame entry snapshots user GPRs plus the
|
||||
// CPU exception frame, including user RSP. For a CALL through a null function
|
||||
// debugging null indirect calls. The frame entry snapshots user GPRs plus the
|
||||
// CPU exception frame, including user RSP. For a CALL through a null function
|
||||
// pointer, RIP is 0 and [RSP] is usually the return address after the bad call;
|
||||
// that is the useful caller site.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#pragma once
|
||||
// ke_dpc.h - deferred procedure calls
|
||||
//
|
||||
// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC
|
||||
// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC
|
||||
// after acknowledging/capturing device state; the DPC then runs later at
|
||||
// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not
|
||||
// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not
|
||||
// wait, block, or call PASSIVE_LEVEL-only code.
|
||||
|
||||
#include <stdint.h>
|
||||
@@ -34,11 +34,11 @@ __declspec(dllexport) void KeInitializeDpc(KDPC* dpc,
|
||||
KDEFERRED_ROUTINE routine,
|
||||
void* context);
|
||||
|
||||
// Queue a DPC if it is not already queued. Returns false if the DPC was
|
||||
// Queue a DPC if it is not already queued. Returns false if the DPC was
|
||||
// already pending or the arguments are invalid.
|
||||
__declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
|
||||
void* systemArgument1,
|
||||
void* systemArgument2);
|
||||
|
||||
// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit.
|
||||
// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit.
|
||||
void KiDrainDpcQueue(void);
|
||||
@@ -0,0 +1,24 @@
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <ntlite/input.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/status.h>
|
||||
|
||||
void KeInputInitialize(const HAL_API* hal);
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId);
|
||||
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event);
|
||||
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event);
|
||||
|
||||
INPUT_KEY KeInputKbPoll(void);
|
||||
bool KeInputIsKeyDown(INPUT_KEY key);
|
||||
INPUT_KEY KeInputReadKey(void);
|
||||
char KeInputReadChar(void);
|
||||
uint32_t KeInputGetModifiers(void);
|
||||
|
||||
bool KeInputMousePoll(void);
|
||||
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons);
|
||||
void KeInputMouseGetPosition(int* x, int* y);
|
||||
void KeInputMouseSetPosition(int x, int y);
|
||||
void KeInputMouseSetBounds(int w, int h);
|
||||
@@ -55,7 +55,7 @@ typedef struct IRP {
|
||||
uint32_t Flags; // IRP_FLAG_*
|
||||
uint64_t Lba;
|
||||
uint32_t SectorCount;
|
||||
void *Buffer; // kernel VA; valid until completion
|
||||
void* Buffer; // kernel VA; valid until completion
|
||||
|
||||
// ---- Completion (caller fills before submit; driver writes Status) ----
|
||||
|
||||
@@ -65,17 +65,17 @@ typedef struct IRP {
|
||||
// Forward-declared pointer: callers that create events include ke_event.h.
|
||||
struct EVENT *CompletionEvent;
|
||||
|
||||
// Optional kernel APC. If set, IoCompleteRequest queues it to its
|
||||
// Optional kernel APC. If set, IoCompleteRequest queues it to its
|
||||
// target thread with arg1=irp and arg2=(void*)(uintptr_t)status.
|
||||
struct KAPC *CompletionApc;
|
||||
|
||||
IRP_COMPLETION_FN CompletionFn;
|
||||
void *CompletionCtx;
|
||||
void* CompletionCtx;
|
||||
|
||||
// ---- Driver-internal --------------------------------------------------
|
||||
|
||||
uint8_t HwSlot;
|
||||
void *DriverCtx;
|
||||
void* DriverCtx;
|
||||
} IRP;
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <ntlite/status.h>
|
||||
|
||||
void KeIrqInitialize(const HAL_API *hal);
|
||||
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq);
|
||||
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void));
|
||||
void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm
|
||||
|
||||
// Low-level interrupt entry/exit hooks. These are called by the common IRQ
|
||||
// assembly stub around KeIrqDispatch. The exit hook is where deferred
|
||||
// Low-level interrupt entry/exit hooks. These are called by the common IRQ
|
||||
// assembly stub around KeIrqDispatch. The exit hook is where deferred
|
||||
// interrupt-time scheduling now happens, so IRQ handlers never switch away
|
||||
// before KeIrqDispatch has returned and sent EOI.
|
||||
void KiEnterInterrupt(void);
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
#pragma once
|
||||
// ke_irql.h - software IRQL skeleton
|
||||
//
|
||||
// This is intentionally small for the first IRQL/APC bring-up step. It does
|
||||
// This is intentionally small for the first IRQL/APC bring-up step. It does
|
||||
// not yet program interrupt hardware or replace the spinlock interrupt-disable
|
||||
// path. It gives the kernel one authoritative execution-level state plus
|
||||
// path. It gives the kernel one authoritative execution-level state plus
|
||||
// assertions so blocking code can reject illegal callers early.
|
||||
|
||||
#include <stdint.h>
|
||||
@@ -18,11 +18,11 @@ typedef uint8_t KIRQL;
|
||||
|
||||
__declspec(dllexport) KIRQL KeGetCurrentIrql(void);
|
||||
|
||||
// Raise to newIrql and return the previous level. Raising to a lower level is
|
||||
// Raise to newIrql and return the previous level. Raising to a lower level is
|
||||
// a kernel bug and will panic in checked/debug-style builds.
|
||||
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql);
|
||||
|
||||
// Restore a previously returned level. Lowering to a higher level is a kernel
|
||||
// Restore a previously returned level. Lowering to a higher level is a kernel
|
||||
// bug and will panic.
|
||||
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql);
|
||||
|
||||
|
||||
@@ -75,6 +75,12 @@ typedef struct PROCESS {
|
||||
uintptr_t UserCwdBase;
|
||||
size_t UserCwdSize;
|
||||
|
||||
/* Per-process user virtual regions. The actual VA arena is global because
|
||||
NT-Lite currently uses one shared address space; this list tracks
|
||||
ownership so leaked user mappings can be reclaimed on process exit. */
|
||||
LIST_ENTRY UserRegions;
|
||||
SPINLOCK UserRegionLock;
|
||||
|
||||
/* Per-process loaded modules (user processes only). */
|
||||
PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
|
||||
uint32_t ModuleCount;
|
||||
|
||||
@@ -37,15 +37,15 @@ KSTATUS KiCreateIdleThreadForCpu(struct KCPU* cpu);
|
||||
void KeReadyThread(THREAD* thread);
|
||||
void KeUnreadyThread(THREAD* thread);
|
||||
|
||||
// Called from the timer ISR every SCHED_TICK_MS. This performs timer
|
||||
// Called from the timer ISR every SCHED_TICK_MS. This performs timer
|
||||
// accounting and requests a reschedule only when the current thread has
|
||||
// exhausted its scheduler budget, or when idle work becomes available. The
|
||||
// exhausted its scheduler budget, or when idle work becomes available. The
|
||||
// actual context switch happens from
|
||||
// KiExitInterrupt after the IRQ handler has returned and EOI has been sent.
|
||||
void KeSchedulerTick(void);
|
||||
|
||||
// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule
|
||||
// is cheap and can be called from interrupt context. KiScheduleFromInterrupt
|
||||
// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule
|
||||
// is cheap and can be called from interrupt context. KiScheduleFromInterrupt
|
||||
// consumes the pending request and performs the context switch, if needed.
|
||||
void KiRequestReschedule(void);
|
||||
int KiIsReschedulePending(void);
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
#include <stdint.h>
|
||||
#include <ntlite/status.h>
|
||||
|
||||
// Provide an optional bootloader-reserved low-memory trampoline page.
|
||||
void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys);
|
||||
|
||||
// Start application processors with hardware INIT/SIPI and park them in
|
||||
// kernel-owned state. APs are not runnable until scheduler admission succeeds.
|
||||
KSTATUS KeStartApplicationProcessors(void);
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
// Never hold a spinlock across a wait/blocking operation.
|
||||
#include <stdint.h>
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_disable)
|
||||
#include <kernel/ke_irql.h>
|
||||
|
||||
typedef struct SPINLOCK {
|
||||
|
||||
@@ -71,7 +71,7 @@ typedef struct THREAD {
|
||||
uintptr_t UserArg;
|
||||
|
||||
// Saved scheduler-lock cookie for this thread's suspended switch
|
||||
// continuation. This is deliberately thread-owned, not CPU-owned:
|
||||
// continuation. This is deliberately thread-owned, not CPU-owned:
|
||||
// after KeSwitchContext() resumes a saved KiSchedule() continuation, the
|
||||
// resumed thread must restore the IF/IRQL state captured when that same
|
||||
// thread switched out.
|
||||
@@ -86,7 +86,7 @@ typedef struct THREAD {
|
||||
void* FpuContext;
|
||||
uint8_t FpuStorage[528];
|
||||
|
||||
// Scheduler accounting. QuantaConsumed is kept in the assembly-stable
|
||||
// Scheduler accounting. QuantaConsumed is kept in the assembly-stable
|
||||
// portion of THREAD for layout compatibility; the dynamic scheduler uses
|
||||
// these fields for per-dispatch budget and CPU-burst tracking.
|
||||
uint32_t QuantumTicks;
|
||||
@@ -105,14 +105,14 @@ typedef struct THREAD {
|
||||
uint64_t AffinityMask;
|
||||
|
||||
// Set by wait primitives before blocking and completed by the object
|
||||
// that wakes the thread. Keep this field after the assembly-visible
|
||||
// that wakes the thread. Keep this field after the assembly-visible
|
||||
// context-switch fields: ke_switch.asm hard-codes offsets through
|
||||
// FpuContext.
|
||||
KSTATUS WaitStatus;
|
||||
LIST_ENTRY* WaitListHead;
|
||||
uint8_t AlertableWait;
|
||||
|
||||
// Kernel APC state. APCs are queued to a target thread and delivered
|
||||
// Kernel APC state. APCs are queued to a target thread and delivered
|
||||
// when that thread reaches PASSIVE_LEVEL with APC delivery enabled.
|
||||
LIST_ENTRY ApcQueue;
|
||||
SPINLOCK ApcLock;
|
||||
@@ -124,7 +124,7 @@ typedef struct THREAD {
|
||||
// The pointed-to frame lives on this thread's kernel stack.
|
||||
KTRAP_FRAME* TrapFrame;
|
||||
|
||||
// User APC state. User APCs are queued by kernel completions, but are
|
||||
// User APC state. User APCs are queued by kernel completions, but are
|
||||
// delivered only on an alertable return to user mode through the registered
|
||||
// user-mode dispatcher stub.
|
||||
LIST_ENTRY UserApcQueue;
|
||||
@@ -162,7 +162,7 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code);
|
||||
//
|
||||
// The split form prevents the classic lost wakeup where an ISR/DPC signals
|
||||
// the object after the waiter is queued but before the waiter marks itself
|
||||
// waiting. Alertable waits are explicit at every call site.
|
||||
// waiting. Alertable waits are explicit at every call site.
|
||||
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
|
||||
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead);
|
||||
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
|
||||
|
||||
@@ -32,7 +32,7 @@ typedef struct SYS_TEB {
|
||||
uint32_t ThreadId;
|
||||
uint32_t Magic;
|
||||
|
||||
// Native per-thread extension pointer. The kernel leaves this NULL;
|
||||
// Native per-thread extension pointer. The kernel leaves this NULL;
|
||||
// user-mode personalities/libraries may hang their own thread-local
|
||||
// state here without baking subsystem semantics into SYS_TEB.
|
||||
void* SubsystemData;
|
||||
|
||||
@@ -6,12 +6,16 @@
|
||||
#include <stdbool.h>
|
||||
#include <ntlite/status.h>
|
||||
|
||||
struct PROCESS;
|
||||
|
||||
// Map a PE32+ image from 'buffer' into the kernel virtual address space.
|
||||
// On success, *baseOut receives the virtual base address.
|
||||
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user);
|
||||
KSTATUS LdrLoadImageForProcess(struct PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut);
|
||||
|
||||
// Return the SizeOfImage recorded in the PE header for an already-mapped image.
|
||||
size_t LdrGetImageSize(uintptr_t base);
|
||||
size_t LdrGetMappedImageSize(struct PROCESS* process, uintptr_t base);
|
||||
|
||||
// Look up a named export in a loaded module.
|
||||
// 'moduleName' must match a name in BOOT_INFO.Modules (e.g., "hal.dll").
|
||||
@@ -24,11 +28,11 @@ uintptr_t LdrGetEntryPoint(uintptr_t base);
|
||||
KSTATUS LdrResolveImports(uintptr_t imageBase);
|
||||
|
||||
// Resolve imports using a process-local module registry.
|
||||
struct PROCESS;
|
||||
KSTATUS LdrResolveImportsForProcess(struct PROCESS* process, uintptr_t imageBase);
|
||||
|
||||
// Apply final page permissions for a loaded image after relocations/imports.
|
||||
KSTATUS LdrProtectImage(uintptr_t imageBase, bool user);
|
||||
KSTATUS LdrProtectImageForProcess(struct PROCESS* process, uintptr_t imageBase, bool user);
|
||||
|
||||
// Load a DLL for a specific process, resolve it against that process's
|
||||
// loaded modules, and register it in the process-local registry.
|
||||
|
||||
@@ -17,8 +17,10 @@ bool MmInitializePhysical(const MEMORY_MAP* map,
|
||||
|
||||
// Allocate / free a single 4 KB frame.
|
||||
// MmAllocateFrame returns 0 on failure.
|
||||
PHYSICAL_ADDRESS MmAllocateFrame(void);
|
||||
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void);
|
||||
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive);
|
||||
PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
|
||||
PHYSICAL_ADDRESS maxExclusive);
|
||||
void MmFreeFrame(PHYSICAL_ADDRESS frame);
|
||||
|
||||
// Allocate 'count' physically contiguous 4 KB frames.
|
||||
|
||||
@@ -6,6 +6,25 @@
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <ntlite/status.h>
|
||||
|
||||
struct PROCESS;
|
||||
typedef struct PROCESS PROCESS;
|
||||
|
||||
typedef enum USER_REGION_TYPE {
|
||||
UserRegionGeneric = 0,
|
||||
UserRegionHeap,
|
||||
UserRegionImage,
|
||||
UserRegionStack,
|
||||
UserRegionTeb,
|
||||
UserRegionPeb,
|
||||
UserRegionEnvironment,
|
||||
UserRegionMappedFile,
|
||||
UserRegionShared,
|
||||
} USER_REGION_TYPE;
|
||||
|
||||
#define USER_REGION_GUARD_LOW 0x00000001u
|
||||
#define USER_REGION_GUARD_HIGH 0x00000002u
|
||||
#define USER_REGION_ZERO 0x00000004u
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Page flags (x86-64 PTE bits)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -32,10 +51,10 @@
|
||||
// Called by KernelMain after MmInitializePhysical.
|
||||
// bootPml4Phys: the PML4 set up by the bootloader.
|
||||
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
|
||||
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
|
||||
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
|
||||
|
||||
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
|
||||
void MmActivate(void);
|
||||
void MmActivate(void);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Page mapping
|
||||
@@ -89,16 +108,37 @@ __declspec(dllexport) void MmFreePool(void* ptr);
|
||||
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// User heap - shared flat space, ring 3 accessible.
|
||||
// Allocations are writable user pages with NX set by default; PE image
|
||||
// sections opt back into execution through the loader.
|
||||
// User virtual regions.
|
||||
// The kernel has one shared user VA arena, but every user mapping is owned by
|
||||
// a PROCESS so process teardown can reclaim leaked memory. Guard pages are
|
||||
// reserved VA with no present PTE.
|
||||
// ---------------------------------------------------------------------------
|
||||
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
|
||||
uint64_t pageFlags, uint32_t regionFlags,
|
||||
USER_REGION_TYPE type, void** outBase);
|
||||
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
|
||||
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
|
||||
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base);
|
||||
void MmDestroyAllUserRegions(PROCESS* process);
|
||||
size_t MmGetUserRegionSize(PROCESS* process, const void* base);
|
||||
size_t MmGetUserRegionSizeByBase(const void* base);
|
||||
|
||||
// Debug-only ownership/address diagnostics used by fault and launch paths.
|
||||
void MmDebugDescribeAddress(uintptr_t addr);
|
||||
void MmDebugDumpUserRegions(PROCESS* process);
|
||||
|
||||
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size);
|
||||
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr);
|
||||
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize);
|
||||
|
||||
// Convenience wrappers for syscalls/current-process allocations.
|
||||
__declspec(dllexport) void* MmAllocateUserMemory(size_t size);
|
||||
__declspec(dllexport) void MmFreeUserMemory(void* ptr);
|
||||
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize);
|
||||
|
||||
// Page-aligned user image mapping. PE section protection is page based, so
|
||||
// executable images must not be allocated from the normal sub-page heap.
|
||||
// Page-aligned user image mapping. PE section protection is page based.
|
||||
__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size);
|
||||
__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size);
|
||||
__declspec(dllexport) void* MmAllocateUserImage(size_t size);
|
||||
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
#define SYS_KB_POLL 64
|
||||
#define SYS_KB_IS_KEY_DOWN 65
|
||||
#define SYS_KB_READ_CHAR 66
|
||||
#define SYS_KB_READ_SCANCODE 67
|
||||
#define SYS_KB_READ_KEY 67
|
||||
#define SYS_KB_GET_MODIFIERS 68
|
||||
#define SYS_MOUSE_POLL 69
|
||||
#define SYS_MOUSE_READ 70
|
||||
|
||||
@@ -3,10 +3,137 @@
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
|
||||
|
||||
#define MAX_PUBLISHED_DEVICES 64
|
||||
#define MAX_DRIVER_MATCHERS 32
|
||||
|
||||
typedef struct IO_DRIVER_MATCHER_ENTRY {
|
||||
bool Used;
|
||||
const char* DriverName;
|
||||
IO_BUS_TYPE BusType;
|
||||
IO_DRIVER_MATCH_CALLBACK Match;
|
||||
IO_DRIVER_BIND_CALLBACK Bind;
|
||||
void* Context;
|
||||
} IO_DRIVER_MATCHER_ENTRY;
|
||||
|
||||
static IO_PUBLISHED_DEVICE gPublishedDevices[MAX_PUBLISHED_DEVICES];
|
||||
static uint32_t gPublishedDeviceCount;
|
||||
static IO_DRIVER_MATCHER_ENTRY gDriverMatchers[MAX_DRIVER_MATCHERS];
|
||||
static uint32_t gDriverMatcherCount;
|
||||
|
||||
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
|
||||
switch (busType) {
|
||||
case IoBusPci: return "pci";
|
||||
case IoBusUsb: return "usb";
|
||||
default: return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
static bool IoTryBindDevice(IO_PUBLISHED_DEVICE* device,
|
||||
IO_DRIVER_MATCHER_ENTRY* matcher) {
|
||||
if (!device || !matcher || !matcher->Used || device->Bound)
|
||||
return false;
|
||||
if (matcher->BusType != IoBusUnknown && matcher->BusType != device->BusType)
|
||||
return false;
|
||||
if (!matcher->Match || !matcher->Bind)
|
||||
return false;
|
||||
if (!matcher->Match(device, matcher->Context))
|
||||
return false;
|
||||
|
||||
KSTATUS st = matcher->Bind(device, matcher->Context);
|
||||
if (KSUCCESS(st)) {
|
||||
device->Bound = true;
|
||||
device->BoundDriverName = matcher->DriverName;
|
||||
DbgPrint("io: %s device %s bound by %s\n",
|
||||
IoBusTypeName(device->BusType),
|
||||
device->Name ? device->Name : "<unnamed>",
|
||||
matcher->DriverName ? matcher->DriverName : "<unnamed>");
|
||||
return true;
|
||||
}
|
||||
|
||||
DbgPrint("io: %s matcher %s accepted %s but bind failed st=0x%x\n",
|
||||
IoBusTypeName(device->BusType),
|
||||
matcher->DriverName ? matcher->DriverName : "<unnamed>",
|
||||
device->Name ? device->Name : "<unnamed>",
|
||||
st);
|
||||
return false;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
|
||||
IO_BUS_TYPE busType,
|
||||
IO_DRIVER_MATCH_CALLBACK match,
|
||||
IO_DRIVER_BIND_CALLBACK bind,
|
||||
void* context) {
|
||||
if (!driverName || !match || !bind)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (gDriverMatcherCount >= MAX_DRIVER_MATCHERS)
|
||||
return STATUS_NO_MEMORY;
|
||||
|
||||
IO_DRIVER_MATCHER_ENTRY* entry = &gDriverMatchers[gDriverMatcherCount++];
|
||||
memset(entry, 0, sizeof(*entry));
|
||||
entry->Used = true;
|
||||
entry->DriverName = driverName;
|
||||
entry->BusType = busType;
|
||||
entry->Match = match;
|
||||
entry->Bind = bind;
|
||||
entry->Context = context;
|
||||
|
||||
DbgPrint("io: registered %s matcher %s\n",
|
||||
IoBusTypeName(busType), driverName);
|
||||
|
||||
for (uint32_t i = 0; i < gPublishedDeviceCount; i++) {
|
||||
IoTryBindDevice(&gPublishedDevices[i], entry);
|
||||
}
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device) {
|
||||
if (!device || device->BusType == IoBusUnknown)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (gPublishedDeviceCount >= MAX_PUBLISHED_DEVICES)
|
||||
return STATUS_NO_MEMORY;
|
||||
|
||||
IO_PUBLISHED_DEVICE* stored = &gPublishedDevices[gPublishedDeviceCount++];
|
||||
memcpy(stored, device, sizeof(*stored));
|
||||
stored->Bound = false;
|
||||
stored->BoundDriverName = (const char*)0;
|
||||
|
||||
|
||||
for (uint32_t i = 0; i < gDriverMatcherCount; i++) {
|
||||
if (IoTryBindDevice(stored, &gDriverMatchers[i]))
|
||||
break;
|
||||
}
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) uint32_t IoPublishedDeviceCount(void) {
|
||||
return gPublishedDeviceCount;
|
||||
}
|
||||
|
||||
#define MAX_DISK_DEVICES 8
|
||||
#define MAX_DISK_ARRIVAL_CALLBACKS 16
|
||||
|
||||
typedef struct IO_DISK_CALLBACK_ENTRY {
|
||||
bool Used;
|
||||
IO_DISK_ARRIVAL_CALLBACK Callback;
|
||||
void* Context;
|
||||
} IO_DISK_CALLBACK_ENTRY;
|
||||
|
||||
static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES];
|
||||
static IO_DISK_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_ARRIVAL_CALLBACKS];
|
||||
static uint32_t gDiskArrivalCallbackCount;
|
||||
|
||||
static void IoNotifyDiskArrived(uint8_t driveIndex, DEVICE_OBJECT* device) {
|
||||
for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
|
||||
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
|
||||
if (entry->Used && entry->Callback)
|
||||
entry->Callback(driveIndex, device, entry->Context);
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
|
||||
size_t extensionSize,
|
||||
@@ -43,6 +170,41 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
|
||||
device->Flags |= DEVICE_OBJECT_DISK;
|
||||
device->DiskIndex = driveIndex;
|
||||
gDiskDevices[driveIndex] = device;
|
||||
IoNotifyDiskArrived(driveIndex, device);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback,
|
||||
void* context) {
|
||||
if (!callback)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
if (gDiskArrivalCallbackCount >= MAX_DISK_ARRIVAL_CALLBACKS)
|
||||
return STATUS_NO_MEMORY;
|
||||
|
||||
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
|
||||
memset(entry, 0, sizeof(*entry));
|
||||
entry->Used = true;
|
||||
entry->Callback = callback;
|
||||
entry->Context = context;
|
||||
|
||||
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
|
||||
if (gDiskDevices[i])
|
||||
callback(i, gDiskDevices[i], context);
|
||||
}
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback,
|
||||
void* context) {
|
||||
if (!callback)
|
||||
return STATUS_INVALID_PARAMETER;
|
||||
|
||||
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
|
||||
if (gDiskDevices[i])
|
||||
callback(i, gDiskDevices[i], context);
|
||||
}
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
@@ -150,7 +150,7 @@ static void KiIoAsyncFileWorker(void* arg) {
|
||||
}
|
||||
|
||||
if (req->CancelRequested) {
|
||||
// The operation may already have reached the filesystem. The
|
||||
// The operation may already have reached the filesystem. The
|
||||
// completion contract still reports cancellation so waiters/APCs do
|
||||
// not mistake a cancelled request for useful data.
|
||||
s = STATUS_CANCELLED;
|
||||
@@ -628,7 +628,7 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Explicit-offset file I/O. Async I/O is built on these routines so it does
|
||||
// Explicit-offset file I/O. Async I/O is built on these routines so it does
|
||||
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
|
||||
// ---------------------------------------------------------------------------
|
||||
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
|
||||
|
||||
@@ -74,13 +74,13 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
|
||||
KeReleaseSpinlock(&thread->ApcLock, flags);
|
||||
|
||||
// If the target opted into an alertable wait, make the APC itself the
|
||||
// wait completion. The target thread will deliver the APC before
|
||||
// wait completion. The target thread will deliver the APC before
|
||||
// KeWaitForEvent(..., true) returns STATUS_KERNEL_APC.
|
||||
if (wakeAlertable)
|
||||
KiWakeThreadForKernelApc(thread);
|
||||
|
||||
// If the caller queued an APC to itself at PASSIVE_LEVEL, try to deliver
|
||||
// immediately. Cross-thread APCs run the next time the target thread
|
||||
// immediately. Cross-thread APCs run the next time the target thread
|
||||
// returns to PASSIVE_LEVEL.
|
||||
if (thread == KeGetCurrentThread())
|
||||
KiDeliverKernelApcs();
|
||||
@@ -165,7 +165,7 @@ void KiDeliverKernelApcs(void) {
|
||||
&arg2);
|
||||
|
||||
// After the kernel routine has had a chance to adjust the callout,
|
||||
// stop touching APC storage before invoking the normal routine. The
|
||||
// stop touching APC storage before invoking the normal routine. The
|
||||
// normal routine is allowed to free the APC's containing allocation.
|
||||
apc->SystemArgument1 = (void*)0;
|
||||
apc->SystemArgument2 = (void*)0;
|
||||
|
||||
@@ -26,7 +26,7 @@ void KeLoadKernelGsBase(KCPU* cpu) {
|
||||
}
|
||||
|
||||
void KeSetUserGsBase(uintptr_t userGsBase) {
|
||||
// While running in the kernel, GS_BASE is the current KCPU. The user TEB
|
||||
// While running in the kernel, GS_BASE is the current KCPU. The user TEB
|
||||
// base is parked in IA32_KERNEL_GS_BASE so swapgs can exchange it with the
|
||||
// KCPU pointer on return to ring 3.
|
||||
__writemsr(IA32_KERNEL_GS_BASE, (uint64_t)userGsBase);
|
||||
|
||||
+30
-11
@@ -5,6 +5,7 @@
|
||||
#include <ntlite/ddi/hal_api.h>
|
||||
#include <kernel/hal/hal_apic.h>
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_disable)
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_cpu.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
@@ -31,8 +32,8 @@ static int KiPanicIsActive(void) {
|
||||
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
|
||||
}
|
||||
|
||||
// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
|
||||
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
|
||||
// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
|
||||
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
|
||||
// unexpected NMI is not handled here and will fall through to the normal fault
|
||||
// path in the assembly stub.
|
||||
__declspec(dllexport) int KeHandleNmiIpi(void) {
|
||||
@@ -72,13 +73,13 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
|
||||
// Silence this CPU first.
|
||||
_disable();
|
||||
|
||||
// Only one CPU owns the visible panic report. CPUs that fault while the
|
||||
// Only one CPU owns the visible panic report. CPUs that fault while the
|
||||
// system is already panicking immediately join the stopped set.
|
||||
if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) != 0) {
|
||||
KiPanicSecondaryHalt();
|
||||
}
|
||||
|
||||
// Stop every other CPU as early as possible. This uses an NMI IPI so it
|
||||
// Stop every other CPU as early as possible. This uses an NMI IPI so it
|
||||
// still reaches CPUs with IF cleared inside spinlock/high-IRQL regions.
|
||||
HalApicSendPanicIpi();
|
||||
|
||||
@@ -98,7 +99,7 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
|
||||
(unsigned)cpu->CpuIndex,
|
||||
(unsigned)cpu->LapicId);
|
||||
if (thread)
|
||||
printf(" thread=%p tid=%llu",
|
||||
printf(" thread=0x%016llX tid=%llu",
|
||||
thread,
|
||||
(unsigned long long)thread->ThreadId);
|
||||
printf("\n");
|
||||
@@ -192,7 +193,7 @@ static int KiFaultHasErrorCode(int vector) {
|
||||
|
||||
// Walk the process module list to find which image owns an address.
|
||||
// Returns the module name string, or "unknown" if the address is outside all
|
||||
// known image ranges. Base/size are optional and are useful for offset dumps.
|
||||
// known image ranges. Base/size are optional and are useful for offset dumps.
|
||||
static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
|
||||
uintptr_t* baseOut, size_t* sizeOut) {
|
||||
if (baseOut) *baseOut = 0;
|
||||
@@ -230,7 +231,7 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
|
||||
const char* mod = KiResolveModuleEx(proc, addr, &base, &size);
|
||||
|
||||
if (base) {
|
||||
printf("%s%016llx <%s+0x%llx>\n",
|
||||
printf("%s%016llx <%s+0x%016llX>\n",
|
||||
label,
|
||||
(unsigned long long)addr,
|
||||
mod,
|
||||
@@ -374,7 +375,7 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
|
||||
size_t size = 0;
|
||||
const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size);
|
||||
if (base) {
|
||||
printf(" [%02d] %016llx : %016llx <%s+0x%llx>\n",
|
||||
printf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n",
|
||||
i,
|
||||
(unsigned long long)slot,
|
||||
(unsigned long long)value,
|
||||
@@ -486,7 +487,7 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
|
||||
}
|
||||
|
||||
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
|
||||
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
|
||||
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
|
||||
// the interrupted context on a secondary CPU once another CPU owns panic.
|
||||
if (KiPanicIsActive())
|
||||
KiPanicSecondaryHalt();
|
||||
@@ -503,10 +504,28 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
|
||||
int vector = (int)frame->Vector;
|
||||
const char* name = KiFaultVectorName(vector);
|
||||
|
||||
DbgPrint("FAULT: kernel exception vector=%d name=%s rip=%016llx rsp=%016llx err=%04llx cr2=%016llx thread=%p pid=%u proc=%s\n",
|
||||
vector,
|
||||
name,
|
||||
(unsigned long long)frame->Rip,
|
||||
(unsigned long long)frame->Rsp,
|
||||
(unsigned long long)frame->ErrorCode,
|
||||
(unsigned long long)frame->Cr2,
|
||||
thread,
|
||||
proc ? proc->ProcessId : 0,
|
||||
proc ? proc->Name : "<none>");
|
||||
KiDumpFaultFrame(proc, frame);
|
||||
if (vector == 14) {
|
||||
KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
|
||||
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
|
||||
if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
|
||||
MmDebugDumpUserRegions(proc);
|
||||
}
|
||||
|
||||
if (vector == 14) {
|
||||
KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | RSP: %016llx | Code: %04llx | CR2: %016llx\n",
|
||||
name,
|
||||
(unsigned long long)frame->Rip,
|
||||
(unsigned long long)frame->Rsp,
|
||||
(unsigned long long)frame->ErrorCode,
|
||||
(unsigned long long)frame->Cr2);
|
||||
} else if (KiFaultHasErrorCode(vector)) {
|
||||
@@ -547,7 +566,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
|
||||
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
|
||||
//
|
||||
// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and
|
||||
// call here. We check the CS RPL to decide: kernel fault → panic and halt,
|
||||
// call here. We check the CS RPL to decide: kernel fault → panic and halt,
|
||||
// user fault → KeBugCheckProcess (kill process, resume scheduler).
|
||||
// ---------------------------------------------------------------------------
|
||||
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
|
||||
|
||||
@@ -46,7 +46,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
|
||||
|
||||
KiEnsureDpcInitialized();
|
||||
|
||||
// Coalesce repeated inserts of the same KDPC. The first insert owns the
|
||||
// Coalesce repeated inserts of the same KDPC. The first insert owns the
|
||||
// argument pair until the DPC is popped by the dispatcher.
|
||||
if (_InterlockedCompareExchange(&dpc->Inserted, 1, 0) != 0)
|
||||
return false;
|
||||
|
||||
@@ -38,7 +38,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
|
||||
} else {
|
||||
if (!LIST_IS_EMPTY(&event->WaitList)) {
|
||||
// Auto-reset handoff: transfer the signal directly to exactly one
|
||||
// waiter. Do not leave Signaled=1, because a later waiter can steal
|
||||
// waiter. Do not leave Signaled=1, because a later waiter can steal
|
||||
// the disk/APC completion before the selected thread resumes.
|
||||
event->Signaled = 0;
|
||||
LIST_ENTRY* node = ListPopHead(&event->WaitList);
|
||||
|
||||
@@ -140,7 +140,7 @@ void KeLoadGdtForCurrentCpu(KCPU* cpu) {
|
||||
KeFlushGdt(&cpu->Gdtr, 0x08, 0x10, 0x28);
|
||||
|
||||
// Reloading GS as a segment register during GDT reload can disturb the
|
||||
// hidden GS base. Reinstall the kernel per-CPU base immediately after
|
||||
// hidden GS base. Reinstall the kernel per-CPU base immediately after
|
||||
// the descriptor reload so KeGetCurrentCpu() remains GS-backed.
|
||||
KeLoadKernelGsBase(cpu);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,352 @@
|
||||
// ke_input.c - kernel-owned canonical input core
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <kernel/ke_input.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
|
||||
#define KEY_QUEUE_SIZE 256u
|
||||
#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u)
|
||||
#define MOUSE_QUEUE_SIZE 128u
|
||||
#define MOUSE_QUEUE_MASK (MOUSE_QUEUE_SIZE - 1u)
|
||||
#define MAX_INPUT_KEYS 256u
|
||||
#define MAX_INPUT_DEVICES 16u
|
||||
#define KEY_REPEAT_DELAY_US 500000ull
|
||||
#define KEY_REPEAT_PERIOD_US 33000ull
|
||||
|
||||
typedef struct KEY_CHAR_PAIR {
|
||||
INPUT_KEY key;
|
||||
char normal;
|
||||
char shifted;
|
||||
} KEY_CHAR_PAIR;
|
||||
|
||||
static const HAL_API* gInputHal;
|
||||
|
||||
static volatile uint8_t gKeyDown[MAX_INPUT_KEYS];
|
||||
static volatile uint32_t gModifiers;
|
||||
|
||||
typedef struct INPUT_KEY_QUEUE_ENTRY {
|
||||
INPUT_KEY key;
|
||||
char ch;
|
||||
} INPUT_KEY_QUEUE_ENTRY;
|
||||
|
||||
static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE];
|
||||
static volatile int gKeyHead;
|
||||
static volatile int gKeyTail;
|
||||
static volatile INPUT_KEY gRepeatKey;
|
||||
static volatile uint64_t gRepeatNextTick;
|
||||
|
||||
static INPUT_MOUSE_EVENT gMouseQueue[MOUSE_QUEUE_SIZE];
|
||||
static volatile int gMouseHead;
|
||||
static volatile int gMouseTail;
|
||||
static volatile int gMouseX;
|
||||
static volatile int gMouseY;
|
||||
static volatile int gMouseBoundsW = 1920;
|
||||
static volatile int gMouseBoundsH = 1080;
|
||||
static volatile uint8_t gMouseButtons;
|
||||
|
||||
static volatile uint32_t gNextInputDeviceId = 1;
|
||||
|
||||
static char KiInputKeyToChar(INPUT_KEY key);
|
||||
|
||||
static const KEY_CHAR_PAIR gCharMap[] = {
|
||||
{ INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' },
|
||||
{ INPUT_KEY_3, '3', '#' }, { INPUT_KEY_4, '4', '$' },
|
||||
{ INPUT_KEY_5, '5', '%' }, { INPUT_KEY_6, '6', '^' },
|
||||
{ INPUT_KEY_7, '7', '&' }, { INPUT_KEY_8, '8', '*' },
|
||||
{ INPUT_KEY_9, '9', '(' }, { INPUT_KEY_0, '0', ')' },
|
||||
{ INPUT_KEY_MINUS, '-', '_' }, { INPUT_KEY_EQUAL, '=', '+' },
|
||||
{ INPUT_KEY_LEFT_BRACKET, '[', '{' }, { INPUT_KEY_RIGHT_BRACKET, ']', '}' },
|
||||
{ INPUT_KEY_BACKSLASH, '\\', '|' }, { INPUT_KEY_SEMICOLON, ';', ':' },
|
||||
{ INPUT_KEY_APOSTROPHE, '\'', '"' }, { INPUT_KEY_GRAVE, '`', '~' },
|
||||
{ INPUT_KEY_COMMA, ',', '<' }, { INPUT_KEY_PERIOD, '.', '>' },
|
||||
{ INPUT_KEY_SLASH, '/', '?' },
|
||||
{ INPUT_KEY_SPACE, ' ', ' ' }, { INPUT_KEY_TAB, '\t', '\t' },
|
||||
{ INPUT_KEY_ENTER, '\n', '\n' }, { INPUT_KEY_BACKSPACE, '\b', '\b' },
|
||||
{ INPUT_KEY_KP_DIVIDE, '/', '/' }, { INPUT_KEY_KP_MULTIPLY, '*', '*' },
|
||||
{ INPUT_KEY_KP_MINUS, '-', '-' }, { INPUT_KEY_KP_PLUS, '+', '+' },
|
||||
{ INPUT_KEY_KP_ENTER, '\n', '\n' }, { INPUT_KEY_KP_1, '1', '1' },
|
||||
{ INPUT_KEY_KP_2, '2', '2' }, { INPUT_KEY_KP_3, '3', '3' },
|
||||
{ INPUT_KEY_KP_4, '4', '4' }, { INPUT_KEY_KP_5, '5', '5' },
|
||||
{ INPUT_KEY_KP_6, '6', '6' }, { INPUT_KEY_KP_7, '7', '7' },
|
||||
{ INPUT_KEY_KP_8, '8', '8' }, { INPUT_KEY_KP_9, '9', '9' },
|
||||
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
|
||||
};
|
||||
|
||||
static void KiInputQueueKey(INPUT_KEY key, char ch) {
|
||||
if (key == INPUT_KEY_NONE) return;
|
||||
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
|
||||
if (next == gKeyHead) return;
|
||||
gKeyQueue[gKeyTail].key = key;
|
||||
gKeyQueue[gKeyTail].ch = ch;
|
||||
gKeyTail = next;
|
||||
}
|
||||
|
||||
static bool KiInputQueueHasSpace(void) {
|
||||
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
|
||||
return next != gKeyHead;
|
||||
}
|
||||
|
||||
static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
|
||||
if (key == INPUT_KEY_NONE) return false;
|
||||
|
||||
switch (key) {
|
||||
case INPUT_KEY_LEFT_SHIFT:
|
||||
case INPUT_KEY_RIGHT_SHIFT:
|
||||
case INPUT_KEY_LEFT_CTRL:
|
||||
case INPUT_KEY_RIGHT_CTRL:
|
||||
case INPUT_KEY_LEFT_ALT:
|
||||
case INPUT_KEY_RIGHT_ALT:
|
||||
case INPUT_KEY_LEFT_GUI:
|
||||
case INPUT_KEY_RIGHT_GUI:
|
||||
case INPUT_KEY_CAPS_LOCK:
|
||||
case INPUT_KEY_NUM_LOCK:
|
||||
return false;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
static uint64_t KiInputNowTicks(void) {
|
||||
if (gInputHal && gInputHal->Timer.QueryTicks)
|
||||
return gInputHal->Timer.QueryTicks();
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void KiInputStartRepeat(INPUT_KEY key, uint64_t now) {
|
||||
if (!KiInputKeyIsRepeatable(key)) return;
|
||||
gRepeatKey = key;
|
||||
gRepeatNextTick = now + KEY_REPEAT_DELAY_US;
|
||||
}
|
||||
|
||||
static void KiInputStopRepeat(INPUT_KEY key) {
|
||||
if (gRepeatKey == key) {
|
||||
gRepeatKey = INPUT_KEY_NONE;
|
||||
gRepeatNextTick = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void KiInputServiceRepeat(void) {
|
||||
INPUT_KEY key = gRepeatKey;
|
||||
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key]) {
|
||||
gRepeatKey = INPUT_KEY_NONE;
|
||||
gRepeatNextTick = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
uint64_t now = KiInputNowTicks();
|
||||
if (now == 0 || now < gRepeatNextTick)
|
||||
return;
|
||||
|
||||
if (!KiInputQueueHasSpace()) {
|
||||
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
|
||||
return;
|
||||
}
|
||||
|
||||
KiInputQueueKey(key, KiInputKeyToChar(key));
|
||||
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
|
||||
}
|
||||
|
||||
static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
|
||||
uint32_t bit = 0;
|
||||
switch (key) {
|
||||
case INPUT_KEY_LEFT_SHIFT: bit = INPUT_MOD_LSHIFT; break;
|
||||
case INPUT_KEY_RIGHT_SHIFT: bit = INPUT_MOD_RSHIFT; break;
|
||||
case INPUT_KEY_LEFT_CTRL: bit = INPUT_MOD_LCTRL; break;
|
||||
case INPUT_KEY_RIGHT_CTRL: bit = INPUT_MOD_RCTRL; break;
|
||||
case INPUT_KEY_LEFT_ALT: bit = INPUT_MOD_LALT; break;
|
||||
case INPUT_KEY_RIGHT_ALT: bit = INPUT_MOD_RALT; break;
|
||||
case INPUT_KEY_LEFT_GUI: bit = INPUT_MOD_LGUI; break;
|
||||
case INPUT_KEY_RIGHT_GUI: bit = INPUT_MOD_RGUI; break;
|
||||
default: break;
|
||||
}
|
||||
|
||||
if (bit) {
|
||||
if (pressed) gModifiers |= bit;
|
||||
else gModifiers &= ~bit;
|
||||
}
|
||||
|
||||
if (pressed && key == INPUT_KEY_CAPS_LOCK) gModifiers ^= INPUT_MOD_CAPS_LOCK;
|
||||
if (pressed && key == INPUT_KEY_NUM_LOCK) gModifiers ^= INPUT_MOD_NUM_LOCK;
|
||||
}
|
||||
|
||||
static char KiInputKeyToChar(INPUT_KEY key) {
|
||||
if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) {
|
||||
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
|
||||
bool caps = (gModifiers & INPUT_MOD_CAPS_LOCK) != 0;
|
||||
char c = (char)('a' + (key - INPUT_KEY_A));
|
||||
if (shifted ^ caps) c = (char)(c - 'a' + 'A');
|
||||
return c;
|
||||
}
|
||||
|
||||
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
|
||||
for (size_t i = 0; i < sizeof(gCharMap) / sizeof(gCharMap[0]); i++) {
|
||||
if (gCharMap[i].key == key)
|
||||
return shifted ? gCharMap[i].shifted : gCharMap[i].normal;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void KeInputInitialize(const HAL_API* hal) {
|
||||
gInputHal = hal;
|
||||
memset((void*)gKeyDown, 0, sizeof(gKeyDown));
|
||||
gModifiers = 0;
|
||||
gKeyHead = gKeyTail = 0;
|
||||
gRepeatKey = INPUT_KEY_NONE;
|
||||
gRepeatNextTick = 0;
|
||||
gMouseHead = gMouseTail = 0;
|
||||
gMouseButtons = 0;
|
||||
|
||||
int w = 1920, h = 1080;
|
||||
if (hal && hal->Screen.GetSize) hal->Screen.GetSize(&w, &h);
|
||||
KeInputMouseSetBounds(w, h);
|
||||
KeInputMouseSetPosition(w / 2, h / 2);
|
||||
|
||||
if (hal && hal->Screen.DrawCursor)
|
||||
hal->Screen.DrawCursor(gMouseX, gMouseY);
|
||||
|
||||
DbgPrint("Input: kernel input core initialized (%dx%d)\n", w, h);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) {
|
||||
(void)flags;
|
||||
uint32_t id = gNextInputDeviceId++;
|
||||
if (id >= MAX_INPUT_DEVICES) return STATUS_NO_MEMORY;
|
||||
if (outDeviceId) *outDeviceId = id;
|
||||
DbgPrint("Input: registered device %u (%s)\n", id, name ? name : "unnamed");
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
|
||||
if (!event) return;
|
||||
INPUT_KEY key = event->Key;
|
||||
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return;
|
||||
|
||||
bool wasDown = gKeyDown[key] != 0;
|
||||
bool pressed = event->Pressed;
|
||||
|
||||
if (pressed) {
|
||||
// The input core owns typematic repeat. Treat repeated hardware make
|
||||
// events as idempotent so PS/2 typematic and USB HID share one policy.
|
||||
if (wasDown)
|
||||
return;
|
||||
|
||||
gKeyDown[key] = 1u;
|
||||
KiInputSetModifier(key, true);
|
||||
KiInputQueueKey(key, KiInputKeyToChar(key));
|
||||
KiInputStartRepeat(key, KiInputNowTicks());
|
||||
} else {
|
||||
if (!wasDown)
|
||||
return;
|
||||
|
||||
gKeyDown[key] = 0u;
|
||||
KiInputSetModifier(key, false);
|
||||
KiInputStopRepeat(key);
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
|
||||
if (!event) return;
|
||||
|
||||
int nx = gMouseX + event->dx;
|
||||
int ny = gMouseY + event->dy;
|
||||
int bw = gMouseBoundsW;
|
||||
int bh = gMouseBoundsH;
|
||||
|
||||
if (bw < 1) bw = 1;
|
||||
if (bh < 1) bh = 1;
|
||||
if (nx < 0) nx = 0;
|
||||
if (ny < 0) ny = 0;
|
||||
if (nx >= bw) nx = bw - 1;
|
||||
if (ny >= bh) ny = bh - 1;
|
||||
|
||||
if (gInputHal && gInputHal->Screen.EraseCursor)
|
||||
gInputHal->Screen.EraseCursor();
|
||||
|
||||
gMouseX = nx;
|
||||
gMouseY = ny;
|
||||
gMouseButtons = event->buttons;
|
||||
|
||||
if (gInputHal && gInputHal->Screen.DrawCursor)
|
||||
gInputHal->Screen.DrawCursor(nx, ny);
|
||||
|
||||
int next = (gMouseTail + 1) & (int)MOUSE_QUEUE_MASK;
|
||||
if (next != gMouseHead) {
|
||||
gMouseQueue[gMouseTail] = *event;
|
||||
gMouseTail = next;
|
||||
}
|
||||
}
|
||||
|
||||
INPUT_KEY KeInputKbPoll(void) {
|
||||
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
|
||||
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
|
||||
}
|
||||
|
||||
bool KeInputIsKeyDown(INPUT_KEY key) {
|
||||
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
|
||||
return gKeyDown[key] != 0;
|
||||
}
|
||||
|
||||
INPUT_KEY KeInputReadKey(void) {
|
||||
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
|
||||
if (gKeyHead == gKeyTail) return INPUT_KEY_NONE;
|
||||
INPUT_KEY key = gKeyQueue[gKeyHead].key;
|
||||
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
|
||||
return key;
|
||||
}
|
||||
|
||||
char KeInputReadChar(void) {
|
||||
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
|
||||
if (gKeyHead == gKeyTail) return 0;
|
||||
char ch = gKeyQueue[gKeyHead].ch;
|
||||
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
|
||||
return ch;
|
||||
}
|
||||
|
||||
uint32_t KeInputGetModifiers(void) {
|
||||
return gModifiers;
|
||||
}
|
||||
|
||||
bool KeInputMousePoll(void) {
|
||||
return gMouseHead != gMouseTail;
|
||||
}
|
||||
|
||||
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
|
||||
if (!dx || !dy || !buttons) return false;
|
||||
if (gMouseHead == gMouseTail) return false;
|
||||
|
||||
INPUT_MOUSE_EVENT ev = gMouseQueue[gMouseHead];
|
||||
gMouseHead = (gMouseHead + 1) & (int)MOUSE_QUEUE_MASK;
|
||||
*dx = ev.dx;
|
||||
*dy = ev.dy;
|
||||
*buttons = ev.buttons;
|
||||
return true;
|
||||
}
|
||||
|
||||
void KeInputMouseGetPosition(int* x, int* y) {
|
||||
if (x) *x = gMouseX;
|
||||
if (y) *y = gMouseY;
|
||||
}
|
||||
|
||||
void KeInputMouseSetPosition(int x, int y) {
|
||||
int bw = gMouseBoundsW;
|
||||
int bh = gMouseBoundsH;
|
||||
if (bw < 1) bw = 1;
|
||||
if (bh < 1) bh = 1;
|
||||
if (x < 0) x = 0;
|
||||
if (y < 0) y = 0;
|
||||
if (x >= bw) x = bw - 1;
|
||||
if (y >= bh) y = bh - 1;
|
||||
gMouseX = x;
|
||||
gMouseY = y;
|
||||
}
|
||||
|
||||
void KeInputMouseSetBounds(int w, int h) {
|
||||
if (w < 1) w = 1;
|
||||
if (h < 1) h = 1;
|
||||
gMouseBoundsW = w;
|
||||
gMouseBoundsH = h;
|
||||
KeInputMouseSetPosition(gMouseX, gMouseY);
|
||||
}
|
||||
+24
-3
@@ -10,6 +10,8 @@
|
||||
|
||||
#define MAX_IRQS 24
|
||||
#define MAX_IRQ_HANDLERS 8
|
||||
#define MSI_IRQ_FIRST 16u
|
||||
#define MSI_IRQ_LAST 23u
|
||||
|
||||
typedef struct IRQ_HANDLER_LIST {
|
||||
void (*Handlers[MAX_IRQ_HANDLERS])(void);
|
||||
@@ -17,6 +19,7 @@ typedef struct IRQ_HANDLER_LIST {
|
||||
} IRQ_HANDLER_LIST;
|
||||
|
||||
static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
|
||||
static bool gIrqSoftwareVector[MAX_IRQS];
|
||||
static const HAL_API *gHal;
|
||||
|
||||
static KIRQL KiIrqToIrql(uint8_t irq) {
|
||||
@@ -30,6 +33,23 @@ void KeIrqInitialize(const HAL_API *hal) {
|
||||
gHal = hal;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
|
||||
if (!outIrq) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
for (uint8_t irq = MSI_IRQ_FIRST; irq <= MSI_IRQ_LAST; irq++) {
|
||||
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
|
||||
if (!gIrqSoftwareVector[irq] && list->Count == 0) {
|
||||
gIrqSoftwareVector[irq] = true;
|
||||
*outIrq = irq;
|
||||
DbgPrint("KeIrq: allocated MSI vector slot IRQ %u vector 0x%02x\n",
|
||||
irq, 0x20u + irq);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
|
||||
if (irq >= MAX_IRQS || !handler) return;
|
||||
|
||||
@@ -45,7 +65,8 @@ __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
|
||||
}
|
||||
|
||||
list->Handlers[list->Count++] = handler;
|
||||
gHal->Intc.RouteIrq(irq);
|
||||
if (!gIrqSoftwareVector[irq])
|
||||
gHal->Intc.RouteIrq(irq);
|
||||
}
|
||||
|
||||
void KeIrqDispatch(uint8_t irq) {
|
||||
@@ -71,9 +92,9 @@ void KeIrqDispatch(uint8_t irq) {
|
||||
}
|
||||
}
|
||||
|
||||
// Handlers are no longer allowed to context-switch directly. Timer IRQs
|
||||
// Handlers are no longer allowed to context-switch directly. Timer IRQs
|
||||
// only request rescheduling; KiExitInterrupt performs the switch after this
|
||||
// function returns, so the EOI is guaranteed to be sent first. Keep the
|
||||
// function returns, so the EOI is guaranteed to be sent first. Keep the
|
||||
// software level at DIRQL until after the controller has been acknowledged.
|
||||
gHal->Intc.SendEoi();
|
||||
|
||||
|
||||
@@ -61,7 +61,7 @@ void KeReleaseMutex(MUTEX* mutex) {
|
||||
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
|
||||
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||
// Direct handoff: the selected waiter owns the mutex when its wait
|
||||
// returns. Do not drop Owner to NULL, or another thread can steal it.
|
||||
// returns. Do not drop Owner to NULL, or another thread can steal it.
|
||||
mutex->Owner = waiter;
|
||||
KeReleaseSpinlock(&mutex->Lock, flags);
|
||||
KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS);
|
||||
|
||||
+33
-52
@@ -96,45 +96,21 @@ static void KiProcessDestroy(OBJECT* obj) {
|
||||
}
|
||||
}
|
||||
|
||||
// Release the main executable image.
|
||||
if (proc->ImageBase && proc->ImageSize) {
|
||||
MmFreeUserImage((void *)proc->ImageBase, proc->ImageSize);
|
||||
proc->ImageBase = 0;
|
||||
proc->ImageSize = 0;
|
||||
}
|
||||
|
||||
// Release per-process modules.
|
||||
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
|
||||
if (proc->Modules[i].Base && proc->Modules[i].Size) {
|
||||
MmFreeUserImage((void *)proc->Modules[i].Base, proc->Modules[i].Size);
|
||||
}
|
||||
proc->Modules[i].Base = 0;
|
||||
proc->Modules[i].Size = 0;
|
||||
proc->Modules[i].Name[0] = '\0';
|
||||
}
|
||||
// Release every user VA region owned by this process. This includes the
|
||||
// main image, DLLs, TEBs/stacks that survived thread teardown, process
|
||||
// environment strings, and leaked user allocations.
|
||||
MmDestroyAllUserRegions(proc);
|
||||
proc->ImageBase = 0;
|
||||
proc->ImageSize = 0;
|
||||
proc->ModuleCount = 0;
|
||||
|
||||
// Release the user-mode process environment.
|
||||
if (proc->UserPebBase && proc->UserPebSize) {
|
||||
MmFreeUserMemory((void *)proc->UserPebBase);
|
||||
proc->UserPebBase = 0;
|
||||
proc->UserPebSize = 0;
|
||||
}
|
||||
if (proc->UserImagePathBase && proc->UserImagePathSize) {
|
||||
MmFreeUserMemory((void *)proc->UserImagePathBase);
|
||||
proc->UserImagePathBase = 0;
|
||||
proc->UserImagePathSize = 0;
|
||||
}
|
||||
if (proc->UserCmdlineBase && proc->UserCmdlineSize) {
|
||||
MmFreeUserMemory((void *)proc->UserCmdlineBase);
|
||||
proc->UserCmdlineBase = 0;
|
||||
proc->UserCmdlineSize = 0;
|
||||
}
|
||||
if (proc->UserCwdBase && proc->UserCwdSize) {
|
||||
MmFreeUserMemory((void *)proc->UserCwdBase);
|
||||
proc->UserCwdBase = 0;
|
||||
proc->UserCwdSize = 0;
|
||||
}
|
||||
proc->UserPebBase = 0;
|
||||
proc->UserPebSize = 0;
|
||||
proc->UserImagePathBase = 0;
|
||||
proc->UserImagePathSize = 0;
|
||||
proc->UserCmdlineBase = 0;
|
||||
proc->UserCmdlineSize = 0;
|
||||
proc->UserCwdBase = 0;
|
||||
proc->UserCwdSize = 0;
|
||||
|
||||
// Release the exit event.
|
||||
if (proc->ExitEvent) {
|
||||
@@ -189,6 +165,8 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
|
||||
proc->CurrentDirectoryLength = 0;
|
||||
proc->CurrentDrive = 0;
|
||||
proc->CwdLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
LIST_INIT(&proc->UserRegions);
|
||||
proc->UserRegionLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
|
||||
// Slot 0 is the invalid handle - leave it NULL.
|
||||
memset(proc->Handles, 0, sizeof(proc->Handles));
|
||||
@@ -213,7 +191,8 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
|
||||
// ---------------------------------------------------------------------------
|
||||
// KeSetupUserProcessEnvironment
|
||||
// ---------------------------------------------------------------------------
|
||||
static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src,
|
||||
static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
|
||||
SYS_STRING_VIEW src,
|
||||
uintptr_t* baseOut,
|
||||
size_t* sizeOut) {
|
||||
if (!baseOut || !sizeOut) return STATUS_INVALID_PARAMETER;
|
||||
@@ -225,7 +204,7 @@ static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src,
|
||||
if (src.Length == (size_t)-1) return STATUS_BUFFER_TOO_SMALL;
|
||||
|
||||
size_t len = src.Length + 1;
|
||||
char* dst = (char*)MmAllocateUserMemory(len);
|
||||
char* dst = (char*)MmAllocateUserMemoryForProcess(proc, len);
|
||||
if (!dst) return STATUS_NO_MEMORY;
|
||||
|
||||
if (src.Length) memcpy(dst, src.Data, src.Length);
|
||||
@@ -236,7 +215,8 @@ static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src,
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static KSTATUS KiCopyEnvironmentStringViewFixed(SYS_STRING_VIEW src,
|
||||
static KSTATUS KiCopyEnvironmentStringViewFixed(PROCESS* proc,
|
||||
SYS_STRING_VIEW src,
|
||||
size_t capacity,
|
||||
uintptr_t* baseOut,
|
||||
size_t* sizeOut) {
|
||||
@@ -248,7 +228,7 @@ static KSTATUS KiCopyEnvironmentStringViewFixed(SYS_STRING_VIEW src,
|
||||
if (SysSvContainsByte(src, '\0')) return STATUS_INVALID_PARAMETER;
|
||||
if (src.Length + 1 > capacity) return STATUS_BUFFER_TOO_SMALL;
|
||||
|
||||
char* dst = (char*)MmAllocateUserMemory(capacity);
|
||||
char* dst = (char*)MmAllocateUserMemoryForProcess(proc, capacity);
|
||||
if (!dst) return STATUS_NO_MEMORY;
|
||||
memset(dst, 0, capacity);
|
||||
|
||||
@@ -264,22 +244,22 @@ static void KiFreePartialProcessEnvironment(PROCESS* proc) {
|
||||
if (!proc) return;
|
||||
|
||||
if (proc->UserPebBase && proc->UserPebSize) {
|
||||
MmFreeUserMemory((void*)proc->UserPebBase);
|
||||
MmFreeUserMemoryForProcess(proc, (void*)proc->UserPebBase);
|
||||
proc->UserPebBase = 0;
|
||||
proc->UserPebSize = 0;
|
||||
}
|
||||
if (proc->UserImagePathBase && proc->UserImagePathSize) {
|
||||
MmFreeUserMemory((void*)proc->UserImagePathBase);
|
||||
MmFreeUserMemoryForProcess(proc, (void*)proc->UserImagePathBase);
|
||||
proc->UserImagePathBase = 0;
|
||||
proc->UserImagePathSize = 0;
|
||||
}
|
||||
if (proc->UserCmdlineBase && proc->UserCmdlineSize) {
|
||||
MmFreeUserMemory((void*)proc->UserCmdlineBase);
|
||||
MmFreeUserMemoryForProcess(proc, (void*)proc->UserCmdlineBase);
|
||||
proc->UserCmdlineBase = 0;
|
||||
proc->UserCmdlineSize = 0;
|
||||
}
|
||||
if (proc->UserCwdBase && proc->UserCwdSize) {
|
||||
MmFreeUserMemory((void*)proc->UserCwdBase);
|
||||
MmFreeUserMemoryForProcess(proc, (void*)proc->UserCwdBase);
|
||||
proc->UserCwdBase = 0;
|
||||
proc->UserCwdSize = 0;
|
||||
}
|
||||
@@ -337,19 +317,20 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
|
||||
|
||||
KiFreePartialProcessEnvironment(proc);
|
||||
|
||||
s = KiCopyEnvironmentStringView(imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
|
||||
s = KiCopyEnvironmentStringView(proc, imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
|
||||
if (!KSUCCESS(s)) goto fail;
|
||||
|
||||
s = KiCopyEnvironmentStringView(finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize);
|
||||
s = KiCopyEnvironmentStringView(proc, finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize);
|
||||
if (!KSUCCESS(s)) goto fail;
|
||||
|
||||
s = KiCopyEnvironmentStringViewFixed(SysSvFromParts(finalCwd, finalCwdLength),
|
||||
s = KiCopyEnvironmentStringViewFixed(proc,
|
||||
SysSvFromParts(finalCwd, finalCwdLength),
|
||||
SYS_CWD_MAX,
|
||||
&proc->UserCwdBase,
|
||||
&proc->UserCwdSize);
|
||||
if (!KSUCCESS(s)) goto fail;
|
||||
|
||||
SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemory(sizeof(SYS_PEB));
|
||||
SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemoryForProcess(proc, sizeof(SYS_PEB));
|
||||
if (!peb) { s = STATUS_NO_MEMORY; goto fail; }
|
||||
memset(peb, 0, sizeof(SYS_PEB));
|
||||
|
||||
@@ -389,7 +370,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
|
||||
process->ExitCode = exitCode;
|
||||
_InterlockedExchange(&process->Terminating, 1);
|
||||
|
||||
// The calling thread exits immediately after this in SysExitProcess. Do
|
||||
// The calling thread exits immediately after this in SysExitProcess. Do
|
||||
// not drop the process object here: each thread owns one process reference,
|
||||
// and KiThreadDestroy releases that reference after unlinking the thread.
|
||||
KiCompleteProcessExitIfLastThread(process);
|
||||
@@ -402,7 +383,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
|
||||
_InterlockedExchange(&process->Terminating, 1);
|
||||
|
||||
// Wake alertable waiters so they can notice Terminating at the syscall/
|
||||
// wait boundary. This is intentionally conservative: fully asynchronous
|
||||
// wait boundary. This is intentionally conservative: fully asynchronous
|
||||
// forced thread teardown needs object-specific wait-list locking and is a
|
||||
// larger process-lifecycle project.
|
||||
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
|
||||
|
||||
+11
-10
@@ -2,6 +2,7 @@
|
||||
#include <stddef.h>
|
||||
#include <stdbool.h>
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_enable)
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_cpu.h>
|
||||
#include <kernel/ke_process.h>
|
||||
@@ -84,7 +85,7 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) {
|
||||
uint32_t quantum = KiBaseQuantumForPriority(thread->Priority);
|
||||
|
||||
// Low-priority CPU-bound threads get longer batches once they have proven
|
||||
// they mostly burn CPU. Higher-priority work still preempts them as soon
|
||||
// they mostly burn CPU. Higher-priority work still preempts them as soon
|
||||
// as it becomes ready, so this reduces switch overhead without hurting
|
||||
// interactive wake latency.
|
||||
if (thread->Priority == PriorityLow && thread->CpuBurstTicks >=
|
||||
@@ -113,7 +114,7 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
|
||||
|
||||
|
||||
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
|
||||
// The scheduler lock is intentionally held across KeSwitchContext(). The
|
||||
// The scheduler lock is intentionally held across KeSwitchContext(). The
|
||||
// unlock cookie belongs to the thread continuation that will resume after
|
||||
// the switch, not to the physical CPU. Another dispatch may resume this
|
||||
// thread from an unrelated interrupt, so using a KCPU slot here would
|
||||
@@ -129,7 +130,7 @@ static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
|
||||
|
||||
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
|
||||
KePanic(KE_PANIC_CODE_SCHEDULER,
|
||||
"scheduler invariant failed: %s: thread=0x%llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
|
||||
"scheduler invariant failed: %s: thread=0x%016llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
|
||||
where,
|
||||
(uint64_t)thread,
|
||||
thread ? thread->ThreadId : 0,
|
||||
@@ -201,7 +202,7 @@ static void KiIdleLoop(void) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// HLT only wakes for maskable IPIs when IF=1. A CPU that reaches
|
||||
// HLT only wakes for maskable IPIs when IF=1. A CPU that reaches
|
||||
// the idle thread through an unusual switch path must not sleep with
|
||||
// interrupts disabled, otherwise reschedule/TLB shootdown IPIs can
|
||||
// be delayed forever.
|
||||
@@ -282,7 +283,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
|
||||
KeInitThreadFpuContext(idle);
|
||||
|
||||
// The hand-built CPU idle thread is not created through KeCreateThread(),
|
||||
// so register it manually with the idle process. This keeps PID 0 visible
|
||||
// so register it manually with the idle process. This keeps PID 0 visible
|
||||
// as the real idle process while kernel worker threads live under PID 1.
|
||||
ObReferenceObject(&gIdleProcess->Header);
|
||||
uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock);
|
||||
@@ -418,8 +419,8 @@ static void KiSchedule(void) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
|
||||
KiArmPostSwitchUnlock(cur, flags);
|
||||
|
||||
// The currently running thread is not present in a run queue. If it is
|
||||
// still runnable and nobody else is ready, keep executing it. Otherwise a
|
||||
// The currently running thread is not present in a run queue. If it is
|
||||
// still runnable and nobody else is ready, keep executing it. Otherwise a
|
||||
// lone CPU-bound thread gets alternated with the idle thread on every timer
|
||||
// tick, effectively turning a periodic preemption check into a 50% throttle.
|
||||
if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableForCpuLocked(cpu)) {
|
||||
@@ -508,7 +509,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
|
||||
|
||||
// Complete the wait while holding the scheduler lock. On SMP a wake can
|
||||
// Complete the wait while holding the scheduler lock. On SMP a wake can
|
||||
// race the prepare->commit window on another CPU: the target thread is
|
||||
// already on the object's wait list and has State=ThreadWaiting, but it is
|
||||
// still executing with RunningCpu set until it actually calls KeYield().
|
||||
@@ -529,7 +530,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
|
||||
thread->QuantumRemaining = 0;
|
||||
|
||||
if (thread->RunningCpu != KE_INVALID_CPU) {
|
||||
// Prepare->commit race: the waiter is still running somewhere. Its
|
||||
// Prepare->commit race: the waiter is still running somewhere. Its
|
||||
// KeCommitBlockThread() will see State != ThreadWaiting and return the
|
||||
// completed wait status without ever entering the ready queue.
|
||||
thread->State = ThreadRunning;
|
||||
@@ -598,7 +599,7 @@ void KeSchedulerTick(void) {
|
||||
}
|
||||
|
||||
// Periodic aging pass to prevent indefinite starvation of PriorityLow and
|
||||
// PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
|
||||
// PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
|
||||
// burst history; they earned the boost by waiting, not by burning CPU.
|
||||
gAgingCounter++;
|
||||
if (gAgingCounter >= SCHED_AGING_INTERVAL_TICKS) {
|
||||
|
||||
+16
-14
@@ -42,7 +42,7 @@ static void KiApcSelfTestKernelRoutine(KAPC* apc,
|
||||
void** normalContext,
|
||||
void** systemArgument1,
|
||||
void** systemArgument2) {
|
||||
DbgPrint("APC selftest: kernel routine apc=0x%llX normal=0x%llX ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n",
|
||||
DbgPrint("APC selftest: kernel routine apc=0x%016llX normal=0x%016llX ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
|
||||
(uint64_t)apc,
|
||||
(uint64_t)*normalRoutine,
|
||||
(uint64_t)*normalContext,
|
||||
@@ -56,7 +56,7 @@ static void KiApcSelfTestNormalRoutine(void* normalContext,
|
||||
void* systemArgument2) {
|
||||
volatile int* counter = (volatile int*)normalContext;
|
||||
|
||||
DbgPrint("APC selftest: normal routine ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n",
|
||||
DbgPrint("APC selftest: normal routine ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
|
||||
(uint64_t)normalContext,
|
||||
(uint64_t)systemArgument1,
|
||||
(uint64_t)systemArgument2,
|
||||
@@ -73,7 +73,7 @@ static void KiApcSelfTestCrossThreadRoutine(void* normalContext,
|
||||
void* systemArgument2) {
|
||||
KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext;
|
||||
|
||||
DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n",
|
||||
DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
|
||||
(uint64_t)systemArgument1,
|
||||
(uint64_t)systemArgument2,
|
||||
KeGetCurrentIrql());
|
||||
@@ -124,7 +124,7 @@ static void KiApcAlertableSelfTestNormalRoutine(void* normalContext,
|
||||
KI_APC_ALERTABLE_SELFTEST_STATE* st =
|
||||
(KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext;
|
||||
|
||||
DbgPrint("APC selftest: alertable APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n",
|
||||
DbgPrint("APC selftest: alertable APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
|
||||
(uint64_t)systemArgument1,
|
||||
(uint64_t)systemArgument2,
|
||||
KeGetCurrentIrql());
|
||||
@@ -366,7 +366,7 @@ static void KiIrpApcSelfTestKernelRoutine(KAPC* apc,
|
||||
void** normalContext,
|
||||
void** systemArgument1,
|
||||
void** systemArgument2) {
|
||||
DbgPrint("IRP/APC selftest: kernel routine apc=0x%llX irp=0x%llX status=%u irql=%u\n",
|
||||
DbgPrint("IRP/APC selftest: kernel routine apc=0x%016llX irp=0x%016llX status=%u irql=%u\n",
|
||||
(uint64_t)apc,
|
||||
(uint64_t)*systemArgument1,
|
||||
(uint32_t)(uintptr_t)*systemArgument2,
|
||||
@@ -383,7 +383,7 @@ static void KiIrpApcSelfTestNormalRoutine(void* normalContext,
|
||||
IRP* irp = (IRP*)systemArgument1;
|
||||
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
|
||||
|
||||
DbgPrint("IRP/APC selftest: APC delivered irp=0x%llX status=%u info=%llu irql=%u\n",
|
||||
DbgPrint("IRP/APC selftest: APC delivered irp=0x%016llX status=%u info=%llu irql=%u\n",
|
||||
(uint64_t)irp,
|
||||
status,
|
||||
(uint64_t)(irp ? irp->Information : 0),
|
||||
@@ -553,7 +553,7 @@ static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc,
|
||||
void** normalContext,
|
||||
void** systemArgument1,
|
||||
void** systemArgument2) {
|
||||
DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%llX iosb=0x%llX status=%u irql=%u\n",
|
||||
DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%016llX iosb=0x%016llX status=%u irql=%u\n",
|
||||
(uint64_t)apc,
|
||||
(uint64_t)*systemArgument1,
|
||||
(uint32_t)(uintptr_t)*systemArgument2,
|
||||
@@ -570,7 +570,7 @@ static void KiFileAsyncSelfTestNormalRoutine(void* normalContext,
|
||||
(KI_FILE_ASYNC_SELFTEST_STATE*)normalContext;
|
||||
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
|
||||
|
||||
DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%llX status=%u iosb=%u/%llu irql=%u\n",
|
||||
DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%016llX status=%u iosb=%u/%llu irql=%u\n",
|
||||
(uint64_t)systemArgument1,
|
||||
status,
|
||||
st->Iosb.Status,
|
||||
@@ -759,10 +759,10 @@ void KiRunCoreHardeningSelfTests(void) {
|
||||
|
||||
KCPU* cpu = KeGetCurrentCpu();
|
||||
if (cpu != &gCpus[0])
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%llX cpu0=0x%llX\n",
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%016llX cpu0=0x%016llX\n",
|
||||
(uint64_t)cpu, (uint64_t)&gCpus[0]);
|
||||
if (cpu->Self != cpu)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%llX cpu=0x%llX\n",
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%016llX cpu=0x%016llX\n",
|
||||
(uint64_t)cpu->Self, (uint64_t)cpu);
|
||||
if (KeGetCurrentCpuIndex() != 0)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n");
|
||||
@@ -776,15 +776,17 @@ void KiRunCoreHardeningSelfTests(void) {
|
||||
if (KeGetCurrentTss() != &cpu->Tss)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n");
|
||||
if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%llX iomap=%u\n",
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%016llX iomap=%u\n",
|
||||
cpu->Tss.RSP0, cpu->Tss.IoMapBase);
|
||||
if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%llX gdt=0x%llX\n",
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%016llX gdt=0x%016llX\n",
|
||||
cpu->Gdtr.Base, (uint64_t)cpu->Gdt);
|
||||
|
||||
for (uint32_t i = 1; i < gCpuCount; i++) {
|
||||
KCPU* ap = &gCpus[i];
|
||||
if (!ap->Present || !ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
|
||||
if (!ap->Present)
|
||||
continue;
|
||||
if (!ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n",
|
||||
i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp);
|
||||
@@ -795,7 +797,7 @@ void KiRunCoreHardeningSelfTests(void) {
|
||||
if (!ap->KernelStack || !ap->KernelStackTop)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
|
||||
if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%llX iomap=%u\n",
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%016llX iomap=%u\n",
|
||||
i, ap->Tss.RSP0, ap->Tss.IoMapBase);
|
||||
if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i);
|
||||
|
||||
@@ -57,7 +57,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
|
||||
if (!sem) return;
|
||||
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
|
||||
if (!LIST_IS_EMPTY(&sem->WaitList)) {
|
||||
// Transfer this semaphore release directly to the selected waiter. The
|
||||
// Transfer this semaphore release directly to the selected waiter. The
|
||||
// count stays unchanged; the waiter returns success without looping and
|
||||
// consuming a count later.
|
||||
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
|
||||
|
||||
+135
-20
@@ -8,6 +8,8 @@
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_disable)
|
||||
#pragma intrinsic(_enable)
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_smp.h>
|
||||
#include <kernel/ke_cpu.h>
|
||||
@@ -22,6 +24,8 @@
|
||||
#include <kernel/hal/hal_apic.h>
|
||||
|
||||
#define AP_TRAMPOLINE_MAX_PHYS 0x000A0000ULL
|
||||
#define AP_TRAMPOLINE_PREFERRED_MIN 0x00070000ULL
|
||||
#define AP_TRAMPOLINE_FALLBACK_MIN 0x00010000ULL
|
||||
#define AP_TRAMPOLINE_BOOT_OFF 0x0800u
|
||||
#define AP_STACK_PAGES 8u
|
||||
#define AP_START_TIMEOUT_SPINS 20000000u
|
||||
@@ -74,6 +78,7 @@ _Static_assert(offsetof(AP_BOOT_BLOCK, State) == 48, "AP_BOOT_BLOCK.State offset
|
||||
_Static_assert(offsetof(AP_BOOT_BLOCK, Stage) == 52, "AP_BOOT_BLOCK.Stage offset");
|
||||
|
||||
static PHYSICAL_ADDRESS gApTrampolinePhys;
|
||||
static PHYSICAL_ADDRESS gReservedApTrampolinePhys;
|
||||
static AP_BOOT_BLOCK* gApBootBlock;
|
||||
|
||||
static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
|
||||
@@ -94,13 +99,55 @@ static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, u
|
||||
((uint64_t)((base >> 24) & 0xFF) << 56);
|
||||
}
|
||||
|
||||
void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys) {
|
||||
if (trampolinePhys >= PAGE_SIZE &&
|
||||
trampolinePhys < AP_TRAMPOLINE_MAX_PHYS &&
|
||||
!(trampolinePhys & (PAGE_SIZE - 1))) {
|
||||
gReservedApTrampolinePhys = (PHYSICAL_ADDRESS)trampolinePhys;
|
||||
} else {
|
||||
gReservedApTrampolinePhys = 0;
|
||||
}
|
||||
}
|
||||
|
||||
static KSTATUS KiBuildApTrampoline(void) {
|
||||
if (gApTrampolinePhys)
|
||||
return STATUS_SUCCESS;
|
||||
|
||||
PHYSICAL_ADDRESS frame = MmAllocateFrameBelow(AP_TRAMPOLINE_MAX_PHYS);
|
||||
if (!frame || frame < PAGE_SIZE || (frame & (PAGE_SIZE - 1)))
|
||||
PHYSICAL_ADDRESS frame = gReservedApTrampolinePhys;
|
||||
if (frame) {
|
||||
if (frame < AP_TRAMPOLINE_FALLBACK_MIN ||
|
||||
frame >= AP_TRAMPOLINE_MAX_PHYS ||
|
||||
(frame & (PAGE_SIZE - 1))) {
|
||||
DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
|
||||
(uint64_t)frame);
|
||||
frame = 0;
|
||||
} else {
|
||||
DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
|
||||
(uint64_t)frame);
|
||||
}
|
||||
}
|
||||
|
||||
if (!frame) {
|
||||
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
|
||||
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
|
||||
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
|
||||
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_PREFERRED_MIN,
|
||||
AP_TRAMPOLINE_MAX_PHYS);
|
||||
}
|
||||
if (!frame) {
|
||||
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
|
||||
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
|
||||
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
|
||||
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_FALLBACK_MIN,
|
||||
AP_TRAMPOLINE_PREFERRED_MIN);
|
||||
}
|
||||
if (!frame) {
|
||||
DbgPrint("SMP: no free PMM frame available below 0x%016llX for AP trampoline\n",
|
||||
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
|
||||
|
||||
uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame);
|
||||
memset(page, 0, PAGE_SIZE);
|
||||
@@ -150,7 +197,7 @@ static KSTATUS KiBuildApTrampoline(void) {
|
||||
while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90);
|
||||
uint32_t long64Off = (uint32_t)(p - page);
|
||||
|
||||
// --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). ---
|
||||
// --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). ---
|
||||
KiEmit8(&p, 0x48); KiEmit8(&p, 0xBB); size_t bootBlockPatch = (size_t)(p - page); KiEmit64(&p, 0); // mov rbx, bootblock
|
||||
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Stage)); KiEmit32(&p, AP_STAGE_LONG64);
|
||||
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, State)); KiEmit32(&p, AP_STATE_LONG_MODE);
|
||||
@@ -201,7 +248,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
|
||||
}
|
||||
|
||||
// APs start from the SIPI reset state, so do not assume the firmware/BSP
|
||||
// CR4 feature bits are present. The scheduler and context-switch paths
|
||||
// CR4 feature bits are present. The scheduler and context-switch paths
|
||||
// use FXSAVE/FXRSTOR on every CPU; without OSFXSR, those instructions
|
||||
// raise #UD as soon as an AP enters its idle thread.
|
||||
uintptr_t cr4 = __readcr4();
|
||||
@@ -282,12 +329,20 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
|
||||
}
|
||||
|
||||
static KSTATUS KiAllocateApStack(KCPU* cpu) {
|
||||
PHYSICAL_ADDRESS stackPhys = MmAllocateContiguous(AP_STACK_PAGES);
|
||||
if (!stackPhys) return STATUS_NO_MEMORY;
|
||||
// AP kernel stacks do not need physically contiguous memory. Requiring
|
||||
// MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
|
||||
// available, which is fragile on real machines and unrelated to AP startup.
|
||||
size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE;
|
||||
void* stack = MmAllocatePool(stackBytes);
|
||||
if (!stack) {
|
||||
DbgPrint("SMP: AP%u stack allocation failed bytes=%llu\n",
|
||||
cpu ? cpu->CpuIndex : 0xFFFFFFFFu, (uint64_t)stackBytes);
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
cpu->KernelStack = (void*)(PHYS_MAP_BASE + stackPhys);
|
||||
cpu->KernelStackTop = (uintptr_t)cpu->KernelStack + AP_STACK_PAGES * PAGE_SIZE;
|
||||
memset(cpu->KernelStack, 0, AP_STACK_PAGES * PAGE_SIZE);
|
||||
cpu->KernelStack = stack;
|
||||
cpu->KernelStackTop = (uintptr_t)stack + stackBytes;
|
||||
memset(cpu->KernelStack, 0, stackBytes);
|
||||
memset(&cpu->Tss, 0, sizeof(cpu->Tss));
|
||||
cpu->Tss.IoMapBase = sizeof(TSS64);
|
||||
cpu->Tss.RSP0 = cpu->KernelStackTop;
|
||||
@@ -295,6 +350,20 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static const char* KiApStageName(uint32_t stage) {
|
||||
switch (stage) {
|
||||
case AP_STAGE_NONE: return "none";
|
||||
case AP_STAGE_LONG64: return "long64";
|
||||
case AP_STAGE_ENTRY: return "entry";
|
||||
case AP_STAGE_GS: return "gs";
|
||||
case AP_STAGE_GDT: return "gdt";
|
||||
case AP_STAGE_IDT: return "idt";
|
||||
case AP_STAGE_PARK: return "park";
|
||||
default: return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) {
|
||||
memset(gApBootBlock, 0, sizeof(*gApBootBlock));
|
||||
gApBootBlock->Magic = AP_BOOT_MAGIC;
|
||||
@@ -326,30 +395,53 @@ KSTATUS KeStartApplicationProcessors(void) {
|
||||
bsp->SchedulerCommand = 0;
|
||||
bsp->SchedulerActive = 1;
|
||||
|
||||
DbgPrint("SMP: APIC mode=%s BSP LAPIC=%u MADT CPUs=%u\n",
|
||||
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
|
||||
bspLapic, topo.CpuCount);
|
||||
|
||||
uint32_t count = 1;
|
||||
for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) {
|
||||
HAL_CPU_INFO* info = &topo.Cpus[i];
|
||||
DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
|
||||
i, info->ProcessorUid, info->LapicId, info->Flags,
|
||||
info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
|
||||
|
||||
if (!info->Enabled) continue;
|
||||
if (info->LapicId == bspLapic) {
|
||||
bsp->AcpiProcessorId = info->ProcessorUid;
|
||||
continue;
|
||||
}
|
||||
if (!HalApicCanAddressId(info->LapicId)) {
|
||||
DbgPrint("SMP: skipping AP uid=%u lapic=%u; current APIC mode cannot address it\n",
|
||||
info->ProcessorUid, info->LapicId);
|
||||
continue;
|
||||
}
|
||||
KiPrepareCpuRecord(count, info);
|
||||
count++;
|
||||
}
|
||||
gCpuCount = count;
|
||||
|
||||
DbgPrint("SMP: discovered %u CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic);
|
||||
DbgPrint("SMP: discovered %u addressable CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic);
|
||||
if (gCpuCount <= 1) {
|
||||
DbgPrint("SMP: no application processors present\n");
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if ((__readcr3() >> 32) != 0)
|
||||
return STATUS_NOT_SUPPORTED;
|
||||
if ((__readcr3() >> 32) != 0) {
|
||||
DbgPrint("SMP: disabled; bootstrap CR3 is above 4GiB (cr3=0x%016llX), AP trampoline cannot load it yet\n",
|
||||
(uint64_t)__readcr3());
|
||||
gCpuCount = 1;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
|
||||
(uint64_t)gReservedApTrampolinePhys);
|
||||
st = KiBuildApTrampoline();
|
||||
if (!KSUCCESS(st)) return st;
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("SMP: disabled; AP trampoline allocation/build failed status=0x%x\n", st);
|
||||
gCpuCount = 1;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
uint8_t sipiVector = (uint8_t)(gApTrampolinePhys >> 12);
|
||||
uint32_t claimed = 0;
|
||||
@@ -359,13 +451,29 @@ KSTATUS KeStartApplicationProcessors(void) {
|
||||
if (!cpu->Present) continue;
|
||||
|
||||
st = KiAllocateApStack(cpu);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("SMP: AP%u LAPIC=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
|
||||
i, cpu->LapicId, st);
|
||||
cpu->Present = 0;
|
||||
cpu->Started = 0;
|
||||
cpu->Parked = 0;
|
||||
cpu->Online = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
KiFillBootBlockForCpu(cpu, i);
|
||||
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
|
||||
|
||||
st = HalStartApplicationProcessor(cpu->LapicId, sipiVector);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("SMP: AP%u LAPIC=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
|
||||
i, cpu->LapicId, st);
|
||||
cpu->Present = 0;
|
||||
cpu->Started = 0;
|
||||
cpu->Parked = 0;
|
||||
cpu->Online = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
uint64_t lastHeartbeat = gApBootBlock->Heartbeat;
|
||||
int parked = 0;
|
||||
@@ -382,10 +490,14 @@ KSTATUS KeStartApplicationProcessors(void) {
|
||||
}
|
||||
|
||||
if (!parked) {
|
||||
KePanic(KE_PANIC_CODE_INIT,
|
||||
"SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x heartbeat=%llu last=%llu\n",
|
||||
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
|
||||
gApBootBlock->Heartbeat, lastHeartbeat);
|
||||
DbgPrint("SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
|
||||
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
|
||||
KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
|
||||
cpu->Present = 0;
|
||||
cpu->Started = 0;
|
||||
cpu->Parked = 0;
|
||||
cpu->Online = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
cpu->Started = 1;
|
||||
@@ -395,6 +507,9 @@ KSTATUS KeStartApplicationProcessors(void) {
|
||||
DbgPrint("SMP: AP%u reached kernel park\n", i);
|
||||
}
|
||||
|
||||
if (claimed == 0 && gCpuCount > 1)
|
||||
DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n");
|
||||
|
||||
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
@@ -482,7 +597,7 @@ struct SMP_AP_WORK_TEST {
|
||||
static void KiSmpApControlledWorker(void* context) {
|
||||
SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context;
|
||||
if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%llX\n", (uint64_t)arg);
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%016llX\n", (uint64_t)arg);
|
||||
}
|
||||
|
||||
_InterlockedExchange(&arg->Ready, 1);
|
||||
|
||||
@@ -50,7 +50,7 @@ KeSwitchContext PROC
|
||||
; rsp = to->SavedContext
|
||||
mov rsp, [rdx + THREAD_SavedContext]
|
||||
|
||||
; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the
|
||||
; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the
|
||||
; incoming thread's user TEB in IA32_KERNEL_GS_BASE; user-mode entry/exit
|
||||
; swaps it with the kernel GS base using swapgs.
|
||||
mov rax, [rdx + THREAD_Teb] ; Fetch the incoming user GS/TEB pointer
|
||||
@@ -80,9 +80,9 @@ KeSwitchContext ENDP
|
||||
; void KeStartInitialThread(THREAD* to)
|
||||
; RCX = to
|
||||
;
|
||||
; Used by AP scheduler admission. The AP is already in kernel long mode on a
|
||||
; Used by AP scheduler admission. The AP is already in kernel long mode on a
|
||||
; AP bootstrap stack; this helper installs the idle thread's stack/TSS
|
||||
; state and returns directly into that thread's saved context. There is no
|
||||
; state and returns directly into that thread's saved context. There is no
|
||||
; outgoing THREAD and no scheduler lock to release.
|
||||
; -------------------------------------------------------------------------
|
||||
KeStartInitialThread PROC
|
||||
@@ -94,7 +94,7 @@ KeStartInitialThread PROC
|
||||
call KeSetCurrentTssRsp0
|
||||
add rsp, 40
|
||||
|
||||
; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel
|
||||
; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel
|
||||
; threads have a NULL TEB, which is fine.
|
||||
mov rax, [rbx + THREAD_Teb]
|
||||
mov ecx, 0C0000102h
|
||||
|
||||
+15
-10
@@ -1,7 +1,7 @@
|
||||
; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64)
|
||||
;
|
||||
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
|
||||
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
|
||||
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
|
||||
; slots used by user APC return redirection.
|
||||
|
||||
EXTERN KeSyscallDispatch:PROC
|
||||
@@ -147,8 +147,8 @@ KeSyscallEntry ENDP
|
||||
; Fault stubs - snapshot a full user fault frame before entering C.
|
||||
;
|
||||
; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless
|
||||
; by itself. The captured user RSP lets the C crash dumper read [RSP], which is
|
||||
; usually the return address immediately after the bad CALL. The GPR snapshot
|
||||
; by itself. The captured user RSP lets the C crash dumper read [RSP], which is
|
||||
; usually the return address immediately after the bad CALL. The GPR snapshot
|
||||
; also shows which register held the null target.
|
||||
;
|
||||
; KeFaultDispatchFrame signature (MS x64):
|
||||
@@ -239,7 +239,8 @@ KeFaultDE PROC
|
||||
jmp de_call
|
||||
|
||||
de_kernel_frame:
|
||||
mov qword ptr [rax + KF_Rsp], 0
|
||||
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
|
||||
mov [rax + KF_Rsp], r10
|
||||
mov qword ptr [rax + KF_Ss], 0
|
||||
|
||||
de_call:
|
||||
@@ -304,7 +305,8 @@ KeFaultGP PROC
|
||||
jmp gp_call
|
||||
|
||||
gp_kernel_frame:
|
||||
mov qword ptr [rax + KF_Rsp], 0
|
||||
lea r10, [rdx+32] ; same-privilege fault: original RSP is above ERR/RIP/CS/RFLAGS
|
||||
mov [rax + KF_Rsp], r10
|
||||
mov qword ptr [rax + KF_Ss], 0
|
||||
|
||||
gp_call:
|
||||
@@ -370,7 +372,8 @@ KeFaultPF PROC
|
||||
jmp pf_call
|
||||
|
||||
pf_kernel_frame:
|
||||
mov qword ptr [rax + KF_Rsp], 0
|
||||
lea r10, [rdx+32] ; same-privilege #PF frame has ERR/RIP/CS/RFLAGS
|
||||
mov [rax + KF_Rsp], r10
|
||||
mov qword ptr [rax + KF_Ss], 0
|
||||
|
||||
pf_call:
|
||||
@@ -438,7 +441,8 @@ StubName PROC
|
||||
jmp noerr_call
|
||||
|
||||
noerr_kernel_frame:
|
||||
mov qword ptr [rax + KF_Rsp], 0
|
||||
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
|
||||
mov [rax + KF_Rsp], r10
|
||||
mov qword ptr [rax + KF_Ss], 0
|
||||
|
||||
noerr_call:
|
||||
@@ -502,7 +506,8 @@ StubName PROC
|
||||
jmp err_call
|
||||
|
||||
err_kernel_frame:
|
||||
mov qword ptr [rax + KF_Rsp], 0
|
||||
lea r10, [rdx+32] ; same-privilege error-code frame has ERR/RIP/CS/RFLAGS
|
||||
mov [rax + KF_Rsp], r10
|
||||
mov qword ptr [rax + KF_Ss], 0
|
||||
|
||||
err_call:
|
||||
@@ -516,8 +521,8 @@ ENDM
|
||||
EXTERN KeHandleNmiIpi:PROC
|
||||
PUBLIC KeNmiIpiStub
|
||||
KeNmiIpiStub PROC
|
||||
; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
|
||||
; the NMI was a panic/TLB-control event and is fully handled. If it
|
||||
; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
|
||||
; the NMI was a panic/TLB-control event and is fully handled. If it
|
||||
; returns 0, restore the interrupted context and fall through to the normal
|
||||
; #NMI fault stub so unexpected NMIs still panic with a useful frame.
|
||||
push rax
|
||||
|
||||
+61
-51
@@ -42,6 +42,16 @@
|
||||
#include <kernel/ke_apc.h>
|
||||
#include <kernel/ke_queue.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
|
||||
#ifndef PROCESS_LAUNCH_DEBUG
|
||||
#define PROCESS_LAUNCH_DEBUG 0
|
||||
#endif
|
||||
|
||||
#if PROCESS_LAUNCH_DEBUG
|
||||
#define ProcDbg(...) DbgPrint(__VA_ARGS__)
|
||||
#else
|
||||
#define ProcDbg(...) ((void)0)
|
||||
#endif
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ldr_image.h>
|
||||
#include <kernel/syscall_nr.h>
|
||||
@@ -49,6 +59,7 @@
|
||||
#include <kernel/io_file.h>
|
||||
#include <kernel/net_socket.h>
|
||||
#include <kernel/ke_trap.h>
|
||||
#include <kernel/ke_input.h>
|
||||
#include <ntlite/syscall.h>
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -121,7 +132,7 @@ static KSTATUS KiAllocHandleForCurrentWithRights(OBJECT* obj, uint32_t rights, u
|
||||
KSTATUS s = KeAllocateHandleWithRights(proc, obj, rights, handleOut);
|
||||
|
||||
// Most syscall create/open paths receive a fresh creator reference from the
|
||||
// subsystem. The handle table takes its own reference; this helper always
|
||||
// subsystem. The handle table takes its own reference; this helper always
|
||||
// drops the creator reference so the object is owned only by the returned
|
||||
// handle (and any subsystem-internal refs).
|
||||
ObDereferenceObject(obj);
|
||||
@@ -752,7 +763,7 @@ static void SyscallUserAsyncKernelRoutine(KAPC* apc,
|
||||
(void)systemArgument2;
|
||||
|
||||
SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc);
|
||||
DbgPrint("USER/APC async: kernel routine ctx=0x%llX iosb=0x%llX status=%u info=%llu irql=%u\n",
|
||||
DbgPrint("USER/APC async: kernel routine ctx=0x%016llX iosb=0x%016llX status=%u info=%llu irql=%u\n",
|
||||
(uint64_t)ctx,
|
||||
(uint64_t)(uintptr_t)apc->SystemArgument1,
|
||||
(uint32_t)(uintptr_t)apc->SystemArgument2,
|
||||
@@ -1150,81 +1161,72 @@ static uint64_t SyscallScreenPutChar(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_KB_POLL (40) -> 1 or 0
|
||||
// SYS_KB_POLL (64) -> next INPUT_KEY or INPUT_KEY_NONE
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallKbPoll(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
|
||||
if (!gHal) return 0;
|
||||
return gHal->Keyboard.Poll();
|
||||
return (uint64_t)KeInputKbPoll();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_KB_IS_KEY_DOWN (41) a0=scancode a1=extended -> 1 or 0
|
||||
// SYS_KB_IS_KEY_DOWN (65) a0=INPUT_KEY -> 1 or 0
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallKbIsKeyDown(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a2; (void)a3; (void)a4; (void)a5;
|
||||
|
||||
// Check gHal and ensure the HAL struct actually has the new function pointer
|
||||
if (!gHal || !gHal->Keyboard.IsKeyDown) return 0;
|
||||
|
||||
return gHal->Keyboard.IsKeyDown((uint8_t)a0, (bool)a1) ? 1u : 0u;
|
||||
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
|
||||
return KeInputIsKeyDown((INPUT_KEY)a0) ? 1u : 0u;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_KB_READ_CHAR (42) -> char (blocks, yields while waiting)
|
||||
// SYS_KB_READ_CHAR (66) -> char (blocks, yields while waiting)
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
|
||||
if (!gHal) return 0;
|
||||
while (!gHal->Keyboard.Poll()) KeYield();
|
||||
return (uint64_t)(uint8_t)gHal->Keyboard.ReadChar();
|
||||
while (!KeInputKbPoll()) KeYield();
|
||||
return (uint64_t)(uint8_t)KeInputReadChar();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_KB_READ_SCANCODE (43) -> uint8_t (blocks, yields while waiting)
|
||||
// SYS_KB_READ_KEY (67) -> INPUT_KEY (blocks, yields while waiting)
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallKbReadScancode(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
|
||||
if (!gHal) return 0;
|
||||
while (!gHal->Keyboard.Poll()) KeYield();
|
||||
return (uint64_t)gHal->Keyboard.ReadScancode();
|
||||
while (!KeInputKbPoll()) KeYield();
|
||||
return (uint64_t)KeInputReadKey();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_KB_GET_MODIFIERS (44) -> uint32_t
|
||||
// SYS_KB_GET_MODIFIERS (68) -> uint32_t INPUT_MOD_* flags
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallKbGetModifiers(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
|
||||
if (!gHal) return 0;
|
||||
return (uint64_t)gHal->Keyboard.GetModifiers();
|
||||
return (uint64_t)KeInputGetModifiers();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_MOUSE_POLL (45) -> 1 or 0
|
||||
// SYS_MOUSE_POLL (69) -> 1 or 0
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallMousePoll(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
|
||||
if (!gHal) return 0;
|
||||
return gHal->Mouse.Poll() ? 1u : 0u;
|
||||
return KeInputMousePoll() ? 1u : 0u;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_MOUSE_READ (46) a0=*dx a1=*dy a2=*buttons -> 1 or 0
|
||||
// SYS_MOUSE_READ (70) a0=*dx a1=*dy a2=*buttons -> 1 or 0
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallMouseRead(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a3; (void)a4; (void)a5;
|
||||
if (!gHal || !a0 || !a1 || !a2) return 0;
|
||||
if (!a0 || !a1 || !a2) return 0;
|
||||
|
||||
int dx = 0, dy = 0;
|
||||
uint8_t buttons = 0;
|
||||
if (!gHal->Mouse.Read(&dx, &dy, &buttons)) return 0;
|
||||
if (!KeInputMouseRead(&dx, &dy, &buttons)) return 0;
|
||||
|
||||
if (!KSUCCESS(CopyToUser((void*)(uintptr_t)a0, &dx, sizeof(dx)))) return 0;
|
||||
if (!KSUCCESS(CopyToUser((void*)(uintptr_t)a1, &dy, sizeof(dy)))) return 0;
|
||||
@@ -1233,15 +1235,15 @@ static uint64_t SyscallMouseRead(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SYS_MOUSE_GET_POS (47) a0=*x a1=*y
|
||||
// SYS_MOUSE_GET_POS (71) a0=*x a1=*y
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallMouseGetPos(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uint64_t a3, uint64_t a4, uint64_t a5) {
|
||||
(void)a2; (void)a3; (void)a4; (void)a5;
|
||||
if (!gHal || !a0 || !a1) return STATUS_INVALID_PARAMETER;
|
||||
if (!a0 || !a1) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
int x = 0, y = 0;
|
||||
gHal->Mouse.GetPosition(&x, &y);
|
||||
KeInputMouseGetPosition(&x, &y);
|
||||
KSTATUS s = KiCopyIntToUser(a0, x);
|
||||
if (!KSUCCESS(s)) return s;
|
||||
return KiCopyIntToUser(a1, y);
|
||||
@@ -1334,22 +1336,25 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
|
||||
// Read the executable from disk.
|
||||
FILE_OBJECT* f;
|
||||
ProcDbg("CreateProcess: path='%v' cmd='%v' callerPid=%u\n", path, cmdlineView, KeGetCurrentProcess() ? KeGetCurrentProcess()->ProcessId : 0);
|
||||
s = IoOpenFile(path, &f);
|
||||
if (!KSUCCESS(s)) return s;
|
||||
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: IoOpenFile failed st=%d\n", s); return s; }
|
||||
|
||||
uint64_t size = IoGetFileSize(f);
|
||||
ProcDbg("CreateProcess: fileSize=%llu\n", size);
|
||||
void* mem = MmAllocatePool(size);
|
||||
if (!mem) { IoCloseFile(f); return STATUS_NO_MEMORY; }
|
||||
if (!mem) { IoCloseFile(f); ProcDbg("CreateProcess: raw image pool alloc failed size=%llu\n", size); return STATUS_NO_MEMORY; }
|
||||
|
||||
size_t got;
|
||||
s = IoReadFile(f, mem, size, &got);
|
||||
IoCloseFile(f);
|
||||
if (!KSUCCESS(s)) { MmFreePool(mem); return s; }
|
||||
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: IoReadFile failed st=%d got=%llu\n", s, (unsigned long long)got); MmFreePool(mem); return s; }
|
||||
|
||||
// Create the process object.
|
||||
PROCESS* proc;
|
||||
s = KeCreateProcess(pathBuf, &proc);
|
||||
if (!KSUCCESS(s)) { MmFreePool(mem); return s; }
|
||||
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateProcess failed st=%d\n", s); MmFreePool(mem); return s; }
|
||||
ProcDbg("CreateProcess: created pid=%u name=%s\n", proc->ProcessId, proc->Name);
|
||||
|
||||
// Inherit CWD from parent
|
||||
PROCESS* parent = KeGetCurrentProcess();
|
||||
@@ -1368,34 +1373,37 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
path,
|
||||
cmdlineView,
|
||||
SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength));
|
||||
if (!KSUCCESS(s)) { MmFreePool(mem); ObDereferenceObject(&proc->Header); return s; }
|
||||
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: env setup failed st=%d\n", s); MmFreePool(mem); ObDereferenceObject(&proc->Header); return s; }
|
||||
|
||||
// Map the PE image into memory.
|
||||
uintptr_t imageBase = 0;
|
||||
s = LdrLoadImage(mem, size, &imageBase, true);
|
||||
s = LdrLoadImageForProcess(proc, mem, size, &imageBase);
|
||||
MmFreePool(mem);
|
||||
if (!KSUCCESS(s)) { ObDereferenceObject(&proc->Header); return s; }
|
||||
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: LdrLoadImage failed st=%d\n", s); ObDereferenceObject(&proc->Header); return s; }
|
||||
ProcDbg("CreateProcess: image mapped pid=%u base=%016llx size=%llx\n", proc->ProcessId, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(proc, imageBase));
|
||||
|
||||
s = LdrResolveImportsForProcess(proc, imageBase);
|
||||
if (!KSUCCESS(s)) {
|
||||
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
|
||||
ProcDbg("CreateProcess: resolve imports failed st=%d pid=%u image=%016llx\n", s, proc->ProcessId, (unsigned long long)imageBase);
|
||||
MmDebugDumpUserRegions(proc);
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return s;
|
||||
}
|
||||
ProcDbg("CreateProcess: imports resolved pid=%u modules=%u\n", proc->ProcessId, proc->ModuleCount);
|
||||
|
||||
s = LdrProtectImage(imageBase, true);
|
||||
s = LdrProtectImageForProcess(proc, imageBase, true);
|
||||
if (!KSUCCESS(s)) {
|
||||
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
|
||||
ProcDbg("CreateProcess: protect image failed st=%d pid=%u\n", s, proc->ProcessId);
|
||||
MmDebugDumpUserRegions(proc);
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return s;
|
||||
}
|
||||
|
||||
proc->ImageBase = imageBase;
|
||||
proc->ImageSize = LdrGetImageSize(imageBase);
|
||||
proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
|
||||
|
||||
PROCESS* caller = KeGetCurrentProcess();
|
||||
if (!caller) {
|
||||
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return STATUS_FAILURE;
|
||||
}
|
||||
@@ -1405,7 +1413,6 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
if (!KSUCCESS(s)) {
|
||||
proc->ImageBase = 0;
|
||||
proc->ImageSize = 0;
|
||||
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return s;
|
||||
}
|
||||
@@ -1414,10 +1421,13 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
|
||||
s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread);
|
||||
if (!KSUCCESS(s)) {
|
||||
ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint);
|
||||
MmDebugDumpUserRegions(proc);
|
||||
KeCloseHandle(caller, handle);
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return s;
|
||||
}
|
||||
ProcDbg("CreateProcess: thread created pid=%u tid=%llu entry=%016llx handle=%u\n", proc->ProcessId, (unsigned long long)thread->ThreadId, (unsigned long long)entryPoint, handle);
|
||||
|
||||
s = KiCopyHandleToUser((uint64_t)(uintptr_t)req.HandleOut, handle);
|
||||
if (!KSUCCESS(s)) {
|
||||
@@ -1427,7 +1437,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
return s;
|
||||
}
|
||||
|
||||
// Drop the creator's local reference. The returned handle and live thread
|
||||
// Drop the creator's local reference. The returned handle and live thread
|
||||
// references now own the process lifetime.
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return STATUS_SUCCESS;
|
||||
@@ -1666,7 +1676,7 @@ static uint64_t SyscallGetProcessPid(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
// Named Queue Registry
|
||||
//
|
||||
// A flat, fixed-size table of { name -> QUEUE* } pairs protected by a
|
||||
// spinlock. The registry holds one reference to each published queue for
|
||||
// spinlock. The registry holds one reference to each published queue for
|
||||
// the lifetime of the entry, so the queue object cannot be destroyed while
|
||||
// it is registered even if every process closes its own handle to it.
|
||||
//
|
||||
@@ -1747,7 +1757,7 @@ static uint64_t SyscallPublishQueue(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
// SYS_OPEN_NAMED_QUEUE (61) a0=*name a1=*handleOut
|
||||
//
|
||||
// Looks up 'name' in the registry and allocates a handle to the queue in the
|
||||
// calling process's handle table. Returns STATUS_NOT_FOUND if no match.
|
||||
// calling process's handle table. Returns STATUS_NOT_FOUND if no match.
|
||||
// Callers that race against the publisher should retry on NOT_FOUND.
|
||||
// ---------------------------------------------------------------------------
|
||||
static uint64_t SyscallOpenNamedQueue(uint64_t a0, uint64_t a1, uint64_t a2,
|
||||
@@ -2629,7 +2639,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
|
||||
/* 64 SYS_KB_POLL */ SyscallKbPoll,
|
||||
/* 65 SYS_KB_IS_KEY_DOWN */ SyscallKbIsKeyDown,
|
||||
/* 66 SYS_KB_READ_CHAR */ SyscallKbReadChar,
|
||||
/* 67 SYS_KB_READ_SCANCODE */ SyscallKbReadScancode,
|
||||
/* 67 SYS_KB_READ_KEY */ SyscallKbReadKey,
|
||||
/* 68 SYS_KB_GET_MODIFIERS */ SyscallKbGetModifiers,
|
||||
/* 69 SYS_MOUSE_POLL */ SyscallMousePoll,
|
||||
/* 70 SYS_MOUSE_READ */ SyscallMouseRead,
|
||||
|
||||
+31
-22
@@ -16,7 +16,7 @@
|
||||
#define FPU_FCW_DEFAULT 0x037Fu
|
||||
#define FPU_MXCSR_DEFAULT 0x1F80u
|
||||
|
||||
// These offsets are consumed directly by ke_switch.asm. Keep the asserts
|
||||
// These offsets are consumed directly by ke_switch.asm. Keep the asserts
|
||||
// here so THREAD layout edits fail at build time instead of corrupting
|
||||
// context switches at runtime.
|
||||
#define STATIC_ASSERT_OFFSET(type, field, expected) \
|
||||
@@ -41,6 +41,20 @@ static void KiThreadDestroy(OBJECT* obj) {
|
||||
thread->ExitEvent = (EVENT*)0;
|
||||
}
|
||||
|
||||
PROCESS* ownerForUserRegions = thread->Owner;
|
||||
if (ownerForUserRegions) {
|
||||
if (thread->Teb && thread->TebSize) {
|
||||
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb);
|
||||
thread->Teb = (void*)0;
|
||||
thread->TebSize = 0;
|
||||
}
|
||||
|
||||
if (thread->UserStackBase) {
|
||||
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase);
|
||||
thread->UserStackBase = (void *)0;
|
||||
}
|
||||
}
|
||||
|
||||
if (thread->Owner) {
|
||||
PROCESS* owner = thread->Owner;
|
||||
uint32_t remaining = 0;
|
||||
@@ -63,17 +77,6 @@ static void KiThreadDestroy(OBJECT* obj) {
|
||||
MmFreePool(thread->KernelStackBase);
|
||||
thread->KernelStackBase = (void *)0;
|
||||
}
|
||||
|
||||
if (thread->Teb && thread->TebSize) {
|
||||
MmFreeUserMemory(thread->Teb);
|
||||
thread->Teb = (void*)0;
|
||||
thread->TebSize = 0;
|
||||
}
|
||||
|
||||
if (thread->UserStackBase) {
|
||||
MmFreeUserMemory(thread->UserStackBase);
|
||||
thread->UserStackBase = (void *)0;
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
||||
@@ -119,8 +122,14 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
|
||||
thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
|
||||
thread->TssRsp0 = thread->KernelStackTop;
|
||||
|
||||
void* ustack = MmAllocateUserMemory(USTACK_SIZE);
|
||||
if (!ustack) {
|
||||
KSTATUS s;
|
||||
void* ustack = (void*)0;
|
||||
s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
||||
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
|
||||
UserRegionStack,
|
||||
&ustack);
|
||||
if (!KSUCCESS(s) || !ustack) {
|
||||
MmFreePool(kstack);
|
||||
thread->KernelStackBase = (void*)0;
|
||||
ObDereferenceObject(&thread->Header);
|
||||
@@ -128,14 +137,14 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
|
||||
}
|
||||
thread->UserStackBase = ustack;
|
||||
// Ring-3 starts at the image entry via IRET, not CALL, so there is no
|
||||
// pushed return address. The x64 ABI expects RSP % 16 == 8 at function
|
||||
// pushed return address. The x64 ABI expects RSP % 16 == 8 at function
|
||||
// entry; bias the initial stack as if a call had happened so compiler
|
||||
// generated aligned SSE spills (movaps) are safe.
|
||||
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8;
|
||||
|
||||
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemory(sizeof(SYS_TEB));
|
||||
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
|
||||
if (!teb) {
|
||||
MmFreeUserMemory(ustack);
|
||||
MmFreeUserMemoryForProcess(owner, ustack);
|
||||
thread->UserStackBase = (void*)0;
|
||||
MmFreePool(kstack);
|
||||
thread->KernelStackBase = (void*)0;
|
||||
@@ -177,12 +186,12 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
|
||||
// Allocate the exit event before the thread becomes visible. Manual-reset
|
||||
// so multiple joiners (if any) all wake up when the thread exits.
|
||||
EVENT* exitEvent;
|
||||
KSTATUS s = KeEventCreate(EventManualReset, 0, &exitEvent);
|
||||
s = KeEventCreate(EventManualReset, 0, &exitEvent);
|
||||
if (!KSUCCESS(s)) {
|
||||
if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb);
|
||||
if (thread->Teb && thread->TebSize) MmFreeUserMemoryForProcess(owner, thread->Teb);
|
||||
thread->Teb = (void*)0;
|
||||
thread->TebSize = 0;
|
||||
MmFreeUserMemory(ustack);
|
||||
MmFreeUserMemoryForProcess(owner, ustack);
|
||||
thread->UserStackBase = (void*)0;
|
||||
MmFreePool(kstack);
|
||||
thread->KernelStackBase = (void*)0;
|
||||
@@ -238,7 +247,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
|
||||
THREAD_PRIORITY priority,
|
||||
uint64_t affinityMask,
|
||||
THREAD** out) {
|
||||
// Kernel worker threads are owned by the system process. The scheduler's
|
||||
// Kernel worker threads are owned by the system process. The scheduler's
|
||||
// hand-built CPU idle thread stays attached to PID 0 (idle), while driver,
|
||||
// async I/O, and init worker threads are grouped under PID 1 (system).
|
||||
PROCESS* owner = KeGetSystemProcess();
|
||||
@@ -331,7 +340,7 @@ __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uin
|
||||
|
||||
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
|
||||
// Called with the synchronization object's spinlock held, so the caller is
|
||||
// normally at DISPATCH_LEVEL here. This function only marks intent to
|
||||
// normally at DISPATCH_LEVEL here. This function only marks intent to
|
||||
// block; the actual scheduling point is KeCommitBlockThread at PASSIVE.
|
||||
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
|
||||
if (!thread) return;
|
||||
|
||||
@@ -56,7 +56,7 @@ KTRAP_FRAME* KeGetCurrentTrapFrame(void) {
|
||||
|
||||
void KiEnterTrapFrame(KTRAP_FRAME* frame) {
|
||||
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
|
||||
KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame);
|
||||
KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
|
||||
|
||||
THREAD* t = KeGetCurrentThread();
|
||||
if (!t) return;
|
||||
@@ -68,13 +68,13 @@ void KiEnterTrapFrame(KTRAP_FRAME* frame) {
|
||||
|
||||
void KiExitTrapFrame(KTRAP_FRAME* frame) {
|
||||
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
|
||||
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame);
|
||||
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
|
||||
|
||||
// Give alertable user-mode returns a chance to redirect through the
|
||||
// registered user APC dispatcher before committing IRET slots.
|
||||
KiCheckForUserApcDelivery(frame);
|
||||
|
||||
// Commit editable return state. Kernel-mode interrupts have no RSP/SS
|
||||
// Commit editable return state. Kernel-mode interrupts have no RSP/SS
|
||||
// hardware slots, so those pointers are NULL in that case.
|
||||
if (frame->RipSlot) *frame->RipSlot = frame->Rip;
|
||||
if (frame->CsSlot) *frame->CsSlot = frame->Cs;
|
||||
@@ -85,7 +85,7 @@ void KiExitTrapFrame(KTRAP_FRAME* frame) {
|
||||
THREAD* t = KeGetCurrentThread();
|
||||
if (t) {
|
||||
if (t->TrapFrame != frame)
|
||||
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: trap-frame stack mismatch top=0x%llX frame=0x%llX\n",
|
||||
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: trap-frame stack mismatch top=0x%016llX frame=0x%016llX\n",
|
||||
(uint64_t)t->TrapFrame, (uint64_t)frame);
|
||||
t->TrapFrame = frame->PreviousFrame;
|
||||
}
|
||||
@@ -98,11 +98,11 @@ void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
|
||||
if (!f)
|
||||
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: no current trap frame for nr=%llu\n", nr);
|
||||
if (f->Magic != KTRAP_FRAME_MAGIC)
|
||||
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: bad magic 0x%llX\n", f->Magic);
|
||||
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: bad magic 0x%016llX\n", f->Magic);
|
||||
if (f->Kind != KTRAP_KIND_SYSCALL)
|
||||
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: kind=%u, expected syscall\n", f->Kind);
|
||||
if (!KeTrapFrameFromUser(f) || f->PreviousMode != KTRAP_MODE_USER)
|
||||
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: syscall did not come from user cs=0x%llX mode=%u\n",
|
||||
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: syscall did not come from user cs=0x%016llX mode=%u\n",
|
||||
f->Cs, f->PreviousMode);
|
||||
if (f->Rax != nr || f->Rcx != a0 || f->Rdx != a1 || f->R8 != a2 ||
|
||||
f->R9 != a3 || f->R10 != a4 || f->R11 != a5) {
|
||||
|
||||
+10
-73
@@ -2,6 +2,8 @@
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_disable)
|
||||
#pragma intrinsic(_enable)
|
||||
#include <ntlite/boot_info.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
@@ -22,6 +24,7 @@
|
||||
#include <kernel/io_file.h>
|
||||
#include <kernel/ke_selftest.h>
|
||||
#include <kernel/ke_smp.h>
|
||||
#include <kernel/ke_input.h>
|
||||
#include <common/ini_parser.h>
|
||||
|
||||
typedef struct {
|
||||
@@ -31,52 +34,9 @@ typedef struct {
|
||||
|
||||
typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *);
|
||||
|
||||
// HAL pointer cached at boot so kernel threads can reach it without
|
||||
// threading a parameter through every call chain.
|
||||
// HAL pointer cached at boot for subsystems that still need platform services.
|
||||
static const HAL_API* gKmainHal;
|
||||
|
||||
// Kernel-owned DPC wrappers for HAL input devices. The HAL keeps the tiny
|
||||
// hardware-specific ISR fast and exposes a bottom-half entry point; the kernel
|
||||
// owns the actual DPC objects and scheduling policy.
|
||||
static KDPC gKeyboardDpc;
|
||||
static KDPC gMouseDpc;
|
||||
|
||||
static void KeyboardDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
|
||||
(void)dpc;
|
||||
(void)context;
|
||||
(void)systemArgument1;
|
||||
(void)systemArgument2;
|
||||
|
||||
if (gKmainHal && gKmainHal->Keyboard.Dpc)
|
||||
gKmainHal->Keyboard.Dpc();
|
||||
}
|
||||
|
||||
static void KiMouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
|
||||
(void)dpc;
|
||||
(void)context;
|
||||
(void)systemArgument1;
|
||||
(void)systemArgument2;
|
||||
|
||||
if (gKmainHal && gKmainHal->Mouse.Dpc)
|
||||
gKmainHal->Mouse.Dpc();
|
||||
}
|
||||
|
||||
static void KeyboardIrqWrapper(void) {
|
||||
if (gKmainHal && gKmainHal->Keyboard.Isr)
|
||||
gKmainHal->Keyboard.Isr();
|
||||
|
||||
if (gKmainHal && gKmainHal->Keyboard.Dpc)
|
||||
KeInsertQueueDpc(&gKeyboardDpc, (void*)0, (void*)0);
|
||||
}
|
||||
|
||||
static void KiMouseIrqWrapper(void) {
|
||||
if (gKmainHal && gKmainHal->Mouse.Isr)
|
||||
gKmainHal->Mouse.Isr();
|
||||
|
||||
if (gKmainHal && gKmainHal->Mouse.Dpc)
|
||||
KeInsertQueueDpc(&gMouseDpc, (void*)0, (void*)0);
|
||||
}
|
||||
|
||||
static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
|
||||
for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) {
|
||||
const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i];
|
||||
@@ -288,17 +248,14 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
|
||||
// Read the executable file into a raw buffer.
|
||||
s = IoOpenFile(SysSvFromCstr(filePath), &f);
|
||||
DbgPrint("KiLaunchUserProcess: IoOpenFile: %d\n", s);
|
||||
if (s != STATUS_SUCCESS) {
|
||||
DbgPrint("Failed to load file: \"%s\"!\n", filePath);
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t size = IoGetFileSize(f);
|
||||
DbgPrint("KiLaunchUserProcess: IoGetFileSize: %llu\n", size);
|
||||
|
||||
void* mem = MmAllocatePool(size);
|
||||
DbgPrint("KiLaunchUserProcess: MmAllocatePool: 0x%llX\n", mem);
|
||||
if (!mem) {
|
||||
IoCloseFile(f);
|
||||
return 0;
|
||||
@@ -306,7 +263,6 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
|
||||
size_t got = 0;
|
||||
s = IoReadFile(f, mem, size, &got);
|
||||
DbgPrint("KiLaunchUserProcess: IoReadFile: %d\n", s);
|
||||
IoCloseFile(f);
|
||||
if (!KSUCCESS(s)) {
|
||||
MmFreePool(mem);
|
||||
@@ -338,7 +294,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
|
||||
// Parse the raw PE file and map it into executable memory.
|
||||
uintptr_t imageBase = 0;
|
||||
s = LdrLoadImage(mem, size, &imageBase, true);
|
||||
s = LdrLoadImageForProcess(proc, mem, size, &imageBase);
|
||||
MmFreePool(mem);
|
||||
if (!KSUCCESS(s)) {
|
||||
DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s);
|
||||
@@ -349,16 +305,14 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
s = LdrResolveImportsForProcess(proc, imageBase);
|
||||
if (!KSUCCESS(s)) {
|
||||
DbgPrint("KiLaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s);
|
||||
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return 0;
|
||||
}
|
||||
|
||||
proc->ImageBase = imageBase;
|
||||
proc->ImageSize = LdrGetImageSize(imageBase);
|
||||
proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
|
||||
|
||||
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
|
||||
DbgPrint("KiLaunchUserProcess: PE Image mapped! Base 0x%llx, Entry 0x%llX\n", imageBase, entryPoint);
|
||||
|
||||
// Create the main thread last. KeCreateThread queues it before returning.
|
||||
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread);
|
||||
@@ -366,7 +320,6 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
|
||||
proc->ImageBase = 0;
|
||||
proc->ImageSize = 0;
|
||||
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return 0;
|
||||
}
|
||||
@@ -388,19 +341,6 @@ void SystemInitThread(void* arg) {
|
||||
KiResolveSystemImports(args->bootInfo);
|
||||
KiReclaimBootloaderMemory(args->bootInfo);
|
||||
|
||||
args->hal->Screen.Clear(0x000000);
|
||||
|
||||
// Print memory status now that the system is fully alive
|
||||
DbgPrint(
|
||||
"Memory Status:\n"
|
||||
" -> Free (PMM): %.2f MB\n"
|
||||
" -> Kernel Pool: %.2f MB\n"
|
||||
" -> User Heap: %.2f MB\n",
|
||||
(float)MmFreeFrameCount() * PAGE_SIZE / (1024.0f * 1024.0f),
|
||||
(float)MmGetKernelPoolUsage() / (1024.0f * 1024.0f),
|
||||
(float)MmGetUserHeapUsage() / (1024.0f * 1024.0f)
|
||||
);
|
||||
|
||||
if (args->bootInfo->Debug & 2)
|
||||
KiRunKernelSelfTests(args->hal);
|
||||
|
||||
@@ -464,7 +404,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
|
||||
.FramebufferHeight = bootInfo->FramebufferHeight,
|
||||
.FramebufferStride = bootInfo->FramebufferStride,
|
||||
.RsdpPhysical = bootInfo->RsdpPhysical,
|
||||
.Debug = bootInfo->Debug
|
||||
.Debug = bootInfo->Debug,
|
||||
.AllocatePool = MmAllocatePool
|
||||
};
|
||||
|
||||
KSTATUS s = HalInitialize(&halParams);
|
||||
@@ -482,6 +423,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
|
||||
// the kernel owns KCPU records, per-CPU stacks, GDT/TSS/IDT state, and the
|
||||
// AP park loop. External device IRQs remain routed to the BSP during this
|
||||
// step; APs only become runnable after scheduler admission below.
|
||||
KeSetApplicationProcessorTrampoline((uint64_t)bootInfo->ApTrampolinePhys);
|
||||
KSTATUS smpStatus = KeStartApplicationProcessors();
|
||||
if (!KSUCCESS(smpStatus)) {
|
||||
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to start application processors, status=0x%x\n", smpStatus);
|
||||
@@ -490,14 +432,9 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
|
||||
KeIrqInitialize(hal);
|
||||
KeInitializeDpcSystem();
|
||||
|
||||
KeInitializeDpc(&gKeyboardDpc, KeyboardDpcRoutine, (void*)0);
|
||||
KeInitializeDpc(&gMouseDpc, KiMouseDpcRoutine, (void*)0);
|
||||
KeInputInitialize(hal);
|
||||
|
||||
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr);
|
||||
KeIrqRegister(hal->Keyboard.IrqNumber,
|
||||
hal->Keyboard.Dpc ? KeyboardIrqWrapper : hal->Keyboard.Isr);
|
||||
KeIrqRegister(hal->Mouse.IrqNumber,
|
||||
hal->Mouse.Dpc ? KiMouseIrqWrapper : hal->Mouse.Isr);
|
||||
|
||||
HalRegisterScheduler(KeSchedulerTick);
|
||||
|
||||
|
||||
+157
-21
@@ -8,6 +8,16 @@
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
|
||||
#ifndef LDR_DEBUG_VERBOSE
|
||||
#define LDR_DEBUG_VERBOSE 0
|
||||
#endif
|
||||
|
||||
#if LDR_DEBUG_VERBOSE
|
||||
#define LdrDbg(...) DbgPrint(__VA_ARGS__)
|
||||
#else
|
||||
#define LdrDbg(...) ((void)0)
|
||||
#endif
|
||||
|
||||
int _stricmp(const char* a, const char* b);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -78,6 +88,42 @@ typedef struct {
|
||||
uint32_t Characteristics;
|
||||
} SECTION_HEADER;
|
||||
|
||||
static size_t KiAlignPeSize(size_t value) {
|
||||
return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
||||
}
|
||||
|
||||
static KSTATUS KiComputeRequiredImageSize(const PE_HEADER* pe, size_t fileSize, size_t* outSize) {
|
||||
if (!pe || !outSize) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
const OPT_HEADER64* opt = &pe->OptionalHeader;
|
||||
size_t required = KiAlignPeSize(opt->SizeOfImage);
|
||||
size_t headers = KiAlignPeSize(opt->SizeOfHeaders);
|
||||
if (headers > required) required = headers;
|
||||
|
||||
const SECTION_HEADER* sec = (const SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
|
||||
for (int i = 0; i < pe->NumberOfSections; i++) {
|
||||
size_t va = (size_t)sec[i].VirtualAddress;
|
||||
size_t virt = (size_t)sec[i].VirtualSize;
|
||||
size_t raw = (size_t)sec[i].SizeOfRawData;
|
||||
size_t span = virt ? virt : raw;
|
||||
if (raw > span) span = raw;
|
||||
if (!span) continue;
|
||||
|
||||
if (va > (size_t)-1 - span) return STATUS_NO_MEMORY;
|
||||
size_t end = KiAlignPeSize(va + span);
|
||||
if (end > required) required = end;
|
||||
|
||||
if (raw) {
|
||||
size_t rawBase = (size_t)sec[i].PointerToRawData;
|
||||
if (rawBase > fileSize || raw > fileSize - rawBase) return STATUS_FAILURE;
|
||||
}
|
||||
}
|
||||
|
||||
if (!required) return STATUS_FAILURE;
|
||||
*outSize = required;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
uint32_t Characteristics;
|
||||
uint32_t TimeDateStamp;
|
||||
@@ -246,33 +292,77 @@ void* LdrGetExportForProcess(PROCESS* process, const char* moduleName, const cha
|
||||
}
|
||||
|
||||
size_t LdrGetImageSize(uintptr_t base) {
|
||||
if (!base) return 0;
|
||||
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
|
||||
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
|
||||
if (pe->Signature != PE_SIGNATURE) return 0;
|
||||
return (size_t)pe->OptionalHeader.SizeOfImage;
|
||||
}
|
||||
|
||||
|
||||
static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
|
||||
if (!imageBase) return;
|
||||
size_t imageSize = LdrGetImageSize(imageBase);
|
||||
if (user) MmFreeUserImage((void*)imageBase, imageSize);
|
||||
else MmFreePool((void*)imageBase);
|
||||
size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
|
||||
if (!base) return 0;
|
||||
if (process) {
|
||||
size_t regionSize = MmGetUserRegionSize(process, (const void*)base);
|
||||
if (regionSize) return regionSize;
|
||||
}
|
||||
size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base);
|
||||
if (globalRegionSize) return globalRegionSize;
|
||||
return LdrGetImageSize(base);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) {
|
||||
static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
|
||||
if (!imageBase) return;
|
||||
size_t imageSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase);
|
||||
if (user) MmFreeUserImageForProcess(process, (void*)imageBase, imageSize);
|
||||
else MmFreePool((void*)imageBase);
|
||||
}
|
||||
|
||||
static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
|
||||
KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user);
|
||||
}
|
||||
|
||||
|
||||
KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
|
||||
if (!imageBase) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
size_t mappedSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase);
|
||||
if (!mappedSize) return STATUS_FAILURE;
|
||||
|
||||
uint32_t peOffset = *(uint32_t *)(imageBase + 0x3C);
|
||||
if ((size_t)peOffset > mappedSize || mappedSize - (size_t)peOffset < sizeof(uint32_t) + 20)
|
||||
return STATUS_FAILURE;
|
||||
|
||||
PE_HEADER* pe = (PE_HEADER *)(imageBase + peOffset);
|
||||
if (pe->Signature != PE_SIGNATURE) return STATUS_FAILURE;
|
||||
|
||||
SECTION_HEADER* sec = (SECTION_HEADER *)((uint8_t *)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
|
||||
size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader;
|
||||
size_t sectionTableSize = (size_t)pe->NumberOfSections * sizeof(SECTION_HEADER);
|
||||
if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset)
|
||||
return STATUS_FAILURE;
|
||||
|
||||
SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset);
|
||||
for (int i = 0; i < pe->NumberOfSections; i++) {
|
||||
size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData;
|
||||
if (!protectSize) continue;
|
||||
|
||||
uintptr_t secBase = imageBase + (uintptr_t)sec[i].VirtualAddress;
|
||||
uintptr_t secEnd = secBase + protectSize;
|
||||
uintptr_t mapEnd = imageBase + mappedSize;
|
||||
|
||||
if (secBase < imageBase || secBase >= mapEnd) {
|
||||
LdrDbg("LDR: protect skipping out-of-range section %d base=%016llx size=%llx image=%016llx mapped=%llx\n",
|
||||
i, (unsigned long long)secBase, (unsigned long long)protectSize,
|
||||
(unsigned long long)imageBase, (unsigned long long)mappedSize);
|
||||
continue;
|
||||
}
|
||||
if (secEnd < secBase) return STATUS_NO_MEMORY;
|
||||
if (secEnd > mapEnd) {
|
||||
LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n",
|
||||
i, (unsigned long long)secEnd, (unsigned long long)mapEnd);
|
||||
protectSize = (size_t)(mapEnd - secBase);
|
||||
}
|
||||
|
||||
uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
|
||||
if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE)
|
||||
prot |= PAGE_WRITE;
|
||||
@@ -280,12 +370,27 @@ KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) {
|
||||
(sec[i].Characteristics & IMAGE_SCN_CNT_CODE)))
|
||||
prot |= PAGE_NX;
|
||||
|
||||
KSTATUS s = MmSetPageProtect(imageBase + sec[i].VirtualAddress, protectSize, prot);
|
||||
if (!KSUCCESS(s)) return s;
|
||||
KSTATUS s;
|
||||
if (user) {
|
||||
s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot);
|
||||
} else {
|
||||
s = MmSetPageProtect(secBase, protectSize, prot);
|
||||
}
|
||||
if (!KSUCCESS(s)) {
|
||||
LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
|
||||
i, (unsigned long long)secBase, (unsigned long long)protectSize,
|
||||
(unsigned long long)prot, s,
|
||||
(unsigned long long)imageBase, (unsigned long long)mappedSize);
|
||||
return s;
|
||||
}
|
||||
}
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) {
|
||||
return LdrProtectImageForProcess(user ? (PROCESS*)0 : (PROCESS*)0, imageBase, user);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Entry point lookup
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -368,6 +473,7 @@ static KSTATUS LdrResolveImportsCommon(
|
||||
|
||||
//Compatibility remap.
|
||||
dllName = KiCanonicalModuleName(dllName);
|
||||
LdrDbg("LDR: resolve import dll=%s image=%016llx pid=%u\n", dllName, (unsigned long long)imageBase, process ? process->ProcessId : 0);
|
||||
|
||||
//
|
||||
// Ensure dependency is loaded into the process
|
||||
@@ -469,7 +575,7 @@ static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uint
|
||||
strncpy(mod->Name, name, 31);
|
||||
mod->Name[31] = '\0';
|
||||
mod->Base = base;
|
||||
mod->Size = LdrGetImageSize(base);
|
||||
mod->Size = LdrGetMappedImageSize(process, base);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
@@ -500,6 +606,7 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
|
||||
}
|
||||
|
||||
uint64_t fileSize = IoGetFileSize(f);
|
||||
LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize);
|
||||
void* mem = MmAllocatePool((size_t)fileSize);
|
||||
if (!mem) {
|
||||
IoCloseFile(f);
|
||||
@@ -515,39 +622,45 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
|
||||
}
|
||||
|
||||
uintptr_t imageBase = 0;
|
||||
s = LdrLoadImage(mem, (size_t)fileSize, &imageBase, true);
|
||||
s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase);
|
||||
MmFreePool(mem);
|
||||
if (!KSUCCESS(s)) {
|
||||
DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s);
|
||||
return s;
|
||||
}
|
||||
LdrDbg("LDR: module mapped pid=%u file=%s base=%016llx size=%llx\n",
|
||||
process->ProcessId, filename, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(process, imageBase));
|
||||
|
||||
|
||||
s = LdrResolveImportsForProcess(process, imageBase);
|
||||
if (!KSUCCESS(s)) {
|
||||
KiFreeLoadedImage(imageBase, true);
|
||||
KiFreeLoadedImageForProcess(process, imageBase, true);
|
||||
return s;
|
||||
}
|
||||
|
||||
s = LdrProtectImage(imageBase, true);
|
||||
s = LdrProtectImageForProcess(process, imageBase, true);
|
||||
if (!KSUCCESS(s)) {
|
||||
KiFreeLoadedImage(imageBase, true);
|
||||
KiFreeLoadedImageForProcess(process, imageBase, true);
|
||||
return s;
|
||||
}
|
||||
|
||||
|
||||
s = LdrRegisterProcessModule(process, filename, imageBase);
|
||||
if (!KSUCCESS(s)) {
|
||||
KiFreeLoadedImage(imageBase, true);
|
||||
KiFreeLoadedImageForProcess(process, imageBase, true);
|
||||
return s;
|
||||
}
|
||||
|
||||
LdrDbg("LDR: module registered pid=%u name=%s base=%016llx modules=%u\n",
|
||||
process->ProcessId, filename, (unsigned long long)imageBase, process->ModuleCount);
|
||||
if (baseOut) *baseOut = imageBase;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// LdrLoadImage - load a PE from a raw buffer into virtual space.
|
||||
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
|
||||
static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
|
||||
if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER;
|
||||
if (user && !process) return STATUS_INVALID_PARAMETER;
|
||||
if (size < 0x40) return STATUS_FAILURE;
|
||||
|
||||
const uint8_t* bytes = (const uint8_t*)buffer;
|
||||
@@ -579,10 +692,23 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u
|
||||
|
||||
PE_HEADER* pe = (PE_HEADER *)(bytes + peOffset);
|
||||
OPT_HEADER64* opt = &pe->OptionalHeader;
|
||||
size_t imageSize = opt->SizeOfImage;
|
||||
size_t imageSize = 0;
|
||||
KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize);
|
||||
if (!KSUCCESS(sizeStatus)) return sizeStatus;
|
||||
|
||||
void* mem = user ? MmAllocateUserImage(imageSize) : MmAllocatePool(imageSize);
|
||||
if (!mem) return STATUS_NO_MEMORY;
|
||||
LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n",
|
||||
opt->SizeOfImage,
|
||||
(unsigned long long)imageSize,
|
||||
(unsigned long long)opt->ImageBase,
|
||||
pe->NumberOfSections,
|
||||
user ? 1 : 0,
|
||||
process ? process->ProcessId : 0);
|
||||
void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize);
|
||||
if (!mem) {
|
||||
DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n",
|
||||
(unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0);
|
||||
return STATUS_NO_MEMORY;
|
||||
}
|
||||
memset(mem, 0, imageSize);
|
||||
|
||||
memcpy(mem, buffer, opt->SizeOfHeaders);
|
||||
@@ -591,7 +717,9 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u
|
||||
void* dst = (uint8_t *)mem + sec[i].VirtualAddress;
|
||||
const void* src = (const uint8_t *)buffer + sec[i].PointerToRawData;
|
||||
if (sec[i].SizeOfRawData) {
|
||||
memcpy(dst, src, sec[i].SizeOfRawData < sec[i].VirtualSize ? sec[i].SizeOfRawData : sec[i].VirtualSize);
|
||||
size_t copySize = sec[i].SizeOfRawData;
|
||||
if (sec[i].VirtualSize && sec[i].VirtualSize < copySize) copySize = sec[i].VirtualSize;
|
||||
memcpy(dst, src, copySize);
|
||||
}
|
||||
|
||||
uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0);
|
||||
@@ -608,3 +736,11 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u
|
||||
*baseOut = (uintptr_t)mem;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) {
|
||||
return KiLdrLoadImageCommon(process, buffer, size, baseOut, true);
|
||||
}
|
||||
|
||||
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
|
||||
return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user);
|
||||
}
|
||||
+29
-8
@@ -101,27 +101,48 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) {
|
||||
if (maxExclusive <= PAGE_SIZE) return (PHYSICAL_ADDRESS)0;
|
||||
static PHYSICAL_ADDRESS KiPopFrameAtIndexLocked(size_t idx) {
|
||||
PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx];
|
||||
size_t tail = gFrameStack.Top - idx - 1;
|
||||
for (size_t j = 0; j < tail; j++)
|
||||
gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j];
|
||||
gFrameStack.Top--;
|
||||
return frame;
|
||||
}
|
||||
|
||||
PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
|
||||
PHYSICAL_ADDRESS maxExclusive) {
|
||||
minInclusive = (minInclusive + PAGE_SIZE - 1) & PAGE_MASK;
|
||||
maxExclusive &= PAGE_MASK;
|
||||
|
||||
if (maxExclusive <= minInclusive)
|
||||
return (PHYSICAL_ADDRESS)0;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
|
||||
|
||||
// Search high-to-low within the requested window. For low-memory users
|
||||
// such as the AP trampoline this naturally prefers the traditional
|
||||
// 0x70000-0x9f000 area and avoids consuming lower real-mode space first.
|
||||
for (size_t i = gFrameStack.Top; i > 0; i--) {
|
||||
size_t idx = i - 1;
|
||||
PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx];
|
||||
if (frame && frame < maxExclusive && !(frame & (PAGE_SIZE - 1))) {
|
||||
size_t tail = gFrameStack.Top - idx - 1;
|
||||
for (size_t j = 0; j < tail; j++)
|
||||
gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j];
|
||||
gFrameStack.Top--;
|
||||
if (frame >= minInclusive && frame < maxExclusive &&
|
||||
!(frame & (PAGE_SIZE - 1))) {
|
||||
frame = KiPopFrameAtIndexLocked(idx);
|
||||
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||
return frame;
|
||||
}
|
||||
}
|
||||
|
||||
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||
return (PHYSICAL_ADDRESS)0;
|
||||
}
|
||||
|
||||
PHYSICAL_ADDRESS MmAllocateFrame(void) {
|
||||
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) {
|
||||
return MmAllocateFrameInRange(PAGE_SIZE, maxExclusive);
|
||||
}
|
||||
|
||||
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
|
||||
if (gFrameStack.Top == 0) {
|
||||
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||
|
||||
+628
-94
@@ -16,6 +16,7 @@
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_process.h>
|
||||
#include <kernel/ke_cpu.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
@@ -139,7 +140,7 @@ void MmFlushTlbForKernelMapping(uintptr_t virt) {
|
||||
|
||||
if (acks < targets) {
|
||||
KePanic(KE_PANIC_CODE_GENERIC,
|
||||
"TLB shootdown timeout: va=0x%llX targets=%ld acks=%ld selfCpu=%u\n",
|
||||
"TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n",
|
||||
(uint64_t)virt, targets, acks, selfIndex);
|
||||
}
|
||||
}
|
||||
@@ -506,6 +507,35 @@ typedef struct HEAP {
|
||||
SPINLOCK Lock;
|
||||
} HEAP;
|
||||
|
||||
static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) {
|
||||
if (!h || !size) return 1;
|
||||
if (start < h->Start) return 0;
|
||||
if (start > (uintptr_t)-1 - (size - 1)) return 0;
|
||||
|
||||
uintptr_t begin = start & PAGE_MASK;
|
||||
uintptr_t end = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
|
||||
if (end < begin || end > h->Limit) return 0;
|
||||
|
||||
for (uintptr_t va = begin; va < end; va += PAGE_SIZE) {
|
||||
PHYSICAL_ADDRESS phys = 0;
|
||||
uint64_t flags = 0;
|
||||
KSTATUS qs = MmQueryPage(va, &phys, &flags);
|
||||
if (KSUCCESS(qs) && (flags & PAGE_PRESENT))
|
||||
continue;
|
||||
|
||||
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
||||
if (!frame) return 0;
|
||||
KSTATUS ms = MmMapPage(va, frame, h->MapFlags);
|
||||
if (!KSUCCESS(ms)) {
|
||||
MmFreeFrame(frame);
|
||||
return 0;
|
||||
}
|
||||
memset((void*)va, 0, PAGE_SIZE);
|
||||
DbgPrint("MM: repaired missing heap page va=%016llx\n", (unsigned long long)va);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
// Extend heap by at least 'need' bytes (rounded up to page boundary).
|
||||
static int KiHeapGrow(HEAP* h, size_t need) {
|
||||
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
||||
@@ -521,6 +551,9 @@ static int KiHeapGrow(HEAP* h, size_t need) {
|
||||
memset((void *)va, 0, PAGE_SIZE);
|
||||
}
|
||||
|
||||
if (!KiHeapEnsureMapped(h, h->Brk, grow))
|
||||
return 0;
|
||||
|
||||
// Make the new pages a single free block.
|
||||
BLOCK_HDR* blk = (BLOCK_HDR *)h->Brk;
|
||||
blk->Magic = BLOCK_MAGIC;
|
||||
@@ -547,34 +580,18 @@ static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
|
||||
}
|
||||
|
||||
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
|
||||
(void)h;
|
||||
if (!blk || blk->Free == 0) return;
|
||||
|
||||
while (h->Brk > h->Start) {
|
||||
uintptr_t blockEnd = (uintptr_t)blk + HDR_SIZE + blk->Size + FTR_SIZE;
|
||||
if (blockEnd != h->Brk) break;
|
||||
|
||||
// Keep at least one page for the block header/footer and a small payload.
|
||||
if (blk->Size < PAGE_SIZE + sizeof(void*)) break;
|
||||
|
||||
uintptr_t pageStart = h->Brk - PAGE_SIZE;
|
||||
if (pageStart <= (uintptr_t)blk) break; // would unmap the page containing the header
|
||||
|
||||
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(pageStart);
|
||||
if (!phys) break;
|
||||
if (!KSUCCESS(MmUnmapPage(pageStart))) {
|
||||
break;
|
||||
}
|
||||
MmFreeFrame(phys);
|
||||
|
||||
h->Brk -= PAGE_SIZE;
|
||||
blk->Size -= PAGE_SIZE;
|
||||
}
|
||||
|
||||
if (blk->Free) {
|
||||
KiHdrToFtr(blk)->Size = blk->Size;
|
||||
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
||||
KiHdrToFtr(blk)->Free = 1;
|
||||
}
|
||||
// Do not decommit kernel-pool tail pages for now. The pool allocator keeps
|
||||
// free-list nodes and boundary tags inside the managed arena; shrinking the
|
||||
// mapped tail while large launch-failure cleanup is walking/freeing many
|
||||
// objects can leave stale metadata pointers into unmapped pages. Physical
|
||||
// pages are still reclaimed for user regions by MmDestroyUserRegion(); pool
|
||||
// pages remain committed until we have a dedicated, audited heap shrinker.
|
||||
KiHdrToFtr(blk)->Size = blk->Size;
|
||||
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
||||
KiHdrToFtr(blk)->Free = 1;
|
||||
}
|
||||
|
||||
static void* KiHeapAlloc(HEAP* h, size_t size) {
|
||||
@@ -583,6 +600,11 @@ static void* KiHeapAlloc(HEAP* h, size_t size) {
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
|
||||
|
||||
if (h->Brk > h->Start && !KiHeapEnsureMapped(h, h->Start, h->Brk - h->Start)) {
|
||||
KeReleaseSpinlock(&h->Lock, flags);
|
||||
return (void*)0;
|
||||
}
|
||||
|
||||
// Best-fit search.
|
||||
BLOCK_HDR* best = (BLOCK_HDR *)0;
|
||||
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
|
||||
@@ -735,12 +757,29 @@ static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
|
||||
static HEAP gPoolHeap;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// User heap
|
||||
// User virtual region arena
|
||||
// ---------------------------------------------------------------------------
|
||||
static HEAP gUserHeap;
|
||||
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
|
||||
static SPINLOCK gUserImageLock = SPINLOCK_INIT;
|
||||
static uintptr_t gUserImageNext = USER_LIMIT;
|
||||
typedef struct USER_VA_REGION {
|
||||
LIST_ENTRY ProcessLink;
|
||||
LIST_ENTRY GlobalLink;
|
||||
PROCESS* Owner;
|
||||
|
||||
uintptr_t ReservationBase;
|
||||
size_t ReservationSize;
|
||||
uintptr_t Base;
|
||||
size_t Size;
|
||||
size_t CommittedSize;
|
||||
|
||||
uint64_t PageFlags;
|
||||
USER_REGION_TYPE Type;
|
||||
uint32_t Flags;
|
||||
} USER_VA_REGION;
|
||||
|
||||
static LIST_ENTRY gUserRegionList;
|
||||
static SPINLOCK gUserRegionLock = SPINLOCK_INIT;
|
||||
static int gUserRegionListReady = 0;
|
||||
|
||||
#define USER_REGION_ARENA_BASE 0x0000100000000000ULL
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Module init
|
||||
@@ -763,20 +802,11 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
|
||||
KiHeapGrow(&gPoolHeap, poolSize);
|
||||
}
|
||||
|
||||
// User heap.
|
||||
gUserHeap.FreeList = (BLOCK_HDR *)0;
|
||||
gUserHeap.Start = USER_BASE;
|
||||
gUserHeap.Brk = USER_BASE;
|
||||
gUserHeap.Limit = USER_LIMIT;
|
||||
gUserHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX;
|
||||
gUserHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
|
||||
|
||||
// Executable image mappings grow downward from the top of user space.
|
||||
// The regular user heap grows upward from USER_BASE. Keeping them in
|
||||
// separate page-aligned regions prevents W^X section protection from
|
||||
// changing heap metadata or adjacent allocations.
|
||||
gUserImageLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
gUserImageNext = USER_LIMIT;
|
||||
// User VA arena. One global lower-half virtual-address bucket, with
|
||||
// per-process ownership tracked by USER_VA_REGION descriptors.
|
||||
LIST_INIT(&gUserRegionList);
|
||||
gUserRegionLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
gUserRegionListReady = 1;
|
||||
}
|
||||
|
||||
void MmActivate(void) {
|
||||
@@ -791,69 +821,86 @@ __declspec(dllexport) void MmFreePool(void* ptr) { KiHeap
|
||||
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public user heap API
|
||||
// User virtual region API
|
||||
// ---------------------------------------------------------------------------
|
||||
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
return KiHeapAlloc(&gUserHeap, size);
|
||||
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
|
||||
if (align < PAGE_SIZE) align = PAGE_SIZE;
|
||||
if (align & (align - 1)) return 0;
|
||||
return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
KiHeapFree(&gUserHeap, ptr);
|
||||
static inline size_t KiAlignUpSize(size_t v, size_t align) {
|
||||
if (align < PAGE_SIZE) align = PAGE_SIZE;
|
||||
if (align & (align - 1)) return 0;
|
||||
return (v + align - 1) & ~(align - 1);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
return KiHeapRealloc(&gUserHeap, ptr, newSize);
|
||||
static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) {
|
||||
uintptr_t aEnd = aBase + aSize;
|
||||
uintptr_t bEnd = bBase + bSize;
|
||||
return aBase < bEnd && bBase < aEnd;
|
||||
}
|
||||
|
||||
static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) {
|
||||
if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER;
|
||||
if (!gUserRegionListReady) return STATUS_FAILURE;
|
||||
|
||||
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
if (!size) return (void*)0;
|
||||
uintptr_t candidate = KiAlignUpPtr(USER_REGION_ARENA_BASE, alignment);
|
||||
if (!candidate) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
||||
if (pagesSize < size) return (void*)0;
|
||||
for (;;) {
|
||||
if (candidate < USER_REGION_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate))
|
||||
return STATUS_NO_MEMORY;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gUserImageLock);
|
||||
uintptr_t next = candidate;
|
||||
int collided = 0;
|
||||
|
||||
if (pagesSize > gUserImageNext || gUserImageNext - pagesSize < gUserHeap.Brk) {
|
||||
KeReleaseSpinlock(&gUserImageLock, flags);
|
||||
return (void*)0;
|
||||
}
|
||||
|
||||
uintptr_t base = gUserImageNext - pagesSize;
|
||||
gUserImageNext = base;
|
||||
KeReleaseSpinlock(&gUserImageLock, flags);
|
||||
|
||||
for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) {
|
||||
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
||||
if (!frame) {
|
||||
MmFreeUserImage((void*)base, va - base);
|
||||
return (void*)0;
|
||||
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
||||
if (KiRangesOverlap(candidate, reservationSize, r->ReservationBase, r->ReservationSize)) {
|
||||
uintptr_t end = r->ReservationBase + r->ReservationSize;
|
||||
next = KiAlignUpPtr(end, alignment);
|
||||
if (!next || next <= candidate) return STATUS_NO_MEMORY;
|
||||
collided = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KSTATUS s = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
|
||||
if (!KSUCCESS(s)) {
|
||||
MmFreeFrame(frame);
|
||||
MmFreeUserImage((void*)base, va - base);
|
||||
return (void*)0;
|
||||
}
|
||||
}
|
||||
|
||||
memset((void*)base, 0, pagesSize);
|
||||
return (void*)base;
|
||||
if (!collided) {
|
||||
*baseOut = candidate;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
candidate = next;
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
if (!ptr || !size) return;
|
||||
static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) {
|
||||
if (!process || !base) return (USER_VA_REGION*)0;
|
||||
uintptr_t b = (uintptr_t)base;
|
||||
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
||||
if (r->Base == b) return r;
|
||||
}
|
||||
return (USER_VA_REGION*)0;
|
||||
}
|
||||
|
||||
uintptr_t base = (uintptr_t)ptr & PAGE_MASK;
|
||||
size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
||||
if (pagesSize < size) return;
|
||||
static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) {
|
||||
if (!process || !base || !size) return (USER_VA_REGION*)0;
|
||||
if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0;
|
||||
uintptr_t end = base + size;
|
||||
|
||||
for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) {
|
||||
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
||||
uintptr_t regionEnd = r->Base + r->CommittedSize;
|
||||
if (base >= r->Base && end <= regionEnd) return r;
|
||||
}
|
||||
return (USER_VA_REGION*)0;
|
||||
}
|
||||
|
||||
static void KiUnmapUserRegionPayload(USER_VA_REGION* r) {
|
||||
if (!r) return;
|
||||
uintptr_t end = r->Base + r->CommittedSize;
|
||||
for (uintptr_t va = r->Base; va < end; va += PAGE_SIZE) {
|
||||
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va);
|
||||
if (!phys) continue;
|
||||
if (KSUCCESS(MmUnmapPage(va))) {
|
||||
@@ -862,12 +909,499 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
|
||||
uint64_t pageFlags, uint32_t regionFlags,
|
||||
USER_REGION_TYPE type, void** outBase) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
if (!process || !outBase || !size) return STATUS_INVALID_PARAMETER;
|
||||
*outBase = (void*)0;
|
||||
|
||||
if (!(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER;
|
||||
if (!(pageFlags & PAGE_PRESENT)) pageFlags |= PAGE_PRESENT;
|
||||
|
||||
size_t payloadSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
|
||||
if (payloadSize < size) return STATUS_NO_MEMORY;
|
||||
|
||||
size_t guardLow = (regionFlags & USER_REGION_GUARD_LOW) ? PAGE_SIZE : 0;
|
||||
size_t guardHigh = (regionFlags & USER_REGION_GUARD_HIGH) ? PAGE_SIZE : 0;
|
||||
size_t reservationSize = guardLow + payloadSize + guardHigh;
|
||||
if (reservationSize < payloadSize) return STATUS_NO_MEMORY;
|
||||
|
||||
if (alignment < PAGE_SIZE) alignment = PAGE_SIZE;
|
||||
if (alignment & (alignment - 1)) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
USER_VA_REGION* r = (USER_VA_REGION*)MmAllocatePool(sizeof(USER_VA_REGION));
|
||||
if (!r) return STATUS_NO_MEMORY;
|
||||
memset(r, 0, sizeof(*r));
|
||||
LIST_INIT(&r->ProcessLink);
|
||||
LIST_INIT(&r->GlobalLink);
|
||||
|
||||
uintptr_t lockFlags = KeAcquireSpinlock(&gUserRegionLock);
|
||||
uintptr_t reservationBase = 0;
|
||||
KSTATUS s = KiFindUserVaRange(reservationSize, alignment, &reservationBase);
|
||||
if (KSUCCESS(s)) {
|
||||
r->Owner = process;
|
||||
r->ReservationBase = reservationBase;
|
||||
r->ReservationSize = reservationSize;
|
||||
r->Base = reservationBase + guardLow;
|
||||
r->Size = size;
|
||||
r->CommittedSize = payloadSize;
|
||||
r->PageFlags = pageFlags;
|
||||
r->Type = type;
|
||||
r->Flags = regionFlags;
|
||||
ListInsertTail(&gUserRegionList, &r->GlobalLink);
|
||||
}
|
||||
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
|
||||
|
||||
if (!KSUCCESS(s)) {
|
||||
MmFreePool(r);
|
||||
return s;
|
||||
}
|
||||
|
||||
uintptr_t va = r->Base;
|
||||
for (; va < r->Base + payloadSize; va += PAGE_SIZE) {
|
||||
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
||||
if (!frame) {
|
||||
s = STATUS_NO_MEMORY;
|
||||
break;
|
||||
}
|
||||
s = MmMapPage(va, frame, pageFlags);
|
||||
if (!KSUCCESS(s)) {
|
||||
MmFreeFrame(frame);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!KSUCCESS(s)) {
|
||||
r->CommittedSize = (va > r->Base) ? (size_t)(va - r->Base) : 0;
|
||||
KiUnmapUserRegionPayload(r);
|
||||
lockFlags = KeAcquireSpinlock(&gUserRegionLock);
|
||||
ListRemove(&r->GlobalLink);
|
||||
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
|
||||
MmFreePool(r);
|
||||
return s;
|
||||
}
|
||||
|
||||
if (regionFlags & USER_REGION_ZERO) {
|
||||
memset((void*)r->Base, 0, payloadSize);
|
||||
}
|
||||
|
||||
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
||||
ListInsertTail(&process->UserRegions, &r->ProcessLink);
|
||||
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
||||
|
||||
*outBase = (void*)r->Base;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uintptr_t start = (uintptr_t)base;
|
||||
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
|
||||
|
||||
uintptr_t alignedStart = start & PAGE_MASK;
|
||||
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
|
||||
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
|
||||
|
||||
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
||||
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
|
||||
uintptr_t regionBase = r ? r->Base : 0;
|
||||
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
|
||||
uint64_t defaultFlags = r ? r->PageFlags : 0;
|
||||
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
||||
|
||||
if (!r) {
|
||||
DbgPrint("MM: ensure user mapping failed: pid=%u range=%016llx-%016llx not inside owned committed region\n",
|
||||
process->ProcessId,
|
||||
(unsigned long long)alignedStart,
|
||||
(unsigned long long)alignedEnd);
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
uint64_t flags = pageFlags ? pageFlags : defaultFlags;
|
||||
flags |= PAGE_PRESENT | PAGE_USER;
|
||||
|
||||
uint32_t committed = 0;
|
||||
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
|
||||
if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED;
|
||||
|
||||
PHYSICAL_ADDRESS phys = 0;
|
||||
uint64_t existingFlags = 0;
|
||||
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
|
||||
if (KSUCCESS(qs) && (existingFlags & PAGE_PRESENT))
|
||||
continue;
|
||||
|
||||
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
||||
if (!frame) return STATUS_NO_MEMORY;
|
||||
|
||||
KSTATUS ms = MmMapPage(va, frame, flags);
|
||||
if (!KSUCCESS(ms)) {
|
||||
MmFreeFrame(frame);
|
||||
return ms;
|
||||
}
|
||||
memset((void*)va, 0, PAGE_SIZE);
|
||||
committed++;
|
||||
}
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uintptr_t start = (uintptr_t)base;
|
||||
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
|
||||
|
||||
uintptr_t alignedStart = start & PAGE_MASK;
|
||||
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
|
||||
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
|
||||
|
||||
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
||||
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
|
||||
uintptr_t regionBase = r ? r->Base : 0;
|
||||
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
|
||||
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
||||
|
||||
if (!r) {
|
||||
DbgPrint("MM: protect user region failed: pid=%u range=%016llx-%016llx not owned/committed\n",
|
||||
process->ProcessId,
|
||||
(unsigned long long)alignedStart,
|
||||
(unsigned long long)alignedEnd);
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
uint64_t finalFlags = pageFlags | PAGE_PRESENT | PAGE_USER;
|
||||
|
||||
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
|
||||
if (va < regionBase || va >= regionEnd) {
|
||||
DbgPrint("MM: protect user region hit guard/outside page pid=%u va=%016llx region=%016llx-%016llx\n",
|
||||
process->ProcessId,
|
||||
(unsigned long long)va,
|
||||
(unsigned long long)regionBase,
|
||||
(unsigned long long)regionEnd);
|
||||
return STATUS_ACCESS_DENIED;
|
||||
}
|
||||
|
||||
PHYSICAL_ADDRESS phys = 0;
|
||||
uint64_t existingFlags = 0;
|
||||
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
|
||||
if (!KSUCCESS(qs) || !(existingFlags & PAGE_PRESENT)) {
|
||||
PHYSICAL_ADDRESS frame = MmAllocateFrame();
|
||||
if (!frame) return STATUS_NO_MEMORY;
|
||||
KSTATUS ms = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
|
||||
if (!KSUCCESS(ms)) {
|
||||
MmFreeFrame(frame);
|
||||
return ms;
|
||||
}
|
||||
memset((void*)va, 0, PAGE_SIZE);
|
||||
DbgPrint("MM: committed missing user page during protect pid=%u va=%016llx\n",
|
||||
process->ProcessId, (unsigned long long)va);
|
||||
}
|
||||
|
||||
KSTATUS ps = MmSetPageProtect(va, PAGE_SIZE, finalFlags);
|
||||
if (!KSUCCESS(ps)) {
|
||||
PHYSICAL_ADDRESS dbgPhys = 0;
|
||||
uint64_t dbgFlags = 0;
|
||||
KSTATUS dbg = MmQueryPage(va, &dbgPhys, &dbgFlags);
|
||||
DbgPrint("MM: protect user page failed pid=%u va=%016llx st=%d query=%d phys=%016llx flags=%016llx final=%016llx\n",
|
||||
process->ProcessId,
|
||||
(unsigned long long)va,
|
||||
ps, dbg,
|
||||
(unsigned long long)dbgPhys,
|
||||
(unsigned long long)dbgFlags,
|
||||
(unsigned long long)finalFlags);
|
||||
return ps;
|
||||
}
|
||||
}
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
if (!process || !base) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
||||
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
|
||||
if (!r) {
|
||||
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
||||
return STATUS_NOT_FOUND;
|
||||
}
|
||||
ListRemove(&r->ProcessLink);
|
||||
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
||||
|
||||
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
|
||||
ListRemove(&r->GlobalLink);
|
||||
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
|
||||
|
||||
KiUnmapUserRegionPayload(r);
|
||||
MmFreePool(r);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void MmDestroyAllUserRegions(PROCESS* process) {
|
||||
KeAssertIrql(PASSIVE_LEVEL);
|
||||
if (!process) return;
|
||||
|
||||
for (;;) {
|
||||
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
||||
LIST_ENTRY* node = ListPopHead(&process->UserRegions);
|
||||
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
||||
if (!node) break;
|
||||
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
||||
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
|
||||
ListRemove(&r->GlobalLink);
|
||||
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
|
||||
KiUnmapUserRegionPayload(r);
|
||||
MmFreePool(r);
|
||||
}
|
||||
}
|
||||
|
||||
size_t MmGetUserRegionSize(PROCESS* process, const void* base) {
|
||||
if (!process || !base) return 0;
|
||||
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
|
||||
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
|
||||
size_t size = r ? r->Size : 0;
|
||||
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
|
||||
return size;
|
||||
}
|
||||
|
||||
size_t MmGetUserRegionSizeByBase(const void* base) {
|
||||
if (!base) return 0;
|
||||
uintptr_t b = (uintptr_t)base;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
|
||||
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
||||
if (r->Base == b) {
|
||||
size_t size = r->Size;
|
||||
KeReleaseSpinlock(&gUserRegionLock, flags);
|
||||
return size;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gUserRegionLock, flags);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
static const char* KiUserRegionTypeName(USER_REGION_TYPE type) {
|
||||
switch (type) {
|
||||
case UserRegionGeneric: return "generic";
|
||||
case UserRegionHeap: return "heap";
|
||||
case UserRegionImage: return "image";
|
||||
case UserRegionStack: return "stack";
|
||||
case UserRegionTeb: return "teb";
|
||||
case UserRegionPeb: return "peb";
|
||||
case UserRegionEnvironment: return "env";
|
||||
case UserRegionMappedFile: return "mapped";
|
||||
case UserRegionShared: return "shared";
|
||||
default: return "?";
|
||||
}
|
||||
}
|
||||
|
||||
void MmDebugDumpUserRegions(PROCESS* process) {
|
||||
if (!process) {
|
||||
DbgPrint("MM: user regions: <no process>\n");
|
||||
return;
|
||||
}
|
||||
|
||||
DbgPrint("MM: user regions for pid=%u name=%s:\n", process->ProcessId, process->Name);
|
||||
uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock);
|
||||
uint32_t count = 0;
|
||||
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
|
||||
DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx flags=%x pte=%llx\n",
|
||||
count,
|
||||
KiUserRegionTypeName(r->Type),
|
||||
(unsigned long long)r->ReservationBase,
|
||||
(unsigned long long)(r->ReservationBase + r->ReservationSize),
|
||||
(unsigned long long)r->Base,
|
||||
(unsigned long long)r->Size,
|
||||
(unsigned long long)r->CommittedSize,
|
||||
r->Flags,
|
||||
(unsigned long long)r->PageFlags);
|
||||
count++;
|
||||
}
|
||||
KeReleaseSpinlock(&process->UserRegionLock, flags);
|
||||
if (!count) DbgPrint(" <none>\n");
|
||||
}
|
||||
|
||||
static void KiDebugDescribePoolAddress(uintptr_t addr) {
|
||||
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
|
||||
(unsigned long long)gPoolHeap.Start,
|
||||
(unsigned long long)gPoolHeap.Brk,
|
||||
(unsigned long long)gPoolHeap.Limit,
|
||||
(unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
|
||||
|
||||
if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
|
||||
if (addr >= gPoolHeap.Brk) {
|
||||
DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
|
||||
(unsigned long long)addr,
|
||||
(unsigned long long)(addr - gPoolHeap.Brk));
|
||||
return;
|
||||
}
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
|
||||
for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
|
||||
(uintptr_t)cur < gPoolHeap.Brk;
|
||||
cur = KiNextBlock(cur)) {
|
||||
if (cur->Magic != BLOCK_MAGIC) {
|
||||
DbgPrint("MM: pool walk corrupt at hdr=%p magic=%08x\n", cur, cur->Magic);
|
||||
break;
|
||||
}
|
||||
uintptr_t hdr = (uintptr_t)cur;
|
||||
uintptr_t data = hdr + HDR_SIZE;
|
||||
uintptr_t end = data + cur->Size;
|
||||
uintptr_t ftr = end;
|
||||
if (addr >= hdr && addr < ftr + FTR_SIZE) {
|
||||
DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx free=%u\n",
|
||||
(unsigned long long)addr,
|
||||
(unsigned long long)hdr,
|
||||
(unsigned long long)data,
|
||||
(unsigned long long)end,
|
||||
(unsigned long long)cur->Size,
|
||||
cur->Free);
|
||||
break;
|
||||
}
|
||||
if (KiNextBlock(cur) <= cur) {
|
||||
DbgPrint("MM: pool walk stopped: non-advancing block hdr=%p size=%llx\n", cur, (unsigned long long)cur->Size);
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
|
||||
}
|
||||
|
||||
void MmDebugDescribeAddress(uintptr_t addr) {
|
||||
PHYSICAL_ADDRESS phys = 0;
|
||||
uint64_t flags = 0;
|
||||
KSTATUS qs = MmQueryPage(addr, &phys, &flags);
|
||||
DbgPrint("MM: describe va=%016llx query=%d phys=%016llx flags=%016llx\n",
|
||||
(unsigned long long)addr,
|
||||
qs,
|
||||
(unsigned long long)phys,
|
||||
(unsigned long long)flags);
|
||||
|
||||
if (addr >= POOL_BASE && addr < POOL_LIMIT) {
|
||||
KiDebugDescribePoolAddress(addr);
|
||||
return;
|
||||
}
|
||||
|
||||
if (addr >= USER_REGION_ARENA_BASE && addr < USER_LIMIT) {
|
||||
uintptr_t gl = KeAcquireSpinlock(&gUserRegionLock);
|
||||
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
||||
if (addr >= r->ReservationBase && addr < r->ReservationBase + r->ReservationSize) {
|
||||
DbgPrint("MM: address belongs to pid=%u name=%s region=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n",
|
||||
r->Owner ? r->Owner->ProcessId : 0,
|
||||
r->Owner ? r->Owner->Name : "?",
|
||||
KiUserRegionTypeName(r->Type),
|
||||
(unsigned long long)r->ReservationBase,
|
||||
(unsigned long long)(r->ReservationBase + r->ReservationSize),
|
||||
(unsigned long long)r->Base,
|
||||
(unsigned long long)r->Size,
|
||||
r->Flags);
|
||||
KeReleaseSpinlock(&gUserRegionLock, gl);
|
||||
return;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gUserRegionLock, gl);
|
||||
DbgPrint("MM: address is in user arena but not reserved by any USER_VA_REGION\n");
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) {
|
||||
void* base = (void*)0;
|
||||
if (!process || !size) return (void*)0;
|
||||
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
||||
USER_REGION_ZERO,
|
||||
UserRegionHeap,
|
||||
&base);
|
||||
return KSUCCESS(s) ? base : (void*)0;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) {
|
||||
if (!process || !ptr) return;
|
||||
(void)MmDestroyUserRegion(process, ptr);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) {
|
||||
if (!process) return (void*)0;
|
||||
if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize);
|
||||
if (!newSize) {
|
||||
MmFreeUserMemoryForProcess(process, ptr);
|
||||
return (void*)0;
|
||||
}
|
||||
|
||||
size_t oldSize = MmGetUserRegionSize(process, ptr);
|
||||
if (!oldSize) return (void*)0;
|
||||
if (newSize <= oldSize) return ptr;
|
||||
|
||||
void* newPtr = MmAllocateUserMemoryForProcess(process, newSize);
|
||||
if (!newPtr) return (void*)0;
|
||||
memcpy(newPtr, ptr, oldSize);
|
||||
MmFreeUserMemoryForProcess(process, ptr);
|
||||
return newPtr;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
return MmAllocateUserMemoryForProcess(process, size);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
MmFreeUserMemoryForProcess(process, ptr);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
return MmReallocateUserMemoryForProcess(process, ptr, newSize);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) {
|
||||
void* base = (void*)0;
|
||||
if (!process || !size) return (void*)0;
|
||||
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
||||
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
|
||||
UserRegionImage,
|
||||
&base);
|
||||
return KSUCCESS(s) ? base : (void*)0;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) {
|
||||
(void)size;
|
||||
if (!process || !ptr) return;
|
||||
(void)MmDestroyUserRegion(process, ptr);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
return MmAllocateUserImageForProcess(process, size);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
MmFreeUserImageForProcess(process, ptr, size);
|
||||
}
|
||||
|
||||
size_t MmGetKernelPoolUsage(void) {
|
||||
// Returns bytes committed to the kernel pool
|
||||
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
|
||||
}
|
||||
|
||||
size_t MmGetUserHeapUsage(void) {
|
||||
// Returns bytes committed to the user heap
|
||||
return (size_t)(gUserHeap.Brk - gUserHeap.Start);
|
||||
size_t total = 0;
|
||||
if (!gUserRegionListReady) return 0;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
|
||||
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
|
||||
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
|
||||
total += r->CommittedSize;
|
||||
}
|
||||
KeReleaseSpinlock(&gUserRegionLock, flags);
|
||||
return total;
|
||||
}
|
||||
@@ -39,6 +39,9 @@ typedef struct BOOT_INFO {
|
||||
/* Direct physical window base - always 0xFFFF880000000000. */
|
||||
uintptr_t PhysicalMapBase;
|
||||
|
||||
/* Reserved low-memory page for the AP startup trampoline, or zero. */
|
||||
uintptr_t ApTrampolinePhys;
|
||||
|
||||
/* From boot.ini */
|
||||
uint32_t Debug;
|
||||
} BOOT_INFO;
|
||||
@@ -20,42 +20,6 @@ typedef struct HAL_SCREEN {
|
||||
void (*EraseCursor)(void);
|
||||
} HAL_SCREEN;
|
||||
|
||||
typedef struct HAL_KEYBOARD {
|
||||
uint8_t (*Poll)(void);
|
||||
uint8_t (*ReadScancode)(void);
|
||||
char (*ReadChar)(void);
|
||||
uint32_t (*GetModifiers)(void);
|
||||
void (*SetCallback)(void (*cb)(uint8_t scancode));
|
||||
bool (*IsKeyDown)(uint8_t scancode, bool extended);
|
||||
void (*Isr)(void);
|
||||
// Optional bottom half, queued by the kernel after IRQ dispatch.
|
||||
void (*Dpc)(void);
|
||||
uint8_t IrqNumber;
|
||||
} HAL_KEYBOARD;
|
||||
|
||||
// HAL_MOUSE – PS/2 auxiliary mouse on IRQ 12.
|
||||
// Packet format: standard 3-byte PS/2 (buttons, deltaX, deltaY).
|
||||
// Y is already screen-oriented (positive = down) after ISR inversion.
|
||||
typedef struct HAL_MOUSE {
|
||||
// Non-blocking: returns true if a packet is waiting in the ring buffer.
|
||||
bool (*Poll)(void);
|
||||
// Non-blocking dequeue. Returns false when buffer is empty.
|
||||
// buttons: bit0=left, bit1=right, bit2=middle
|
||||
bool (*Read)(int* dx, int* dy, uint8_t* buttons);
|
||||
// Callback invoked from DPC context after every completed packet.
|
||||
void (*SetCallback)(void (*cb)(int dx, int dy, uint8_t buttons));
|
||||
// Absolute cursor position, clamped to screen bounds.
|
||||
void (*GetPosition)(int* x, int* y);
|
||||
void (*SetPosition)(int x, int y);
|
||||
// Update the clamping rectangle (call after mode change / resize).
|
||||
void (*SetBounds)(int w, int h);
|
||||
// Raw ISR – called by KeIrqDispatch for IRQ 12.
|
||||
void (*Isr)(void);
|
||||
// Optional bottom half, queued by the kernel after IRQ dispatch.
|
||||
void (*Dpc)(void);
|
||||
uint8_t IrqNumber; // always 12
|
||||
} HAL_MOUSE;
|
||||
|
||||
typedef struct HAL_TIMER {
|
||||
uint64_t (*QueryTicks)(void);
|
||||
void (*Stall)(uint32_t us);
|
||||
@@ -75,6 +39,10 @@ typedef struct HAL_PCI {
|
||||
// Returns the IRQ line from PCI config space (register 0x3C, byte 0).
|
||||
// Returns 0xFF if no interrupt is wired.
|
||||
uint8_t (*GetIrqLine)(uint8_t bus, uint8_t dev, uint8_t fn);
|
||||
|
||||
// Program a PCI MSI capability to deliver to vector (0x20 + irqSlot).
|
||||
// irqSlot must correspond to a kernel IRQ stub/vector slot.
|
||||
KSTATUS (*EnableMsi)(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t irqSlot);
|
||||
} HAL_PCI;
|
||||
|
||||
typedef struct HAL_INTC {
|
||||
@@ -94,8 +62,6 @@ typedef struct HAL_API {
|
||||
uint32_t Version;
|
||||
void *(*PhysToVirt)(uint64_t physAddr);
|
||||
HAL_SCREEN Screen;
|
||||
HAL_KEYBOARD Keyboard;
|
||||
HAL_MOUSE Mouse;
|
||||
HAL_TIMER Timer;
|
||||
HAL_PCI Pci;
|
||||
HAL_INTC Intc;
|
||||
@@ -117,9 +83,11 @@ typedef struct HAL_INIT_PARAMS {
|
||||
|
||||
// Debug
|
||||
int Debug;
|
||||
|
||||
void* (*AllocatePool)(size_t size);
|
||||
} HAL_INIT_PARAMS;
|
||||
|
||||
#define HAL_API_VERSION 2
|
||||
#define HAL_API_VERSION 3
|
||||
|
||||
__declspec(dllexport) const HAL_API* HalGetApi(void);
|
||||
__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void));
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
#pragma once
|
||||
// input_device.h - driver-facing input submission DDI.
|
||||
|
||||
#include <ntlite/input.h>
|
||||
#include <ntlite/status.h>
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId);
|
||||
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event);
|
||||
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event);
|
||||
@@ -1,9 +1,9 @@
|
||||
#pragma once
|
||||
// io_driver.h - definitive kernel driver/device I/O model
|
||||
//
|
||||
// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through
|
||||
// Disk controllers register DEVICE_OBJECTs. IRPs are submitted only through
|
||||
// IoCallDriver()/IoSubmitIrp(); drivers complete accepted IRPs by calling
|
||||
// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table.
|
||||
// IoCompleteRequest(). There is no legacy DISK_DRIVER submit table.
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
@@ -33,6 +33,75 @@ struct DRIVER_OBJECT {
|
||||
DRIVER_UNLOAD_FN DriverUnload;
|
||||
};
|
||||
|
||||
|
||||
|
||||
typedef enum IO_BUS_TYPE {
|
||||
IoBusUnknown = 0,
|
||||
IoBusPci = 1,
|
||||
IoBusUsb = 2,
|
||||
} IO_BUS_TYPE;
|
||||
|
||||
#define IO_PUBLISHED_USB_MAX_ENDPOINTS 8u
|
||||
|
||||
typedef struct IO_PUBLISHED_USB_ENDPOINT {
|
||||
uint8_t Address;
|
||||
uint8_t Attributes;
|
||||
uint16_t MaxPacketSize;
|
||||
uint8_t Interval;
|
||||
} IO_PUBLISHED_USB_ENDPOINT;
|
||||
|
||||
typedef struct IO_PUBLISHED_USB_DEVICE {
|
||||
uint16_t VendorId;
|
||||
uint16_t ProductId;
|
||||
uint16_t BcdUsb;
|
||||
uint16_t BcdDevice;
|
||||
|
||||
uint8_t DeviceClass;
|
||||
uint8_t DeviceSubclass;
|
||||
uint8_t DeviceProtocol;
|
||||
|
||||
uint8_t SlotId;
|
||||
uint8_t Port;
|
||||
uint8_t Speed;
|
||||
uint8_t ConfigurationValue;
|
||||
|
||||
uint8_t InterfaceNumber;
|
||||
uint8_t AlternateSetting;
|
||||
uint8_t InterfaceClass;
|
||||
uint8_t InterfaceSubclass;
|
||||
uint8_t InterfaceProtocol;
|
||||
|
||||
uint8_t EndpointCount;
|
||||
IO_PUBLISHED_USB_ENDPOINT Endpoints[IO_PUBLISHED_USB_MAX_ENDPOINTS];
|
||||
} IO_PUBLISHED_USB_DEVICE;
|
||||
|
||||
typedef struct IO_PUBLISHED_DEVICE {
|
||||
IO_BUS_TYPE BusType;
|
||||
const char* Name;
|
||||
void* BusContext;
|
||||
void* DeviceContext;
|
||||
uint32_t Flags;
|
||||
bool Bound;
|
||||
const char* BoundDriverName;
|
||||
|
||||
union {
|
||||
IO_PUBLISHED_USB_DEVICE Usb;
|
||||
} u;
|
||||
} IO_PUBLISHED_DEVICE;
|
||||
|
||||
typedef bool (*IO_DRIVER_MATCH_CALLBACK)(const IO_PUBLISHED_DEVICE* device,
|
||||
void* context);
|
||||
typedef KSTATUS (*IO_DRIVER_BIND_CALLBACK)(IO_PUBLISHED_DEVICE* device,
|
||||
void* context);
|
||||
|
||||
__declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
|
||||
IO_BUS_TYPE busType,
|
||||
IO_DRIVER_MATCH_CALLBACK match,
|
||||
IO_DRIVER_BIND_CALLBACK bind,
|
||||
void* context);
|
||||
__declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device);
|
||||
__declspec(dllexport) uint32_t IoPublishedDeviceCount(void);
|
||||
|
||||
struct DEVICE_OBJECT {
|
||||
DRIVER_OBJECT* DriverObject;
|
||||
void* DeviceExtension;
|
||||
@@ -56,6 +125,15 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex);
|
||||
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex);
|
||||
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex);
|
||||
|
||||
typedef void (*IO_DISK_ARRIVAL_CALLBACK)(uint8_t driveIndex,
|
||||
DEVICE_OBJECT* device,
|
||||
void* context);
|
||||
|
||||
__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback,
|
||||
void* context);
|
||||
__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback,
|
||||
void* context);
|
||||
|
||||
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device,
|
||||
struct IRP* irp);
|
||||
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
#pragma once
|
||||
// input.h - canonical keyboard/mouse input ABI shared by kernel, drivers, and user-mode.
|
||||
//
|
||||
// Key values intentionally follow USB HID Keyboard/Keypad usage IDs. PS/2,
|
||||
// USB HID, and future input drivers all translate their hardware format into
|
||||
// these values before entering the kernel input core.
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
|
||||
typedef uint16_t INPUT_KEY;
|
||||
|
||||
#define INPUT_KEY_NONE 0x0000u
|
||||
#define INPUT_KEY_A 0x0004u
|
||||
#define INPUT_KEY_B 0x0005u
|
||||
#define INPUT_KEY_C 0x0006u
|
||||
#define INPUT_KEY_D 0x0007u
|
||||
#define INPUT_KEY_E 0x0008u
|
||||
#define INPUT_KEY_F 0x0009u
|
||||
#define INPUT_KEY_G 0x000Au
|
||||
#define INPUT_KEY_H 0x000Bu
|
||||
#define INPUT_KEY_I 0x000Cu
|
||||
#define INPUT_KEY_J 0x000Du
|
||||
#define INPUT_KEY_K 0x000Eu
|
||||
#define INPUT_KEY_L 0x000Fu
|
||||
#define INPUT_KEY_M 0x0010u
|
||||
#define INPUT_KEY_N 0x0011u
|
||||
#define INPUT_KEY_O 0x0012u
|
||||
#define INPUT_KEY_P 0x0013u
|
||||
#define INPUT_KEY_Q 0x0014u
|
||||
#define INPUT_KEY_R 0x0015u
|
||||
#define INPUT_KEY_S 0x0016u
|
||||
#define INPUT_KEY_T 0x0017u
|
||||
#define INPUT_KEY_U 0x0018u
|
||||
#define INPUT_KEY_V 0x0019u
|
||||
#define INPUT_KEY_W 0x001Au
|
||||
#define INPUT_KEY_X 0x001Bu
|
||||
#define INPUT_KEY_Y 0x001Cu
|
||||
#define INPUT_KEY_Z 0x001Du
|
||||
#define INPUT_KEY_1 0x001Eu
|
||||
#define INPUT_KEY_2 0x001Fu
|
||||
#define INPUT_KEY_3 0x0020u
|
||||
#define INPUT_KEY_4 0x0021u
|
||||
#define INPUT_KEY_5 0x0022u
|
||||
#define INPUT_KEY_6 0x0023u
|
||||
#define INPUT_KEY_7 0x0024u
|
||||
#define INPUT_KEY_8 0x0025u
|
||||
#define INPUT_KEY_9 0x0026u
|
||||
#define INPUT_KEY_0 0x0027u
|
||||
#define INPUT_KEY_ENTER 0x0028u
|
||||
#define INPUT_KEY_ESCAPE 0x0029u
|
||||
#define INPUT_KEY_BACKSPACE 0x002Au
|
||||
#define INPUT_KEY_TAB 0x002Bu
|
||||
#define INPUT_KEY_SPACE 0x002Cu
|
||||
#define INPUT_KEY_MINUS 0x002Du
|
||||
#define INPUT_KEY_EQUAL 0x002Eu
|
||||
#define INPUT_KEY_LEFT_BRACKET 0x002Fu
|
||||
#define INPUT_KEY_RIGHT_BRACKET 0x0030u
|
||||
#define INPUT_KEY_BACKSLASH 0x0031u
|
||||
#define INPUT_KEY_SEMICOLON 0x0033u
|
||||
#define INPUT_KEY_APOSTROPHE 0x0034u
|
||||
#define INPUT_KEY_GRAVE 0x0035u
|
||||
#define INPUT_KEY_COMMA 0x0036u
|
||||
#define INPUT_KEY_PERIOD 0x0037u
|
||||
#define INPUT_KEY_SLASH 0x0038u
|
||||
#define INPUT_KEY_CAPS_LOCK 0x0039u
|
||||
#define INPUT_KEY_F1 0x003Au
|
||||
#define INPUT_KEY_F2 0x003Bu
|
||||
#define INPUT_KEY_F3 0x003Cu
|
||||
#define INPUT_KEY_F4 0x003Du
|
||||
#define INPUT_KEY_F5 0x003Eu
|
||||
#define INPUT_KEY_F6 0x003Fu
|
||||
#define INPUT_KEY_F7 0x0040u
|
||||
#define INPUT_KEY_F8 0x0041u
|
||||
#define INPUT_KEY_F9 0x0042u
|
||||
#define INPUT_KEY_F10 0x0043u
|
||||
#define INPUT_KEY_F11 0x0044u
|
||||
#define INPUT_KEY_F12 0x0045u
|
||||
#define INPUT_KEY_INSERT 0x0049u
|
||||
#define INPUT_KEY_HOME 0x004Au
|
||||
#define INPUT_KEY_PAGE_UP 0x004Bu
|
||||
#define INPUT_KEY_DELETE 0x004Cu
|
||||
#define INPUT_KEY_END 0x004Du
|
||||
#define INPUT_KEY_PAGE_DOWN 0x004Eu
|
||||
#define INPUT_KEY_RIGHT 0x004Fu
|
||||
#define INPUT_KEY_LEFT 0x0050u
|
||||
#define INPUT_KEY_DOWN 0x0051u
|
||||
#define INPUT_KEY_UP 0x0052u
|
||||
#define INPUT_KEY_NUM_LOCK 0x0053u
|
||||
#define INPUT_KEY_KP_DIVIDE 0x0054u
|
||||
#define INPUT_KEY_KP_MULTIPLY 0x0055u
|
||||
#define INPUT_KEY_KP_MINUS 0x0056u
|
||||
#define INPUT_KEY_KP_PLUS 0x0057u
|
||||
#define INPUT_KEY_KP_ENTER 0x0058u
|
||||
#define INPUT_KEY_KP_1 0x0059u
|
||||
#define INPUT_KEY_KP_2 0x005Au
|
||||
#define INPUT_KEY_KP_3 0x005Bu
|
||||
#define INPUT_KEY_KP_4 0x005Cu
|
||||
#define INPUT_KEY_KP_5 0x005Du
|
||||
#define INPUT_KEY_KP_6 0x005Eu
|
||||
#define INPUT_KEY_KP_7 0x005Fu
|
||||
#define INPUT_KEY_KP_8 0x0060u
|
||||
#define INPUT_KEY_KP_9 0x0061u
|
||||
#define INPUT_KEY_KP_0 0x0062u
|
||||
#define INPUT_KEY_KP_PERIOD 0x0063u
|
||||
|
||||
#define INPUT_KEY_LEFT_CTRL 0x00E0u
|
||||
#define INPUT_KEY_LEFT_SHIFT 0x00E1u
|
||||
#define INPUT_KEY_LEFT_ALT 0x00E2u
|
||||
#define INPUT_KEY_LEFT_GUI 0x00E3u
|
||||
#define INPUT_KEY_RIGHT_CTRL 0x00E4u
|
||||
#define INPUT_KEY_RIGHT_SHIFT 0x00E5u
|
||||
#define INPUT_KEY_RIGHT_ALT 0x00E6u
|
||||
#define INPUT_KEY_RIGHT_GUI 0x00E7u
|
||||
|
||||
#define INPUT_MOD_LSHIFT (1u << 0)
|
||||
#define INPUT_MOD_RSHIFT (1u << 1)
|
||||
#define INPUT_MOD_LCTRL (1u << 2)
|
||||
#define INPUT_MOD_RCTRL (1u << 3)
|
||||
#define INPUT_MOD_LALT (1u << 4)
|
||||
#define INPUT_MOD_RALT (1u << 5)
|
||||
#define INPUT_MOD_CAPS_LOCK (1u << 6)
|
||||
#define INPUT_MOD_NUM_LOCK (1u << 7)
|
||||
#define INPUT_MOD_LGUI (1u << 8)
|
||||
#define INPUT_MOD_RGUI (1u << 9)
|
||||
|
||||
#define INPUT_MOD_SHIFT (INPUT_MOD_LSHIFT | INPUT_MOD_RSHIFT)
|
||||
#define INPUT_MOD_CTRL (INPUT_MOD_LCTRL | INPUT_MOD_RCTRL)
|
||||
#define INPUT_MOD_ALT (INPUT_MOD_LALT | INPUT_MOD_RALT)
|
||||
#define INPUT_MOD_GUI (INPUT_MOD_LGUI | INPUT_MOD_RGUI)
|
||||
|
||||
typedef struct INPUT_KEY_EVENT {
|
||||
INPUT_KEY Key;
|
||||
bool Pressed;
|
||||
} INPUT_KEY_EVENT;
|
||||
|
||||
typedef struct INPUT_MOUSE_EVENT {
|
||||
int dx;
|
||||
int dy;
|
||||
int wheel;
|
||||
uint8_t buttons; // bit0=left, bit1=right, bit2=middle
|
||||
} INPUT_MOUSE_EVENT;
|
||||
@@ -7,6 +7,7 @@
|
||||
#include <intrin.h>
|
||||
#include <ntlite/string_view.h>
|
||||
#include <ntlite/dirent.h>
|
||||
#include <ntlite/input.h>
|
||||
|
||||
#define SYS_STATUS_SUCCESS 0u
|
||||
#define SYS_STATUS_TIMEOUT 3u
|
||||
@@ -20,7 +21,7 @@
|
||||
// ---------------------------------------------------------------------------
|
||||
// User process/thread environment
|
||||
// ---------------------------------------------------------------------------
|
||||
// GS base points at SYS_TEB. GS:[0] is the process PEB pointer.
|
||||
// GS base points at SYS_TEB. GS:[0] is the process PEB pointer.
|
||||
// This is this OS' ABI, not a Windows compatibility layout.
|
||||
#define SYS_PEB_MAGIC 0x30424550u /* "PEB0" */
|
||||
#define SYS_TEB_MAGIC 0x30424554u /* "TEB0" */
|
||||
@@ -48,7 +49,7 @@ typedef struct SYS_TEB {
|
||||
uint32_t ThreadId;
|
||||
uint32_t Magic;
|
||||
|
||||
// Native per-thread extension pointer. The kernel leaves this NULL;
|
||||
// Native per-thread extension pointer. The kernel leaves this NULL;
|
||||
// user-mode personalities/libraries may hang their own thread-local
|
||||
// state here without baking subsystem semantics into SYS_TEB.
|
||||
void* SubsystemData;
|
||||
@@ -278,9 +279,9 @@ uint32_t SysScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg);
|
||||
// Input (62-69)
|
||||
// ---------------------------------------------------------------------------
|
||||
uint64_t SysKbPoll(void);
|
||||
uint64_t SysKbIsKeyDown(uint8_t scancode, bool extended);
|
||||
uint64_t SysKbIsKeyDown(INPUT_KEY key);
|
||||
char SysKbReadChar(void);
|
||||
uint8_t SysKbReadScancode(void);
|
||||
INPUT_KEY SysKbReadKey(void);
|
||||
uint32_t SysKbGetModifiers(void);
|
||||
|
||||
uint64_t SysMousePoll(void);
|
||||
|
||||
+1
-1
@@ -665,7 +665,7 @@ netconn_recv_data(struct netconn *conn, void **new_buf, u8_t apiflags)
|
||||
/* Register event with callback */
|
||||
API_EVENT(conn, NETCONN_EVT_RCVMINUS, len);
|
||||
|
||||
LWIP_DEBUGF(API_LIB_DEBUG, ("netconn_recv_data: received %p, len=%"U16_F"\n", buf, len));
|
||||
LWIP_DEBUGF(API_LIB_DEBUG, ("netconn_recv_data: received 0x%016llX, len=%"U16_F"\n", buf, len));
|
||||
|
||||
*new_buf = buf;
|
||||
/* don't set conn->last_err: it's only ERR_OK, anyway */
|
||||
|
||||
+2
-2
@@ -126,11 +126,11 @@ lwip_gethostbyname(const char *name)
|
||||
#if DNS_DEBUG
|
||||
/* dump hostent */
|
||||
LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_name == %s\n", s_hostent.h_name));
|
||||
LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_aliases == %p\n", (void *)s_hostent.h_aliases));
|
||||
LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_aliases == 0x%016llX\n", (void *)s_hostent.h_aliases));
|
||||
/* h_aliases are always empty */
|
||||
LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_addrtype == %d\n", s_hostent.h_addrtype));
|
||||
LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_length == %d\n", s_hostent.h_length));
|
||||
LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_addr_list == %p\n", (void *)s_hostent.h_addr_list));
|
||||
LWIP_DEBUGF(DNS_DEBUG, ("hostent.h_addr_list == 0x%016llX\n", (void *)s_hostent.h_addr_list));
|
||||
if (s_hostent.h_addr_list != NULL) {
|
||||
u8_t idx;
|
||||
for (idx = 0; s_hostent.h_addr_list[idx]; idx++) {
|
||||
|
||||
+14
-14
@@ -975,7 +975,7 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
|
||||
err_t err;
|
||||
u16_t copylen;
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: top while sock->lastdata=%p\n", (void *)sock->lastdata.pbuf));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: top while sock->lastdata=0x%016llX\n", (void *)sock->lastdata.pbuf));
|
||||
/* Check if there is data left from the last recv operation. */
|
||||
if (sock->lastdata.pbuf) {
|
||||
p = sock->lastdata.pbuf;
|
||||
@@ -983,7 +983,7 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
|
||||
/* No data was left from the previous operation, so we try to get
|
||||
some from the network. */
|
||||
err = netconn_recv_tcp_pbuf_flags(sock->conn, &p, apiflags);
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: netconn_recv err=%d, pbuf=%p\n",
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: netconn_recv err=%d, pbuf=0x%016llX\n",
|
||||
err, (void *)p));
|
||||
|
||||
if (err != ERR_OK) {
|
||||
@@ -1037,10 +1037,10 @@ lwip_recv_tcp(struct lwip_sock *sock, void *mem, size_t len, int flags)
|
||||
/* If so, it should be saved in the sock structure for the next recv call.
|
||||
We store the pbuf but hide/free the consumed data: */
|
||||
sock->lastdata.pbuf = pbuf_free_header(p, copylen);
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: lastdata now pbuf=%p\n", (void *)sock->lastdata.pbuf));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: lastdata now pbuf=0x%016llX\n", (void *)sock->lastdata.pbuf));
|
||||
} else {
|
||||
sock->lastdata.pbuf = NULL;
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: deleting pbuf=%p\n", (void *)p));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recv_tcp: deleting pbuf=0x%016llX\n", (void *)p));
|
||||
pbuf_free(p);
|
||||
}
|
||||
}
|
||||
@@ -1142,14 +1142,14 @@ lwip_recvfrom_udp_raw(struct lwip_sock *sock, int flags, struct msghdr *msg, u16
|
||||
apiflags = 0;
|
||||
}
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: top sock->lastdata=%p\n", (void *)sock->lastdata.netbuf));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: top sock->lastdata=0x%016llX\n", (void *)sock->lastdata.netbuf));
|
||||
/* Check if there is data left from the last recv operation. */
|
||||
buf = sock->lastdata.netbuf;
|
||||
if (buf == NULL) {
|
||||
/* No data was left from the previous operation, so we try to get
|
||||
some from the network. */
|
||||
err = netconn_recv_udp_raw_netbuf_flags(sock->conn, &buf, apiflags);
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: netconn_recv err=%d, netbuf=%p\n",
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom_udp_raw[UDP/RAW]: netconn_recv err=%d, netbuf=0x%016llX\n",
|
||||
err, (void *)buf));
|
||||
|
||||
if (err != ERR_OK) {
|
||||
@@ -1242,7 +1242,7 @@ lwip_recvfrom(int s, void *mem, size_t len, int flags,
|
||||
struct lwip_sock *sock;
|
||||
ssize_t ret;
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom(%d, %p, %"SZT_F", 0x%x, ..)\n", s, mem, len, flags));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvfrom(%d, 0x%016llX, %"SZT_F", 0x%x, ..)\n", s, mem, len, flags));
|
||||
sock = get_socket(s);
|
||||
if (!sock) {
|
||||
return -1;
|
||||
@@ -1324,7 +1324,7 @@ lwip_recvmsg(int s, struct msghdr *message, int flags)
|
||||
msg_iovlen_t i;
|
||||
ssize_t buflen;
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvmsg(%d, message=%p, flags=0x%x)\n", s, (void *)message, flags));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_recvmsg(%d, message=0x%016llX, flags=0x%x)\n", s, (void *)message, flags));
|
||||
LWIP_ERROR("lwip_recvmsg: invalid message pointer", message != NULL, return ERR_ARG;);
|
||||
LWIP_ERROR("lwip_recvmsg: unsupported flags", (flags & ~(MSG_PEEK|MSG_DONTWAIT)) == 0,
|
||||
set_errno(EOPNOTSUPP); return -1;);
|
||||
@@ -1426,7 +1426,7 @@ lwip_send(int s, const void *data, size_t size, int flags)
|
||||
u8_t write_flags;
|
||||
size_t written;
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_send(%d, data=%p, size=%"SZT_F", flags=0x%x)\n",
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_send(%d, data=0x%016llX, size=%"SZT_F", flags=0x%x)\n",
|
||||
s, data, size, flags));
|
||||
|
||||
sock = get_socket(s);
|
||||
@@ -1675,7 +1675,7 @@ lwip_sendto(int s, const void *data, size_t size, int flags,
|
||||
netbuf_fromport(&buf) = remote_port;
|
||||
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_sendto(%d, data=%p, short_size=%"U16_F", flags=0x%x to=",
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_sendto(%d, data=0x%016llX, short_size=%"U16_F", flags=0x%x to=",
|
||||
s, data, short_size, flags));
|
||||
ip_addr_debug_print_val(SOCKETS_DEBUG, buf.addr);
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, (" port=%"U16_F"\n", remote_port));
|
||||
@@ -2018,7 +2018,7 @@ lwip_select(int maxfdp1, fd_set *readset, fd_set *writeset, fd_set *exceptset,
|
||||
#endif
|
||||
SYS_ARCH_DECL_PROTECT(lev);
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_select(%d, %p, %p, %p, tvsec=%"S32_F" tvusec=%"S32_F")\n",
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_select(%d, 0x%016llX, 0x%016llX, 0x%016llX, tvsec=%"S32_F" tvusec=%"S32_F")\n",
|
||||
maxfdp1, (void *)readset, (void *) writeset, (void *) exceptset,
|
||||
timeout ? (s32_t)timeout->tv_sec : (s32_t) - 1,
|
||||
timeout ? (s32_t)timeout->tv_usec : (s32_t) - 1));
|
||||
@@ -2372,7 +2372,7 @@ lwip_poll(struct pollfd *fds, nfds_t nfds, int timeout)
|
||||
int waited = 0;
|
||||
#endif
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_poll(%p, %d, %d)\n",
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_poll(0x%016llX, %d, %d)\n",
|
||||
(void*)fds, (int)nfds, timeout));
|
||||
LWIP_ERROR("lwip_poll: invalid fds", ((fds != NULL && nfds > 0) || (fds == NULL && nfds == 0)),
|
||||
set_errno(EINVAL); return -1;);
|
||||
@@ -3882,7 +3882,7 @@ lwip_ioctl(int s, long cmd, void *argp)
|
||||
}
|
||||
*((int *)argp) = recv_avail;
|
||||
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, FIONREAD, %p) = %"U16_F"\n", s, argp, *((u16_t *)argp)));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, FIONREAD, 0x%016llX) = %"U16_F"\n", s, argp, *((u16_t *)argp)));
|
||||
set_errno(0);
|
||||
done_socket(sock);
|
||||
return 0;
|
||||
@@ -3905,7 +3905,7 @@ lwip_ioctl(int s, long cmd, void *argp)
|
||||
default:
|
||||
break;
|
||||
} /* switch (cmd) */
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, UNIMPL: 0x%lx, %p)\n", s, cmd, argp));
|
||||
LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_ioctl(%d, UNIMPL: 0x%lx, 0x%016llX)\n", s, cmd, argp));
|
||||
set_errno(ENOSYS); /* not yet implemented */
|
||||
done_socket(sock);
|
||||
return -1;
|
||||
|
||||
+9
-9
@@ -167,16 +167,16 @@ tcpip_thread_handle_msg(struct tcpip_msg *msg)
|
||||
switch (msg->type) {
|
||||
#if !LWIP_TCPIP_CORE_LOCKING
|
||||
case TCPIP_MSG_API:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API message %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API message 0x%016llX\n", (void *)msg));
|
||||
msg->msg.api_msg.function(msg->msg.api_msg.msg);
|
||||
break;
|
||||
case TCPIP_MSG_API_CALL:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API CALL message %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: API CALL message 0x%016llX\n", (void *)msg));
|
||||
msg->msg.api_call.arg->err = msg->msg.api_call.function(msg->msg.api_call.arg);
|
||||
sys_sem_signal(msg->msg.api_call.sem);
|
||||
break;
|
||||
case TCPIP_MSG_CALLBACK_STATIC_WAIT:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK WAIT message %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK WAIT message 0x%016llX\n", (void *)msg));
|
||||
msg->msg.cb_wait.function(msg->msg.cb_wait.ctx);
|
||||
sys_sem_signal(msg->msg.cb_wait.sem);
|
||||
break;
|
||||
@@ -184,7 +184,7 @@ tcpip_thread_handle_msg(struct tcpip_msg *msg)
|
||||
|
||||
#if !LWIP_TCPIP_CORE_LOCKING_INPUT
|
||||
case TCPIP_MSG_INPKT:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: PACKET %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: PACKET 0x%016llX\n", (void *)msg));
|
||||
if (msg->msg.inp.input_fn(msg->msg.inp.p, msg->msg.inp.netif) != ERR_OK) {
|
||||
pbuf_free(msg->msg.inp.p);
|
||||
}
|
||||
@@ -194,25 +194,25 @@ tcpip_thread_handle_msg(struct tcpip_msg *msg)
|
||||
|
||||
#if LWIP_TCPIP_TIMEOUT && LWIP_TIMERS
|
||||
case TCPIP_MSG_TIMEOUT:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: TIMEOUT %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: TIMEOUT 0x%016llX\n", (void *)msg));
|
||||
sys_timeout(msg->msg.tmo.msecs, msg->msg.tmo.h, msg->msg.tmo.arg);
|
||||
memp_free(MEMP_TCPIP_MSG_API, msg);
|
||||
break;
|
||||
case TCPIP_MSG_UNTIMEOUT:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: UNTIMEOUT %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: UNTIMEOUT 0x%016llX\n", (void *)msg));
|
||||
sys_untimeout(msg->msg.tmo.h, msg->msg.tmo.arg);
|
||||
memp_free(MEMP_TCPIP_MSG_API, msg);
|
||||
break;
|
||||
#endif /* LWIP_TCPIP_TIMEOUT && LWIP_TIMERS */
|
||||
|
||||
case TCPIP_MSG_CALLBACK:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK 0x%016llX\n", (void *)msg));
|
||||
msg->msg.cb.function(msg->msg.cb.ctx);
|
||||
memp_free(MEMP_TCPIP_MSG_API, msg);
|
||||
break;
|
||||
|
||||
case TCPIP_MSG_CALLBACK_STATIC:
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK_STATIC %p\n", (void *)msg));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_thread: CALLBACK_STATIC 0x%016llX\n", (void *)msg));
|
||||
msg->msg.cb.function(msg->msg.cb.ctx);
|
||||
break;
|
||||
|
||||
@@ -255,7 +255,7 @@ tcpip_inpkt(struct pbuf *p, struct netif *inp, netif_input_fn input_fn)
|
||||
{
|
||||
#if LWIP_TCPIP_CORE_LOCKING_INPUT
|
||||
err_t ret;
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_inpkt: PACKET %p/%p\n", (void *)p, (void *)inp));
|
||||
LWIP_DEBUGF(TCPIP_DEBUG, ("tcpip_inpkt: PACKET 0x%016llX/0x%016llX\n", (void *)p, (void *)inp));
|
||||
LOCK_TCPIP_CORE();
|
||||
ret = input_fn(p, inp);
|
||||
UNLOCK_TCPIP_CORE();
|
||||
|
||||
+8
-8
@@ -623,7 +623,7 @@ static err_t
|
||||
http_close_or_abort_conn(struct altcp_pcb *pcb, struct http_state *hs, u8_t abort_conn)
|
||||
{
|
||||
err_t err;
|
||||
LWIP_DEBUGF(HTTPD_DEBUG, ("Closing connection %p\n", (void *)pcb));
|
||||
LWIP_DEBUGF(HTTPD_DEBUG, ("Closing connection 0x%016llX\n", (void *)pcb));
|
||||
|
||||
http_state_close_post(hs);
|
||||
|
||||
@@ -642,7 +642,7 @@ http_close_or_abort_conn(struct altcp_pcb *pcb, struct http_state *hs, u8_t abor
|
||||
}
|
||||
err = altcp_close(pcb);
|
||||
if (err != ERR_OK) {
|
||||
LWIP_DEBUGF(HTTPD_DEBUG, ("Error %d closing %p\n", err, (void *)pcb));
|
||||
LWIP_DEBUGF(HTTPD_DEBUG, ("Error %d closing 0x%016llX\n", err, (void *)pcb));
|
||||
/* error closing, try again later in poll */
|
||||
altcp_poll(pcb, http_poll, HTTPD_POLL_INTERVAL);
|
||||
}
|
||||
@@ -1590,7 +1590,7 @@ http_send(struct altcp_pcb *pcb, struct http_state *hs)
|
||||
{
|
||||
u8_t data_to_send = HTTP_NO_DATA_TO_SEND;
|
||||
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_send: pcb=%p hs=%p left=%d\n", (void *)pcb,
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_send: pcb=0x%016llX hs=0x%016llX left=%d\n", (void *)pcb,
|
||||
(void *)hs, hs != NULL ? (int)hs->left : 0));
|
||||
|
||||
#if LWIP_HTTPD_SUPPORT_POST && LWIP_HTTPD_POST_MANUAL_WND
|
||||
@@ -2480,7 +2480,7 @@ http_sent(void *arg, struct altcp_pcb *pcb, u16_t len)
|
||||
{
|
||||
struct http_state *hs = (struct http_state *)arg;
|
||||
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_sent %p\n", (void *)pcb));
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_sent 0x%016llX\n", (void *)pcb));
|
||||
|
||||
LWIP_UNUSED_ARG(len);
|
||||
|
||||
@@ -2506,7 +2506,7 @@ static err_t
|
||||
http_poll(void *arg, struct altcp_pcb *pcb)
|
||||
{
|
||||
struct http_state *hs = (struct http_state *)arg;
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_poll: pcb=%p hs=%p pcb_state=%s\n",
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_poll: pcb=0x%016llX hs=0x%016llX pcb_state=%s\n",
|
||||
(void *)pcb, (void *)hs, tcp_debug_state_str(altcp_dbg_get_tcp_state(pcb))));
|
||||
|
||||
if (hs == NULL) {
|
||||
@@ -2554,7 +2554,7 @@ static err_t
|
||||
http_recv(void *arg, struct altcp_pcb *pcb, struct pbuf *p, err_t err)
|
||||
{
|
||||
struct http_state *hs = (struct http_state *)arg;
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: pcb=%p pbuf=%p err=%s\n", (void *)pcb,
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: pcb=0x%016llX pbuf=0x%016llX err=%s\n", (void *)pcb,
|
||||
(void *)p, lwip_strerr(err)));
|
||||
|
||||
if ((err != ERR_OK) || (p == NULL) || (hs == NULL)) {
|
||||
@@ -2616,7 +2616,7 @@ http_recv(void *arg, struct altcp_pcb *pcb, struct pbuf *p, err_t err)
|
||||
if (hs->post_content_len_left == 0)
|
||||
#endif /* LWIP_HTTPD_SUPPORT_POST */
|
||||
{
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: data %p len %"S32_F"\n", (const void *)hs->file, hs->left));
|
||||
LWIP_DEBUGF(HTTPD_DEBUG | LWIP_DBG_TRACE, ("http_recv: data 0x%016llX len %"S32_F"\n", (const void *)hs->file, hs->left));
|
||||
http_send(pcb, hs);
|
||||
}
|
||||
} else if (parsed == ERR_ARG) {
|
||||
@@ -2641,7 +2641,7 @@ http_accept(void *arg, struct altcp_pcb *pcb, err_t err)
|
||||
struct http_state *hs;
|
||||
LWIP_UNUSED_ARG(err);
|
||||
LWIP_UNUSED_ARG(arg);
|
||||
LWIP_DEBUGF(HTTPD_DEBUG, ("http_accept %p / %p\n", (void *)pcb, arg));
|
||||
LWIP_DEBUGF(HTTPD_DEBUG, ("http_accept 0x%016llX / 0x%016llX\n", (void *)pcb, arg));
|
||||
|
||||
if ((err != ERR_OK) || (pcb == NULL)) {
|
||||
return ERR_VAL;
|
||||
|
||||
+1
-1
@@ -1017,7 +1017,7 @@ mqtt_tcp_err_cb(void *arg, err_t err)
|
||||
{
|
||||
mqtt_client_t *client = (mqtt_client_t *)arg;
|
||||
LWIP_UNUSED_ARG(err); /* only used for debug output */
|
||||
LWIP_DEBUGF(MQTT_DEBUG_TRACE, ("mqtt_tcp_err_cb: TCP error callback: error %d, arg: %p\n", err, arg));
|
||||
LWIP_DEBUGF(MQTT_DEBUG_TRACE, ("mqtt_tcp_err_cb: TCP error callback: error %d, arg: 0x%016llX\n", err, arg));
|
||||
LWIP_ASSERT("mqtt_tcp_err_cb: client != NULL", client != NULL);
|
||||
/* Set conn to null before calling close as pcb is already deallocated*/
|
||||
client->conn = NULL;
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@
|
||||
@code{.c}
|
||||
void my_smtp_result_fn(void *arg, u8_t smtp_result, u16_t srv_err, err_t err)
|
||||
{
|
||||
printf("mail (%p) sent with results: 0x%02x, 0x%04x, 0x%08x\n", arg,
|
||||
printf("mail (0x%016llX) sent with results: 0x%02x, 0x%04x, 0x%08x\n", arg,
|
||||
smtp_result, srv_err, err);
|
||||
}
|
||||
static void my_smtp_test(void)
|
||||
|
||||
+2
-2
@@ -264,7 +264,7 @@ inet_cksum_pseudo_base(struct pbuf *p, u8_t proto, u16_t proto_len, u32_t acc)
|
||||
|
||||
/* iterate through all pbuf in chain */
|
||||
for (q = p; q != NULL; q = q->next) {
|
||||
LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf %p (has next %p) \n",
|
||||
LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf 0x%016llX (has next 0x%016llX) \n",
|
||||
(void *)q, (void *)q->next));
|
||||
acc += LWIP_CHKSUM(q->payload, q->len);
|
||||
/*LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): unwrapped lwip_chksum()=%"X32_F" \n", acc));*/
|
||||
@@ -405,7 +405,7 @@ inet_cksum_pseudo_partial_base(struct pbuf *p, u8_t proto, u16_t proto_len,
|
||||
|
||||
/* iterate through all pbuf in chain */
|
||||
for (q = p; (q != NULL) && (chksum_len > 0); q = q->next) {
|
||||
LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf %p (has next %p) \n",
|
||||
LWIP_DEBUGF(INET_DEBUG, ("inet_chksum_pseudo(): checksumming pbuf 0x%016llX (has next 0x%016llX) \n",
|
||||
(void *)q, (void *)q->next));
|
||||
chklen = q->len;
|
||||
if (chklen > chksum_len) {
|
||||
|
||||
+1
-1
@@ -185,7 +185,7 @@ acd_start(struct netif *netif, struct acd *acd, ip4_addr_t ipaddr)
|
||||
|
||||
LWIP_UNUSED_ARG(netif);
|
||||
LWIP_DEBUGF(ACD_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE,
|
||||
("acd_start(netif=%p) %c%c%"U16_F"\n",
|
||||
("acd_start(netif=0x%016llX) %c%c%"U16_F"\n",
|
||||
(void *)netif, netif->name[0],
|
||||
netif->name[1], (u16_t)netif->num));
|
||||
|
||||
|
||||
+3
-3
@@ -190,7 +190,7 @@ autoip_bind(struct netif *netif)
|
||||
autoip->state = AUTOIP_STATE_BOUND;
|
||||
|
||||
LWIP_DEBUGF(AUTOIP_DEBUG | LWIP_DBG_TRACE,
|
||||
("autoip_bind(netif=%p) %c%c%"U16_F" %"U16_F".%"U16_F".%"U16_F".%"U16_F"\n",
|
||||
("autoip_bind(netif=0x%016llX) %c%c%"U16_F" %"U16_F".%"U16_F".%"U16_F".%"U16_F"\n",
|
||||
(void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num,
|
||||
ip4_addr1_16(&autoip->llipaddr), ip4_addr2_16(&autoip->llipaddr),
|
||||
ip4_addr3_16(&autoip->llipaddr), ip4_addr4_16(&autoip->llipaddr)));
|
||||
@@ -265,7 +265,7 @@ autoip_start(struct netif *netif)
|
||||
|
||||
if (autoip->state == AUTOIP_STATE_OFF) {
|
||||
LWIP_DEBUGF(AUTOIP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE,
|
||||
("autoip_start(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0],
|
||||
("autoip_start(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0],
|
||||
netif->name[1], (u16_t)netif->num));
|
||||
|
||||
/* add acd struct to list*/
|
||||
@@ -282,7 +282,7 @@ autoip_start(struct netif *netif)
|
||||
acd_start(netif, &autoip->acd, autoip->llipaddr);
|
||||
} else {
|
||||
LWIP_DEBUGF(AUTOIP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE,
|
||||
("autoip_start(): already started on netif=%p %c%c%"U16_F"\n",
|
||||
("autoip_start(): already started on netif=0x%016llX %c%c%"U16_F"\n",
|
||||
(void *)netif, netif->name[0],
|
||||
netif->name[1], (u16_t)netif->num));
|
||||
}
|
||||
|
||||
+8
-8
@@ -325,7 +325,7 @@ dhcp_handle_nak(struct netif *netif)
|
||||
{
|
||||
struct dhcp *dhcp = netif_dhcp_data(netif);
|
||||
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_nak(netif=%p) %c%c%"U16_F"\n",
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_nak(netif=0x%016llX) %c%c%"U16_F"\n",
|
||||
(void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
/* Change to a defined state - set this before assigning the address
|
||||
to ensure the callback can use dhcp_supplied_address() */
|
||||
@@ -394,7 +394,7 @@ dhcp_check(struct netif *netif)
|
||||
{
|
||||
struct dhcp *dhcp = netif_dhcp_data(netif);
|
||||
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_check(netif=%p) %c%c\n", (void *)netif, (s16_t)netif->name[0],
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_check(netif=0x%016llX) %c%c\n", (void *)netif, (s16_t)netif->name[0],
|
||||
(s16_t)netif->name[1]));
|
||||
dhcp_set_state(dhcp, DHCP_STATE_CHECKING);
|
||||
|
||||
@@ -413,7 +413,7 @@ dhcp_handle_offer(struct netif *netif, struct dhcp_msg *msg_in)
|
||||
{
|
||||
struct dhcp *dhcp = netif_dhcp_data(netif);
|
||||
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_offer(netif=%p) %c%c%"U16_F"\n",
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_handle_offer(netif=0x%016llX) %c%c%"U16_F"\n",
|
||||
(void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
/* obtain the server address */
|
||||
if (dhcp_option_given(dhcp, DHCP_OPTION_IDX_SERVER_ID)) {
|
||||
@@ -430,7 +430,7 @@ dhcp_handle_offer(struct netif *netif, struct dhcp_msg *msg_in)
|
||||
dhcp_select(netif);
|
||||
} else {
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_LEVEL_SERIOUS,
|
||||
("dhcp_handle_offer(netif=%p) did not get server ID!\n", (void *)netif));
|
||||
("dhcp_handle_offer(netif=0x%016llX) did not get server ID!\n", (void *)netif));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,7 +456,7 @@ dhcp_select(struct netif *netif)
|
||||
dhcp = netif_dhcp_data(netif);
|
||||
LWIP_ERROR("dhcp_select: dhcp != NULL", (dhcp != NULL), return ERR_VAL;);
|
||||
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_select(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_select(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
dhcp_set_state(dhcp, DHCP_STATE_REQUESTING);
|
||||
|
||||
/* create and initialize the DHCP message header */
|
||||
@@ -816,7 +816,7 @@ dhcp_start(struct netif *netif)
|
||||
LWIP_ERROR("netif != NULL", (netif != NULL), return ERR_ARG;);
|
||||
LWIP_ERROR("netif is not up, old style port?", netif_is_up(netif), return ERR_ARG;);
|
||||
dhcp = netif_dhcp_data(netif);
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp_start(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE | LWIP_DBG_STATE, ("dhcp_start(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
|
||||
/* check MTU of the netif */
|
||||
if (netif->mtu < DHCP_MAX_MSG_LEN_MIN_REQUIRED) {
|
||||
@@ -1087,7 +1087,7 @@ dhcp_bind(struct netif *netif)
|
||||
LWIP_ERROR("dhcp_bind: netif != NULL", (netif != NULL), return;);
|
||||
dhcp = netif_dhcp_data(netif);
|
||||
LWIP_ERROR("dhcp_bind: dhcp != NULL", (dhcp != NULL), return;);
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_bind(netif=%p) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_bind(netif=0x%016llX) %c%c%"U16_F"\n", (void *)netif, netif->name[0], netif->name[1], (u16_t)netif->num));
|
||||
|
||||
/* reset time used of lease */
|
||||
dhcp->lease_used = 0;
|
||||
@@ -1777,7 +1777,7 @@ dhcp_recv(void *arg, struct udp_pcb *pcb, struct pbuf *p, const ip_addr_t *addr,
|
||||
|
||||
LWIP_ASSERT("invalid server address type", IP_IS_V4(addr));
|
||||
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_recv(pbuf = %p) from DHCP server %"U16_F".%"U16_F".%"U16_F".%"U16_F" port %"U16_F"\n", (void *)p,
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("dhcp_recv(pbuf = 0x%016llX) from DHCP server %"U16_F".%"U16_F".%"U16_F".%"U16_F" port %"U16_F"\n", (void *)p,
|
||||
ip4_addr1_16(ip_2_ip4(addr)), ip4_addr2_16(ip_2_ip4(addr)), ip4_addr3_16(ip_2_ip4(addr)), ip4_addr4_16(ip_2_ip4(addr)), port));
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("pbuf->len = %"U16_F"\n", p->len));
|
||||
LWIP_DEBUGF(DHCP_DEBUG | LWIP_DBG_TRACE, ("pbuf->tot_len = %"U16_F"\n", p->tot_len));
|
||||
|
||||
Loaded 100 of 130 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user