maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite comments into one, unified style

Browse Source
This commit is contained in:
maksw committed 2026-06-15 10:17:48 +02:00
1 parent 6260de0085
commit 467a58c9c6
163 files changed
+3560 -2556
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/boot/uefi/main.c
+93 -95
View File
@@ -1,5 +1,5 @@
// boot/main.c - UEFI bootloader
//
// UEFI bootloader entry point. This file parses configuration, loads PE images, builds the initial page tables, converts the EFI memory map, and hands a BOOT_INFO block to the kernel.
// Responsibilities (see design §5.2):
// 1. Parse boot.ini
// 2. Read drivers/drivers.ini
@@ -10,8 +10,7 @@
// 7. Convert EFI memory map -> MEMORY_MAP
// 8. ExitBootServices
// 9. Set up minimal GDT/IDT, enable NXE, set CR3
// 10. Fill BOOT_INFO and jump to KernelMain
//
// 10. Fill BOOT_INFO and jump to KernelMain.
#include <efi.h>
#include <efilib.h>
@@ -28,6 +27,7 @@ int _fltused = 0;
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
#define PAGE_SIZE 4096ULL
// Implements the ascii Str To Unicode Str S operation.
static int AsciiStrToUnicodeStrS(const char* Source, uint16_t* Dest, size_t DestMax) {
if (!Source || !Dest || !DestMax) return -1;
@@ -39,6 +39,7 @@ static int AsciiStrToUnicodeStrS(const char* Source, uint16_t* Dest, size_t Dest
return -1; // Buffer too small
}
// Implements the ascii Str Cpy S operation.
int AsciiStrCpyS(char* Dest, size_t DestMax, const char* Source) {
if (!Dest || !Source || !DestMax) return -1;
@@ -50,11 +51,12 @@ int AsciiStrCpyS(char* Dest, size_t DestMax, const char* Source) {
return -1;
}
// Implements the unicode Str To Ascii Str S operation.
int UnicodeStrToAsciiStrS(const uint16_t* Source, char* Dest, size_t DestMax) {
if (!Source || !Dest || !DestMax) return -1;
for (size_t i = 0; i < DestMax; i++) {
// Narrowing: only taking the lower 8 bits
// Narrowing: only taking the lower 8 bits.
Dest[i] = (char)(Source[i] & 0xFF);
if (Source[i] == 0) return 0;
@@ -64,9 +66,7 @@ int UnicodeStrToAsciiStrS(const uint16_t* Source, char* Dest, size_t DestMax) {
return -1;
}
// ---------------------------------------------------------------------------
// PE32+ checksum validation
// ---------------------------------------------------------------------------
// Validates the PE32+ checksum using the same field layout the Windows loader expects.
static UINT32 ComputePeChecksum(const UINT8* buf, UINTN size) {
UINT64 checksum = 0;
const UINT16* words = (const UINT16 *)buf;
@@ -88,20 +88,18 @@ static UINT32 ComputePeChecksum(const UINT8* buf, UINTN size) {
checksum = (checksum & 0xFFFF) + (checksum >> 16);
checksum += (UINT64)size;
// Restore.
// Restore the image buffer before returning so checksum validation is non-destructive.
((UINT16 *)buf)[cksumOff / 2] = savedLo;
((UINT16 *)buf)[cksumOff / 2 + 1] = savedHi;
return (UINT32)checksum;
}
// ---------------------------------------------------------------------------
// Page table helpers
// ---------------------------------------------------------------------------
// Allocates and links the minimal x86-64 page tables used before KernelMain runs.
static UINT64* AllocPage(EFI_PHYSICAL_ADDRESS* physOut) {
VOID* mem = NULL;
gBS->AllocatePages(AllocateAnyPages, EfiLoaderData, 1, (EFI_PHYSICAL_ADDRESS *)&mem);
gBS->SetMem(mem, PAGE_SIZE, 0);
if (physOut) *physOut = (EFI_PHYSICAL_ADDRESS)mem; // identity-mapped in UEFI
if (physOut) *physOut = (EFI_PHYSICAL_ADDRESS)mem; // UEFI identity maps loader allocations here.
return (UINT64 *)mem;
}
@@ -115,6 +113,7 @@ static UINT64* AllocPage(EFI_PHYSICAL_ADDRESS* physOut) {
static UINT64* gPml4;
// Implements the ensure Table operation.
static UINT64* EnsureTable(UINT64* parent, UINTN index) {
UINT64* entry = &parent[index];
if (!(*entry & 1)) {
@@ -126,6 +125,7 @@ static UINT64* EnsureTable(UINT64* parent, UINTN index) {
return (UINT64 *)(*entry & ~0xFFFULL);
}
// Implements the map4 K operation.
static void Map4K(UINT64 virt, UINT64 phys, UINT64 flags) {
UINT64* pdpt = EnsureTable(gPml4, PML4_IDX(virt));
UINT64* pdt = EnsureTable(pdpt, PDPT_IDX(virt));
@@ -133,17 +133,20 @@ static void Map4K(UINT64 virt, UINT64 phys, UINT64 flags) {
pt[PT_IDX(virt)] = phys | flags;
}
// Implements the map2 M operation.
static void Map2M(UINT64 virt, UINT64 phys, UINT64 flags) {
UINT64* pdpt = EnsureTable(gPml4, PML4_IDX(virt));
UINT64* pdt = EnsureTable(pdpt, PDPT_IDX(virt));
pdt[PDT_IDX(virt)] = phys | flags | 0x80ULL; // PS bit
}
// Implements the map1 G operation.
static void Map1G(UINT64 virt, UINT64 phys, UINT64 flags) {
UINT64* pdpt = EnsureTable(gPml4, PML4_IDX(virt));
pdpt[PDPT_IDX(virt)] = phys | flags | 0x80ULL; // PS bit
}
// Maps range Smart.
static void MapRangeSmart(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags) {
while (size) {
if ((virt & (PAGE_1G - 1)) == 0 && (phys & (PAGE_1G - 1)) == 0 && size >= PAGE_1G) {
@@ -195,34 +198,33 @@ static void MapRangeSkipping(UINT64 virt, UINT64 phys, UINT64 size, UINT64 flags
}
}
// ---------------------------------------------------------------------------
// File loading
// ---------------------------------------------------------------------------
// File loading.
static EFI_FILE_PROTOCOL* gRoot;
// Loads file.
static UINT8* LoadFile(CHAR16* path, UINTN* sizeOut) {
EFI_STATUS Status;
EFI_FILE_PROTOCOL* fh;
// Open the file
// Open the file.
Status = gRoot->Open(gRoot, &fh, path, EFI_FILE_MODE_READ, 0);
if (EFI_ERROR(Status)) return NULL;
// Get the correct buffer size for EFI_FILE_INFO
// Get the correct buffer size for EFI_FILE_INFO.
EFI_FILE_INFO* info;
UINTN infoSz = 0;
// This first call will fail but will return the required size in infoSz
// This first call will fail but will return the required size in infoSz.
fh->GetInfo(fh, &gEfiFileInfoGuid, &infoSz, NULL);
// Allocate the buffer for the info struct
// Allocate the buffer for the info struct.
info = AllocatePool(infoSz);
if (info == NULL) {
fh->Close(fh);
return NULL;
}
// Now actually get the info
// Now actually get the info.
Status = fh->GetInfo(fh, &gEfiFileInfoGuid, &infoSz, info);
if (EFI_ERROR(Status)) {
FreePool(info);
@@ -230,7 +232,7 @@ static UINT8* LoadFile(CHAR16* path, UINTN* sizeOut) {
return NULL;
}
// Allocate buffer for file content
// Allocate buffer for file content.
UINTN sz = (UINTN)info->FileSize;
FreePool(info); // Don't need this anymore
@@ -240,7 +242,7 @@ static UINT8* LoadFile(CHAR16* path, UINTN* sizeOut) {
return NULL;
}
// Read the file into the buffer
// Read the file into the buffer.
Status = fh->Read(fh, &sz, buf);
fh->Close(fh);
@@ -253,9 +255,7 @@ static UINT8* LoadFile(CHAR16* path, UINTN* sizeOut) {
return buf;
}
// ---------------------------------------------------------------------------
// INI parsing state
// ---------------------------------------------------------------------------
// INI parsing state.
typedef struct {
BOOT_INFO* bi;
CHAR16 drivers[32][64];
@@ -265,6 +265,7 @@ typedef struct {
} PARSE_STATE;
// Parses boot Ini Integer.
static int ParseBootIniInteger(const char* val) {
int out = 0;
if (!val) return 0;
@@ -290,10 +291,10 @@ static int ParseBootIniInteger(const char* val) {
static void BootIniCb(const char* section, const char* key,
const char* val, void* ud) {
PARSE_STATE* s = (PARSE_STATE *)ud;
// [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log
// [kernel] debug = bitmask: 1=console, 2=selftests, 4=file log.
if (!AsciiStrCmp(section, "kernel") && !AsciiStrCmp(key, "debug"))
s->debug = ParseBootIniInteger(val);
// [init] commandline = hello.exe
// [init] commandline = hello.exe.
else if (!AsciiStrCmp(section, "init") && !AsciiStrCmp(key, "commandline"))
AsciiStrCpyS(s->commandLine, 64, val);
}
@@ -308,9 +309,7 @@ static void DriversIniCb(const char* section, const char* key,
}
}
// ---------------------------------------------------------------------------
// EFI Memory Map -> MEMORY_MAP conversion
// ---------------------------------------------------------------------------
// EFI Memory Map -> MEMORY_MAP conversion.
static EFI_PHYSICAL_ADDRESS AllocateApTrampolinePageBelow1M(void) {
// AP startup via SIPI requires the first instruction page to live below
@@ -326,6 +325,7 @@ static EFI_PHYSICAL_ADDRESS AllocateApTrampolinePageBelow1M(void) {
return phys;
}
// Converts memory Map.
static void ConvertMemoryMap(MEMORY_MAP* out) {
UINTN mapSize = 0, descSz = 0, mapKey; UINT32 descVer;
gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer);
@@ -402,7 +402,7 @@ typedef struct {
#define IMAGE_ORDINAL_FLAG64 0x8000000000000000ULL
// --- Essential PE Structures (if not in your headers) ---
// These PE structures are kept local because the bootloader only needs the subset used by image loading.
#pragma pack(push, 1)
typedef struct {
UINT32 VirtualAddress;
@@ -428,36 +428,36 @@ typedef struct {
} IMAGE_SECTION_HEADER;
#pragma pack(pop)
// --- The PE Loader ---
// The PE loader maps image sections into their final virtual addresses before the kernel switches page tables.
EFI_STATUS LoadPeImage(UINT8* rawFile, UINT64 virtDest, EFI_PHYSICAL_ADDRESS* outPhys) {
UINT32 peOff = *(UINT32*)(rawFile + 0x3C);
UINT8* ntHeaders = rawFile + peOff;
// Validate Signature
// Validate Signature.
if (*(UINT32*)ntHeaders != 0x00004550) { Print(L"Signature mismatch\n"); return EFI_INVALID_PARAMETER; } // "PE\0\0"
UINT16 sizeOfOptHeader = *(UINT16*)(ntHeaders + 20);
UINT16 numSections = *(UINT16*)(ntHeaders + 6);
// Offsets for PE32+ (64-bit)
// Offsets for PE32+ (64-bit).
UINT64 imageBase = *(UINT64*)(ntHeaders + 24 + 24);
UINT32 sizeOfImage = *(UINT32*)(ntHeaders + 24 + 56);
UINT32 sizeOfHeaders = *(UINT32*)(ntHeaders + 24 + 60);
IMAGE_DATA_DIRECTORY* dataDirs = (IMAGE_DATA_DIRECTORY*)(ntHeaders + 24 + 112);
IMAGE_DATA_DIRECTORY* relocDir = &dataDirs[5];
//IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1];
//IMAGE_DATA_DIRECTORY* importDir = &dataDirs[1];.
// Allocate Physical Memory for the mapped image
// Allocate Physical Memory for the mapped image.
UINTN pages = (sizeOfImage + PAGE_SIZE - 1) / PAGE_SIZE;
EFI_PHYSICAL_ADDRESS physBase;
gBS->AllocatePages(AllocateAnyPages, EfiLoaderData, pages, &physBase);
gBS->SetMem((VOID*)physBase, pages * PAGE_SIZE, 0);
// Copy Headers
// Copy Headers.
gBS->CopyMem((VOID*)physBase, rawFile, sizeOfHeaders);
// Map Sections to their RVAs
// Map Sections to their RVAs.
IMAGE_SECTION_HEADER* sections = (IMAGE_SECTION_HEADER*)(ntHeaders + 24 + sizeOfOptHeader);
for (int i = 0; i < numSections; i++) {
if (sections[i].SizeOfRawData > 0) {
@@ -497,11 +497,12 @@ EFI_STATUS LoadPeImage(UINT8* rawFile, UINT64 virtDest, EFI_PHYSICAL_ADDRESS* ou
return EFI_SUCCESS;
}
// Returns export Rva.
static UINT32 GetExportRva(UINT8* providerPhys, const char* funcName) {
UINT32 peOff = *(UINT32*)(providerPhys + 0x3C);
UINT8* ntHeaders = providerPhys + peOff;
// Export Dir is index 0 in the DataDirectory array
// Export Dir is index 0 in the DataDirectory array.
UINT32 exportDirRva = *(UINT32*)(ntHeaders + 24 + 112);
if (!exportDirRva) return 0; // No exports!
@@ -511,7 +512,7 @@ static UINT32 GetExportRva(UINT8* providerPhys, const char* funcName) {
UINT32* addrOfNames = (UINT32*)(providerPhys + exports->AddressOfNames);
UINT16* addrOfOrdinals = (UINT16*)(providerPhys + exports->AddressOfNameOrdinals);
// Binary search is faster, but linear is fine for a bootloader
// Binary search is faster, but linear is fine for a bootloader.
for (UINT32 i = 0; i < exports->NumberOfNames; i++) {
char* name = (char*)(providerPhys + addrOfNames[i]);
if (AsciiStrCmp(name, funcName) == 0) {
@@ -526,13 +527,13 @@ EFI_STATUS ResolveImports(UINT8* consumerPhys, UINT8* providerPhys, UINT64 provi
UINT32 peOff = *(UINT32*)(consumerPhys + 0x3C);
UINT8* ntHeaders = consumerPhys + peOff;
// Import Dir is index 1 in the DataDirectory array
// Import Dir is index 1 in the DataDirectory array.
UINT32 importDirRva = *(UINT32*)(ntHeaders + 24 + 112 + 8);
if (!importDirRva) return EFI_SUCCESS; // No imports to resolve
IMAGE_IMPORT_DESCRIPTOR* importDesc = (IMAGE_IMPORT_DESCRIPTOR*)(consumerPhys + importDirRva);
// Iterate through all imported DLLs
// Iterate through all imported DLLs.
while (importDesc->Name != 0) {
char* dllName = (char*)(consumerPhys + importDesc->Name);
@@ -551,11 +552,11 @@ EFI_STATUS ResolveImports(UINT8* consumerPhys, UINT8* providerPhys, UINT64 provi
int i = 0;
while (origFirstThunk[i].u1.Function != 0) {
if (origFirstThunk[i].u1.Ordinal & IMAGE_ORDINAL_FLAG64) {
// Import by Ordinal (Usually you don't do this in modern kernels, but good to handle)
// Import by Ordinal (Usually you don't do this in modern kernels, but good to handle).
Print(L"Error: Ordinal imports not supported in basic loader.\n");
return EFI_UNSUPPORTED;
} else {
// Import by Name
// Import by Name.
IMAGE_IMPORT_BY_NAME* importByName = (IMAGE_IMPORT_BY_NAME*)(consumerPhys + origFirstThunk[i].u1.AddressOfData);
char* funcName = (char*)importByName->Name;
@@ -579,17 +580,15 @@ EFI_STATUS ResolveImports(UINT8* consumerPhys, UINT8* providerPhys, UINT64 provi
return EFI_SUCCESS;
}
// ---------------------------------------------------------------------------
// UefiMain
// ---------------------------------------------------------------------------
// UefiMain.
EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable) {
InitializeLib(ImageHandle, SystemTable);
// Open root filesystem on the device we actually booted from
// Open root filesystem on the device we actually booted from.
EFI_LOADED_IMAGE_PROTOCOL* LoadedImage;
EFI_SIMPLE_FILE_SYSTEM_PROTOCOL* fs;
// Get the LoadedImage protocol for this application
// Get the LoadedImage protocol for this application.
EFI_STATUS Status = gBS->HandleProtocol(
ImageHandle,
&gEfiLoadedImageProtocolGuid,
@@ -601,7 +600,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
return Status;
}
// Get the SimpleFileSystem protocol for the device we were loaded from
// Get the SimpleFileSystem protocol for the device we were loaded from.
Status = gBS->HandleProtocol(
LoadedImage->DeviceHandle,
&gEfiSimpleFileSystemProtocolGuid,
@@ -613,16 +612,16 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
return Status;
}
// Open the volume
// Open the volume.
Status = fs->OpenVolume(fs, &gRoot);
if (EFI_ERROR(Status)) {
Print(L"Failed to open root volume\n");
return Status;
}
//Print(L"1\n");
//Print(L"1\n");.
// --- Parse boot.ini ---
// boot.ini controls debugger and runtime policy before the kernel has a filesystem service.
UINTN sz;
UINT8* bootIniRaw = LoadFile(L"boot.ini", &sz);
PARSE_STATE ps = {0};
@@ -633,9 +632,9 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
FreePool(bootIniRaw);
}
//Print(L"2\n");
//Print(L"2\n");.
// --- Parse drivers/drivers.ini ---
// drivers.ini decides which boot and runtime drivers are loaded into the initial module list.
UINT8* drvIniRaw = LoadFile(L"drivers\\drivers.ini", &sz);
if (drvIniRaw) {
char* txt = (char *)drvIniRaw;
@@ -644,7 +643,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
FreePool(drvIniRaw);
}
//Print(L"3\n");
//Print(L"3\n");.
BOOT_INFO* bi = AllocateZeroPool(sizeof(BOOT_INFO));
bi->Magic = BOOT_INFO_MAGIC;
@@ -676,23 +675,24 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
// Current mapping cursor for upper half.
UINT64 virtCursor = KERNEL_VIRT_BASE;
//Print(L"4\n");
//Print(L"4\n");.
// --- Load and validate PE images ---
// Core images are loaded first because driver import resolution depends on them.
typedef struct {
CHAR16* path;
char name[32];
UINT8* physBase; // Temporary storage for import resolution
UINT64 virtBase;
CHAR16* path;
char name[32];
UINT8* physBase; // Temporary storage for import resolution.
UINT64 virtBase;
} MODULE_LOAD_INFO;
MODULE_LOAD_INFO core[] = {
{ L"system\\colonel.exe", "kernel.exe", NULL, KERNEL_VIRT_BASE },
{ L"system\\hal.dll", "hal.dll", NULL, 0 /* Calculated dynamically */ },
// The HAL virtual size is filled after the image is loaded because it depends on the PE headers.
{ L"system\\hal.dll", "hal.dll", NULL, 0 },
};
for (int i = 0; i < 2; i++) {
//Print(L"Loading %s...\n", core[i].path);
// The firmware console can be noisy here, so per-module load prints stay disabled unless early loader debugging is needed.
UINTN rawSize;
UINT8* rawBuf = LoadFile(core[i].path, &rawSize);
if (!rawBuf) {
@@ -701,7 +701,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
return EFI_NOT_FOUND;
}
// Map sections and apply base relocations
// Map sections and apply base relocations.
EFI_PHYSICAL_ADDRESS physBase;
EFI_STATUS status = LoadPeImage(rawBuf, virtCursor, &physBase);
if (EFI_ERROR(status)) {
@@ -710,30 +710,30 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
return status;
}
// Store locations for the Import Resolver
// Store image locations so the import resolver can patch cross-module references.
core[i].physBase = (UINT8*)physBase;
core[i].virtBase = virtCursor;
// Map the pages into the new PML4
// We need the SizeOfImage from the mapped headers
// Images are mapped by SizeOfImage so zero-filled sections occupy their final virtual range.
UINT32 peOff = *(UINT32*)(rawBuf + 0x3C);
UINT32 imgLen = *(UINT32*)(rawBuf + peOff + 4 + 20 + 56); // SizeOfImage
imgLen = (imgLen + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
for (UINT64 off = 0; off < imgLen; off += PAGE_SIZE) {
Map4K(virtCursor + off, physBase + off, 0x03 /* P|W */);
// The bootloader keeps the flag literal local because this file owns its minimal page-table encoder.
Map4K(virtCursor + off, physBase + off, 0x03);
}
// Fill BOOT_INFO
// Fill BOOT_INFO.
AsciiStrCpyS(bi->Modules[bi->ModuleCount].Name, 32, core[i].name);
bi->Modules[bi->ModuleCount].VirtBase = virtCursor;
bi->Modules[bi->ModuleCount].ImageSize = imgLen;
bi->ModuleCount++;
// Increment cursor (1MB alignment for safety)
// Increment cursor (1MB alignment for safety).
virtCursor = (virtCursor + imgLen + 0xFFFFF) & ~0xFFFFF;
// We can free the raw file buffer now; sections are in physBase
// We can free the raw file buffer now; sections are in physBase.
FreePool(rawBuf);
}
@@ -742,9 +742,9 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
while(true);
}
//Print(L"5\n");
//Print(L"5\n");.
// Load driver .sys files into raw buffers
// Load driver .sys files into raw buffers.
for (int i = 0; i < ps.driverCount; i++) {
CHAR16 path[128];
UnicodeSPrint(path, sizeof(path), L"drivers\\%s", ps.drivers[i]);
@@ -756,7 +756,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
continue;
}
// Store file info for the kernel to process later
// Store file info for the kernel to process later.
char name[32];
UnicodeStrToAsciiStrS(ps.drivers[i], name, 32);
@@ -767,9 +767,9 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
mod->VirtBase = 0; // Kernel will map this later
}
//Print(L"6\n");
//Print(L"6\n");.
// Query GOP FIRST, before the GetMemoryMap loop
// Query GOP FIRST, before the GetMemoryMap loop.
EFI_GRAPHICS_OUTPUT_PROTOCOL* gop;
UINT64 fbBase = 0;
UINT64 fbBytes = 0;
@@ -795,11 +795,11 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
}
}
//Print(L"7\n");
//Print(L"7\n");.
bi->ApTrampolinePhys = (uintptr_t)AllocateApTrampolinePageBelow1M();
// --- Map physical RAM as direct window AND identity map ---
// Usable RAM is mapped both through the physical direct window and identity mappings until the final CR3 switch.
UINTN mapSize = 0, descSz = 0, mapKey; UINT32 descVer;
gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer);
mapSize += 8 * descSz;
@@ -886,9 +886,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
}
FreePool(efiMap);
// -------------------------------------------------------------------------
// Explicitly Map Standard Architectural MMIO
// -------------------------------------------------------------------------
// Motherboards frequently omit the APIC/IOAPIC/HPET from the UEFI memory map.
// We explicitly map the region from 0xFEC00000 up through 0xFEE00FFF
// to guarantee the HAL can access them via the physical window.
@@ -897,24 +895,24 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
Map4K(PHYS_MAP_BASE + phys, phys, 0x13); // Physical Window Map, P|W|PCD
}
//Print(L"8\n");
//Print(L"8\n");.
// --- Convert EFI memory map -> MEMORY_MAP ---
// The kernel consumes a compact memory map so it does not need EFI descriptor semantics after boot.
ConvertMemoryMap(&bi->MemoryMap);
// --- Locate ACPI RSDP ---
// Define the standard UEFI ACPI GUIDs
// The RSDP physical address is captured before firmware services are unavailable.
// Define the standard UEFI ACPI GUIDs.
EFI_GUID acpi2Guid = { 0x8868e871, 0xe4f1, 0x11d3, {0xbc, 0x22, 0x00, 0x80, 0xc7, 0x3c, 0x88, 0x81} };
EFI_GUID acpi1Guid = { 0xeb9d2d30, 0x2d88, 0x11d3, {0x9a, 0x16, 0x00, 0x90, 0x27, 0x3f, 0xc1, 0x4d} };
bi->RsdpPhysical = 0;
void* acpi1Table = NULL;
// Search the UEFI Configuration Table
// Search the UEFI Configuration Table.
for (UINTN i = 0; i < SystemTable->NumberOfTableEntries; i++) {
EFI_GUID* vendorGuid = &SystemTable->ConfigurationTable[i].VendorGuid;
// Check for ACPI 2.0 (Preferred)
// Check for ACPI 2.0 (Preferred).
if (vendorGuid->Data1 == acpi2Guid.Data1 &&
vendorGuid->Data2 == acpi2Guid.Data2 &&
vendorGuid->Data3 == acpi2Guid.Data3 &&
@@ -925,7 +923,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
break; // Found the best one, stop searching
}
// Check for ACPI 1.0 (Fallback)
// Check for ACPI 1.0 (Fallback).
if (vendorGuid->Data1 == acpi1Guid.Data1 &&
vendorGuid->Data2 == acpi1Guid.Data2 &&
vendorGuid->Data3 == acpi1Guid.Data3 &&
@@ -936,24 +934,24 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
}
}
// Fallback to ACPI 1.0 if ACPI 2.0 wasn't found
// Fallback to ACPI 1.0 if ACPI 2.0 wasn't found.
if (bi->RsdpPhysical == 0 && acpi1Table != NULL) {
bi->RsdpPhysical = (UINT64)acpi1Table;
}
// Print it out to verify the bootloader actually found it before launching the kernel
//Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);
//Print(L"Found RSDP at: 0x%016llX\n", bi->RsdpPhysical);.
bi->PhysicalMapBase = PHYS_MAP_BASE;
bi->Debug = (UINT32)ps.debug;
//Print(L"9\n");
//Print(L"9\n");.
// --- ExitBootServices ---
// ExitBootServices is the point of no return for firmware allocation, file I/O, and protocol access.
gBS->GetMemoryMap(&mapSize, NULL, &mapKey, &descSz, &descVer);
gBS->ExitBootServices(ImageHandle, mapKey);
// --- Load minimal GDT / IDT, enable NXE, set CR3 ---
// The bootloader installs the CPU state the kernel expects before jumping into the upper half.
// (These are inline since we have no runtime services after ExitBootServices.)
// GDT is set up by the kernel's KeInitializeGdt; we just need a valid one to
// reach KernelMain. The bootloader installed a flat 64-bit GDT earlier via
@@ -966,7 +964,7 @@ EFI_STATUS EFIAPI UefiMain(EFI_HANDLE ImageHandle, EFI_SYSTEM_TABLE* SystemTable
// Switch to our new PML4.
__writecr3((UINT64)gPml4);
// --- Jump to KernelMain ---
// KernelMain receives BOOT_INFO in the new address space and takes ownership of the machine.
typedef void (*KERNEL_ENTRY)(BOOT_INFO *);
KERNEL_ENTRY KernelMain = (KERNEL_ENTRY)bi->Modules[0].VirtBase;
code/build/build.c
+23 -22
View File
@@ -1,10 +1,10 @@
// Project build orchestrator. This file describes the Windows/MSVC build graph and keeps driver dependencies explicit.
#define NOB_IMPLEMENTATION
#include "nob.h"
// Replicate your compiler and linker flags. Keep the literals in macros so
// they can be used in file-scope aggregate initializers. MSVC C rejects
// initializing a static object with the value of another object, even if that
// object is a const char*.
// Compiler and linker flags stay in macros so they can be used in file-scope aggregate initializers.
// MSVC C rejects initializing a static object from another object, even when the source object is const.
#define CFLAGS_LITERAL "/nologo /W3 /WX /Od /GS- /Gs999999 /Gz /Gy /EHsc- /GR- /Oi- /Z7 /std:c17 /D_CRT_SECURE_NO_WARNINGS /D_CRT_NONSTDC_NO_WARNINGS /Icode\\public /Icode\\common\\include /Icode\\kernel\\include"
#define LWIP_CFLAGS_LITERAL CFLAGS_LITERAL " /Icode\\drivers\\net\\lwip\\include /Icode\\third_party\\lwip\\src\\include /wd4005 /wd4100 /wd4103 /wd4127 /wd4244 /wd4267 /wd4706"
#define LFLAGS_LITERAL "/nologo /NODEFAULTLIB /DEBUG"
@@ -39,6 +39,7 @@ static const Driver_Target STANDALONE_DRIVERS[] = {
DRIVER_TARGET_LIBS("rtl8139", "obj\\drivers\\rtl8139", "code\\drivers\\net\\rtl8139\\rtl8139.c", "bin\\pci.lib"),
};
// Creates layout.
void create_layout() {
const char *dirs[] = {
"obj", "bin", "obj\\boot", "obj\\kernel", "obj\\lwip", "obj\\hal",
@@ -52,7 +53,7 @@ void create_layout() {
for (size_t i = 0; i < NOB_ARRAY_LEN(STANDALONE_DRIVERS); i++) nob_mkdir_if_not_exists(STANDALONE_DRIVERS[i].obj_dir);
}
// Helper to run a shell command string and wait for it
// Runs a shell command and waits for it to finish.
bool run_sync(const char *cmd_str) {
Nob_Cmd cmd = {0};
nob_cmd_append(&cmd, "cmd.exe", "/c", cmd_str);
@@ -61,7 +62,7 @@ bool run_sync(const char *cmd_str) {
return ok;
}
// Helper to launch a shell command into the background
// Launches a shell command while the build graph continues.
Nob_Proc run_async(const char *cmd_str) {
Nob_Cmd cmd = {0};
nob_cmd_append(&cmd, "cmd.exe", "/c", cmd_str);
@@ -70,18 +71,19 @@ Nob_Proc run_async(const char *cmd_str) {
return proc;
}
// Runs the requested operation.
int main(int argc, char **argv) {
// Magic macro: Checks if build.c is newer than build.exe and recompiles itself if so
// The build tool rebuilds itself before doing any real work so the graph below matches the source.
NOB_GO_REBUILD_URSELF(argc, argv);
// 1. Initial Checks & Setup
// The disk image must exist before the build can copy fresh binaries into it.
if (!nob_file_exists("disk.vhd")) {
nob_log(NOB_INFO, "disk.vhd missing, calling mkimage.bat...");
if (!run_sync("mkimage.bat")) return 1;
}
create_layout();
// 2. Sequential Phase (Bootloader, Core Kernel, and DLLs)
// Core images build sequentially because later images link against earlier import libraries.
nob_log(NOB_INFO, "--- Building Core Subsystems ---");
const char *core_tasks[] = {
@@ -96,16 +98,16 @@ int main(int argc, char **argv) {
nob_temp_sprintf("cl %s /c /Foobj\\runtime\\user_heap.obj code\\user\\runtime\\heap.c", CFLAGS),
nob_temp_sprintf("cl %s /c /Foobj\\runtime\\user_heap_exports.obj code\\user\\runtime\\heap_exports.c", CFLAGS),
// Bootloader
// Bootloader.
nob_temp_sprintf("cl %s /MP /c /Icode\\boot\\uefi\\include /Foobj\\boot\\ code\\boot\\uefi\\*.c && link %s /FORCE:MULTIPLE /SUBSYSTEM:EFI_APPLICATION /ENTRY:UefiMain /OUT:bin\\bootx64.efi obj\\libc.obj obj\\ini_parser.obj code\\boot\\uefi\\efilib.lib obj\\boot\\*.obj", CFLAGS, LFLAGS),
// HAL
// HAL.
nob_temp_sprintf("cl %s /MP /c /Foobj\\hal\\ code\\kernel\\hal\\*.c && link %s /DLL /OUT:bin\\hal.dll /ENTRY:HalInitialize /IGNORE:4216 obj\\printf.obj obj\\libc.obj obj\\hal\\*.obj", CFLAGS, LFLAGS),
// Kernel
// Kernel.
nob_temp_sprintf("cl %s /DKE_DISPLAY_BUILD /MP /c /Foobj\\kernel\\ code\\kernel\\*.c && link %s /SUBSYSTEM:NATIVE /ENTRY:KernelMain /OUT:bin\\kernel.exe bin\\hal.lib obj\\ini_parser.obj obj\\printf.obj obj\\libc.obj obj\\kernel\\*.obj", CFLAGS, LFLAGS),
// Subsystem DLLs
// Subsystem DLLs.
"link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\syscall.dll obj\\syscall\\stubs.obj",
nob_temp_sprintf("cl %s /DUSER_HEAP_EXPORTS /MP /c /Foobj\\win32\\ code\\user\\runtime\\heap.c code\\user\\runtime\\heap_exports.c code\\user\\personalities\\win32\\*.c && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\win32.dll bin\\syscall.lib obj\\libc.obj obj\\win32\\*.obj", CFLAGS),
nob_temp_sprintf("cl %s /DUSER_HEAP_EXPORTS /MP /c /Foobj\\posix\\ code\\user\\runtime\\heap.c code\\user\\runtime\\heap_exports.c code\\user\\personalities\\posix\\*.c && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\posix.dll bin\\syscall.lib obj\\libc.obj obj\\posix\\*.obj", CFLAGS)
@@ -115,8 +117,7 @@ int main(int argc, char **argv) {
if (!run_sync(core_tasks[i])) return 1;
}
// 3. Driver dependency phase. pci.sys owns PCI enumeration and exports
// bin\pci.lib, so it must be built before PCI function drivers link.
// PCI builds before PCI function drivers because it exports the bus-driver import library.
nob_log(NOB_INFO, "--- Building Bus Drivers ---");
const char *pci_cmd = nob_temp_sprintf(
"cl %s /MP /c /Fo%s\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\%s.sys /IMPLIB:bin\\%s.lib /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\printf.obj obj\\libc.obj %s\\*.obj",
@@ -125,7 +126,7 @@ int main(int argc, char **argv) {
);
if (!run_sync(pci_cmd)) return 1;
// 4. Parallel Phase (Drivers and Applications)
// 4. Parallel Phase (Drivers and Applications).
nob_log(NOB_INFO, "--- Compiling Drivers and Apps in Parallel ---");
Nob_Procs procs = {0};
@@ -141,7 +142,7 @@ int main(int argc, char **argv) {
nob_da_append(&procs, run_async(cmd));
}
// Parse lwIP Sources and Queue it
// Parse lwIP Sources and Queue it.
Nob_String_Builder sb = {0}, lwip_files = {0};
if (nob_read_entire_file("code\\drivers\\net\\lwip\\lwip_sources.txt", &sb)) {
Nob_String_View sv = nob_sv_from_parts(sb.items, sb.count);
@@ -157,9 +158,9 @@ int main(int argc, char **argv) {
nob_da_append(&procs, run_async(nob_temp_sprintf("cl %s /MP /c /Foobj\\lwip\\ %s && link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:bin\\lwip.sys /ENTRY:DriverEntry bin\\hal.lib bin\\kernel.lib obj\\libc.obj obj\\lwip\\*.obj", LWIP_CFLAGS, lwip_files.items)));
}
// Queue Subsystem Applications
// Queue Subsystem Applications.
const char *apps[][4] = {
// Folder, Libs, Name, Optional Extra Link Args
// Folder, Libs, Name, Optional Extra Link Args.
{"system\\init", "bin\\syscall.lib obj\\libc.obj obj\\runtime\\user_heap.obj obj\\runtime\\user_heap_exports.obj", "init", "/ENTRY:_start"},
{"apps\\shell_win32", "bin\\win32.lib obj\\libc.obj", "shell_win32", "/ENTRY:main"},
{"apps\\shell_posix", "bin\\posix.lib obj\\libc.obj", "shell_posix", "/ENTRY:main"},
@@ -184,16 +185,16 @@ int main(int argc, char **argv) {
nob_da_append(&procs, run_async(cmd));
}
// Wait for everything to link simultaneously
// Wait for everything to link simultaneously.
if (!nob_procs_wait(procs)) {
nob_log(NOB_ERROR, "One or more parallel build jobs failed!");
return 1;
}
// 4. Mounting & Packing Image
// 4. Mounting & Packing Image.
nob_log(NOB_INFO, "--- Mounting disk.vhd ---");
// Resolve the absolute path to disk.vhd for diskpart
// Resolve the absolute path to disk.vhd for diskpart.
char vhd_path[MAX_PATH];
GetFullPathNameA("disk.vhd", MAX_PATH, vhd_path, NULL);
code/common/include/common/ini_parser.h
+6 -4
View File
@@ -1,4 +1,6 @@
#pragma once
typedef void (*IniCallback)(const char* section, const char* key,
const char* value, void* userdata);
void IniParse(char* buf, IniCallback cb, void* userdata);
// Shared INI parser declarations for boot and kernel configuration readers.
#pragma once
typedef void (*IniCallback)(const char* section, const char* key,
const char* value, void* userdata);
void IniParse(char* buf, IniCallback cb, void* userdata);
code/common/ini_parser.c
+5 -3
View File
@@ -1,16 +1,18 @@
// ini_parser.c - minimal [section] key=value parser
// Used by the bootloader to read boot.ini and drivers/drivers.ini.
// No heap; caller provides fixed-size output buffers.
// Minimal INI parser. This file accepts the small configuration dialect used by boot.ini and drivers.ini.
#include <string.h>
#include <common/ini_parser.h>
// Checks whether ws.
static inline int IsWs(char c) { return c == ' ' || c == '\t' || c == '\r'; }
// Implements the trim Left operation.
static char* TrimLeft(char* s) {
while (*s && IsWs(*s)) s++;
return s;
}
// Implements the trim Right operation.
static void TrimRight(char* s) {
char* e = s + strlen(s);
while (e > s && IsWs(*(e-1))) *--e = '\0';
code/common/libc.c
+226 -210
View File
@@ -1,210 +1,226 @@
// everyone needs a little libc
#include <stdint.h>
#include <stddef.h>
#include <intrin.h>
char* strcpy(char* dest, const char* src) {
char* d = dest;
while (*src) *d++ = *src++;
*d = 0;
return dest;
}
char* strncpy(char* dest, const char* src, size_t n) {
size_t i;
for (i = 0; i < n && src[i]; i++) dest[i] = src[i];
for (; i < n; i++) dest[i] = 0;
return dest;
}
char* strcat(char* dest, const char* src) {
char* ptr = dest;
while (*ptr != '\0') {
ptr++;
}
while (*src != '\0') {
*ptr = *src;
ptr++;
src++;
}
*ptr = '\0';
return dest;
}
char* strncat(char* dest, const char* src, size_t n) {
char* ptr = dest;
while (*ptr != '\0') {
ptr++;
}
while (n > 0 && *src != '\0') {
*ptr = *src;
ptr++;
src++;
n--;
}
*ptr = '\0';
return dest;
}
int strcmp(const char* s1, const char* s2) {
while (*s1 && *s1 == *s2) s1++, s2++;
return (unsigned char)*s1 - (unsigned char)*s2;
}
int strncmp(const char *s1, const char *s2, size_t n) {
while (n && *s1 && (*s1 == *s2)) {
s1++;
s2++;
n--;
}
return n == 0 ? 0 : *(unsigned char *)s1 - *(unsigned char *)s2;
}
int _stricmp(const char* s1, const char* s2) {
// 0x20 (32) is the bit that differs between 'A' and 'a'
while (*s1 && ((*s1 | 0x20) == (*s2 | 0x20))) {
s1++, s2++;
}
return (unsigned char)(*s1 | 0x20) - (unsigned char)(*s2 | 0x20);
}
size_t strlen(const char* s) {
size_t n = 0;
while (*s++) n++;
return n;
}
size_t strnlen(const char *s, size_t maxlen) {
size_t len;
for (len = 0; len < maxlen; len++) {
if (s[len] == '\0') {
break;
}
}
return len;
}
char* strchr(const char* s, int c) {
while (*s) {
if (*s == (char)c) return (char *)s;
s++;
}
return NULL;
}
char* strrchr(const char* s, int c) {
char ch = (char)c;
const char* last = NULL;
do {
if (*s == ch) last = s;
} while (*s++);
return (char*)last;
}
void* memset(void* dest, int c, size_t n) {
unsigned char* d = (unsigned char*)dest;
size_t qwords = n / 8;
size_t tail = n % 8;
if (qwords > 0) {
uint64_t value = (unsigned char)c;
value |= value << 8;
value |= value << 16;
value |= value << 32;
__stosq((unsigned __int64*)d, value, qwords);
d += qwords * 8;
}
if (tail > 0) {
__stosb(d, (unsigned char)c, tail);
}
return dest;
}
void* memcpy(void* dest, const void* src, size_t n) {
unsigned char* d = (unsigned char*)dest;
const unsigned char* s = (const unsigned char*)src;
size_t qwords = n / 8;
size_t tail = n % 8;
if (qwords > 0) {
__movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords);
d += qwords * 8;
s += qwords * 8;
}
if (tail > 0) {
__movsb(d, s, tail);
}
return dest;
}
void* memmove(void* dest, const void* src, size_t n) {
unsigned char* d = (unsigned char*)dest;
const unsigned char* s = (const unsigned char*)src;
if (d < s) {
// Forward copy: safe to use chunked __movsq + tail
size_t qwords = n / 8;
size_t tail = n % 8;
if (qwords > 0) {
__movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords);
d += qwords * 8;
s += qwords * 8;
}
if (tail > 0) {
__movsb(d, s, tail);
}
} else if (d > s) {
// Backward copy: must process from end to avoid corruption
size_t qwords = n / 8;
size_t tail = n % 8;
// Copy tail bytes first from the very end
if (tail > 0) {
for (size_t i = tail; i > 0; i--) {
d[qwords * 8 + i - 1] = s[qwords * 8 + i - 1];
}
}
// Copy 64-bit chunks backward
if (qwords > 0) {
unsigned __int64* dq = (unsigned __int64*)d;
const unsigned __int64* sq = (const unsigned __int64*)s;
for (size_t i = qwords; i > 0; i--) {
dq[i - 1] = sq[i - 1];
}
}
}
return dest;
}
int memcmp(const void* s1, const void* s2, size_t n) {
const unsigned __int64* a64 = (const unsigned __int64*)s1;
const unsigned __int64* b64 = (const unsigned __int64*)s2;
size_t qwords = n / 8;
size_t tail = n % 8;
// Compare 8-byte blocks
for (size_t i = 0; i < qwords; i++) {
if (a64[i] != b64[i]) {
// Locate the exact mismatching byte within the quadword
const unsigned char* a8 = (const unsigned char*)&a64[i];
const unsigned char* b8 = (const unsigned char*)&b64[i];
for (size_t j = 0; j < 8; j++) {
if (a8[j] != b8[j]) return (int)a8[j] - (int)b8[j];
}
}
}
// Compare remaining tail bytes
const unsigned char* a8 = (const unsigned char*)(a64 + qwords);
const unsigned char* b8 = (const unsigned char*)(b64 + qwords);
for (size_t i = 0; i < tail; i++) {
if (a8[i] != b8[i]) return (int)a8[i] - (int)b8[i];
}
return 0;
}
// Small freestanding C runtime subset shared by boot, kernel, drivers, and user support libraries.
#include <stdint.h>
#include <stddef.h>
#include <intrin.h>
// Implements the strcpy operation.
char* strcpy(char* dest, const char* src) {
char* d = dest;
while (*src) *d++ = *src++;
*d = 0;
return dest;
}
// Implements the strncpy operation.
char* strncpy(char* dest, const char* src, size_t n) {
size_t i;
for (i = 0; i < n && src[i]; i++) dest[i] = src[i];
for (; i < n; i++) dest[i] = 0;
return dest;
}
// Implements the strcat operation.
char* strcat(char* dest, const char* src) {
char* ptr = dest;
while (*ptr != '\0') {
ptr++;
}
while (*src != '\0') {
*ptr = *src;
ptr++;
src++;
}
*ptr = '\0';
return dest;
}
// Implements the strncat operation.
char* strncat(char* dest, const char* src, size_t n) {
char* ptr = dest;
while (*ptr != '\0') {
ptr++;
}
while (n > 0 && *src != '\0') {
*ptr = *src;
ptr++;
src++;
n--;
}
*ptr = '\0';
return dest;
}
// Implements the strcmp operation.
int strcmp(const char* s1, const char* s2) {
while (*s1 && *s1 == *s2) s1++, s2++;
return (unsigned char)*s1 - (unsigned char)*s2;
}
// Implements the strncmp operation.
int strncmp(const char *s1, const char *s2, size_t n) {
while (n && *s1 && (*s1 == *s2)) {
s1++;
s2++;
n--;
}
return n == 0 ? 0 : *(unsigned char *)s1 - *(unsigned char *)s2;
}
// Implements the stricmp operation.
int _stricmp(const char* s1, const char* s2) {
// ASCII case folding works here because bit 0x20 distinguishes 'A' from 'a'.
while (*s1 && ((*s1 | 0x20) == (*s2 | 0x20))) {
s1++, s2++;
}
return (unsigned char)(*s1 | 0x20) - (unsigned char)(*s2 | 0x20);
}
// Implements the strlen operation.
size_t strlen(const char* s) {
size_t n = 0;
while (*s++) n++;
return n;
}
// Implements the strnlen operation.
size_t strnlen(const char *s, size_t maxlen) {
size_t len;
for (len = 0; len < maxlen; len++) {
if (s[len] == '\0') {
break;
}
}
return len;
}
// Implements the strchr operation.
char* strchr(const char* s, int c) {
while (*s) {
if (*s == (char)c) return (char *)s;
s++;
}
return NULL;
}
// Implements the strrchr operation.
char* strrchr(const char* s, int c) {
char ch = (char)c;
const char* last = NULL;
do {
if (*s == ch) last = s;
} while (*s++);
return (char*)last;
}
// Implements the memset operation.
void* memset(void* dest, int c, size_t n) {
unsigned char* d = (unsigned char*)dest;
size_t qwords = n / 8;
size_t tail = n % 8;
if (qwords > 0) {
uint64_t value = (unsigned char)c;
value |= value << 8;
value |= value << 16;
value |= value << 32;
__stosq((unsigned __int64*)d, value, qwords);
d += qwords * 8;
}
if (tail > 0) {
__stosb(d, (unsigned char)c, tail);
}
return dest;
}
// Implements the memcpy operation.
void* memcpy(void* dest, const void* src, size_t n) {
unsigned char* d = (unsigned char*)dest;
const unsigned char* s = (const unsigned char*)src;
size_t qwords = n / 8;
size_t tail = n % 8;
if (qwords > 0) {
__movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords);
d += qwords * 8;
s += qwords * 8;
}
if (tail > 0) {
__movsb(d, s, tail);
}
return dest;
}
// Implements the memmove operation.
void* memmove(void* dest, const void* src, size_t n) {
unsigned char* d = (unsigned char*)dest;
const unsigned char* s = (const unsigned char*)src;
if (d < s) {
// Forward copy: safe to use chunked __movsq + tail.
size_t qwords = n / 8;
size_t tail = n % 8;
if (qwords > 0) {
__movsq((unsigned __int64*)d, (const unsigned __int64*)s, qwords);
d += qwords * 8;
s += qwords * 8;
}
if (tail > 0) {
__movsb(d, s, tail);
}
} else if (d > s) {
// Backward copy: must process from end to avoid corruption.
size_t qwords = n / 8;
size_t tail = n % 8;
// Copy tail bytes first from the very end.
if (tail > 0) {
for (size_t i = tail; i > 0; i--) {
d[qwords * 8 + i - 1] = s[qwords * 8 + i - 1];
}
}
// Copy 64-bit chunks backward.
if (qwords > 0) {
unsigned __int64* dq = (unsigned __int64*)d;
const unsigned __int64* sq = (const unsigned __int64*)s;
for (size_t i = qwords; i > 0; i--) {
dq[i - 1] = sq[i - 1];
}
}
}
return dest;
}
// Implements the memcmp operation.
int memcmp(const void* s1, const void* s2, size_t n) {
const unsigned __int64* a64 = (const unsigned __int64*)s1;
const unsigned __int64* b64 = (const unsigned __int64*)s2;
size_t qwords = n / 8;
size_t tail = n % 8;
// Compare 8-byte blocks.
for (size_t i = 0; i < qwords; i++) {
if (a64[i] != b64[i]) {
// Locate the exact mismatching byte within the quadword.
const unsigned char* a8 = (const unsigned char*)&a64[i];
const unsigned char* b8 = (const unsigned char*)&b64[i];
for (size_t j = 0; j < 8; j++) {
if (a8[j] != b8[j]) return (int)a8[j] - (int)b8[j];
}
}
}
// Compare remaining tail bytes.
const unsigned char* a8 = (const unsigned char*)(a64 + qwords);
const unsigned char* b8 = (const unsigned char*)(b64 + qwords);
for (size_t i = 0; i < tail; i++) {
if (a8[i] != b8[i]) return (int)a8[i] - (int)b8[i];
}
return 0;
}
code/drivers/bus/pci/pci.c
+39 -6
View File
@@ -1,4 +1,5 @@
// pci.c - PCI bus driver, PDO enumerator, and exported PCI DDI
// PCI bus driver. This file owns PCI enumeration, PDO creation, BAR decoding, and explicit MMIO mappings for function drivers.
#include "pci.h"
#include <stdint.h>
@@ -20,24 +21,26 @@ static const HAL_API* gHal;
static DRIVER_OBJECT gPciDriver;
static uint32_t gPciDeviceCount;
/*
* PCI MMIO BARs use a private virtual aperture instead of the physical
* direct map. This avoids cacheability aliasing with large direct-map pages
* and keeps BAR mappings page-granular.
*/
// PCI MMIO BARs use a private virtual aperture instead of the physical
// direct map. This avoids cacheability aliasing with large direct-map pages
// and keeps BAR mappings page-granular.
#define PCI_MMIO_VA_BASE 0xFFFF900000000000ULL
#define PCI_MMIO_VA_LIMIT 0xFFFF980000000000ULL
static uintptr_t gPciMmioNext = PCI_MMIO_VA_BASE;
// Implements the align Up64 operation.
static uint64_t PciAlignUp64(uint64_t value, uint64_t alignment) {
return (value + alignment - 1u) & ~(alignment - 1u);
}
// Implements the default Mmio Flags operation.
static uint64_t PciDefaultMmioFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_WRITETHROUGH | PAGE_NOCACHE | PAGE_NX;
}
// Creates mmio Va.
static KSTATUS PciAllocateMmioVa(uint64_t length, uintptr_t* outVa) {
if (!outVa || !length)
return STATUS_INVALID_PARAMETER;
@@ -54,6 +57,7 @@ static KSTATUS PciAllocateMmioVa(uint64_t length, uintptr_t* outVa) {
return STATUS_SUCCESS;
}
// Checks whether pdo.
static bool PciIsPdo(DEVICE_OBJECT* pdo, PCI_PDO_EXTENSION** outExt) {
if (!pdo || pdo->Type != DeviceObjectPhysical || pdo->u.Pdo.BusType != IoBusPci)
return false;
@@ -63,26 +67,31 @@ static bool PciIsPdo(DEVICE_OBJECT* pdo, PCI_PDO_EXTENSION** outExt) {
return true;
}
// Implements the raw Read32 operation.
static uint32_t PciRawRead32(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
if (!gHal || offset > 0xFCu) return 0xFFFFFFFFu;
return gHal->Pci.Read32(0, bus, dev, fn, offset & 0xFCu);
}
// Implements the raw Write32 operation.
static void PciRawWrite32(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset, uint32_t value) {
if (!gHal || offset > 0xFCu) return;
gHal->Pci.Write32(0, bus, dev, fn, offset & 0xFCu, value);
}
// Implements the raw Read8 operation.
static uint8_t PciRawRead8(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
uint32_t v = PciRawRead32(bus, dev, fn, offset);
return (uint8_t)(v >> ((offset & 3u) * 8u));
}
// Implements the raw Read16 operation.
static uint16_t PciRawRead16(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
uint32_t v = PciRawRead32(bus, dev, fn, offset);
return (uint16_t)(v >> ((offset & 2u) * 8u));
}
// Implements the raw Write8 operation.
static void PciRawWrite8(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset, uint8_t value) {
uint16_t aligned = offset & ~3u;
uint32_t shift = (offset & 3u) * 8u;
@@ -92,6 +101,7 @@ static void PciRawWrite8(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset,
PciRawWrite32(bus, dev, fn, aligned, old);
}
// Implements the raw Write16 operation.
static void PciRawWrite16(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset, uint16_t value) {
uint16_t aligned = offset & ~3u;
uint32_t shift = (offset & 2u) * 8u;
@@ -101,6 +111,7 @@ static void PciRawWrite16(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset,
PciRawWrite32(bus, dev, fn, aligned, old);
}
// Implements the bar Size From Mask operation.
static uint64_t PciBarSizeFromMask(uint64_t mask, uint64_t baseMask) {
uint64_t sizeMask = mask & baseMask;
if (!sizeMask || sizeMask == baseMask)
@@ -108,6 +119,7 @@ static uint64_t PciBarSizeFromMask(uint64_t mask, uint64_t baseMask) {
return (~sizeMask + 1u) & baseMask;
}
// Implements the header Bar Count operation.
static uint8_t PciHeaderBarCount(uint8_t headerType) {
switch (headerType & PCI_HEADER_TYPE_MASK) {
case 0x00: return 6;
@@ -117,6 +129,7 @@ static uint8_t PciHeaderBarCount(uint8_t headerType) {
}
}
// Implements the decode Bars operation.
static void PciDecodeBars(PCI_PDO_EXTENSION* ext) {
uint8_t barCount = PciHeaderBarCount(ext->Info.HeaderType);
for (uint8_t i = 0; i < PCI_MAX_BARS; i++) {
@@ -255,6 +268,7 @@ static void PciDecodeBars(PCI_PDO_EXTENSION* ext) {
}
}
// Returns capability Offset.
static uint8_t PciFindCapabilityOffset(PCI_PDO_EXTENSION* ext) {
uint16_t status = (uint16_t)(PciRawRead32(ext->Info.Address.Bus,
ext->Info.Address.Device,
@@ -270,6 +284,7 @@ static uint8_t PciFindCapabilityOffset(PCI_PDO_EXTENSION* ext) {
return cap;
}
// Creates ids.
static void PciMakeIds(const PCI_DEVICE_INFO* info, IO_DEVICE_ID* ids, char storage[][IO_MAX_DEVICE_ID_CHARS], size_t* outCount) {
size_t count = 0;
@@ -311,6 +326,7 @@ static void PciMakeIds(const PCI_DEVICE_INFO* info, IO_DEVICE_ID* ids, char stor
*outCount = count;
}
// Creates pdo.
static KSTATUS PciCreatePdo(uint8_t bus, uint8_t dev, uint8_t fn) {
uint32_t id = PciRawRead32(bus, dev, fn, PCI_REG_VENDOR_DEVICE);
uint16_t vendor = (uint16_t)(id & 0xFFFFu);
@@ -385,6 +401,7 @@ static KSTATUS PciCreatePdo(uint8_t bus, uint8_t dev, uint8_t fn) {
return STATUS_SUCCESS;
}
// Implements the enumerate Bus operation.
static void PciEnumerateBus(void) {
for (uint32_t bus = 0; bus < 256u; bus++) {
for (uint32_t dev = 0; dev < 32u; dev++) {
@@ -406,6 +423,7 @@ static void PciEnumerateBus(void) {
}
}
// Dispatches pnp.
static KSTATUS PciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
@@ -425,10 +443,12 @@ static KSTATUS PciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the driver Abi Version operation.
__declspec(dllexport) uint32_t PciDriverAbiVersion(void) {
return 1u;
}
// Returns device Info.
__declspec(dllexport) KSTATUS PciGetDeviceInfo(DEVICE_OBJECT* pdo, PCI_DEVICE_INFO* outInfo) {
PCI_PDO_EXTENSION* ext;
if (!outInfo || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -436,6 +456,7 @@ __declspec(dllexport) KSTATUS PciGetDeviceInfo(DEVICE_OBJECT* pdo, PCI_DEVICE_IN
return STATUS_SUCCESS;
}
// Returns config8.
__declspec(dllexport) KSTATUS PciReadConfig8(DEVICE_OBJECT* pdo, uint16_t offset, uint8_t* outValue) {
PCI_PDO_EXTENSION* ext;
if (!outValue || offset > 0xFFu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -443,6 +464,7 @@ __declspec(dllexport) KSTATUS PciReadConfig8(DEVICE_OBJECT* pdo, uint16_t offset
return STATUS_SUCCESS;
}
// Returns config16.
__declspec(dllexport) KSTATUS PciReadConfig16(DEVICE_OBJECT* pdo, uint16_t offset, uint16_t* outValue) {
PCI_PDO_EXTENSION* ext;
if (!outValue || offset > 0xFEu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -450,6 +472,7 @@ __declspec(dllexport) KSTATUS PciReadConfig16(DEVICE_OBJECT* pdo, uint16_t offse
return STATUS_SUCCESS;
}
// Returns config32.
__declspec(dllexport) KSTATUS PciReadConfig32(DEVICE_OBJECT* pdo, uint16_t offset, uint32_t* outValue) {
PCI_PDO_EXTENSION* ext;
if (!outValue || offset > 0xFCu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -457,6 +480,7 @@ __declspec(dllexport) KSTATUS PciReadConfig32(DEVICE_OBJECT* pdo, uint16_t offse
return STATUS_SUCCESS;
}
// Updates config8.
__declspec(dllexport) KSTATUS PciWriteConfig8(DEVICE_OBJECT* pdo, uint16_t offset, uint8_t value) {
PCI_PDO_EXTENSION* ext;
if (offset > 0xFFu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -464,6 +488,7 @@ __declspec(dllexport) KSTATUS PciWriteConfig8(DEVICE_OBJECT* pdo, uint16_t offse
return STATUS_SUCCESS;
}
// Updates config16.
__declspec(dllexport) KSTATUS PciWriteConfig16(DEVICE_OBJECT* pdo, uint16_t offset, uint16_t value) {
PCI_PDO_EXTENSION* ext;
if (offset > 0xFEu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -471,6 +496,7 @@ __declspec(dllexport) KSTATUS PciWriteConfig16(DEVICE_OBJECT* pdo, uint16_t offs
return STATUS_SUCCESS;
}
// Updates config32.
__declspec(dllexport) KSTATUS PciWriteConfig32(DEVICE_OBJECT* pdo, uint16_t offset, uint32_t value) {
PCI_PDO_EXTENSION* ext;
if (offset > 0xFCu || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -478,6 +504,7 @@ __declspec(dllexport) KSTATUS PciWriteConfig32(DEVICE_OBJECT* pdo, uint16_t offs
return STATUS_SUCCESS;
}
// Returns bar.
__declspec(dllexport) KSTATUS PciGetBar(DEVICE_OBJECT* pdo, uint8_t index, PCI_BAR* outBar) {
PCI_PDO_EXTENSION* ext;
if (!outBar || index >= PCI_MAX_BARS || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -486,6 +513,7 @@ __declspec(dllexport) KSTATUS PciGetBar(DEVICE_OBJECT* pdo, uint8_t index, PCI_B
return STATUS_SUCCESS;
}
// Maps a PCI BAR through the private MMIO aperture with explicit cache attributes.
__declspec(dllexport) KSTATUS PciMapBar(DEVICE_OBJECT* pdo, uint8_t index, uint64_t pageFlags, void** outVirtual, uint64_t* outLength) {
PCI_BAR bar;
if (!outVirtual) return STATUS_INVALID_PARAMETER;
@@ -522,6 +550,7 @@ __declspec(dllexport) KSTATUS PciMapBar(DEVICE_OBJECT* pdo, uint8_t index, uint6
return STATUS_SUCCESS;
}
// Enables device.
__declspec(dllexport) KSTATUS PciEnableDevice(DEVICE_OBJECT* pdo, uint32_t flags) {
PCI_PDO_EXTENSION* ext;
if (!PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -544,6 +573,7 @@ __declspec(dllexport) KSTATUS PciEnableDevice(DEVICE_OBJECT* pdo, uint32_t flags
return STATUS_SUCCESS;
}
// Returns interrupt.
__declspec(dllexport) KSTATUS PciGetInterrupt(DEVICE_OBJECT* pdo, uint8_t* outIrq) {
PCI_PDO_EXTENSION* ext;
if (!outIrq || !PciIsPdo(pdo, &ext)) return STATUS_INVALID_PARAMETER;
@@ -552,6 +582,7 @@ __declspec(dllexport) KSTATUS PciGetInterrupt(DEVICE_OBJECT* pdo, uint8_t* outIr
return STATUS_SUCCESS;
}
// Returns capability.
static uint8_t PciFindCapability(PCI_PDO_EXTENSION* ext, uint8_t capId) {
if (!ext || !ext->CapabilityOffset)
return 0;
@@ -572,6 +603,7 @@ static uint8_t PciFindCapability(PCI_PDO_EXTENSION* ext, uint8_t capId) {
return 0;
}
// Enables msi.
__declspec(dllexport) KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot) {
PCI_PDO_EXTENSION* ext;
if (!PciIsPdo(pdo, &ext))
@@ -627,6 +659,7 @@ __declspec(dllexport) KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot)
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal) return STATUS_INVALID_PARAMETER;
gHal = hal;
code/drivers/bus/pci/pci.h
+2
View File
@@ -1,3 +1,5 @@
// PCI bus-driver declarations shared with PCI function drivers.
#pragma once
#include <stdint.h>
code/drivers/bus/usb/usb.c
+4 -1
View File
@@ -1,4 +1,5 @@
// usb.c - USB bus driver entry point and host-controller driver registration.
// USB driver package entry point. This file registers the USB bus, class, hub, HID, storage, and xHCI drivers.
#include "usb.h"
DRIVER_OBJECT gXhciDriver;
@@ -23,6 +24,7 @@ const char* UsbControllerName(uint8_t progIf) {
}
}
// Initializes xhci Driver.
static KSTATUS UsbInitializeXhciDriver(void) {
memset(&gXhciDriver, 0, sizeof(gXhciDriver));
IoInitializeDriverObject(&gXhciDriver, SYS_SV_LITERAL("xhci"));
@@ -32,6 +34,7 @@ static KSTATUS UsbInitializeXhciDriver(void) {
return IoRegisterDriverObject(&gXhciDriver);
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal) return STATUS_INVALID_PARAMETER;
DbgPrint("usb: driver loading\n");
code/drivers/bus/usb/usb.h
+4 -2
View File
@@ -1,4 +1,5 @@
// usb.h - internal USB/xHCI driver interfaces shared across usb.sys translation units.
// USB core declarations shared by bus, class, hub, HID, storage, and xHCI code.
#pragma once
#include <stdint.h>
@@ -29,6 +30,7 @@
#if USB_DEBUG_VERBOSE
#define UsbTrace DbgPrint
#else
// Implements the trace operation.
static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#endif
@@ -221,7 +223,7 @@ static __inline void UsbTrace(const char* fmt, ...) { (void)fmt; }
#define USB_MAX_TRACKED_ENDPOINTS 8u
#define USB_MAX_DEVICE_ID_TEXT 96u
#define USB_MAX_PIPES 32u
#define USB_IOCTL_SUBMIT_URB 0x55534201u /* USB1 */
#define USB_IOCTL_SUBMIT_URB 0x55534201u // USB1
#define USB_ROOT_HUB_MAX_PORTS XHCI_MAX_ROOT_PORTS
#define USB_EXTERNAL_HUB_MAX_PORTS 31u
#define USB_ROOT_PORT_ENUM_RETRIES 3u
code/drivers/bus/usb/usb_bus.c
+37 -2
View File
@@ -1,5 +1,5 @@
// usb_bus.c - USB bus/HCD/PDO/URB core model.
//
// USB bus manager. This file owns device addressing, descriptor reads, and child PDO creation.
// This is the structural USB bus layer used by the upcoming composite, hub,
// and hotplug stages. The legacy direct xHCI class path is still present for
// boot HID/storage during migration, but newly enumerated xHCI devices now also
@@ -11,6 +11,7 @@ DRIVER_OBJECT gUsbBusDriver;
static bool gUsbBusModelReady;
static uint8_t gUsbNextControllerIndex;
// Implements the device Destroy operation.
static void UsbDeviceDestroy(OBJECT* object) {
USB_DEVICE* dev = (USB_DEVICE*)object;
if (dev->RemoveLockReady) {
@@ -23,6 +24,7 @@ static void UsbDeviceDestroy(OBJECT* object) {
}
}
// Implements the pipe Destroy operation.
static void UsbPipeDestroy(OBJECT* object) {
USB_PIPE* pipe = (USB_PIPE*)object;
if (pipe->Device) {
@@ -31,28 +33,34 @@ static void UsbPipeDestroy(OBJECT* object) {
}
}
// Implements the reference Device operation.
void UsbReferenceDevice(USB_DEVICE* dev) {
if (dev) ObReferenceObject(&dev->Header);
}
// Implements the dereference Device operation.
void UsbDereferenceDevice(USB_DEVICE* dev) {
if (dev) ObDereferenceObject(&dev->Header);
}
// Implements the reference Pipe operation.
void UsbReferencePipe(USB_PIPE* pipe) {
if (pipe) ObReferenceObject(&pipe->Header);
}
// Implements the dereference Pipe operation.
void UsbDereferencePipe(USB_PIPE* pipe) {
if (pipe) ObDereferenceObject(&pipe->Header);
}
// Creates dma Buffer.
void* UsbAllocateDmaBuffer(USB_DEVICE* dev, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!dev || !dev->Hcd || !dev->Hcd->Operations || !dev->Hcd->Operations->AllocateDma)
return (void*)0;
return dev->Hcd->Operations->AllocateDma(dev->Hcd, pages, outPhys);
}
// Creates device.
static USB_DEVICE* UsbAllocateDevice(void) {
USB_DEVICE* dev = (USB_DEVICE*)ObAllocateObject(ObjectTypeUsbDevice, sizeof(USB_DEVICE));
if (!dev) return (USB_DEVICE*)0;
@@ -90,10 +98,12 @@ KSTATUS UsbCreatePipe(USB_DEVICE* dev,
return STATUS_SUCCESS;
}
// Implements the id From Buffer operation.
static SYS_STRING_VIEW UsbIdFromBuffer(char* buf) {
return SysSvFromCstr(buf);
}
// Implements the id Put Char operation.
static void UsbIdPutChar(char* buf, size_t cap, size_t* pos, char ch) {
if (!buf || !pos || cap == 0)
return;
@@ -106,6 +116,7 @@ static void UsbIdPutChar(char* buf, size_t cap, size_t* pos, char ch) {
buf[cap - 1] = '\0';
}
// Implements the id Append Literal operation.
static void UsbIdAppendLiteral(char* buf, size_t cap, size_t* pos, const char* text) {
if (!text)
return;
@@ -113,6 +124,7 @@ static void UsbIdAppendLiteral(char* buf, size_t cap, size_t* pos, const char* t
UsbIdPutChar(buf, cap, pos, *text++);
}
// Implements the id Append Hex operation.
static void UsbIdAppendHex(char* buf, size_t cap, size_t* pos, uint32_t value, uint8_t digits) {
static const char kHex[] = "0123456789ABCDEF";
while (digits) {
@@ -121,6 +133,7 @@ static void UsbIdAppendHex(char* buf, size_t cap, size_t* pos, uint32_t value, u
}
}
// Implements the format Vid Pid Rev operation.
static void UsbFormatVidPidRev(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint16_t rev) {
size_t pos = 0;
if (cap) buf[0] = '\0';
@@ -132,6 +145,7 @@ static void UsbFormatVidPidRev(char* buf, size_t cap, uint16_t vid, uint16_t pid
UsbIdAppendHex(buf, cap, &pos, rev, 4);
}
// Implements the format Vid Pid operation.
static void UsbFormatVidPid(char* buf, size_t cap, uint16_t vid, uint16_t pid) {
size_t pos = 0;
if (cap) buf[0] = '\0';
@@ -141,6 +155,7 @@ static void UsbFormatVidPid(char* buf, size_t cap, uint16_t vid, uint16_t pid) {
UsbIdAppendHex(buf, cap, &pos, pid, 4);
}
// Implements the format Vid Pid Mi operation.
static void UsbFormatVidPidMi(char* buf, size_t cap, uint16_t vid, uint16_t pid, uint8_t mi) {
size_t pos = 0;
if (cap) buf[0] = '\0';
@@ -152,6 +167,7 @@ static void UsbFormatVidPidMi(char* buf, size_t cap, uint16_t vid, uint16_t pid,
UsbIdAppendHex(buf, cap, &pos, mi, 2);
}
// Implements the format Class Sub Prot operation.
static void UsbFormatClassSubProt(char* buf, size_t cap, uint8_t cls, uint8_t sub, uint8_t proto) {
size_t pos = 0;
if (cap) buf[0] = '\0';
@@ -163,6 +179,7 @@ static void UsbFormatClassSubProt(char* buf, size_t cap, uint8_t cls, uint8_t su
UsbIdAppendHex(buf, cap, &pos, proto, 2);
}
// Implements the format Class operation.
static void UsbFormatClass(char* buf, size_t cap, uint8_t cls) {
size_t pos = 0;
if (cap) buf[0] = '\0';
@@ -171,18 +188,21 @@ static void UsbFormatClass(char* buf, size_t cap, uint8_t cls) {
}
// Implements the format Composite operation.
static void UsbFormatComposite(char* buf, size_t cap) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\COMPOSITE");
}
// Implements the format Root Hub operation.
static void UsbFormatRootHub(char* buf, size_t cap) {
size_t pos = 0;
if (cap) buf[0] = '\0';
UsbIdAppendLiteral(buf, cap, &pos, "USB\\ROOT_HUB");
}
// Creates device Ids.
static void UsbMakeDeviceIds(const XHCI_DEVICE* xdev, IO_DEVICE_ID* ids, char idbuf[][USB_MAX_DEVICE_ID_TEXT], size_t* outCount) {
size_t n = 0;
UsbFormatVidPidRev(idbuf[n], USB_MAX_DEVICE_ID_TEXT, xdev->VendorId, xdev->ProductId, xdev->BcdDevice);
@@ -233,6 +253,7 @@ static void UsbMakeInterfaceIds(const XHCI_DEVICE* xdev,
*outCount = n;
}
// Initializes bus Model.
KSTATUS UsbInitializeBusModel(void) {
if (gUsbBusModelReady)
return STATUS_SUCCESS;
@@ -263,6 +284,7 @@ void UsbInitializeHcd(USB_HCD* hcd,
hcd->MaxRootPorts = maxRootPorts;
}
// Creates root Hub Pdo.
KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) {
if (!hcd) return STATUS_INVALID_PARAMETER;
if (hcd->RootHubPdo) return STATUS_ALREADY_EXISTS;
@@ -298,6 +320,7 @@ KSTATUS UsbCreateRootHubPdo(USB_HCD* hcd) {
return STATUS_SUCCESS;
}
// Creates device Pdo.
KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
if (!hcd || !xdev) return STATUS_INVALID_PARAMETER;
if (xdev->UsbPdo) return STATUS_ALREADY_EXISTS;
@@ -365,6 +388,7 @@ KSTATUS UsbCreateDevicePdo(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* paren
return STATUS_SUCCESS;
}
// Creates interface Pdos.
KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* parent) {
if (!hcd || !xdev || !xdev->UsbDevice) return STATUS_INVALID_PARAMETER;
DEVICE_OBJECT* deviceParent = parent ? parent : xdev->UsbPdo;
@@ -419,6 +443,7 @@ KSTATUS UsbCreateInterfacePdos(USB_HCD* hcd, XHCI_DEVICE* xdev, DEVICE_OBJECT* p
}
// Implements the reference First Child operation.
static DEVICE_OBJECT* UsbReferenceFirstChild(DEVICE_OBJECT* parent) {
if (!parent) return (DEVICE_OBJECT*)0;
uintptr_t flags = KeAcquireSpinlock(&parent->Lock);
@@ -429,12 +454,14 @@ static DEVICE_OBJECT* UsbReferenceFirstChild(DEVICE_OBJECT* parent) {
return child;
}
// Implements the send Pnp If Alive operation.
static void UsbSendPnpIfAlive(DEVICE_OBJECT* pdo, IO_PNP_MINOR_FUNCTION minor) {
if (!pdo || pdo->DeletePending)
return;
(void)IoSendPnpIrpSync(pdo, minor);
}
// Releases device Object Children.
KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) {
if (!parent)
return STATUS_INVALID_PARAMETER;
@@ -454,6 +481,7 @@ KSTATUS UsbRemoveDeviceObjectChildren(DEVICE_OBJECT* parent, bool surprise) {
return STATUS_SUCCESS;
}
// Releases pdo Subtree.
KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise) {
if (!pdo || pdo->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -471,6 +499,7 @@ KSTATUS UsbRemovePdoSubtree(DEVICE_OBJECT* pdo, bool surprise) {
return STATUS_SUCCESS;
}
// Submits urb To Hcd.
static KSTATUS UsbSubmitUrbToHcd(USB_PDO_EXTENSION* ext, USB_URB* urb) {
if (!ext || !urb) return STATUS_INVALID_PARAMETER;
USB_HCD* hcd = ext->Hcd;
@@ -485,6 +514,7 @@ static KSTATUS UsbSubmitUrbToHcd(USB_PDO_EXTENSION* ext, USB_URB* urb) {
return hcd->Operations->SubmitUrb(hcd, urb);
}
// Cancels urb.
KSTATUS UsbCancelUrb(USB_URB* urb) {
if (!urb || !urb->Device || !urb->Device->Hcd)
return STATUS_INVALID_PARAMETER;
@@ -494,6 +524,7 @@ KSTATUS UsbCancelUrb(USB_URB* urb) {
return hcd->Operations->CancelUrb(hcd, urb);
}
// Submits urb Sync.
KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb) {
if (!target || !urb) return STATUS_INVALID_PARAMETER;
@@ -510,11 +541,13 @@ KSTATUS UsbSubmitUrbSync(DEVICE_OBJECT* target, USB_URB* urb) {
return KSUCCESS(st) ? urb->Status : st;
}
// Implements the async Urb Irp Complete operation.
static void UsbAsyncUrbIrpComplete(IRP* irp, void* context) {
(void)context;
IoFreeIrp(irp);
}
// Submits urb Async.
KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb) {
if (!target || !urb) return STATUS_INVALID_PARAMETER;
@@ -539,6 +572,7 @@ KSTATUS UsbSubmitUrbAsync(DEVICE_OBJECT* target, USB_URB* urb) {
return st;
}
// Dispatches internal Device Control.
KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
@@ -570,6 +604,7 @@ KSTATUS UsbDispatchInternalDeviceControl(DEVICE_OBJECT* device, IRP* irp) {
return STATUS_PENDING;
}
// Dispatches pdo Pnp.
KSTATUS UsbDispatchPdoPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
code/drivers/bus/usb/usb_composite.c
+7 -1
View File
@@ -1,4 +1,5 @@
// usb_composite.c - USB composite function driver and interface-PDO bus.
// USB composite driver. This file owns multi-interface device attachment and function handoff.
#include "usb.h"
DRIVER_OBJECT gUsbCompositeDriver;
@@ -12,6 +13,7 @@ static const IO_DRIVER_ID_TABLE gUsbCompositeIdTable = {
sizeof(gUsbCompositeIds) / sizeof(gUsbCompositeIds[0]),
};
// Implements the composite Create Interfaces operation.
static KSTATUS UsbCompositeCreateInterfaces(USB_COMPOSITE_EXTENSION* ext) {
if (!ext || !ext->Device || ext->InterfacesCreated)
return STATUS_SUCCESS;
@@ -26,6 +28,7 @@ static KSTATUS UsbCompositeCreateInterfaces(USB_COMPOSITE_EXTENSION* ext) {
return st;
}
// Implements the composite Forward Pnp operation.
static KSTATUS UsbCompositeForwardPnp(DEVICE_OBJECT* device, IRP* irp, KSTATUS localStatus) {
if (!KSUCCESS(localStatus)) {
IoCompleteRequest(irp, localStatus, 0);
@@ -34,6 +37,7 @@ static KSTATUS UsbCompositeForwardPnp(DEVICE_OBJECT* device, IRP* irp, KSTATUS l
return IoForwardIrpToLower(device, irp);
}
// Implements the composite Dispatch Pnp operation.
KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
@@ -110,6 +114,7 @@ KSTATUS UsbCompositeDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the composite Add Device operation.
KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -139,6 +144,7 @@ KSTATUS UsbCompositeAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevi
return STATUS_SUCCESS;
}
// Initializes composite Driver.
KSTATUS UsbInitializeCompositeDriver(void) {
memset(&gUsbCompositeDriver, 0, sizeof(gUsbCompositeDriver));
IoInitializeDriverObject(&gUsbCompositeDriver, SYS_SV_LITERAL("usb-composite"));
code/drivers/bus/usb/usb_core.c
+16 -10
View File
@@ -1,4 +1,5 @@
// usb_core.c - USB descriptor/configuration parsing, interface publishing, and root-port enumeration.
// USB core helpers. This file owns common descriptor, string, and request utilities.
#include "usb.h"
KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
@@ -25,6 +26,7 @@ KSTATUS XhciSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
// Returns tracked Interface By Number.
static USB_TRACKED_INTERFACE* UsbFindTrackedInterfaceByNumber(XHCI_DEVICE* dev, uint8_t interfaceNumber) {
if (!dev) return (USB_TRACKED_INTERFACE*)0;
for (uint8_t i = 0; i < dev->InterfaceCount && i < USB_MAX_TRACKED_INTERFACES; i++) {
@@ -108,15 +110,15 @@ void XhciLogConfigurationTree(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
dev->SlotId, iface->bInterfaceNumber);
}
} else {
/*
* Alternate settings are not independent USB interfaces. Real
* UVC cameras often expose one VideoStreaming interface with
* many alternate settings, and publishing each alternate setting
* as a separate PDO causes duplicate interface devices and can
* exhaust USB_MAX_TRACKED_INTERFACES. Track only alternate
* setting zero until a function driver explicitly selects a
* different setting.
*/
// Alternate settings are not independent USB interfaces. Real
// UVC cameras often expose one VideoStreaming interface with
// many alternate settings, and publishing each alternate setting
// as a separate PDO causes duplicate interface devices and can
// exhaust USB_MAX_TRACKED_INTERFACES. Track only alternate
// setting zero until a function driver explicitly selects a
// different setting.
tracked = (USB_TRACKED_INTERFACE*)0;
}
@@ -253,6 +255,7 @@ KSTATUS XhciDiscoverAndSetConfiguration(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
// Implements the rollback Enumerated Slot operation.
static void XhciRollbackEnumeratedSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0 || slotId > XHCI_MAX_TRACKED_SLOTS)
return;
@@ -377,6 +380,7 @@ static KSTATUS XhciEnumeratePathEx(XHCI_CONTROLLER* xhci,
return STATUS_SUCCESS;
}
// Implements the enumerate Port Ex operation.
KSTATUS XhciEnumeratePortEx(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSlotId) {
if (outSlotId) *outSlotId = 0;
@@ -426,11 +430,13 @@ KSTATUS XhciEnumerateHubPortEx(XHCI_CONTROLLER* xhci,
return XhciEnumeratePathEx(xhci, hubPort, speed, &path, hub->Self, outSlotId);
}
// Implements the enumerate Port operation.
KSTATUS XhciEnumeratePort(XHCI_CONTROLLER* xhci, uint8_t port) {
return XhciEnumeratePortEx(xhci, port, (uint8_t*)0);
}
// Implements the enumerate Root Ports operation.
void XhciEnumerateRootPorts(XHCI_CONTROLLER* xhci) {
uint32_t attempted = 0;
uint32_t configured = 0;
code/drivers/bus/usb/usb_hid.c
+28 -8
View File
@@ -1,6 +1,8 @@
// usb_hid.c - USB HID boot keyboard/mouse class binding and report handling.
// USB HID class driver. This file owns HID interface probing and input report translation.
#include "usb.h"
// Implements the hid Key In Report operation.
bool XhciHidKeyInReport(const uint8_t* keys, uint8_t key) {
if (key == 0 || key == 1) return false;
for (uint8_t i = 0; i < 6; i++) {
@@ -25,6 +27,7 @@ INPUT_KEY XhciHidModifierKey(uint8_t bit) {
}
// Submits hid Key.
void XhciSubmitHidKey(INPUT_KEY key, bool pressed) {
if (key == INPUT_KEY_NONE) return;
INPUT_KEY_EVENT ev;
@@ -34,6 +37,7 @@ void XhciSubmitHidKey(INPUT_KEY key, bool pressed) {
}
// Processes boot Keyboard Report.
void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) {
if (len < 8) return;
@@ -63,6 +67,7 @@ void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint
}
// Processes boot Mouse Report.
void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) {
if (len < 3) return;
@@ -84,6 +89,7 @@ void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_
}
// Implements the arm Hid Interrupt In operation.
KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (!dev->HidActive || dev->HidArmed || !dev->HidRing || !dev->HidBuffer)
return STATUS_SUCCESS;
@@ -112,6 +118,7 @@ KSTATUS XhciArmHidInterruptIn(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
}
// Implements the arm All Hid operation.
void XhciArmAllHid(XHCI_CONTROLLER* xhci) {
for (uint8_t slot = 1; slot <= XHCI_MAX_TRACKED_SLOTS; slot++) {
XHCI_DEVICE* dev = &xhci->Devices[slot];
@@ -157,6 +164,7 @@ static void XhciHandleLegacyHidTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trb
}
// Starts hid Interrupts.
KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
if (!xhci || xhci->HidDeviceCount == 0)
return STATUS_SUCCESS;
@@ -166,6 +174,7 @@ KSTATUS XhciStartHidInterrupts(XHCI_CONTROLLER* xhci) {
}
// Updates hid Boot Protocol.
KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
USB_SETUP_PACKET setup;
setup.bmRequestType = USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_INTERFACE;
@@ -197,6 +206,7 @@ KSTATUS XhciSetHidBootProtocol(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
}
// Creates hid Memory.
KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->HidRing && dev->HidBuffer) return STATUS_SUCCESS;
@@ -223,6 +233,7 @@ KSTATUS XhciAllocateHidMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
}
// Implements the configure Hid Endpoint operation.
KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (!dev->HidCandidate)
return STATUS_SUCCESS;
@@ -298,9 +309,7 @@ KSTATUS XhciConfigureHidEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
}
// -------------------------------------------------------------------------
// PDO/FDO boot HID function driver path.
// -------------------------------------------------------------------------
DRIVER_OBJECT gUsbHidDriver;
@@ -316,6 +325,7 @@ static const IO_DRIVER_ID_TABLE gUsbHidIdTable = {
};
// Implements the hid Is Qemu Absolute Tablet Pdo operation.
static bool UsbHidIsQemuAbsoluteTabletPdo(const USB_PDO_EXTENSION* pdoExt, const USB_ENDPOINT_DESCRIPTOR* ep) {
if (!pdoExt || !pdoExt->Device || !ep)
return false;
@@ -327,14 +337,14 @@ static bool UsbHidIsQemuAbsoluteTabletPdo(const USB_PDO_EXTENSION* pdoExt, const
if (pdoExt->Device->VendorId != 0x0627u || pdoExt->Device->ProductId != 0x0001u)
return false;
/*
* QEMU's usb-kbd and usb-tablet commonly share VID:PID 0627:0001.
* Do not classify by VID:PID alone: the boot keyboard is subclass/protocol
* 01/01, while usb-tablet is the non-boot HID interface 00/00.
*/
// QEMU's usb-kbd and usb-tablet commonly share VID:PID 0627:0001.
// Do not classify by VID:PID alone: the boot keyboard is subclass/protocol
// 01/01, while usb-tablet is the non-boot HID interface 00/00.
return (ep->wMaxPacketSize & 0x07FFu) >= 6u;
}
// Implements the hid Find Interrupt In Pdo operation.
static const USB_ENDPOINT_DESCRIPTOR* UsbHidFindInterruptInPdo(const USB_PDO_EXTENSION* ext) {
if (!ext) return (const USB_ENDPOINT_DESCRIPTOR*)0;
@@ -349,6 +359,7 @@ static const USB_ENDPOINT_DESCRIPTOR* UsbHidFindInterruptInPdo(const USB_PDO_EXT
}
// Implements the hid Process Boot Keyboard Report operation.
static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 8) return;
@@ -377,6 +388,7 @@ static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_
memcpy(ext->LastKeys, keys, sizeof(ext->LastKeys));
}
// Implements the hid Process Boot Mouse Report operation.
static void UsbHidProcessBootMouseReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 3) return;
@@ -397,6 +409,7 @@ static void UsbHidProcessBootMouseReport(USB_HID_EXTENSION* ext, const uint8_t*
ext->LastButtons = buttons;
}
// Implements the hid Process Qemu Tablet Report operation.
static void UsbHidProcessQemuTabletReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 5) return;
@@ -417,6 +430,7 @@ static void UsbHidProcessQemuTabletReport(USB_HID_EXTENSION* ext, const uint8_t*
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext);
// Implements the hid Interrupt Complete operation.
static void UsbHidInterruptComplete(USB_URB* urb, void* context) {
USB_HID_EXTENSION* ext = (USB_HID_EXTENSION*)context;
if (!ext || urb != &ext->InterruptUrb)
@@ -451,6 +465,7 @@ static void UsbHidInterruptComplete(USB_URB* urb, void* context) {
(void)UsbHidArmInterrupt(ext);
}
// Implements the hid Send Boot Protocol operation.
static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) {
if (!ext || !ext->LowerDevice || !ext->Device)
return STATUS_INVALID_PARAMETER;
@@ -494,6 +509,7 @@ static KSTATUS UsbHidSendBootProtocol(USB_HID_EXTENSION* ext) {
return STATUS_SUCCESS;
}
// Implements the hid Arm Interrupt operation.
static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
if (!ext || !ext->Started || ext->SurpriseRemoved || ext->Removed)
return STATUS_OBJECT_DEAD;
@@ -524,6 +540,7 @@ static KSTATUS UsbHidArmInterrupt(USB_HID_EXTENSION* ext) {
return STATUS_SUCCESS;
}
// Implements the hid Start Device operation.
static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext) {
if (!device || !ext || !ext->Device || !ext->InterfacePdo)
return STATUS_INVALID_PARAMETER;
@@ -614,6 +631,7 @@ static KSTATUS UsbHidStartDevice(DEVICE_OBJECT* device, USB_HID_EXTENSION* ext)
}
// Implements the hid Dispatch Pnp operation.
KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
@@ -699,6 +717,7 @@ KSTATUS UsbHidDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the hid Add Device operation.
KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -759,6 +778,7 @@ KSTATUS UsbHidAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObje
return STATUS_SUCCESS;
}
// Initializes hid Driver.
KSTATUS UsbInitializeHidDriver(void) {
memset(&gUsbHidDriver, 0, sizeof(gUsbHidDriver));
IoInitializeDriverObject(&gUsbHidDriver, SYS_SV_LITERAL("usb-hid"));
code/drivers/bus/usb/usb_hub.c
+42 -10
View File
@@ -1,5 +1,5 @@
// usb_hub.c - USB hub function driver.
//
// USB hub driver. This file owns hub probing, port status handling, and child-device enumeration.
// Root hubs and external hubs are represented by the same FDO abstraction. The
// xHCI root hub feeds port-change events directly into this driver, while an
// external hub feeds the same port state machine from its interrupt-IN change
@@ -21,16 +21,19 @@ static const IO_DRIVER_ID_TABLE gUsbHubIdTable = {
#define USB_HUB_PORT_ENUMERATING 0xffu
#define USB_HUB_PORT_FAILED 0xfeu
// Implements the hub Slot Busy operation.
static bool UsbHubSlotBusy(uint8_t slot) {
return slot != 0 && slot != USB_HUB_PORT_ENUMERATING && slot != USB_HUB_PORT_FAILED;
}
// Implements the hub Root Port Connected operation.
static bool UsbHubRootPortConnected(XHCI_CONTROLLER* xhci, uint8_t port) {
if (!xhci || !xhci->Op || port == 0) return false;
uint32_t portsc = XhciRead32(xhci->Op, XhciPortReg(port));
return (portsc & XHCI_PORTSC_CCS) != 0;
}
// Implements the hub Clear Root Port Failure operation.
static void UsbHubClearRootPortFailure(USB_HUB_EXTENSION* hub, uint8_t port) {
if (!hub || port == 0 || port > USB_ROOT_HUB_MAX_PORTS) return;
hub->RootPortFailCount[port] = 0;
@@ -38,6 +41,7 @@ static void UsbHubClearRootPortFailure(USB_HUB_EXTENSION* hub, uint8_t port) {
hub->RootPortSlot[port] = 0;
}
// Implements the hub Mark Root Port Failed operation.
static void UsbHubMarkRootPortFailed(USB_HUB_EXTENSION* hub, uint8_t port) {
if (!hub || port == 0 || port > USB_ROOT_HUB_MAX_PORTS) return;
hub->RootPortSlot[port] = USB_HUB_PORT_FAILED;
@@ -71,14 +75,13 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub,
hub->RootPortFailCount[port] = attempt;
/*
* On real Intel laptop controllers some internal full-speed devices
* respond with xHCI USB Transaction Error during Address Device while
* the port is still settling. Retry that boundedly while the port is
* held in ENUMERATING state so queued PSC events cannot start parallel
* enumerations. Descriptor/control timeouts are not retried here; they
* are usually dead/firmware-owned ports and retrying them stalls boot.
*/
// On real Intel laptop controllers some internal full-speed devices
// respond with xHCI USB Transaction Error during Address Device while
// the port is still settling. Retry that boundedly while the port is
// held in ENUMERATING state so queued PSC events cannot start parallel
// enumerations. Descriptor/control timeouts are not retried here; they
// are usually dead/firmware-owned ports and retrying them stalls boot.
if (last != STATUS_DEVICE_ERROR || attempt == maxAttempts)
break;
@@ -97,6 +100,7 @@ static KSTATUS UsbHubEnumerateRootPortWithRetries(USB_HUB_EXTENSION* hub,
return last;
}
// Implements the hub Pop Pending Root Port operation.
static bool UsbHubPopPendingRootPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) {
if (!hub || !outPort) return false;
@@ -118,6 +122,7 @@ static bool UsbHubPopPendingRootPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) {
return false;
}
// Implements the hub Pop Pending External Port operation.
static bool UsbHubPopPendingExternalPort(USB_HUB_EXTENSION* hub, uint8_t* outPort) {
if (!hub || !outPort) return false;
@@ -142,6 +147,7 @@ static bool UsbHubPopPendingExternalPort(USB_HUB_EXTENSION* hub, uint8_t* outPor
return false;
}
// Implements the hub Record Root Port Change operation.
static void UsbHubRecordRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
if (!hub || port == 0 || port > hub->PortCount)
return;
@@ -154,6 +160,7 @@ static void UsbHubRecordRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
KeSignalSemaphore(hub->WorkerSemaphore);
}
// Implements the hub Queue External Port Change operation.
void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
if (!hub || port == 0 || port > hub->PortCount || port > USB_EXTERNAL_HUB_MAX_PORTS)
return;
@@ -164,6 +171,7 @@ void UsbHubQueueExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
KeSignalSemaphore(hub->WorkerSemaphore);
}
// Implements the hub Remove Slot Child operation.
static void UsbHubRemoveSlotChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t slot) {
if (!hub || !hub->Hcd || slot == 0)
return;
@@ -197,6 +205,7 @@ static void UsbHubRemoveSlotChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t
}
}
// Implements the hub Remove Root Child operation.
static void UsbHubRemoveRootChild(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t slot) {
UsbHubRemoveSlotChild(hub, port, slot);
}
@@ -249,6 +258,7 @@ static KSTATUS UsbHubControlNoData(USB_HUB_EXTENSION* hub,
return UsbSubmitUrbSync(hub->LowerDevice, &urb);
}
// Implements the hub Read Descriptor operation.
static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR* outDesc) {
if (!hub || !outDesc || !hub->ChangeBitmap || !hub->ChangeBitmapPhys)
return STATUS_INVALID_PARAMETER;
@@ -271,6 +281,7 @@ static KSTATUS UsbHubReadDescriptor(USB_HUB_EXTENSION* hub, USB_HUB_DESCRIPTOR*
return STATUS_SUCCESS;
}
// Implements the hub Get Port Status operation.
static KSTATUS UsbHubGetPortStatus(USB_HUB_EXTENSION* hub, uint8_t port, USB_HUB_PORT_STATUS* outStatus) {
if (!hub || !outStatus || port == 0)
return STATUS_INVALID_PARAMETER;
@@ -291,6 +302,7 @@ static KSTATUS UsbHubGetPortStatus(USB_HUB_EXTENSION* hub, uint8_t port, USB_HUB
return STATUS_SUCCESS;
}
// Implements the hub Set Port Feature operation.
static KSTATUS UsbHubSetPortFeature(USB_HUB_EXTENSION* hub, uint8_t port, uint16_t feature) {
return UsbHubControlNoData(hub,
USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_OTHER,
@@ -299,6 +311,7 @@ static KSTATUS UsbHubSetPortFeature(USB_HUB_EXTENSION* hub, uint8_t port, uint16
port);
}
// Implements the hub Clear Port Feature operation.
static KSTATUS UsbHubClearPortFeature(USB_HUB_EXTENSION* hub, uint8_t port, uint16_t feature) {
return UsbHubControlNoData(hub,
USB_DIR_OUT | USB_TYPE_CLASS | USB_RECIP_OTHER,
@@ -307,6 +320,7 @@ static KSTATUS UsbHubClearPortFeature(USB_HUB_EXTENSION* hub, uint8_t port, uint
port);
}
// Implements the hub Speed From Port Status operation.
static uint8_t UsbHubSpeedFromPortStatus(uint16_t status) {
if (status & USB_HUB_PORT_STATUS_LOW_SPEED)
return 2u;
@@ -315,6 +329,7 @@ static uint8_t UsbHubSpeedFromPortStatus(uint16_t status) {
return 1u;
}
// Implements the hub Reset External Port operation.
static KSTATUS UsbHubResetExternalPort(USB_HUB_EXTENSION* hub, uint8_t port, uint8_t* outSpeed) {
if (outSpeed) *outSpeed = 0;
if (!hub || port == 0)
@@ -355,6 +370,7 @@ static KSTATUS UsbHubResetExternalPort(USB_HUB_EXTENSION* hub, uint8_t port, uin
return STATUS_SUCCESS;
}
// Implements the hub Handle External Port Change operation.
static void UsbHubHandleExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
if (!hub || hub->Kind != UsbHubKindExternal || port == 0 || port > hub->PortCount || port > USB_EXTERNAL_HUB_MAX_PORTS)
return;
@@ -433,6 +449,7 @@ static void UsbHubHandleExternalPortChange(USB_HUB_EXTENSION* hub, uint8_t port)
}
}
// Implements the hub Handle Root Port Change operation.
static void UsbHubHandleRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
if (!hub || !hub->Hcd || port == 0 || port > hub->PortCount || port > USB_ROOT_HUB_MAX_PORTS)
return;
@@ -509,6 +526,7 @@ static void UsbHubHandleRootPortChange(USB_HUB_EXTENSION* hub, uint8_t port) {
static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub);
static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd);
// Implements the hub Take Initial Root Scan operation.
static bool UsbHubTakeInitialRootScan(USB_HUB_EXTENSION* hub) {
if (!hub || hub->Kind != UsbHubKindRoot)
return false;
@@ -522,6 +540,7 @@ static bool UsbHubTakeInitialRootScan(USB_HUB_EXTENSION* hub) {
return take;
}
// Implements the hub Worker operation.
static void UsbHubWorker(void* arg) {
USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)arg;
while (hub && hub->WorkerRunning) {
@@ -546,6 +565,7 @@ static void UsbHubWorker(void* arg) {
}
}
// Implements the hub Do Root Port Scan operation.
static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd) {
if (!hcd || !hcd->RootHubContext)
return STATUS_INVALID_PARAMETER;
@@ -592,6 +612,7 @@ static KSTATUS UsbHubDoRootPortScan(USB_HCD* hcd) {
return STATUS_SUCCESS;
}
// Implements the hub Schedule Root Scan operation.
KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd) {
if (!hcd || !hcd->RootHubContext)
return STATUS_INVALID_PARAMETER;
@@ -609,6 +630,7 @@ KSTATUS UsbHubScheduleRootScan(USB_HCD* hcd) {
return STATUS_SUCCESS;
}
// Implements the hub Queue Root Port Change operation.
void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port) {
if (!hcd || !hcd->RootHubContext)
return;
@@ -616,6 +638,7 @@ void UsbHubQueueRootPortChange(USB_HCD* hcd, uint8_t port) {
UsbHubRecordRootPortChange(hub, port);
}
// Implements the hub Scan External Ports operation.
static KSTATUS UsbHubScanExternalPorts(USB_HUB_EXTENSION* hub) {
if (!hub || hub->Kind != UsbHubKindExternal)
return STATUS_INVALID_PARAMETER;
@@ -643,6 +666,7 @@ static KSTATUS UsbHubScanExternalPorts(USB_HUB_EXTENSION* hub) {
return STATUS_SUCCESS;
}
// Implements the hub Find Interrupt In operation.
static const USB_ENDPOINT_DESCRIPTOR* UsbHubFindInterruptIn(USB_PDO_EXTENSION* pdoExt) {
if (!pdoExt)
return (const USB_ENDPOINT_DESCRIPTOR*)0;
@@ -676,6 +700,7 @@ static const USB_ENDPOINT_DESCRIPTOR* UsbHubFindInterruptIn(USB_PDO_EXTENSION* p
return (const USB_ENDPOINT_DESCRIPTOR*)0;
}
// Implements the hub External Change Complete operation.
static void UsbHubExternalChangeComplete(USB_URB* urb, void* context) {
USB_HUB_EXTENSION* hub = (USB_HUB_EXTENSION*)context;
if (!hub || urb != &hub->ChangeUrb)
@@ -697,6 +722,7 @@ static void UsbHubExternalChangeComplete(USB_URB* urb, void* context) {
}
}
// Implements the hub Arm External Interrupt operation.
static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub) {
if (!hub || !hub->InterruptPipe || !hub->ChangeBitmap || hub->ChangeUrbPending)
return;
@@ -724,6 +750,7 @@ static void UsbHubArmExternalInterrupt(USB_HUB_EXTENSION* hub) {
}
}
// Implements the hub Start Root Device operation.
static KSTATUS UsbHubStartRootDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* hub) {
hub->PortCount = hub->Hcd ? hub->Hcd->MaxRootPorts : 0;
if (hub->PortCount > USB_ROOT_HUB_MAX_PORTS)
@@ -752,6 +779,7 @@ static KSTATUS UsbHubStartRootDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* h
return STATUS_SUCCESS;
}
// Implements the hub Start External Device operation.
static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* hub) {
if (!hub || !hub->Device)
return STATUS_INVALID_PARAMETER;
@@ -821,6 +849,7 @@ static KSTATUS UsbHubStartExternalDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSIO
return STATUS_SUCCESS;
}
// Implements the hub Start Device operation.
static KSTATUS UsbHubStartDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* hub) {
if (!device || !hub || hub->Started)
return STATUS_SUCCESS;
@@ -830,6 +859,7 @@ static KSTATUS UsbHubStartDevice(DEVICE_OBJECT* device, USB_HUB_EXTENSION* hub)
return UsbHubStartExternalDevice(device, hub);
}
// Implements the hub Dispatch Pnp operation.
KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
@@ -932,6 +962,7 @@ KSTATUS UsbHubDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the hub Add Device operation.
KSTATUS UsbHubAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -1017,6 +1048,7 @@ KSTATUS UsbHubAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObje
return STATUS_SUCCESS;
}
// Initializes hub Driver.
KSTATUS UsbInitializeHubDriver(void) {
memset(&gUsbHubDriver, 0, sizeof(gUsbHubDriver));
IoInitializeDriverObject(&gUsbHubDriver, SYS_SV_LITERAL("usb-hub"));
code/drivers/bus/usb/usb_storage.c
+25 -2
View File
@@ -1,5 +1,5 @@
// usb_storage.c - USB mass-storage BOT/SCSI class driver.
//
// USB mass-storage driver. This file owns bulk-only transport and storage-device exposure.
// The storage driver now binds as a USB function driver to an interface PDO
// (USB\CLASS_08&SUBCLASS_06&PROT_50). It creates a disk FDO above that
// interface stack and issues BOT transfers through USB URBs. The xHCI HCD
@@ -18,6 +18,7 @@ static const IO_DRIVER_ID_TABLE gUsbStorageIdTable = {
sizeof(gUsbStorageIds) / sizeof(gUsbStorageIds[0]),
};
// Implements the storage From Disk Device operation.
static USB_STORAGE_EXTENSION* UsbStorageFromDiskDevice(DEVICE_OBJECT* device) {
if (!device || !device->DeviceExtension)
return (USB_STORAGE_EXTENSION*)0;
@@ -28,6 +29,7 @@ static USB_STORAGE_EXTENSION* UsbStorageFromDiskDevice(DEVICE_OBJECT* device) {
return diskExt->Storage;
}
// Implements the storage From Interface Device operation.
static USB_STORAGE_EXTENSION* UsbStorageFromInterfaceDevice(DEVICE_OBJECT* device) {
if (!device || !device->DeviceExtension)
return (USB_STORAGE_EXTENSION*)0;
@@ -38,6 +40,7 @@ static USB_STORAGE_EXTENSION* UsbStorageFromInterfaceDevice(DEVICE_OBJECT* devic
return ext;
}
// Implements the storage Find Bulk Endpoint operation.
static const USB_ENDPOINT_DESCRIPTOR* UsbStorageFindBulkEndpoint(const USB_PDO_EXTENSION* pdoExt, bool wantIn) {
if (!pdoExt) return (const USB_ENDPOINT_DESCRIPTOR*)0;
@@ -150,11 +153,13 @@ KSTATUS UsbStorageBotCommand(USB_STORAGE_EXTENSION* storage,
return STATUS_SUCCESS;
}
// Implements the be32 operation.
uint32_t UsbBe32(const uint8_t* p) {
return ((uint32_t)p[0] << 24) | ((uint32_t)p[1] << 16) |
((uint32_t)p[2] << 8) | (uint32_t)p[3];
}
// Updates be32.
void UsbStoreBe32(uint8_t* p, uint32_t v) {
p[0] = (uint8_t)(v >> 24);
p[1] = (uint8_t)(v >> 16);
@@ -162,11 +167,13 @@ void UsbStoreBe32(uint8_t* p, uint32_t v) {
p[3] = (uint8_t)v;
}
// Updates be16.
void UsbStoreBe16(uint8_t* p, uint16_t v) {
p[0] = (uint8_t)(v >> 8);
p[1] = (uint8_t)v;
}
// Implements the storage Inquiry operation.
KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
@@ -188,6 +195,7 @@ KSTATUS UsbStorageInquiry(USB_STORAGE_EXTENSION* storage) {
return STATUS_SUCCESS;
}
// Implements the storage Request Sense operation.
KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
@@ -199,6 +207,7 @@ KSTATUS UsbStorageRequestSense(USB_STORAGE_EXTENSION* storage) {
dev->MassBuffer, dev->MassBufferPhys, 18, true);
}
// Implements the storage Test Unit Ready operation.
KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
@@ -220,6 +229,7 @@ KSTATUS UsbStorageTestUnitReady(USB_STORAGE_EXTENSION* storage) {
return last;
}
// Implements the storage Read Capacity operation.
KSTATUS UsbStorageReadCapacity(USB_STORAGE_EXTENSION* storage) {
if (!storage || !storage->XhciDevice)
return STATUS_INVALID_PARAMETER;
@@ -287,6 +297,7 @@ KSTATUS UsbStorageReadWriteSectors(USB_STORAGE_EXTENSION* storage,
return STATUS_SUCCESS;
}
// Implements the storage Acquire Io operation.
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
if (!storage) return;
XHCI_CONTROLLER* xhci = storage->Device ? (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
@@ -295,10 +306,12 @@ void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
}
}
// Implements the storage Release Io operation.
void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage) {
if (storage) _InterlockedExchange(&storage->IoBusy, 0);
}
// Implements the storage Dispatch Read Write operation.
KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
@@ -362,6 +375,7 @@ KSTATUS UsbStorageDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
return STATUS_PENDING;
}
// Creates mass Storage Memory.
KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (dev->MassInRing && dev->MassOutRing && dev->MassBuffer)
return STATUS_SUCCESS;
@@ -382,6 +396,7 @@ KSTATUS XhciAllocateMassStorageMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return STATUS_SUCCESS;
}
// Implements the configure Mass Storage Endpoints operation.
KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
if (!dev->MassCandidate)
return STATUS_SUCCESS;
@@ -451,6 +466,7 @@ KSTATUS XhciConfigureMassStorageEndpoints(XHCI_CONTROLLER* xhci, XHCI_DEVICE* de
return STATUS_SUCCESS;
}
// Returns free Disk Index.
uint8_t UsbFindFreeDiskIndex(void) {
for (uint8_t i = 0; i < 8; i++) {
if (!IoDiskDevicePresent(i))
@@ -459,6 +475,7 @@ uint8_t UsbFindFreeDiskIndex(void) {
return 0xFFu;
}
// Implements the storage Register Disk operation.
KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage) {
if (!storage || storage->BlockSize != USB_STORAGE_SECTOR_SIZE || storage->BlockCount == 0)
return STATUS_NOT_SUPPORTED;
@@ -502,6 +519,7 @@ KSTATUS UsbStorageRegisterDisk(USB_STORAGE_EXTENSION* storage) {
return STATUS_SUCCESS;
}
// Implements the storage Start Device operation.
static KSTATUS UsbStorageStartDevice(DEVICE_OBJECT* device, USB_STORAGE_EXTENSION* storage) {
if (!device || !storage || !storage->InterfacePdo || !storage->Device)
return STATUS_INVALID_PARAMETER;
@@ -618,6 +636,7 @@ fail:
return st;
}
// Implements the storage Disk Pnp operation.
static KSTATUS UsbStorageDiskPnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
if (!device || !irp || !storage)
return STATUS_INVALID_PARAMETER;
@@ -668,6 +687,7 @@ static KSTATUS UsbStorageDiskPnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EX
}
}
// Implements the storage Interface Pnp operation.
static KSTATUS UsbStorageInterfacePnp(DEVICE_OBJECT* device, IRP* irp, USB_STORAGE_EXTENSION* storage, IO_PNP_MINOR_FUNCTION minor) {
if (!device || !irp || !storage)
return STATUS_INVALID_PARAMETER;
@@ -745,6 +765,7 @@ static KSTATUS UsbStorageInterfacePnp(DEVICE_OBJECT* device, IRP* irp, USB_STORA
}
}
// Implements the storage Dispatch Pnp operation.
KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp)
return STATUS_INVALID_PARAMETER;
@@ -765,6 +786,7 @@ KSTATUS UsbStorageDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
// Implements the storage Add Device operation.
KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -815,6 +837,7 @@ KSTATUS UsbStorageAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevice
return STATUS_SUCCESS;
}
// Initializes storage Driver.
KSTATUS UsbInitializeStorageDriver(void) {
memset(&gUsbStorageDriver, 0, sizeof(gUsbStorageDriver));
IoInitializeDriverObject(&gUsbStorageDriver, SYS_SV_LITERAL("usb-storage"));
code/drivers/bus/usb/usb_xhci.c
+79 -14
View File
@@ -1,11 +1,14 @@
// usb_xhci.c - xHCI host-controller, MMIO, rings, commands, event dispatch, and generic transfers.
// xHCI USB host-controller driver. This file owns MMIO setup, rings, slots, endpoints, and transfer completion.
#include "usb.h"
// Implements the align Up32 operation.
uint32_t UsbAlignUp32(uint32_t value, uint32_t align) {
return (value + align - 1u) & ~(align - 1u);
}
// Implements the barrier operation.
void XhciBarrier(void) {
_ReadWriteBarrier();
_mm_mfence();
@@ -13,26 +16,31 @@ void XhciBarrier(void) {
}
// Implements the read8 operation.
uint8_t XhciRead8(volatile uint8_t* base, uint32_t reg) {
return *(volatile uint8_t*)(base + reg);
}
// Implements the read16 operation.
uint16_t XhciRead16(volatile uint8_t* base, uint32_t reg) {
return *(volatile uint16_t*)(base + reg);
}
// Implements the read32 operation.
uint32_t XhciRead32(volatile uint8_t* base, uint32_t reg) {
return *(volatile uint32_t*)(base + reg);
}
// Implements the write32 operation.
void XhciWrite32(volatile uint8_t* base, uint32_t reg, uint32_t value) {
*(volatile uint32_t*)(base + reg) = value;
}
// Implements the read64 operation.
uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg) {
uint32_t lo = XhciRead32(base, reg);
uint32_t hi = XhciRead32(base, reg + 4u);
@@ -40,12 +48,14 @@ uint64_t XhciRead64(volatile uint8_t* base, uint32_t reg) {
}
// Implements the write64 operation.
void XhciWrite64(volatile uint8_t* base, uint32_t reg, uint64_t value) {
XhciWrite32(base, reg, (uint32_t)value);
XhciWrite32(base, reg + 4u, (uint32_t)(value >> 32));
}
// Implements the delay Us operation.
void UsbDelayUs(const HAL_API* hal, uint32_t us) {
if (hal && hal->Timer.Stall) hal->Timer.Stall(us);
else {
@@ -103,42 +113,50 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
}
// Implements the scratchpad Count operation.
uint16_t XhciScratchpadCount(uint32_t hcs2) {
return (uint16_t)((((hcs2 >> 16) & 0x3E0u) | ((hcs2 >> 27) & 0x1Fu)) & 0x3FFu);
}
// Implements the trb Type operation.
uint32_t XhciTrbType(const XHCI_TRB* trb) {
return (trb->Control >> XHCI_TRB_TYPE_SHIFT) & XHCI_TRB_TYPE_MASK;
}
// Implements the completion Code operation.
uint32_t XhciCompletionCode(const XHCI_TRB* trb) {
return (trb->Status >> 24) & 0xFFu;
}
// Implements the event Slot Id operation.
uint8_t XhciEventSlotId(const XHCI_TRB* trb) {
return (uint8_t)(trb->Control >> 24);
}
// Implements the event Endpoint Id operation.
uint8_t XhciEventEndpointId(const XHCI_TRB* trb) {
return (uint8_t)((trb->Control >> 16) & 0x1Fu);
}
// Implements the transfer Residue operation.
uint32_t XhciTransferResidue(const XHCI_TRB* trb) {
return trb->Status & 0xFFFFFFu;
}
// Implements the input Control Context operation.
uint32_t* XhciInputControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
(void)xhci;
return (uint32_t*)dev->InputContext;
}
// Implements the input Slot Context operation.
uint32_t* XhciInputSlotContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
return (uint32_t*)(dev->InputContext + xhci->ContextBytes);
}
@@ -150,6 +168,7 @@ uint32_t* XhciInputEndpointContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
// Implements the ep Max Packet operation.
uint32_t XhciEpMaxPacket(uint8_t speed) {
switch (speed) {
case 2: return 8; // low-speed
@@ -162,6 +181,7 @@ uint32_t XhciEpMaxPacket(uint8_t speed) {
}
// Implements the setup Packet To U64 operation.
uint64_t UsbSetupPacketToU64(const USB_SETUP_PACKET* setup) {
uint64_t v = 0;
v |= (uint64_t)setup->bmRequestType;
@@ -186,6 +206,7 @@ const char* XhciPortSpeedName(uint32_t speed) {
// Parses supported Protocols.
void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) {
memset(xhci->PortProtocolMajor, 0, sizeof(xhci->PortProtocolMajor));
memset(xhci->PortProtocolMinor, 0, sizeof(xhci->PortProtocolMinor));
@@ -233,6 +254,7 @@ void XhciParseSupportedProtocols(XHCI_CONTROLLER* xhci) {
}
// Implements the port Protocol Major operation.
uint8_t XhciPortProtocolMajor(XHCI_CONTROLLER* xhci, uint8_t port, uint32_t portsc) {
if (port <= XHCI_MAX_ROOT_PORTS && xhci->PortProtocolMajor[port])
return xhci->PortProtocolMajor[port];
@@ -276,6 +298,7 @@ const char* UsbInterfaceClassName(uint8_t klass, uint8_t subclass, uint8_t proto
}
// Implements the log Port Status operation.
void XhciLogPortStatus(XHCI_CONTROLLER* xhci) {
for (uint8_t port = 1; port <= xhci->MaxPorts; port++) {
uint32_t off = XHCI_OP_PORT_BASE + ((uint32_t)port - 1u) * XHCI_OP_PORT_STRIDE;
@@ -297,6 +320,7 @@ void XhciLogPortStatus(XHCI_CONTROLLER* xhci) {
}
}
// Implements the power Root Ports operation.
void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
if (!xhci || !xhci->Op)
return;
@@ -323,6 +347,7 @@ void XhciPowerRootPorts(XHCI_CONTROLLER* xhci) {
}
// Implements the ack Interrupter0 operation.
void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
if (!xhci->Runtime) return;
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
@@ -331,6 +356,7 @@ void XhciAckInterrupter0(XHCI_CONTROLLER* xhci) {
}
// Updates erdp.
void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
uint64_t erdp = xhci->EventRingPhys + (uint64_t)xhci->EventDequeue * sizeof(XHCI_TRB);
@@ -338,6 +364,7 @@ void XhciUpdateErdp(XHCI_CONTROLLER* xhci) {
}
// Returns command Waiter Locked.
static XHCI_WAIT_TARGET* XhciFindCommandWaiterLocked(XHCI_CONTROLLER* xhci, uint64_t commandPhys) {
for (XHCI_WAIT_TARGET* w = xhci->WaiterList; w; w = w->Next) {
if (w->CommandPhys && w->CommandPhys == commandPhys)
@@ -401,6 +428,7 @@ static XHCI_WAIT_TARGET* XhciClaimTransferWaiter(XHCI_CONTROLLER* xhci,
}
// Implements the drain Event Ring operation.
bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
(void)target;
bool processed = false;
@@ -457,6 +485,7 @@ bool XhciDrainEventRing(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
}
// Implements the drain Event Ring Locked operation.
bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci) return false;
uintptr_t flags = KeAcquireSpinlock(&xhci->EventLock);
@@ -466,6 +495,7 @@ bool XhciDrainEventRingLocked(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
}
// Waits for target Complete.
bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
if (!target) return false;
if (target->CommandPhys && !target->CommandMatched) return false;
@@ -474,6 +504,7 @@ bool XhciWaitTargetComplete(const XHCI_WAIT_TARGET* target) {
}
// Registers wait Target.
void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || target->Registered)
return;
@@ -486,6 +517,7 @@ void XhciRegisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
}
// Implements the unregister Wait Target operation.
void XhciUnregisterWaitTarget(XHCI_CONTROLLER* xhci, XHCI_WAIT_TARGET* target) {
if (!xhci || !target || !target->Registered)
return;
@@ -561,12 +593,13 @@ void XhciEventDpcRoutine(KDPC* dpc, void* context,
}
#define XHCI_CONTROLLER_MAGIC 0x58484349u /* XHCI */
#define XHCI_CONTROLLER_MAGIC 0x58484349u // XHCI
#define XHCI_MAX_CONTROLLERS 8u
static XHCI_CONTROLLER* gXhciControllers[XHCI_MAX_CONTROLLERS];
static uint32_t gXhciControllerCount;
// Registers controller.
static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) {
if (!xhci) return STATUS_INVALID_PARAMETER;
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
@@ -579,6 +612,7 @@ static KSTATUS XhciRegisterController(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Implements the isr operation.
void XhciIsr(void) {
for (uint32_t i = 0; i < gXhciControllerCount; i++) {
XHCI_CONTROLLER* xhci = gXhciControllers[i];
@@ -597,6 +631,7 @@ void XhciIsr(void) {
}
// Implements the bios Handoff operation.
KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) {
uint32_t xecp = (xhci->Hcc1 >> 16) & 0xFFFFu;
if (!xecp) return STATUS_SUCCESS;
@@ -641,6 +676,7 @@ KSTATUS XhciBiosHandoff(XHCI_CONTROLLER* xhci) {
}
// Implements the stop And Reset operation.
KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
if (cmd & XHCI_USBCMD_RS) {
@@ -674,6 +710,7 @@ KSTATUS XhciStopAndReset(XHCI_CONTROLLER* xhci) {
}
// Creates scratchpads.
KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
if (xhci->ScratchpadCount == 0) return STATUS_SUCCESS;
if (xhci->ScratchpadCount > XHCI_MAX_SCRATCHPADS) {
@@ -708,6 +745,7 @@ KSTATUS XhciAllocateScratchpads(XHCI_CONTROLLER* xhci) {
}
// Creates runtime Memory.
KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
xhci->Dcbaa = (uint64_t*)UsbDmaAllocPages(xhci, 1, &xhci->DcbaaPhys);
xhci->CommandRing = (XHCI_TRB*)UsbDmaAllocPages(xhci, 1, &xhci->CommandRingPhys);
@@ -749,6 +787,7 @@ KSTATUS XhciAllocateRuntimeMemory(XHCI_CONTROLLER* xhci) {
}
// Implements the program Runtime operation.
KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
uint8_t slots = xhci->MaxSlots;
if (slots == 0) return STATUS_DEVICE_ERROR;
@@ -770,6 +809,7 @@ KSTATUS XhciProgramRuntime(XHCI_CONTROLLER* xhci) {
}
// Implements the command Trb Phys operation.
uint64_t XhciCommandTrbPhys(XHCI_CONTROLLER* xhci, uint32_t index) {
return xhci->CommandRingPhys + (uint64_t)index * sizeof(XHCI_TRB);
}
@@ -853,6 +893,7 @@ KSTATUS XhciSubmitCommand(XHCI_CONTROLLER* xhci,
}
// Submits noop Command.
KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci) {
XHCI_COMMAND_RESULT result;
KSTATUS st = XhciSubmitCommand(
@@ -873,6 +914,7 @@ KSTATUS XhciSubmitNoopCommand(XHCI_CONTROLLER* xhci) {
}
// Enables slot.
KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
@@ -898,6 +940,7 @@ KSTATUS XhciEnableSlot(XHCI_CONTROLLER* xhci, uint8_t* outSlotId) {
// Disables slot.
KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
if (!xhci || slotId == 0)
return STATUS_INVALID_PARAMETER;
@@ -919,6 +962,7 @@ KSTATUS XhciDisableSlot(XHCI_CONTROLLER* xhci, uint8_t slotId) {
return STATUS_SUCCESS;
}
// Implements the address Device operation.
KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
XHCI_COMMAND_RESULT result;
memset(&result, 0, sizeof(result));
@@ -939,6 +983,7 @@ KSTATUS XhciAddressDevice(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
}
// Starts the requested operation.
KSTATUS XhciStart(XHCI_CONTROLLER* xhci) {
uint32_t cmd = XhciRead32(xhci->Op, XHCI_OP_USBCMD);
cmd |= XHCI_USBCMD_RS;
@@ -956,6 +1001,7 @@ KSTATUS XhciStart(XHCI_CONTROLLER* xhci) {
}
// Enables interrupts.
void XhciEnableInterrupts(XHCI_CONTROLLER* xhci) {
volatile uint8_t* ir0 = xhci->Runtime + 0x20u;
XhciWrite32(xhci->Op, XHCI_OP_USBSTS,
@@ -966,27 +1012,30 @@ void XhciEnableInterrupts(XHCI_CONTROLLER* xhci) {
}
// Implements the port Reg operation.
uint32_t XhciPortReg(uint8_t port) {
return XHCI_OP_PORT_BASE + ((uint32_t)port - 1u) * XHCI_OP_PORT_STRIDE;
}
// Clears port Changes.
void XhciClearPortChanges(XHCI_CONTROLLER* xhci, uint8_t port) {
uint32_t off = XhciPortReg(port);
uint32_t portsc = XhciRead32(xhci->Op, off);
uint32_t changes = portsc & XHCI_PORTSC_CHANGE_MASK;
if (changes) {
/*
* PORTSC is a mixed register. The change bits are RW1C, but many
* other bits are either RO or have side effects when written as one
* (PED disables the port; PR starts a reset). Preserve only Port
* Power and write one to the RW1C change bits.
*/
// PORTSC is a mixed register. The change bits are RW1C, but many
// other bits are either RO or have side effects when written as one
// (PED disables the port; PR starts a reset). Preserve only Port
// Power and write one to the RW1C change bits.
XhciWrite32(xhci->Op, off, (portsc & XHCI_PORTSC_PP) | changes);
}
}
// Implements the reset Port operation.
KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
uint32_t off = XhciPortReg(port);
uint32_t portsc = XhciRead32(xhci->Op, off);
@@ -1000,12 +1049,12 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
portsc = XhciRead32(xhci->Op, off);
if (protocol >= 3u || speed >= 4u) {
/*
* SuperSpeed ports are enabled by link training. Do not issue a
* USB2-style PORTSC.PR reset; on real Intel controllers that can make
* the link drop during early enumeration. If firmware/controller
* already has PR set, just wait for it to settle.
*/
// SuperSpeed ports are enabled by link training. Do not issue a
// USB2-style PORTSC.PR reset; on real Intel controllers that can make
// the link drop during early enumeration. If firmware/controller
// already has PR set, just wait for it to settle.
bool enabled = false;
for (uint32_t i = 0; i < 1000000u; i++) {
portsc = XhciRead32(xhci->Op, off);
@@ -1070,6 +1119,7 @@ KSTATUS XhciResetPort(XHCI_CONTROLLER* xhci, uint8_t port, uint8_t* outSpeed) {
}
// Creates device Memory.
KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
dev->InputContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->InputContextPhys);
dev->DeviceContext = (uint8_t*)UsbDmaAllocPages(xhci, 1, &dev->DeviceContextPhys);
@@ -1094,6 +1144,7 @@ KSTATUS XhciAllocateDeviceMemory(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
}
// Builds default Control Context.
void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
memset(dev->InputContext, 0, PAGE_SIZE);
memset(dev->DeviceContext, 0, PAGE_SIZE);
@@ -1137,6 +1188,7 @@ void XhciBuildDefaultControlContext(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev) {
}
// Implements the ep0 Trb Phys operation.
uint64_t XhciEp0TrbPhys(XHCI_DEVICE* dev, uint32_t index) {
return dev->Ep0RingPhys + (uint64_t)index * sizeof(XHCI_TRB);
}
@@ -1405,6 +1457,7 @@ KSTATUS XhciGetConfigurationDescriptor(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
}
// Implements the endpoint Id From Address operation.
uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress) {
uint8_t number = endpointAddress & USB_ENDPOINT_NUMBER_MASK;
if (number == 0) return 1;
@@ -1412,6 +1465,7 @@ uint8_t XhciEndpointIdFromAddress(uint8_t endpointAddress) {
}
// Implements the interrupt Interval operation.
uint8_t XhciInterruptInterval(uint8_t speed, uint8_t usbInterval) {
if (usbInterval == 0) usbInterval = 1;
@@ -1544,6 +1598,7 @@ KSTATUS XhciBulkTransfer(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev,
// Implements the configure Pipe Endpoint operation.
KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_PIPE* pipe) {
if (!xhci || !dev || !pipe || !pipe->Open)
return STATUS_INVALID_PARAMETER;
@@ -1631,6 +1686,7 @@ KSTATUS XhciConfigurePipeEndpoint(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_P
return STATUS_SUCCESS;
}
// Implements the hcd Submit Interrupt Urb operation.
static KSTATUS XhciHcdSubmitInterruptUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
return STATUS_INVALID_PARAMETER;
@@ -1725,6 +1781,7 @@ void XhciHandleAsyncTransferEvent(XHCI_CONTROLLER* xhci, uint64_t trbPhys,
IoCompleteRequest(completeIrp, st, KSUCCESS(st) ? actual : 0);
}
// Implements the hcd Submit Bulk Urb operation.
static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB_URB* urb) {
if (!xhci || !dev || !urb || !urb->Pipe || !urb->BufferPhys || urb->Length == 0)
return STATUS_INVALID_PARAMETER;
@@ -1764,6 +1821,7 @@ static KSTATUS XhciHcdSubmitBulkUrb(XHCI_CONTROLLER* xhci, XHCI_DEVICE* dev, USB
return st;
}
// Implements the hcd Submit Urb operation.
static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
if (!hcd || !urb || !urb->Device)
return STATUS_INVALID_PARAMETER;
@@ -1806,6 +1864,7 @@ static KSTATUS XhciHcdSubmitUrb(USB_HCD* hcd, USB_URB* urb) {
return st;
}
// Implements the hcd Cancel Urb operation.
static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
(void)hcd;
if (!urb) return STATUS_INVALID_PARAMETER;
@@ -1815,16 +1874,19 @@ static KSTATUS XhciHcdCancelUrb(USB_HCD* hcd, USB_URB* urb) {
return STATUS_SUCCESS;
}
// Implements the hcd Allocate Dma operation.
static void* XhciHcdAllocateDma(USB_HCD* hcd, size_t pages, PHYSICAL_ADDRESS* outPhys) {
if (!hcd) return (void*)0;
return UsbDmaAllocPages((XHCI_CONTROLLER*)hcd->ControllerContext, pages, outPhys);
}
// Implements the hcd Reset Root Port operation.
static KSTATUS XhciHcdResetRootPort(USB_HCD* hcd, uint8_t port, uint8_t* outSpeed) {
if (!hcd) return STATUS_INVALID_PARAMETER;
return XhciResetPort((XHCI_CONTROLLER*)hcd->ControllerContext, port, outSpeed);
}
// Implements the hcd Clear Root Port Changes operation.
static KSTATUS XhciHcdClearRootPortChanges(USB_HCD* hcd, uint8_t port) {
if (!hcd) return STATUS_INVALID_PARAMETER;
XhciClearPortChanges((XHCI_CONTROLLER*)hcd->ControllerContext, port);
@@ -1839,6 +1901,7 @@ const USB_HCD_OPERATIONS gXhciHcdOperations = {
XhciHcdAllocateDma,
};
// Starts controller.
KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
if (!xhci || xhci->Magic != XHCI_CONTROLLER_MAGIC || !xhci->Pdo || !xhci->Fdo)
return STATUS_INVALID_PARAMETER;
@@ -2011,6 +2074,7 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return STATUS_SUCCESS;
}
// Dispatches pnp.
KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return STATUS_INVALID_PARAMETER;
@@ -2061,6 +2125,7 @@ KSTATUS XhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the add Device operation.
KSTATUS XhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
code/drivers/display/vmwaresvga/vmwaresvga.c
+114 -67
View File
@@ -1,9 +1,4 @@
// vmwaresvga.c - VMware SVGA-II PCI display provider.
//
// The default renderer is the legacy guest-framebuffer path: draw into BAR1,
// enqueue SVGA_CMD_UPDATE through BAR2, and synchronize with the device.
// Screen objects remain available behind VMWSVGA_ENABLE_SCREEN_OBJECTS for
// future validation.
// VMware SVGA-II PCI display provider. This file owns adapter setup, FIFO command submission, framebuffer presentation, and cursor integration.
#include "vmwaresvga.h"
@@ -23,16 +18,19 @@ static const IO_DRIVER_ID_TABLE gVmwSvgaIdTable = {
static KSTATUS VmwSvgaSyncDevice(VMWSVGA_DEVICE* v);
static void VmwSvgaWakeFifo(VMWSVGA_DEVICE* v);
// Reads a 32-bit SVGA register through the indexed I/O port pair.
static uint32_t VmwSvgaReadReg(VMWSVGA_DEVICE* v, uint32_t reg) {
__outdword((unsigned short)(v->IoBase + SVGA_INDEX_PORT), reg);
return __indword((unsigned short)(v->IoBase + SVGA_VALUE_PORT));
}
// Writes a 32-bit SVGA register through the indexed I/O port pair.
static void VmwSvgaWriteReg(VMWSVGA_DEVICE* v, uint32_t reg, uint32_t value) {
__outdword((unsigned short)(v->IoBase + SVGA_INDEX_PORT), reg);
__outdword((unsigned short)(v->IoBase + SVGA_VALUE_PORT), value);
}
// Negotiates the highest SVGA device ID supported by the adapter.
static uint32_t VmwSvgaProbeId(VMWSVGA_DEVICE* v) {
for (uint32_t id = SVGA_ID_2; id >= SVGA_ID_0; id--) {
VmwSvgaWriteReg(v, SVGA_REG_ID, id);
@@ -45,6 +43,7 @@ static uint32_t VmwSvgaProbeId(VMWSVGA_DEVICE* v) {
return SVGA_ID_INVALID;
}
// Chooses the runtime display mode, preferring the current boot framebuffer size.
static KSTATUS VmwSvgaChooseMode(uint32_t* outWidth, uint32_t* outHeight) {
if (!outWidth || !outHeight)
return STATUS_INVALID_PARAMETER;
@@ -64,11 +63,12 @@ static KSTATUS VmwSvgaChooseMode(uint32_t* outWidth, uint32_t* outHeight) {
return STATUS_SUCCESS;
}
// Programs the visible guest display rectangle used by the legacy framebuffer path.
static void VmwSvgaProgramLegacyDisplayTopology(VMWSVGA_DEVICE* v, uint32_t width, uint32_t height, uint32_t caps) {
if (!v || (caps & SVGA_CAP_DISPLAY_TOPOLOGY) == 0)
return;
/* Legacy GFB rendering still needs one visible primary display rectangle. */
// Legacy GFB rendering still needs one visible primary display rectangle.
VmwSvgaWriteReg(v, SVGA_REG_NUM_GUEST_DISPLAYS, 1u);
VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_ID, 0u);
VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_IS_PRIMARY, 1u);
@@ -78,6 +78,7 @@ static void VmwSvgaProgramLegacyDisplayTopology(VMWSVGA_DEVICE* v, uint32_t widt
VmwSvgaWriteReg(v, SVGA_REG_DISPLAY_HEIGHT, height);
}
// Programs the SVGA device mode before FIFO and framebuffer use.
static KSTATUS VmwSvgaSetMode(VMWSVGA_DEVICE* v) {
if (!v)
return STATUS_INVALID_PARAMETER;
@@ -118,6 +119,7 @@ static KSTATUS VmwSvgaSetMode(VMWSVGA_DEVICE* v) {
return STATUS_SUCCESS;
}
// Clips a display rectangle to the current framebuffer bounds.
static bool VmwSvgaClipRect(VMWSVGA_DEVICE* v, DISPLAY_RECT* r) {
if (!v || !r || r->W <= 0 || r->H <= 0)
return false;
@@ -141,6 +143,7 @@ static bool VmwSvgaClipRect(VMWSVGA_DEVICE* v, DISPLAY_RECT* r) {
return true;
}
// Returns the next FIFO dword offset, wrapping at FIFO_MAX.
static uint32_t VmwSvgaFifoNext(VMWSVGA_DEVICE* v, uint32_t offset) {
uint32_t next = offset + sizeof(uint32_t);
uint32_t min = v->Fifo[SVGA_FIFO_MIN];
@@ -150,6 +153,7 @@ static uint32_t VmwSvgaFifoNext(VMWSVGA_DEVICE* v, uint32_t offset) {
return next;
}
// Writes one command dword into the SVGA FIFO and waits for space if needed.
static KSTATUS VmwSvgaFifoWrite(VMWSVGA_DEVICE* v, uint32_t value) {
if (!v || !v->FifoReady || !v->Fifo)
return STATUS_NOT_SUPPORTED;
@@ -182,6 +186,7 @@ static KSTATUS VmwSvgaFifoWrite(VMWSVGA_DEVICE* v, uint32_t value) {
return STATUS_TIMEOUT;
}
// Checks whether an extended FIFO register is valid for this device.
static bool VmwSvgaHasFifoRegister(VMWSVGA_DEVICE* v, uint32_t index) {
if (!v || !v->Fifo || !v->FifoReady)
return false;
@@ -189,6 +194,7 @@ static bool VmwSvgaHasFifoRegister(VMWSVGA_DEVICE* v, uint32_t index) {
v->Fifo[SVGA_FIFO_MIN] >= (index + 1u) * sizeof(uint32_t);
}
// Wakes the host side of the SVGA FIFO after new commands are queued.
static void VmwSvgaWakeFifo(VMWSVGA_DEVICE* v) {
if (!v || !v->FifoReady)
return;
@@ -207,6 +213,7 @@ static void VmwSvgaWakeFifo(VMWSVGA_DEVICE* v) {
static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels);
// Implements the premultiply Argb For Bgra operation.
static uint32_t VmwSvgaPremultiplyArgbForBgra(uint32_t argb) {
uint32_t a = argb >> 24;
if (a == 0)
@@ -224,6 +231,7 @@ static uint32_t VmwSvgaPremultiplyArgbForBgra(uint32_t argb) {
return (a << 24) | (r << 16) | (g << 8) | b;
}
// Implements the cursor Commit operation.
static void VmwSvgaCursorCommit(VMWSVGA_DEVICE* v) {
if (!v || !v->HardwareCursor || !v->CursorImageDefined)
return;
@@ -231,6 +239,7 @@ static void VmwSvgaCursorCommit(VMWSVGA_DEVICE* v) {
if (v->CursorBypass3 &&
VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_ON) &&
VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_X) &&
// Checks whether an extended FIFO register is valid for this device.
VmwSvgaHasFifoRegister(v, SVGA_FIFO_CURSOR_Y)) {
v->Fifo[SVGA_FIFO_CURSOR_X] = (uint32_t)v->CursorX;
v->Fifo[SVGA_FIFO_CURSOR_Y] = (uint32_t)v->CursorY;
@@ -245,19 +254,19 @@ static void VmwSvgaCursorCommit(VMWSVGA_DEVICE* v) {
return;
}
/*
* Legacy QEMU SVGA cursors are frontend/host cursors. Re-defining the
* cursor image from the input path can stall the FIFO/frontend after a
* GTK focus/grab transition, so cursor activity must only update the
* legacy position/visibility registers. The image is defined once when
* SetCursorImage runs.
*/
// Legacy QEMU SVGA cursors are frontend/host cursors. Re-defining the
// cursor image from the input path can stall the FIFO/frontend after a
// GTK focus/grab transition, so cursor activity must only update the
// legacy position/visibility registers. The image is defined once when
// SetCursorImage runs.
VmwSvgaWriteReg(v, SVGA_REG_CURSOR_ID, 0u);
VmwSvgaWriteReg(v, SVGA_REG_CURSOR_X, (uint32_t)v->CursorX);
VmwSvgaWriteReg(v, SVGA_REG_CURSOR_Y, (uint32_t)v->CursorY);
VmwSvgaWriteReg(v, SVGA_REG_CURSOR_ON, v->CursorVisible ? SVGA_CURSOR_ON_SHOW : SVGA_CURSOR_ON_HIDE);
}
// Implements the cursor Set Visible operation.
static void VmwSvgaCursorSetVisible(VMWSVGA_DEVICE* v, bool visible) {
if (!v || !v->HardwareCursor)
return;
@@ -265,6 +274,7 @@ static void VmwSvgaCursorSetVisible(VMWSVGA_DEVICE* v, bool visible) {
VmwSvgaCursorCommit(v);
}
// Implements the cursor Set Position operation.
static void VmwSvgaCursorSetPosition(VMWSVGA_DEVICE* v, int x, int y) {
if (!v || !v->HardwareCursor)
return;
@@ -273,6 +283,7 @@ static void VmwSvgaCursorSetPosition(VMWSVGA_DEVICE* v, int x, int y) {
VmwSvgaCursorCommit(v);
}
// Implements the define Alpha Cursor operation.
static KSTATUS VmwSvgaDefineAlphaCursor(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_ALPHA_CURSOR);
if (!KSUCCESS(st)) return st;
@@ -303,13 +314,13 @@ static uint32_t VmwSvgaPackMonoCursorDword(const uint8_t* bits,
int height,
int y,
uint32_t dwordIndex) {
/*
* SVGA classic cursor scanlines are padded to 32 bits. QEMU stores
* FIFO dwords in guest-endian order and then interprets the result as
* bytes via cursor_set_mono(), whose first pixel is bit 0x80 of byte 0.
* Therefore byte 0 of the numeric dword must contain pixels 0..7,
* byte 1 pixels 8..15, and so on.
*/
// SVGA classic cursor scanlines are padded to 32 bits. QEMU stores
// FIFO dwords in guest-endian order and then interprets the result as
// bytes via cursor_set_mono(), whose first pixel is bit 0x80 of byte 0.
// Therefore byte 0 of the numeric dword must contain pixels 0..7,
// byte 1 pixels 8..15, and so on.
uint8_t bytes[4] = { 0, 0, 0, 0 };
uint32_t baseX = dwordIndex * 32u;
@@ -331,17 +342,18 @@ static uint32_t VmwSvgaPackMonoCursorDword(const uint8_t* bits,
((uint32_t)bytes[3] << 24);
}
// Implements the classic Cursor Width operation.
static int VmwSvgaClassicCursorWidth(int width) {
/*
* QEMU's VMware SVGA implementation reads classic cursor masks as
* SVGA dword-padded scanlines, then passes the raw byte stream to
* cursor_set_mono(), which advances rows by ceil(width / 8) bytes.
* For widths that are not a multiple of 32 pixels, those two row
* strides differ and every other row is decoded from padding bytes.
* Define the classic cursor with a 32-pixel-aligned visible width and
* leave the extra pixels transparent. The real cursor image remains
* at the left edge and the hotspot is unchanged.
*/
// QEMU's VMware SVGA implementation reads classic cursor masks as
// SVGA dword-padded scanlines, then passes the raw byte stream to
// cursor_set_mono(), which advances rows by ceil(width / 8) bytes.
// For widths that are not a multiple of 32 pixels, those two row
// strides differ and every other row is decoded from padding bytes.
// Define the classic cursor with a 32-pixel-aligned visible width and
// leave the extra pixels transparent. The real cursor image remains
// at the left edge and the hotspot is unchanged.
return (width + 31) & ~31;
}
@@ -351,15 +363,15 @@ static void VmwSvgaBuildMonoCursorBits(const uint32_t* argbPixels,
int height,
uint8_t* andBits,
uint8_t* xorBits) {
/*
* Classic SVGA cursors are AND/XOR masks. The most compatible path is
* a 1bpp AND mask plus a 1bpp XOR mask:
* transparent: AND=1 XOR=0
* black: AND=0 XOR=0
* white: AND=0 XOR=1
* This intentionally discards antialiasing, but avoids QEMU/GTK's
* fragile 32bpp classic host-cursor path.
*/
// Classic SVGA cursors are AND/XOR masks. The most compatible path is
// a 1bpp AND mask plus a 1bpp XOR mask:
// transparent: AND=1 XOR=0
// black: AND=0 XOR=0
// white: AND=0 XOR=1
// This intentionally discards antialiasing, but avoids QEMU/GTK's
// fragile 32bpp classic host-cursor path.
for (int y = 0; y < height; y++) {
for (int x = 0; x < encodedWidth; x++) {
size_t out = (size_t)y * (size_t)encodedWidth + (size_t)x;
@@ -387,6 +399,7 @@ static void VmwSvgaBuildMonoCursorBits(const uint32_t* argbPixels,
}
}
// Implements the define Classic Cursor operation.
static KSTATUS VmwSvgaDefineClassicCursor(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
int encodedWidth = VmwSvgaClassicCursorWidth(width);
uint32_t dwordsPerRow = ((uint32_t)encodedWidth + 31u) / 32u;
@@ -447,6 +460,7 @@ done:
return st;
}
// Implements the upload Cursor Image operation.
static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
if (!v || !v->HardwareCursor || !argbPixels || width <= 0 || height <= 0)
return STATUS_INVALID_PARAMETER;
@@ -464,6 +478,7 @@ static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height
return st;
}
// Implements the define Cursor Image operation.
static KSTATUS VmwSvgaDefineCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
if (!v || !v->HardwareCursor || !argbPixels || width <= 0 || height <= 0)
return STATUS_INVALID_PARAMETER;
@@ -495,10 +510,12 @@ static KSTATUS VmwSvgaDefineCursorImage(VMWSVGA_DEVICE* v, int width, int height
}
// Implements the align Up32 operation.
static uint32_t VmwSvgaAlignUp32(uint32_t value, uint32_t alignment) {
return (value + alignment - 1u) & ~(alignment - 1u);
}
// Implements the define Screen Object operation.
static KSTATUS VmwSvgaDefineScreenObject(VMWSVGA_DEVICE* v) {
if (!v || !v->FifoReady || !v->UseScreenObject)
return STATUS_INVALID_PARAMETER;
@@ -506,50 +523,52 @@ static KSTATUS VmwSvgaDefineScreenObject(VMWSVGA_DEVICE* v) {
KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_SCREEN);
if (!KSUCCESS(st)) return st;
/* SVGAScreenObject, structSize = 11 dwords with backingStore+cloneCount. */
st = VmwSvgaFifoWrite(v, 44u); /* structSize */
// SVGAScreenObject, structSize = 11 dwords with backingStore+cloneCount.
st = VmwSvgaFifoWrite(v, 44u); // structSize
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->ScreenId); /* id */
st = VmwSvgaFifoWrite(v, v->ScreenId); // id
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, SVGA_SCREEN_MUST_BE_SET | SVGA_SCREEN_IS_PRIMARY);
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->Width); /* size.width */
st = VmwSvgaFifoWrite(v, v->Width); // size.width
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->Height); /* size.height */
st = VmwSvgaFifoWrite(v, v->Height); // size.height
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, 0u); /* root.x */
st = VmwSvgaFifoWrite(v, 0u); // root.x
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, 0u); /* root.y */
st = VmwSvgaFifoWrite(v, 0u); // root.y
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, SVGA_GMR_FRAMEBUFFER); /* backingStore.ptr.gmrId */
st = VmwSvgaFifoWrite(v, SVGA_GMR_FRAMEBUFFER); // backingStore.ptr.gmrId
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->ScreenBackingOffset); /* backingStore.ptr.offset */
st = VmwSvgaFifoWrite(v, v->ScreenBackingOffset); // backingStore.ptr.offset
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->BytesPerLine); /* backingStore.pitch */
st = VmwSvgaFifoWrite(v, v->BytesPerLine); // backingStore.pitch
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, 0u); /* cloneCount */
st = VmwSvgaFifoWrite(v, 0u); // cloneCount
if (!KSUCCESS(st)) return st;
return VmwSvgaSyncDevice(v);
}
// Implements the define Gmrfb operation.
static KSTATUS VmwSvgaDefineGmrfb(VMWSVGA_DEVICE* v) {
if (!v || !v->FifoReady || !v->UseScreenObject)
return STATUS_INVALID_PARAMETER;
KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_GMRFB);
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, SVGA_GMR_FRAMEBUFFER); /* ptr.gmrId */
st = VmwSvgaFifoWrite(v, SVGA_GMR_FRAMEBUFFER); // ptr.gmrId
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->ScreenSourceOffset); /* ptr.offset */
st = VmwSvgaFifoWrite(v, v->ScreenSourceOffset); // ptr.offset
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->BytesPerLine); /* bytesPerLine */
st = VmwSvgaFifoWrite(v, v->BytesPerLine); // bytesPerLine
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, SVGA_GMRFB_FORMAT_32BPP_24DEPTH);
if (!KSUCCESS(st)) return st;
return STATUS_SUCCESS;
}
// Blits gmrfb To Screen.
static KSTATUS VmwSvgaBlitGmrfbToScreen(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
@@ -557,25 +576,26 @@ static KSTATUS VmwSvgaBlitGmrfbToScreen(VMWSVGA_DEVICE* v, int x, int y, int w,
KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_BLIT_GMRFB_TO_SCREEN);
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, (uint32_t)r.X); /* srcOrigin.x */
st = VmwSvgaFifoWrite(v, (uint32_t)r.X); // srcOrigin.x
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, (uint32_t)r.Y); /* srcOrigin.y */
st = VmwSvgaFifoWrite(v, (uint32_t)r.Y); // srcOrigin.y
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, (uint32_t)r.X); /* destRect.left */
st = VmwSvgaFifoWrite(v, (uint32_t)r.X); // destRect.left
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, (uint32_t)r.Y); /* destRect.top */
st = VmwSvgaFifoWrite(v, (uint32_t)r.Y); // destRect.top
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, (uint32_t)(r.X + r.W)); /* destRect.right */
st = VmwSvgaFifoWrite(v, (uint32_t)(r.X + r.W)); // destRect.right
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, (uint32_t)(r.Y + r.H)); /* destRect.bottom */
st = VmwSvgaFifoWrite(v, (uint32_t)(r.Y + r.H)); // destRect.bottom
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, v->ScreenId); /* destScreenId */
st = VmwSvgaFifoWrite(v, v->ScreenId); // destScreenId
if (!KSUCCESS(st)) return st;
VmwSvgaWakeFifo(v);
return STATUS_SUCCESS;
}
// Initializes screen Object.
static KSTATUS VmwSvgaInitializeScreenObject(VMWSVGA_DEVICE* v) {
if (!v || !v->FifoReady)
return STATUS_INVALID_PARAMETER;
@@ -586,12 +606,11 @@ static KSTATUS VmwSvgaInitializeScreenObject(VMWSVGA_DEVICE* v) {
bool hasScreenObject2 = ((v->FifoCaps & SVGA_FIFO_CAP_SCREEN_OBJECT_2) != 0) &&
((v->Capabilities & SVGA_CAP_SCREEN_OBJECT_2) != 0);
/*
* Real VMware can advertise SCREEN_OBJECT_2 without also setting the
* original FIFO_CAP_SCREEN_OBJECT bit. The command set is still the
* Screen Object command set: DEFINE_SCREEN, DEFINE_GMRFB, and
* BLIT_GMRFB_TO_SCREEN. Treat either capability generation as usable.
*/
// Real VMware can advertise SCREEN_OBJECT_2 without also setting the
// original FIFO_CAP_SCREEN_OBJECT bit. The command set is still the
// Screen Object command set: DEFINE_SCREEN, DEFINE_GMRFB, and
// BLIT_GMRFB_TO_SCREEN. Treat either capability generation as usable.
if (!hasScreenObject1 && !hasScreenObject2) {
DbgPrint("VMWSVGA: screen object unavailable: screenObject1=%u screenObject2=%u caps=0x%x fifoCaps=0x%x\n",
hasScreenObject1 ? 1u : 0u,
@@ -640,6 +659,7 @@ static KSTATUS VmwSvgaInitializeScreenObject(VMWSVGA_DEVICE* v) {
return STATUS_SUCCESS;
}
// Implements the legacy Update operation.
static KSTATUS VmwSvgaLegacyUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
@@ -660,6 +680,7 @@ static KSTATUS VmwSvgaLegacyUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h
return STATUS_SUCCESS;
}
// Updates the requested value.
static KSTATUS VmwSvgaUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
@@ -676,6 +697,7 @@ static KSTATUS VmwSvgaUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
return VmwSvgaLegacyUpdate(v, r.X, r.Y, r.W, r.H);
}
// Waits until the SVGA host has consumed the queued FIFO work.
static KSTATUS VmwSvgaSyncDevice(VMWSVGA_DEVICE* v) {
if (!v)
return STATUS_INVALID_PARAMETER;
@@ -691,6 +713,7 @@ static KSTATUS VmwSvgaSyncDevice(VMWSVGA_DEVICE* v) {
return STATUS_TIMEOUT;
}
// Implements the blend Pixel operation.
static uint32_t VmwSvgaBlendPixel(uint32_t dst, uint32_t src) {
uint8_t a = (uint8_t)(src >> 24);
if (a == 0) return dst;
@@ -709,10 +732,12 @@ static uint32_t VmwSvgaBlendPixel(uint32_t dst, uint32_t src) {
return ((uint32_t)rr << 16) | ((uint32_t)gg << 8) | bb;
}
// Implements the from Display operation.
static VMWSVGA_DEVICE* VmwSvgaFromDisplay(DISPLAY_DEVICE* device) {
return device ? (VMWSVGA_DEVICE*)device->DriverContext : NULL;
}
// Implements the display Get Mode operation.
static KSTATUS VmwSvgaDisplayGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !outMode || !v->Framebuffer)
@@ -724,6 +749,7 @@ static KSTATUS VmwSvgaDisplayGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMo
return STATUS_SUCCESS;
}
// Implements the display Blit operation.
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0)
@@ -747,6 +773,7 @@ static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, i
return STATUS_SUCCESS;
}
// Implements the display Fill Rect operation.
static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || w <= 0 || h <= 0)
@@ -767,6 +794,7 @@ static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int
return STATUS_SUCCESS;
}
// Implements the display Copy Rect operation.
static KSTATUS VmwSvgaDisplayCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || w <= 0 || h <= 0)
@@ -803,6 +831,7 @@ static KSTATUS VmwSvgaDisplayCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY
return STATUS_SUCCESS;
}
// Implements the display Read Rect operation.
static KSTATUS VmwSvgaDisplayReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
@@ -825,6 +854,7 @@ static KSTATUS VmwSvgaDisplayReadRect(DISPLAY_DEVICE* device, int x, int y, int
return STATUS_SUCCESS;
}
// Implements the display Present operation.
static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Framebuffer || !v->Backbuffer)
@@ -866,6 +896,7 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT*
return result;
}
// Implements the display Sync operation.
static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
KSTATUS st = VmwSvgaSyncDevice(v);
@@ -874,6 +905,7 @@ static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) {
return st;
}
// Implements the display Set Cursor Image operation.
static KSTATUS VmwSvgaDisplaySetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->HardwareCursor)
@@ -881,6 +913,7 @@ static KSTATUS VmwSvgaDisplaySetCursorImage(DISPLAY_DEVICE* device, int width, i
return VmwSvgaDefineCursorImage(v, width, height, hotX, hotY, argbPixels);
}
// Implements the display Set Cursor Position operation.
static KSTATUS VmwSvgaDisplaySetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v)
@@ -893,6 +926,7 @@ static KSTATUS VmwSvgaDisplaySetCursorPosition(DISPLAY_DEVICE* device, int x, in
return STATUS_SUCCESS;
}
// Implements the restore Cursor operation.
static void VmwSvgaRestoreCursor(VMWSVGA_DEVICE* v) {
if (!v || !v->CursorDrawn || !v->Backbuffer || !v->Framebuffer)
return;
@@ -912,6 +946,7 @@ static void VmwSvgaRestoreCursor(VMWSVGA_DEVICE* v) {
v->CursorDrawY = -1;
}
// Draws cursor.
static void VmwSvgaDrawCursor(VMWSVGA_DEVICE* v) {
if (!v || !v->Backbuffer || !v->Framebuffer)
return;
@@ -944,6 +979,7 @@ static void VmwSvgaDrawCursor(VMWSVGA_DEVICE* v) {
(void)VmwSvgaUpdate(v, dirty.X, dirty.Y, dirty.W, dirty.H);
}
// Implements the display Show Cursor operation.
static KSTATUS VmwSvgaDisplayShowCursor(DISPLAY_DEVICE* device, bool show) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v)
@@ -978,6 +1014,7 @@ static KSTATUS VmwSvgaDisplayShowCursor(DISPLAY_DEVICE* device, bool show) {
return STATUS_SUCCESS;
}
// Implements the display Query Stats operation.
static KSTATUS VmwSvgaDisplayQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !outStats)
@@ -1000,6 +1037,7 @@ static const DISPLAY_DEVICE_OPS gVmwSvgaDisplayOps = {
.QueryStats = VmwSvgaDisplayQueryStats,
};
// Implements the select Bars operation.
static KSTATUS VmwSvgaSelectBars(VMWSVGA_DEVICE* v) {
PCI_BAR bar;
memset(&bar, 0, sizeof(bar));
@@ -1029,10 +1067,12 @@ static KSTATUS VmwSvgaSelectBars(VMWSVGA_DEVICE* v) {
return STATUS_SUCCESS;
}
// Implements the mmio Page Flags operation.
static uint64_t VmwSvgaMmioPageFlags(void) {
return PAGE_PRESENT | PAGE_WRITE | PAGE_WRITETHROUGH | PAGE_NOCACHE | PAGE_NX;
}
// Maps bars.
static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) {
uint64_t mmioFlags = VmwSvgaMmioPageFlags();
KSTATUS st = PciMapBar(v->Pdo, 1, mmioFlags,
@@ -1068,6 +1108,7 @@ static KSTATUS VmwSvgaMapBars(VMWSVGA_DEVICE* v) {
return STATUS_SUCCESS;
}
// Returns mode.
static KSTATUS VmwSvgaReadMode(VMWSVGA_DEVICE* v) {
v->Width = VmwSvgaReadReg(v, SVGA_REG_WIDTH);
v->Height = VmwSvgaReadReg(v, SVGA_REG_HEIGHT);
@@ -1133,6 +1174,7 @@ static KSTATUS VmwSvgaReadMode(VMWSVGA_DEVICE* v) {
return STATUS_SUCCESS;
}
// Returns the active FIFO size reported by the device, not the larger PCI BAR aperture.
static uint32_t VmwSvgaGetFifoMaxBytes(VMWSVGA_DEVICE* v) {
uint64_t limit = v->FifoBytes;
if (v->FifoDeviceBytes != 0 && (uint64_t)v->FifoDeviceBytes < limit)
@@ -1142,6 +1184,7 @@ static uint32_t VmwSvgaGetFifoMaxBytes(VMWSVGA_DEVICE* v) {
return (uint32_t)limit & ~3u;
}
// Initializes fifo.
static KSTATUS VmwSvgaInitializeFifo(VMWSVGA_DEVICE* v) {
if (!v->Fifo || v->FifoBytes < 4096u)
return STATUS_DEVICE_ERROR;
@@ -1188,6 +1231,7 @@ static KSTATUS VmwSvgaInitializeFifo(VMWSVGA_DEVICE* v) {
return STATUS_SUCCESS;
}
// Starts device.
static KSTATUS VmwSvgaStartDevice(VMWSVGA_DEVICE* v) {
if (!v || v->Magic != VMWSVGA_DEVICE_MAGIC || !v->Fdo || !v->Pdo)
return STATUS_INVALID_PARAMETER;
@@ -1322,6 +1366,7 @@ static KSTATUS VmwSvgaStartDevice(VMWSVGA_DEVICE* v) {
return STATUS_SUCCESS;
}
// Dispatches pnp.
KSTATUS VmwSvgaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return STATUS_INVALID_PARAMETER;
@@ -1368,6 +1413,7 @@ KSTATUS VmwSvgaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the add Device operation.
KSTATUS VmwSvgaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -1396,6 +1442,7 @@ KSTATUS VmwSvgaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal)
return STATUS_INVALID_PARAMETER;
code/drivers/display/vmwaresvga/vmwaresvga.h
+7 -6
View File
@@ -1,3 +1,5 @@
// VMware SVGA-II device definitions shared by the display provider implementation.
#pragma once
// vmwaresvga.h - VMware SVGA-II display provider internals.
@@ -19,7 +21,7 @@
#define VMWSVGA_VENDOR_VMWARE 0x15ADu
#define VMWSVGA_DEVICE_SVGA2 0x0405u
#define VMWSVGA_DEVICE_MAGIC 0x564D5356u /* VMSV */
#define VMWSVGA_DEVICE_MAGIC 0x564D5356u // VMSV
#define SVGA_ID_INVALID 0xFFFFFFFFu
#define SVGA_ID_0 0x90000000u
@@ -125,11 +127,10 @@
#define SVGA_SCREEN_OBJECT_ID_PRIMARY 0u
#define SVGA_GMRFB_FORMAT_32BPP_24DEPTH ((32u) | (24u << 8))
/*
* The production path is legacy GFB + UPDATE. Screen objects are kept behind
* an opt-in switch until the SO/GMRFB path is independently validated on real
* VMware hosts.
*/
// The production path is legacy GFB + UPDATE. Screen objects are kept behind
// an opt-in switch until the SO/GMRFB path is independently validated on real
// VMware hosts.
#ifndef VMWSVGA_ENABLE_SCREEN_OBJECTS
#define VMWSVGA_ENABLE_SCREEN_OBJECTS 0
#endif
code/drivers/fs/fat32/fat32.c
+73 -63
View File
@@ -1,9 +1,8 @@
// fat32.c - FAT32 filesystem core, directory, cache, and I/O implementation.
// FAT32 filesystem driver. This file owns filesystem operations and FAT-backed file access.
#include "fat32.h"
// ---------------------------------------------------------------------------
// On-disk structures
// ---------------------------------------------------------------------------
// On-disk structures.
#pragma pack(push, 1)
typedef struct {
@@ -110,9 +109,7 @@ typedef struct {
// FAT32_VOL forward declaration (needed by cache helpers).
typedef struct FAT32_VOL_S FAT32_VOL;
// ---------------------------------------------------------------------------
// FAT sector cache (fixes #1 and #5)
// ---------------------------------------------------------------------------
// 32 direct-mapped slots, indexed by (lba % FAT_CACHE_SLOTS).
// Write-back: WriteFatEntry modifies the in-cache bytes and marks the slot
// dirty. FlushFatCache() propagates dirty slots to every FAT copy at once,
@@ -164,12 +161,14 @@ static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) {
return STATUS_SUCCESS;
}
// Implements the fat Cache Mark Dirty operation.
static void FatCacheMarkDirty(FAT32_VOL *v, uint64_t lba) {
uint32_t slot = (uint32_t)(lba % FAT_CACHE_SLOTS);
if (gFatCache[slot].valid && gFatCache[slot].vol == v && gFatCache[slot].lba == lba)
gFatCache[slot].dirty = true;
}
// Implements the fat Cache Invalidate operation.
static void FatCacheInvalidate(void) {
for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
gFatCache[i].vol = (FAT32_VOL*)0;
@@ -182,9 +181,7 @@ static void FatCacheInvalidate(void) {
// Defined after FAT32_VOL is fully declared (below).
static void FlushFatCache(FAT32_VOL *v);
// ---------------------------------------------------------------------------
// Driver state
// ---------------------------------------------------------------------------
// Driver state.
struct FAT32_VOL_S {
uint8_t DriveIndex; // physical disk DEVICE_OBJECT index
uint64_t PartitionLba; // absolute LBA of the mounted volume
@@ -208,9 +205,7 @@ typedef struct {
uint32_t DirentOffset;
} FAT32_FILE;
// ---------------------------------------------------------------------------
// Raw sector I/O (bypasses FAT cache; used for file data and cache eviction)
// ---------------------------------------------------------------------------
// Raw sector I/O (bypasses FAT cache; used for file data and cache eviction).
static KSTATUS ReadSectorsRaw(FAT32_VOL *v, uint64_t lba,
uint32_t count, void *buf) {
if (!v) return STATUS_INVALID_PARAMETER;
@@ -225,15 +220,15 @@ static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba,
// Convenience aliases for file-data I/O.
static KSTATUS ReadSectors (FAT32_VOL *v, uint64_t lba, uint32_t n, void *b) { return ReadSectorsRaw (v, lba, n, b); }
// Updates sectors.
static KSTATUS WriteSectors(FAT32_VOL *v, uint64_t lba, uint32_t n, const void *b) { return WriteSectorsRaw(v, lba, n, b); }
// Implements the cluster To Lba operation.
static uint64_t ClusterToLba(FAT32_VOL *v, uint32_t cluster) {
return v->DataStart + (uint64_t)(cluster - 2) * v->SectorsPerCluster;
}
// ---------------------------------------------------------------------------
// FlushFatCache — needs full FAT32_VOL definition
// ---------------------------------------------------------------------------
// FlushFatCache — needs full FAT32_VOL definition.
static void FlushFatCache(FAT32_VOL *v) {
uint64_t fat0Start = (uint64_t)v->PartitionLba + v->ReservedSectors;
@@ -253,9 +248,7 @@ static void FlushFatCache(FAT32_VOL *v) {
}
}
// ---------------------------------------------------------------------------
// FAT read / write (through cache)
// ---------------------------------------------------------------------------
// FAT read / write (through cache).
static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) {
uint32_t fatOffset = cluster * 4;
@@ -268,6 +261,7 @@ static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) {
return *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
}
// Updates fat Entry.
static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) {
uint32_t fatOffset = cluster * 4;
uint64_t sector = (uint64_t)v->PartitionLba + v->ReservedSectors
@@ -285,9 +279,7 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan)
// ---------------------------------------------------------------------------
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan).
static uint32_t AllocCluster(FAT32_VOL *v) {
if (v->TotalClusters == 0) return 0;
@@ -317,6 +309,7 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
return 0;
}
// Appends cluster.
static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
uint32_t newCluster = AllocCluster(v);
if (!newCluster) return 0;
@@ -324,6 +317,7 @@ static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
return newCluster;
}
// Returns chain End.
static uint32_t GetChainEnd(FAT32_VOL *v, uint32_t start, uint32_t *outCount) {
uint32_t cluster = start, prev = start, count = 0;
while (cluster >= 2 && cluster < FAT32_EOC) {
@@ -334,6 +328,7 @@ static uint32_t GetChainEnd(FAT32_VOL *v, uint32_t start, uint32_t *outCount) {
return prev;
}
// Releases cluster Chain.
static void FreeClusterChain(FAT32_VOL *v, uint32_t start) {
uint32_t cluster = start;
while (cluster >= 2 && cluster < FAT32_EOC) {
@@ -344,9 +339,7 @@ static void FreeClusterChain(FAT32_VOL *v, uint32_t start) {
}
}
// ---------------------------------------------------------------------------
// FAT timestamp and DIRENT metadata helpers
// ---------------------------------------------------------------------------
// FAT timestamp and DIRENT metadata helpers.
typedef struct FAT32_ENCODED_TIME {
uint16_t Date;
@@ -367,10 +360,12 @@ typedef struct FAT32_DIRENT_UPDATE {
#define FAT32_DIRENT_UPDATE_WRITE_TIME 0x00000008u
#define FAT32_DIRENT_UPDATE_ACCESS_DATE 0x00000010u
// Implements the fat Is Leap Year operation.
static int FatIsLeapYear(uint32_t year) {
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
}
// Implements the fat Days In Month operation.
static uint32_t FatDaysInMonth(uint32_t year, uint32_t month) {
static const uint8_t days[12] = { 31,28,31,30,31,30,31,31,30,31,30,31 };
if (month < 1u || month > 12u) return 31u;
@@ -389,11 +384,13 @@ static uint64_t FatCalendarToEpochSeconds(uint32_t year, uint32_t month, uint32_
return (((days * 24u) + hour) * 60u + minute) * 60u + second;
}
// Implements the fat Day Of Week operation.
static uint8_t FatDayOfWeek(uint32_t year, uint32_t month, uint32_t day) {
uint64_t epoch = FatCalendarToEpochSeconds(year, month, day, 0, 0, 0);
return (uint8_t)(((epoch / 86400u) + 4u) % 7u);
}
// Implements the fat Encode Date operation.
static uint16_t FatEncodeDate(const SYS_SYSTEM_TIME* t) {
uint32_t year = t && t->Year ? t->Year : 1980u;
uint32_t month = t && t->Month ? t->Month : 1u;
@@ -408,6 +405,7 @@ static uint16_t FatEncodeDate(const SYS_SYSTEM_TIME* t) {
return (uint16_t)(((year - 1980u) << 9) | (month << 5) | day);
}
// Implements the fat Encode Time operation.
static uint16_t FatEncodeTime(const SYS_SYSTEM_TIME* t) {
uint32_t hour = t ? t->Hour : 0u;
uint32_t minute = t ? t->Minute : 0u;
@@ -418,6 +416,7 @@ static uint16_t FatEncodeTime(const SYS_SYSTEM_TIME* t) {
return (uint16_t)((hour << 11) | (minute << 5) | (second / 2u));
}
// Implements the fat Encode Create Time Tenth operation.
static uint8_t FatEncodeCreateTimeTenth(const SYS_SYSTEM_TIME* t) {
uint32_t second = t ? t->Second : 0u;
uint32_t ms = t ? t->Millisecond : 0u;
@@ -428,6 +427,7 @@ static uint8_t FatEncodeCreateTimeTenth(const SYS_SYSTEM_TIME* t) {
return (uint8_t)v;
}
// Implements the fat Query Encoded Time operation.
static void FatQueryEncodedTime(FAT32_ENCODED_TIME* out) {
SYS_SYSTEM_TIME now;
memset(&now, 0, sizeof(now));
@@ -472,6 +472,7 @@ static void FatDecodeDateTime(uint16_t date, uint16_t time, uint8_t createTenth,
out->EpochSeconds = FatCalendarToEpochSeconds(year, month, day, hour, minute, second);
}
// Implements the fat Decode Access Date operation.
static void FatDecodeAccessDate(uint16_t date, SYS_SYSTEM_TIME* out) {
FatDecodeDateTime(date, 0, 0, out);
}
@@ -527,9 +528,7 @@ static KSTATUS DirentUpdateFirstCluster(FAT32_VOL *v, uint64_t lba,
return DirentUpdateMetadata(v, lba, offset, &u);
}
// ---------------------------------------------------------------------------
// 8.3 name matching / assembly
// ---------------------------------------------------------------------------
// 8.3 name matching / assembly.
static int MatchRaw83(const char dirName[8], const char dirExt[3],
const char wantName[8], const char wantExt[3]) {
@@ -541,6 +540,7 @@ static char FatUpperChar(char c);
static int FatIsLowerChar(char c);
static int FatIsValidShortChar(char c);
// Implements the make83 Name V operation.
static void Make83NameV(SYS_STRING_VIEW src, char name8[8], char ext3[3]) {
memset(name8, ' ', 8);
memset(ext3, ' ', 3);
@@ -571,19 +571,20 @@ static void Make83NameV(SYS_STRING_VIEW src, char name8[8], char ext3[3]) {
}
}
// ---------------------------------------------------------------------------
// LFN assembly / matching
// ---------------------------------------------------------------------------
// LFN assembly / matching.
#define MAX_LFN 256
// Implements the fat Upper Char operation.
static char FatUpperChar(char c) {
return (c >= 'a' && c <= 'z') ? (char)(c - 32) : c;
}
// Implements the fat Is Lower Char operation.
static int FatIsLowerChar(char c) {
return c >= 'a' && c <= 'z';
}
// Implements the fat Is Valid Short Char operation.
static int FatIsValidShortChar(char c) {
c = FatUpperChar(c);
if (c >= 'A' && c <= 'Z') return 1;
@@ -621,12 +622,14 @@ static int FatNameIsStructurallyShort83V(SYS_STRING_VIEW name) {
return baseLen >= 1 && baseLen <= 8 && extLen <= 3;
}
// Implements the fat Component Has Lower V operation.
static int FatComponentHasLowerV(SYS_STRING_VIEW name, size_t start, size_t end) {
for (size_t i = start; i < end; i++)
if (FatIsLowerChar(name.Data[i])) return 1;
return 0;
}
// Implements the fat Component Has Upper V operation.
static int FatComponentHasUpperV(SYS_STRING_VIEW name, size_t start, size_t end) {
for (size_t i = start; i < end; i++)
if (name.Data[i] >= 'A' && name.Data[i] <= 'Z') return 1;
@@ -667,6 +670,7 @@ static int FatNameNeedsLfnForCreateV(SYS_STRING_VIEW name) {
return 0;
}
// Implements the fat Short Name Case Flags V operation.
static uint8_t FatShortNameCaseFlagsV(SYS_STRING_VIEW name) {
if (!FatNameIsStructurallyShort83V(name)) return 0;
size_t baseEnd, extStart, extEnd;
@@ -682,6 +686,7 @@ static uint8_t FatShortNameCaseFlagsV(SYS_STRING_VIEW name) {
return flags;
}
// Implements the lfn Checksum operation.
static uint8_t LfnChecksum(const char name8[8], const char ext3[3]) {
uint8_t sum = 0;
for (int i = 0; i < 11; i++) {
@@ -691,6 +696,7 @@ static uint8_t LfnChecksum(const char name8[8], const char ext3[3]) {
return sum;
}
// Implements the lfn Set Char operation.
static void LfnSetChar(LFN_ENTRY *e, int idx, uint16_t value) {
if (idx < 5) e->Name1[idx] = value;
else if (idx < 11) e->Name2[idx - 5] = value;
@@ -719,6 +725,7 @@ static void BuildLfnEntry(LFN_ENTRY *e, const char *name, size_t nameLen,
}
}
// Creates legacy Broken Dot83.
static void MakeLegacyBrokenDot83(const char *src, char name8[8], char ext3[3]) {
// This exactly models the pre-fix bug for ".doomrc": empty name + "DOO".
memset(name8, ' ', 8);
@@ -758,12 +765,14 @@ static int ShortNameExistsRaw(FAT32_VOL *v, uint32_t dirCluster,
return 0;
}
// Implements the fat Alias Append Char operation.
static void FatAliasAppendChar(char *dst, int *len, int maxLen, char c) {
c = FatUpperChar(c);
if (!FatIsValidShortChar(c)) return;
if (*len < maxLen) dst[(*len)++] = c;
}
// Builds alias Stem Ext V.
static void BuildAliasStemExtV(SYS_STRING_VIEW src, char stem[9], char ext3[3]) {
memset(stem, 0, 9);
memset(ext3, ' ', 3);
@@ -838,6 +847,7 @@ static int MakeUniqueShortAliasV(FAT32_VOL *v, uint32_t dirCluster,
return 0;
}
// Creates legacy Broken Dot83 V.
static void MakeLegacyBrokenDot83V(SYS_STRING_VIEW src, char name8[8], char ext3[3]) {
// This exactly models the pre-fix bug for ".doomrc": empty name + "DOO".
memset(name8, ' ', 8);
@@ -894,6 +904,7 @@ static KSTATUS DeleteLegacyBrokenDotAliasV(FAT32_VOL *v, uint32_t dirCluster,
return STATUS_SUCCESS;
}
// Implements the lfn Append operation.
static void LfnAppend(char *lfn, const LFN_ENTRY *e) {
int seq = (e->Order & 0x3F) - 1;
if (seq < 0 || seq >= (MAX_LFN / 13)) return;
@@ -910,13 +921,12 @@ static void LfnAppend(char *lfn, const LFN_ENTRY *e) {
}
// ---------------------------------------------------------------------------
// Directory search and view-native path traversal
// ---------------------------------------------------------------------------
// Directory search and view-native path traversal.
static int Fat32IsPathSep(char c) {
return c == '\\' || c == '/';
}
// Implements the name View Is Valid operation.
static int Fat32NameViewIsValid(SYS_STRING_VIEW name) {
if (!SysSvIsValid(name)) return 0;
if (name.Length == 0 || name.Length >= MAX_LFN) return 0;
@@ -927,6 +937,7 @@ static int Fat32NameViewIsValid(SYS_STRING_VIEW name) {
return 1;
}
// Implements the lfn Match V operation.
static int LfnMatchV(const char *lfn, SYS_STRING_VIEW want) {
if (!lfn || !SysSvIsValid(want)) return 0;
size_t i = 0;
@@ -1104,9 +1115,7 @@ static KSTATUS SplitPathV(SYS_STRING_VIEW path, SYS_STRING_VIEW* parent,
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Find a free DIRENT slot
// ---------------------------------------------------------------------------
// Find a free DIRENT slot.
static int FindFreeDirent(FAT32_VOL *v, uint32_t dirCluster,
uint64_t *outLba, uint32_t *outOffset) {
uint8_t *clusterBuf = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
@@ -1197,9 +1206,7 @@ static int FindFreeDirentRun(FAT32_VOL *v, uint32_t dirCluster, uint32_t needed,
return 0;
}
// ---------------------------------------------------------------------------
// Open / Close
// ---------------------------------------------------------------------------
// Open / Close.
KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
@@ -1217,6 +1224,7 @@ KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
return STATUS_SUCCESS;
}
// Releases the requested object.
void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
@@ -1224,14 +1232,12 @@ void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
MmFreePool(fileCtx);
}
// ---------------------------------------------------------------------------
// Read (fix #4: contiguous-run coalescing)
// ---------------------------------------------------------------------------
// We peek ahead in the FAT chain: if cluster N+1 == cluster N physically,
// extend the current run instead of issuing a new ReadSectors call.
// Run length is capped at MAX_RUN_SECTORS to stay within the AHCI bounce
// buffer (128 sectors = 64 KB).
//
// FatEntry() calls in the peek loop are nearly free because they hit the
// FAT sector cache (fix #1).
@@ -1326,9 +1332,7 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Write (fix #4: contiguous-run coalescing)
// ---------------------------------------------------------------------------
// Same strategy as read. Whole-cluster-aligned runs skip the read-modify-
// write; partial head/tail clusters still require it.
@@ -1442,11 +1446,10 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
return (written == size) ? STATUS_SUCCESS : STATUS_DEVICE_ERROR;
}
// ---------------------------------------------------------------------------
// Create
// ---------------------------------------------------------------------------
// Create.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
// Creates the requested object.
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
@@ -1559,9 +1562,7 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Truncate
// ---------------------------------------------------------------------------
// Truncate.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
(void)fs;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
@@ -1625,9 +1626,7 @@ update_size:
return s; }
}
// ---------------------------------------------------------------------------
// Delete
// ---------------------------------------------------------------------------
// Delete.
KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (!fs || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
@@ -1650,13 +1649,12 @@ KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
return STATUS_SUCCESS;
}
// Returns size.
uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs; return ((FAT32_FILE *)fileCtx)->FileSize;
}
// ---------------------------------------------------------------------------
// Directory enumeration (fix #2: cluster buffer cached in context)
// ---------------------------------------------------------------------------
// Previously: allocate + ReadSectors + return 1 entry + free, every call.
// Now: buffer lives in FAT32_DIR_CTX; ReadSectors fires only on cluster
// boundaries.
@@ -1672,6 +1670,7 @@ typedef struct {
uint8_t *ClusterBuf; // allocated once in OpenDir, freed in CloseDir
} FAT32_DIR_CTX;
// Opens dir.
KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx) {
if (!fs || !dirCtx || !SysSvIsValid(dirPath))
return STATUS_INVALID_PARAMETER;
@@ -1703,6 +1702,7 @@ KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx)
return STATUS_SUCCESS;
}
// Returns dir.
KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
(void)fs;
FAT32_DIR_CTX *ctx = (FAT32_DIR_CTX *)dirCtx;
@@ -1764,6 +1764,7 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
return STATUS_NO_MORE_FILES;
}
// Releases dir.
void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx) {
(void)fs;
FAT32_DIR_CTX *ctx = (FAT32_DIR_CTX *)dirCtx;
@@ -1771,9 +1772,7 @@ void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx) {
MmFreePool(ctx);
}
// ---------------------------------------------------------------------------
// Volume scan (MBR + GPT + superfloppy)
// ---------------------------------------------------------------------------
// Volume scan (MBR + GPT + superfloppy).
static const EFI_GUID GUID_EFI_SYSTEM = { 0xC12A7328, 0xF81F, 0x11D2, {0xBA,0x4B,0x00,0xA0,0xC9,0x3E,0xC9,0x3B} };
static const EFI_GUID GUID_MS_BASIC_DATA = { 0xEBD0A0A2, 0xB9E5, 0x4433, {0x87,0xC0,0x68,0xB6,0xB7,0x26,0x99,0xC7} };
static const EFI_GUID GUID_LINUX_BASIC_DATA = { 0x0FC63DAF, 0x8483, 0x4772, {0x8E,0x79,0x3D,0x69,0xD8,0x47,0x7D,0xE4} };
@@ -1785,6 +1784,7 @@ typedef struct FAT32_PARTITION_CANDIDATE {
uint64_t SectorCount;
} FAT32_PARTITION_CANDIDATE;
// Returns le32.
static uint32_t ReadLe32(const void* p) {
const uint8_t* b = (const uint8_t*)p;
return (uint32_t)b[0]
@@ -1793,6 +1793,7 @@ static uint32_t ReadLe32(const void* p) {
| ((uint32_t)b[3] << 24);
}
// Checks whether fat32 Vbr.
static int IsFat32Vbr(FAT32_VOL *v, uint64_t lba) {
uint8_t sector[512];
if (!KSUCCESS(ReadSectors(v, lba, 1, sector))) return 0;
@@ -1946,9 +1947,7 @@ static uint32_t Fat32EnumerateDiskVolumes(uint8_t diskIndex,
return count;
}
// ---------------------------------------------------------------------------
// Native async read/write
// ---------------------------------------------------------------------------
// Metadata work (FAT traversal/allocation and partial-cluster read-modify
// preparation) still runs synchronously in the caller/completion context, but
// file data transfers are submitted as disk IRPs and completion is chained from
@@ -1990,6 +1989,7 @@ static volatile int gFat32AsyncInitState;
static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx);
static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx);
// Implements the async Worker operation.
static void Fat32AsyncWorker(void* arg) {
(void)arg;
@@ -2013,6 +2013,7 @@ static void Fat32AsyncWorker(void* arg) {
}
}
// Implements the ensure Async Worker Started operation.
static KSTATUS Fat32EnsureAsyncWorkerStarted(void) {
uintptr_t flags = KeAcquireSpinlock(&gFat32AsyncLock);
if (gFat32AsyncInitState == 2) {
@@ -2044,6 +2045,7 @@ static KSTATUS Fat32EnsureAsyncWorkerStarted(void) {
return s;
}
// Implements the async Queue Continuation operation.
static void Fat32AsyncQueueContinuation(FAT32_ASYNC_RW* ctx) {
if (!ctx) return;
@@ -2053,6 +2055,7 @@ static void Fat32AsyncQueueContinuation(FAT32_ASYNC_RW* ctx) {
KeSignalSemaphore(gFat32AsyncSemaphore);
}
// Implements the async Finish operation.
static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t information) {
if (!ctx) return;
if (ctx->CurrentDevice) {
@@ -2066,6 +2069,7 @@ static void Fat32AsyncFinish(FAT32_ASYNC_RW* ctx, KSTATUS status, size_t informa
if (complete) complete(completeContext, status, information);
}
// Implements the async Read Irp Complete operation.
static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
if (!ctx) return;
@@ -2089,6 +2093,7 @@ static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
Fat32AsyncQueueContinuation(ctx);
}
// Implements the async Read Submit Next operation.
static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
if (ctx->Remaining == 0) {
Fat32AsyncFinish(ctx, STATUS_SUCCESS, ctx->Done);
@@ -2206,6 +2211,7 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
return STATUS_PENDING;
}
// Implements the async Write Irp Complete operation.
static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) {
FAT32_ASYNC_RW* ctx = (FAT32_ASYNC_RW*)completeCtx;
if (!ctx) return;
@@ -2228,6 +2234,7 @@ static void Fat32AsyncWriteIrpComplete(IRP* irp, void* completeCtx) {
Fat32AsyncQueueContinuation(ctx);
}
// Implements the async Write Submit Next operation.
static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
if (ctx->Remaining == 0) {
uint64_t newEnd = ctx->Offset + ctx->Done;
@@ -2396,9 +2403,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
return STATUS_PENDING;
}
// ---------------------------------------------------------------------------
// DriverEntry
// ---------------------------------------------------------------------------
// DriverEntry.
#define FAT32_MAX_VOLUMES 8
static FAT32_VOL gVols[FAT32_MAX_VOLUMES];
@@ -2408,6 +2413,7 @@ static bool gVolumeMounted[FAT32_MAX_VOLUMES];
static uint32_t gMountedCount;
static bool gDiskScanned[FAT32_MAX_VOLUMES];
// Returns free Volume Index.
static int Fat32FindFreeVolumeIndex(void) {
for (uint8_t i = 0; i < FAT32_MAX_VOLUMES; i++) {
if (!gVolumeMounted[i] && !IoFilesystemPresent(i))
@@ -2416,6 +2422,7 @@ static int Fat32FindFreeVolumeIndex(void) {
return -1;
}
// Implements the try Mount Disk operation.
static uint32_t Fat32TryMountDisk(uint8_t disk) {
if (disk >= FAT32_MAX_VOLUMES)
return 0;
@@ -2513,6 +2520,7 @@ static uint32_t Fat32TryMountDisk(uint8_t disk) {
return mountedHere;
}
// Implements the disk Arrival operation.
static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* context) {
(void)device;
(void)context;
@@ -2526,6 +2534,7 @@ static void Fat32DiskArrival(uint8_t driveIndex, DEVICE_OBJECT* device, void* co
}
}
// Implements the unmount Disk operation.
static void Fat32UnmountDisk(uint8_t diskIndex, IO_DEVICE_CHANGE change) {
uint32_t removed = 0;
@@ -2580,6 +2589,7 @@ static void Fat32DiskChange(uint8_t driveIndex,
}
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API *hal) {
DbgPrint("FAT32: Init\n");
if (!hal) return STATUS_INVALID_PARAMETER;
code/drivers/fs/fat32/fat32.h
+2 -1
View File
@@ -1,4 +1,5 @@
// fat32.h - shared FAT32 driver internals.
// FAT32 driver internals shared by mount, directory, and file I/O code.
#pragma once
#include <stdint.h>
code/drivers/fs/fat32/fat32_mount.c
+3 -1
View File
@@ -1,6 +1,8 @@
// fat32_mount.c - filesystem vtable/mount wiring.
// FAT32 mount logic. This file owns volume probing, BPB validation, and mount-table insertion.
#include "fat32.h"
// Initializes vtable.
void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
memset(fs, 0, sizeof(*fs));
fs->Open = Fat32Open;
code/drivers/input/ps2/ps2.c
+11 -1
View File
@@ -1,4 +1,5 @@
// ps2.c - PS/2 keyboard/mouse event decoding and kernel input binding.
// PS/2 input driver package entry point. This file registers keyboard and mouse devices.
#include "ps2.h"
static KDPC gMouseDpc;
@@ -9,6 +10,7 @@ static PS2_MOUSE_PACKET gMouseDpcQueue[MOUSE_DPC_QUEUE_SIZE];
static volatile int gMouseDpcHead;
static volatile int gMouseDpcTail;
// Implements the ps2 Set1 To Key operation.
static INPUT_KEY Ps2Set1ToKey(uint8_t make, bool extended) {
if (extended) {
switch (make) {
@@ -119,6 +121,7 @@ static INPUT_KEY Ps2Set1ToKey(uint8_t make, bool extended) {
}
}
// Implements the ps2 Keyboard Isr operation.
static void Ps2KeyboardIsr(void) {
uint8_t sc = __inbyte(PS2_DATA);
if (sc == SC1_EXT_PREFIX) {
@@ -141,6 +144,7 @@ static void Ps2KeyboardIsr(void) {
KeInputSubmitKey(&event);
}
// Implements the ps2 Mouse Dpc Routine operation.
static void Ps2MouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
@@ -161,6 +165,7 @@ static void Ps2MouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1,
}
}
// Implements the ps2 Mouse Isr operation.
static void Ps2MouseIsr(void) {
uint8_t data = __inbyte(PS2_DATA);
@@ -190,6 +195,7 @@ static void Ps2MouseIsr(void) {
}
}
// Implements the ps2 Controller Prepare operation.
static void Ps2ControllerPrepare(void) {
Ps2WriteCommand(0xADu); // disable keyboard port
Ps2WriteCommand(0xA7u); // disable aux port
@@ -208,6 +214,7 @@ static void Ps2ControllerPrepare(void) {
Ps2FlushOutput();
}
// Implements the ps2 Controller Enable Interrupts operation.
static void Ps2ControllerEnableInterrupts(void) {
uint8_t commandByte = Ps2ReadCommandByte();
commandByte |= 0x01u;
@@ -215,6 +222,7 @@ static void Ps2ControllerEnableInterrupts(void) {
Ps2WriteCommandByte(commandByte);
}
// Implements the ps2 Keyboard Initialize operation.
static void Ps2KeyboardInitialize(void) {
// Enable scanning. If the keyboard is absent, the timeout/ACK failure is
// non-fatal; the IRQ path simply never produces events.
@@ -222,6 +230,7 @@ static void Ps2KeyboardInitialize(void) {
DbgPrint("ps2: keyboard enable did not ACK\n");
}
// Implements the ps2 Mouse Initialize operation.
static void Ps2MouseInitialize(void) {
if (!Ps2SendMouse(0xFFu)) {
DbgPrint("ps2: mouse reset did not ACK\n");
@@ -237,6 +246,7 @@ static void Ps2MouseInitialize(void) {
DbgPrint("ps2: mouse enable did not ACK\n");
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
(void)hal;
code/drivers/input/ps2/ps2.h
+2 -1
View File
@@ -1,4 +1,5 @@
// ps2.h - shared i8042/PS/2 driver internals.
// PS/2 driver internals shared by the package entry point and i8042 controller code.
#pragma once
#include <stdint.h>
code/drivers/input/ps2/ps2_i8042.c
+13 -1
View File
@@ -1,10 +1,13 @@
// ps2_i8042.c - 8042 controller and device command helpers.
// i8042 controller driver. This file owns PS/2 controller commands, IRQ handling, and packet decoding.
#include "ps2.h"
// Implements the ps2 Pause operation.
void Ps2Pause(void) {
_mm_pause();
}
// Implements the ps2 Wait Input operation.
void Ps2WaitInput(void) {
int timeout = 100000;
while ((__inbyte(PS2_STAT) & PS2_STAT_IN_FULL) && --timeout) {
@@ -12,6 +15,7 @@ void Ps2WaitInput(void) {
}
}
// Implements the ps2 Wait Output operation.
bool Ps2WaitOutput(void) {
int timeout = 5000000;
while (!(__inbyte(PS2_STAT) & PS2_STAT_OUT_FULL) && --timeout) {
@@ -20,6 +24,7 @@ bool Ps2WaitOutput(void) {
return timeout != 0;
}
// Implements the ps2 Flush Output operation.
void Ps2FlushOutput(void) {
int timeout = 100000;
while ((__inbyte(PS2_STAT) & PS2_STAT_OUT_FULL) && --timeout) {
@@ -27,36 +32,43 @@ void Ps2FlushOutput(void) {
}
}
// Implements the ps2 Read Data operation.
uint8_t Ps2ReadData(void) {
if (!Ps2WaitOutput()) return 0xFFu;
return __inbyte(PS2_DATA);
}
// Implements the ps2 Write Command operation.
void Ps2WriteCommand(uint8_t command) {
Ps2WaitInput();
__outbyte(PS2_CMD, command);
}
// Implements the ps2 Write Data operation.
void Ps2WriteData(uint8_t data) {
Ps2WaitInput();
__outbyte(PS2_DATA, data);
}
// Implements the ps2 Read Command Byte operation.
uint8_t Ps2ReadCommandByte(void) {
Ps2WriteCommand(0x20u);
return Ps2ReadData();
}
// Implements the ps2 Write Command Byte operation.
void Ps2WriteCommandByte(uint8_t commandByte) {
Ps2WriteCommand(0x60u);
Ps2WriteData(commandByte);
}
// Implements the ps2 Send Keyboard operation.
bool Ps2SendKeyboard(uint8_t byte) {
Ps2WriteData(byte);
return Ps2ReadData() == 0xFAu;
}
// Implements the ps2 Send Mouse operation.
bool Ps2SendMouse(uint8_t byte) {
Ps2WriteCommand(0xD4u);
Ps2WriteData(byte);
code/drivers/net/lwip/lwip_stack.c
+34 -1
View File
@@ -1,4 +1,4 @@
// lwip_stack.c - lwIP network stack provider driver.
// lwIP stack provider glue. This file bridges the kernel network-device model into lwIP.
#include <stdint.h>
#include <stddef.h>
@@ -130,6 +130,7 @@ static const NET_STACK_PROVIDER gLwipProvider = {
LwipProviderResolveHostIpv4,
};
// Implements the err To Status operation.
static KSTATUS LwipErrToStatus(err_t err) {
switch (err) {
case ERR_OK: return STATUS_SUCCESS;
@@ -150,10 +151,12 @@ static KSTATUS LwipErrToStatus(err_t err) {
}
}
// Implements the ascii Is Digit operation.
static int LwipAsciiIsDigit(char c) {
return c >= '0' && c <= '9';
}
// Parses ipv4 Part.
static int LwipParseIpv4Part(const char** cursor, uint8_t* out) {
const char* p = *cursor;
uint32_t value = 0;
@@ -175,6 +178,7 @@ static int LwipParseIpv4Part(const char** cursor, uint8_t* out) {
return 1;
}
// Parses ipv4.
static int LwipParseIpv4(const char* text, ip4_addr_t* out) {
if (!text || !out)
return 0;
@@ -205,6 +209,7 @@ static KSTATUS LwipGetSettingIpv4(const char* section, const char* key,
return STATUS_SUCCESS;
}
// Copies section Name.
static KSTATUS LwipCopySectionName(char* out, size_t outSize, const char* prefix, const char* name) {
if (!out || !outSize || !prefix || !name)
return STATUS_INVALID_PARAMETER;
@@ -219,6 +224,7 @@ static KSTATUS LwipCopySectionName(char* out, size_t outSize, const char* prefix
return STATUS_SUCCESS;
}
// Loads config From Section.
static KSTATUS LwipLoadConfigFromSection(const char* section, LWIP_NETIF_CONFIG* cfg) {
if (!section || !cfg)
return STATUS_INVALID_PARAMETER;
@@ -262,6 +268,7 @@ static KSTATUS LwipLoadConfigFromSection(const char* section, LWIP_NETIF_CONFIG*
return STATUS_INVALID_PARAMETER;
}
// Loads netif Config.
static KSTATUS LwipLoadNetifConfig(const NET_DEVICE* device, LWIP_NETIF_CONFIG* cfg) {
if (!device || !cfg)
return STATUS_INVALID_PARAMETER;
@@ -291,17 +298,20 @@ static KSTATUS LwipLoadNetifConfig(const NET_DEVICE* device, LWIP_NETIF_CONFIG*
return STATUS_SUCCESS;
}
// Implements the socket Lock operation.
static KSTATUS LwipSocketLock(LWIP_PROVIDER_SOCKET* s) {
if (!s || !s->Lock)
return STATUS_INVALID_PARAMETER;
return KeWaitSemaphore(s->Lock);
}
// Implements the socket Unlock operation.
static void LwipSocketUnlock(LWIP_PROVIDER_SOCKET* s) {
if (s && s->Lock)
KeSignalSemaphore(s->Lock);
}
// Implements the socket Callback operation.
static void LwipSocketCallback(struct netconn* conn, enum netconn_evt evt, u16_t len) {
(void)len;
if (!conn)
@@ -362,10 +372,12 @@ static KSTATUS LwipSocketAllocateWithConn(uint32_t domain, uint32_t type, uint32
return STATUS_SUCCESS;
}
// Implements the socket Make Ip operation.
static void LwipSocketMakeIp(const SYS_SOCKET_ADDRESS_IPV4_K* address, ip_addr_t* out) {
IP_ADDR4(out, address->Address[0], address->Address[1], address->Address[2], address->Address[3]);
}
// Implements the socket Fill Address operation.
static void LwipSocketFillAddress(SYS_SOCKET_ADDRESS_IPV4_K* out, const ip_addr_t* ip, u16_t port) {
if (!out)
return;
@@ -378,6 +390,7 @@ static void LwipSocketFillAddress(SYS_SOCKET_ADDRESS_IPV4_K* out, const ip_addr_
out->Address[3] = (uint8_t)ip4_addr4(ip_2_ip4(ip));
}
// Implements the low Level Output operation.
static err_t LwipLowLevelOutput(struct netif* netif, struct pbuf* p) {
LWIP_BINDING* b = (LWIP_BINDING*)netif->state;
if (!b || !b->Device || !p || p->tot_len < NET_ETHERNET_HEADER_LENGTH)
@@ -405,6 +418,7 @@ static err_t LwipLowLevelOutput(struct netif* netif, struct pbuf* p) {
return KSUCCESS(st) ? ERR_OK : ERR_IF;
}
// Implements the netif Init operation.
static err_t LwipNetifInit(struct netif* netif) {
LWIP_BINDING* b = (LWIP_BINDING*)netif->state;
if (!b || !b->Device)
@@ -422,6 +436,7 @@ static err_t LwipNetifInit(struct netif* netif) {
return ERR_OK;
}
// Attaches callback.
static void LwipAttachCallback(void* arg) {
LWIP_ATTACH_CTX* ctx = (LWIP_ATTACH_CTX*)arg;
LWIP_BINDING* b = ctx->Binding;
@@ -490,11 +505,13 @@ static void LwipAttachCallback(void* arg) {
KeSignalSemaphore(ctx->Done);
}
// Initializes done.
static void LwipInitDone(void* arg) {
SEMAPHORE* sem = (SEMAPHORE*)arg;
KeSignalSemaphore(sem);
}
// Implements the stack Initialize operation.
static KSTATUS LwipStackInitialize(const HAL_API* hal) {
if (gLwipInitialized)
return STATUS_SUCCESS;
@@ -519,6 +536,7 @@ static KSTATUS LwipStackInitialize(const HAL_API* hal) {
return STATUS_SUCCESS;
}
// Implements the provider Attach Device operation.
static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device) {
if (!gLwipInitialized)
return STATUS_NOT_SUPPORTED;
@@ -576,6 +594,7 @@ static KSTATUS LwipProviderAttachDevice(NET_DEVICE* device) {
return STATUS_SUCCESS;
}
// Implements the provider Receive Frame operation.
static KSTATUS LwipProviderReceiveFrame(NET_DEVICE* device, const void* frame, size_t length) {
if (!gLwipInitialized)
return STATUS_NOT_SUPPORTED;
@@ -609,6 +628,7 @@ static KSTATUS LwipProviderReceiveFrame(NET_DEVICE* device, const void* frame, s
return STATUS_SUCCESS;
}
// Implements the provider Socket Create operation.
static KSTATUS LwipProviderSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, void** outSocket) {
if (!outSocket)
return STATUS_INVALID_PARAMETER;
@@ -655,6 +675,7 @@ static KSTATUS LwipProviderSocketCreate(uint32_t domain, uint32_t type, uint32_t
return STATUS_SUCCESS;
}
// Implements the provider Socket Close operation.
static void LwipProviderSocketClose(void* socket) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s)
@@ -681,6 +702,7 @@ static void LwipProviderSocketClose(void* socket) {
MmFreePool(s);
}
// Implements the provider Bind operation.
static KSTATUS LwipProviderBind(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !address)
@@ -711,6 +733,7 @@ static KSTATUS LwipProviderBind(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* a
return LwipErrToStatus(err);
}
// Implements the provider Listen operation.
static KSTATUS LwipProviderListen(void* socket, uint32_t backlog) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s)
@@ -739,6 +762,7 @@ static KSTATUS LwipProviderListen(void* socket, uint32_t backlog) {
return LwipErrToStatus(err);
}
// Implements the provider Accept operation.
static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !acceptedOut)
@@ -795,6 +819,7 @@ static KSTATUS LwipProviderAccept(void* socket, void** acceptedOut, SYS_SOCKET_A
return STATUS_SUCCESS;
}
// Implements the provider Connect operation.
static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !address)
@@ -830,6 +855,7 @@ static KSTATUS LwipProviderConnect(void* socket, const SYS_SOCKET_ADDRESS_IPV4_K
return LwipErrToStatus(err);
}
// Implements the provider Send operation.
static KSTATUS LwipProviderSend(void* socket, const void* buffer, size_t length, uint32_t flags, size_t* sentOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (sentOut)
@@ -913,6 +939,7 @@ static KSTATUS LwipReceiveFromBuffered(LWIP_PROVIDER_SOCKET* s, uint8_t* out,
return STATUS_SUCCESS;
}
// Implements the provider Receive operation.
static KSTATUS LwipProviderReceive(void* socket, void* buffer, size_t length, uint32_t flags, size_t* receivedOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (receivedOut)
@@ -1103,6 +1130,7 @@ static KSTATUS LwipProviderReceiveFrom(void* socket, void* buffer, size_t length
return LwipErrToStatus(err);
}
// Implements the provider Set Option operation.
static KSTATUS LwipProviderSetOption(void* socket, uint32_t option, const void* value, size_t valueSize) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !value)
@@ -1141,6 +1169,7 @@ static KSTATUS LwipProviderSetOption(void* socket, uint32_t option, const void*
return STATUS_SUCCESS;
}
// Implements the provider Get Option operation.
static KSTATUS LwipProviderGetOption(void* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (actualSizeOut)
@@ -1177,6 +1206,7 @@ static KSTATUS LwipProviderGetOption(void* socket, uint32_t option, void* value,
return STATUS_SUCCESS;
}
// Implements the provider Poll operation.
static KSTATUS LwipProviderPoll(void* socket, uint32_t events, uint32_t* returnedEventsOut) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s || !returnedEventsOut)
@@ -1215,6 +1245,7 @@ static KSTATUS LwipProviderPoll(void* socket, uint32_t events, uint32_t* returne
return STATUS_SUCCESS;
}
// Implements the provider Shutdown operation.
static KSTATUS LwipProviderShutdown(void* socket, uint32_t how) {
LWIP_PROVIDER_SOCKET* s = (LWIP_PROVIDER_SOCKET*)socket;
if (!s)
@@ -1240,6 +1271,7 @@ static KSTATUS LwipProviderShutdown(void* socket, uint32_t how) {
return LwipErrToStatus(err);
}
// Implements the provider Resolve Host Ipv4 operation.
static KSTATUS LwipProviderResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut) {
if (!name || !addressOut)
return STATUS_INVALID_PARAMETER;
@@ -1263,6 +1295,7 @@ static KSTATUS LwipProviderResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
DbgPrint("lwIP: driver loading...\n");
code/drivers/net/lwip/lwip_sys_arch.c
+37 -27
View File
@@ -1,4 +1,5 @@
// lwip_sys_arch.c - lwIP OS abstraction layer for the kernel.
// lwIP OS-abstraction layer. This file maps lwIP synchronization and timing primitives onto kernel services.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
@@ -42,18 +43,20 @@ struct sys_mbox {
static uint64_t (*gLwipQueryTicksUs)(void) = 0;
static uint32_t gRandState = 0x4b4e4554u; /* 'KNET' */
static uint32_t gRandState = 0x4b4e4554u; // 'KNET'
// Implements the port Set Timer Query operation.
void LwipPortSetTimerQuery(uint64_t (*query_ticks_us)(void)) {
gLwipQueryTicksUs = query_ticks_us;
}
unsigned int lwip_port_rand(void) {
/* Tiny deterministic LCG. Good enough for ephemeral port randomisation. */
// Tiny deterministic LCG. Good enough for ephemeral port randomisation.
gRandState = gRandState * 1664525u + 1013904223u;
return gRandState;
}
// Implements the sys init operation.
void sys_init(void) {
}
@@ -67,22 +70,20 @@ u32_t sys_jiffies(void) {
return sys_now();
}
/*
* lwIP's core/sys.c provides sys_msleep() when it is not a macro. Do not
* define it here as well, or MSVC links both sys.obj and lwip_sys_arch.obj
* with a duplicate sys_msleep symbol.
*/
// lwIP's core/sys.c provides sys_msleep() when it is not a macro. Do not
// define it here as well, or MSVC links both sys.obj and lwip_sys_arch.obj
// with a duplicate sys_msleep symbol.
/*
* First bring-up errno storage for lwIP's socket/netdb compatibility layer.
* This is global, not per-thread; the kernel socket-object wrapper should
* eventually translate lwIP errors to KSTATUS instead of exposing this.
*/
// fuckass microsoft
// First bring-up errno storage for lwIP's socket/netdb compatibility layer.
// This is global, not per-thread; the kernel socket-object wrapper should
// eventually translate lwIP errors to KSTATUS instead of exposing this.
// fuckass microsoft.
#ifdef errno
#undef errno
#endif
int errno;
// Implements the errno operation.
int* __cdecl _errno(void) { return &errno; }
err_t sys_sem_new(sys_sem_t* sem, u8_t count) {
@@ -105,6 +106,7 @@ err_t sys_sem_new(sys_sem_t* sem, u8_t count) {
return ERR_OK;
}
// Implements the sys sem signal operation.
void sys_sem_signal(sys_sem_t* sem) {
if (sem && *sem && (*sem)->Sem)
KeSignalSemaphore((*sem)->Sem);
@@ -132,6 +134,7 @@ u32_t sys_arch_sem_wait(sys_sem_t* sem, u32_t timeout) {
}
}
// Implements the sys sem free operation.
void sys_sem_free(sys_sem_t* sem) {
if (!sem || !*sem)
return;
@@ -161,16 +164,19 @@ err_t sys_mutex_new(sys_mutex_t* mutex) {
return ERR_OK;
}
// Implements the sys mutex lock operation.
void sys_mutex_lock(sys_mutex_t* mutex) {
if (mutex && *mutex && (*mutex)->Sem)
(void)KeWaitSemaphore((*mutex)->Sem);
}
// Implements the sys mutex unlock operation.
void sys_mutex_unlock(sys_mutex_t* mutex) {
if (mutex && *mutex && (*mutex)->Sem)
KeSignalSemaphore((*mutex)->Sem);
}
// Implements the sys mutex free operation.
void sys_mutex_free(sys_mutex_t* mutex) {
if (!mutex || !*mutex)
return;
@@ -180,6 +186,7 @@ void sys_mutex_free(sys_mutex_t* mutex) {
*mutex = 0;
}
// Implements the mbox Try Post operation.
static err_t LwipMboxTryPost(sys_mbox_t* mbox, void* msg) {
if (!mbox || !*mbox || !(*mbox)->Valid)
return ERR_ARG;
@@ -232,8 +239,9 @@ err_t sys_mbox_new(sys_mbox_t* mbox, int size) {
return ERR_OK;
}
// Implements the sys mbox post operation.
void sys_mbox_post(sys_mbox_t* mbox, void* msg) {
/* lwIP promises this is not called from ISR context. */
// lwIP promises this is not called from ISR context.
while (LwipMboxTryPost(mbox, msg) != ERR_OK)
KeYield();
}
@@ -246,6 +254,7 @@ err_t sys_mbox_trypost_fromisr(sys_mbox_t* mbox, void* msg) {
return LwipMboxTryPost(mbox, msg);
}
// Implements the mbox Pop After Item operation.
static u32_t LwipMboxPopAfterItem(sys_mbox_t* mbox, void** msg, u32_t waited) {
struct sys_mbox* m = *mbox;
uintptr_t flags = KeAcquireSpinlock(&m->Lock);
@@ -299,6 +308,7 @@ u32_t sys_arch_mbox_tryfetch(sys_mbox_t* mbox, void** msg) {
return LwipMboxPopAfterItem(mbox, msg, 0);
}
// Implements the sys mbox free operation.
void sys_mbox_free(sys_mbox_t* mbox) {
if (!mbox || !*mbox)
return;
@@ -327,18 +337,17 @@ sys_thread_t sys_thread_new(const char* name, lwip_thread_fn thread, void* arg,
else if (prio <= 0)
kp = PriorityLow;
/*
* lwIP thread entries already use the same ABI as kernel worker threads:
*
* void entry(void* arg)
*
* Do not bounce through a heap-allocated thunk. The first scaffold used a
* tiny { function, argument } heap block and a wrapper, but an early boot
* crash showed the wrapper sometimes loaded a bogus function pointer and
* jumped into lwIP's .rdata strings. Passing the lwIP entry directly keeps
* the initial thread stack simple: KeKernelThreadEntry calls exactly the
* function pointer that lwIP supplied, with exactly the argument it supplied.
*/
// lwIP thread entries already use the same ABI as kernel worker threads:
// void entry(void* arg)
// Do not bounce through a heap-allocated thunk. The first scaffold used a
// tiny { function, argument } heap block and a wrapper, but an early boot
// crash showed the wrapper sometimes loaded a bogus function pointer and
// jumped into lwIP's .rdata strings. Passing the lwIP entry directly keeps
// the initial thread stack simple: KeKernelThreadEntry calls exactly the
// function pointer that lwIP supplied, with exactly the argument it supplied.
THREAD* t = 0;
KSTATUS st = KeCreateKernelThread((uintptr_t)thread,
(uintptr_t)arg,
@@ -356,6 +365,7 @@ sys_prot_t sys_arch_protect(void) {
return (sys_prot_t)flags;
}
// Implements the sys arch unprotect operation.
void sys_arch_unprotect(sys_prot_t pval) {
if (((uintptr_t)pval) & 0x200u)
_enable();
code/drivers/net/rtl8139/rtl8139.c
+26 -1
View File
@@ -1,4 +1,5 @@
// rtl8139.c - packet I/O, interrupt, and DriverEntry path.
// Realtek RTL8139 network driver. This file owns PCI attachment, MMIO setup, RX/TX rings, and net-device registration.
#include "rtl8139.h"
const HAL_API* gHal;
@@ -16,40 +17,47 @@ static const IO_DRIVER_ID_TABLE gRtl8139IdTable = {
sizeof(gRtl8139Ids) / sizeof(gRtl8139Ids[0]),
};
// Implements the memory Barrier operation.
static __forceinline void Rtl8139MemoryBarrier(void) {
_ReadWriteBarrier();
_mm_mfence();
_ReadWriteBarrier();
}
// Implements the read8 operation.
static uint8_t Rtl8139Read8(RTL8139_DEVICE* r, uint32_t reg) {
if (r->UseMmio)
return *(volatile uint8_t*)(r->Mmio + reg);
return __inbyte((unsigned short)(r->IoBase + reg));
}
// Implements the read16 operation.
static uint16_t Rtl8139Read16(RTL8139_DEVICE* r, uint32_t reg) {
if (r->UseMmio)
return *(volatile uint16_t*)(r->Mmio + reg);
return __inword((unsigned short)(r->IoBase + reg));
}
// Implements the read32 operation.
static uint32_t Rtl8139Read32(RTL8139_DEVICE* r, uint32_t reg) {
if (r->UseMmio)
return *(volatile uint32_t*)(r->Mmio + reg);
return __indword((unsigned short)(r->IoBase + reg));
}
// Implements the write8 operation.
static void Rtl8139Write8(RTL8139_DEVICE* r, uint32_t reg, uint8_t value) {
if (r->UseMmio) *(volatile uint8_t*)(r->Mmio + reg) = value;
else __outbyte((unsigned short)(r->IoBase + reg), value);
}
// Implements the write16 operation.
static void Rtl8139Write16(RTL8139_DEVICE* r, uint32_t reg, uint16_t value) {
if (r->UseMmio) *(volatile uint16_t*)(r->Mmio + reg) = value;
else __outword((unsigned short)(r->IoBase + reg), value);
}
// Implements the write32 operation.
static void Rtl8139Write32(RTL8139_DEVICE* r, uint32_t reg, uint32_t value) {
if (r->UseMmio) *(volatile uint32_t*)(r->Mmio + reg) = value;
else __outdword((unsigned short)(r->IoBase + reg), value);
@@ -59,6 +67,7 @@ static void Rtl8139Write32(RTL8139_DEVICE* r, uint32_t reg, uint32_t value) {
// Implements the reset operation.
static KSTATUS Rtl8139Reset(RTL8139_DEVICE* r) {
Rtl8139Write8(r, RTL8139_REG_CMD, RTL8139_CMD_RST);
for (uint32_t i = 0; i < 1000; i++) {
@@ -70,12 +79,14 @@ static KSTATUS Rtl8139Reset(RTL8139_DEVICE* r) {
return STATUS_TIMEOUT;
}
// Returns mac.
static void Rtl8139ReadMac(RTL8139_DEVICE* r) {
for (uint32_t i = 0; i < 6; i++)
r->NetDevice.HardwareAddress[i] = Rtl8139Read8(r, RTL8139_REG_IDR0 + i);
r->NetDevice.HardwareAddressLength = 6;
}
// Creates dma.
static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
r->RxPhys = MmAllocateContiguous(RTL8139_RX_PAGES);
r->TxPhys = MmAllocateContiguous(RTL8139_TX_PAGES);
@@ -101,6 +112,7 @@ static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
return STATUS_SUCCESS;
}
// Implements the rx Read16 operation.
static uint16_t Rtl8139RxRead16(RTL8139_DEVICE* r, uint32_t offset) {
offset %= RTL8139_RX_RING_SIZE;
uint16_t lo = r->RxBuffer[offset];
@@ -108,6 +120,7 @@ static uint16_t Rtl8139RxRead16(RTL8139_DEVICE* r, uint32_t offset) {
return (uint16_t)(lo | (hi << 8));
}
// Implements the rx Copy operation.
static void Rtl8139RxCopy(RTL8139_DEVICE* r, uint32_t offset, void* dst, uint32_t length) {
uint8_t* out = (uint8_t*)dst;
offset %= RTL8139_RX_RING_SIZE;
@@ -120,6 +133,7 @@ static void Rtl8139RxCopy(RTL8139_DEVICE* r, uint32_t offset, void* dst, uint32_
memcpy(out + first, r->RxBuffer, length - first);
}
// Implements the program Hardware operation.
static void Rtl8139ProgramHardware(RTL8139_DEVICE* r) {
// Wake the device and clear stale interrupts before installing buffers.
Rtl8139Write8(r, RTL8139_REG_CONFIG1, 0x00);
@@ -148,6 +162,7 @@ static void Rtl8139ProgramHardware(RTL8139_DEVICE* r) {
Rtl8139Write16(r, RTL8139_REG_IMR, RTL8139_INTR_ENABLES);
}
// Handles rx.
static void Rtl8139HandleRx(RTL8139_DEVICE* r) {
for (uint32_t budget = 0; budget < 64; budget++) {
uint8_t cmd = Rtl8139Read8(r, RTL8139_REG_CMD);
@@ -192,6 +207,7 @@ static void Rtl8139HandleRx(RTL8139_DEVICE* r) {
}
}
// Implements the dpc Routine operation.
static void Rtl8139DpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)systemArgument1;
@@ -212,6 +228,7 @@ static void Rtl8139DpcRoutine(KDPC* dpc, void* context, void* systemArgument1, v
DbgPrint("RTL8139: interrupt error bits=0x%04x\n", status);
}
// Registers device.
static KSTATUS Rtl8139RegisterDevice(RTL8139_DEVICE* r) {
if (!r) return STATUS_INVALID_PARAMETER;
@@ -233,6 +250,7 @@ static KSTATUS Rtl8139RegisterDevice(RTL8139_DEVICE* r) {
return STATUS_SUCCESS;
}
// Implements the unregister Device operation.
static void Rtl8139UnregisterDevice(RTL8139_DEVICE* r) {
if (!r) return;
@@ -248,6 +266,7 @@ static void Rtl8139UnregisterDevice(RTL8139_DEVICE* r) {
KeReleaseSpinlock(&gRtl8139RegistryLock, flags);
}
// Implements the isr operation.
static void Rtl8139Isr(void) {
RTL8139_DEVICE* snapshot[RTL8139_MAX_DEVICES];
uint32_t count = 0;
@@ -276,6 +295,7 @@ static void Rtl8139Isr(void) {
}
}
// Implements the send Frame operation.
static KSTATUS Rtl8139SendFrame(NET_DEVICE* device, const void* frame, size_t length) {
if (!device || !frame)
return STATUS_INVALID_PARAMETER;
@@ -312,6 +332,7 @@ static KSTATUS Rtl8139SendFrame(NET_DEVICE* device, const void* frame, size_t le
return STATUS_SUCCESS;
}
// Implements the select Register Bar operation.
static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
if (!r || !r->Pdo)
return STATUS_INVALID_PARAMETER;
@@ -377,6 +398,7 @@ static KSTATUS Rtl8139SelectRegisterBar(RTL8139_DEVICE* r) {
return STATUS_NOT_FOUND;
}
// Starts device.
static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) {
if (!r || r->Magic != RTL8139_DEVICE_MAGIC || !r->Pdo || !r->Fdo)
return STATUS_INVALID_PARAMETER;
@@ -462,6 +484,7 @@ static KSTATUS Rtl8139StartDevice(RTL8139_DEVICE* r) {
return STATUS_SUCCESS;
}
// Dispatches pnp.
KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return STATUS_INVALID_PARAMETER;
@@ -504,6 +527,7 @@ KSTATUS Rtl8139DispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the add Device operation.
KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -532,6 +556,7 @@ KSTATUS Rtl8139AddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal)
return STATUS_INVALID_PARAMETER;
code/drivers/net/rtl8139/rtl8139.h
+2 -1
View File
@@ -1,4 +1,5 @@
// rtl8139.h - shared rtl8139 driver internals.
// Realtek RTL8139 driver internals shared by the PCI and packet paths.
#pragma once
#include <stdint.h>
code/drivers/net/rtl8139/rtl8139_pci.c
+2 -2
View File
@@ -1,2 +1,2 @@
// rtl8139_pci.c intentionally empty.
// PCI enumeration, BAR decoding, mapping, and command-register policy now live in pci.sys.
// Driver support for rtl8139 pci.
code/drivers/net/rtl816x/rtl816x.c
+25 -1
View File
@@ -1,4 +1,5 @@
// rtl816x.c - rtl8168/rtl8169 packet I/O, interrupt, and PCI function-driver path.
// Realtek RTL816x network driver. This file owns PCI attachment, descriptor rings, and net-device registration.
#include "rtl816x.h"
const HAL_API* gHal;
@@ -17,45 +18,53 @@ static const IO_DRIVER_ID_TABLE gRtl816xIdTable = {
sizeof(gRtl816xIds) / sizeof(gRtl816xIds[0]),
};
// Implements the rtl Memory Barrier operation.
static __forceinline void RtlMemoryBarrier(void) {
_ReadWriteBarrier();
_mm_mfence();
_ReadWriteBarrier();
}
// Implements the rtl Read8 operation.
static uint8_t RtlRead8(RTL816X_DEVICE* r, uint32_t reg) {
if (r->UseMmio)
return *(volatile uint8_t*)(r->Mmio + reg);
return __inbyte((unsigned short)(r->IoBase + reg));
}
// Implements the rtl Read16 operation.
static uint16_t RtlRead16(RTL816X_DEVICE* r, uint32_t reg) {
if (r->UseMmio)
return *(volatile uint16_t*)(r->Mmio + reg);
return __inword((unsigned short)(r->IoBase + reg));
}
// Implements the rtl Read32 operation.
static uint32_t RtlRead32(RTL816X_DEVICE* r, uint32_t reg) {
if (r->UseMmio)
return *(volatile uint32_t*)(r->Mmio + reg);
return __indword((unsigned short)(r->IoBase + reg));
}
// Implements the rtl Write8 operation.
static void RtlWrite8(RTL816X_DEVICE* r, uint32_t reg, uint8_t value) {
if (r->UseMmio) *(volatile uint8_t*)(r->Mmio + reg) = value;
else __outbyte((unsigned short)(r->IoBase + reg), value);
}
// Implements the rtl Write16 operation.
static void RtlWrite16(RTL816X_DEVICE* r, uint32_t reg, uint16_t value) {
if (r->UseMmio) *(volatile uint16_t*)(r->Mmio + reg) = value;
else __outword((unsigned short)(r->IoBase + reg), value);
}
// Implements the rtl Write32 operation.
static void RtlWrite32(RTL816X_DEVICE* r, uint32_t reg, uint32_t value) {
if (r->UseMmio) *(volatile uint32_t*)(r->Mmio + reg) = value;
else __outdword((unsigned short)(r->IoBase + reg), value);
}
// Implements the rtl Reset operation.
static KSTATUS RtlReset(RTL816X_DEVICE* r) {
RtlWrite8(r, RTL_REG_CHIP_CMD, RTL_CMD_RESET);
for (uint32_t i = 0; i < 1000; i++) {
@@ -67,12 +76,14 @@ static KSTATUS RtlReset(RTL816X_DEVICE* r) {
return STATUS_TIMEOUT;
}
// Implements the rtl Read Mac operation.
static void RtlReadMac(RTL816X_DEVICE* r) {
for (uint32_t i = 0; i < 6; i++)
r->NetDevice.HardwareAddress[i] = RtlRead8(r, RTL_REG_MAC0 + i);
r->NetDevice.HardwareAddressLength = 6;
}
// Implements the rtl Allocate Dma operation.
static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
r->RxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
r->TxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
@@ -100,6 +111,7 @@ static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
return STATUS_SUCCESS;
}
// Implements the rtl Init Descriptors operation.
static void RtlInitDescriptors(RTL816X_DEVICE* r) {
for (uint32_t i = 0; i < RTL_RX_COUNT; i++) {
PHYSICAL_ADDRESS p = r->RxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE;
@@ -122,6 +134,7 @@ static void RtlInitDescriptors(RTL816X_DEVICE* r) {
RtlMemoryBarrier();
}
// Implements the rtl Program Hardware operation.
static void RtlProgramHardware(RTL816X_DEVICE* r) {
RtlWrite8(r, RTL_REG_CFG9346, 0xC0);
@@ -151,6 +164,7 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) {
RtlWrite16(r, RTL_REG_INTR_MASK, RTL_INTR_ENABLES);
}
// Implements the rtl Handle Rx operation.
static void RtlHandleRx(RTL816X_DEVICE* r) {
for (uint32_t budget = 0; budget < RTL_RX_COUNT; budget++) {
RTL_DESC* d = &r->RxRing[r->RxIndex];
@@ -188,6 +202,7 @@ static void RtlHandleRx(RTL816X_DEVICE* r) {
}
}
// Implements the rtl Dpc Routine operation.
static void RtlDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)systemArgument1;
@@ -212,6 +227,7 @@ static void RtlDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void*
DbgPrint("RTL816X: interrupt status error bits=0x%04x\n", status);
}
// Implements the rtl Register Device operation.
static KSTATUS RtlRegisterDevice(RTL816X_DEVICE* r) {
if (!r) return STATUS_INVALID_PARAMETER;
@@ -233,6 +249,7 @@ static KSTATUS RtlRegisterDevice(RTL816X_DEVICE* r) {
return STATUS_SUCCESS;
}
// Implements the rtl Unregister Device operation.
static void RtlUnregisterDevice(RTL816X_DEVICE* r) {
if (!r) return;
@@ -248,6 +265,7 @@ static void RtlUnregisterDevice(RTL816X_DEVICE* r) {
KeReleaseSpinlock(&gRtl816xRegistryLock, flags);
}
// Implements the rtl Isr operation.
static void RtlIsr(void) {
RTL816X_DEVICE* snapshot[RTL816X_MAX_DEVICES];
uint32_t count = 0;
@@ -276,6 +294,7 @@ static void RtlIsr(void) {
}
}
// Implements the rtl Send Frame operation.
static KSTATUS RtlSendFrame(NET_DEVICE* device, const void* frame, size_t length) {
if (!device || !frame)
return STATUS_INVALID_PARAMETER;
@@ -315,6 +334,7 @@ static KSTATUS RtlSendFrame(NET_DEVICE* device, const void* frame, size_t length
return STATUS_SUCCESS;
}
// Implements the rtl Select Register Bar operation.
static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
if (!r || !r->Pdo)
return STATUS_INVALID_PARAMETER;
@@ -380,6 +400,7 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
return STATUS_NOT_FOUND;
}
// Implements the rtl Start Device operation.
static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
if (!r || r->Magic != RTL816X_DEVICE_MAGIC || !r->Pdo || !r->Fdo)
return STATUS_INVALID_PARAMETER;
@@ -465,6 +486,7 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
return STATUS_SUCCESS;
}
// Dispatches pnp.
KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return STATUS_INVALID_PARAMETER;
@@ -507,6 +529,7 @@ KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Implements the add Device operation.
KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -535,6 +558,7 @@ KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal)
return STATUS_INVALID_PARAMETER;
code/drivers/net/rtl816x/rtl816x.h
+2 -1
View File
@@ -1,4 +1,5 @@
// rtl816x.h - shared rtl8168/rtl8169 driver internals.
// Realtek RTL816x driver internals shared by the PCI and packet paths.
#pragma once
#include <stdint.h>
code/drivers/storage/ahci/ahci.c
+51 -61
View File
@@ -1,4 +1,5 @@
// ahci.c - AHCI disk command path and DriverEntry.
// AHCI storage driver. This file owns controller attachment, drive discovery, command submission, and worker-driven IRP completion.
#include "ahci.h"
const HAL_API *g_hal;
@@ -17,21 +18,21 @@ static const IO_DRIVER_ID_TABLE g_ahciIdTable = {
sizeof(g_ahciIds) / sizeof(g_ahciIds[0]),
};
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
// Helpers.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl);
// Accessors for port structure memory at known offsets from structVirt/Phys
// Accessors for port structure memory at known offsets from structVirt/Phys.
static inline HBA_CMD_HEADER *cmd_list(AHCI_DRIVE *drv) {
return (HBA_CMD_HEADER *)((uint8_t *)drv->structVirt + PORT_OFF_CMDLIST);
}
// Implements the cmd list phys operation.
static inline uint64_t cmd_list_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_CMDLIST;
}
// Implements the fis phys operation.
static inline uint64_t fis_phys(AHCI_DRIVE *drv) {
return drv->structPhys + PORT_OFF_FIS;
}
@@ -42,20 +43,17 @@ static inline HBA_CMD_TBL *cmd_tbl(AHCI_DRIVE *drv, int slot) {
+ (uint32_t)slot * PORT_CMDTBL_SIZE);
}
// Implements the cmd tbl phys operation.
static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
return drv->structPhys + PORT_OFF_CMDTBLS
+ (uint64_t)slot * PORT_CMDTBL_SIZE;
}
// ---------------------------------------------------------------------------
// Port engine start / stop (polling safe)
// ---------------------------------------------------------------------------
// Port engine start / stop (polling safe).
// ---------------------------------------------------------------------------
// Port memory initialisation
// ---------------------------------------------------------------------------
// Port memory initialisation.
static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
@@ -63,7 +61,7 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT);
if (!drv->structPhys) return STATUS_NO_MEMORY;
drv->structVirt = g_hal->PhysToVirt(drv->structPhys);
// Force NOCACHE on DMA descriptor memory
// Force NOCACHE on DMA descriptor memory.
MmSetPageProtect((uintptr_t)drv->structVirt,
PORT_PAGES_STRUCT * 4096, flags);
memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096);
@@ -72,7 +70,7 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
if (!drv->bouncePhys) return STATUS_NO_MEMORY;
drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys);
// Bounce buffer: NOCACHE to ensure CPU writes reach physical memory
// before the HBA's DMA engine reads them
// before the HBA's DMA engine reads them.
MmSetPageProtect((uintptr_t)drv->bounceVirt,
BOUNCE_PAGES * 4096, flags);
memset(drv->bounceVirt, 0, BOUNCE_BYTES);
@@ -81,15 +79,15 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
port_stop(port);
// Set command list base
// Set command list base.
port->clb = (uint32_t)(cmd_list_phys(drv) & 0xFFFFFFFF);
port->clbu = (uint32_t)(cmd_list_phys(drv) >> 32);
// Set FIS base
// Set FIS base.
port->fb = (uint32_t)(fis_phys(drv) & 0xFFFFFFFF);
port->fbu = (uint32_t)(fis_phys(drv) >> 32);
// Initialise all 32 command headers, pointing each to its command table
// Initialise all 32 command headers, pointing each to its command table.
HBA_CMD_HEADER *hdrs = cmd_list(drv);
for (int i = 0; i < NUM_CMD_SLOTS; i++) {
hdrs[i].prdtl = PORT_PRDT_COUNT;
@@ -98,7 +96,7 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
hdrs[i].ctbau = (uint32_t)(tbl >> 32);
}
// Clear any stale interrupts then start the port engine
// Clear any stale interrupts then start the port engine.
port->serr = (uint32_t)-1;
port->is = (uint32_t)-1;
@@ -107,19 +105,17 @@ static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Issue a command - polling fallback (used during early init / IDENTIFY only,
// before the scheduler and interrupt handler are live).
// Returns STATUS_SUCCESS, STATUS_DEVICE_ERROR, or STATUS_TIMEOUT.
// ---------------------------------------------------------------------------
static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
// Clear pending interrupts before issuing
// Clear pending interrupts before issuing.
port->is = (uint32_t)-1;
// Wait for port to be idle (not BSY or DRQ)
// Wait for port to be idle (not BSY or DRQ).
int spin = 0;
while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT)
spin++;
@@ -128,10 +124,10 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
return STATUS_TIMEOUT;
}
// Issue slot 0
// Issue slot 0.
port->ci = (1u << 0);
// Poll until slot 0 clears
// Poll until slot 0 clears.
while (true) {
if (!(port->ci & (1u << 0))) break;
if (port->is & (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)) {
@@ -152,10 +148,8 @@ static KSTATUS port_issue_cmd_poll(AHCI_DRIVE *drv) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Interrupt bottom half. The ISR owns hardware acknowledgement and status
// capture; this DPC wakes the worker thread after the hard IRQ has unwound.
// ---------------------------------------------------------------------------
static void AhciCompletionDpc(KDPC *dpc, void *context,
void *systemArgument1, void *systemArgument2) {
@@ -168,11 +162,9 @@ static void AhciCompletionDpc(KDPC *dpc, void *context,
KeSignalEvent(drv->completionEvent);
}
// ---------------------------------------------------------------------------
// ISR - called from KeIrqDispatch on the AHCI PCI IRQ line.
// Walks all registered drives, checks if their port raised an interrupt,
// captures IS, clears it, and queues the per-drive completion DPC.
// ---------------------------------------------------------------------------
static void AhciIsr(void) {
for (int c = 0; c < g_controllerCount; c++) {
@@ -206,11 +198,9 @@ static void AhciIsr(void) {
}
}
// ---------------------------------------------------------------------------
// Issue a command - interrupt-driven (runtime path).
// Caller has already built slot 0 via build_rw_cmd / port_identify.
// Blocks the current thread until the ISR signals completionEvent.
// ---------------------------------------------------------------------------
static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
@@ -220,7 +210,7 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
port->is = (uint32_t)-1;
drv->lastIs = 0;
// Wait for port idle
// Wait for port idle.
int spin = 0;
while ((port->tfd & (ATA_SR_BUSY | ATA_SR_DRQ)) && spin < POLL_TIMEOUT)
spin++;
@@ -245,9 +235,7 @@ static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Build a standard DMA read or write command in slot 0
// ---------------------------------------------------------------------------
// Build a standard DMA read or write command in slot 0.
static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
uint64_t bufPhys, bool write) {
@@ -256,7 +244,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
memset(tbl, 0, PORT_CMDTBL_SIZE);
// How many PRDT entries we need (each covers up to 8K = 16 sectors)
// How many PRDT entries we need (each covers up to 8K = 16 sectors).
uint32_t prdtl = (sectors + 15) / 16;
if (prdtl > PORT_PRDT_COUNT) prdtl = PORT_PRDT_COUNT; // capped (see caller)
@@ -265,7 +253,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
hdr->prdtl = (uint16_t)prdtl;
hdr->prdbc = 0;
// Fill PRDT entries
// Fill PRDT entries.
uint32_t remaining = sectors;
uint64_t buf = bufPhys;
for (uint32_t i = 0; i < prdtl; i++) {
@@ -281,7 +269,7 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
remaining -= sects;
}
// Build H2D FIS
// Build H2D FIS.
FIS_REG_H2D *fis = (FIS_REG_H2D *)tbl->cfis;
memset(fis, 0, sizeof(*fis));
fis->fis_type = FIS_TYPE_REG_H2D;
@@ -300,12 +288,10 @@ static void build_rw_cmd(AHCI_DRIVE *drv, uint64_t lba, uint32_t sectors,
fis->counth = (uint8_t)(sectors >> 8);
}
// ---------------------------------------------------------------------------
// IDENTIFY command - fills a 512-byte buffer and returns it
// ---------------------------------------------------------------------------
// IDENTIFY command - fills a 512-byte buffer and returns it.
static KSTATUS port_identify(AHCI_DRIVE *drv) {
// Reuse bounce buffer for IDENTIFY data (512 bytes)
// Reuse bounce buffer for IDENTIFY data (512 bytes).
memset(drv->bounceVirt, 0, 512);
HBA_CMD_HEADER *hdr = cmd_list(drv);
@@ -334,17 +320,17 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
KSTATUS st = port_issue_cmd_poll(drv);
if (!KSUCCESS(st)) return st;
// Parse sector count from IDENTIFY data
// Parse sector count from IDENTIFY data.
const uint16_t *id = (const uint16_t *)drv->bounceVirt;
if (id[ATA_IDENT_W83] & (1u << 10)) {
// 48-bit LBA supported
// 48-bit LBA supported.
drv->sectorCount = (uint64_t)id[ATA_IDENT_W100]
| ((uint64_t)id[ATA_IDENT_W100 + 1] << 16)
| ((uint64_t)id[ATA_IDENT_W100 + 2] << 32)
| ((uint64_t)id[ATA_IDENT_W100 + 3] << 48);
} else {
// 28-bit LBA
// 28-bit LBA.
drv->sectorCount = (uint64_t)id[ATA_IDENT_W60]
| ((uint64_t)id[ATA_IDENT_W60 + 1] << 16);
}
@@ -352,7 +338,7 @@ static KSTATUS port_identify(AHCI_DRIVE *drv) {
return STATUS_SUCCESS;
}
// Core read logic
// Core read logic.
static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba,
uint32_t count, void *buf) {
uint8_t *out = (uint8_t *)buf;
@@ -372,7 +358,7 @@ static KSTATUS do_read(AHCI_DRIVE *drv, uint64_t lba,
return STATUS_SUCCESS;
}
// Core write logic
// Core write logic.
static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
uint32_t count, const void *buf) {
const uint8_t *in = (const uint8_t *)buf;
@@ -392,9 +378,7 @@ static KSTATUS do_write(AHCI_DRIVE *drv, uint64_t lba,
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Async IRP dispatch
// ---------------------------------------------------------------------------
// Async IRP dispatch.
static KSTATUS AhciGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
@@ -449,12 +433,14 @@ static KSTATUS AhciDispatchReadWrite(DEVICE_OBJECT *device, IRP *irp) {
return STATUS_PENDING;
}
// Cancels release Remove Lock.
static void AhciCancelReleaseRemoveLock(IRP* irp, void* context) {
AHCI_DRIVE* drv = (AHCI_DRIVE*)context;
if (drv)
IoReleaseRemoveLock(&drv->removeLock, irp);
}
// Dispatches drive Pnp.
static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
AHCI_DRIVE* drv = (AHCI_DRIVE*)device->DeviceExtension;
@@ -492,6 +478,7 @@ static KSTATUS AhciDispatchDrivePnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Dispatches controller Pnp.
static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
AHCI_CONTROLLER* ctl = (AHCI_CONTROLLER*)device->DeviceExtension;
@@ -545,6 +532,7 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
}
}
// Dispatches pnp.
static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DeviceExtension)
return STATUS_INVALID_PARAMETER;
@@ -562,14 +550,14 @@ static KSTATUS AhciDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
// single-slot interrupt-driven path. Completion fires the IRP's event
// and/or callback so the original caller can observe the result without
// ever having blocked in a driver call.
//
// NCQ TODO: replace the serialised loop with a multi-slot dispatcher:
// NCQ remains serialized until the driver grows a multi-slot dispatcher:
// 1. KeWaitSemaphore() to wait for a free slot AND a pending IRP.
// 2. Pick the oldest IRP from the queue.
// 3. Grab a free bit in drv->slotMask (under the slot spinlock).
// 4. Copy write data into slotBounce[slot], build the NCQ FIS (FPDMA),
// record slotIrp[slot] = irp, and issue via port->sact + port->ci.
// 5. Return immediately - the ISR drives all completions.
// 5. Return immediately so the ISR can drive all completions.
static void AhciWorker(uintptr_t arg) {
AHCI_DRIVE *drv = (AHCI_DRIVE *)arg;
@@ -599,9 +587,7 @@ static void AhciWorker(uintptr_t arg) {
}
}
// ---------------------------------------------------------------------------
// AHCI controller initialisation
// ---------------------------------------------------------------------------
// AHCI controller initialisation.
@@ -622,11 +608,12 @@ static void AhciWorker(uintptr_t arg) {
// TFES (bit 30) - Task File Error Status - error
// HBFS (bit 29) - Host Bus Fatal error
// HBDS (bit 28) - Host Bus Data error
// IFS (bit 27) - Interface Fatal error
// IFS (bit 27) - Interface Fatal error.
#define PORT_IE_NORMAL (0x0000000Fu)
#define PORT_IE_ERRORS (PORT_IS_TFES | PORT_IS_HBFS | PORT_IS_HBDS | PORT_IS_IFS)
#define PORT_IE_ALL (PORT_IE_NORMAL | PORT_IE_ERRORS)
// Registers controller.
static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
if (!ctl) return STATUS_INVALID_PARAMETER;
for (int i = 0; i < g_controllerCount; i++) {
@@ -639,6 +626,7 @@ static KSTATUS AhciRegisterController(AHCI_CONTROLLER* ctl) {
return STATUS_SUCCESS;
}
// Starts controller.
static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
if (!ctl || ctl->magic != AHCI_CONTROLLER_MAGIC || !ctl->pdo || !ctl->fdo)
return STATUS_INVALID_PARAMETER;
@@ -723,7 +711,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
HBA_PORT *port = &ctl->hba->ports[portno];
// Phase 1: physical link check
// Phase 1: physical link check.
if (!port_link_up(port)) {
DbgPrint("AHCI: port %u: no link (ssts=0x%08x)\n", portno, port->ssts);
continue;
@@ -739,28 +727,28 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
drv->port = port;
drv->portNo = (uint8_t)portno;
// Phase 2: allocate command list, FIS buffer, bounce buffer
// Phase 2: allocate command list, FIS buffer, bounce buffer.
st = port_init_memory(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: memory init failed (0x%x)\n", portno, st);
continue;
}
// Phase 3: COMRESET - forces device to re-send D2H FIS for valid SIG
// Phase 3: COMRESET - forces device to re-send D2H FIS for valid SIG.
st = port_comreset(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: COMRESET failed\n", portno);
continue;
}
// Phase 4: filter to SATA disks
// Phase 4: filter to SATA disks.
if (!port_is_sata_disk(port)) {
DbgPrint("AHCI: port %u: not a SATA disk (sig=0x%08x), skipping\n",
portno, port->sig);
continue;
}
// Phase 5: create per-drive completion event (auto-reset, initially clear)
// Phase 5: create per-drive completion event (auto-reset, initially clear).
st = KeEventCreate(EventAutoReset, 0, &drv->completionEvent);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: event create failed (0x%x)\n", portno, st);
@@ -783,7 +771,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
continue;
}
// Phase 6: IDENTIFY (polled - scheduler not yet running)
// Phase 6: IDENTIFY (polled - scheduler not yet running).
st = port_identify(drv);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: port %u: IDENTIFY failed (0x%x)\n", portno, st);
@@ -852,6 +840,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
return STATUS_SUCCESS;
}
// Implements the add Device operation.
static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -879,6 +868,7 @@ static KSTATUS AhciAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDevic
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API *hal) {
DbgPrint("AHCI: driver loading...\n");
g_hal = hal;
code/drivers/storage/ahci/ahci.h
+41 -47
View File
@@ -1,4 +1,5 @@
// ahci.h - shared AHCI driver internals.
// AHCI driver state and register definitions shared across controller and port code.
#pragma once
#include <stdint.h>
@@ -16,7 +17,6 @@
#include <kernel/ke_irp.h>
#include <kernel/ke_dpc.h>
// ---------------------------------------------------------------------------
typedef enum {
FIS_TYPE_REG_H2D = 0x27,
FIS_TYPE_REG_D2H = 0x34,
@@ -29,7 +29,7 @@ typedef enum {
} FIS_TYPE;
#pragma pack(push, 1)
// Register FIS - Host to Device (H2D)
// Register FIS - Host to Device (H2D).
typedef struct {
uint8_t fis_type; // FIS_TYPE_REG_H2D
uint8_t pmport : 4;
@@ -37,21 +37,21 @@ typedef struct {
uint8_t c : 1; // 1 = Command, 0 = Control
uint8_t command;
uint8_t featurel;
// DW1
// DW1.
uint8_t lba0, lba1, lba2;
uint8_t device;
// DW2
// DW2.
uint8_t lba3, lba4, lba5;
uint8_t featureh;
// DW3
// DW3.
uint8_t countl, counth;
uint8_t icc;
uint8_t control;
// DW4
// DW4.
uint8_t rsv1[4];
} FIS_REG_H2D;
// Register FIS - Device to Host (D2H)
// Register FIS - Device to Host (D2H).
typedef struct {
uint8_t fis_type;
uint8_t pmport : 4;
@@ -69,7 +69,7 @@ typedef struct {
uint8_t rsv4[4];
} FIS_REG_D2H;
// DMA Setup FIS
// DMA Setup FIS.
typedef struct {
uint8_t fis_type;
uint8_t pmport : 4;
@@ -85,7 +85,7 @@ typedef struct {
uint32_t resvd;
} FIS_DMA_SETUP;
// PIO Setup FIS
// PIO Setup FIS.
typedef struct {
uint8_t fis_type;
uint8_t pmport : 4;
@@ -106,7 +106,7 @@ typedef struct {
uint8_t rsv4[2];
} FIS_PIO_SETUP;
// Set Device Bits FIS
// Set Device Bits FIS.
typedef struct {
uint8_t fis_type;
uint8_t pmport : 4;
@@ -121,11 +121,9 @@ typedef struct {
uint32_t protocol;
} FIS_DEV_BITS;
// ---------------------------------------------------------------------------
// HBA structures (volatile - MMIO)
// ---------------------------------------------------------------------------
// HBA structures (volatile - MMIO).
// Received FIS structure in host memory
// Received FIS structure in host memory.
typedef volatile struct {
FIS_DMA_SETUP dsfis; // 0x00 DMA Setup FIS
uint8_t pad0[4];
@@ -138,9 +136,9 @@ typedef volatile struct {
uint8_t rsv[0x100-0xA0];
} HBA_FIS;
// Command List header (one entry = one slot)
// Command List header (one entry = one slot).
typedef struct {
// DW0
// DW0.
uint8_t cfl : 5; // Command FIS length in DWORDs (2..16)
uint8_t a : 1; // ATAPI
uint8_t w : 1; // Write (H2D=1, D2H=0)
@@ -151,16 +149,16 @@ typedef struct {
uint8_t rsv0 : 1;
uint8_t pmp : 4; // Port multiplier port
uint16_t prdtl; // PRDT length in entries
// DW1
// DW1.
volatile uint32_t prdbc; // PRDT byte count transferred
// DW2,3
// DW2,3.
uint32_t ctba; // Command table base address (128-byte aligned)
uint32_t ctbau; // Upper 32 bits
// DW4-7
// DW4-7.
uint32_t rsv1[4];
} HBA_CMD_HEADER;
// Physical Region Descriptor Table entry
// Physical Region Descriptor Table entry.
typedef struct {
uint32_t dba; // Data base address
uint32_t dbau; // Upper 32 bits
@@ -170,7 +168,7 @@ typedef struct {
uint32_t i : 1; // Interrupt on completion
} HBA_PRDT_ENTRY;
// Command Table
// Command Table.
typedef struct {
uint8_t cfis[64]; // Command FIS (up to 64 bytes)
uint8_t acmd[16]; // ATAPI command (12 or 16 bytes)
@@ -179,7 +177,7 @@ typedef struct {
} HBA_CMD_TBL;
#pragma pack(pop)
// Port registers
// Port registers.
typedef volatile struct {
uint32_t clb; // 0x00 Command list base (1K-aligned)
uint32_t clbu; // 0x04 Command list base upper
@@ -202,7 +200,7 @@ typedef volatile struct {
uint32_t vendor[4];
} HBA_PORT;
// Generic Host Control registers (starts at ABAR)
// Generic Host Control registers (starts at ABAR).
typedef volatile struct {
uint32_t cap; // 0x00 Host capability
uint32_t ghc; // 0x04 Global host control
@@ -220,9 +218,7 @@ typedef volatile struct {
HBA_PORT ports[32]; // 0x100 Port control registers
} HBA_MEM;
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
// Constants.
// PCI/AHCI binding. PCI enumeration, BAR decoding, interrupt line
// discovery, and command-register policy are owned by pci.sys.
@@ -231,20 +227,20 @@ typedef volatile struct {
#define AHCI_DRIVE_MAGIC 0x41484452u // 'AHDR'
#define MAX_AHCI_CONTROLLERS 8
// GHC register bits
// GHC register bits.
#define GHC_AE (1u << 31) // AHCI Enable
#define GHC_IE (1u << 1) // Interrupt Enable
#define GHC_HR (1u << 0) // HBA Reset
// BOHC register bits
// BOHC register bits.
#define BOHC_BOS (1u << 0) // BIOS-owned semaphore
#define BOHC_OOS (1u << 1) // OS-owned semaphore
#define BOHC_BB (1u << 4) // BIOS Busy
// CAP2 bits
// CAP2 bits.
#define CAP2_BOH (1u << 0) // BIOS/OS handoff capable
// Port CMD register bits
// Port CMD register bits.
#define PORT_CMD_ST (1u << 0) // Start
#define PORT_CMD_FRE (1u << 4) // FIS Receive Enable
#define PORT_CMD_FR (1u << 14) // FIS Receive Running
@@ -253,34 +249,34 @@ typedef volatile struct {
#define PORT_CMD_SUD (1u << 1) // Spin-Up Device
#define PORT_CMD_ICC_ACTIVE (0x1u << 28)
// Port IS register bits (interrupt status)
// Port IS register bits (interrupt status).
#define PORT_IS_TFES (1u << 30) // Task File Error Status
#define PORT_IS_HBFS (1u << 29) // Host Bus Fatal Error
#define PORT_IS_HBDS (1u << 28) // Host Bus Data Error
#define PORT_IS_IFS (1u << 27) // Interface Fatal Error
// Port SSTS register
// Port SSTS register.
#define PORT_DET_PRESENT 0x3
#define PORT_IPM_ACTIVE 0x1
// Port signatures
// Port signatures.
#define SATA_SIG_ATA 0x00000101
#define SATA_SIG_ATAPI 0xEB140101
#define SATA_SIG_SEMB 0xC33C0101
#define SATA_SIG_PM 0x96690101
// ATA commands
// ATA commands.
#define ATA_CMD_READ_DMA_EX 0x25
#define ATA_CMD_WRITE_DMA_EX 0x35
#define ATA_CMD_IDENTIFY 0xEC
#define ATA_CMD_FLUSH_EXT 0xEA
// ATA status bits
// ATA status bits.
#define ATA_SR_BUSY 0x80
#define ATA_SR_DRQ 0x08
#define ATA_SR_ERR 0x01
// IDENTIFY word offsets
// IDENTIFY word offsets.
#define ATA_IDENT_W83 83 // bit 10 = 48-bit LBA supported
#define ATA_IDENT_W100 100 // 48-bit LBA sector count (4 words)
#define ATA_IDENT_W60 60 // 28-bit LBA sector count (2 words)
@@ -289,7 +285,7 @@ typedef volatile struct {
// Offset 0: Command List 1024 bytes (32 × 32-byte headers)
// Offset 1024: Received FIS 256 bytes
// Offset 2048: Command Tables 32 × 256 bytes = 8192 bytes
// Total: 10240 bytes → 3 pages (12288 bytes)
// Total: 10240 bytes → 3 pages (12288 bytes).
#define PORT_PAGES_STRUCT 3
#define PORT_OFF_CMDLIST 0
#define PORT_OFF_FIS 1024
@@ -297,12 +293,12 @@ typedef volatile struct {
#define PORT_CMDTBL_SIZE 256 // 64 CFIS + 16 ACMD + 48 rsv + 8×16 PRDT
#define PORT_PRDT_COUNT 8 // PRDT entries per command table
// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages
// Bounce buffer: 128 sectors × 512 bytes = 64 KB = 16 pages.
#define BOUNCE_SECTORS 128
#define BOUNCE_BYTES (BOUNCE_SECTORS * 512)
#define BOUNCE_PAGES (BOUNCE_BYTES / 4096)
// Polling timeout (spin iterations)
// Polling timeout (spin iterations).
#define POLL_TIMEOUT 1000000
#define MAX_AHCI_DRIVES 8
@@ -316,7 +312,7 @@ typedef volatile struct {
typedef struct AHCI_CONTROLLER AHCI_CONTROLLER;
// Per-drive context
// Per-drive context.
typedef struct {
uint32_t magic; // AHCI_DRIVE_MAGIC
AHCI_CONTROLLER* controller; // owning AHCI controller FDO extension
@@ -324,11 +320,11 @@ typedef struct {
uint64_t sectorCount; // Total LBA sectors on device
uint8_t portNo; // Physical port index (for ISR dispatch)
// Structures block (command list + FIS + cmd tables)
// Structures block (command list + FIS + cmd tables).
uint64_t structPhys; // Physical base
void* structVirt; // Virtual base
// Bounce buffer for DMA data
// Bounce buffer for DMA data.
uint64_t bouncePhys;
void* bounceVirt;
@@ -342,9 +338,7 @@ typedef struct {
// Bottom-half used to wake the worker outside hard IRQ/DIRQL context.
KDPC completionDpc;
// -----------------------------------------------------------------------
// IRP queue - async dispatch
// -----------------------------------------------------------------------
// AhciWorker blocks here waiting for work. AHCI dispatch enqueues an IRP
// and signals the semaphore to wake the worker.
IRP_QUEUE irpQueue;
@@ -354,12 +348,12 @@ typedef struct {
THREAD* workerThread;
DEVICE_OBJECT* deviceObject;
// NCQ TODO: for multi-slot NCQ, add:
// Multi-slot NCQ needs per-slot ownership state in addition to the serialized queue:
// uint32_t slotMask; // bitmask of in-flight slots
// IRP* slotIrp[NUM_CMD_SLOTS]; // slot → pending IRP
// uint64_t slotBouncePhys[NUM_CMD_SLOTS];
// void* slotBounceVirt[NUM_CMD_SLOTS];
// volatile uint32_t prevSact; // previous port->sact snapshot
// volatile uint32_t prevSact; // previous port->sact snapshot.
} AHCI_DRIVE;
struct AHCI_CONTROLLER {
code/drivers/storage/ahci/ahci_port.c
+24 -15
View File
@@ -1,6 +1,8 @@
// ahci_port.c - AHCI MMIO mapping, HBA reset, and port link bring-up helpers.
// AHCI port helpers. This file owns port reset, command-table setup, and per-port command execution.
#include "ahci.h"
// Implements the ahci map mmio operation.
KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
if (length == 0)
length = AHCI_MMIO_PAGES * 4096ull;
@@ -27,24 +29,26 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
}
// Implements the port stop operation.
void port_stop(HBA_PORT *port) {
// Clear ST (bit 0)
// Clear ST (bit 0).
port->cmd &= ~PORT_CMD_ST;
// Wait for CR (bit 15) to clear (command list DMA stops)
// Wait for CR (bit 15) to clear (command list DMA stops).
for (int i = 0; i < POLL_TIMEOUT; i++) {
if (!(port->cmd & PORT_CMD_CR)) break;
}
// Clear FRE (bit 4)
// Clear FRE (bit 4).
port->cmd &= ~PORT_CMD_FRE;
// Wait for FR (bit 14) to clear
// Wait for FR (bit 14) to clear.
for (int i = 0; i < POLL_TIMEOUT; i++) {
if (!(port->cmd & PORT_CMD_FR)) break;
}
}
// Implements the port start operation.
void port_start(HBA_PORT *port) {
// Wait until CR is clear before setting ST
// Wait until CR is clear before setting ST.
for (int i = 0; i < POLL_TIMEOUT; i++) {
if (!(port->cmd & PORT_CMD_CR)) break;
}
@@ -53,24 +57,25 @@ void port_start(HBA_PORT *port) {
}
// Implements the hba reset operation.
KSTATUS hba_reset(HBA_MEM *hba) {
// BIOS/OS handoff (spec §10.6.3)
// BIOS/OS handoff (spec §10.6.3).
if (hba->cap2 & CAP2_BOH) {
hba->bohc |= BOHC_OOS;
// Wait for BIOS to clear BOS
// Wait for BIOS to clear BOS.
for (int i = 0; i < POLL_TIMEOUT; i++) {
if (!(hba->bohc & BOHC_BOS)) break;
}
// Give the BIOS another moment to clear Busy
// Give the BIOS another moment to clear Busy.
for (int i = 0; i < 800000; i++) {
if (!(hba->bohc & BOHC_BB)) break;
}
}
// Enable AHCI mode before reset so the reset actually takes effect
// Enable AHCI mode before reset so the reset actually takes effect.
hba->ghc |= GHC_AE;
// HBA reset
// HBA reset.
hba->ghc |= GHC_HR;
for (int i = 0; i < POLL_TIMEOUT; i++) {
if (!(hba->ghc & GHC_HR)) break;
@@ -87,6 +92,7 @@ KSTATUS hba_reset(HBA_MEM *hba) {
}
// Implements the port link up operation.
bool port_link_up(HBA_PORT *port) {
uint32_t det = port->ssts & 0x0F;
uint32_t ipm = (port->ssts >> 8) & 0x0F;
@@ -94,17 +100,20 @@ bool port_link_up(HBA_PORT *port) {
}
// Implements the port is sata disk operation.
bool port_is_sata_disk(HBA_PORT *port) {
return port->sig == SATA_SIG_ATA;
}
// Implements the ahci delay ms operation.
void ahci_delay_ms(uint32_t ms) {
for (uint32_t i = 0; i < ms * 1000; i++)
__outbyte(0x80, 0);
}
// Implements the port comreset operation.
KSTATUS port_comreset(AHCI_DRIVE *drv) {
HBA_PORT *port = drv->port;
@@ -112,14 +121,14 @@ KSTATUS port_comreset(AHCI_DRIVE *drv) {
for (int i = 0; i < POLL_TIMEOUT; i++)
if (!(port->cmd & PORT_CMD_CR)) break;
// Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms
// Assert COMRESET: SControl.DET = 1, hold for ≥ 1ms.
port->sctl = (port->sctl & ~0x0Fu) | 0x1u;
ahci_delay_ms(1);
// Deassert: DET = 0
// Deassert: DET = 0.
port->sctl = port->sctl & ~0x0Fu;
// Poll for DET = 3, up to 1 second
// Poll for DET = 3, up to 1 second.
bool link_ok = false;
for (int i = 0; i < 1000; i++) {
if ((port->ssts & 0x0Fu) == PORT_DET_PRESENT) { link_ok = true; break; }
@@ -134,7 +143,7 @@ KSTATUS port_comreset(AHCI_DRIVE *drv) {
port->is = (uint32_t)-1;
port->cmd |= PORT_CMD_ST;
// Wait up to 500ms for a valid signature (D2H FIS from device)
// Wait up to 500ms for a valid signature (D2H FIS from device).
for (int i = 0; i < 500; i++) {
uint32_t sig = port->sig;
if (sig != 0xFFFFFFFFu && sig != 0x00000000u) break;
code/drivers/storage/ata/ata.c
+31 -1
View File
@@ -1,4 +1,5 @@
// ata.c - PCI IDE / ATA disk driver.
// ATA compatibility storage driver. This file owns legacy IDE probing and PIO disk access for older virtual hardware.
#include "ata.h"
const HAL_API* gAtaHal;
@@ -15,30 +16,37 @@ static const IO_DRIVER_ID_TABLE gAtaIdTable = {
sizeof(gAtaIds) / sizeof(gAtaIds[0]),
};
// Implements the in8 operation.
static uint8_t AtaIn8(ATA_CHANNEL* ch, uint8_t reg) {
return __inbyte((unsigned short)(ch->CommandBase + reg));
}
// Implements the in16 operation.
static uint16_t AtaIn16(ATA_CHANNEL* ch, uint8_t reg) {
return __inword((unsigned short)(ch->CommandBase + reg));
}
// Implements the out8 operation.
static void AtaOut8(ATA_CHANNEL* ch, uint8_t reg, uint8_t value) {
__outbyte((unsigned short)(ch->CommandBase + reg), value);
}
// Implements the out16 operation.
static void AtaOut16(ATA_CHANNEL* ch, uint8_t reg, uint16_t value) {
__outword((unsigned short)(ch->CommandBase + reg), value);
}
// Implements the alt Status operation.
static uint8_t AtaAltStatus(ATA_CHANNEL* ch) {
return __inbyte((unsigned short)(ch->ControlBase + ATA_REG_ALT_STATUS));
}
// Implements the device Control operation.
static void AtaDeviceControl(ATA_CHANNEL* ch, uint8_t value) {
__outbyte((unsigned short)(ch->ControlBase + ATA_REG_DEVICE_CONTROL), value);
}
// Implements the delay400ns operation.
static void AtaDelay400ns(ATA_CHANNEL* ch) {
(void)AtaAltStatus(ch);
(void)AtaAltStatus(ch);
@@ -46,6 +54,7 @@ static void AtaDelay400ns(ATA_CHANNEL* ch) {
(void)AtaAltStatus(ch);
}
// Waits for not Busy.
static KSTATUS AtaWaitNotBusy(ATA_CHANNEL* ch) {
for (uint32_t i = 0; i < ATA_POLL_LIMIT; i++) {
if (!(AtaIn8(ch, ATA_REG_STATUS) & ATA_STATUS_BSY))
@@ -55,6 +64,7 @@ static KSTATUS AtaWaitNotBusy(ATA_CHANNEL* ch) {
return STATUS_TIMEOUT;
}
// Waits for drq.
static KSTATUS AtaWaitDrq(ATA_CHANNEL* ch) {
for (uint32_t i = 0; i < ATA_POLL_LIMIT; i++) {
uint8_t st = AtaIn8(ch, ATA_REG_STATUS);
@@ -69,12 +79,14 @@ static KSTATUS AtaWaitDrq(ATA_CHANNEL* ch) {
return STATUS_TIMEOUT;
}
// Implements the select Drive operation.
static void AtaSelectDrive(ATA_DEVICE* dev) {
ATA_CHANNEL* ch = dev->Channel;
AtaOut8(ch, ATA_REG_DRIVE_HEAD, (uint8_t)(0xA0u | (dev->Drive ? 0x10u : 0u)));
AtaDelay400ns(ch);
}
// Implements the identify operation.
static KSTATUS AtaIdentify(ATA_DEVICE* dev) {
ATA_CHANNEL* ch = dev->Channel;
uint16_t ident[256];
@@ -139,6 +151,7 @@ static KSTATUS AtaIdentify(ATA_DEVICE* dev) {
return dev->SectorCount ? STATUS_SUCCESS : STATUS_NOT_FOUND;
}
// Implements the program Lba operation.
static KSTATUS AtaProgramLba(ATA_DEVICE* dev, uint64_t lba, uint16_t count, bool write) {
ATA_CHANNEL* ch = dev->Channel;
@@ -168,6 +181,7 @@ static KSTATUS AtaProgramLba(ATA_DEVICE* dev, uint64_t lba, uint16_t count, bool
return STATUS_SUCCESS;
}
// Returns pio.
static KSTATUS AtaReadPio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, void* buffer) {
if (!dev || !buffer)
return STATUS_INVALID_PARAMETER;
@@ -210,6 +224,7 @@ static KSTATUS AtaReadPio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, void* b
return st;
}
// Flushes the requested operation.
static KSTATUS AtaFlush(ATA_DEVICE* dev) {
ATA_CHANNEL* ch = dev->Channel;
KSTATUS st = AtaWaitNotBusy(ch);
@@ -219,6 +234,7 @@ static KSTATUS AtaFlush(ATA_DEVICE* dev) {
return AtaWaitNotBusy(ch);
}
// Updates pio.
static KSTATUS AtaWritePio(ATA_DEVICE* dev, uint64_t lba, uint32_t count, const void* buffer) {
if (!dev || !buffer)
return STATUS_INVALID_PARAMETER;
@@ -285,6 +301,7 @@ static KSTATUS AtaGetIrpReadWrite(IRP* irp, bool* outWrite, uint64_t* outLba,
return STATUS_SUCCESS;
}
// Implements the worker operation.
static void AtaWorker(uintptr_t arg) {
ATA_DEVICE* dev = (ATA_DEVICE*)arg;
if (!dev)
@@ -315,6 +332,7 @@ static void AtaWorker(uintptr_t arg) {
}
}
// Dispatches read Write.
KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return STATUS_INVALID_PARAMETER;
@@ -345,6 +363,7 @@ KSTATUS AtaDispatchReadWrite(DEVICE_OBJECT* device, IRP* irp) {
return STATUS_PENDING;
}
// Returns free Disk Index.
static uint8_t AtaFindFreeDiskIndex(void) {
for (uint8_t i = 0; i < 8; i++) {
if (!IoDiskDevicePresent(i))
@@ -353,6 +372,7 @@ static uint8_t AtaFindFreeDiskIndex(void) {
return 0xFFu;
}
// Implements the bar Io Base operation.
static uint16_t AtaBarIoBase(ATA_CONTROLLER* ctl, uint8_t barIndex, uint16_t fallback) {
PCI_BAR bar;
KSTATUS st = PciGetBar(ctl->Pdo, barIndex, &bar);
@@ -361,6 +381,7 @@ static uint16_t AtaBarIoBase(ATA_CONTROLLER* ctl, uint8_t barIndex, uint16_t fal
return fallback;
}
// Initializes channel.
static KSTATUS AtaInitChannel(ATA_CONTROLLER* ctl, uint8_t index) {
ATA_CHANNEL* ch = &ctl->Channels[index];
memset(ch, 0, sizeof(*ch));
@@ -398,6 +419,7 @@ static KSTATUS AtaInitChannel(ATA_CONTROLLER* ctl, uint8_t index) {
return STATUS_SUCCESS;
}
// Registers disk.
static KSTATUS AtaRegisterDisk(ATA_DEVICE* dev) {
DEVICE_OBJECT* disk = (DEVICE_OBJECT*)0;
KSTATUS st = IoCreateDevice(&gAtaDriver, 0, &disk);
@@ -433,6 +455,7 @@ static KSTATUS AtaRegisterDisk(ATA_DEVICE* dev) {
return STATUS_SUCCESS;
}
// Probes drive.
static KSTATUS AtaProbeDrive(ATA_CONTROLLER* ctl, ATA_CHANNEL* ch, uint8_t drive) {
ATA_DEVICE* dev = &ch->Devices[drive];
memset(dev, 0, sizeof(*dev));
@@ -470,6 +493,7 @@ static KSTATUS AtaProbeDrive(ATA_CONTROLLER* ctl, ATA_CHANNEL* ch, uint8_t drive
return STATUS_SUCCESS;
}
// Registers controller.
static KSTATUS AtaRegisterController(ATA_CONTROLLER* ctl) {
if (!ctl) return STATUS_INVALID_PARAMETER;
for (uint32_t i = 0; i < gAtaControllerCount; i++) {
@@ -482,6 +506,7 @@ static KSTATUS AtaRegisterController(ATA_CONTROLLER* ctl) {
return STATUS_SUCCESS;
}
// Starts controller.
static KSTATUS AtaStartController(ATA_CONTROLLER* ctl) {
if (!ctl || ctl->Magic != ATA_CONTROLLER_MAGIC || !ctl->Pdo || !ctl->Fdo)
return STATUS_INVALID_PARAMETER;
@@ -538,12 +563,14 @@ static KSTATUS AtaStartController(ATA_CONTROLLER* ctl) {
return STATUS_SUCCESS;
}
// Cancels release Remove Lock.
static void AtaCancelReleaseRemoveLock(IRP* irp, void* context) {
ATA_DEVICE* dev = (ATA_DEVICE*)context;
if (dev)
IoReleaseRemoveLock(&dev->RemoveLock, irp);
}
// Releases controller Devices.
static void AtaRemoveControllerDevices(ATA_CONTROLLER* ctl, IO_DEVICE_CHANGE change) {
if (!ctl) return;
for (uint8_t c = 0; c < ATA_CHANNEL_COUNT; c++) {
@@ -566,6 +593,7 @@ static void AtaRemoveControllerDevices(ATA_CONTROLLER* ctl, IO_DEVICE_CHANGE cha
}
}
// Dispatches pnp.
KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->DeviceExtension)
return STATUS_INVALID_PARAMETER;
@@ -637,6 +665,7 @@ KSTATUS AtaDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
return STATUS_INVALID_PARAMETER;
}
// Implements the add Device operation.
KSTATUS AtaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject) {
if (!driver || !physicalDeviceObject || physicalDeviceObject->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -663,6 +692,7 @@ KSTATUS AtaAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObject)
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal)
return STATUS_INVALID_PARAMETER;
code/drivers/storage/ata/ata.h
+2 -1
View File
@@ -1,4 +1,5 @@
// ata.h - PCI IDE / ATA driver internals.
// ATA compatibility driver definitions shared by the storage implementation.
#pragma once
#include <stdint.h>
code/kernel/hal/hal_apic.c
+24 -3
View File
@@ -1,4 +1,5 @@
// hal_apic.c - Local APIC + I/O APIC
// HAL APIC support. This file owns local APIC, I/O APIC, and interrupt-controller setup.
#include <kernel/hal/hal_apic.h>
#include <ntlite/ddi/hal_api.h>
#include <stdint.h>
@@ -6,7 +7,7 @@
extern HAL_API gApi;
// ---- Local APIC ----
// ---- Local APIC ----.
static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
#define LAPIC_ID 0x020
@@ -28,7 +29,7 @@ static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define APIC_VECTOR_RESCHEDULE 0xF0u
// ---- I/O APIC ----
// ---- I/O APIC ----.
static volatile uint32_t* gIoapic = (volatile uint32_t *)0;
static uint32_t gBspLapicId = 0;
static uint8_t gX2ApicSupported = 0;
@@ -36,15 +37,18 @@ static uint8_t gX2ApicEnabled = 0;
#define IOAPIC_IDX 0x00
#define IOAPIC_DATA 0x10
// Implements the ioapic Read operation.
static uint32_t IoapicRead(uint8_t reg) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg;
return *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA);
}
// Implements the ioapic Write operation.
static void IoapicWrite(uint8_t reg, uint32_t val) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg;
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val;
}
// Implements the apic Detect Mode operation.
static void HalApicDetectMode(void) {
int regs[4] = {0, 0, 0, 0};
__cpuid(regs, 1);
@@ -54,17 +58,20 @@ static void HalApicDetectMode(void) {
gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u;
}
// Implements the apic Stall Fallback operation.
static void HalApicStallFallback(uint32_t loops) {
for (volatile uint32_t i = 0; i < loops; i++) {
_mm_pause();
}
}
// Implements the apic Stall Us operation.
static void HalApicStallUs(uint32_t us) {
if (gApi.Timer.Stall) gApi.Timer.Stall(us);
else HalApicStallFallback(us * 128u + 1u);
}
// Implements the apic Wait Icr Idle operation.
static void HalApicWaitIcrIdle(void) {
if (gX2ApicEnabled)
return;
@@ -76,6 +83,7 @@ static void HalApicWaitIcrIdle(void) {
}
}
// Implements the apic Send Ipi operation.
static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low);
@@ -94,20 +102,24 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
#define IRQ_VECTOR_BASE 0x20
// Returns bsp Lapic Id.
__declspec(dllexport) uint32_t HalGetBspLapicId(void) {
return gBspLapicId;
}
// Implements the apic Using X2 Apic operation.
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void) {
return gX2ApicEnabled;
}
// Implements the apic Can Address Id operation.
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId) {
if (gX2ApicEnabled)
return 1u;
return (lapicId < 256u) ? 1u : 0u;
}
// Implements the apic Enable Current Cpu operation.
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE;
@@ -123,6 +135,7 @@ __declspec(dllexport) void HalApicEnableCurrentCpu(void) {
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
}
// Implements the apic Send Panic Ipi operation.
__declspec(dllexport) void HalApicSendPanicIpi(void) {
if (!gLapic && !gX2ApicEnabled) return;
@@ -133,11 +146,13 @@ __declspec(dllexport) void HalApicSendPanicIpi(void) {
HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI
}
// Implements the apic Send Reschedule Ipi operation.
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) {
if (!gLapic && !gX2ApicEnabled) return;
HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination
}
// Implements the apic Send Tlb Shootdown Ipi operation.
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
if (!gLapic && !gX2ApicEnabled) return;
@@ -148,6 +163,7 @@ __declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination
}
// Starts application Processor.
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) {
if ((!gLapic && !gX2ApicEnabled) || !startupVector || lapicId == gBspLapicId)
return STATUS_INVALID_PARAMETER;
@@ -166,6 +182,7 @@ __declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uin
return STATUS_SUCCESS;
}
// Implements the apic Route Irq operation.
void HalApicRouteIrq(uint8_t irq) {
// Full initial setup of the redirection entry.
// Sets the vector and clears the mask bit in one write.
@@ -174,12 +191,14 @@ void HalApicRouteIrq(uint8_t irq) {
IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed
}
// Implements the apic Mask Irq operation.
void HalApicMaskIrq(uint8_t irq) {
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
lo |= (1 << 16);
IoapicWrite(IOREDTBL_LO(irq), lo);
}
// Implements the apic Unmask Irq operation.
void HalApicUnmaskIrq(uint8_t irq) {
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
lo &= ~(1 << 16); // clear mask bit
@@ -188,6 +207,7 @@ void HalApicUnmaskIrq(uint8_t irq) {
IoapicWrite(IOREDTBL_LO(irq), lo);
}
// Implements the apic Send Eoi operation.
void HalApicSendEoi(void) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_EOI, 0);
@@ -196,6 +216,7 @@ void HalApicSendEoi(void) {
LAPIC_REG(LAPIC_EOI) = 0;
}
// Implements the apic Initialize operation.
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
HalApicDetectMode();
code/kernel/hal/hal_disk.c
+2 -2
View File
@@ -1,2 +1,2 @@
// hal_disk.c intentionally left empty.
// Disk registration and dispatch live in kernel/io_device.c.
// HAL disk stub support. This file preserves the old HAL disk API while storage lives in drivers.
code/kernel/hal/hal_keyboard.c
+2 -1
View File
@@ -1 +1,2 @@
// hal_keyboard.c - input moved out of HAL into kernel input core and ps2.sys.
// HAL keyboard stub support. Keyboard input is handled by the kernel input core and ps2.sys.
code/kernel/hal/hal_main.c
+34 -23
View File
@@ -1,4 +1,5 @@
// hal_main.c - HAL initialisation and HalGetApi
// HAL initialization entry point. This file discovers platform tables and publishes the HAL API to the kernel.
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <string.h>
@@ -23,15 +24,13 @@ HAL_API gApi;
static int gInitialised = 0;
extern void (*gTimerCallback)(void);
// ---------------------------------------------------------------------------
// ACPI table walker - minimal, just enough to find APIC and HPET addresses.
// ---------------------------------------------------------------------------
#define ACPI_SIG(a,b,c,d) ((uint32_t)(a)|((uint32_t)(b)<<8)|((uint32_t)(c)<<16)|((uint32_t)(d)<<24))
#pragma pack(push, 1)
#pragma warning(disable : 4200)
// ACPI RSDP (Root System Description Pointer)
// ACPI RSDP (Root System Description Pointer).
typedef struct {
char Signature[8];
uint8_t Checksum;
@@ -44,7 +43,7 @@ typedef struct {
uint8_t Padding[3];
} RSDP;
// ACPI SDT Header
// ACPI SDT Header.
typedef struct {
char Signature[4];
uint32_t Length;
@@ -57,19 +56,19 @@ typedef struct {
uint32_t CreatorRevision;
} ACPISDTHeader;
// RSDT (Root System Description Table)
// RSDT (Root System Description Table).
typedef struct {
ACPISDTHeader Header;
uint32_t PointerToOtherSDT[];
} RSDT;
// XSDT (Extended System Description Table)
// XSDT (Extended System Description Table).
typedef struct {
ACPISDTHeader Header;
uint64_t PointerToOtherSDT[];
} XSDT;
// MADT (Multiple APIC Description Table)
// MADT (Multiple APIC Description Table).
typedef struct {
ACPISDTHeader Header;
uint32_t LocalAPICAddress;
@@ -94,9 +93,11 @@ static int gS5Found = 0;
int debugEnabled = 0;
int _fltused; // MSVC floating-point helper anchor
// Implements the hal putchar operation.
static void hal_putchar(char c) {
gApi.Screen.PrintChar(c, 0xCCCCCC, 0x012456);
}
// Implements the dbg Print operation.
void HalDbgPrint(const char* fmt, ...) {
if (!debugEnabled) return;
@@ -131,6 +132,7 @@ typedef struct {
} ACPI_GAS;
#pragma pack(pop)
// Implements the acpi Read U32 operation.
static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) {
uint32_t value = 0;
if (off + sizeof(value) <= tableLen)
@@ -138,6 +140,7 @@ static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off)
return value;
}
// Implements the acpi Read U64 operation.
static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off) {
uint64_t value = 0;
if (off + sizeof(value) <= tableLen)
@@ -145,6 +148,7 @@ static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off)
return value;
}
// Implements the acpi Read Gas Io Port operation.
static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t off) {
ACPI_GAS gas;
if (off + sizeof(gas) > tableLen) return 0;
@@ -214,6 +218,7 @@ static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
return p;
}
// Implements the aml Read Integer operation.
static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
const uint8_t* p = *pp;
if (p >= end) return 0;
@@ -290,6 +295,7 @@ static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
HalDbgPrint("HAL: _S5_ not found in DSDT; ACPI shutdown disabled\n");
}
// Validates checksum.
static int validate_checksum(void* table, uint32_t length) {
uint8_t sum = 0;
uint8_t* bytes = (uint8_t*)table;
@@ -299,6 +305,7 @@ static int validate_checksum(void* table, uint32_t length) {
return sum == 0;
}
// Parses fadt.
static void ParseFadt(const ACPISDTHeader* header) {
const void* fadt = header;
uint32_t len = header->Length;
@@ -326,6 +333,7 @@ static void ParseFadt(const ACPISDTHeader* header) {
if (dsdtPhys) {
const ACPISDTHeader* dsdt = (const ACPISDTHeader*)(dsdtPhys + gPhysBase);
if (memcmp(dsdt->Signature, "DSDT", 4) == 0 &&
// Validates checksum.
validate_checksum((void*)dsdt, dsdt->Length)) {
const uint8_t* aml = (const uint8_t*)(dsdt + 1);
ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader));
@@ -336,6 +344,7 @@ static void ParseFadt(const ACPISDTHeader* header) {
}
// Implements the add Cpu Record operation.
static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
if (gCpuTopology.CpuCount >= HAL_MAX_CPUS)
return;
@@ -353,6 +362,7 @@ static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
cpu->Enabled = (flags & 1u) ? 1u : 0u;
}
// Parses madt.
static void ParseMadt(MADT* madt) {
gLapicPhys = madt->LocalAPICAddress;
uint8_t* p = (uint8_t *)(madt + 1);
@@ -383,10 +393,11 @@ static void ParseMadt(MADT* madt) {
}
}
// Implements the walk Acpi operation.
static void WalkAcpi(uintptr_t rsdpPhys) {
if (!rsdpPhys) return;
// Map RSDP into virtual memory
// Map RSDP into virtual memory.
RSDP* rsdp = (RSDP*)(rsdpPhys + gPhysBase);
if (memcmp(rsdp->Signature, "RSD PTR ", 8) != 0) {
@@ -398,11 +409,12 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
uint32_t count = 0;
int isXsdt = 0;
// Safely find and validate the XSDT (ACPI 2.0+)
// Safely find and validate the XSDT (ACPI 2.0+).
if (rsdp->Revision >= 2 && rsdp->XsdtAddress != 0) {
XSDT* xsdt = (XSDT*)(rsdp->XsdtAddress + gPhysBase);
if (validate_checksum(xsdt, xsdt->Header.Length) &&
// Implements the memcmp operation.
memcmp(xsdt->Header.Signature, "XSDT", 4) == 0) {
sdtHeader = &xsdt->Header;
@@ -411,11 +423,12 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
}
}
// Fallback to safely finding and validating the RSDT (ACPI 1.0)
// Fallback to safely finding and validating the RSDT (ACPI 1.0).
if (sdtHeader == NULL && rsdp->RsdtAddress != 0) {
RSDT* rsdt = (RSDT*)((uintptr_t)rsdp->RsdtAddress + gPhysBase);
if (validate_checksum(rsdt, rsdt->Header.Length) &&
// Implements the memcmp operation.
memcmp(rsdt->Header.Signature, "RSDT", 4) == 0) {
sdtHeader = &rsdt->Header;
@@ -429,7 +442,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
return;
}
// Walk the tables
// Walk the tables.
for (uint32_t i = 0; i < count; i++) {
uintptr_t tablePhys = 0;
@@ -439,34 +452,32 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
tablePhys = ((RSDT*)sdtHeader)->PointerToOtherSDT[i];
}
// Paranoid alignment check to reject firmware garbage pointers
// Paranoid alignment check to reject firmware garbage pointers.
if (!tablePhys || (tablePhys & 3) != 0) continue;
ACPISDTHeader* header = (ACPISDTHeader*)(tablePhys + gPhysBase);
// Ensure the table isn't corrupt before parsing
// Ensure the table isn't corrupt before parsing.
if (!validate_checksum(header, header->Length)) continue;
// Parse MADT
// Parse MADT.
if (memcmp(header->Signature, "APIC", 4) == 0) {
ParseMadt((MADT*)header);
}
// Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer
// Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer.
else if (memcmp(header->Signature, "FACP", 4) == 0) {
ParseFadt(header);
}
// Parse HPET
// Parse HPET.
else if (memcmp(header->Signature, "HPET", 4) == 0) {
// HPET base address block is located at offset 44
// (which is 8 bytes past the 36-byte ACPISDTHeader)
// (which is 8 bytes past the 36-byte ACPISDTHeader).
gHpetPhys = *(uint64_t*)((uint8_t*)header + 44) & ~0xFFFULL;
}
}
}
// ---------------------------------------------------------------------------
// HalShutdown - ACPI S5 soft power-off. Does not return.
// ---------------------------------------------------------------------------
static void HalShutdown(void) {
// Silence! Stop the world!
_disable();
@@ -477,7 +488,7 @@ static void HalShutdown(void) {
while (true) __halt();
}
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10).
uint16_t valA = (uint16_t)(((uint16_t)gSlpTypA << 10) | (1 << 13));
uint16_t valB = (uint16_t)(((uint16_t)gSlpTypB << 10) | (1 << 13));
@@ -488,7 +499,6 @@ static void HalShutdown(void) {
while (true) __halt();
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
if (!out) return STATUS_INVALID_PARAMETER;
@@ -526,7 +536,6 @@ __declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
if (gInitialised) return STATUS_ALREADY_EXISTS;
gPhysBase = initParams->PhysicalWindowBase;
@@ -585,8 +594,10 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
return STATUS_SUCCESS;
}
// Registers scheduler.
__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)) {
gTimerCallback = pKeSchedulerTick;
}
// Returns api.
__declspec(dllexport) const HAL_API* HalGetApi(void) { return &gApi; }
code/kernel/hal/hal_mouse.c
+2 -1
View File
@@ -1 +1,2 @@
// hal_mouse.c - input moved out of HAL into kernel input core and ps2.sys.
// HAL mouse stub support. Mouse input is handled by the kernel input core and ps2.sys.
code/kernel/hal/hal_pci.c
+4 -1
View File
@@ -1,10 +1,12 @@
// hal_pci.c - raw PCI configuration access via legacy Mechanism 1 port I/O
// HAL PCI configuration access. This file owns legacy mechanism-1 PCI config port I/O.
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#define PCI_CONFIG_ADDR 0xCF8
#define PCI_CONFIG_DATA 0xCFC
// Creates addr.
static uint32_t MakeAddr(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
return 0x80000000UL
| ((uint32_t)bus << 16)
@@ -38,6 +40,7 @@ static void PciWrite32(uint16_t segment,
__outdword(PCI_CONFIG_DATA, value);
}
// Implements the pci Initialize operation.
void HalPciInitialize(HAL_PCI* out) {
out->Read32 = PciRead32;
out->Write32 = PciWrite32;
code/kernel/hal/hal_rtc.c
+12 -1
View File
@@ -1,4 +1,5 @@
// hal_rtc.c - legacy CMOS RTC fallback clock
// HAL RTC support. This file reads the CMOS clock as the fallback wall-clock source.
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
@@ -22,19 +23,23 @@
static uint8_t gCenturyRegister;
// Implements the rtc Read operation.
static uint8_t RtcRead(uint8_t reg) {
__outbyte(CMOS_INDEX_PORT, (uint8_t)(reg | 0x80u)); // keep NMI masked while selecting.
return __inbyte(CMOS_DATA_PORT);
}
// Implements the rtc Update In Progress operation.
static bool RtcUpdateInProgress(void) {
return (RtcRead(RTC_REG_STATUS_A) & 0x80u) != 0;
}
// Implements the rtc Bcd To Binary operation.
static uint8_t RtcBcdToBinary(uint8_t value) {
return (uint8_t)(((value >> 4) * 10u) + (value & 0x0Fu));
}
// Implements the rtc Read Stable operation.
static bool RtcReadStable(HAL_RTC_TIME* out) {
if (!out) return false;
@@ -111,10 +116,12 @@ static bool RtcReadStable(HAL_RTC_TIME* out) {
return true;
}
// Implements the rtc Leap Year operation.
static bool RtcLeapYear(uint16_t year) {
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
}
// Implements the rtc Days In Month operation.
static uint8_t RtcDaysInMonth(uint16_t year, uint8_t month) {
static const uint8_t days[12] = { 31,28,31,30,31,30,31,31,30,31,30,31 };
if (month < 1u || month > 12u) return 0;
@@ -122,6 +129,7 @@ static uint8_t RtcDaysInMonth(uint16_t year, uint8_t month) {
return days[month - 1u];
}
// Implements the rtc Valid operation.
static bool RtcValid(const HAL_RTC_TIME* t) {
if (!t) return false;
if (t->Year < 1970u || t->Year > 9999u) return false;
@@ -132,6 +140,7 @@ static bool RtcValid(const HAL_RTC_TIME* t) {
return true;
}
// Implements the rtc Query Time operation.
static KSTATUS HalRtcQueryTime(HAL_RTC_TIME* outTime) {
if (!outTime) return STATUS_INVALID_PARAMETER;
@@ -144,10 +153,12 @@ static KSTATUS HalRtcQueryTime(HAL_RTC_TIME* outTime) {
return STATUS_SUCCESS;
}
// Implements the rtc Set Century Register operation.
void HalRtcSetCenturyRegister(uint8_t centuryRegister) {
gCenturyRegister = centuryRegister;
}
// Implements the rtc Initialize operation.
void HalRtcInitialize(HAL_RTC* out) {
if (!out) return;
out->QueryTime = HalRtcQueryTime;
code/kernel/hal/hal_screen.c
+20 -21
View File
@@ -1,5 +1,5 @@
// hal_screen.c - GOP framebuffer screen driver with embedded 8x16 font
//
// HAL screen support. This file owns GOP framebuffer drawing before a runtime display provider takes over.
// The 8x16 bitmap font is a standard VGA/PC font. We embed just the
// printable ASCII range (0x20–0x7E). Each glyph is 16 bytes; bit 7 of
// each byte is the leftmost pixel.
@@ -22,18 +22,15 @@ static int gCursorX = 0, gCursorY = 0;
#define SCREEN_DEFAULT_FG 0xCCCCCCu
#define SCREEN_DEFAULT_BG 0x012456u
// ---------------------------------------------------------------------------
// 8x16 VGA font - printable ASCII only (glyph 0 = space = 0x20)
// ---------------------------------------------------------------------------
// 8x16 VGA font - printable ASCII only (glyph 0 = space = 0x20).
#include <kernel/hal/font8x16.h> // see font8x16.h
// ---------------------------------------------------------------------------
// Internals
// ---------------------------------------------------------------------------
// Internals.
static inline uint32_t* ScreenBackbufferPixel(int x, int y) {
return gBackbuffer ? (gBackbuffer + y * gStride + x) : (gFb + y * gStride + x);
}
// Updates pixel Raw.
static inline void PutPixelRaw(int x, int y, uint32_t color) {
if ((unsigned)x >= (unsigned)gWidth || (unsigned)y >= (unsigned)gHeight)
return;
@@ -42,6 +39,7 @@ static inline void PutPixelRaw(int x, int y, uint32_t color) {
gFb[y * gStride + x] = color;
}
// Implements the screen Flush Rows operation.
static void ScreenFlushRows(int startY, int rows) {
if (!gBackbuffer || rows <= 0)
return;
@@ -60,9 +58,7 @@ static void ScreenFlushRows(int startY, int rows) {
(size_t)rows * rowBytes);
}
// ---------------------------------------------------------------------------
// HAL_SCREEN implementations
// ---------------------------------------------------------------------------
// HAL_SCREEN implementations.
static void ScreenClear(uint32_t color) {
for (int y = 0; y < gHeight; y++)
for (int x = 0; x < gWidth; x++)
@@ -77,12 +73,14 @@ static void ScreenClear(uint32_t color) {
gCursorX = 0; gCursorY = 0;
}
// Implements the screen Put Pixel operation.
static void ScreenPutPixel(int x, int y, uint32_t color) {
PutPixelRaw(x, y, color);
}
// Implements the screen Blit operation.
static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
// Calculate clipping bounds
// Calculate clipping bounds.
int startX = (x < 0) ? 0 : x;
int startY = (y < 0) ? 0 : y;
int endX = (x + w > gWidth) ? gWidth : (x + w);
@@ -91,16 +89,16 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
int drawW = endX - startX;
int drawH = endY - startY;
// If the image is entirely off-screen, do nothing
// If the image is entirely off-screen, do nothing.
if (drawW <= 0 || drawH <= 0) {
return;
}
// Adjust the source pointer if the image was clipped on the top or left
// Adjust the source pointer if the image was clipped on the top or left.
int srcOffsetX = startX - x;
int srcOffsetY = startY - y;
// Calculate bytes to copy per row
// Calculate bytes to copy per row.
size_t bytesPerRow = (size_t)drawW * sizeof(uint32_t);
for (int row = 0; row < drawH; row++) {
@@ -113,8 +111,10 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
}
}
// Implements the screen Get Size operation.
static void ScreenGetSize(int* w, int* h) { *w = gWidth; *h = gHeight; }
// Implements the screen Get Framebuffer operation.
static KSTATUS ScreenGetFramebuffer(HAL_FRAMEBUFFER_INFO* outInfo) {
if (!outInfo || !gFb || gWidth <= 0 || gHeight <= 0 || gStride <= 0)
return STATUS_INVALID_PARAMETER;
@@ -127,6 +127,7 @@ static KSTATUS ScreenGetFramebuffer(HAL_FRAMEBUFFER_INFO* outInfo) {
return STATUS_SUCCESS;
}
// Implements the screen Scroll operation.
static void ScreenScroll(int lines) {
if (lines <= 0)
return;
@@ -152,6 +153,7 @@ static void ScreenScroll(int lines) {
ScreenFlushRows(0, gHeight);
}
// Implements the screen Put Char operation.
static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16);
for (int row = 0; row < 16; row++) {
@@ -163,11 +165,12 @@ static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
}
}
// Implements the screen Print Char operation.
static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
int cols = gWidth / FONT_W;
int rows = gHeight / FONT_H;
// Mirror to serial
// Mirror to serial.
while ((__inbyte(0x3FD) & 0x20) == 0);
if (c == '\n') __outbyte(0x3F8, '\r');
__outbyte(0x3F8, c);
@@ -194,11 +197,9 @@ static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
}
}
// ---------------------------------------------------------------------------
// Mouse cursor – 12×19 monochrome arrow sprite
// 0 = transparent 1 = black outline 2 = white fill
// The hotspot is at (0,0) (top-left pixel = tip of the arrow).
// ---------------------------------------------------------------------------
#define CURSOR_W 16
#define CURSOR_H 25
@@ -317,9 +318,7 @@ void HalScreenEraseCursor(void) {
gCursorDrawY = -1;
}
// ---------------------------------------------------------------------------
// Populate the HAL_SCREEN vtable
// ---------------------------------------------------------------------------
// Populate the HAL_SCREEN vtable.
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out) {
gFb = (uint32_t *)fb;
code/kernel/hal/hal_timer.c
+24 -16
View File
@@ -1,4 +1,5 @@
// hal_timer.c - Local APIC timer
// HAL timer support. This file owns APIC timer calibration and early tick delivery.
#include <stdint.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
@@ -30,15 +31,19 @@ static volatile uint64_t* gHpet = NULL;
static volatile uint64_t gTicks = 0;
void (* volatile gTimerCallback)(void) = NULL;
// Implements the outb operation.
static inline void outb(uint16_t port, uint8_t val) { __outbyte(port, val); }
// Implements the inb operation.
static inline uint8_t inb(uint16_t port) { return __inbyte(port); }
// Implements the timer Isr operation.
static void TimerIsr(void) {
gTicks++;
if (gTimerCallback) gTimerCallback();
// EOI is sent by KeIrqDispatch after this returns
// EOI is sent by KeIrqDispatch after this returns.
}
// Returns ticks.
static uint64_t QueryTicks(void) {
// Prefer HPET when present: it is already a monotonically increasing main
// counter and avoids races between the LAPIC periodic tick and the current
@@ -76,8 +81,9 @@ static uint64_t QueryTicks(void) {
return (baseTicks1 * TIMER_PERIOD_MS * 1000ull) + subQuantumUs;
}
// Implements the stall operation.
static void Stall(uint32_t us) {
// Priority 1: Use HPET (Highest Precision)
// Priority 1: Use HPET (Highest Precision).
if (gHpet && gHpetTicksPerUs > 0) {
uint64_t start = gHpet[30]; // Offset 0xF0 is the Main Counter
uint64_t delta = (uint64_t)us * gHpetTicksPerUs;
@@ -87,7 +93,7 @@ static void Stall(uint32_t us) {
return;
}
// Priority 2: Use APIC Timer (High Precision, but handles wrap-around)
// Priority 2: Use APIC Timer (High Precision, but handles wrap-around).
if (gApicTicksPerUs > 0) {
uint32_t total_ticks = us * gApicTicksPerUs;
uint32_t last_val = LAPIC_REG(LAPIC_TIMER_CUR);
@@ -98,7 +104,7 @@ static void Stall(uint32_t us) {
if (current < last_val) {
elapsed += (last_val - current);
} else if (current > last_val) {
// Handled the reset/wrap at the end of TIMER_PERIOD_MS
// Handled the reset/wrap at the end of TIMER_PERIOD_MS.
uint32_t init_val = LAPIC_REG(LAPIC_TIMER_INIT);
elapsed += (last_val + (init_val - current));
}
@@ -108,7 +114,7 @@ static void Stall(uint32_t us) {
return;
}
// Priority 3: Fallback to Port 0x80
// Priority 3: Fallback to Port 0x80.
for (uint32_t i = 0; i < us; i++) {
__outbyte(0x80, 0);
}
@@ -116,16 +122,16 @@ static void Stall(uint32_t us) {
// Returns HPET ticks/second, or 0 if HPET is unavailable.
static uint64_t HpetInit(uintptr_t hpetPhys) {
// HPET register block, accessed through physical window
// HPET register block, accessed through physical window.
volatile uint64_t *hpet = (volatile uint64_t *)gApi.PhysToVirt(hpetPhys);
// General Capabilities register (offset 0):
// Bits 63:32 = counter clock period in femtoseconds
// Bits 63:32 = counter clock period in femtoseconds.
uint64_t caps = hpet[0];
uint32_t periodFs = (uint32_t)(caps >> 32);
if (!periodFs) return 0;
// Enable the HPET main counter (General Configuration, offset 0x10, bit 0)
// Enable the HPET main counter (General Configuration, offset 0x10, bit 0).
hpet[2] |= 1; // offset 0x10 / 8 = index 2
return (uint64_t)1e15 / periodFs; // ticks per second
@@ -133,21 +139,21 @@ static uint64_t HpetInit(uintptr_t hpetPhys) {
// Use PIT channel 2 one-shot to measure how many APIC ticks elapse in 10 ms, then return ticks/ms.
static uint32_t CalibrateApicTimerViaPit(void) {
// Divide by 16
// Divide by 16.
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
// Mask, set vector - leave masked during calibration
// Mask, set vector - leave masked during calibration.
LAPIC_REG(LAPIC_TIMER_LVT) = TIMER_MASKED | TIMER_VECTOR;
// Load max count
// Load max count.
LAPIC_REG(LAPIC_TIMER_INIT) = 0xFFFFFFFF;
// PIT channel 2: one-shot, ~10 ms
// PIT channel 2: one-shot, ~10 ms.
uint16_t pit_count = (uint16_t)(PIT_HZ / 100); // 11931 ≈ 10 ms
outb(0x61, (inb(0x61) & ~0x02) | 0x01); // gate on, speaker off
outb(0x43, 0xB0); // ch2, lo/hi, mode 0, binary
outb(0x42, (uint8_t)(pit_count & 0xFF));
outb(0x42, (uint8_t)(pit_count >> 8));
// Wait for OUT2 (bit 5 of port 0x61) to go high
// Wait for OUT2 (bit 5 of port 0x61) to go high.
while (!(inb(0x61) & 0x20)) _mm_pause();
uint32_t elapsed = 0xFFFFFFFF - LAPIC_REG(LAPIC_TIMER_CUR);
@@ -160,14 +166,14 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
uint64_t hpetHz = HpetInit(hpetPhys);
if (!hpetHz) return 0;
// Ticks in 10 ms = hpetHz / 100
// Ticks in 10 ms = hpetHz / 100.
uint64_t ticksFor10ms = hpetHz / 100;
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
LAPIC_REG(LAPIC_TIMER_LVT) = TIMER_MASKED | TIMER_VECTOR;
LAPIC_REG(LAPIC_TIMER_INIT) = 0xFFFFFFFF;
// HPET main counter is at offset 0xF0, index 0xF0/8 = 30
// HPET main counter is at offset 0xF0, index 0xF0/8 = 30.
uint64_t hpetStart = hpet[30];
while ((hpet[30] - hpetStart) < ticksFor10ms) _mm_pause();
@@ -175,6 +181,7 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
return elapsed / 10; // APIC ticks per ms
}
// Implements the timer Configure Current Cpu operation.
__declspec(dllexport) void HalTimerConfigureCurrentCpu(void) {
if (!gLapic || !gApicTicksPerQuantum) return;
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
@@ -182,6 +189,7 @@ __declspec(dllexport) void HalTimerConfigureCurrentCpu(void) {
LAPIC_REG(LAPIC_TIMER_INIT) = gApicTicksPerQuantum;
}
// Implements the timer Initialize operation.
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out) {
gLapic = (volatile uint32_t *)gApi.PhysToVirt(lapicPhys);
HalDbgPrint("HAL: lapicPhys=0x%016llX, lapicVirt=0x%016llX, hpetPhys=0x%016llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
code/kernel/include/kernel/display_cursor.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel display cursor declarations shared by implementation files and kernel clients.
#pragma once
// display_cursor.h - shared default software cursor image
// display_cursor.h - shared default software cursor image.
#include <stdint.h>
code/kernel/include/kernel/hal/font8x16.h
+523 -521
View File
File diff suppressed because it is too large. Load diff
code/kernel/include/kernel/hal/hal_apic.h
+2 -1
View File
@@ -1,4 +1,5 @@
// hal_apic.h
// Kernel hal apic declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
code/kernel/include/kernel/hal/hal_debug.h
+5 -3
View File
@@ -1,4 +1,6 @@
#pragma once
// hal_debug.h - HAL debug output function
// Kernel hal debug declarations shared by implementation files and kernel clients.
#pragma once
// hal_debug.h - HAL debug output function.
void HalDbgPrint(const char* fmt, ...);
code/kernel/include/kernel/io_file.h
+9 -25
View File
@@ -1,3 +1,5 @@
// Kernel io file declarations shared by implementation files and kernel clients.
#pragma once
// io_file.h - VFS layer
// Filesystem drivers register an IO_FILESYSTEM vtable via IoRegisterFilesystem.
@@ -23,9 +25,7 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
KSTATUS status,
size_t information);
// ---------------------------------------------------------------------------
// Filesystem driver vtable - filled in by a filesystem driver.
// ---------------------------------------------------------------------------
typedef struct IO_FILESYSTEM {
// Open a file by absolute path (e.g. "apps\\shell.exe").
// Returns an opaque per-file driver cookie in* fileCtx.
@@ -80,9 +80,7 @@ typedef struct IO_FILESYSTEM {
// Optional driver-private data.
void* DriverCtx;
// ---------------------------------------------------------------------------
// Directory enumeration (optional - may be NULL if not supported)
// ---------------------------------------------------------------------------
// Directory enumeration (optional - may be NULL if not supported).
// Open a directory iterator for 'dirPath' (relative or drive-prefixed).
// Returns an opaque per-iteration driver cookie in *dirCtx.
@@ -100,9 +98,7 @@ typedef struct IO_FILESYSTEM {
} IO_FILESYSTEM;
// ---------------------------------------------------------------------------
// IO_STATUS_BLOCK - completion status visible to async callers
// ---------------------------------------------------------------------------
// IO_STATUS_BLOCK - completion status visible to async callers.
typedef struct IO_STATUS_BLOCK {
volatile KSTATUS Status;
volatile size_t Information;
@@ -112,9 +108,7 @@ struct EVENT;
struct KAPC;
typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
// ---------------------------------------------------------------------------
// FILE_OBJECT - the kernel object returned to callers
// ---------------------------------------------------------------------------
// FILE_OBJECT - the kernel object returned to callers.
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
@@ -124,9 +118,7 @@ typedef struct FILE_OBJECT {
uint64_t Offset; // current position for sequential reads
} FILE_OBJECT;
// ---------------------------------------------------------------------------
// DIR_OBJECT - kernel object for an open directory iterator
// ---------------------------------------------------------------------------
// DIR_OBJECT - kernel object for an open directory iterator.
typedef struct DIR_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
@@ -139,18 +131,14 @@ typedef struct IO_VOLUME_PARAMS {
uint64_t PartitionSize; // sectors, for bounds checking
} IO_VOLUME_PARAMS;
// ---------------------------------------------------------------------------
// Registration - called by fat32.sys from DriverEntry
// ---------------------------------------------------------------------------
// driveIndex matches the index passed to IoRegisterDiskDevice.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol);
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
__declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex);
// ---------------------------------------------------------------------------
// Kernel-side API
// ---------------------------------------------------------------------------
// Kernel-side API.
__declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out);
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead);
@@ -183,15 +171,11 @@ __declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// ---------------------------------------------------------------------------
// Directory enumeration API
// ---------------------------------------------------------------------------
// Directory enumeration API.
__declspec(dllexport) KSTATUS IoOpenDir (SYS_STRING_VIEW dirPath, DIR_OBJECT** out);
__declspec(dllexport) KSTATUS IoReadDir (DIR_OBJECT* dir, KDIR_ENTRY* entryOut);
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir);
// ---------------------------------------------------------------------------
// Current Working Directory
// ---------------------------------------------------------------------------
// Current Working Directory.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size);
code/kernel/include/kernel/ke_apc.h
+4 -2
View File
@@ -1,6 +1,8 @@
// Kernel ke apc declarations shared by implementation files and kernel clients.
#pragma once
// ke_apc.h - kernel APC infrastructure
//
// Kernel APCs are per-thread callbacks that run in the target thread's
// context when that thread reaches PASSIVE_LEVEL and APC delivery is enabled.
// They are the first APC layer; user APCs will build on this once trap-frame
@@ -43,7 +45,7 @@ typedef struct SYS_USER_APC_FRAME_K {
uint64_t SystemArgument2;
} SYS_USER_APC_FRAME_K;
#define SYS_USER_APC_FRAME_MAGIC 0x3143504152455355ULL /* "USERAPC1" little-ish */
#define SYS_USER_APC_FRAME_MAGIC 0x3143504152455355ULL // "USERAPC1" little-ish
struct KUSER_APC {
LIST_ENTRY Link;
code/kernel/include/kernel/ke_cpu.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke cpu declarations shared by implementation files and kernel clients.
#pragma once
// ke_cpu.h - per-CPU execution state
//
// Every processor that reaches kernel-owned state has a KCPU record. The record
// owns CPU-local scheduler, IRQL, GDT, and TSS state.
code/kernel/include/kernel/ke_debug.h
+3 -3
View File
@@ -1,3 +1,5 @@
// Kernel ke debug declarations shared by implementation files and kernel clients.
#ifndef KE_DEBUG_H
#define KE_DEBUG_H
@@ -31,15 +33,13 @@ typedef enum KE_PANIC_CODE {
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...);
__declspec(dllexport) int KeHandleNmiIpi(void);
// ---------------------------------------------------------------------------
// Fault dispatch - called from the IDT fault stubs (ke_syscall.asm).
//
// The old scalar entry only reported RIP/error/CR2, which is not enough for
// debugging null indirect calls. The frame entry snapshots user GPRs plus the
// CPU exception frame, including user RSP. For a CALL through a null function
// pointer, RIP is 0 and [RSP] is usually the return address after the bad call;
// that is the useful caller site.
// ---------------------------------------------------------------------------
typedef struct KFAULT_FRAME {
uint64_t Magic;
uint64_t Vector;
code/kernel/include/kernel/ke_display.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke display declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <stdbool.h>
code/kernel/include/kernel/ke_dpc.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke dpc declarations shared by implementation files and kernel clients.
#pragma once
// ke_dpc.h - deferred procedure calls
//
// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC
// after acknowledging/capturing device state; the DPC then runs later at
// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not
code/kernel/include/kernel/ke_event.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel ke event declarations shared by implementation files and kernel clients.
#pragma once
// ke_event.h - auto-reset and manual-reset events
// ke_event.h - auto-reset and manual-reset events.
#include <stdint.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
code/kernel/include/kernel/ke_idt.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel ke idt declarations shared by implementation files and kernel clients.
#pragma once
// ke_idt.h - IDT, GDT, and TSS setup
// ke_idt.h - IDT, GDT, and TSS setup.
#include <stdint.h>
// 64-bit TSS - minimal; we only care about RSP0.
code/kernel/include/kernel/ke_input.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke input declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <stdbool.h>
code/kernel/include/kernel/ke_irp.h
+11 -25
View File
@@ -1,6 +1,8 @@
// Kernel ke irp declarations shared by implementation files and kernel clients.
#pragma once
// ke_irp.h - I/O Request Packet and remove-lock primitives
//
// IRPs are stack-based. A caller builds an IRP with one or more
// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O
// manager/driver stack advances CurrentLocation as the IRP is forwarded.
@@ -18,9 +20,7 @@ struct SEMAPHORE;
struct KAPC;
typedef struct DEVICE_OBJECT DEVICE_OBJECT;
// -------------------------------------------------------------------------
// IRP major/minor functions
// -------------------------------------------------------------------------
// IRP major/minor functions.
typedef enum IO_MAJOR_FUNCTION {
IRP_MJ_CREATE = 0,
@@ -50,17 +50,13 @@ typedef enum IO_DEVICE_RELATION_TYPE {
IoRemovalRelations = 1,
} IO_DEVICE_RELATION_TYPE;
// -------------------------------------------------------------------------
// IRP flags
// -------------------------------------------------------------------------
// IRP flags.
#define IRP_FLAG_PRIORITY (1u << 0) // insert at head of a driver queue
#define IRP_FLAG_CANCELLED (1u << 1)
#define IRP_FLAG_COMPLETED (1u << 2)
// -------------------------------------------------------------------------
// IRP stack parameters
// -------------------------------------------------------------------------
// IRP stack parameters.
typedef struct IO_STACK_LOCATION {
IO_MAJOR_FUNCTION MajorFunction;
@@ -91,18 +87,14 @@ typedef struct IO_STACK_LOCATION {
} Parameters;
} IO_STACK_LOCATION;
// -------------------------------------------------------------------------
// Forward declaration / callbacks
// -------------------------------------------------------------------------
// Forward declaration / callbacks.
typedef struct IRP IRP;
typedef void (*IRP_COMPLETION_FN)(IRP* irp, void* ctx);
typedef void (*IRP_CANCEL_ROUTINE)(DEVICE_OBJECT* device, IRP* irp);
// -------------------------------------------------------------------------
// I/O Request Packet
// -------------------------------------------------------------------------
// I/O Request Packet.
typedef struct IRP {
LIST_ENTRY Link;
@@ -135,9 +127,7 @@ typedef struct IRP {
IO_STACK_LOCATION Stack[1];
} IRP;
// -------------------------------------------------------------------------
// Remove lock
// -------------------------------------------------------------------------
// Remove lock.
typedef struct IO_REMOVE_LOCK {
SPINLOCK Lock;
@@ -152,9 +142,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp);
// -------------------------------------------------------------------------
// IRP_QUEUE (per-device, driver-internal)
// -------------------------------------------------------------------------
// IRP_QUEUE (per-device, driver-internal).
typedef struct IRP_QUEUE {
SPINLOCK Lock;
@@ -172,9 +160,7 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
void* context);
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status);
// -------------------------------------------------------------------------
// IRP allocation / stack helpers
// -------------------------------------------------------------------------
// IRP allocation / stack helpers.
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp);
__declspec(dllexport) void IoFreeIrp(IRP* irp);
code/kernel/include/kernel/ke_irq.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke irq declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
code/kernel/include/kernel/ke_irql.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke irql declarations shared by implementation files and kernel clients.
#pragma once
// ke_irql.h - software IRQL skeleton
//
// This is intentionally small for the first IRQL/APC bring-up step. It does
// not yet program interrupt hardware or replace the spinlock interrupt-disable
// path. It gives the kernel one authoritative execution-level state plus
code/kernel/include/kernel/ke_list.h
+49 -44
View File
@@ -1,44 +1,49 @@
#pragma once
// ke_list.h - intrusive doubly-linked list
// Used throughout the kernel for run queues, wait lists, thread lists, etc.
#include <stddef.h>
typedef struct LIST_ENTRY {
struct LIST_ENTRY* Fwd;
struct LIST_ENTRY* Bwd;
} LIST_ENTRY;
#define LIST_INIT(head) ((head)->Fwd = (head)->Bwd = (head))
#define LIST_IS_EMPTY(head) ((head)->Fwd == (head))
// Recover the containing struct pointer from an embedded LIST_ENTRY.
#define CONTAINING_RECORD(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
static inline void ListInsertTail(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Bwd = head->Bwd;
node->Fwd = head;
head->Bwd->Fwd = node;
head->Bwd = node;
}
static inline void ListInsertHead(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Fwd = head->Fwd;
node->Bwd = head;
head->Fwd->Bwd = node;
head->Fwd = node;
}
static inline void ListRemove(LIST_ENTRY* node) {
node->Bwd->Fwd = node->Fwd;
node->Fwd->Bwd = node->Bwd;
node->Fwd = node->Bwd = node; /* self-loop: poison against double-remove */
}
// Pop the first entry off the list, or NULL if empty.
static inline LIST_ENTRY* ListPopHead(LIST_ENTRY* head) {
if (LIST_IS_EMPTY(head)) return (LIST_ENTRY *)0;
LIST_ENTRY* node = head->Fwd;
ListRemove(node);
return node;
}
// Kernel ke list declarations shared by implementation files and kernel clients.
#pragma once
// ke_list.h - intrusive doubly-linked list
// Used throughout the kernel for run queues, wait lists, thread lists, etc.
#include <stddef.h>
typedef struct LIST_ENTRY {
struct LIST_ENTRY* Fwd;
struct LIST_ENTRY* Bwd;
} LIST_ENTRY;
#define LIST_INIT(head) ((head)->Fwd = (head)->Bwd = (head))
#define LIST_IS_EMPTY(head) ((head)->Fwd == (head))
// Recover the containing struct pointer from an embedded LIST_ENTRY.
#define CONTAINING_RECORD(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
// Implements the list Insert Tail operation.
static inline void ListInsertTail(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Bwd = head->Bwd;
node->Fwd = head;
head->Bwd->Fwd = node;
head->Bwd = node;
}
// Implements the list Insert Head operation.
static inline void ListInsertHead(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Fwd = head->Fwd;
node->Bwd = head;
head->Fwd->Bwd = node;
head->Fwd = node;
}
// Implements the list Remove operation.
static inline void ListRemove(LIST_ENTRY* node) {
node->Bwd->Fwd = node->Fwd;
node->Fwd->Bwd = node->Bwd;
node->Fwd = node->Bwd = node; // self-loop: poison against double-remove
}
// Pop the first entry off the list, or NULL if empty.
static inline LIST_ENTRY* ListPopHead(LIST_ENTRY* head) {
if (LIST_IS_EMPTY(head)) return (LIST_ENTRY *)0;
LIST_ENTRY* node = head->Fwd;
ListRemove(node);
return node;
}
code/kernel/include/kernel/ke_mutex.h
+4 -2
View File
@@ -1,5 +1,7 @@
// Kernel ke mutex declarations shared by implementation files and kernel clients.
#pragma once
// ke_mutex.h - binary ownership mutex
// ke_mutex.h - binary ownership mutex.
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_list.h>
@@ -8,7 +10,7 @@
typedef struct MUTEX {
OBJECT Header;
SPINLOCK Lock;
struct THREAD* Owner; /* NULL if unowned */
struct THREAD* Owner; // NULL if unowned
LIST_ENTRY WaitList;
} MUTEX;
code/kernel/include/kernel/ke_process.h
+22 -20
View File
@@ -1,3 +1,5 @@
// Kernel ke process declarations shared by implementation files and kernel clients.
#pragma once
// ke_process.h - process model
// A process owns a handle table and a list of threads.
@@ -37,23 +39,23 @@ typedef struct PROCESS_MODULE {
} PROCESS_MODULE;
typedef struct PROCESS {
OBJECT Header; /* must be first */
OBJECT Header; // must be first
uint32_t ProcessId;
char Name[PROCESS_NAME_MAX]; /* UTF-8 */
char Name[PROCESS_NAME_MAX]; // UTF-8
char CurrentDirectory[PROCESS_CWD_MAX];
size_t CurrentDirectoryLength;
uint8_t CurrentDrive;
SPINLOCK CwdLock;
/* Thread list. */
LIST_ENTRY ThreadList; /* links THREAD.ProcessLink */
// Thread list.
LIST_ENTRY ThreadList; // links THREAD.ProcessLink
uint32_t ThreadCount;
/* Handle table.
Index 0 is always empty (the invalid handle).
Each live entry holds one object reference plus rights metadata. */
// Handle table.
// Index 0 is always empty (the invalid handle).
// Each live entry holds one object reference plus rights metadata.
HANDLE_ENTRY Handles[HANDLE_TABLE_SIZE];
SPINLOCK HandleLock;
@@ -61,11 +63,11 @@ typedef struct PROCESS {
volatile long Terminating;
volatile long Exited;
/* Main executable image. Used for cleanup. */
// Main executable image. Used for cleanup.
uintptr_t ImageBase;
size_t ImageSize;
/* User-mode process environment. The initial thread sees this through GS->TEB->PEB. */
// User-mode process environment. The initial thread sees this through GS->TEB->PEB.
uintptr_t UserPebBase;
size_t UserPebSize;
uintptr_t UserImagePathBase;
@@ -75,21 +77,21 @@ typedef struct PROCESS {
uintptr_t UserCwdBase;
size_t UserCwdSize;
/* Per-process user virtual regions. The actual VA arena is global because
NT-Lite currently uses one shared address space; this list tracks
ownership so leaked user mappings can be reclaimed on process exit. */
// Per-process user virtual regions. The actual VA arena is global because
// NT-Lite currently uses one shared address space; this list tracks
// ownership so leaked user mappings can be reclaimed on process exit.
LIST_ENTRY UserRegions;
SPINLOCK UserRegionLock;
/* Per-process loaded modules (user processes only). */
// Per-process loaded modules (user processes only).
PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
uint32_t ModuleCount;
/* Signaled (manual-reset) when the process exits.
Waiters block on this via KeWaitForEvent. */
// Signaled (manual-reset) when the process exits.
// Waiters block on this via KeWaitForEvent.
EVENT* ExitEvent;
LIST_ENTRY GlobalLink; /* node in the global process list */
LIST_ENTRY GlobalLink; // node in the global process list
} PROCESS;
KSTATUS KeCreateProcess(const char* name, PROCESS **out);
@@ -103,7 +105,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode);
int KeIsProcessTerminating(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut);
/* Handle table operations. */
// Handle table operations.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut);
KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t rights, uint32_t* handleOut);
void KeCloseHandle(PROCESS* process, uint32_t handle);
@@ -111,13 +113,13 @@ OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights);
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle);
/* Return the global process list head (kernel-internal). */
// Return the global process list head (kernel-internal).
LIST_ENTRY* KeGetProcessList(void);
PROCESS* KeGetCurrentProcess(void);
PROCESS* KeGetSystemProcess(void);
struct SYS_PROCESS_INFO;
/* Copy a snapshot of rich process information into caller-provided storage. */
// Copy a snapshot of rich process information into caller-provided storage.
size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount);
/* Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject. */
// Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject.
PROCESS* KeGetProcessById(uint32_t pid);
code/kernel/include/kernel/ke_queue.h
+5 -3
View File
@@ -1,3 +1,5 @@
// Kernel ke queue declarations shared by implementation files and kernel clients.
#pragma once
// ke_queue.h - fixed-depth message queue
// Processes share an address space, so a pointer in Param[] is valid
@@ -9,14 +11,14 @@
#include <kernel/ke_list.h>
#include <ntlite/status.h>
#define QUEUE_MESSAGE_SIZE 64 /* bytes, fixed */
#define QUEUE_MESSAGE_SIZE 64 // bytes, fixed
#define QUEUE_DEPTH 256
typedef struct QUEUE_MESSAGE {
uint32_t Type;
uint32_t Flags;
uint64_t Param[6]; /* 48 bytes of payload */
} QUEUE_MESSAGE; /* total: 64 bytes */
uint64_t Param[6]; // 48 bytes of payload
} QUEUE_MESSAGE; // total: 64 bytes
typedef struct QUEUE {
OBJECT Header;
code/kernel/include/kernel/ke_sched.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke sched declarations shared by implementation files and kernel clients.
#pragma once
// ke_sched.h - Multi-Level Feedback Queue scheduler
// Three run queues (Low / Normal / High). IRQ 0 drives KeSchedulerTick.
code/kernel/include/kernel/ke_selftest.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke selftest declarations shared by implementation files and kernel clients.
#pragma once
// ke_selftest.h - kernel bring-up and regression self-tests.
code/kernel/include/kernel/ke_semaphore.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel ke semaphore declarations shared by implementation files and kernel clients.
#pragma once
// ke_semaphore.h - counting semaphore
// ke_semaphore.h - counting semaphore.
#include <stdint.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
code/kernel/include/kernel/ke_settings.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke settings declarations shared by implementation files and kernel clients.
#pragma once
#include <stddef.h>
code/kernel/include/kernel/ke_smp.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke smp declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
code/kernel/include/kernel/ke_spinlock.h
+9 -2
View File
@@ -1,12 +1,14 @@
// Kernel ke spinlock declarations shared by implementation files and kernel clients.
#pragma once
// ke_spinlock.h - spinlock for short, non-sleeping critical sections
//
// Step 3 IRQL bring-up:
// * Acquiring a spinlock disables maskable interrupts and raises the software
// IRQL to at least DISPATCH_LEVEL.
// * Releasing restores both the previous IRQL and the previous RFLAGS.IF
// state through the returned cookie.
//
// The public type remains uintptr_t so existing callers do not need to change.
// Never hold a spinlock across a wait/blocking operation.
#include <stdint.h>
@@ -23,15 +25,18 @@ typedef struct SPINLOCK {
#define KI_SPINLOCK_IRQL_SHIFT ((unsigned)(sizeof(uintptr_t) * 8u - 8u))
#define KI_SPINLOCK_FLAGS_MASK (~(((uintptr_t)0xFFu) << KI_SPINLOCK_IRQL_SHIFT))
// Implements the pack Spinlock Cookie operation.
static __forceinline uintptr_t KiPackSpinlockCookie(uintptr_t flags, KIRQL oldIrql) {
return (flags & KI_SPINLOCK_FLAGS_MASK) |
(((uintptr_t)oldIrql) << KI_SPINLOCK_IRQL_SHIFT);
}
// Implements the spinlock Cookie Flags operation.
static __forceinline uintptr_t KiSpinlockCookieFlags(uintptr_t cookie) {
return cookie & KI_SPINLOCK_FLAGS_MASK;
}
// Implements the spinlock Cookie Irql operation.
static __forceinline KIRQL KiSpinlockCookieIrql(uintptr_t cookie) {
return (KIRQL)(cookie >> KI_SPINLOCK_IRQL_SHIFT);
}
@@ -62,6 +67,7 @@ static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t cookie) {
__writeeflags(flags);
}
// Implements the try Acquire Spinlock operation.
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie) {
uintptr_t flags = __readeflags();
_disable();
@@ -85,6 +91,7 @@ static __forceinline uintptr_t KeAcquireSpinlockRaiseToDpc(SPINLOCK* sl) {
return KeAcquireSpinlock(sl);
}
// Releases spinlock From Dpc.
static __forceinline void KeReleaseSpinlockFromDpc(SPINLOCK* sl, uintptr_t cookie) {
KeReleaseSpinlock(sl, cookie);
}
code/kernel/include/kernel/ke_syscall.h
+4 -2
View File
@@ -1,11 +1,13 @@
// Kernel ke syscall declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
// Set the HAL API pointer for syscalls
// Set the HAL API pointer for syscalls.
void KeSyscallInit(const HAL_API* hal);
// Syscall dispatcher, called from assembly entry point
// Syscall dispatcher, called from assembly entry point.
uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5);
code/kernel/include/kernel/ke_thread.h
+10 -8
View File
@@ -1,3 +1,5 @@
// Kernel ke thread declarations shared by implementation files and kernel clients.
#pragma once
// ke_thread.h - thread model
// A THREAD is the unit of scheduling. Every thread belongs to exactly one process.
@@ -9,8 +11,8 @@
#include <kernel/ke_spinlock.h>
#include <kernel/ke_trap.h>
#define KSTACK_SIZE (64 * 1024) /* 64 KB kernel stack */
#define USTACK_SIZE (512 * 1024) /* 512 KB user stack */
#define KSTACK_SIZE (64 * 1024) // 64 KB kernel stack
#define USTACK_SIZE (512 * 1024) // 512 KB user stack
typedef enum THREAD_PRIORITY {
PriorityLow = 0,
@@ -31,11 +33,11 @@ typedef enum THREAD_STATE {
typedef struct KCONTEXT {
uint64_t R15, R14, R13, R12;
uint64_t Rbp, Rbx;
uint64_t Rip; /* return address back into KeSwitchContext */
uint64_t Rip; // return address back into KeSwitchContext
} KCONTEXT;
typedef struct THREAD {
OBJECT Header; /* must be first */
OBJECT Header; // must be first
uint32_t ThreadId;
THREAD_STATE State;
@@ -46,8 +48,8 @@ typedef struct THREAD {
uint32_t QuantaConsumed;
PROCESS* Owner;
LIST_ENTRY ProcessLink; /* node in Owner->ThreadList */
LIST_ENTRY SchedulerLink; /* node in run queue or wait list */
LIST_ENTRY ProcessLink; // node in Owner->ThreadList
LIST_ENTRY SchedulerLink; // node in run queue or wait list
// Kernel stack - never PAGE_USER, pointed to by TssRsp0.
void* KernelStackBase;
@@ -152,14 +154,14 @@ __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uin
__declspec(dllexport) void KeExitThread(THREAD* thread, int code);
// Block / unblock primitives used by sync objects.
//
// Correct wait pattern:
// object lock held
// KePrepareBlockThread(self, object, alertable);
// insert self into object's wait list
// release object lock
// KeCommitBlockThread(self, alertable);
//
// The split form prevents the classic lost wakeup where an ISR/DPC signals
// the object after the waiter is queued but before the waiter marks itself
// waiting. Alertable waits are explicit at every call site.
code/kernel/include/kernel/ke_time.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke time declarations shared by implementation files and kernel clients.
#pragma once
#include <ntlite/status.h>
#include <ntlite/time.h>
code/kernel/include/kernel/ke_trap.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke trap declarations shared by implementation files and kernel clients.
#pragma once
// ke_trap.h - trap-frame metadata for syscall/interrupt return paths
//
// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally
// an editable return frame: user APC delivery can rewrite Rip/Rsp in the frame,
// and KiExitTrapFrame commits those values back to the hardware IRET slots.
code/kernel/include/kernel/ke_userenv.h
+4 -2
View File
@@ -1,3 +1,5 @@
// Kernel ke userenv declarations shared by implementation files and kernel clients.
#pragma once
// ke_userenv.h - kernel view of the user-mode process/thread environment.
// This is this OS' ABI, not a Windows compatibility structure.
@@ -6,8 +8,8 @@
#include <stddef.h>
#include <ntlite/string_view.h>
#define SYS_PEB_MAGIC 0x30424550u /* "PEB0" */
#define SYS_TEB_MAGIC 0x30424554u /* "TEB0" */
#define SYS_PEB_MAGIC 0x30424550u // "PEB0"
#define SYS_TEB_MAGIC 0x30424554u // "TEB0"
#define SYS_ENV_VERSION 1u
#define SYS_CWD_MAX 520u
code/kernel/include/kernel/ke_util.h
+4 -5
View File
@@ -1,23 +1,22 @@
// ke_util.h - kernel utility macros (dynamic arrays, etc.)
// Kernel ke util declarations shared by implementation files and kernel clients.
#pragma once
#include <string.h>
#include <kernel/mm_virt.h>
// ---------------------------------------------------------------------------
// Dynamic array (stretchy-buffer style)
//
// T *buf = NULL;
// arr_push(buf, value); // appends; grows automatically
// arr_push_front(buf, value); // prepends
// arr_remove(buf, i); // removes index i, shifts tail left
// size_t n = arr_len(buf); // element count (0 if buf == NULL)
// arr_free(buf); // frees header + data
//
// The allocation layout is:
// [ Header (count, capacity) ][ T[0] ][ T[1] ] ...
// buf always points at T[0]; the Header lives one slot before it.
// ---------------------------------------------------------------------------
typedef struct {
size_t count;
code/kernel/include/kernel/ldr_image.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ldr image declarations shared by implementation files and kernel clients.
#pragma once
// ldr_image.h - minimal PE32+ loader
// Loads an already-mapped PE image and resolves its imports.
code/kernel/include/kernel/mm_phys.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel mm phys declarations shared by implementation files and kernel clients.
#pragma once
// mm_phys.h - physical memory manager (PMM)
// Bitmap allocator over 4 KB frames, seeded from MEMORY_MAP.
code/kernel/include/kernel/mm_virt.h
+5 -17
View File
@@ -1,3 +1,5 @@
// Kernel mm virt declarations shared by implementation files and kernel clients.
#pragma once
// mm_virt.h - virtual memory manager (VMM)
// Manages the single shared PML4, kernel pool, and page-granular user VM regions.
@@ -30,9 +32,7 @@ typedef enum USER_REGION_TYPE {
#define USER_REGION_GUARD_HIGH 0x00000002u
#define USER_REGION_ZERO 0x00000004u
// ---------------------------------------------------------------------------
// Page flags (x86-64 PTE bits)
// ---------------------------------------------------------------------------
// Page flags (x86-64 PTE bits).
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
@@ -40,9 +40,7 @@ typedef enum USER_REGION_TYPE {
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_NX (1ULL << 63)
// ---------------------------------------------------------------------------
// Virtual address space constants
// ---------------------------------------------------------------------------
// Virtual address space constants.
#define USER_BASE 0x0000000000001000ULL
#define USER_LIMIT 0x00007FFFFFFFF000ULL
@@ -51,9 +49,7 @@ typedef enum USER_REGION_TYPE {
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
// ---------------------------------------------------------------------------
// Initialisation
// ---------------------------------------------------------------------------
// Called by KernelMain after MmInitializePhysical.
// bootPml4Phys: the PML4 set up by the bootloader.
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
@@ -62,9 +58,7 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
void MmActivate(void);
// ---------------------------------------------------------------------------
// Page mapping
// ---------------------------------------------------------------------------
// Page mapping.
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode);
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt);
@@ -86,9 +80,7 @@ void MmFlushTlbForKernelMapping(uintptr_t virt);
int MmHandleTlbShootdownNmi(void);
// ---------------------------------------------------------------------------
// User/kernel copy boundary helpers
// ---------------------------------------------------------------------------
// MmQueryPage returns the physical address and effective PAGE_* bits for a
// mapped virtual address. The reported PAGE_USER/PAGE_WRITE bits account for
// every page-table level, not just the leaf entry.
@@ -105,20 +97,16 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size);
__declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size);
__declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size);
// ---------------------------------------------------------------------------
// Kernel pool - ring 0 only, never PAGE_USER.
// Best-fit free-list allocator, thread-safe via spinlock.
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocatePool(size_t size);
__declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// ---------------------------------------------------------------------------
// User virtual regions.
// The kernel has one shared user VA arena, but every user mapping is owned by
// a PROCESS so process teardown can reclaim leaked memory. Guard pages are
// reserved VA with no present PTE.
// ---------------------------------------------------------------------------
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t regionFlags,
USER_REGION_TYPE type, void** outBase);
code/kernel/include/kernel/net_socket.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel net socket declarations shared by implementation files and kernel clients.
#pragma once
// net_socket.h - process-owned, provider-neutral kernel socket objects.
//
// The native networking ABI lives in net_abi.h. Protocol implementation lives
// in a network stack driver registered through net_stack.h.
code/kernel/include/kernel/ob_object.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ob object declarations shared by implementation files and kernel clients.
#pragma once
// ob_object.h - executive object model
// Every resource the kernel tracks is a typed, reference-counted OBJECT.
code/kernel/include/kernel/syscall_nr.h
+3 -1
View File
@@ -1,10 +1,12 @@
// Kernel syscall nr declarations shared by implementation files and kernel clients.
#pragma once
// syscall_nr.h
// INT 0x40 calling convention, MS x64:
// RAX = syscall number
// RCX, RDX, R8, R9 = args 0-3
// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6)
// RAX = return value
// RAX = return value.
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
code/kernel/io_device.c
+29 -19
View File
@@ -1,4 +1,5 @@
// io_device.c - I/O manager: driver/device objects, stacks, PnP skeleton, disk namespace
// I/O manager. This file owns driver objects, device stacks, PnP dispatch, and the disk namespace.
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
@@ -10,10 +11,12 @@
#define MAX_DRIVER_OBJECTS 64
#define MAX_PDO_OBJECTS 128
// Implements the sv Or Unnamed operation.
static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) {
return SysSvIsEmpty(name) ? SYS_SV_LITERAL("<unnamed>") : name;
}
// Implements the bus Type Name operation.
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
switch (busType) {
case IoBusPci: return "pci";
@@ -22,15 +25,14 @@ static const char* IoBusTypeName(IO_BUS_TYPE busType) {
}
}
// -------------------------------------------------------------------------
// Driver object registry / PDO AddDevice binding
// -------------------------------------------------------------------------
// Driver object registry / PDO AddDevice binding.
static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS];
static uint32_t gDriverObjectCount;
static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS];
static uint32_t gPhysicalDeviceCount;
// Implements the driver Matches Pdo operation.
static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical)
return false;
@@ -47,6 +49,7 @@ static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT*
return false;
}
// Implements the try Add Device For Driver operation.
static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) {
if (!driver || !pdo || !driver->DriverAddDevice)
return false;
@@ -81,6 +84,7 @@ __declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
driver->Name = name;
}
// Registers driver Object.
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
if (!driver) return STATUS_INVALID_PARAMETER;
if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
@@ -100,9 +104,7 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
return STATUS_SUCCESS;
}
// -------------------------------------------------------------------------
// Device object lifetime
// -------------------------------------------------------------------------
// Device object lifetime.
static void IoDeviceDestroy(OBJECT* object) {
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
@@ -221,6 +223,7 @@ __declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
return STATUS_SUCCESS;
}
// Registers physical Device.
__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
if (!pdo || pdo->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -264,14 +267,17 @@ __declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
return STATUS_SUCCESS;
}
// Implements the reference Device Object operation.
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObReferenceObject(&device->Header);
}
// Implements the dereference Device Object operation.
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObDereferenceObject(&device->Header);
}
// Releases physical Device Registry Entry.
static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
if (!device || device->Type != DeviceObjectPhysical)
return;
@@ -288,6 +294,7 @@ static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
}
}
// Implements the unlink Device From Parent operation.
static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
if (!device || !device->Parent)
return;
@@ -309,6 +316,7 @@ static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
IoDereferenceDeviceObject(parent);
}
// Releases device.
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
if (device->DeletePending)
@@ -321,9 +329,7 @@ __declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
IoDereferenceDeviceObject(device);
}
// -------------------------------------------------------------------------
// Device stack management
// -------------------------------------------------------------------------
// Device stack management.
__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
DEVICE_OBJECT* target) {
@@ -339,6 +345,7 @@ __declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT*
return top;
}
// Implements the detach Device operation.
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
if (!source || !source->LowerDevice) return;
@@ -349,6 +356,7 @@ __declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
IoDereferenceDeviceObject(lower);
}
// Returns attached Device Reference.
__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device) {
if (!device) return (DEVICE_OBJECT*)0;
DEVICE_OBJECT* top = device;
@@ -358,9 +366,7 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT*
return top;
}
// -------------------------------------------------------------------------
// Device-stack helpers / PnP dispatch
// -------------------------------------------------------------------------
// Device-stack helpers / PnP dispatch.
__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
uint8_t count = 0;
@@ -369,6 +375,7 @@ __declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
return count ? count : 1;
}
// Implements the forward Irp To Lower operation.
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->LowerDevice)
return STATUS_NOT_SUPPORTED;
@@ -413,9 +420,7 @@ __declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
return st;
}
// -------------------------------------------------------------------------
// Disk namespace
// -------------------------------------------------------------------------
// Disk namespace.
#define MAX_DISK_DEVICES 8
#define MAX_DISK_CALLBACKS 16
@@ -438,6 +443,7 @@ static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
// Implements the notify Disk Change operation.
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i];
@@ -477,6 +483,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
return STATUS_SUCCESS;
}
// Implements the notify Disk Surprise Removed operation.
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
@@ -489,6 +496,7 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
return STATUS_SUCCESS;
}
// Implements the unregister Disk Device operation.
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
@@ -562,29 +570,31 @@ __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK ca
return STATUS_SUCCESS;
}
// Returns disk Device.
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES) return (DEVICE_OBJECT*)0;
return gDiskDevices[driveIndex];
}
// Implements the reference Disk Device operation.
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
if (dev) IoReferenceDeviceObject(dev);
return dev;
}
// Implements the disk Device Present operation.
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
}
// Implements the disk Sector Count operation.
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
return dev ? dev->u.Disk.SectorCount : 0;
}
// -------------------------------------------------------------------------
// Dispatch
// -------------------------------------------------------------------------
// Dispatch.
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DriverObject || !irp)
code/kernel/io_file.c
+36 -37
View File
@@ -1,4 +1,5 @@
// io_file.c - VFS dispatch layer
// Kernel file I/O layer. This file owns file handles, filesystem dispatch, and asynchronous request plumbing.
#include <string.h>
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
@@ -22,9 +23,7 @@ typedef struct {
static VOLUME_ENTRY gVolumes[MAX_DRIVES];
// ---------------------------------------------------------------------------
// Async file I/O worker
// ---------------------------------------------------------------------------
// Async file I/O worker.
typedef enum IO_ASYNC_FILE_OP {
IoAsyncFileRead = 0,
IoAsyncFileWrite = 1,
@@ -61,6 +60,7 @@ static SPINLOCK gAsyncFileQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gAsyncFileQueue;
static SEMAPHORE* gAsyncFileSemaphore;
// Implements the io Async File Request Destroy operation.
static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header);
@@ -101,6 +101,7 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
KeSignalEvent(req->CompletionEvent);
}
// Implements the io Async File Dequeue operation.
static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
LIST_ENTRY* node = ListPopHead(&gAsyncFileQueue);
@@ -121,6 +122,7 @@ static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
return req;
}
// Implements the io Async File Worker operation.
static void KiIoAsyncFileWorker(void* arg) {
(void)arg;
@@ -162,6 +164,7 @@ static void KiIoAsyncFileWorker(void* arg) {
}
}
// Implements the io Ensure Async File Io Started operation.
static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileInitLock);
if (gAsyncFileInitState == 2) {
@@ -337,10 +340,12 @@ __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const vo
requestOut);
}
// Implements the async File Request Object operation.
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? &req->Header : (OBJECT*)0;
}
// Cancels async File Request.
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
if (!req) return STATUS_INVALID_PARAMETER;
@@ -382,11 +387,9 @@ __declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJE
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Current Working Directory
// Stored as (CurrentDrive, drive-relative path with no trailing backslash).
// Exported as native "N:/path" so personalities can round-trip volumes.
// ---------------------------------------------------------------------------
#define CWD_MAX PROCESS_CWD_MAX
typedef struct KRESOLVED_PATH {
@@ -395,6 +398,7 @@ typedef struct KRESOLVED_PATH {
SYS_STRING_VIEW Path;
} KRESOLVED_PATH;
// Implements the format Cwd operation.
static size_t KiFormatCwd(uint8_t drive, SYS_STRING_VIEW rel, char* buf, size_t size) {
if (!buf || size == 0 || !SysSvIsValid(rel)) return 0;
if (drive > 7) drive = 0;
@@ -410,6 +414,7 @@ static size_t KiFormatCwd(uint8_t drive, SYS_STRING_VIEW rel, char* buf, size_t
return needed;
}
// Returns cwd.
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
if (!buf || size == 0) return STATUS_INVALID_PARAMETER;
@@ -423,6 +428,7 @@ __declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
return needed && needed <= size ? STATUS_SUCCESS : STATUS_BUFFER_TOO_SMALL;
}
// Implements the sv Has Drive Root operation.
static int KiSvHasDriveRoot(SYS_STRING_VIEW path) {
return path.Length >= 3 &&
path.Data[0] >= '0' && path.Data[0] <= '7' &&
@@ -449,6 +455,7 @@ static int KiAppendComponent(char* work, size_t* workLen, size_t cap,
return KiAppendBytes(work, workLen, cap, data, len);
}
// Implements the pop Component operation.
static void KiPopComponent(char* work, size_t* workLen) {
if (!work || !workLen || *workLen == 0) return;
size_t i = *workLen;
@@ -529,6 +536,7 @@ static KSTATUS KiResolvePathV(SYS_STRING_VIEW input, KRESOLVED_PATH* out) {
return STATUS_SUCCESS;
}
// Updates cwd.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path) {
if (path.Length && !path.Data) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -571,9 +579,7 @@ __declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Registration
// ---------------------------------------------------------------------------
// Registration.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
IO_FILESYSTEM *fs,
const IO_VOLUME_PARAMS *vol) {
@@ -586,6 +592,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
return STATUS_SUCCESS;
}
// Implements the unregister Filesystem operation.
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return STATUS_INVALID_PARAMETER;
@@ -596,6 +603,7 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
return STATUS_SUCCESS;
}
// Implements the unregister Filesystems For Disk operation.
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
KSTATUS last = STATUS_NOT_FOUND;
for (uint8_t i = 0; i < MAX_DRIVES; i++) {
@@ -607,23 +615,20 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex)
return last;
}
// Implements the filesystem Present operation.
__declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return false;
return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
}
// ---------------------------------------------------------------------------
// Destructor
// ---------------------------------------------------------------------------
// Destructor.
static void KiFileDestroy(OBJECT* obj) {
FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx);
}
// ---------------------------------------------------------------------------
// IoOpenFile
// ---------------------------------------------------------------------------
// IoOpenFile.
__declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out) {
if ((!path.Data && path.Length) || !out) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -640,7 +645,8 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
s = fs->Open(fs, resolved.Path, &fileCtx);
if (!KSUCCESS(s)) return s;
FILE_OBJECT* f = (FILE_OBJECT *)ObAllocateObject(ObjectTypeFile /* reuse slot */,
// File objects reuse the generic object slot because the handle table only needs the shared header contract here.
FILE_OBJECT* f = (FILE_OBJECT *)ObAllocateObject(ObjectTypeFile,
sizeof(FILE_OBJECT));
if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; }
f->Header.Destroy = KiFileDestroy;
@@ -654,10 +660,8 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
void* buf, size_t size,
size_t* bytesRead) {
@@ -700,9 +704,7 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
return s;
}
// ---------------------------------------------------------------------------
// IoReadFile - sequential read at current offset
// ---------------------------------------------------------------------------
// IoReadFile - sequential read at current offset.
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead) {
if (bytesRead) *bytesRead = 0;
@@ -721,9 +723,7 @@ __declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
return s;
}
// ---------------------------------------------------------------------------
// IoWriteFile - sequential write at current offset
// ---------------------------------------------------------------------------
// IoWriteFile - sequential write at current offset.
__declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
size_t size, size_t* bytesWritten) {
if (bytesWritten) *bytesWritten = 0;
@@ -742,9 +742,7 @@ __declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
return s;
}
// ---------------------------------------------------------------------------
// IoCreateFile - create a new file and return it open for writing
// ---------------------------------------------------------------------------
// IoCreateFile - create a new file and return it open for writing.
__declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** out) {
if ((!path.Data && path.Length) || !out || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -775,9 +773,7 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// IoDeleteFile - delete a file by path (does not require an open handle)
// ---------------------------------------------------------------------------
// IoDeleteFile - delete a file by path (does not require an open handle).
__declspec(dllexport) KSTATUS IoDeleteFile(SYS_STRING_VIEW path) {
if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -793,6 +789,7 @@ __declspec(dllexport) KSTATUS IoDeleteFile(SYS_STRING_VIEW path) {
return fs->Delete(fs, resolved.Path);
}
// Implements the rename File operation.
__declspec(dllexport) KSTATUS IoRenameFile(SYS_STRING_VIEW oldPath, SYS_STRING_VIEW newPath) {
if ((!oldPath.Data && oldPath.Length) || (!newPath.Data && newPath.Length) ||
oldPath.Length == 0 || newPath.Length == 0) return STATUS_INVALID_PARAMETER;
@@ -813,6 +810,7 @@ __declspec(dllexport) KSTATUS IoRenameFile(SYS_STRING_VIEW oldPath, SYS_STRING_V
return fs->Rename(fs, oldResolved.Path, newResolved.Path);
}
// Creates dir.
__declspec(dllexport) KSTATUS IoCreateDir(SYS_STRING_VIEW path) {
if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -828,6 +826,7 @@ __declspec(dllexport) KSTATUS IoCreateDir(SYS_STRING_VIEW path) {
return fs->CreateDir(fs, resolved.Path);
}
// Releases dir.
__declspec(dllexport) KSTATUS IoRemoveDir(SYS_STRING_VIEW path) {
if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -843,9 +842,7 @@ __declspec(dllexport) KSTATUS IoRemoveDir(SYS_STRING_VIEW path) {
return fs->RemoveDir(fs, resolved.Path);
}
// ---------------------------------------------------------------------------
// IoTruncateFile - resize the file; updates the cached Size on success
// ---------------------------------------------------------------------------
// IoTruncateFile - resize the file; updates the cached Size on success.
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize) {
if (!file) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -861,7 +858,6 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize
return s;
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) {
if (!file) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
@@ -870,6 +866,7 @@ __declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) {
return STATUS_SUCCESS;
}
// Returns file Size.
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
if (!file) return 0;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
@@ -878,19 +875,19 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
return size;
}
// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
if (file) ObDereferenceObject(&file->Header);
}
// ---------------------------------------------------------------------------
// Directory enumeration
// ---------------------------------------------------------------------------
// Directory enumeration.
static void KiDirDestroy(OBJECT* obj) {
DIR_OBJECT* d = (DIR_OBJECT*)obj; // Header is first member
if (d->Fs && d->DirCtx) d->Fs->CloseDir(d->Fs, d->DirCtx);
}
// Opens dir.
__declspec(dllexport) KSTATUS IoOpenDir(SYS_STRING_VIEW dirPath, DIR_OBJECT** out) {
if (!out) return STATUS_INVALID_PARAMETER;
if (dirPath.Length && !dirPath.Data) return STATUS_INVALID_PARAMETER;
@@ -937,6 +934,7 @@ __declspec(dllexport) KSTATUS IoOpenDir(SYS_STRING_VIEW dirPath, DIR_OBJECT** ou
return STATUS_SUCCESS;
}
// Returns dir.
__declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) {
if (!dir || !entryOut) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -944,6 +942,7 @@ __declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) {
return dir->Fs->ReadDir(dir->Fs, dir->DirCtx, entryOut);
}
// Releases dir.
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir) {
if (dir) ObDereferenceObject(&dir->Header);
}
code/kernel/ke_apc.c
+13 -1
View File
@@ -1,4 +1,5 @@
// ke_apc.c - kernel APC infrastructure
// Kernel APC infrastructure. This file owns APC queuing, delivery, and user-mode return transitions.
#include <kernel/ke_apc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
@@ -9,6 +10,7 @@
#include <string.h>
#include <intrin.h>
// Initializes apc State.
void KiInitializeApcState(THREAD* thread) {
if (!thread) return;
LIST_INIT(&thread->ApcQueue);
@@ -88,6 +90,7 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
return true;
}
// Implements the enter Critical Region operation.
__declspec(dllexport) void KeEnterCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
@@ -97,6 +100,7 @@ __declspec(dllexport) void KeEnterCriticalRegion(void) {
KeReleaseSpinlock(&thread->ApcLock, flags);
}
// Implements the leave Critical Region operation.
__declspec(dllexport) void KeLeaveCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
@@ -111,12 +115,14 @@ __declspec(dllexport) void KeLeaveCriticalRegion(void) {
KiDeliverKernelApcs();
}
// Implements the are Kernel Apcs Disabled operation.
__declspec(dllexport) bool KeAreKernelApcsDisabled(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return true;
return thread->KernelApcDisable != 0;
}
// Implements the pop Kernel Apc operation.
static KAPC* KiPopKernelApc(THREAD* thread) {
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
LIST_ENTRY* node = ListPopHead(&thread->ApcQueue);
@@ -129,6 +135,7 @@ static KAPC* KiPopKernelApc(THREAD* thread) {
return CONTAINING_RECORD(node, KAPC, Link);
}
// Implements the deliver Kernel Apcs operation.
void KiDeliverKernelApcs(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread)
@@ -178,6 +185,7 @@ void KiDeliverKernelApcs(void) {
}
// Updates user Apc Dispatcher.
__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return STATUS_FAILURE;
@@ -221,6 +229,7 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
return STATUS_SUCCESS;
}
// Implements the pop User Apc operation.
static KUSER_APC* KiPopUserApc(THREAD* thread) {
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue);
@@ -233,6 +242,7 @@ static KUSER_APC* KiPopUserApc(THREAD* thread) {
return CONTAINING_RECORD(node, KUSER_APC, Link);
}
// Implements the check For User Apc Delivery operation.
void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
if (!frame || !KeTrapFrameFromUser(frame))
return;
@@ -288,6 +298,7 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
MmFreePool(apc);
}
// Implements the rundown Apc State operation.
void KiRundownApcState(THREAD* thread) {
if (!thread) return;
@@ -322,6 +333,7 @@ void KiRundownApcState(THREAD* thread) {
thread->UserApcDeliveryActive = 0;
}
// Implements the user Apc Return operation.
__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* userFrame) {
SYS_USER_APC_FRAME_K frameCopy;
KSTATUS s = CopyFromUser(&frameCopy, userFrame, sizeof(frameCopy));
code/kernel/ke_cpu.c
+9 -1
View File
@@ -1,4 +1,5 @@
// ke_cpu.c - bootstrap/per-CPU state plumbing
// CPU-local kernel state. This file owns bootstrap and per-CPU bookkeeping shared by scheduler and SMP code.
#include <stdint.h>
#include <string.h>
#include <intrin.h>
@@ -16,6 +17,7 @@ static volatile long gKernelGsReady = 0;
extern KCPU* KeReadCurrentCpuFromGs(void);
// Loads kernel Gs Base.
void KeLoadKernelGsBase(KCPU* cpu) {
if (!cpu)
cpu = &gCpus[0];
@@ -25,6 +27,7 @@ void KeLoadKernelGsBase(KCPU* cpu) {
gKernelGsReady = 1;
}
// Updates user Gs Base.
void KeSetUserGsBase(uintptr_t userGsBase) {
// While running in the kernel, GS_BASE is the current KCPU. The user TEB
// base is parked in IA32_KERNEL_GS_BASE so swapgs can exchange it with the
@@ -32,10 +35,12 @@ void KeSetUserGsBase(uintptr_t userGsBase) {
__writemsr(IA32_KERNEL_GS_BASE, (uint64_t)userGsBase);
}
// Returns user Gs Base.
uintptr_t KeGetUserGsBase(void) {
return (uintptr_t)__readmsr(IA32_KERNEL_GS_BASE);
}
// Initializes bootstrap Cpu.
void KeInitializeBootstrapCpu(void) {
memset(gCpus, 0, sizeof(gCpus));
@@ -73,6 +78,7 @@ void KeInitializeBootstrapCpu(void) {
KeSetUserGsBase(0);
}
// Returns current Cpu.
KCPU* KeGetCurrentCpu(void) {
if (gKernelGsReady) {
KCPU* cpu = KeReadCurrentCpuFromGs();
@@ -84,12 +90,14 @@ KCPU* KeGetCurrentCpu(void) {
return &gCpus[0];
}
// Returns cpu By Index.
KCPU* KeGetCpuByIndex(uint32_t index) {
if (index >= KE_MAX_CPUS)
return (KCPU*)0;
return &gCpus[index];
}
// Returns current Cpu Index.
uint32_t KeGetCurrentCpuIndex(void) {
return KeGetCurrentCpu()->CpuIndex;
}
code/kernel/ke_debug.c
+34 -8
View File
@@ -1,4 +1,5 @@
// ke_debug.c - debugging utilities
// Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.
#include <stdarg.h>
#include <stdint.h>
#include <string.h>
@@ -31,6 +32,7 @@ static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile long gDebugLogFlushing;
// Implements the debug Log Append Locked operation.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
return;
@@ -53,6 +55,7 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
gDebugLogUsed += len;
}
// Implements the debug Console Write operation.
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
return;
@@ -64,12 +67,14 @@ static volatile long gPanicInProgress = 0;
static volatile uintptr_t gPanicOwnerToken = 0;
static volatile long gPanicStoppedCpuCount = 0;
// Implements the panic Secondary Halt operation.
static _Noreturn void KiPanicSecondaryHalt(void) {
_disable();
_InterlockedIncrement(&gPanicStoppedCpuCount);
for (;;) __halt();
}
// Implements the panic Current Token operation.
static uintptr_t KiPanicCurrentToken(void) {
THREAD* thread = KeGetCurrentThread();
if (thread)
@@ -80,15 +85,18 @@ static uintptr_t KiPanicCurrentToken(void) {
return 1u;
}
// Implements the panic Is Active operation.
static int KiPanicIsActive(void) {
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
}
// Implements the panic Current Cpu Owns Report operation.
static int KiPanicCurrentCpuOwnsReport(void) {
uintptr_t token = KiPanicCurrentToken();
return token != 0 && gPanicOwnerToken == token;
}
// Implements the begin Fatal Report operation.
static void KiBeginFatalReport(void) {
_disable();
uintptr_t token = KiPanicCurrentToken();
@@ -114,11 +122,13 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
return 0;
}
// Implements the kernel Put Char operation.
void KiKernelPutChar(char c) {
if (PrintChar)
PrintChar(c, 0xCCCCCC, 0x012456);
}
// Initializes debug.
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
gDebugFlags = flags;
gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
@@ -136,10 +146,12 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
}
}
// Returns debug Flags.
uint32_t KeGetDebugFlags(void) {
return gDebugFlags;
}
// Updates debug Console Writer.
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
if (!PrintPtr)
return;
@@ -147,6 +159,7 @@ void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
printf_set_putchar(KiKernelPutChar);
}
// Implements the dbg Print operation.
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled || !fmt) return;
@@ -179,6 +192,7 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
KeReleaseSpinlock(&gDebugPrintLock, flags);
}
// Flushes debug Log To File.
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
@@ -228,7 +242,7 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// KePanic
// KePanic.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
KiBeginFatalReport();
@@ -276,13 +290,11 @@ __declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
PrintChar(buf[i], 0xCCCCCC, 0x012456);
}
// ---------------------------------------------------------------------------
// KeBugCheckProcess - user-mode fault handler
//
// Resolves the faulting RIP to a module name, prints a yellow crash banner,
// kills the offending process, then exits the current thread so the scheduler
// can continue running everything else.
// ---------------------------------------------------------------------------
// Return the short exception name for a CPU exception vector.
// Keep this table complete for vectors 0..31 so faults never collapse into
@@ -326,6 +338,7 @@ static const char* KiFaultVectorName(int vector) {
}
}
// Implements the fault Has Error Code operation.
static int KiFaultHasErrorCode(int vector) {
switch (vector) {
case 8: // #DF
@@ -374,10 +387,12 @@ static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
return "unknown";
}
// Implements the resolve Module operation.
static const char* KiResolveModule(PROCESS* proc, uintptr_t addr) {
return KiResolveModuleEx(proc, addr, (uintptr_t*)0, (size_t*)0);
}
// Prints address.
static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
uintptr_t base = 0;
size_t size = 0;
@@ -396,6 +411,7 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
}
}
// Implements the user Read U64 operation.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
if (!out) return 0;
if (!addr) return 0;
@@ -405,6 +421,7 @@ static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
return 1;
}
// Implements the user Read U8 operation.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
if (!out) return 0;
if (!addr) return 0;
@@ -414,6 +431,7 @@ static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
return 1;
}
// Dumps bytes.
static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
if (!center) {
printf("%s: <null>\n", label);
@@ -438,6 +456,7 @@ static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
}
}
// Implements the user Read I32 operation.
static int KiUserReadI32(uintptr_t addr, int32_t* out) {
if (!out) return 0;
if (!addr) return 0;
@@ -447,6 +466,7 @@ static int KiUserReadI32(uintptr_t addr, int32_t* out) {
return 1;
}
// Dumps indirect Call Slot.
static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
if (!ret) return;
@@ -461,12 +481,14 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
if (KiUserReadU8(ret - 6, &b0) &&
KiUserReadU8(ret - 5, &b1) &&
b0 == 0xff && b1 == 0x15 &&
// Implements the user Read I32 operation.
KiUserReadI32(ret - 4, &disp)) {
insn = ret - 6;
} else if (KiUserReadU8(ret - 7, &b0) &&
KiUserReadU8(ret - 6, &b1) &&
KiUserReadU8(ret - 5, &b2) &&
b0 == 0x48 && b1 == 0xff && b2 == 0x15 &&
// Implements the user Read I32 operation.
KiUserReadI32(ret - 4, &disp)) {
insn = ret - 7;
} else {
@@ -490,6 +512,7 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
KiDumpBytes(proc, "Bytes near indirect slot", slot);
}
// Dumps modules.
static void KiDumpModules(PROCESS* proc) {
if (!proc) return;
@@ -513,6 +536,7 @@ static void KiDumpModules(PROCESS* proc) {
}
}
// Dumps user Stack.
static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
printf("User stack @ %016llx:\n", (unsigned long long)rsp);
if (!rsp) {
@@ -548,6 +572,7 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
}
}
// Dumps fault Frame.
static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
if (!f) return;
@@ -586,6 +611,7 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
KiDumpUserStack(proc, (uintptr_t)f->Rsp);
}
// Prints crash Char.
static void KiPrintCrashChar(char c) {
if (PrintChar)
PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic
@@ -621,6 +647,7 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
(unsigned long long)errCode);
}
// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread) {
printf_set_putchar(KiKernelPutChar);
@@ -640,6 +667,7 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
KiFinishFaultedProcess(proc, thread);
}
// Implements the fault Dispatch Frame operation.
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
// the interrupted context on a secondary CPU once another CPU owns panic.
@@ -718,13 +746,11 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
KiFinishFaultedProcess(proc, thread);
}
// ---------------------------------------------------------------------------
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
//
// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and
// call here. We check the CS RPL to decide: kernel fault → panic and halt,
// user fault → KeBugCheckProcess (kill process, resume scheduler).
// ---------------------------------------------------------------------------
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
uintptr_t errCode, uintptr_t cr2,
uintptr_t cs) {
code/kernel/ke_display.c
+48 -6
View File
@@ -1,9 +1,4 @@
// ke_display.c - kernel display manager
//
// The display manager owns the primary display device, cursor policy, dirty
// rectangles, and the kernel/user screen API. The first provider is a boot
// framebuffer backend over the HAL GOP framebuffer; later GPU drivers register
// DISPLAY_DEVICE providers here.
// Kernel display manager. This file owns provider registration, framebuffer routing, dirty rectangles, text console state, and cursor delegation.
#include <stdint.h>
#include <stdbool.h>
@@ -96,28 +91,33 @@ static const DISPLAY_DEVICE_OPS gBootFbOps = {
};
// Checks whether a primary display provider is available.
static bool KiDisplayHasDevice(void) {
return gPrimaryDisplay && gPrimaryDisplay->Ops;
}
// Checks whether the active provider expects the manager to draw the cursor into the framebuffer.
static bool KiDisplayUsesSoftwareCursor(void) {
return KiDisplayHasDevice() &&
(gPrimaryDisplay->Caps & DISPLAY_CAP_SOFTWARE_CURSOR) &&
gPrimaryDisplay->Ops->ShowCursor;
}
// Checks whether the active provider owns cursor presentation in hardware or device firmware.
static bool KiDisplayUsesHardwareCursor(void) {
return KiDisplayHasDevice() &&
(gPrimaryDisplay->Caps & DISPLAY_CAP_HARDWARE_CURSOR) &&
gPrimaryDisplay->Ops->ShowCursor;
}
// Checks whether the active provider exposes any cursor-control path.
static bool KiDisplayHasCursorOps(void) {
return KiDisplayHasDevice() &&
(gPrimaryDisplay->Caps & (DISPLAY_CAP_SOFTWARE_CURSOR | DISPLAY_CAP_HARDWARE_CURSOR)) &&
gPrimaryDisplay->Ops->ShowCursor;
}
// Clips a rectangle to a display mode so providers never see negative or out-of-bounds regions.
static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) {
if (!mode || !rect || rect->W <= 0 || rect->H <= 0)
return false;
@@ -141,12 +141,14 @@ static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) {
return true;
}
// Implements the display Dirty All operation.
static void KiDisplayDirtyAll(void) {
gDisplayStats.DirtyAllCount++;
gDirtyAll = true;
gDirtyRectCount = 0;
}
// Implements the display Add Dirty operation.
static void KiDisplayAddDirty(int x, int y, int w, int h) {
if (!KiDisplayHasDevice() || gDirtyAll || w <= 0 || h <= 0)
return;
@@ -174,6 +176,7 @@ static void KiDisplayAddDirty(int x, int y, int w, int h) {
gDirtyRects[gDirtyRectCount++] = rect;
}
// Implements the display Flush Dirty operation.
static KSTATUS KiDisplayFlushDirty(void) {
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Present)
return STATUS_SUCCESS;
@@ -207,6 +210,7 @@ static KSTATUS KiDisplayFlushDirty(void) {
return s;
}
// Implements the display Hide Cursor For Draw operation.
static void KiDisplayHideCursorForDraw(void) {
if (!gCursorDrawn || !KiDisplayUsesSoftwareCursor())
return;
@@ -215,6 +219,7 @@ static void KiDisplayHideCursorForDraw(void) {
gCursorDrawn = false;
}
// Implements the display Restore Cursor After Draw operation.
static void KiDisplayRestoreCursorAfterDraw(void) {
if (!gCursorVisible || gCursorDrawn || !KiDisplayHasCursorOps())
return;
@@ -226,11 +231,13 @@ static void KiDisplayRestoreCursorAfterDraw(void) {
gCursorDrawn = true;
}
// Implements the display Begin Draw operation.
static void KiDisplayBeginDraw(void) {
if (gDrawDepth++ == 0)
KiDisplayHideCursorForDraw();
}
// Implements the display End Draw operation.
static KSTATUS KiDisplayEndDraw(void) {
KSTATUS s = STATUS_SUCCESS;
if (gDrawDepth > 0)
@@ -243,6 +250,7 @@ static KSTATUS KiDisplayEndDraw(void) {
return s;
}
// Implements the display Draw Blit No Cursor operation.
static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint32_t* pixels) {
if (!pixels || w <= 0 || h <= 0)
return STATUS_INVALID_PARAMETER;
@@ -258,6 +266,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3
return s;
}
// Implements the display Fill Rect No Cursor operation.
static KSTATUS KiDisplayFillRectNoCursor(int x, int y, int w, int h, uint32_t color) {
if (w <= 0 || h <= 0)
return STATUS_INVALID_PARAMETER;
@@ -273,6 +282,7 @@ static KSTATUS KiDisplayFillRectNoCursor(int x, int y, int w, int h, uint32_t co
return s;
}
// Implements the display Copy Rect No Cursor operation.
static KSTATUS KiDisplayCopyRectNoCursor(int dstX, int dstY, int srcX, int srcY, int w, int h) {
if (w <= 0 || h <= 0)
return STATUS_INVALID_PARAMETER;
@@ -288,10 +298,12 @@ static KSTATUS KiDisplayCopyRectNoCursor(int dstX, int dstY, int srcX, int srcY,
return s;
}
// Implements the display Console Index operation.
static size_t KiDisplayConsoleIndex(int col, int row) {
return (size_t)row * DISPLAY_CONSOLE_MAX_COLS + (size_t)col;
}
// Implements the display Console Color operation.
static uint32_t KiDisplayConsoleColor(uint8_t index) {
static const uint32_t colors[16] = {
0x00000000u, 0x000000AAu, 0x0000AA00u, 0x0000AAAAu,
@@ -302,14 +314,17 @@ static uint32_t KiDisplayConsoleColor(uint8_t index) {
return colors[index & 0x0Fu];
}
// Implements the display Console Fg operation.
static uint32_t KiDisplayConsoleFg(uint16_t attr) {
return KiDisplayConsoleColor((uint8_t)(attr & 0x0Fu));
}
// Implements the display Console Bg operation.
static uint32_t KiDisplayConsoleBg(uint16_t attr) {
return KiDisplayConsoleColor((uint8_t)((attr >> 4) & 0x0Fu));
}
// Implements the display Get Console Grid Locked operation.
static bool KiDisplayGetConsoleGridLocked(int* outCols, int* outRows) {
DISPLAY_MODE mode;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode)
@@ -329,6 +344,7 @@ static bool KiDisplayGetConsoleGridLocked(int* outCols, int* outRows) {
return true;
}
// Implements the display Console Reset Cells Locked operation.
static void KiDisplayConsoleResetCellsLocked(void) {
for (size_t i = 0; i < DISPLAY_CONSOLE_MAX_CELLS; i++) {
gConsoleCells[i] = ' ';
@@ -337,6 +353,7 @@ static void KiDisplayConsoleResetCellsLocked(void) {
gConsoleAttribute = DISPLAY_CONSOLE_DEFAULT_ATTR;
}
// Implements the display Console Scroll Cells Locked operation.
static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines) {
if (cols <= 0 || rows <= 0 || cellLines <= 0)
return;
@@ -370,6 +387,7 @@ static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines)
}
}
// Implements the display Draw Char Pixels Locked operation.
static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int x, int y, uint32_t fg, uint32_t bg) {
uint32_t glyphPixels[FONT_W * FONT_H];
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u);
@@ -383,6 +401,7 @@ static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int x, int y, uint32_t fg,
return KiDisplayDrawBlitNoCursor(x, y, FONT_W, FONT_H, glyphPixels);
}
// Implements the display Draw Console Cell Locked operation.
static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t attr, bool updateCell) {
if (updateCell) {
size_t i = KiDisplayConsoleIndex(col, row);
@@ -395,6 +414,7 @@ static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t
KiDisplayConsoleBg(attr));
}
// Implements the display Serial Write Char operation.
static void KiDisplaySerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n')
@@ -402,6 +422,7 @@ static void KiDisplaySerialWriteChar(char c) {
__outbyte(0x3F8, (uint8_t)c);
}
// Implements the display Initialize operation.
void KeDisplayInitialize(const HAL_API* hal) {
gDisplayHal = hal;
gDisplayList = NULL;
@@ -498,6 +519,7 @@ DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
return result;
}
// Implements the display Seed New Primary operation.
static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE* newDevice) {
if (!oldDevice || !newDevice || oldDevice == newDevice)
return STATUS_SUCCESS;
@@ -540,6 +562,7 @@ DISPLAY_API void KeDisplaySuspendConsole(bool suspend) {
_InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, suspend ? 1 : 0);
}
// Implements the display Output Suspended operation.
static bool KiDisplayOutputSuspended(void) {
return _InterlockedCompareExchange((volatile long*)&gDisplayConsoleSuspended, 0, 0) != 0;
}
@@ -644,6 +667,7 @@ DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color
return KSUCCESS(s) ? ps : s;
}
// Implements the display Clear Locked operation.
static KSTATUS KiDisplayClearLocked(uint32_t color) {
DISPLAY_MODE mode;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode)
@@ -675,6 +699,7 @@ DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, in
return KSUCCESS(s) ? ps : s;
}
// Implements the display Scroll Locked operation.
static KSTATUS KiDisplayScrollLocked(int lines) {
if (lines <= 0)
return STATUS_SUCCESS;
@@ -705,6 +730,7 @@ DISPLAY_API KSTATUS KeDisplayScroll(int lines) {
return s;
}
// Implements the display Put Char Locked operation.
static KSTATUS KiDisplayPutCharLocked(char c, int x, int y, uint32_t fg, uint32_t bg) {
KiDisplayBeginDraw();
KSTATUS s = KiDisplayDrawCharPixelsLocked(c, x, y, fg, bg);
@@ -789,6 +815,7 @@ DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
return STATUS_SUCCESS;
}
// Implements the display Debug Print Char operation.
void KeDisplayDebugPrintChar(char c, uint32_t fg, uint32_t bg) {
(void)KeDisplayPrintChar(c, fg, bg);
}
@@ -1069,10 +1096,12 @@ DISPLAY_API KSTATUS KeDisplayShowCursor(bool show) {
return s;
}
// Implements the boot Fb Context operation.
static BOOTFB_CONTEXT* BootFbContext(DISPLAY_DEVICE* device) {
return device ? (BOOTFB_CONTEXT*)device->DriverContext : NULL;
}
// Implements the boot Fb Get Mode operation.
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !outMode)
@@ -1085,6 +1114,7 @@ static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
return STATUS_SUCCESS;
}
// Implements the boot Fb Blit operation.
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0)
@@ -1114,6 +1144,7 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co
return STATUS_SUCCESS;
}
// Implements the boot Fb Fill Rect operation.
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
@@ -1139,6 +1170,7 @@ static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h
return STATUS_SUCCESS;
}
// Implements the boot Fb Copy Rect operation.
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
@@ -1177,6 +1209,7 @@ static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int sr
return STATUS_SUCCESS;
}
// Implements the boot Fb Read Rect operation.
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
@@ -1201,6 +1234,7 @@ static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h
return STATUS_SUCCESS;
}
// Implements the boot Fb Present operation.
static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer)
@@ -1231,12 +1265,14 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects,
return STATUS_SUCCESS;
}
// Implements the boot Fb Sync operation.
static KSTATUS BootFbSync(DISPLAY_DEVICE* device) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (fb) fb->Stats.SyncCount++;
return STATUS_SUCCESS;
}
// Implements the boot Fb Set Cursor Image operation.
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
(void)device;
(void)width;
@@ -1247,6 +1283,7 @@ static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int heigh
return STATUS_NOT_SUPPORTED;
}
// Implements the boot Fb Set Cursor Position operation.
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (fb) fb->Stats.CursorMoveCount++;
@@ -1255,6 +1292,7 @@ static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
return STATUS_SUCCESS;
}
// Implements the display Blend Pixel operation.
static inline uint32_t KiDisplayBlendPixel(uint32_t dst, uint32_t src) {
uint8_t a = (uint8_t)(src >> 24);
if (a == 0) return dst;
@@ -1273,6 +1311,7 @@ static inline uint32_t KiDisplayBlendPixel(uint32_t dst, uint32_t src) {
return ((uint32_t)rr << 16) | ((uint32_t)gg << 8) | bb;
}
// Implements the boot Fb Restore Cursor operation.
static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
if (!fb || !fb->Framebuffer || !fb->Backbuffer || gCursorDrawX < 0)
return;
@@ -1291,6 +1330,7 @@ static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
gCursorDrawY = -1;
}
// Implements the boot Fb Draw Cursor operation.
static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int x, int y) {
if (!fb || !fb->Framebuffer || !fb->Backbuffer)
return;
@@ -1315,6 +1355,7 @@ static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int x, int y) {
}
}
// Implements the boot Fb Show Cursor operation.
static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !fb->Backbuffer)
@@ -1330,6 +1371,7 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
return STATUS_SUCCESS;
}
// Implements the boot Fb Query Stats operation.
static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !outStats)
code/kernel/ke_dpc.c
+5 -1
View File
@@ -1,4 +1,5 @@
// ke_dpc.c - deferred procedure calls
// Deferred procedure call infrastructure. This file owns DPC queuing and execution outside hard IRQ context.
#include <kernel/ke_dpc.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_debug.h>
@@ -9,6 +10,7 @@ static SPINLOCK gDpcLock;
static volatile long gDpcInitialized = 0;
static volatile long gDpcActive = 0;
// Initializes dpc System.
void KeInitializeDpcSystem(void) {
LIST_INIT(&gDpcQueue);
gDpcLock = (SPINLOCK)SPINLOCK_INIT;
@@ -16,6 +18,7 @@ void KeInitializeDpcSystem(void) {
gDpcInitialized = 1;
}
// Implements the ensure Dpc Initialized operation.
static void KiEnsureDpcInitialized(void) {
if (gDpcInitialized)
return;
@@ -60,6 +63,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
return true;
}
// Implements the drain Dpc Queue operation.
void KiDrainDpcQueue(void) {
if (!gDpcInitialized)
return;
code/kernel/ke_event.c
+7 -1
View File
@@ -1,4 +1,5 @@
// ke_event.c - synchronisation events
// Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
#include <stdbool.h>
#include <kernel/ke_event.h>
#include <kernel/ke_thread.h>
@@ -7,8 +8,10 @@
#include <kernel/ke_apc.h>
#include <kernel/ob_object.h>
// Implements the event Destroy operation.
static void KiEventDestroy(OBJECT* obj) { (void)obj; }
// Implements the event Create operation.
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
if (!out) return STATUS_INVALID_PARAMETER;
EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
@@ -22,6 +25,7 @@ __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialSta
return STATUS_SUCCESS;
}
// Signals event.
__declspec(dllexport) void KeSignalEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
@@ -52,6 +56,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
KeReleaseSpinlock(&event->Lock, flags);
}
// Clears event.
__declspec(dllexport) void KeClearEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
@@ -59,6 +64,7 @@ __declspec(dllexport) void KeClearEvent(EVENT* event) {
KeReleaseSpinlock(&event->Lock, flags);
}
// Waits for for Event.
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
if (!event) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
code/kernel/ke_idt.c
+18 -15
View File
@@ -1,5 +1,5 @@
// ke_idt.c - GDT, IDT, and TSS initialisation
//
// Kernel descriptor-table setup. This file owns GDT, IDT, and TSS initialization.
// GDT layout (matches segment selectors in ke_switch.asm / ke_syscall.asm):
// 0x00 null
// 0x08 ring-0 code (64-bit)
@@ -7,13 +7,13 @@
// 0x18 ring-3 code (64-bit, DPL=3)
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x28 TSS low
// 0x30 TSS high
// 0x30 TSS high.
#include <stdint.h>
#include <string.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_cpu.h>
// External assembly procedures
// External assembly procedures.
extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
extern void KeFlushIdt(void* pIdtr);
@@ -76,9 +76,7 @@ extern void KeFaultRSV31(void);
#define NUM_IRQS 24
// ---------------------------------------------------------------------------
// GDT
// ---------------------------------------------------------------------------
// GDT.
static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
uint8_t access, uint8_t flags) {
gdt[idx] =
@@ -90,6 +88,7 @@ static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
((uint64_t)(base >> 24 & 0xFF)<< 56);
}
// Implements the gdt Set Tss operation.
static void KiGdtSetTss(uint64_t* gdt, int idx, uintptr_t base, uint32_t limit) {
// TSS descriptor is 16 bytes (two GDT slots).
gdt[idx] = 0;
@@ -105,6 +104,7 @@ static void KiGdtSetTss(uint64_t* gdt, int idx, uintptr_t base, uint32_t limit)
gdt[idx+1] = (base >> 32);
}
// Initializes gdt For Cpu.
void KeInitializeGdtForCpu(KCPU* cpu) {
if (!cpu) return;
@@ -113,15 +113,15 @@ void KeInitializeGdtForCpu(KCPU* cpu) {
memset(gdt, 0, sizeof(cpu->Gdt));
// 0x00: Null
// 0x00: Null.
KiGdtSetEntry(gdt, 0, 0, 0, 0, 0);
// 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit)
// 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit).
KiGdtSetEntry(gdt, 1, 0, 0xFFFFF, 0x9A, 0xA);
// 0x10: Ring-0 Data
// 0x10: Ring-0 Data.
KiGdtSetEntry(gdt, 2, 0, 0xFFFFF, 0x92, 0xC);
// 0x18: Ring-3 Code
// 0x18: Ring-3 Code.
KiGdtSetEntry(gdt, 3, 0, 0xFFFFF, 0xFA, 0xA);
// 0x20: Ring-3 Data
// 0x20: Ring-3 Data.
KiGdtSetEntry(gdt, 4, 0, 0xFFFFF, 0xF2, 0xC);
// 0x28: CPU-local TSS (occupies slots 0x28 and 0x30).
@@ -135,6 +135,7 @@ void KeInitializeGdtForCpu(KCPU* cpu) {
cpu->Gdtr.Base = (uint64_t)cpu->Gdt;
}
// Loads gdt For Current Cpu.
void KeLoadGdtForCurrentCpu(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
KeFlushGdt(&cpu->Gdtr, 0x08, 0x10, 0x28);
@@ -145,21 +146,21 @@ void KeLoadGdtForCurrentCpu(KCPU* cpu) {
KeLoadKernelGsBase(cpu);
}
// Initializes gdt.
void KeInitializeGdt(void) {
KCPU* cpu = KeGetCurrentCpu();
KeInitializeGdtForCpu(cpu);
KeLoadGdtForCurrentCpu(cpu);
}
// Updates current Tss Rsp0.
void KeSetCurrentTssRsp0(uintptr_t rsp0) {
TSS64* tss = KeGetCurrentTss();
if (tss)
tss->RSP0 = rsp0;
}
// ---------------------------------------------------------------------------
// IDT
// ---------------------------------------------------------------------------
// IDT.
#define IDT_ENTRIES 256
#pragma pack(push, 1)
@@ -191,6 +192,7 @@ static void KiIdtSetGate(int vec, uintptr_t handler,
gIdt[vec].MustBeZero = 0;
}
// Initializes idt.
void KeInitializeIdt(void) {
memset(gIdt, 0, sizeof(gIdt));
@@ -231,6 +233,7 @@ void KeInitializeIdt(void) {
KeLoadIdtForCurrentCpu();
}
// Loads idt For Current Cpu.
void KeLoadIdtForCurrentCpu(void) {
IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt };
KeFlushIdt(&idtr);
code/kernel/ke_input.c
+26 -1
View File
@@ -1,4 +1,5 @@
// ke_input.c - kernel-owned canonical input core
// Kernel input core. This file owns canonical keyboard and mouse state shared by input drivers and syscalls.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
@@ -76,6 +77,7 @@ static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Implements the input Queue Key operation.
static void KiInputQueueKey(INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
@@ -85,11 +87,13 @@ static void KiInputQueueKey(INPUT_KEY key, char ch) {
gKeyTail = next;
}
// Implements the input Queue Has Space operation.
static bool KiInputQueueHasSpace(void) {
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
return next != gKeyHead;
}
// Implements the input Key Is Repeatable operation.
static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return false;
@@ -110,18 +114,21 @@ static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
}
}
// Implements the input Now Ticks operation.
static uint64_t KiInputNowTicks(void) {
if (gInputHal && gInputHal->Timer.QueryTicks)
return gInputHal->Timer.QueryTicks();
return 0;
}
// Implements the input Start Repeat operation.
static void KiInputStartRepeat(INPUT_KEY key, uint64_t now) {
if (!KiInputKeyIsRepeatable(key)) return;
gRepeatKey = key;
gRepeatNextTick = now + KEY_REPEAT_DELAY_US;
}
// Implements the input Stop Repeat operation.
static void KiInputStopRepeat(INPUT_KEY key) {
if (gRepeatKey == key) {
gRepeatKey = INPUT_KEY_NONE;
@@ -129,6 +136,7 @@ static void KiInputStopRepeat(INPUT_KEY key) {
}
}
// Implements the input Service Repeat operation.
static void KiInputServiceRepeat(void) {
INPUT_KEY key = gRepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key]) {
@@ -150,6 +158,7 @@ static void KiInputServiceRepeat(void) {
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
}
// Implements the input Set Modifier operation.
static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
uint32_t bit = 0;
switch (key) {
@@ -173,6 +182,7 @@ static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
if (pressed && key == INPUT_KEY_NUM_LOCK) gModifiers ^= INPUT_MOD_NUM_LOCK;
}
// Implements the input Key To Char operation.
static char KiInputKeyToChar(INPUT_KEY key) {
if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) {
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
@@ -190,6 +200,7 @@ static char KiInputKeyToChar(INPUT_KEY key) {
return 0;
}
// Implements the input Initialize operation.
void KeInputInitialize(const HAL_API* hal) {
gInputHal = hal;
memset((void*)gKeyDown, 0, sizeof(gKeyDown));
@@ -214,6 +225,7 @@ void KeInputInitialize(const HAL_API* hal) {
DbgPrint("Input: kernel input core initialized (%dx%d)\n", w, h);
}
// Implements the input Register Device operation.
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) {
(void)flags;
uint32_t id = gNextInputDeviceId++;
@@ -223,6 +235,7 @@ __declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t f
return STATUS_SUCCESS;
}
// Implements the input Submit Key operation.
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
if (!event) return;
INPUT_KEY key = event->Key;
@@ -251,6 +264,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
}
}
// Implements the input Commit Mouse Position operation.
static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
@@ -268,6 +282,7 @@ static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
KeDisplaySetCursorPosition(nx, ny);
}
// Implements the input Submit Mouse operation.
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
if (!event) return;
@@ -285,6 +300,7 @@ __declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
}
}
// Implements the input Submit Mouse Absolute operation.
__declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE_EVENT* event) {
if (!event) return;
@@ -305,6 +321,7 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
}
}
// Implements the input Submit Mouse Absolute Scaled operation.
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
@@ -324,6 +341,7 @@ INPUT_KEY KeInputKbPoll(void) {
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
return gKeyDown[key] != 0;
@@ -337,6 +355,7 @@ INPUT_KEY KeInputReadKey(void) {
return key;
}
// Implements the input Read Char operation.
char KeInputReadChar(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return 0;
@@ -345,14 +364,17 @@ char KeInputReadChar(void) {
return ch;
}
// Implements the input Get Modifiers operation.
uint32_t KeInputGetModifiers(void) {
return gModifiers;
}
// Implements the input Mouse Poll operation.
bool KeInputMousePoll(void) {
return gMouseHead != gMouseTail;
}
// Implements the input Mouse Read operation.
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
if (!dx || !dy || !buttons) return false;
if (gMouseHead == gMouseTail) return false;
@@ -365,11 +387,13 @@ bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
return true;
}
// Implements the input Mouse Get Position operation.
void KeInputMouseGetPosition(int* x, int* y) {
if (x) *x = gMouseX;
if (y) *y = gMouseY;
}
// Implements the input Mouse Set Position operation.
void KeInputMouseSetPosition(int x, int y) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
@@ -384,6 +408,7 @@ void KeInputMouseSetPosition(int x, int y) {
KeDisplaySetCursorPosition(x, y);
}
// Implements the input Mouse Set Bounds operation.
void KeInputMouseSetBounds(int w, int h) {
if (w < 1) w = 1;
if (h < 1) h = 1;
code/kernel/ke_irp.c
+24 -13
View File
@@ -1,4 +1,4 @@
// ke_irp.c - I/O Request Packet dispatch, queue, cancellation, remove-locks
// I/O request packet infrastructure. This file owns IRP dispatch, queuing, cancellation, and remove-lock helpers.
#include <string.h>
#include <intrin.h>
@@ -10,9 +10,7 @@
#include <kernel/mm_virt.h>
#include <ntlite/ddi/io_driver.h>
// -------------------------------------------------------------------------
// IRP_QUEUE operations
// -------------------------------------------------------------------------
// IRP_QUEUE operations.
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
q->Lock = (SPINLOCK)SPINLOCK_INIT;
@@ -20,6 +18,7 @@ __declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
LIST_INIT(&q->Head);
}
// Implements the irp Queue Enqueue operation.
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
if (irp->Flags & IRP_FLAG_PRIORITY)
@@ -30,6 +29,7 @@ __declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
KeSignalSemaphore((SEMAPHORE*)q->Semaphore);
}
// Implements the irp Queue Dequeue operation.
__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY* node = ListPopHead(&q->Head);
@@ -58,13 +58,12 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
}
}
// Implements the irp Queue Cancel All operation.
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
}
// -------------------------------------------------------------------------
// Remove locks
// -------------------------------------------------------------------------
// Remove locks.
__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return STATUS_INVALID_PARAMETER;
@@ -76,6 +75,7 @@ __declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
return STATUS_SUCCESS;
}
// Releases remove Lock.
__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return;
if (lock->RemovedEvent) {
@@ -84,6 +84,7 @@ __declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
}
}
// Implements the acquire Remove Lock operation.
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return STATUS_INVALID_PARAMETER;
@@ -97,6 +98,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
return STATUS_SUCCESS;
}
// Releases remove Lock.
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return;
@@ -105,6 +107,7 @@ __declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
KeSignalEvent(lock->RemovedEvent);
}
// Releases remove Lock And Wait.
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp) {
(void)removeIrp;
if (!lock) return;
@@ -124,15 +127,14 @@ __declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP*
KeWaitForEvent(lock->RemovedEvent, 0);
}
// -------------------------------------------------------------------------
// IRP helpers
// -------------------------------------------------------------------------
// IRP helpers.
static size_t IoIrpSize(uint8_t stackCount) {
if (stackCount == 0) stackCount = 1;
return sizeof(IRP) + ((size_t)stackCount - 1u) * sizeof(IO_STACK_LOCATION);
}
// Creates irp.
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
if (!outIrp) return STATUS_INVALID_PARAMETER;
if (stackCount == 0) stackCount = 1;
@@ -143,10 +145,12 @@ __declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
return STATUS_SUCCESS;
}
// Releases irp.
__declspec(dllexport) void IoFreeIrp(IRP* irp) {
if (irp) MmFreePool(irp);
}
// Initializes irp.
__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
if (!irp) return;
if (stackCount == 0) stackCount = 1;
@@ -159,22 +163,26 @@ __declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
irp->CurrentLocation = 0;
}
// Returns current Irp Stack Location.
__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation];
}
// Returns next Irp Stack Location.
__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation + 1u >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation + 1u];
}
// Implements the skip Current Irp Stack Location operation.
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) {
if (!irp) return;
if (irp->CurrentLocation + 1u < irp->StackCount)
irp->CurrentLocation++;
}
// Copies current Irp Stack Location To Next.
__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
@@ -183,12 +191,14 @@ __declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
irp->CurrentLocation++;
}
// Updates completion Routine.
__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx) {
if (!irp) return;
irp->CompletionFn = fn;
irp->CompletionCtx = ctx;
}
// Updates cancel Routine.
__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine) {
if (!irp) return (IRP_CANCEL_ROUTINE)0;
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
@@ -198,6 +208,7 @@ __declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL
return old;
}
// Cancels irp.
__declspec(dllexport) void IoCancelIrp(IRP* irp) {
if (!irp) return;
@@ -262,9 +273,7 @@ __declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
ios->Parameters.DeviceIoControl.OutputLength = outputLength;
}
// -------------------------------------------------------------------------
// Completion/submission
// -------------------------------------------------------------------------
// Completion/submission.
__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information) {
if (!irp) return STATUS_INVALID_PARAMETER;
@@ -293,6 +302,7 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t
return STATUS_SUCCESS;
}
// Submits irp.
__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
irp->Status = STATUS_FAILURE;
@@ -301,6 +311,7 @@ __declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
return IoCallDriver(device, irp);
}
// Submits irp Sync.
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
code/kernel/ke_irq.c
+9 -1
View File
@@ -1,4 +1,5 @@
// ke_irq.c - IRQ dispatch, kernel-side
// Interrupt manager. This file owns IRQ routing, handler dispatch, and post-interrupt scheduling hooks.
#include <kernel/ke_irq.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
@@ -24,6 +25,7 @@ static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
static bool gIrqSoftwareVector[MAX_IRQS];
static const HAL_API *gHal;
// Implements the irq To Irql operation.
static KIRQL KiIrqToIrql(uint8_t irq) {
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)irq;
if (level >= HIGH_LEVEL)
@@ -31,10 +33,12 @@ static KIRQL KiIrqToIrql(uint8_t irq) {
return (KIRQL)level;
}
// Implements the irq Initialize operation.
void KeIrqInitialize(const HAL_API *hal) {
gHal = hal;
}
// Implements the irq Allocate Msi Vector operation.
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
if (!outIrq) return STATUS_INVALID_PARAMETER;
@@ -52,6 +56,7 @@ __declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
return STATUS_NO_MEMORY;
}
// Implements the irq Register operation.
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
@@ -71,6 +76,7 @@ __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
gHal->Intc.RouteIrq(irq);
}
// Implements the irq Dispatch operation.
void KeIrqDispatch(uint8_t irq) {
KIRQL targetIrql = KiIrqToIrql(irq);
KIRQL currentIrql = KeGetCurrentIrql();
@@ -103,10 +109,12 @@ void KeIrqDispatch(uint8_t irq) {
KeLowerIrql(oldIrql);
}
// Implements the enter Interrupt operation.
void KiEnterInterrupt(void) {
KeGetCurrentCpu()->InterruptDepth++;
}
// Implements the exit Interrupt operation.
void KiExitInterrupt(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu->InterruptDepth)
code/kernel/ke_irql.c
+9 -1
View File
@@ -1,9 +1,11 @@
// ke_irql.c - software IRQL skeleton
// Kernel IRQL model. This file owns the software interrupt-priority state used by IRQ and scheduler code.
#include <kernel/ke_irql.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_apc.h>
// Implements the irql Name operation.
__declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
switch (irql) {
case PASSIVE_LEVEL: return "PASSIVE_LEVEL";
@@ -17,10 +19,12 @@ __declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
}
}
// Returns current Irql.
__declspec(dllexport) KIRQL KeGetCurrentIrql(void) {
return KeGetCurrentCpu()->CurrentIrql;
}
// Implements the raise Irql operation.
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
KCPU* cpu = KeGetCurrentCpu();
KIRQL oldIrql = cpu->CurrentIrql;
@@ -39,6 +43,7 @@ __declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
return oldIrql;
}
// Implements the lower Irql operation.
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KCPU* cpu = KeGetCurrentCpu();
KIRQL currentIrql = cpu->CurrentIrql;
@@ -59,6 +64,7 @@ __declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KiDeliverKernelApcs();
}
// Implements the assert Irql operation.
__declspec(dllexport) void KeAssertIrql(KIRQL expected) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql != expected) {
@@ -68,6 +74,7 @@ __declspec(dllexport) void KeAssertIrql(KIRQL expected) {
}
}
// Implements the assert Irql At Or Below operation.
__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql > maxIrql) {
@@ -77,6 +84,7 @@ __declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
}
}
// Checks whether wait.
__declspec(dllexport) int KeCanWait(void) {
return KeGetCurrentIrql() == PASSIVE_LEVEL;
}
code/kernel/ke_mutex.c
+7 -1
View File
@@ -1,4 +1,5 @@
// ke_mutex.c - binary ownership mutex
// Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups.
#include <stdbool.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_thread.h>
@@ -7,8 +8,10 @@
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
// Implements the mutex Destroy operation.
static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
// Implements the mutex Create operation.
KSTATUS KeMutexCreate(MUTEX **out) {
if (!out) return STATUS_INVALID_PARAMETER;
MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
@@ -21,6 +24,7 @@ KSTATUS KeMutexCreate(MUTEX **out) {
return STATUS_SUCCESS;
}
// Implements the acquire Mutex operation.
KSTATUS KeAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -40,6 +44,7 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) {
return KeCommitBlockThread(self, 0);
}
// Implements the try Acquire Mutex operation.
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
@@ -53,6 +58,7 @@ KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
return STATUS_FAILURE;
}
// Releases mutex.
void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
code/kernel/ke_process.c
+27 -22
View File
@@ -1,4 +1,5 @@
// ke_process.c - process model
// Kernel process model. This file owns address-space metadata, handle tables, and process lifetime transitions.
#include <string.h>
#include <intrin.h>
#include <kernel/ke_process.h>
@@ -12,14 +13,13 @@
#include <kernel/hal/hal_apic.h>
#include <ntlite/syscall.h>
// ---------------------------------------------------------------------------
// Global process list
// ---------------------------------------------------------------------------
// Global process list.
static LIST_ENTRY gProcessList;
static SPINLOCK gProcessListLock;
static uint32_t gNextProcessId = 0;
static PROCESS* gSystemProcess = (PROCESS*)0;
// Processes init.
static void KiProcessInit(void) {
static int inited = 0;
if (!inited) {
@@ -29,11 +29,13 @@ static void KiProcessInit(void) {
}
}
// Returns process List.
LIST_ENTRY* KeGetProcessList(void) {
KiProcessInit();
return &gProcessList;
}
// Returns current Process.
PROCESS* KeGetCurrentProcess(void) {
THREAD* currentThread = KeGetCurrentThread();
if (currentThread) {
@@ -42,10 +44,12 @@ PROCESS* KeGetCurrentProcess(void) {
return NULL;
}
// Returns system Process.
PROCESS* KeGetSystemProcess(void) {
return gSystemProcess;
}
// Creates system Process.
KSTATUS KeCreateSystemProcess(PROCESS** out) {
if (!out) return STATUS_INVALID_PARAMETER;
if (gSystemProcess) {
@@ -65,6 +69,7 @@ KSTATUS KeCreateSystemProcess(PROCESS** out) {
return STATUS_SUCCESS;
}
// Implements the count Process Handles Locked operation.
static uint32_t KiCountProcessHandlesLocked(PROCESS* process) {
uint32_t count = 0;
if (!process) return 0;
@@ -75,6 +80,7 @@ static uint32_t KiCountProcessHandlesLocked(PROCESS* process) {
return count;
}
// Copies truncated String.
static void KiCopyTruncatedString(char* dst, size_t dstSize, const char* src, size_t srcLen) {
if (!dst || dstSize == 0) return;
size_t n = 0;
@@ -87,6 +93,7 @@ static void KiCopyTruncatedString(char* dst, size_t dstSize, const char* src, si
dst[n] = '\0';
}
// Copies process Info.
size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
KiProcessInit();
@@ -151,9 +158,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
return count;
}
// ---------------------------------------------------------------------------
// Destructor (called by ObDereferenceObject when RefCount hits zero)
// ---------------------------------------------------------------------------
// Destructor (called by ObDereferenceObject when RefCount hits zero).
static void KiProcessDestroy(OBJECT* obj) {
PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
@@ -201,9 +206,7 @@ static void KiProcessDestroy(OBJECT* obj) {
// Destroy returns.
}
// ---------------------------------------------------------------------------
// KeCreateProcess
// ---------------------------------------------------------------------------
// KeCreateProcess.
KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
KiProcessInit();
@@ -262,9 +265,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
}
// ---------------------------------------------------------------------------
// KeSetupUserProcessEnvironment
// ---------------------------------------------------------------------------
// KeSetupUserProcessEnvironment.
static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
SYS_STRING_VIEW src,
uintptr_t* baseOut,
@@ -314,6 +315,7 @@ static KSTATUS KiCopyEnvironmentStringViewFixed(PROCESS* proc,
return STATUS_SUCCESS;
}
// Releases partial Process Environment.
static void KiFreePartialProcessEnvironment(PROCESS* proc) {
if (!proc) return;
@@ -424,9 +426,7 @@ fail:
return s;
}
// ---------------------------------------------------------------------------
// Process exit / termination
// ---------------------------------------------------------------------------
// Process exit / termination.
static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
if (!process) return;
@@ -439,6 +439,7 @@ static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
}
}
// Implements the exit Process operation.
void KeExitProcess(PROCESS* process, int exitCode) {
if (!process) return;
process->ExitCode = exitCode;
@@ -450,6 +451,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
KiCompleteProcessExitIfLastThread(process);
}
// Implements the terminate Process operation.
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
if (!process) return STATUS_INVALID_PARAMETER;
@@ -483,22 +485,19 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
return STATUS_SUCCESS;
}
// Checks whether process Terminating.
int KeIsProcessTerminating(PROCESS* process) {
return process && process->Terminating != 0;
}
// ---------------------------------------------------------------------------
// KeGetProcessExitCode
// ---------------------------------------------------------------------------
// KeGetProcessExitCode.
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut) {
if (!process || !exitCodeOut) return STATUS_INVALID_PARAMETER;
*exitCodeOut = process->ExitCode;
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Handle table
// ---------------------------------------------------------------------------
// Handle table.
KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t rights, uint32_t* handleOut) {
if (!process || !object || !handleOut || rights == 0) return STATUS_INVALID_PARAMETER;
@@ -520,10 +519,12 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri
return STATUS_NO_MEMORY;
}
// Creates handle.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut) {
return KeAllocateHandleWithRights(process, object, HANDLE_RIGHT_ALL, handleOut);
}
// Releases handle.
void KeCloseHandle(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return;
@@ -540,6 +541,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) {
KeReleaseSpinlock(&process->HandleLock, flags);
}
// Returns handle With Rights.
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE)
return (OBJECT*)0;
@@ -554,10 +556,12 @@ OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t req
return obj;
}
// Returns handle.
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) {
return KeLookupHandleWithRights(process, handle, 0);
}
// Returns handle Rights.
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return 0;
@@ -567,6 +571,7 @@ uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle) {
return rights;
}
// Returns process By Id.
PROCESS* KeGetProcessById(uint32_t pid) {
KiProcessInit();
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
code/kernel/ke_queue.c
+6 -1
View File
@@ -1,4 +1,5 @@
// ke_queue.c - ring-buffer message queue
// Kernel message queue primitive. This file owns bounded queue storage and blocking send/receive behavior.
#include <string.h>
#include <kernel/ke_queue.h>
#include <kernel/ke_thread.h>
@@ -6,8 +7,10 @@
#include <kernel/ke_irql.h>
#include <kernel/ob_object.h>
// Implements the queue Destroy operation.
static void KiQueueDestroy(OBJECT* obj) { (void)obj; }
// Implements the queue Create operation.
KSTATUS KeQueueCreate(QUEUE **out) {
if (!out) return STATUS_INVALID_PARAMETER;
QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
@@ -21,6 +24,7 @@ KSTATUS KeQueueCreate(QUEUE **out) {
return STATUS_SUCCESS;
}
// Implements the send Message operation.
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
@@ -56,6 +60,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
}
}
// Implements the receive Message operation.
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
code/kernel/ke_sched.c
+43 -2
View File
@@ -1,4 +1,5 @@
// ke_sched.c - MLFQ scheduler
// Kernel scheduler. This file owns run queues, dispatch decisions, preemption, and CPU-local scheduling state.
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
@@ -22,6 +23,7 @@ static uint32_t gAgingCounter = 0;
static KE_SCHED_STATS gSchedStats;
static PROCESS* gIdleProcess;
// Implements the default Thread Affinity Mask operation.
static uint64_t KiDefaultThreadAffinityMask(void) {
// Normal kernel, user, and driver-created threads are BSP-affine by
// default. Code that is explicitly audited for AP execution can widen its
@@ -29,10 +31,12 @@ static uint64_t KiDefaultThreadAffinityMask(void) {
return 1ULL;
}
// Implements the thread Is Killed Locked operation.
static int KiThreadIsKilledLocked(const THREAD* thread) {
return thread && !thread->IsIdleThread && thread->Owner && KeIsProcessTerminating(thread->Owner);
}
// Implements the thread Can Run On Cpu Locked operation.
static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
if (!thread || !cpu) return 0;
if (thread->IsIdleThread) return thread == cpu->IdleThread;
@@ -40,6 +44,7 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
}
// Implements the cpu Is Idle Locked operation.
static int KiCpuIsIdleLocked(const KCPU* cpu) {
if (!cpu || !cpu->Online)
return 0;
@@ -47,6 +52,7 @@ static int KiCpuIsIdleLocked(const KCPU* cpu) {
return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
}
// Implements the any Idle Primary For Thread Locked operation.
static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
if (!thread || thread->IsIdleThread)
return 0;
@@ -65,6 +71,7 @@ static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
return 0;
}
// Checks whether defer To Idle Primary Locked.
static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
if (!cpu || !thread || thread->IsIdleThread)
return 0;
@@ -78,6 +85,7 @@ static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
return KiAnyIdlePrimaryForThreadLocked(thread);
}
// Implements the request Reschedule Cpu Locked operation.
static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || !target->Online)
return;
@@ -90,6 +98,7 @@ static void KiRequestRescheduleCpuLocked(KCPU* target) {
}
}
// Implements the poke Runnable Cpus Locked operation.
static void KiPokeRunnableCpusLocked(THREAD* thread) {
if (!thread || thread->IsIdleThread)
return;
@@ -137,6 +146,7 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
}
}
// Implements the base Quantum For Priority operation.
static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
switch (priority) {
case PriorityHigh: return SCHED_HIGH_QUANTUM_TICKS;
@@ -146,6 +156,7 @@ static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
}
}
// Implements the compute Quantum Locked operation.
static uint32_t KiComputeQuantumLocked(THREAD* thread) {
if (!thread || thread == KeGetCurrentCpu()->IdleThread)
return 1;
@@ -164,15 +175,19 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) {
return quantum ? quantum : 1;
}
// Implements the refill Quantum Locked operation.
static void KiRefillQuantumLocked(THREAD* thread) {
if (!thread) return;
thread->QuantumTicks = KiComputeQuantumLocked(thread);
thread->QuantumRemaining = thread->QuantumTicks;
}
// Returns current Thread.
THREAD* KeGetCurrentThread(void) { return KeGetCurrentCpu()->CurrentThread; }
// Updates current Thread.
void KeSetCurrentThread(THREAD* t) { KeGetCurrentCpu()->CurrentThread = t; }
// Returns scheduler Stats.
void KeGetSchedulerStats(KE_SCHED_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -181,6 +196,7 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
}
// Implements the arm Post Switch Unlock operation.
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
// The scheduler lock is intentionally held across KeSwitchContext(). The
// unlock cookie belongs to the thread continuation that will resume after
@@ -191,11 +207,13 @@ static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
thread->SwitchLockCookie = cookie;
}
// Implements the post Switch Unlock Cookie operation.
static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
return thread ? thread->SwitchLockCookie : 0x202ULL;
}
// Implements the scheduler Bug operation.
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"scheduler invariant failed: %s: thread=0x%016llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
@@ -209,6 +227,7 @@ static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
why ? why : "?");
}
// Implements the assert Ready Queue Candidate Locked operation.
static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null thread");
@@ -222,6 +241,7 @@ static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread)
KiSchedulerBug(where, thread, "thread still reserved/running on a CPU");
}
// Implements the assert Picked Thread Locked operation.
static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null picked thread");
@@ -235,6 +255,7 @@ static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) {
KiSchedulerBug(where, thread, "picked thread was not ready");
}
// Inserts ready Locked.
static void KiInsertReadyLocked(THREAD* thread) {
KiAssertReadyQueueCandidateLocked("KiInsertReadyLocked", thread);
@@ -249,6 +270,7 @@ static void KiInsertReadyLocked(THREAD* thread) {
static int KiAnyRunnableForCpuLocked(KCPU* cpu);
// Implements the any Runnable For Current Cpu operation.
static int KiAnyRunnableForCurrentCpu(void) {
KCPU* cpu = KeGetCurrentCpu();
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -257,6 +279,7 @@ static int KiAnyRunnableForCurrentCpu(void) {
return any;
}
// Implements the idle Loop operation.
static void KiIdleLoop(void) {
for (;;) {
KCPU* cpu = KeGetCurrentCpu();
@@ -279,6 +302,7 @@ static void KiIdleLoop(void) {
}
}
// Initializes scheduler Global.
void KiInitializeSchedulerGlobal(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
@@ -294,6 +318,7 @@ void KiInitializeSchedulerGlobal(void) {
(void)systemProc;
}
// Creates idle Thread For Cpu.
KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
if (!cpu || !gIdleProcess)
return STATUS_INVALID_PARAMETER;
@@ -366,6 +391,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
return STATUS_SUCCESS;
}
// Initializes scheduler.
void KeInitializeScheduler(void) {
KiInitializeSchedulerGlobal();
@@ -376,7 +402,7 @@ void KeInitializeScheduler(void) {
(unsigned)cpu->CpuIndex, s);
}
// Safely release the lock and clean up any dead threads natively post-switch
// Safely release the lock and clean up any dead threads natively post-switch.
void KeSchedulerPostSwitch(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -392,6 +418,7 @@ void KeSchedulerPostSwitch(void) {
}
}
// Updates thread Affinity Mask.
void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask) {
if (!thread || thread->IsIdleThread) return;
@@ -405,10 +432,12 @@ void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Returns default Thread Affinity Mask.
uint64_t KeGetDefaultThreadAffinityMask(void) {
return KiDefaultThreadAffinityMask();
}
// Implements the ready Thread operation.
void KeReadyThread(THREAD* thread) {
if (!thread) return;
@@ -419,6 +448,7 @@ void KeReadyThread(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the unready Thread operation.
void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -430,6 +460,7 @@ void KeUnreadyThread(THREAD* thread) {
}
// Implements the any Runnable For Cpu Locked operation.
static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -447,6 +478,7 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
return 0;
}
// Implements the pick Next For Cpu operation.
static THREAD* KiPickNextForCpu(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -472,17 +504,20 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
}
// Implements the request Reschedule operation.
void KiRequestReschedule(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu)
_InterlockedExchange(&cpu->ReschedulePending, 1);
}
// Checks whether reschedule Pending.
int KiIsReschedulePending(void) {
KCPU* cpu = KeGetCurrentCpu();
return cpu && _InterlockedCompareExchange(&cpu->ReschedulePending, 0, 0) != 0;
}
// Implements the schedule operation.
static void KiSchedule(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -547,12 +582,14 @@ static void KiSchedule(void) {
KeSchedulerPostSwitch();
}
// Implements the schedule From Interrupt operation.
void KiScheduleFromInterrupt(void) {
if (!KiIsReschedulePending()) return;
KiSchedule();
}
// Implements the scheduler Mark Thread Waiting operation.
void KiSchedulerMarkThreadWaiting(THREAD* thread) {
if (!thread) return;
@@ -574,6 +611,7 @@ void KiSchedulerMarkThreadWaiting(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the scheduler Note Voluntary Block operation.
void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -583,6 +621,7 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
}
// Implements the scheduler Wake Thread operation.
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
if (!thread) return;
@@ -627,6 +666,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the scheduler Tick operation.
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -705,6 +745,7 @@ void KeSchedulerTick(void) {
KiRequestReschedule();
}
// Implements the yield operation.
__declspec(dllexport) void KeYield(void) {
KeAssertIrql(PASSIVE_LEVEL);
KiSchedule();
Loaded 100 of 163 files, more files were not shown because too many files have changed in this diff. Show more