maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

redo boot around an explicit system volume and fixed kernel/HAL images, redo volumes and filesystem paths around capabilities instead of drive numbers and kernel CWD state, add a capability/security model with derived and revocable handles and security contexts, add port IPC and rssrv to manage the userspace resource namespace and publish the system volume, move process, dll and path loading to userspace and add process builder syscalls, init now supervises rssrv and the session process, redo the syscall ABI around request structs and four arguments, add a volume manager and proper partition arrival/removal handling so FAT and NTFS mount dynamically, clean up the HAL/DDK boundary so drivers don't depend on HAL internals, save fpu state properly now, the serial driver doesn't hang on a missing port, fix AHCI startup races, fix timeout accounting, fix shared mmap writeback, more smoke tests

Browse Source
This commit is contained in:
maksw committed 2026-08-20 12:33:17 +02:00
1 parent 4add6789ba
commit f882d339fb
232 files changed
+20934 -8898
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/hal/hal_apic.c
+62 -56
View File
@@ -1,15 +1,16 @@
// HAL APIC support. This file owns local APIC, I/O APIC, and interrupt-controller setup.
#include <kernel/hal/hal_apic.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_timer.h>
#include <stdint.h>
#include <intrin.h>
extern HAL_API gApi;
// ---- Local APIC ----.
static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
static volatile uint32_t* gLapic = (volatile uint32_t*)0;
#define LAPIC_REG(off) (*(volatile uint32_t*)((uint8_t*)gLapic + (off)))
#define LAPIC_ID 0x020
#define LAPIC_EOI 0x0B0
#define LAPIC_SPURIOUS 0x0F0
@@ -27,10 +28,9 @@ static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define CPUID_FEATURE_ECX_X2APIC (1u << 21)
#define APIC_VECTOR_RESCHEDULE 0xF0u
// ---- I/O APIC ----.
static volatile uint32_t* gIoapic = (volatile uint32_t *)0;
static volatile uint32_t* gIoapic = (volatile uint32_t*)0;
static uint32_t gBspLapicId = 0;
static uint8_t gX2ApicSupported = 0;
static uint8_t gX2ApicEnabled = 0;
@@ -39,13 +39,13 @@ static uint8_t gX2ApicEnabled = 0;
// Implements the ioapic Read operation.
static uint32_t IoapicRead(uint8_t reg) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg;
return *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA);
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
return *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA);
}
// Implements the ioapic Write operation.
static void IoapicWrite(uint8_t reg, uint32_t val) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg;
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val;
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA) = val;
}
// Implements the apic Detect Mode operation.
@@ -58,17 +58,9 @@ static void HalApicDetectMode(void) {
gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u;
}
// Implements the apic Stall Fallback operation.
static void HalApicStallFallback(uint32_t loops) {
for (volatile uint32_t i = 0; i < loops; i++) {
_mm_pause();
}
}
// Implements the apic Stall Us operation.
// Stalls for the requested bootstrap interval using the HAL timer fallback chain.
static void HalApicStallUs(uint32_t us) {
if (gApi.Timer.Stall) gApi.Timer.Stall(us);
else HalApicStallFallback(us * 128u + 1u);
HalTimerStall(us);
}
// Implements the apic Wait Icr Idle operation.
@@ -102,25 +94,25 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
#define IRQ_VECTOR_BASE 0x20
// Returns bsp Lapic Id.
__declspec(dllexport) uint32_t HalGetBspLapicId(void) {
// Returns the local interrupt-controller hardware ID of the boot processor.
uint32_t HalApicGetBootProcessorId(void) {
return gBspLapicId;
}
// Implements the apic Using X2 Apic operation.
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void) {
// Returns whether the local interrupt controller is using x2APIC mode.
uint8_t HalApicUsingX2Apic(void) {
return gX2ApicEnabled;
}
// Implements the apic Can Address Id operation.
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId) {
// Returns whether the active local interrupt-controller mode can address one hardware ID.
uint8_t HalApicCanAddressId(uint32_t hardwareId) {
if (gX2ApicEnabled)
return 1u;
return (lapicId < 256u) ? 1u : 0u;
return (hardwareId < 256u) ? 1u : 0u;
}
// Implements the apic Enable Current Cpu operation.
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
// Initializes the local interrupt controller on the current processor.
void HalApicInitializeCurrentProcessor(void) {
uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE;
if (gX2ApicEnabled) {
@@ -135,8 +127,8 @@ __declspec(dllexport) void HalApicEnableCurrentCpu(void) {
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
}
// Implements the apic Send Panic Ipi operation.
__declspec(dllexport) void HalApicSendPanicIpi(void) {
// Broadcasts an NMI to every processor except the caller.
void HalApicBroadcastNmi(void) {
if (!gLapic && !gX2ApicEnabled) return;
// Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed
@@ -146,42 +138,56 @@ __declspec(dllexport) void HalApicSendPanicIpi(void) {
HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI
}
// Implements the apic Send Reschedule Ipi operation.
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) {
if (!gLapic && !gX2ApicEnabled) return;
HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination
// Sends one fixed-vector processor interrupt without exposing APIC mechanics.
void HalApicSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector) {
if ((!gLapic && !gX2ApicEnabled) || vector < 0x20u || !HalApicCanAddressId(hardwareId))
return;
HalApicSendIpi(hardwareId, vector);
}
// Implements the apic Send Tlb Shootdown Ipi operation.
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
if (!gLapic && !gX2ApicEnabled) return;
// TLB shootdown must reach CPUs that may be spinning with IF=0 inside
// high-IRQL/spinlock regions, so use NMI delivery rather than a fixed
// vector. The NMI stub handles active shootdowns and returns; unexpected
// NMIs still go through the normal fault/panic path.
HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination
// Sends one non-maskable processor interrupt.
void HalApicSendProcessorNmi(uint32_t hardwareId) {
if ((!gLapic && !gX2ApicEnabled) || !HalApicCanAddressId(hardwareId))
return;
HalApicSendIpi(hardwareId, (4u << 8));
}
// Starts application Processor.
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) {
if ((!gLapic && !gX2ApicEnabled) || !startupVector || lapicId == gBspLapicId)
// Starts one secondary processor through the local interrupt-controller startup protocol.
KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical) {
if ((!gLapic && !gX2ApicEnabled) || !startPhysical ||
(startPhysical & 0xFFFu) != 0 || startPhysical >= 0x100000u ||
hardwareId == gBspLapicId) {
return NTLSTATUS_INVALID_PARAMETER;
if (!HalApicCanAddressId(lapicId))
}
if (!HalApicCanAddressId(hardwareId))
return NTLSTATUS_NOT_SUPPORTED;
// INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12.
HalApicSendIpi(lapicId, 0x0000C500u); // INIT, level assert, level-triggered
uint8_t startupVector = (uint8_t)(startPhysical >> 12);
// INIT assert/deassert, then two SIPIs. The target begins at startPhysical.
HalApicSendIpi(hardwareId, 0x0000C500u); // INIT, level assert, level-triggered
HalApicStallUs(10000);
HalApicSendIpi(lapicId, 0x00008500u); // INIT, level deassert
HalApicSendIpi(hardwareId, 0x00008500u); // INIT, level deassert
HalApicStallUs(200);
HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI
HalApicSendIpi(hardwareId, 0x00000600u | startupVector); // SIPI
HalApicStallUs(200);
HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI retry
HalApicSendIpi(hardwareId, 0x00000600u | startupVector); // SIPI retry
HalApicStallUs(200);
return NTLSTATUS_SUCCESS;
}
// Composes an MSI message targeting the boot processor without exposing APIC mode.
KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut) {
if (!messageOut || vector < 0x20u)
return NTLSTATUS_INVALID_PARAMETER;
if (gBspLapicId >= 256u)
return NTLSTATUS_NOT_SUPPORTED;
messageOut->Address = 0xFEE00000ull | ((uint64_t)(gBspLapicId & 0xFFu) << 12);
messageOut->Data = vector;
messageOut->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
// Implements the apic Route Irq operation.
void HalApicRouteIrq(uint8_t irq) {
// Full initial setup of the redirection entry.
@@ -220,8 +226,8 @@ void HalApicSendEoi(void) {
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
HalApicDetectMode();
gLapic = (volatile uint32_t *)(gApi.PhysToVirt(lapicPhys));
gIoapic = (volatile uint32_t *)(gApi.PhysToVirt(ioapicPhys));
gLapic = (volatile uint32_t*)(HalPhysicalToVirtual(lapicPhys));
gIoapic = (volatile uint32_t*)(HalPhysicalToVirtual(ioapicPhys));
// The BSP is the CPU executing HAL init. Keep all external/device IRQs
// routed to this LAPIC; CPU-local IPIs use the local APIC directly.
@@ -230,7 +236,7 @@ void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
else
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
HalApicEnableCurrentCpu();
HalApicInitializeCurrentProcessor();
// Mask all IOAPIC lines initially.
uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
code/kernel/hal/hal_main.c
+76 -55
View File
@@ -1,8 +1,11 @@
// HAL initialization entry point. This file discovers platform tables and publishes the HAL API to the kernel.
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_output.h>
#include <kernel/hal/hal_processor.h>
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_timer.h>
#include <string.h>
#include <stdint.h>
#include <stdarg.h>
@@ -11,7 +14,6 @@
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out);
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out);
void HalPciInitialize(HAL_PCI* out);
void HalRtcInitialize(HAL_RTC* out);
void HalRtcSetCenturyRegister(uint8_t centuryRegister);
@@ -20,9 +22,34 @@ extern void HalSetDebugEnabled(int v);
// Defined in mm_virt.c - HAL needs it for MMIO window access.
static uint64_t gPhysBase = 0xFFFF880000000000ULL;
HAL_API gApi;
static KSTATUS HalQueryProcessorTopology(HAL_PROCESSOR_TOPOLOGY* out);
static HAL_API gApi = {
.Size = sizeof(HAL_API),
.Version = HAL_API_VERSION,
.Processor = {
.DisableInterrupts = HalDisableInterrupts,
.RestoreInterrupts = HalRestoreInterrupts,
.EnableInterrupts = HalEnableInterrupts,
.WaitForInterrupt = HalWaitForInterrupt,
.Halt = HalHalt,
.Stop = HalStopProcessor,
.QueryTopology = HalQueryProcessorTopology,
.Start = HalApicStartProcessor,
},
.InterruptController = {
.RouteIrq = HalApicRouteIrq,
.MaskIrq = HalApicMaskIrq,
.UnmaskIrq = HalApicUnmaskIrq,
.SendEoi = HalApicSendEoi,
.InitializeCurrentProcessor = HalApicInitializeCurrentProcessor,
.SendProcessorInterrupt = HalApicSendProcessorInterrupt,
.SendProcessorNmi = HalApicSendProcessorNmi,
.BroadcastNmi = HalApicBroadcastNmi,
.ComposeMsi = HalApicComposeMsi,
},
};
static int gInitialised = 0;
extern void (*gTimerCallback)(void);
// ACPI table walker - minimal, just enough to find APIC and HPET addresses.
#define ACPI_SIG(a,b,c,d) ((uint32_t)(a)|((uint32_t)(b)<<8)|((uint32_t)(c)<<16)|((uint32_t)(d)<<24))
@@ -81,7 +108,7 @@ typedef struct {
static uintptr_t gLapicPhys = 0xFEE00000;
static uintptr_t gIoapicPhys = 0xFEC00000;
static uintptr_t gHpetPhys = 0xFED00000;
static HAL_CPU_TOPOLOGY gCpuTopology;
static HAL_PROCESSOR_TOPOLOGY gCpuTopology;
// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5).
// Do not guess SLP_TYP on real hardware: these values are firmware-specific.
@@ -122,8 +149,8 @@ void HalDbgPrint(const char* fmt, ...) {
gApi.Output.Write(&record);
}
static void *HalPhysToVirt(uint64_t phys) {
return (void *)(phys + gPhysBase);
void* HalPhysicalToVirtual(uint64_t phys) {
return (void*)(phys + gPhysBase);
}
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
@@ -360,18 +387,18 @@ static void ParseFadt(const ACPISDTHeader* header) {
// Implements the add Cpu Record operation.
static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
if (gCpuTopology.CpuCount >= HAL_MAX_CPUS)
if (gCpuTopology.Count >= HAL_MAX_PROCESSORS)
return;
for (uint32_t i = 0; i < gCpuTopology.CpuCount; i++) {
if (gCpuTopology.Cpus[i].LapicId == lapicId)
for (uint32_t i = 0; i < gCpuTopology.Count; i++) {
if (gCpuTopology.Processors[i].HardwareId == lapicId)
return;
}
HAL_CPU_INFO* cpu = &gCpuTopology.Cpus[gCpuTopology.CpuCount++];
HAL_PROCESSOR_INFO* cpu = &gCpuTopology.Processors[gCpuTopology.Count++];
memset(cpu, 0, sizeof(*cpu));
cpu->ProcessorUid = uid;
cpu->LapicId = lapicId;
cpu->FirmwareId = uid;
cpu->HardwareId = lapicId;
cpu->Flags = flags;
cpu->Enabled = (flags & 1u) ? 1u : 0u;
}
@@ -379,8 +406,8 @@ static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
// Parses madt.
static void ParseMadt(MADT* madt) {
gLapicPhys = madt->LocalAPICAddress;
uint8_t* p = (uint8_t *)(madt + 1);
uint8_t* end = (uint8_t *)madt + madt->Header.Length;
uint8_t* p = (uint8_t*)(madt + 1);
uint8_t* end = (uint8_t*)madt + madt->Header.Length;
while (p + 2 <= end) {
uint8_t type = p[0], len = p[1];
if (!len || p + len > end) break;
@@ -388,17 +415,17 @@ static void ParseMadt(MADT* madt) {
if (type == 0 && len >= 8) { // Processor Local APIC
uint32_t uid = p[2];
uint32_t lapicId = p[3];
uint32_t flags = *(uint32_t *)(p + 4);
uint32_t flags = *(uint32_t*)(p + 4);
if (flags & 1u)
HalAddCpuRecord(uid, lapicId, flags);
} else if (type == 1 && len >= 12) { // I/O APIC entry
gIoapicPhys = *(uint32_t *)(p + 4);
gIoapicPhys = *(uint32_t*)(p + 4);
} else if (type == 5 && len >= 12) { // Local APIC Address Override
gLapicPhys = (uintptr_t)*(uint64_t *)(p + 4);
gLapicPhys = (uintptr_t)*(uint64_t*)(p + 4);
} else if (type == 9 && len >= 16) { // Processor Local x2APIC
uint32_t lapicId = *(uint32_t *)(p + 4);
uint32_t flags = *(uint32_t *)(p + 8);
uint32_t uid = *(uint32_t *)(p + 12);
uint32_t lapicId = *(uint32_t*)(p + 4);
uint32_t flags = *(uint32_t*)(p + 8);
uint32_t uid = *(uint32_t*)(p + 12);
if (flags & 1u)
HalAddCpuRecord(uid, lapicId, flags);
}
@@ -512,47 +539,52 @@ static void HalShutdown(void) {
HalStopProcessor();
}
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
static KSTATUS HalQueryProcessorTopology(HAL_PROCESSOR_TOPOLOGY* out) {
if (!out) return NTLSTATUS_INVALID_PARAMETER;
memset(out, 0, sizeof(*out));
*out = gCpuTopology;
if (!out->CpuCount) {
out->CpuCount = 1;
out->BspLapicId = HalGetBspLapicId();
out->Cpus[0].LapicId = out->BspLapicId;
out->Cpus[0].ProcessorUid = 0;
out->Cpus[0].Flags = 1;
out->Cpus[0].Enabled = 1;
out->Cpus[0].IsBsp = 1;
if (!out->Count) {
out->Count = 1;
out->BootProcessorHardwareId = HalApicGetBootProcessorId();
out->Processors[0].HardwareId = out->BootProcessorHardwareId;
out->Processors[0].FirmwareId = 0;
out->Processors[0].Flags = 1;
out->Processors[0].Enabled = 1;
out->Processors[0].IsBootProcessor = 1;
out->Processors[0].Startable = 1;
return NTLSTATUS_SUCCESS;
}
uint32_t bsp = HalGetBspLapicId();
out->BspLapicId = bsp;
uint32_t bsp = HalApicGetBootProcessorId();
out->BootProcessorHardwareId = bsp;
int foundBsp = 0;
for (uint32_t i = 0; i < out->CpuCount; i++) {
out->Cpus[i].IsBsp = (out->Cpus[i].LapicId == bsp) ? 1u : 0u;
if (out->Cpus[i].IsBsp) foundBsp = 1;
for (uint32_t i = 0; i < out->Count; i++) {
out->Processors[i].IsBootProcessor = (out->Processors[i].HardwareId == bsp) ? 1u : 0u;
out->Processors[i].Startable = out->Processors[i].IsBootProcessor || HalApicCanAddressId(out->Processors[i].HardwareId);
if (out->Processors[i].IsBootProcessor) foundBsp = 1;
}
if (!foundBsp && out->CpuCount < HAL_MAX_CPUS) {
HAL_CPU_INFO* cpu = &out->Cpus[out->CpuCount++];
if (!foundBsp && out->Count < HAL_MAX_PROCESSORS) {
HAL_PROCESSOR_INFO* cpu = &out->Processors[out->Count++];
memset(cpu, 0, sizeof(*cpu));
cpu->LapicId = bsp;
cpu->ProcessorUid = bsp;
cpu->HardwareId = bsp;
cpu->FirmwareId = bsp;
cpu->Flags = 1;
cpu->Enabled = 1;
cpu->IsBsp = 1;
cpu->IsBootProcessor = 1;
cpu->Startable = 1;
}
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
if (!initParams || initParams->Size != sizeof(HAL_INIT_PARAMS) || initParams->Flags != 0 ||
!initParams->PhysicalWindowBase || !initParams->AllocatePool)
return NTLSTATUS_INVALID_PARAMETER;
if (gInitialised) return NTLSTATUS_ALREADY_EXISTS;
gPhysBase = initParams->PhysicalWindowBase;
gApi.PhysToVirt = HalPhysToVirt;
debugEnabled = initParams->Debug;
HalOutputInitialize(&gApi.Output);
@@ -577,21 +609,15 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
HalApicInitialize(gLapicPhys, gIoapicPhys);
HalDbgPrint("HAL: local APIC mode=%s BSP_ID=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
HalGetBspLapicId());
HalApicGetBootProcessorId());
KSTATUS ts = HalTimerInitialize(gLapicPhys, gHpetPhys, &gApi.Timer);
if (!KSUCCESS(ts)) HalDbgPrint("HAL: HPET unavailable, using fallback\n");
HalRtcInitialize(&gApi.Rtc);
HalPciInitialize(&gApi.Pci);
gApi.Intc.RouteIrq = HalApicRouteIrq;
gApi.Intc.MaskIrq = HalApicMaskIrq;
gApi.Intc.UnmaskIrq = HalApicUnmaskIrq;
gApi.Intc.SendEoi = HalApicSendEoi;
gApi.Power.Shutdown = HalShutdown;
gApi.Version = HAL_API_VERSION;
gInitialised = 1;
// The 8042 can leave output bytes pending while the keyboard/mouse drivers
@@ -609,10 +635,5 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
return NTLSTATUS_SUCCESS;
}
// Registers scheduler.
__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)) {
gTimerCallback = pKeSchedulerTick;
}
// Returns api.
__declspec(dllexport) const HAL_API* HalGetApi(void) { return &gApi; }
HAL_ENTRY const HAL_API* HalGetApi(void) { return &gApi; }
code/kernel/hal/hal_output.c
+1 -1
View File
@@ -3,7 +3,7 @@
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_output.h>
#define HAL_OUTPUT_ENDPOINT_CAPACITY 8u
code/kernel/hal/hal_pci.c
+2 -1
View File
@@ -1,7 +1,8 @@
// HAL PCI configuration access. This file owns legacy mechanism-1 PCI config port I/O.
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#include <intrin.h>
#include <kernel/hal/hal_api.h>
#define PCI_CONFIG_ADDR 0xCF8
#define PCI_CONFIG_DATA 0xCFC
code/kernel/hal/hal_processor.c
+8 -7
View File
@@ -1,41 +1,42 @@
// HAL processor control exports. This file owns interrupt masking and processor halt instructions.
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_processor.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
// Masks interrupts and returns the previous RFLAGS value.
__declspec(dllexport) uintptr_t HalDisableInterrupts(void) {
uintptr_t HalDisableInterrupts(void) {
uintptr_t flags = __readeflags();
_disable();
return flags;
}
// Restores a previously saved RFLAGS value.
__declspec(dllexport) void HalRestoreInterrupts(uintptr_t flags) {
void HalRestoreInterrupts(uintptr_t flags) {
__writeeflags(flags);
}
// Enables maskable interrupts.
__declspec(dllexport) void HalEnableInterrupts(void) {
void HalEnableInterrupts(void) {
_enable();
}
// Enables interrupts and halts until an interrupt arrives.
__declspec(dllexport) void HalWaitForInterrupt(void) {
void HalWaitForInterrupt(void) {
_enable();
__halt();
}
// Halts once without changing interrupt state.
__declspec(dllexport) void HalHalt(void) {
void HalHalt(void) {
__halt();
}
// Masks interrupts and permanently stops the current processor.
__declspec(dllexport) _Noreturn void HalStopProcessor(void) {
_Noreturn void HalStopProcessor(void) {
_disable();
while (true)
__halt();
code/kernel/hal/hal_rtc.c
+1 -1
View File
@@ -5,7 +5,7 @@
#include <string.h>
#include <intrin.h>
#include <ntlite/status.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_debug.h>
#define CMOS_INDEX_PORT 0x70
code/kernel/hal/hal_screen.c
+4 -4
View File
@@ -5,12 +5,12 @@
// each byte is the leftmost pixel.
#include <stdint.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_output.h>
static uint32_t* gFb = (uint32_t *)0;
static uint32_t* gBackbuffer = (uint32_t *)0;
static uint32_t* gFb = (uint32_t*)0;
static uint32_t* gBackbuffer = (uint32_t*)0;
static int gWidth = 0;
static int gHeight = 0;
static int gStride = 0; // pixels per row
@@ -343,7 +343,7 @@ void HalScreenRegisterOutputEndpoint(HAL_OUTPUT_ROUTER* output) {
// Populate the HAL_SCREEN vtable.
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out) {
gFb = (uint32_t *)fb;
gFb = (uint32_t*)fb;
gWidth = w;
gHeight = h;
gStride = stride;
code/kernel/hal/hal_serial.c
+56 -16
View File
@@ -1,26 +1,51 @@
// HAL serial output endpoint. This file owns polling COM1 writes and subscribes serial to HAL text streams.
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_output.h>
#include <kernel/hal/hal_processor.h>
#define HAL_COM1_DATA_PORT 0x03F8u
#define HAL_COM1_STATUS_PORT 0x03FDu
#define HAL_COM1_TX_READY 0x20u
#define HAL_COM1_BASE 0x03F8u
#define HAL_COM1_DATA_PORT (HAL_COM1_BASE + 0u)
#define HAL_COM1_STATUS_PORT (HAL_COM1_BASE + 5u)
#define HAL_COM1_SCRATCH_PORT (HAL_COM1_BASE + 7u)
#define HAL_COM1_TX_READY 0x20u
#define HAL_SERIAL_TX_SPIN_LIMIT 1000000u
static volatile long gSerialWriteLock;
static volatile long gSerialPresent;
// Waits until COM1 can accept one byte.
static void HalSerialWaitForTransmit(void) {
while ((__inbyte(HAL_COM1_STATUS_PORT) & HAL_COM1_TX_READY) == 0)
_mm_pause();
// Probes the 16550 scratch register without relying on line-status values from an absent port.
static bool HalSerialProbeCom1(void) {
uint8_t original = __inbyte(HAL_COM1_SCRATCH_PORT);
__outbyte(HAL_COM1_SCRATCH_PORT, 0x5Au);
uint8_t first = __inbyte(HAL_COM1_SCRATCH_PORT);
__outbyte(HAL_COM1_SCRATCH_PORT, 0xA5u);
uint8_t second = __inbyte(HAL_COM1_SCRATCH_PORT);
__outbyte(HAL_COM1_SCRATCH_PORT, original);
return first == 0x5Au && second == 0xA5u;
}
// Writes one byte to COM1.
static void HalSerialWriteByte(uint8_t value) {
HalSerialWaitForTransmit();
// Waits for COM1 transmit readiness without allowing broken or removed hardware to stall the machine forever.
static bool HalSerialWaitForTransmit(void) {
for (uint32_t spin = 0; spin < HAL_SERIAL_TX_SPIN_LIMIT; spin++) {
if ((__inbyte(HAL_COM1_STATUS_PORT) & HAL_COM1_TX_READY) != 0)
return true;
_mm_pause();
}
return false;
}
// Writes one byte to COM1 when the device remains responsive.
static bool HalSerialWriteByte(uint8_t value) {
if (!HalSerialWaitForTransmit())
return false;
__outbyte(HAL_COM1_DATA_PORT, value);
return true;
}
// Acquires the normal serial writer lock.
@@ -37,7 +62,8 @@ static void HalSerialReleaseWriteLock(void) {
// Writes one output record through polling COM1.
static void HalSerialWriteRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer)
if (!record || !record->Buffer ||
_InterlockedCompareExchange(&gSerialPresent, 0, 0) == 0)
return;
bool emergency = (record->Flags & HAL_OUTPUT_WRITE_EMERGENCY) != 0;
@@ -47,24 +73,36 @@ static void HalSerialWriteRecord(void* context, const HAL_OUTPUT_RECORD* record)
HalSerialAcquireWriteLock();
}
for (size_t i = 0; i < record->Length; i++) {
bool responsive = true;
for (size_t i = 0; i < record->Length && responsive; i++) {
char c = record->Buffer[i];
if (c == '\n')
HalSerialWriteByte('\r');
HalSerialWriteByte((uint8_t)c);
responsive = HalSerialWriteByte('\r');
if (responsive)
responsive = HalSerialWriteByte((uint8_t)c);
}
if (!responsive)
_InterlockedExchange(&gSerialPresent, 0);
if (!emergency) {
HalSerialReleaseWriteLock();
HalRestoreInterrupts(interruptFlags);
}
}
// Registers the polling serial endpoint.
// Registers the polling serial endpoint only when a 16550-compatible COM1 device is present.
void HalSerialInitialize(HAL_OUTPUT_ROUTER* output) {
if (!output || !output->RegisterEndpoint)
return;
gSerialWriteLock = 0;
gSerialPresent = 0;
if (!HalSerialProbeCom1())
return;
gSerialPresent = 1;
HAL_OUTPUT_ENDPOINT endpoint;
endpoint.Name = "serial0";
endpoint.StreamMask = HAL_OUTPUT_STREAM_ALL;
@@ -75,4 +113,6 @@ void HalSerialInitialize(HAL_OUTPUT_ROUTER* output) {
uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
if (KSUCCESS(output->RegisterEndpoint(&endpoint, &endpointId)))
output->SerialEndpointId = endpointId;
else
gSerialPresent = 0;
}
code/kernel/hal/hal_timer.c
+19 -21
View File
@@ -2,8 +2,9 @@
#include <stdint.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_timer.h>
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_debug.h>
#define LAPIC_TIMER_LVT 0x320
@@ -19,27 +20,24 @@
#define TIMER_PERIOD_MS 10
#define PIT_HZ 1193182UL
extern HAL_API gApi;
static volatile uint32_t* gLapic = NULL;
static uint32_t gApicTicksPerUs = 0;
static uint32_t gApicTicksPerQuantum = 0;
static uint64_t gHpetTicksPerUs = 0;
static uint64_t gHpetHz = 0;
static volatile uint64_t* gHpet = NULL;
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
#define LAPIC_REG(off) (*(volatile uint32_t*)((uint8_t*)gLapic + (off)))
static volatile uint64_t gTicks = 0;
void (* volatile gTimerCallback)(void) = NULL;
// Implements the outb operation.
static inline void outb(uint16_t port, uint8_t val) { __outbyte(port, val); }
// Implements the inb operation.
static inline uint8_t inb(uint16_t port) { return __inbyte(port); }
// Implements the timer Isr operation.
static void TimerIsr(void) {
// Records one local timer interrupt for HAL timekeeping fallback.
static void TimerInterrupt(void) {
gTicks++;
if (gTimerCallback) gTimerCallback();
// EOI is sent by KeIrqDispatch after this returns.
}
@@ -87,7 +85,7 @@ static uint64_t QueryTickFrequency(void) {
}
// Implements the stall operation.
static void Stall(uint32_t us) {
void HalTimerStall(uint32_t us) {
// Priority 1: Use HPET (Highest Precision).
if (gHpet && gHpetTicksPerUs > 0) {
uint64_t start = gHpet[30]; // Offset 0xF0 is the Main Counter
@@ -128,7 +126,7 @@ static void Stall(uint32_t us) {
// Returns HPET ticks/second, or 0 if HPET is unavailable.
static uint64_t HpetInit(uintptr_t hpetPhys) {
// HPET register block, accessed through physical window.
volatile uint64_t *hpet = (volatile uint64_t *)gApi.PhysToVirt(hpetPhys);
volatile uint64_t* hpet = (volatile uint64_t*)HalPhysicalToVirtual(hpetPhys);
// General Capabilities register (offset 0):
// Bits 63:32 = counter clock period in femtoseconds.
@@ -167,7 +165,7 @@ static uint32_t CalibrateApicTimerViaPit(void) {
// Use HPET main counter to measure how many APIC ticks elapse in 10 ms, then return ticks/ms.
static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
volatile uint64_t *hpet = (volatile uint64_t *)gApi.PhysToVirt(hpetPhys);
volatile uint64_t* hpet = (volatile uint64_t*)HalPhysicalToVirtual(hpetPhys);
uint64_t hpetHz = HpetInit(hpetPhys);
if (!hpetHz) return 0;
@@ -187,7 +185,7 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
}
// Implements the timer Configure Current Cpu operation.
__declspec(dllexport) void HalTimerConfigureCurrentCpu(void) {
void HalTimerConfigureCurrentProcessor(void) {
if (!gLapic || !gApicTicksPerQuantum) return;
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
LAPIC_REG(LAPIC_TIMER_LVT) = TIMER_PERIODIC | TIMER_VECTOR;
@@ -195,8 +193,8 @@ __declspec(dllexport) void HalTimerConfigureCurrentCpu(void) {
}
// Implements the timer Initialize operation.
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out) {
gLapic = (volatile uint32_t *)gApi.PhysToVirt(lapicPhys);
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* out) {
gLapic = (volatile uint32_t*)HalPhysicalToVirtual(lapicPhys);
HalDbgPrint("HAL: lapicPhys=0x%016llX, lapicVirt=0x%016llX, hpetPhys=0x%016llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
uint32_t ticksPerMs = 0;
@@ -217,7 +215,7 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *o
gApicTicksPerQuantum = ticksPerMs * TIMER_PERIOD_MS;
if (hpetPhys) {
gHpet = (volatile uint64_t*)gApi.PhysToVirt(hpetPhys);
gHpet = (volatile uint64_t*)HalPhysicalToVirtual(hpetPhys);
uint64_t hpetHz = HpetInit(hpetPhys);
gHpetHz = hpetHz;
gHpetTicksPerUs = hpetHz / 1000000;
@@ -225,14 +223,14 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *o
HalDbgPrint("HAL: APIC timer: %u ticks/ms\n", ticksPerMs);
// Configure this CPU's LAPIC timer; APs call HalTimerConfigureCurrentCpu()
// after enabling their local APIC.
HalTimerConfigureCurrentCpu();
// Leave the BSP timer masked until the kernel registers its scheduler
// callback and IRQ route. APs arm their local timer after per-CPU setup.
out->QueryTicks = QueryTicks;
out->QueryTickFrequency = QueryTickFrequency;
out->Stall = Stall;
out->Isr = TimerIsr;
out->IrqNumber = 0;
out->Stall = HalTimerStall;
out->HandleInterrupt = TimerInterrupt;
out->ConfigureCurrentProcessor = HalTimerConfigureCurrentProcessor;
out->IrqNumber = 0;
return NTLSTATUS_SUCCESS;
}
code/kernel/include/kernel/hal/hal_api.h
+234
View File
@@ -0,0 +1,234 @@
// Private kernel-to-HAL interface table. This ABI is not exposed to drivers or user mode.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#ifdef NTL_HAL_BUILD
#define HAL_ENTRY __declspec(dllexport)
#else
#define HAL_ENTRY __declspec(dllimport)
#endif
#define HAL_MAX_PROCESSORS 32u
typedef struct HAL_PROCESSOR_INFO {
uint32_t FirmwareId;
uint32_t HardwareId;
uint32_t Flags;
uint8_t Enabled;
uint8_t IsBootProcessor;
uint8_t Startable;
uint8_t Reserved;
} HAL_PROCESSOR_INFO;
typedef struct HAL_PROCESSOR_TOPOLOGY {
uint32_t Count;
uint32_t BootProcessorHardwareId;
HAL_PROCESSOR_INFO Processors[HAL_MAX_PROCESSORS];
} HAL_PROCESSOR_TOPOLOGY;
typedef struct HAL_PROCESSOR {
uintptr_t (*DisableInterrupts)(void);
void (*RestoreInterrupts)(uintptr_t flags);
void (*EnableInterrupts)(void);
void (*WaitForInterrupt)(void);
void (*Halt)(void);
void (*Stop)(void);
KSTATUS (*QueryTopology)(HAL_PROCESSOR_TOPOLOGY* out);
KSTATUS (*Start)(uint32_t hardwareId, uintptr_t startPhysical);
} HAL_PROCESSOR;
typedef struct HAL_MSI_MESSAGE {
uint64_t Address;
uint32_t Data;
uint32_t Reserved;
} HAL_MSI_MESSAGE;
typedef struct HAL_FRAMEBUFFER_INFO {
uintptr_t Base; // kernel virtual address of the visible framebuffer
uint32_t Width;
uint32_t Height;
uint32_t Stride; // pixels per scanline
uint32_t Format; // currently 0x00RRGGBB / X8R8G8B8
} HAL_FRAMEBUFFER_INFO;
typedef struct HAL_SCREEN {
void (*Clear)(uint32_t color);
void (*PutPixel)(int x, int y, uint32_t color);
void (*Blit)(int x, int y, int w, int h, const uint32_t* pixels);
void (*GetSize)(int* width, int* height);
KSTATUS (*GetFramebuffer)(HAL_FRAMEBUFFER_INFO* outInfo);
void (*Scroll)(int lines);
void (*PutChar)(char c, int x, int y, uint32_t fg, uint32_t bg);
// Writes only to the boot framebuffer text console. Stream routing belongs to HAL_OUTPUT_ROUTER.
void (*PrintChar)(char c, uint32_t fg, uint32_t bg);
// Legacy software cursor hooks kept for early-boot/panic fallback only.
// Normal input cursor ownership lives in ke_display.
void (*DrawCursor)(int x, int y);
void (*EraseCursor)(void);
} HAL_SCREEN;
// Output streams describe the producer's intent. Endpoints subscribe by mask.
#define HAL_OUTPUT_STREAM_DEBUG 0x00000001u
#define HAL_OUTPUT_STREAM_CONSOLE 0x00000002u
#define HAL_OUTPUT_STREAM_CRASH 0x00000004u
#define HAL_OUTPUT_STREAM_PANIC 0x00000008u
#define HAL_OUTPUT_STREAM_ALL 0x0000000Fu
// Emergency writes bypass endpoints that cannot run without scheduler services.
#define HAL_OUTPUT_WRITE_EMERGENCY 0x00000001u
#define HAL_OUTPUT_ENDPOINT_PANIC_SAFE 0x00000001u
#define HAL_OUTPUT_ENDPOINT_INVALID UINT32_MAX
typedef struct HAL_OUTPUT_RECORD {
// Exactly one HAL_OUTPUT_STREAM_* value.
uint32_t Stream;
// HAL_OUTPUT_WRITE_* flags.
uint32_t Flags;
// X8R8G8B8 text color hints.
uint32_t Foreground;
uint32_t Background;
// The buffer remains valid only for the duration of Write().
const char* Buffer;
size_t Length;
} HAL_OUTPUT_RECORD;
// Endpoint callbacks consume or copy the record synchronously and must not retain Buffer.
typedef void (*HAL_OUTPUT_ENDPOINT_WRITE)(void* context, const HAL_OUTPUT_RECORD* record);
typedef struct HAL_OUTPUT_ENDPOINT {
// The name remains stable for the endpoint lifetime.
const char* Name;
uint32_t StreamMask;
uint32_t Capabilities;
HAL_OUTPUT_ENDPOINT_WRITE Write;
void* Context;
} HAL_OUTPUT_ENDPOINT;
typedef struct HAL_OUTPUT_STATS {
uint64_t RecordCount;
uint64_t ByteCount;
uint64_t EndpointWriteCount;
uint64_t EmergencyRecordCount;
uint32_t EndpointCount;
} HAL_OUTPUT_STATS;
typedef struct HAL_OUTPUT_ROUTER {
// Registration is append-only; endpoint callbacks and context remain valid until shutdown.
KSTATUS (*RegisterEndpoint)(const HAL_OUTPUT_ENDPOINT* endpoint, uint32_t* endpointIdOut);
KSTATUS (*SetEndpointStreams)(uint32_t endpointId, uint32_t streamMask);
void (*Write)(const HAL_OUTPUT_RECORD* record);
void (*QueryStats)(HAL_OUTPUT_STATS* statsOut);
uint32_t SerialEndpointId;
uint32_t BootFramebufferEndpointId;
} HAL_OUTPUT_ROUTER;
typedef struct HAL_TIMER {
uint64_t (*QueryTicks)(void);
uint64_t (*QueryTickFrequency)(void);
void (*Stall)(uint32_t us);
void (*HandleInterrupt)(void);
void (*ConfigureCurrentProcessor)(void);
uint8_t IrqNumber;
} HAL_TIMER;
#define HAL_RTC_TIME_FLAG_VALID 0x0001u
#define HAL_RTC_TIME_FLAG_RTC 0x0002u
typedef struct HAL_RTC_TIME {
uint16_t Year;
uint8_t Month;
uint8_t Day;
uint8_t Hour;
uint8_t Minute;
uint8_t Second;
uint32_t Nanosecond;
uint16_t Flags;
uint16_t Reserved;
} HAL_RTC_TIME;
typedef struct HAL_RTC {
KSTATUS (*QueryTime)(HAL_RTC_TIME* outTime);
} HAL_RTC;
typedef struct HAL_PCI {
// Raw PCI configuration access. Policy-level operations such as
// enumeration, BAR decoding, interrupt discovery, and MSI programming
// belong to pci.sys.
uint32_t (*Read32)(uint16_t segment,
uint8_t bus,
uint8_t dev,
uint8_t fn,
uint16_t offset);
void (*Write32)(uint16_t segment,
uint8_t bus,
uint8_t dev,
uint8_t fn,
uint16_t offset,
uint32_t value);
} HAL_PCI;
typedef struct HAL_INTERRUPT_CONTROLLER {
void (*RouteIrq)(uint8_t irq);
void (*MaskIrq)(uint8_t irq);
void (*UnmaskIrq)(uint8_t irq);
void (*SendEoi)(void);
void (*InitializeCurrentProcessor)(void);
void (*SendProcessorInterrupt)(uint32_t hardwareId, uint8_t vector);
void (*SendProcessorNmi)(uint32_t hardwareId);
void (*BroadcastNmi)(void);
KSTATUS (*ComposeMsi)(uint8_t vector, HAL_MSI_MESSAGE* messageOut);
} HAL_INTERRUPT_CONTROLLER;
typedef struct HAL_POWER {
// ACPI S5 soft-off. Disables interrupts, writes SLP_EN to PM1a/b_CNT,
// and halts if firmware doesn't respond. Does not return.
void (*Shutdown)(void);
} HAL_POWER;
typedef struct HAL_API {
uint32_t Size;
uint32_t Version;
HAL_PROCESSOR Processor;
HAL_SCREEN Screen;
HAL_OUTPUT_ROUTER Output;
HAL_TIMER Timer;
HAL_RTC Rtc;
HAL_PCI Pci;
HAL_INTERRUPT_CONTROLLER InterruptController;
HAL_POWER Power;
} HAL_API;
typedef struct HAL_INIT_PARAMS {
uint32_t Size;
uint32_t Flags;
// Framebuffer.
uintptr_t FramebufferBase;
uint32_t FramebufferWidth;
uint32_t FramebufferHeight;
uint32_t FramebufferStride;
// ACPI.
uintptr_t RsdpPhysical;
// Memory.
uint64_t PhysicalWindowBase;
// Debug.
int Debug;
void* (*AllocatePool)(size_t size);
} HAL_INIT_PARAMS;
// Increment whenever the kernel/HAL table ABI changes.
#define HAL_API_VERSION 2u
// Returns the HAL table. Processor interrupt-state operations are valid before HalInitialize().
HAL_ENTRY const HAL_API* HalGetApi(void);
// Initializes platform discovery and the runtime portions of the HAL table.
HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* params);
code/kernel/include/kernel/hal/hal_apic.h
+12 -28
View File
@@ -1,38 +1,22 @@
// Kernel hal apic declarations shared by implementation files and kernel clients.
// HAL APIC implementation declarations. APIC details do not cross the HAL ABI.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
#include <kernel/hal/hal_api.h>
void HalApicRouteIrq(uint8_t irq);
void HalApicMaskIrq(uint8_t irq);
void HalApicUnmaskIrq(uint8_t irq);
void HalApicSendEoi(void);
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys);
#define HAL_MAX_CPUS 32u
typedef struct HAL_CPU_INFO {
uint32_t ProcessorUid;
uint32_t LapicId;
uint32_t Flags;
uint8_t Enabled;
uint8_t IsBsp;
} HAL_CPU_INFO;
typedef struct HAL_CPU_TOPOLOGY {
uint32_t CpuCount;
uint32_t BspLapicId;
HAL_CPU_INFO Cpus[HAL_MAX_CPUS];
} HAL_CPU_TOPOLOGY;
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out);
__declspec(dllexport) uint32_t HalGetBspLapicId(void);
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void);
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId);
__declspec(dllexport) void HalApicEnableCurrentCpu(void);
__declspec(dllexport) void HalApicSendPanicIpi(void);
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId);
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId);
__declspec(dllexport) void HalTimerConfigureCurrentCpu(void);
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector);
uint32_t HalApicGetBootProcessorId(void);
uint8_t HalApicUsingX2Apic(void);
uint8_t HalApicCanAddressId(uint32_t hardwareId);
void HalApicInitializeCurrentProcessor(void);
void HalApicBroadcastNmi(void);
void HalApicSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector);
void HalApicSendProcessorNmi(uint32_t hardwareId);
KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical);
KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut);
code/kernel/include/kernel/hal/hal_internal.h
+7
View File
@@ -0,0 +1,7 @@
// Shared HAL implementation helpers that are not part of the kernel/HAL ABI.
#pragma once
#include <stdint.h>
void* HalPhysicalToVirtual(uint64_t physicalAddress);
code/kernel/include/kernel/hal/hal_output.h
+1 -1
View File
@@ -2,7 +2,7 @@
#pragma once
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
// Initializes the fixed HAL output endpoint table.
void HalOutputInitialize(HAL_OUTPUT_ROUTER* output);
code/kernel/include/kernel/hal/hal_processor.h
+12
View File
@@ -0,0 +1,12 @@
// HAL processor-control declarations used within hal.dll.
#pragma once
#include <stdint.h>
uintptr_t HalDisableInterrupts(void);
void HalRestoreInterrupts(uintptr_t flags);
void HalEnableInterrupts(void);
void HalWaitForInterrupt(void);
void HalHalt(void);
_Noreturn void HalStopProcessor(void);
code/kernel/include/kernel/hal/hal_timer.h
+10
View File
@@ -0,0 +1,10 @@
// Internal HAL timer declarations shared by HAL implementation files.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
#include <kernel/hal/hal_api.h>
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* out);
void HalTimerStall(uint32_t microseconds);
code/kernel/include/kernel/io_ddk.h
+2 -1
View File
@@ -1,6 +1,7 @@
// Kernel DDK-table provider declarations.
#pragma once
#include <stdint.h>
#include <ntlite/ddk/ddk.h>
@@ -8,7 +9,7 @@ struct HAL_API;
typedef struct HAL_API HAL_API;
// Initializes the kernel-owned DDK table from platform services.
void IoInitializeDdk(const HAL_API* hal);
void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase);
// Returns the kernel-owned DDK table passed to driver entrypoints.
const NTL_DDK* IoGetDdk(void);
code/kernel/include/kernel/io_file.h
+13 -13
View File
@@ -1,26 +1,26 @@
// Kernel I/O file declarations shared by filesystem, cache, and syscall code.
#pragma once
// io_file.h - VFS and filesystem provider interface.
// Current kernel-mode filesystem drivers still use the legacy file-provider
// layout. The shared declarations live in the DDK while the real provider split
// is being designed; this private header adds kernel-only async and lock entry
// points without redefining the shared types.
// The public DDK defines IO_VOLUME and provider dispatch. This private header
// adds kernel-only async and file-lock entry points.
#include <ntlite/ddk/file.h>
#include <kernel/io_async.h>
struct PROCESS;
// Initializes filesystem namespace change notification before drivers register volumes.
KSTATUS IoInitializeVolumeNamespace(void);
// Capability-relative opens rooted at an already-open directory.
KSTATUS IoOpenFileAtDirectory(DIR_OBJECT* directory, SYS_STRING_VIEW path, FILE_OBJECT** out);
KSTATUS IoCreateFileAtDirectory(DIR_OBJECT* directory, SYS_STRING_VIEW path, FILE_OBJECT** out);
KSTATUS IoOpenDirAtDirectory(DIR_OBJECT* directory, SYS_STRING_VIEW path, DIR_OBJECT** out);
// Returns the generation of the mounted-volume namespace.
uint64_t IoGetVolumeNamespaceGeneration(void);
// Waits until the mounted-volume namespace changes or the timeout expires.
KSTATUS IoWaitForVolumeNamespaceChange(uint64_t observedGeneration,
uint64_t timeoutMilliseconds);
// Generic capability-rooted namespace operations used by the syscall layer.
KSTATUS IoStatAtRoot(OBJECT* root, SYS_STRING_VIEW path, SYS_FILE_STAT* out);
KSTATUS IoDeleteFileAtRoot(OBJECT* root, SYS_STRING_VIEW path);
KSTATUS IoRenameFileAtRoots(OBJECT* oldRoot, SYS_STRING_VIEW oldPath,
OBJECT* newRoot, SYS_STRING_VIEW newPath);
KSTATUS IoCreateDirAtRoot(OBJECT* root, SYS_STRING_VIEW path);
KSTATUS IoRemoveDirAtRoot(OBJECT* root, SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file,
void* buf,
code/kernel/include/kernel/io_partition.h
+12
View File
@@ -0,0 +1,12 @@
// Kernel partition discovery declarations. This file keeps structural partition identity private to the I/O core.
#pragma once
#include <ntlite/status.h>
#include <ntlite/ddi/io_driver.h>
// Initializes dynamic partition discovery and the mandatory structural system-volume selector.
KSTATUS IoInitializePartitionManager(const char* systemVolumeSelector);
// Returns the uniquely selected system partition, or NOT_FOUND/COLLISION while discovery is incomplete or ambiguous.
KSTATUS IoQuerySystemPartition(IO_PARTITION_INFO* partitionOut);
code/kernel/include/kernel/io_volume.h
+111
View File
@@ -0,0 +1,111 @@
// Kernel mounted-volume core declarations.
#pragma once
#include <ntlite/ddk/file.h>
#include <kernel/ke_list.h>
#include <ntlite/ddi/io_driver.h>
typedef enum IO_VOLUME_STATE {
IoVolumeMounted = 1,
IoVolumeDead,
} IO_VOLUME_STATE;
typedef struct IO_VOLUME_MANAGER {
OBJECT Header;
} IO_VOLUME_MANAGER;
typedef struct IO_VOLUME_SNAPSHOT_ENTRY {
uint32_t InstanceId;
} IO_VOLUME_SNAPSHOT_ENTRY;
typedef struct IO_VOLUME_STATS {
uint64_t CreatedVolumes;
uint64_t MountedVolumes;
uint64_t RemovedVolumes;
uint64_t DeadIoRejects;
uint64_t ProviderDestroys;
} IO_VOLUME_STATS;
typedef struct IO_VOLUME_MANAGER_STATS {
uint64_t Snapshots;
uint64_t VolumeOpens;
uint64_t Waits;
uint64_t Wakeups;
} IO_VOLUME_MANAGER_STATS;
struct IO_VOLUME {
OBJECT Header;
LIST_ENTRY RegistryLink;
SPINLOCK Lock;
uint32_t InstanceId;
uint32_t State;
uint32_t RemovalReason;
uint8_t RegistryLinked;
IO_VOLUME_SOURCE Source;
DEVICE_OBJECT* BackingDevice;
IO_VOLUME_PROVIDER Provider;
};
// Initializes dynamic mounted-volume discovery notifications before providers start.
KSTATUS IoInitializeVolumeManager(void);
// References the persistent kernel volume-manager authority object.
KSTATUS IoReferenceVolumeManager(IO_VOLUME_MANAGER** managerOut);
// Returns a stable value snapshot of every currently mounted volume. The caller frees the returned array with MmFreePool.
KSTATUS IoSnapshotVolumes(IO_VOLUME_SNAPSHOT_ENTRY** entriesOut, uint32_t* countOut, uint64_t* generationOut);
// References one mounted volume by its monotonic runtime instance identity.
KSTATUS IoReferenceVolumeByInstanceId(uint32_t instanceId, IO_VOLUME** volumeOut);
// Returns the current dynamic mounted-volume registry generation.
uint64_t IoGetVolumeGeneration(void);
// Waits until the dynamic mounted-volume registry changes or the timeout expires.
KSTATUS IoWaitForVolumeChange(uint64_t observedGeneration, uint64_t timeoutMilliseconds);
// Acquires a reference only while the volume is still mounted.
bool IoVolumeTryReferenceMounted(IO_VOLUME* volume);
// References the unique mounted volume backed by one exact disk extent start.
KSTATUS IoReferenceVolumeByBacking(uint8_t diskIndex, uint64_t startLba, IO_VOLUME** volumeOut);
// Returns whether provider I/O is still permitted on this volume.
bool IoVolumeIsMounted(const IO_VOLUME* volume);
// Returns whether the mounted provider implements native async file I/O.
bool IoVolumeSupportsReadAsync(const IO_VOLUME* volume);
bool IoVolumeSupportsWriteAsync(const IO_VOLUME* volume);
void IoQueryVolumeStats(IO_VOLUME_STATS* out);
void IoQueryVolumeManagerStats(IO_VOLUME_MANAGER_STATS* out);
// Provider dispatch. All operations except Close/CloseDir reject dead volumes.
KSTATUS IoVolumeOpen(IO_VOLUME* volume, SYS_STRING_VIEW path, void** fileCtx);
KSTATUS IoVolumeRead(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
void* buffer, size_t size, size_t* bytesRead);
KSTATUS IoVolumeWrite(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
const void* buffer, size_t size, size_t* bytesWritten);
KSTATUS IoVolumeReadAsync(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
void* buffer, size_t size, IO_FS_ASYNC_COMPLETE complete,
void* completeContext);
KSTATUS IoVolumeWriteAsync(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
const void* buffer, size_t size, IO_FS_ASYNC_COMPLETE complete,
void* completeContext);
KSTATUS IoVolumeCreate(IO_VOLUME* volume, SYS_STRING_VIEW path, void** fileCtx);
KSTATUS IoVolumeDelete(IO_VOLUME* volume, SYS_STRING_VIEW path);
KSTATUS IoVolumeRename(IO_VOLUME* volume, SYS_STRING_VIEW oldPath, SYS_STRING_VIEW newPath);
KSTATUS IoVolumeCreateDir(IO_VOLUME* volume, SYS_STRING_VIEW path);
KSTATUS IoVolumeRemoveDir(IO_VOLUME* volume, SYS_STRING_VIEW path);
KSTATUS IoVolumeTruncate(IO_VOLUME* volume, void* fileCtx, uint64_t newSize, uint32_t fileFlags);
KSTATUS IoVolumeGetSize(IO_VOLUME* volume, void* fileCtx, uint64_t* sizeOut);
KSTATUS IoVolumeQueryInfo(IO_VOLUME* volume, void* fileCtx, SYS_FILE_STAT* out);
KSTATUS IoVolumeStat(IO_VOLUME* volume, SYS_STRING_VIEW path, SYS_FILE_STAT* out);
KSTATUS IoVolumeQueryIdentity(IO_VOLUME* volume, void* fileCtx, IO_FILE_IDENTITY* out);
KSTATUS IoVolumeFlush(IO_VOLUME* volume, void* fileCtx);
void IoVolumeClose(IO_VOLUME* volume, void* fileCtx);
void IoVolumeQueryStats(IO_VOLUME* volume, IO_FILESYSTEM_STATS* out);
KSTATUS IoVolumeOpenDir(IO_VOLUME* volume, SYS_STRING_VIEW path, void** dirCtx);
KSTATUS IoVolumeReadDir(IO_VOLUME* volume, void* dirCtx, KDIR_ENTRY* out);
void IoVolumeCloseDir(IO_VOLUME* volume, void* dirCtx);
code/kernel/include/kernel/ke_audio.h
+5 -8
View File
@@ -1,4 +1,4 @@
// Kernel audio-core declarations shared by the mixer, syscall layer, and selftests.
// Kernel audio-core declarations shared by the mixer and syscall layer.
#pragma once
@@ -52,7 +52,7 @@ typedef struct KE_AUDIO_STREAM KE_AUDIO_STREAM;
// Reports one endpoint snapshot without exposing provider-owned pointers.
typedef struct KE_AUDIO_ENDPOINT_INFO {
uint64_t EndpointId;
uint32_t EndpointId;
IO_AUDIO_ENDPOINT_KIND Kind;
KE_AUDIO_ENDPOINT_STATE State;
uint32_t Capabilities;
@@ -123,10 +123,10 @@ KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints,
uint32_t* countOut);
// Returns the currently selected endpoint identifier, or zero when none is usable.
uint64_t KeAudioGetDefaultEndpointId(void);
uint32_t KeAudioGetDefaultEndpointId(void);
// Creates one fixed-format logical render stream and returns a caller-owned reference.
KSTATUS KeAudioCreateStream(uint64_t endpointId,
KSTATUS KeAudioCreateStream(uint32_t endpointId,
const IO_AUDIO_FORMAT* format,
KE_AUDIO_STREAM** outStream);
@@ -174,10 +174,7 @@ KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream,
bool muted);
// Updates endpoint master software gain and separate logical mute state.
KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted);
KSTATUS KeAudioSetMasterVolume(uint32_t endpointId, float volume, bool muted);
// Copies one coherent stream snapshot.
KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut);
// Runs the final render path selftest behind KE_DEBUG_AUDIO_SELFTEST.
void KiRunAudioFinalPathSelfTest(void);
code/kernel/include/kernel/ke_boot_config.h
+20
View File
@@ -0,0 +1,20 @@
// Kernel boot configuration declarations for parsing the raw mandatory boot.ini handoff.
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <ntlite/status.h>
#define KE_BOOT_COMMAND_LINE_MAX 256u
#define KE_BOOT_SYSTEM_VOLUME_MAX 128u
typedef struct KE_BOOT_CONFIG {
uint32_t Debug;
// Opaque system-volume selector interpreted by storage policy, never as a legacy drive slot.
char SystemVolume[KE_BOOT_SYSTEM_VOLUME_MAX];
char CommandLine[KE_BOOT_COMMAND_LINE_MAX];
} KE_BOOT_CONFIG;
// Parses the kernel-owned boot.ini settings from the raw bootloader string.
KSTATUS KeParseBootConfiguration(const char* text, size_t length, KE_BOOT_CONFIG* configOut);
code/kernel/include/kernel/ke_cpu.h
+4 -4
View File
@@ -33,15 +33,15 @@ typedef struct KCPU {
struct KCPU* Self;
uint32_t CpuIndex;
uint32_t LapicId;
uint32_t AcpiProcessorId;
uint32_t HardwareId;
uint32_t FirmwareId;
uint32_t BootProcessorIndex;
volatile long State;
uint8_t IsBsp;
uint8_t IsBootProcessor;
// CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded
// from the APIC ID using CPUID leaf 0x1F/0x0B when available, with a
// from the processor hardware ID using CPUID leaf 0x1F/0x0B when available, with a
// conservative CPUID(1)/CPUID(4) fallback. PhysicalCoreIndex is a compact
// kernel-local core number used by the scheduler. CoreSiblingMask contains
// all logical CPUs that share this physical core. PackageMask contains all
code/kernel/include/kernel/ke_debug.h
+2 -2
View File
@@ -7,14 +7,14 @@
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <ntlite/ddk/debug.h>
void KeInitDebug(const HAL_API* hal, uint32_t flags);
uint32_t KeGetDebugFlags(void);
__declspec(dllexport) void DbgPrint(const char* fmt, ...);
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len);
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path);
KSTATUS KeFlushDebugLogToSystemRoot(const char* relativePath);
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...);
__declspec(dllexport) int KeHandleNmiIpi(void);
code/kernel/include/kernel/ke_display.h
+1 -1
View File
@@ -2,7 +2,7 @@
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#ifndef DISPLAY_API
#ifdef KE_DISPLAY_BUILD
code/kernel/include/kernel/ke_hal.h
+23
View File
@@ -0,0 +1,23 @@
// Kernel HAL binding. This file keeps direct hal.dll imports out of kernel subsystems.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
#include <ntlite/ddk/irql.h>
#include <kernel/hal/hal_api.h>
KSTATUS KeBindBootstrapHal(void);
KSTATUS KeInitializeHal(const HAL_INIT_PARAMS* params);
const HAL_API* KeGetHal(void);
void KeEnableInterrupts(void);
void KeWaitForInterrupt(void);
void KeHaltProcessor(void);
_Noreturn void KeStopProcessor(void);
KSTATUS KeQueryProcessorTopology(HAL_PROCESSOR_TOPOLOGY* topologyOut);
KSTATUS KeStartProcessor(uint32_t hardwareId, uintptr_t startPhysical);
void KeInitializeCurrentProcessorInterruptController(void);
void KeSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector);
void KeSendProcessorNmi(uint32_t hardwareId);
void KeBroadcastProcessorNmi(void);
void KeConfigureCurrentProcessorTimer(void);
code/kernel/include/kernel/ke_handle.h
+85
View File
@@ -0,0 +1,85 @@
// Kernel capability handle declarations shared by processes and syscall code.
#pragma once
#include <stdint.h>
#include <ntlite/handle.h>
#include <ntlite/status.h>
#include <ntlite/ddk/object.h>
#include <kernel/ke_spinlock.h>
#define KE_HANDLE_BUCKET_COUNT 256u
#define KE_CAPABILITY_MAX_DEPTH 16u
struct PROCESS;
struct KE_HANDLE_ENTRY;
struct KE_CAPTURED_HANDLE;
typedef struct KE_HANDLE_TABLE {
struct KE_HANDLE_ENTRY* Buckets[KE_HANDLE_BUCKET_COUNT];
SYS_HANDLE NextHandle;
uint32_t Count;
uint32_t Rundown;
SPINLOCK Lock;
} KE_HANDLE_TABLE;
// Initializes an empty process-local handle namespace.
void KeInitializeHandleTable(KE_HANDLE_TABLE* table);
// Releases every handle and capability grant owned by one process.
void KeRundownHandleTable(struct PROCESS* process);
// Creates one root capability handle for an object.
KSTATUS KeCreateHandle(struct PROCESS* process, OBJECT* object, SYS_RIGHTS access, SYS_HANDLE* handleOut);
// Closes one process-local handle without reusing its numeric identity.
KSTATUS KeCloseHandle(struct PROCESS* process, SYS_HANDLE handle);
// References the target object after validating type, revocation, and rights.
KSTATUS KeReferenceHandle(struct PROCESS* process,
SYS_HANDLE handle,
OBJECT_TYPE expectedType,
SYS_RIGHTS requiredRights,
OBJECT** objectOut);
// Duplicates one alias of the same capability grant into another process.
KSTATUS KeDuplicateHandle(struct PROCESS* sourceProcess,
SYS_HANDLE sourceHandle,
struct PROCESS* targetProcess,
SYS_RIGHTS desiredRights,
uint32_t duplicateFlags,
SYS_HANDLE* targetHandleOut);
// Derives one child grant whose maximum rights are an attenuated subset.
KSTATUS KeDeriveHandle(struct PROCESS* process,
SYS_HANDLE sourceHandle,
SYS_RIGHTS desiredRights,
SYS_HANDLE* derivedHandleOut);
// Revokes one grant and all grants derived from it.
KSTATUS KeRevokeHandle(struct PROCESS* process, SYS_HANDLE handle);
#define KE_HANDLE_TRANSFER_FLAG_SAME_RIGHTS 0x00000001u
#define KE_HANDLE_TRANSFER_FLAG_DERIVE 0x00000002u
// Captures one transferable capability independently of the sender's handle lifetime.
KSTATUS KeCaptureHandleForTransfer(struct PROCESS* sourceProcess,
SYS_HANDLE sourceHandle,
SYS_RIGHTS desiredRights,
uint32_t flags,
struct KE_CAPTURED_HANDLE** capturedOut);
// Releases one captured capability that was not materialized.
void KeReleaseCapturedHandle(struct KE_CAPTURED_HANDLE* captured);
// Atomically materializes captured capabilities as new handles in one target process.
KSTATUS KeMaterializeCapturedHandles(struct PROCESS* targetProcess,
struct KE_CAPTURED_HANDLE** captured,
uint32_t count,
SYS_HANDLE* handlesOut);
// Returns the number of currently live handles in one process.
uint32_t KeGetHandleCount(struct PROCESS* process);
// Returns the object-type-specific rights mask accepted by the handle core.
SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type);
code/kernel/include/kernel/ke_id.h
+15
View File
@@ -0,0 +1,15 @@
// Kernel monotonic identifier allocator declarations.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
typedef struct KE_ID_ALLOCATOR {
volatile long long Next;
} KE_ID_ALLOCATOR;
#define KE_ID_ALLOCATOR_INIT(first_) { (long long)(uint32_t)(first_) }
// Allocates one 32-bit identifier and never reuses an issued value during the boot.
KSTATUS KeAllocateId32(KE_ID_ALLOCATOR* allocator, uint32_t* idOut);
code/kernel/include/kernel/ke_idt.h
+1 -1
View File
@@ -52,5 +52,5 @@ extern void KeFaultUD(void); // #UD invalid opcode
extern void KeNmiIpiStub(void); // returning NMI SMP-control stub
// CPU-local IPI vector. This uses the common interrupt entry machinery but is
// not an external/device IRQ and is never routed through the I/O APIC.
// not an external/device IRQ and is never routed through the external interrupt controller.
#define KE_VECTOR_RESCHEDULE_IPI 0xF0u
code/kernel/include/kernel/ke_input.h
+1 -1
View File
@@ -4,7 +4,7 @@
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/input.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <ntlite/status.h>
#include <kernel/ob_object.h>
#include <kernel/ke_event.h>
code/kernel/include/kernel/ke_irq.h
+2 -2
View File
@@ -2,10 +2,10 @@
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <ntlite/status.h>
void KeIrqInitialize(const HAL_API *hal);
void KeIrqInitialize(const HAL_API* hal);
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq);
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void));
void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm
code/kernel/include/kernel/ke_named_queue.h
+2 -1
View File
@@ -3,9 +3,10 @@
#pragma once
#include <stdint.h>
#include <ntlite/handle.h>
struct PROCESS;
struct OBJECT;
// Removes registry entries owned by the given process handle/object pair.
void KeNamedQueueUnpublishHandle(struct PROCESS* process, uint32_t handle, struct OBJECT* object);
void KeNamedQueueUnpublishHandle(struct PROCESS* process, SYS_HANDLE handle, struct OBJECT* object);
code/kernel/include/kernel/ke_port.h
+98
View File
@@ -0,0 +1,98 @@
// Kernel Port IPC declarations.
#pragma once
#include <stdint.h>
#include <kernel/ob_object.h>
#include <kernel/ke_event.h>
#include <ntlite/port.h>
struct PROCESS;
struct THREAD;
typedef struct PORT PORT;
typedef struct KE_PORT_STATS {
uint64_t ConnectionsCreated;
uint64_t DirectDeliveries;
uint64_t DirectBytes;
uint64_t QueuedMessages;
uint64_t QueuedBytes;
uint64_t QueuedDeliveries;
uint64_t RepliesCompleted;
uint64_t Disconnects;
uint64_t ProcessQuotaRejects;
} KE_PORT_STATS;
// Copies a lock-free diagnostic snapshot of Port transport counters.
void KePortQueryStats(KE_PORT_STATS* statsOut);
// Creates one waitable connection-listener Port.
KSTATUS KePortCreate(uint32_t backlog, PORT** portOut);
// Returns whether one Port object is a listener rather than a connected endpoint.
bool KePortIsListener(const PORT* port);
// Returns the passive wait event for a listener or endpoint.
EVENT* KePortWaitEvent(PORT* port);
// Connects one process to a listener and returns its new endpoint handle.
KSTATUS KePortConnect(PORT* listener,
struct PROCESS* clientProcess,
struct THREAD* clientThread,
SYS_HANDLE* endpointHandleOut);
// Accepts one queued connection and returns a new server endpoint handle.
KSTATUS KePortAccept(PORT* listener,
struct PROCESS* serverProcess,
uint64_t timeoutMilliseconds,
SYS_HANDLE* endpointHandleOut,
SYS_PORT_CONNECTION_INFO* infoOut);
// Sends one copied one-way notification, with direct user-to-user delivery when a receiver is already waiting.
KSTATUS KePortSend(PORT* endpoint,
struct PROCESS* senderProcess,
struct THREAD* senderThread,
const void* payload,
uint32_t payloadLength,
const SYS_PORT_HANDLE_TRANSFER* transfers,
uint32_t transferCount,
uint64_t timeoutMilliseconds);
// Receives one request or notification and materializes transferred capabilities in the receiving process.
KSTATUS KePortReceive(PORT* endpoint,
struct PROCESS* receiverProcess,
void* payload,
uint32_t payloadCapacity,
SYS_HANDLE* handlesOut,
uint32_t handleCapacity,
SYS_PORT_MESSAGE_INFO* infoOut,
uint64_t timeoutMilliseconds);
// Performs one synchronous copied request/reply exchange.
KSTATUS KePortCall(PORT* endpoint,
struct PROCESS* callerProcess,
struct THREAD* callerThread,
const void* requestPayload,
uint32_t requestLength,
const SYS_PORT_HANDLE_TRANSFER* requestTransfers,
uint32_t requestTransferCount,
void* replyPayload,
uint32_t replyCapacity,
SYS_HANDLE* replyHandlesOut,
uint32_t replyHandleCapacity,
SYS_PORT_MESSAGE_INFO* replyInfoOut,
uint64_t timeoutMilliseconds);
// Replies to one request previously delivered on this endpoint.
KSTATUS KePortReply(PORT* endpoint,
struct PROCESS* senderProcess,
struct THREAD* senderThread,
uint32_t messageId,
const void* payload,
uint32_t payloadLength,
const SYS_PORT_HANDLE_TRANSFER* transfers,
uint32_t transferCount);
// Disconnects one endpoint and wakes peer waiters and pending synchronous callers.
KSTATUS KePortDisconnect(PORT* endpoint);
code/kernel/include/kernel/ke_process.h
+34 -57
View File
@@ -8,48 +8,26 @@
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_event.h>
#include <kernel/ke_handle.h>
#include <stddef.h>
#include <stdint.h>
#include <ntlite/string_view.h>
#include <kernel/ke_userenv.h>
#define PROCESS_NAME_MAX 64
#define HANDLE_TABLE_SIZE 256
#define PROCESS_MODULE_MAX 16
#define PROCESS_MODULE_NAME_MAX 64
#define PROCESS_CWD_MAX 512
#define KE_PROCESS_FLAG_SYSTEM 0x00000001u
#define KE_PROCESS_FLAG_IDLE 0x00000002u
#define KE_PROCESS_FLAG_TERMINATION_PROTECTED 0x00000004u
#define HANDLE_RIGHT_READ 0x00000001u
#define HANDLE_RIGHT_WRITE 0x00000002u
#define HANDLE_RIGHT_WAIT 0x00000004u
#define HANDLE_RIGHT_SIGNAL 0x00000008u
#define HANDLE_RIGHT_DUPLICATE 0x00000010u
#define HANDLE_RIGHT_QUERY 0x00000020u
#define HANDLE_RIGHT_TERMINATE 0x00000040u
#define HANDLE_RIGHT_LOCK 0x00000080u
#define HANDLE_RIGHT_RESUME 0x00000100u
#define HANDLE_RIGHT_ALL 0xFFFFFFFFu
#define KE_PROCESS_BUILD_STARTED 0l
#define KE_PROCESS_BUILD_CONSTRUCTING 1l
#define KE_PROCESS_BUILD_STARTING 2l
#define KE_PROCESS_BUILD_DEAD 3l
struct SYS_EXCEPTION_STATS;
struct SECURITY_CONTEXT;
typedef struct HANDLE_ENTRY {
OBJECT* Object;
uint32_t Rights;
uint32_t Flags;
} HANDLE_ENTRY;
typedef struct PROCESS_MODULE {
char Name[PROCESS_MODULE_NAME_MAX];
uintptr_t Base;
size_t Size;
uintptr_t EntryPoint;
uint32_t Flags;
uint32_t PublicRefCount;
uint32_t ImportRefCount;
} PROCESS_MODULE;
typedef struct PROCESS {
OBJECT Header; // must be first
@@ -58,27 +36,36 @@ typedef struct PROCESS {
uint32_t Flags;
char Name[PROCESS_NAME_MAX]; // UTF-8
char CurrentDirectory[PROCESS_CWD_MAX];
size_t CurrentDirectoryLength;
uint8_t CurrentDrive;
SPINLOCK CwdLock;
// Live thread set. Dead THREAD objects may outlive this membership while
// retained handles remain waitable and queryable.
LIST_ENTRY ThreadList; // links THREAD.ProcessLink
uint32_t ThreadCount;
SPINLOCK ThreadLock;
// Handle table.
// Index 0 is always empty (the invalid handle).
// Each live entry holds one object reference plus rights metadata.
HANDLE_ENTRY Handles[HANDLE_TABLE_SIZE];
SPINLOCK HandleLock;
// Process-local capability namespace. Handle IDs are monotonic and never reused.
KE_HANDLE_TABLE HandleTable;
// Port IPC kernel-resource accounting. Direct and queued messages are charged to the sender.
SPINLOCK PortQuotaLock;
uint32_t PortOutstandingMessages;
uint32_t PortOutstandingBytes;
uint32_t PortOutstandingHandles;
uint32_t PortOutstandingCalls;
// Immutable process authority. Address-space reachability is not security authority.
struct SECURITY_CONTEXT* SecurityContext;
int32_t ExitCode;
volatile long Terminating;
volatile long Exited;
volatile long ResourcesReleased;
// Userspace image construction is authorized by SYS_PROCESS_RIGHT_BUILD.
// BuildOperations closes the race between memory construction and the one-way start transition.
volatile long BuildState;
volatile long BuildOperations;
// Main executable image. Used for cleanup.
uintptr_t ImageBase;
size_t ImageSize;
@@ -90,8 +77,8 @@ typedef struct PROCESS {
size_t UserImagePathSize;
uintptr_t UserCmdlineBase;
size_t UserCmdlineSize;
uintptr_t UserCwdBase;
size_t UserCwdSize;
uintptr_t UserStartupCapabilitiesBase;
size_t UserStartupCapabilitiesSize;
// Per-process user virtual regions. The actual VA arena is global because
// NT-Lite currently uses one shared address space; this list tracks
@@ -99,11 +86,6 @@ typedef struct PROCESS {
LIST_ENTRY VadList;
SPINLOCK VadLock;
// Per-process loaded modules (user processes only).
PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
uint32_t ModuleCount;
SPINLOCK ModuleLock;
// Native structured-exception dispatcher registration. The user-mode
// personality owns VEH, SEH, signal, and unwind policy behind this hook.
uintptr_t ExceptionDispatcher;
@@ -129,12 +111,16 @@ typedef struct PROCESS {
LIST_ENTRY GlobalLink; // node in the global process list
} PROCESS;
KSTATUS KeCreateProcess(const char* name, PROCESS **out);
KSTATUS KeCreateProcess(const char* name, struct SECURITY_CONTEXT* securityContext, PROCESS** out);
KSTATUS KeCreateSystemProcess(PROCESS** out);
KSTATUS KeSetupUserProcessEnvironment(PROCESS* process,
SYS_STRING_VIEW imagePath,
SYS_STRING_VIEW commandLine,
SYS_STRING_VIEW currentDirectory);
SYS_STRING_VIEW commandLine);
KSTATUS KeSetProcessStartupCapabilities(PROCESS* process,
const SYS_STARTUP_CAPABILITY* capabilities,
uint32_t count);
KSTATUS KeSetProcessLoaderData(PROCESS* process, const void* loaderData);
void KeStopProcessConstruction(PROCESS* process);
KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode);
void KeCompleteProcessExitIfLastThread(PROCESS* process);
@@ -143,15 +129,6 @@ int KeIsProcessTerminationProtected(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int32_t* exitCodeOut);
void KeQueryExceptionStats(struct SYS_EXCEPTION_STATS* statsOut);
// Handle table operations.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut);
KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t rights, uint32_t* handleOut);
void KeCloseHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights);
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle);
KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle, PROCESS* targetProcess, uint32_t desiredRights, uint32_t duplicateFlags, uint32_t* targetHandleOut);
// Return the global process list head (kernel-internal).
LIST_ENTRY* KeGetProcessList(void);
PROCESS* KeGetCurrentProcess(void);
code/kernel/include/kernel/ke_queue.h
+1 -1
View File
@@ -33,6 +33,6 @@ typedef struct QUEUE {
EVENT* NotFullEvent;
} QUEUE;
KSTATUS KeQueueCreate(QUEUE **out);
KSTATUS KeQueueCreate(QUEUE** out);
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds);
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds);
code/kernel/include/kernel/ke_sched.h
+2 -2
View File
@@ -2,7 +2,7 @@
#pragma once
// The scheduler combines dynamic normal queues with static real-time queues.
// LAPIC timer IRQs drive KeSchedulerTick.
// Per-processor timer IRQs drive KeSchedulerTick.
#include <kernel/ke_thread.h>
struct KCPU;
@@ -97,7 +97,7 @@ void KeReadyThread(THREAD* thread);
void KeUnreadyThread(THREAD* thread);
// Called from each CPU timer ISR every SCHED_TICK_MS. Every CPU performs local
// quantum accounting and may request a local reschedule. Only the BSP advances
// quantum accounting and may request a local reschedule. Only the boot processor advances
// global scheduler time and services sleepers, waitable timers, input repeat,
// and priority aging. The actual context switch happens from KiExitInterrupt
// after the IRQ handler has returned and EOI has been sent.
code/kernel/include/kernel/ke_selftest.h
+1 -1
View File
@@ -3,7 +3,7 @@
#pragma once
// ke_selftest.h - kernel bring-up and regression self-tests.
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
void KiRunCoreHardeningSelfTests(void);
void KiRunEarlyKernelSelfTests(void);
code/kernel/include/kernel/ke_semaphore.h
+1 -1
View File
@@ -16,6 +16,6 @@ typedef struct SEMAPHORE {
LIST_ENTRY WaitList;
} SEMAPHORE;
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out);
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE** out);
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* semaphore, uint64_t timeoutMilliseconds);
__declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* semaphore);
code/kernel/include/kernel/ke_settings.h
+1 -1
View File
@@ -5,7 +5,7 @@
#include <stddef.h>
#include <ntlite/status.h>
#define KE_SETTINGS_PATH "system\\settings.ini"
#define KE_SETTINGS_PATH "system/settings.ini"
__declspec(dllexport) KSTATUS KeGetSettingString(const char* section,
const char* key,
code/kernel/include/kernel/ke_smp.h
+1 -1
View File
@@ -22,5 +22,5 @@ void KeBuildProcessorTopology(void);
// Run explicit AP-affine kernel workers that exercise the global scheduler,
// reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver
// threads remain BSP-affine unless their affinity is explicitly widened.
// threads remain boot-processor-affine unless their affinity is explicitly widened.
KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void);
code/kernel/include/kernel/ke_syscall.h
+2 -2
View File
@@ -3,11 +3,11 @@
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
// Set the HAL API pointer for syscalls.
void KeSyscallInit(const HAL_API* hal);
// Syscall dispatcher, called from assembly entry point.
uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5);
uint64_t a2, uint64_t a3);
code/kernel/include/kernel/ke_system_root.h
+7 -12
View File
@@ -1,25 +1,20 @@
// System root volume declarations. This file exposes the one selected kernel root volume to loader and process code.
// System root volume declarations. This file exposes the one selected mounted volume to bootstrap code.
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <ntlite/status.h>
#include <kernel/io_file.h>
#define KE_SYSTEM_ROOT_MAX_DRIVES 8
// Selects the mounted volume that contains the kernel/system tree.
// Selects the mounted volume backed by the configured structural system partition.
KSTATUS KeInitializeSystemRootVolume(void);
// Returns whether the system root volume has been selected.
bool KeSystemRootReady(void);
// Returns the selected system root drive, or zero before discovery.
uint8_t KeGetSystemRootDrive(void);
// References the selected mounted system volume. The caller releases the returned object reference.
KSTATUS KeReferenceSystemRootVolume(IO_VOLUME** volumeOut);
// Builds an absolute path rooted at the selected system volume.
bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize);
// Opens a file rooted at the selected system volume.
// Opens, creates, or deletes canonical paths rooted directly at the selected system volume.
KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out);
KSTATUS KeCreateSystemRootFile(const char* relative, FILE_OBJECT** out);
KSTATUS KeDeleteSystemRootFile(const char* relative);
code/kernel/include/kernel/ke_system_service.h
+24
View File
@@ -0,0 +1,24 @@
// Kernel well-known system-service registry declarations.
#pragma once
#include <ntlite/service.h>
#include <ntlite/status.h>
#include <ntlite/security.h>
struct PORT;
// Registers one Port listener under a numeric well-known service identity.
KSTATUS KeSystemServiceRegister(SYS_SERVICE_ID serviceId, struct PORT* listener);
// Removes a registration only when it still refers to the supplied listener.
KSTATUS KeSystemServiceUnregister(SYS_SERVICE_ID serviceId, struct PORT* listener);
// References the listener currently registered for one service identity.
KSTATUS KeSystemServiceReference(SYS_SERVICE_ID serviceId, struct PORT** listenerOut);
// Removes every service entry that refers to a listener being destroyed.
void KeSystemServiceUnregisterListener(struct PORT* listener);
// Returns the security-context claim required to register one service identity.
SYS_SERVICE_CLAIMS KeSystemServiceRequiredClaim(SYS_SERVICE_ID serviceId);
code/kernel/include/kernel/ke_thread.h
+3
View File
@@ -20,6 +20,9 @@
typedef struct THREAD THREAD;
struct MUTEX;
// Allocates one boot-unique thread identifier.
KSTATUS KeAllocateThreadId(uint32_t* threadIdOut);
#ifndef NTL_THREAD_PRIORITY_DEFINED
typedef enum THREAD_PRIORITY {
PriorityLow = 0,
code/kernel/include/kernel/ke_time.h
+6
View File
@@ -14,3 +14,9 @@ __declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void);
// Returns the monotonic platform counter in microseconds.
uint64_t KeQueryMonotonicTicks(void);
// Returns a saturated monotonic deadline for a relative millisecond interval.
uint64_t KeMonotonicDeadlineAfterMilliseconds(uint64_t milliseconds);
// Returns the remaining whole milliseconds, rounded up, until a monotonic deadline.
uint64_t KeMonotonicMillisecondsUntil(uint64_t deadlineMicroseconds);
code/kernel/include/kernel/ke_timer.h
+2 -2
View File
@@ -13,8 +13,8 @@ typedef struct KTIMER {
OBJECT Header;
EVENT* Event;
LIST_ENTRY Link;
uint64_t DueTick;
uint64_t PeriodTicks;
uint64_t DueMicroseconds;
uint64_t PeriodMicroseconds;
uint8_t Active;
uint8_t Linked;
} KTIMER;
code/kernel/include/kernel/ke_trap.h
+1 -2
View File
@@ -82,6 +82,5 @@ void KiExitTrapFrame(KTRAP_FRAME* frame);
KTRAP_FRAME* KeGetCurrentTrapFrame(void);
int KeTrapFrameFromUser(const KTRAP_FRAME* frame);
void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4,
uint64_t a5);
uint64_t a2, uint64_t a3);
KSTATUS KeQueryCurrentTrapFrame(SYS_TRAP_FRAME_INFO_K* out);
code/kernel/include/kernel/ke_userenv.h
+18 -3
View File
@@ -7,15 +7,26 @@
#include <stdint.h>
#include <stddef.h>
#include <ntlite/string_view.h>
#include <ntlite/handle.h>
#define SYS_PEB_MAGIC 0x30424550u // "PEB0"
#define SYS_TEB_MAGIC 0x30424554u // "TEB0"
#define SYS_ENV_VERSION 1u
#define SYS_CWD_MAX 520u
#define SYS_ENV_VERSION 4u
#ifndef SYS_USER_ENV_TYPES_DEFINED
#define SYS_USER_ENV_TYPES_DEFINED 1
#define SYS_STARTUP_CAP_SYSTEM_VOLUME 1u
#define SYS_STARTUP_CAP_SERVICE_CONTROL 2u
#define SYS_STARTUP_CAP_SYSTEM_VOLUME_NAMESPACE 3u
#define SYS_STARTUP_CAP_VOLUME_MANAGER 4u
#define SYS_STARTUP_CAPABILITY_MAX 32u
typedef struct SYS_STARTUP_CAPABILITY {
uint32_t Tag;
SYS_HANDLE Handle;
} SYS_STARTUP_CAPABILITY;
typedef struct SYS_PEB {
uint32_t Magic;
uint32_t Version;
@@ -23,7 +34,11 @@ typedef struct SYS_PEB {
SYS_STRING_VIEW ImagePath;
SYS_STRING_VIEW CommandLine;
SYS_STRING_VIEW CurrentDirectory;
const SYS_STARTUP_CAPABILITY* StartupCapabilities;
uint32_t StartupCapabilityCount;
uint32_t Reserved;
const void* LoaderData;
} SYS_PEB;
typedef struct SYS_TEB {
code/kernel/include/kernel/ke_util.h
+7 -7
View File
@@ -25,25 +25,25 @@ typedef struct {
#define _ARR_INIT_CAPACITY 2
#define _arr_header(arr) ((_ArrHeader *)(arr) - 1)
#define _arr_header(arr) ((_ArrHeader*)(arr) - 1)
#define arr_len(arr) ((arr) ? _arr_header(arr)->count : 0u)
#define _arr_maybe_grow(arr) \
do { \
if (!(arr)) { \
_ArrHeader *_h = MmAllocatePool(sizeof(_ArrHeader) + \
_ArrHeader* _h = MmAllocatePool(sizeof(_ArrHeader) + \
sizeof(*(arr)) * _ARR_INIT_CAPACITY); \
_h->count = 0; \
_h->capacity = _ARR_INIT_CAPACITY; \
(arr) = (void *)(_h + 1); \
(arr) = (void*)(_h + 1); \
} \
_ArrHeader *_h = _arr_header(arr); \
_ArrHeader* _h = _arr_header(arr); \
if (_h->count >= _h->capacity) { \
_h->capacity *= 2; \
_h = MmReallocatePool(_h, sizeof(_ArrHeader) + \
sizeof(*(arr)) * _h->capacity); \
(arr) = (void *)(_h + 1); \
(arr) = (void*)(_h + 1); \
} \
} while (0)
@@ -56,7 +56,7 @@ typedef struct {
#define arr_push_front(arr, x) \
do { \
_arr_maybe_grow(arr); \
_ArrHeader *_h = _arr_header(arr); \
_ArrHeader* _h = _arr_header(arr); \
memmove(&(arr)[1], &(arr)[0], sizeof(*(arr)) * _h->count); \
(arr)[0] = (x); \
_h->count++; \
@@ -65,7 +65,7 @@ typedef struct {
#define arr_remove(arr, index) \
do { \
if ((arr) && (size_t)(index) < arr_len(arr)) { \
_ArrHeader *_h = _arr_header(arr); \
_ArrHeader* _h = _arr_header(arr); \
memmove(&(arr)[index], &(arr)[(index) + 1], \
sizeof(*(arr)) * (_h->count - (size_t)(index) - 1)); \
_h->count--; \
code/kernel/include/kernel/ldr_image.h
+6 -19
View File
@@ -10,7 +10,6 @@
struct PROCESS;
struct FILE_OBJECT;
struct SYS_MODULE_INFO;
// Map a PE32+ image from a bootloader/driver raw buffer into kernel space.
// User images must be loaded from FILE_OBJECTs so image sections have stable identity.
@@ -22,7 +21,7 @@ size_t LdrGetImageSize(uintptr_t base);
size_t LdrGetMappedImageSize(struct PROCESS* process, uintptr_t base);
// Look up a named export in a loaded module.
// 'moduleName' must match a name in BOOT_INFO.Modules (e.g., "hal.dll").
// 'moduleName' must match a registered kernel module name such as "hal.dll".
void* LdrGetExport(const char* moduleName, const char* functionName);
// Get the entry point address of a loaded image.
@@ -31,26 +30,14 @@ uintptr_t LdrGetEntryPoint(uintptr_t base);
// Resolve the import table of an already-loaded image using the global registry.
KSTATUS LdrResolveImports(uintptr_t imageBase);
// Resolve imports using a process-local module registry.
KSTATUS LdrResolveImportsForProcess(struct PROCESS* process, uintptr_t imageBase);
// Requires that one mapped bootstrap user image declare no import table.
KSTATUS LdrRequireNoImports(uintptr_t imageBase);
// Apply final page permissions for a loaded image after relocations/imports.
KSTATUS LdrProtectImageForProcess(struct PROCESS* process, uintptr_t imageBase, bool user);
// Load a DLL for a specific process, resolve it against that process's
// loaded modules, and register it in the process-local registry.
#define LDR_MODULE_LOAD_IMPORT 0x00000001u
#define LDR_MODULE_LOAD_PUBLIC 0x00000002u
KSTATUS LdrLoadModuleForProcess(struct PROCESS* process, const char* filename, uint32_t loadFlags, uintptr_t* baseOut);
KSTATUS LdrUnloadModuleForProcess(struct PROCESS* process, uintptr_t imageBase);
KSTATUS LdrQueryModuleForProcess(struct PROCESS* process, uintptr_t imageBase, struct SYS_MODULE_INFO* infoOut);
KSTATUS LdrCopyProcessModuleInfo(struct PROCESS* process, struct SYS_MODULE_INFO* infoOut, size_t maxCount, size_t* countOut);
void* LdrGetModuleExportForProcess(struct PROCESS* process, uintptr_t imageBase, const char* functionName);
// Called by KernelMain to register all modules from BOOT_INFO so that
// LdrGetExport can find them.
// Called by KernelMain to register the fixed kernel/HAL bootstrap images so LdrGetExport can find them.
struct BOOT_INFO; // forward declaration
void LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
void LdrRegisterModule(const char* name, uintptr_t base);
KSTATUS LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
KSTATUS LdrRegisterModule(const char* name, uintptr_t base);
const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut, size_t* sizeOut);
code/kernel/include/kernel/mm_file_cache.h
+3
View File
@@ -10,6 +10,8 @@
struct FILE_OBJECT;
typedef struct FILE_OBJECT FILE_OBJECT;
struct IO_VOLUME;
typedef struct IO_VOLUME IO_VOLUME;
typedef struct MM_FILE_CACHE_PAGE {
FILE_OBJECT* File;
@@ -74,6 +76,7 @@ KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file);
// by callers before destructive metadata operations.
void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size);
void MmFileCacheInvalidateAll(FILE_OBJECT* file);
void MmFileCacheDiscardVolume(IO_VOLUME* volume);
// Reclaims one cache-owned page if possible. Safe to call under memory pressure;
// it never writes back because v1 stores only clean file data.
code/kernel/include/kernel/mm_swap.h
+1 -2
View File
@@ -5,7 +5,6 @@
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <kernel/mm_phys.h>
typedef struct MM_SWAP_STATS {
@@ -23,7 +22,7 @@ typedef struct MM_SWAP_STATS {
uint64_t SlotAllocationFailures;
} MM_SWAP_STATS;
KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes);
KSTATUS MmSwapInitialize(const char* relativePath, uint64_t sizeBytes);
int MmSwapIsEnabled(void);
KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut);
KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame);
code/kernel/include/kernel/mm_virt.h
+3
View File
@@ -56,6 +56,9 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size);
// kernel-owned buffers, not raw user pointers, unless explicitly documented.
__declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size);
__declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size);
// Copies bytes directly between two process-owned user ranges while pinning both sides.
__declspec(dllexport) KSTATUS MmCopyUserToUser(struct PROCESS* sourceProcess, const void* source,
struct PROCESS* targetProcess, void* target, size_t size);
typedef struct MM_PINNED_USER_PAGE {
uintptr_t UserPage;
code/kernel/include/kernel/net_socket.h
+1 -1
View File
@@ -54,7 +54,7 @@ typedef struct NET_ASYNC_SOCKET_SUBMIT {
size_t OptionValueSize;
size_t* OptionActualSizeOut;
uint32_t* HandleOut;
SYS_HANDLE* HandleOut;
uint32_t* PollReturnedEventsOut;
IO_STATUS_BLOCK* Iosb;
code/kernel/include/kernel/se_security.h
+42
View File
@@ -0,0 +1,42 @@
// Kernel security context declarations.
#pragma once
#include <stdbool.h>
#include <ntlite/ddk/object.h>
#include <ntlite/security.h>
struct PROCESS;
typedef struct SECURITY_CONTEXT {
OBJECT Header;
uint32_t SecurityId;
SYS_PRIVILEGES EffectivePrivileges;
SYS_PRIVILEGES DelegablePrivileges;
SYS_SERVICE_CLAIMS ServiceClaims;
SYS_SERVICE_CLAIMS DelegableServiceClaims;
uint32_t Flags;
} SECURITY_CONTEXT;
// Initializes the immutable root security context used during bootstrap.
KSTATUS SeInitializeSecurity(void);
// Returns the bootstrap root context. The returned pointer is owned by the security subsystem.
SECURITY_CONTEXT* SeGetRootSecurityContext(void);
// Derives one immutable child context without allowing privilege or service-claim amplification.
KSTATUS SeDeriveSecurityContext(SECURITY_CONTEXT* source,
SYS_PRIVILEGES effectivePrivileges,
SYS_PRIVILEGES delegablePrivileges,
SYS_SERVICE_CLAIMS serviceClaims,
SYS_SERVICE_CLAIMS delegableServiceClaims,
SECURITY_CONTEXT** contextOut);
// Returns whether one process currently carries all requested ambient privileges.
bool SeProcessHasPrivilege(const struct PROCESS* process, SYS_PRIVILEGES privileges);
// Returns whether one security context carries all requested service claims.
bool SeContextHasServiceClaims(const SECURITY_CONTEXT* context, SYS_SERVICE_CLAIMS claims);
// Copies one immutable security context into the public query layout.
void SeQuerySecurityContext(const SECURITY_CONTEXT* context, SYS_SECURITY_CONTEXT_INFO* infoOut);
code/kernel/include/kernel/syscall_nr.h
+160 -142
View File
@@ -4,167 +4,185 @@
// INT 0x40 calling convention, MS x64.
// RAX selects the syscall number. RCX, RDX, R8, and R9 carry args 0-3.
// R10 and R11 carry args 4-5 and are loaded by six-argument stubs.
// Syscalls use at most four register arguments; larger operations use request structures.
// RAX carries the return value.
// Process, threads, modules, waits, APC, and exceptions.
// Process, threads, waits, APC, and exceptions.
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_OPEN_PROCESS 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_RESUME_THREAD 9
#define SYS_GET_THREAD_LIST 10
#define SYS_LOAD_MODULE 11
#define SYS_UNLOAD_MODULE 12
#define SYS_GET_MODULE_INFO 13
#define SYS_GET_MODULE_EXPORT 14
#define SYS_GET_MODULE_LIST 15
#define SYS_GET_CURRENT_TID 16
#define SYS_GET_CURRENT_PID 17
#define SYS_SET_THREAD_PRIORITY 18
#define SYS_YIELD 19
#define SYS_SLEEP 20
#define SYS_WAIT_SINGLE 21
#define SYS_WAIT_MANY 22
#define SYS_SET_USER_APC_DISPATCHER 23
#define SYS_APC_RETURN 24
#define SYS_SET_EXCEPTION_DISPATCHER 25
#define SYS_GET_EXCEPTION_DISPATCHER 26
#define SYS_CONTINUE_EXCEPTION 27
#define SYS_RAISE_EXCEPTION 28
#define SYS_CREATE_PROCESS_OBJECT 2
#define SYS_ALLOC_PROCESS_VM 3
#define SYS_PROTECT_PROCESS_VM 4
#define SYS_START_PROCESS 5
#define SYS_OPEN_PROCESS 6
#define SYS_GET_PROCESS_EXIT_CODE 7
#define SYS_KILL_PROCESS 8
#define SYS_GET_PROCESS_LIST 9
#define SYS_GET_PROCESS_PID 10
#define SYS_CREATE_THREAD 11
#define SYS_RESUME_THREAD 12
#define SYS_GET_THREAD_LIST 13
#define SYS_GET_CURRENT_TID 14
#define SYS_GET_CURRENT_PID 15
#define SYS_SET_THREAD_PRIORITY 16
#define SYS_YIELD 17
#define SYS_SLEEP 18
#define SYS_WAIT_SINGLE 19
#define SYS_WAIT_MANY 20
#define SYS_SET_USER_APC_DISPATCHER 21
#define SYS_APC_RETURN 22
#define SYS_SET_EXCEPTION_DISPATCHER 23
#define SYS_GET_EXCEPTION_DISPATCHER 24
#define SYS_CONTINUE_EXCEPTION 25
#define SYS_RAISE_EXCEPTION 26
// Virtual memory, mappings, and handles.
#define SYS_ALLOC_VM 29
#define SYS_FREE_VM 30
#define SYS_PROTECT_VM 31
#define SYS_QUERY_VM 32
#define SYS_CLOSE_HANDLE 33
#define SYS_DUPLICATE_HANDLE 34
#define SYS_CREATE_FILE_MAPPING 35
#define SYS_MAP_FILE 36
#define SYS_ALLOC_VM 27
#define SYS_FREE_VM 28
#define SYS_PROTECT_VM 29
#define SYS_QUERY_VM 30
#define SYS_CLOSE_HANDLE 31
#define SYS_DUPLICATE_HANDLE 32
#define SYS_DERIVE_HANDLE 33
#define SYS_REVOKE_HANDLE 34
#define SYS_DERIVE_SECURITY_CONTEXT 35
#define SYS_QUERY_SECURITY_CONTEXT 36
#define SYS_CREATE_FILE_MAPPING 37
#define SYS_MAP_FILE 38
// Synchronization objects.
#define SYS_CREATE_MUTEX 37
#define SYS_TRY_ACQUIRE_MUTEX 38
#define SYS_RELEASE_MUTEX 39
#define SYS_CREATE_SEM 40
#define SYS_TRY_WAIT_SEM 41
#define SYS_SIGNAL_SEM 42
#define SYS_CREATE_EVENT 43
#define SYS_SIGNAL_EVENT 44
#define SYS_CLEAR_EVENT 45
#define SYS_CREATE_MUTEX 39
#define SYS_TRY_ACQUIRE_MUTEX 40
#define SYS_RELEASE_MUTEX 41
#define SYS_CREATE_SEM 42
#define SYS_TRY_WAIT_SEM 43
#define SYS_SIGNAL_SEM 44
#define SYS_CREATE_EVENT 45
#define SYS_SIGNAL_EVENT 46
#define SYS_CLEAR_EVENT 47
// Queues and named queues.
#define SYS_CREATE_QUEUE 46
#define SYS_SEND_QUEUE 47
#define SYS_RECV_QUEUE 48
#define SYS_PUBLISH_QUEUE 49
#define SYS_OPEN_NAMED_QUEUE 50
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 51
#define SYS_CREATE_QUEUE 48
#define SYS_SEND_QUEUE 49
#define SYS_RECV_QUEUE 50
#define SYS_PUBLISH_QUEUE 51
#define SYS_OPEN_NAMED_QUEUE 52
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 53
#define SYS_UNPUBLISH_QUEUE 54
// Files, directories, and current directory.
#define SYS_OPEN_FILE 52
#define SYS_READ_FILE 53
#define SYS_WRITE_FILE 54
#define SYS_READ_FILE_ASYNC 55
#define SYS_WRITE_FILE_ASYNC 56
#define SYS_CANCEL_IO 57
#define SYS_SEEK_FILE 58
#define SYS_GET_FILE_POS 59
#define SYS_GET_FILE_SIZE 60
#define SYS_GET_FILE_STAT 61
#define SYS_STAT_FILE 62
#define SYS_TRUNCATE_FILE 63
#define SYS_FLUSH_FILE 64
#define SYS_DELETE_FILE 65
#define SYS_RENAME_FILE 66
#define SYS_LOCK_FILE 67
#define SYS_UNLOCK_FILE 68
#define SYS_OPEN_DIR 69
#define SYS_READ_DIR 70
#define SYS_CREATE_DIR 71
#define SYS_REMOVE_DIR 72
#define SYS_SET_CWD 73
#define SYS_REAL_PATH 74
// LPC-like Ports and well-known system services.
#define SYS_CREATE_PORT 55
#define SYS_CONNECT_PORT 56
#define SYS_ACCEPT_PORT 57
#define SYS_SEND_PORT 58
#define SYS_RECEIVE_PORT 59
#define SYS_CALL_PORT 60
#define SYS_REPLY_PORT 61
#define SYS_DISCONNECT_PORT 62
#define SYS_REGISTER_SYSTEM_SERVICE 63
#define SYS_UNREGISTER_SYSTEM_SERVICE 64
#define SYS_CONNECT_SYSTEM_SERVICE 65
// Mounted-volume manager control plane.
#define SYS_QUERY_VOLUMES 66
#define SYS_OPEN_VOLUME 67
#define SYS_WAIT_VOLUME_CHANGE 68
// Files and directories.
#define SYS_OPEN_FILE_AT 69
#define SYS_OPEN_DIR_AT 70
#define SYS_STAT_AT 71
#define SYS_DELETE_AT 72
#define SYS_RENAME_AT 73
#define SYS_CREATE_DIR_AT 74
#define SYS_REMOVE_DIR_AT 75
#define SYS_READ_FILE 76
#define SYS_WRITE_FILE 77
#define SYS_READ_FILE_ASYNC 78
#define SYS_WRITE_FILE_ASYNC 79
#define SYS_CANCEL_IO 80
#define SYS_SEEK_FILE 81
#define SYS_GET_FILE_POS 82
#define SYS_GET_FILE_SIZE 83
#define SYS_GET_FILE_STAT 84
#define SYS_TRUNCATE_FILE 85
#define SYS_FLUSH_FILE 86
#define SYS_LOCK_FILE 87
#define SYS_UNLOCK_FILE 88
#define SYS_READ_DIR 89
// Console, display, and surfaces.
#define SYS_WRITE_CONSOLE 75
#define SYS_GET_CONSOLE_INFO 76
#define SYS_SET_CONSOLE_CURSOR 77
#define SYS_FILL_CONSOLE_CHARACTER 78
#define SYS_FILL_CONSOLE_ATTRIBUTE 79
#define SYS_PUT_SCREEN_CHAR 80
#define SYS_CLEAR_SCREEN 81
#define SYS_GET_SCREEN_SIZE 82
#define SYS_QUERY_DISPLAY_INFO 83
#define SYS_OPEN_DISPLAY_CLOCK 84
#define SYS_CREATE_SURFACE 85
#define SYS_MAP_SURFACE 86
#define SYS_QUERY_SURFACE 87
#define SYS_PRESENT_SURFACE 88
#define SYS_WRITE_CONSOLE 90
#define SYS_GET_CONSOLE_INFO 91
#define SYS_SET_CONSOLE_CURSOR 92
#define SYS_FILL_CONSOLE_CHARACTER 93
#define SYS_FILL_CONSOLE_ATTRIBUTE 94
#define SYS_PUT_SCREEN_CHAR 95
#define SYS_CLEAR_SCREEN 96
#define SYS_GET_SCREEN_SIZE 97
#define SYS_QUERY_DISPLAY_INFO 98
#define SYS_OPEN_DISPLAY_CLOCK 99
#define SYS_CREATE_SURFACE 100
#define SYS_MAP_SURFACE 101
#define SYS_QUERY_SURFACE 102
#define SYS_PRESENT_SURFACE 103
// Native render audio.
#define SYS_AUDIO_ENUMERATE_ENDPOINTS 89
#define SYS_AUDIO_OPEN_STREAM 90
#define SYS_AUDIO_WRITE 91
#define SYS_AUDIO_ARM_WAIT 92
#define SYS_AUDIO_QUERY_STREAM 93
#define SYS_AUDIO_START 94
#define SYS_AUDIO_PAUSE 95
#define SYS_AUDIO_RESUME 96
#define SYS_AUDIO_DRAIN 97
#define SYS_AUDIO_STOP 98
#define SYS_AUDIO_RESET 99
#define SYS_AUDIO_SET_VOLUME 100
#define SYS_AUDIO_SET_MASTER_VOLUME 101
#define SYS_AUDIO_ENUMERATE_ENDPOINTS 104
#define SYS_AUDIO_OPEN_STREAM 105
#define SYS_AUDIO_WRITE 106
#define SYS_AUDIO_ARM_WAIT 107
#define SYS_AUDIO_QUERY_STREAM 108
#define SYS_AUDIO_START 109
#define SYS_AUDIO_PAUSE 110
#define SYS_AUDIO_RESUME 111
#define SYS_AUDIO_DRAIN 112
#define SYS_AUDIO_STOP 113
#define SYS_AUDIO_RESET 114
#define SYS_AUDIO_SET_VOLUME 115
#define SYS_AUDIO_SET_MASTER_VOLUME 116
// Input.
#define SYS_POLL_KEYBOARD 102
#define SYS_IS_KEY_DOWN 103
#define SYS_READ_KEYBOARD_CHAR 104
#define SYS_READ_KEY 105
#define SYS_GET_KEY_MODIFIERS 106
#define SYS_QUERY_GLOBAL_INPUT_STATE 107
#define SYS_OPEN_INPUT_QUEUE 108
#define SYS_READ_INPUT_EVENT 109
#define SYS_PEEK_INPUT_EVENT 110
#define SYS_PEEK_MOUSE 111
#define SYS_READ_MOUSE 112
#define SYS_GET_MOUSE_POS 113
#define SYS_POLL_KEYBOARD 117
#define SYS_IS_KEY_DOWN 118
#define SYS_READ_KEYBOARD_CHAR 119
#define SYS_READ_KEY 120
#define SYS_GET_KEY_MODIFIERS 121
#define SYS_QUERY_GLOBAL_INPUT_STATE 122
#define SYS_OPEN_INPUT_QUEUE 123
#define SYS_READ_INPUT_EVENT 124
#define SYS_PEEK_INPUT_EVENT 125
#define SYS_PEEK_MOUSE 126
#define SYS_READ_MOUSE 127
#define SYS_GET_MOUSE_POS 128
// Time, debug, and system information.
#define SYS_CREATE_TIMER 114
#define SYS_SET_TIMER 115
#define SYS_CANCEL_TIMER 116
#define SYS_GET_TICK 117
#define SYS_GET_TICK_FREQUENCY 118
#define SYS_GET_SYSTEM_TIME 119
#define SYS_PRINT_DEBUG 120
#define SYS_QUERY_TRAP_FRAME 121
#define SYS_GET_MEMORY_STATUS 122
#define SYS_QUERY_KERNEL_STATS 123
#define SYS_CREATE_TIMER 129
#define SYS_SET_TIMER 130
#define SYS_CANCEL_TIMER 131
#define SYS_GET_TICK 132
#define SYS_GET_TICK_FREQUENCY 133
#define SYS_GET_SYSTEM_TIME 134
#define SYS_PRINT_DEBUG 135
#define SYS_QUERY_TRAP_FRAME 136
#define SYS_GET_MEMORY_STATUS 137
#define SYS_QUERY_KERNEL_STATS 138
// Native sockets and polling.
#define SYS_CREATE_SOCKET 124
#define SYS_SOCKET_IO_ASYNC 125
#define SYS_BIND_SOCKET 126
#define SYS_LISTEN_SOCKET 127
#define SYS_ACCEPT_SOCKET 128
#define SYS_CONNECT_SOCKET 129
#define SYS_SEND_SOCKET 130
#define SYS_RECV_SOCKET 131
#define SYS_SHUTDOWN_SOCKET 132
#define SYS_SEND_TO_SOCKET 133
#define SYS_RECV_FROM_SOCKET 134
#define SYS_RESOLVE_HOST 135
#define SYS_SET_SOCKET_OPTION 136
#define SYS_GET_SOCKET_OPTION 137
#define SYS_POLL 138
#define SYS_UNPUBLISH_QUEUE 139
#define SYS_CREATE_SOCKET 139
#define SYS_SOCKET_IO_ASYNC 140
#define SYS_BIND_SOCKET 141
#define SYS_LISTEN_SOCKET 142
#define SYS_ACCEPT_SOCKET 143
#define SYS_CONNECT_SOCKET 144
#define SYS_SEND_SOCKET 145
#define SYS_RECV_SOCKET 146
#define SYS_SHUTDOWN_SOCKET 147
#define SYS_SEND_TO_SOCKET 148
#define SYS_RECV_FROM_SOCKET 149
#define SYS_RESOLVE_HOST 150
#define SYS_SET_SOCKET_OPTION 151
#define SYS_GET_SOCKET_OPTION 152
#define SYS_POLL 153
#define SYS_MAX 140
#define SYS_MAX 154
code/kernel/io_ddk.c
+24 -17
View File
@@ -2,8 +2,7 @@
#include <stdint.h>
#include <ntlite/ddk/ddk.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_api.h>
#include <kernel/io_ddk.h>
#include <kernel/io_dma.h>
#include <kernel/ke_audio.h>
@@ -13,12 +12,13 @@
#include <kernel/mm_pte.h>
static const HAL_API* gIoDdkHal;
static uintptr_t gIoDdkPhysicalWindowBase;
// Returns the kernel direct-map alias for a physical address.
static void* IoDdkPhysToVirt(uint64_t physicalAddress) {
if (!gIoDdkHal || !gIoDdkHal->PhysToVirt)
if (!gIoDdkPhysicalWindowBase || physicalAddress > UINTPTR_MAX - gIoDdkPhysicalWindowBase)
return (void*)0;
return gIoDdkHal->PhysToVirt(physicalAddress);
return (void*)(gIoDdkPhysicalWindowBase + (uintptr_t)physicalAddress);
}
// Maps one kernel page through the DDK memory surface.
@@ -55,21 +55,28 @@ static void IoDdkPciWrite32(uint16_t segment,
gIoDdkHal->Pci.Write32(segment, bus, device, function, offset, value);
}
// Returns the BSP local APIC identifier.
static uint32_t IoDdkGetBspLapicId(void) {
return HalGetBspLapicId();
}
// Returns whether the platform interrupt controller is using x2APIC mode.
static uint8_t IoDdkIsX2ApicEnabled(void) {
return HalApicUsingX2Apic();
}
// Allocates one kernel IRQ slot for MSI delivery.
static KSTATUS IoDdkAllocateMsiVector(uint8_t* outIrq) {
return KeIrqAllocateMsiVector(outIrq);
}
// Composes one platform MSI message for an allocated nt-lite IRQ slot.
static KSTATUS IoDdkComposeMsiMessage(uint8_t irq, NTL_DDK_MSI_MESSAGE* messageOut) {
if (!gIoDdkHal || !messageOut || irq >= 24u)
return NTLSTATUS_INVALID_PARAMETER;
HAL_MSI_MESSAGE message;
KSTATUS status = gIoDdkHal->InterruptController.ComposeMsi((uint8_t)(0x20u + irq), &message);
if (!KSUCCESS(status))
return status;
messageOut->Address = message.Address;
messageOut->Data = message.Data;
messageOut->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
static const NTL_DDK_DEBUG gIoDdkDebug = {
.Print = DbgPrint,
.GetFlags = KeGetDebugFlags,
@@ -77,6 +84,7 @@ static const NTL_DDK_DEBUG gIoDdkDebug = {
static const NTL_DDK_INTERRUPT gIoDdkInterrupt = {
.AllocateMsiVector = IoDdkAllocateMsiVector,
.ComposeMsiMessage = IoDdkComposeMsiMessage,
};
static const NTL_DDK_MEMORY gIoDdkMemory = {
@@ -87,8 +95,6 @@ static const NTL_DDK_MEMORY gIoDdkMemory = {
static const NTL_DDK_PLATFORM gIoDdkPlatform = {
.PciRead32 = IoDdkPciRead32,
.PciWrite32 = IoDdkPciWrite32,
.GetBspLapicId = IoDdkGetBspLapicId,
.IsX2ApicEnabled = IoDdkIsX2ApicEnabled,
};
static const NTL_DDK_SCHEDULER gIoDdkScheduler = {
@@ -114,8 +120,9 @@ static NTL_DDK gIoDdk = {
};
// Initializes the kernel-owned DDK table from platform services.
void IoInitializeDdk(const HAL_API* hal) {
void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase) {
gIoDdkHal = hal;
gIoDdkPhysicalWindowBase = physicalWindowBase;
gIoDdk.Audio = KeAudioGetDdk();
gIoDdk.Dma = IoGetDmaDdk();
gIoDdkTimer.QueryTicks = hal ? hal->Timer.QueryTicks : 0;
code/kernel/io_file.c
+356 -586
View File
File diff suppressed because it is too large. Load diff
code/kernel/io_partition.c
+772
View File
@@ -0,0 +1,772 @@
// Dynamic partition discovery. This file owns MBR/GPT parsing and keeps structural identity out of filesystem providers.
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <common/boot_volume_selector.h>
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddk/mutex.h>
#include <kernel/io_partition.h>
#include <kernel/ke_debug.h>
#include <kernel/mm_pool.h>
#define IOP_SECTOR_SIZE 512u
#define IOP_GPT_SIGNATURE "EFI PART"
#define IOP_GPT_REVISION_1_0 0x00010000u
#define IOP_GPT_MIN_ENTRY_SIZE 128u
#pragma pack(push, 1)
typedef struct IOP_MBR_PARTITION_ENTRY {
uint8_t BootIndicator;
uint8_t StartingChs[3];
uint8_t Type;
uint8_t EndingChs[3];
uint32_t StartingLba;
uint32_t SectorCount;
} IOP_MBR_PARTITION_ENTRY;
typedef struct IOP_GPT_HEADER {
char Signature[8];
uint32_t Revision;
uint32_t HeaderSize;
uint32_t HeaderCrc32;
uint32_t Reserved;
uint64_t MyLba;
uint64_t AlternateLba;
uint64_t FirstUsableLba;
uint64_t LastUsableLba;
BOOT_VOLUME_GUID DiskGuid;
uint64_t PartitionEntryLba;
uint32_t NumberOfPartitionEntries;
uint32_t SizeOfPartitionEntry;
uint32_t PartitionEntryArrayCrc32;
} IOP_GPT_HEADER;
typedef struct IOP_GPT_ENTRY_PREFIX {
BOOT_VOLUME_GUID PartitionTypeGuid;
BOOT_VOLUME_GUID UniquePartitionGuid;
uint64_t StartingLba;
uint64_t EndingLba;
uint64_t Attributes;
} IOP_GPT_ENTRY_PREFIX;
#pragma pack(pop)
_Static_assert(sizeof(IOP_MBR_PARTITION_ENTRY) == 16u, "MBR partition entry layout must match disk format");
_Static_assert(sizeof(IOP_GPT_HEADER) == 92u, "GPT header prefix layout must match disk format");
_Static_assert(sizeof(IOP_GPT_ENTRY_PREFIX) == 56u, "GPT entry prefix layout must match disk format");
typedef struct IOP_PARTITION_RECORD {
struct IOP_PARTITION_RECORD* Next;
IO_PARTITION_INFO Info;
BOOT_VOLUME_SELECTOR Identity;
} IOP_PARTITION_RECORD;
typedef struct IOP_PARTITION_CALLBACK_ENTRY {
struct IOP_PARTITION_CALLBACK_ENTRY* Next;
IO_PARTITION_CHANGE_CALLBACK Callback;
void* Context;
} IOP_PARTITION_CALLBACK_ENTRY;
typedef struct IOP_GPT_VALIDATED_HEADER {
IOP_GPT_HEADER Header;
uint64_t EntryBytes;
uint64_t EntrySectors;
} IOP_GPT_VALIDATED_HEADER;
static MUTEX* gPartitionMutex;
static IOP_PARTITION_RECORD* gPartitions;
static IOP_PARTITION_CALLBACK_ENTRY* gPartitionCallbacks;
static BOOT_VOLUME_SELECTOR gSystemVolumeSelector;
static bool gPartitionManagerInitialized;
// Returns whether two shared GUID values are identical.
static bool IopGuidEquals(const BOOT_VOLUME_GUID* left, const BOOT_VOLUME_GUID* right) {
return left && right && memcmp(left, right, sizeof(*left)) == 0;
}
// Returns whether one shared GUID value is all zeroes.
static bool IopGuidIsZero(const BOOT_VOLUME_GUID* guid) {
static const BOOT_VOLUME_GUID zero = {0};
return guid && IopGuidEquals(guid, &zero);
}
// Returns whether one internal partition identity equals another.
static bool IopIdentityEquals(const BOOT_VOLUME_SELECTOR* left, const BOOT_VOLUME_SELECTOR* right) {
if (!left || !right || left->Kind != right->Kind)
return false;
if (left->Kind == BootVolumeSelectorMbr) {
return left->Identity.Mbr.DiskSignature == right->Identity.Mbr.DiskSignature &&
left->Identity.Mbr.PartitionNumber == right->Identity.Mbr.PartitionNumber;
}
if (left->Kind == BootVolumeSelectorGpt) {
return IopGuidEquals(&left->Identity.Gpt.PartitionGuid,
&right->Identity.Gpt.PartitionGuid);
}
return false;
}
// Acquires the partition-manager mutex for registry and callback serialization.
static bool IopAcquirePartitionMutex(void) {
if (!gPartitionMutex)
return false;
return KE_MUTEX_ACQUIRE_SUCCEEDED(KeAcquireMutex(gPartitionMutex, KE_WAIT_INFINITE));
}
// Frees one dynamically discovered partition list.
static void IopFreePartitionList(IOP_PARTITION_RECORD* head) {
while (head) {
IOP_PARTITION_RECORD* next = head->Next;
MmFreePool(head);
head = next;
}
}
// Updates one CRC32 accumulator using the GPT/IEEE polynomial.
static uint32_t IopCrc32Update(uint32_t crc, const uint8_t* data, size_t length) {
for (size_t i = 0; i < length; ++i) {
crc ^= data[i];
for (uint32_t bit = 0; bit < 8; ++bit)
crc = (crc >> 1) ^ (0xedb88320u & (uint32_t)-(int32_t)(crc & 1u));
}
return crc;
}
// Computes a complete GPT CRC32 value.
static uint32_t IopCrc32(const void* data, size_t length) {
uint32_t crc = IopCrc32Update(0xffffffffu, (const uint8_t*)data, length);
return crc ^ 0xffffffffu;
}
// Reads an arbitrary byte span through the kernel's 512-byte block-device API.
static KSTATUS IopReadDiskBytes(uint8_t diskIndex,
uint64_t diskSectors,
uint64_t byteOffset,
void* buffer,
size_t length) {
if (!buffer || length == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (diskSectors > UINT64_MAX / IOP_SECTOR_SIZE)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t diskBytes = diskSectors * IOP_SECTOR_SIZE;
if (byteOffset > diskBytes || (uint64_t)length > diskBytes - byteOffset)
return NTLSTATUS_INVALID_PARAMETER;
uint8_t sector[IOP_SECTOR_SIZE];
uint8_t* out = (uint8_t*)buffer;
size_t remaining = length;
while (remaining) {
uint64_t lba = byteOffset / IOP_SECTOR_SIZE;
uint32_t inSector = (uint32_t)(byteOffset % IOP_SECTOR_SIZE);
size_t chunk = IOP_SECTOR_SIZE - inSector;
if (chunk > remaining)
chunk = remaining;
KSTATUS status = IoReadDiskSectors(diskIndex, lba, 1, sector);
if (!KSUCCESS(status))
return status;
memcpy(out, sector + inSector, chunk);
out += chunk;
byteOffset += chunk;
remaining -= chunk;
}
return NTLSTATUS_SUCCESS;
}
// Returns whether one MBR type is an extended-partition container we intentionally do not enumerate.
static bool IopMbrTypeIsExtended(uint8_t type) {
return type == 0x05u || type == 0x0fu || type == 0x85u;
}
// Allocates one immutable partition record.
static IOP_PARTITION_RECORD* IopAllocatePartition(uint8_t diskIndex,
uint64_t startLba,
uint64_t sectorCount,
const BOOT_VOLUME_SELECTOR* identity) {
if (!sectorCount || !identity)
return NULL;
IOP_PARTITION_RECORD* record = MmAllocatePool(sizeof(*record));
if (!record)
return NULL;
memset(record, 0, sizeof(*record));
record->Info.DiskIndex = diskIndex;
record->Info.StartLba = startLba;
record->Info.SectorCount = sectorCount;
record->Identity = *identity;
return record;
}
// Inserts one discovered partition in ascending LBA order and rejects overlap or duplicate identity.
static KSTATUS IopInsertDiscoveredPartition(IOP_PARTITION_RECORD** head,
IOP_PARTITION_RECORD* record) {
if (!head || !record || record->Info.SectorCount == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (record->Info.StartLba > UINT64_MAX - record->Info.SectorCount)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t recordEnd = record->Info.StartLba + record->Info.SectorCount;
IOP_PARTITION_RECORD* previous = NULL;
IOP_PARTITION_RECORD* current = *head;
while (current && current->Info.StartLba < record->Info.StartLba) {
if (IopIdentityEquals(&current->Identity, &record->Identity))
return NTLSTATUS_OBJECT_NAME_COLLISION;
previous = current;
current = current->Next;
}
for (IOP_PARTITION_RECORD* scan = current; scan; scan = scan->Next) {
if (IopIdentityEquals(&scan->Identity, &record->Identity))
return NTLSTATUS_OBJECT_NAME_COLLISION;
}
if (previous) {
uint64_t previousEnd = previous->Info.StartLba + previous->Info.SectorCount;
if (previousEnd > record->Info.StartLba)
return NTLSTATUS_INVALID_PARAMETER;
}
if (current && recordEnd > current->Info.StartLba)
return NTLSTATUS_INVALID_PARAMETER;
record->Next = current;
if (previous)
previous->Next = record;
else
*head = record;
return NTLSTATUS_SUCCESS;
}
// Validates the exact CRC32 of one GPT partition-entry array without imposing an entry-count cap.
static KSTATUS IopValidateGptEntryArrayCrc(uint8_t diskIndex,
const IOP_GPT_VALIDATED_HEADER* validated) {
uint8_t sector[IOP_SECTOR_SIZE];
uint64_t remaining = validated->EntryBytes;
uint64_t lba = validated->Header.PartitionEntryLba;
uint32_t crc = 0xffffffffu;
while (remaining) {
KSTATUS status = IoReadDiskSectors(diskIndex, lba++, 1, sector);
if (!KSUCCESS(status))
return status;
size_t chunk = remaining < IOP_SECTOR_SIZE ? (size_t)remaining : IOP_SECTOR_SIZE;
crc = IopCrc32Update(crc, sector, chunk);
remaining -= chunk;
}
crc ^= 0xffffffffu;
return crc == validated->Header.PartitionEntryArrayCrc32
? NTLSTATUS_SUCCESS
: NTLSTATUS_CHECKSUM_MISMATCH;
}
// Validates one primary or backup GPT header and its partition-entry-array CRC.
static KSTATUS IopValidateGptHeader(uint8_t diskIndex,
uint64_t diskSectors,
uint64_t headerLba,
uint64_t expectedAlternateLba,
IOP_GPT_VALIDATED_HEADER* validatedOut) {
if (!validatedOut || diskSectors < 3 || headerLba >= diskSectors ||
expectedAlternateLba >= diskSectors || headerLba == expectedAlternateLba) {
return NTLSTATUS_INVALID_PARAMETER;
}
uint8_t sector[IOP_SECTOR_SIZE];
KSTATUS status = IoReadDiskSectors(diskIndex, headerLba, 1, sector);
if (!KSUCCESS(status))
return status;
IOP_GPT_HEADER header;
memcpy(&header, sector, sizeof(header));
if (memcmp(header.Signature, IOP_GPT_SIGNATURE, 8) != 0 ||
header.Revision < IOP_GPT_REVISION_1_0 ||
header.HeaderSize < sizeof(IOP_GPT_HEADER) ||
header.HeaderSize > IOP_SECTOR_SIZE ||
header.Reserved != 0 ||
header.MyLba != headerLba ||
header.AlternateLba != expectedAlternateLba ||
header.FirstUsableLba > header.LastUsableLba ||
header.LastUsableLba >= diskSectors ||
IopGuidIsZero(&header.DiskGuid) ||
header.NumberOfPartitionEntries == 0 ||
header.SizeOfPartitionEntry < IOP_GPT_MIN_ENTRY_SIZE ||
(header.SizeOfPartitionEntry & 7u) != 0) {
return NTLSTATUS_INVALID_PARAMETER;
}
uint32_t expectedHeaderCrc = header.HeaderCrc32;
memset(sector + offsetof(IOP_GPT_HEADER, HeaderCrc32), 0, sizeof(header.HeaderCrc32));
if (IopCrc32(sector, header.HeaderSize) != expectedHeaderCrc)
return NTLSTATUS_CHECKSUM_MISMATCH;
if ((uint64_t)header.NumberOfPartitionEntries > UINT64_MAX / header.SizeOfPartitionEntry)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t entryBytes = (uint64_t)header.NumberOfPartitionEntries * header.SizeOfPartitionEntry;
if (entryBytes > UINT64_MAX - (IOP_SECTOR_SIZE - 1u))
return NTLSTATUS_INVALID_PARAMETER;
uint64_t entrySectors = (entryBytes + IOP_SECTOR_SIZE - 1u) / IOP_SECTOR_SIZE;
if (entrySectors == 0 || header.PartitionEntryLba >= diskSectors ||
entrySectors > diskSectors - header.PartitionEntryLba) {
return NTLSTATUS_INVALID_PARAMETER;
}
uint64_t entryLastLba = header.PartitionEntryLba + entrySectors - 1u;
if (headerLba == 1u) {
if (header.PartitionEntryLba <= headerLba || entryLastLba >= header.FirstUsableLba)
return NTLSTATUS_INVALID_PARAMETER;
} else {
if (header.PartitionEntryLba <= header.LastUsableLba || entryLastLba >= headerLba)
return NTLSTATUS_INVALID_PARAMETER;
}
IOP_GPT_VALIDATED_HEADER validated = {0};
validated.Header = header;
validated.EntryBytes = entryBytes;
validated.EntrySectors = entrySectors;
status = IopValidateGptEntryArrayCrc(diskIndex, &validated);
if (!KSUCCESS(status))
return status;
*validatedOut = validated;
return NTLSTATUS_SUCCESS;
}
// Returns whether independently valid primary and backup GPT headers describe the same table.
static bool IopGptHeadersAgree(const IOP_GPT_VALIDATED_HEADER* primary,
const IOP_GPT_VALIDATED_HEADER* backup) {
if (!primary || !backup)
return false;
return IopGuidEquals(&primary->Header.DiskGuid, &backup->Header.DiskGuid) &&
primary->Header.FirstUsableLba == backup->Header.FirstUsableLba &&
primary->Header.LastUsableLba == backup->Header.LastUsableLba &&
primary->Header.NumberOfPartitionEntries == backup->Header.NumberOfPartitionEntries &&
primary->Header.SizeOfPartitionEntry == backup->Header.SizeOfPartitionEntry &&
primary->Header.PartitionEntryArrayCrc32 == backup->Header.PartitionEntryArrayCrc32;
}
// Parses all populated GPT entries from one already validated table.
static KSTATUS IopParseGptEntries(uint8_t diskIndex,
uint64_t diskSectors,
const IOP_GPT_VALIDATED_HEADER* validated,
IOP_PARTITION_RECORD** partitionsOut) {
if (!validated || !partitionsOut)
return NTLSTATUS_INVALID_PARAMETER;
IOP_PARTITION_RECORD* partitions = NULL;
if (validated->Header.PartitionEntryLba > UINT64_MAX / IOP_SECTOR_SIZE)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t arrayByteBase = validated->Header.PartitionEntryLba * IOP_SECTOR_SIZE;
for (uint32_t index = 0; index < validated->Header.NumberOfPartitionEntries; ++index) {
uint64_t entryOffset = (uint64_t)index * validated->Header.SizeOfPartitionEntry;
if (arrayByteBase > UINT64_MAX - entryOffset) {
IopFreePartitionList(partitions);
return NTLSTATUS_INVALID_PARAMETER;
}
IOP_GPT_ENTRY_PREFIX entry;
KSTATUS status = IopReadDiskBytes(diskIndex,
diskSectors,
arrayByteBase + entryOffset,
&entry,
sizeof(entry));
if (!KSUCCESS(status)) {
IopFreePartitionList(partitions);
return status;
}
if (IopGuidIsZero(&entry.PartitionTypeGuid))
continue;
if (IopGuidIsZero(&entry.UniquePartitionGuid) ||
entry.EndingLba < entry.StartingLba ||
entry.StartingLba < validated->Header.FirstUsableLba ||
entry.EndingLba > validated->Header.LastUsableLba) {
IopFreePartitionList(partitions);
return NTLSTATUS_INVALID_PARAMETER;
}
uint64_t sectorCount = entry.EndingLba - entry.StartingLba + 1u;
BOOT_VOLUME_SELECTOR identity = {0};
identity.Kind = BootVolumeSelectorGpt;
identity.Identity.Gpt.PartitionGuid = entry.UniquePartitionGuid;
IOP_PARTITION_RECORD* record = IopAllocatePartition(diskIndex,
entry.StartingLba,
sectorCount,
&identity);
if (!record) {
IopFreePartitionList(partitions);
return NTLSTATUS_NO_MEMORY;
}
status = IopInsertDiscoveredPartition(&partitions, record);
if (!KSUCCESS(status)) {
MmFreePool(record);
IopFreePartitionList(partitions);
return status;
}
}
*partitionsOut = partitions;
return NTLSTATUS_SUCCESS;
}
// Discovers one GPT disk from primary/backup metadata and rejects inconsistent valid copies.
static KSTATUS IopDiscoverGptDisk(uint8_t diskIndex,
uint64_t diskSectors,
IOP_PARTITION_RECORD** partitionsOut) {
IOP_GPT_VALIDATED_HEADER primary = {0};
IOP_GPT_VALIDATED_HEADER backup = {0};
KSTATUS primaryStatus = IopValidateGptHeader(diskIndex, diskSectors, 1u,
diskSectors - 1u, &primary);
KSTATUS backupStatus = IopValidateGptHeader(diskIndex, diskSectors, diskSectors - 1u,
1u, &backup);
bool primaryValid = KSUCCESS(primaryStatus);
bool backupValid = KSUCCESS(backupStatus);
if (!primaryValid && !backupValid) {
DbgPrint("partition: disk %u GPT primary=0x%x backup=0x%x\n",
diskIndex, primaryStatus, backupStatus);
return primaryStatus == NTLSTATUS_CHECKSUM_MISMATCH || backupStatus == NTLSTATUS_CHECKSUM_MISMATCH
? NTLSTATUS_CHECKSUM_MISMATCH
: NTLSTATUS_INVALID_PARAMETER;
}
if (primaryValid && backupValid && !IopGptHeadersAgree(&primary, &backup)) {
DbgPrint("partition: disk %u GPT primary/backup metadata disagree\n", diskIndex);
return NTLSTATUS_INVALID_PARAMETER;
}
const IOP_GPT_VALIDATED_HEADER* selected = primaryValid ? &primary : &backup;
if (!primaryValid)
DbgPrint("partition: disk %u using valid backup GPT\n", diskIndex);
return IopParseGptEntries(diskIndex, diskSectors, selected, partitionsOut);
}
// Discovers either an MBR primary-partition map or a protective-MBR GPT map.
static KSTATUS IopDiscoverDiskPartitions(uint8_t diskIndex,
DEVICE_OBJECT* device,
IOP_PARTITION_RECORD** partitionsOut) {
if (!device || !partitionsOut)
return NTLSTATUS_INVALID_PARAMETER;
*partitionsOut = NULL;
if (device->u.Disk.SectorSize != IOP_SECTOR_SIZE || device->u.Disk.SectorCount == 0) {
DbgPrint("partition: disk %u unsupported sector geometry size=%u sectors=%llu\n",
diskIndex, device->u.Disk.SectorSize,
(unsigned long long)device->u.Disk.SectorCount);
return NTLSTATUS_NOT_SUPPORTED;
}
uint64_t diskSectors = device->u.Disk.SectorCount;
uint8_t sector[IOP_SECTOR_SIZE];
KSTATUS status = IoReadDiskSectors(diskIndex, 0, 1, sector);
if (!KSUCCESS(status))
return status;
if (sector[510] != 0x55u || sector[511] != 0xaau)
return NTLSTATUS_NOT_FOUND;
uint32_t diskSignature = 0;
memcpy(&diskSignature, sector + 440u, sizeof(diskSignature));
IOP_MBR_PARTITION_ENTRY entries[4];
memcpy(entries, sector + 446u, sizeof(entries));
bool protective = false;
for (uint32_t i = 0; i < 4; ++i) {
if (entries[i].Type == 0xeeu && entries[i].SectorCount != 0)
protective = true;
}
if (protective)
return IopDiscoverGptDisk(diskIndex, diskSectors, partitionsOut);
// Validate the complete primary table before emitting supported partitions. Extended containers remain
// unsupported, but malformed container extents must not make a partially trusted MBR map acceptable.
for (uint32_t i = 0; i < 4; ++i) {
const IOP_MBR_PARTITION_ENTRY* entry = &entries[i];
if (entry->BootIndicator != 0x00u && entry->BootIndicator != 0x80u)
return NTLSTATUS_INVALID_PARAMETER;
if (entry->Type == 0 && entry->SectorCount == 0)
continue;
if (entry->Type == 0 || entry->SectorCount == 0 ||
entry->StartingLba == 0 || entry->StartingLba >= diskSectors ||
(uint64_t)entry->SectorCount > diskSectors - entry->StartingLba) {
return NTLSTATUS_INVALID_PARAMETER;
}
uint64_t entryEnd = (uint64_t)entry->StartingLba + entry->SectorCount;
for (uint32_t previous = 0; previous < i; ++previous) {
const IOP_MBR_PARTITION_ENTRY* prior = &entries[previous];
if (prior->Type == 0 && prior->SectorCount == 0)
continue;
uint64_t priorEnd = (uint64_t)prior->StartingLba + prior->SectorCount;
if ((uint64_t)entry->StartingLba < priorEnd &&
(uint64_t)prior->StartingLba < entryEnd) {
return NTLSTATUS_INVALID_PARAMETER;
}
}
}
IOP_PARTITION_RECORD* partitions = NULL;
for (uint32_t i = 0; i < 4; ++i) {
const IOP_MBR_PARTITION_ENTRY* entry = &entries[i];
if (entry->Type == 0 || IopMbrTypeIsExtended(entry->Type))
continue;
BOOT_VOLUME_SELECTOR identity = {0};
identity.Kind = BootVolumeSelectorMbr;
identity.Identity.Mbr.DiskSignature = diskSignature;
identity.Identity.Mbr.PartitionNumber = i + 1u;
IOP_PARTITION_RECORD* record = IopAllocatePartition(diskIndex,
entry->StartingLba,
entry->SectorCount,
&identity);
if (!record) {
IopFreePartitionList(partitions);
return NTLSTATUS_NO_MEMORY;
}
status = IopInsertDiscoveredPartition(&partitions, record);
if (!KSUCCESS(status)) {
MmFreePool(record);
IopFreePartitionList(partitions);
return status;
}
}
*partitionsOut = partitions;
return NTLSTATUS_SUCCESS;
}
// Returns whether the registry already contains at least one partition for this disk index.
static bool IopDiskAlreadyDiscovered(uint8_t diskIndex) {
bool found = false;
if (!IopAcquirePartitionMutex())
return false;
for (IOP_PARTITION_RECORD* record = gPartitions; record; record = record->Next) {
if (record->Info.DiskIndex == diskIndex) {
found = true;
break;
}
}
KeReleaseMutex(gPartitionMutex);
return found;
}
// Appends one discovered list to the live registry and synchronously notifies every subscriber.
static KSTATUS IopPublishPartitions(uint8_t diskIndex, IOP_PARTITION_RECORD* discovered) {
if (!IopAcquirePartitionMutex()) {
IopFreePartitionList(discovered);
return NTLSTATUS_FAILURE;
}
for (IOP_PARTITION_RECORD* record = gPartitions; record; record = record->Next) {
if (record->Info.DiskIndex == diskIndex) {
KeReleaseMutex(gPartitionMutex);
IopFreePartitionList(discovered);
return NTLSTATUS_SUCCESS;
}
}
IOP_PARTITION_RECORD** tail = &gPartitions;
while (*tail)
tail = &(*tail)->Next;
*tail = discovered;
for (IOP_PARTITION_RECORD* record = discovered; record; record = record->Next) {
if (record->Identity.Kind == BootVolumeSelectorMbr) {
DbgPrint("partition: disk=%u mbr/%08X/%u start=%llu sectors=%llu\n",
record->Info.DiskIndex,
record->Identity.Identity.Mbr.DiskSignature,
record->Identity.Identity.Mbr.PartitionNumber,
(unsigned long long)record->Info.StartLba,
(unsigned long long)record->Info.SectorCount);
} else {
const BOOT_VOLUME_GUID* guid = &record->Identity.Identity.Gpt.PartitionGuid;
DbgPrint("partition: disk=%u gpt/%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X start=%llu sectors=%llu\n",
record->Info.DiskIndex,
guid->Data1, guid->Data2, guid->Data3,
guid->Data4[0], guid->Data4[1], guid->Data4[2], guid->Data4[3],
guid->Data4[4], guid->Data4[5], guid->Data4[6], guid->Data4[7],
(unsigned long long)record->Info.StartLba,
(unsigned long long)record->Info.SectorCount);
}
if (IopIdentityEquals(&record->Identity, &gSystemVolumeSelector)) {
DbgPrint("partition: configured system_volume matches disk=%u start=%llu sectors=%llu\n",
record->Info.DiskIndex,
(unsigned long long)record->Info.StartLba,
(unsigned long long)record->Info.SectorCount);
}
for (IOP_PARTITION_CALLBACK_ENTRY* entry = gPartitionCallbacks; entry; entry = entry->Next)
entry->Callback(&record->Info, IoPartitionArrived, entry->Context);
}
KeReleaseMutex(gPartitionMutex);
return NTLSTATUS_SUCCESS;
}
// Discovers and publishes all partitions for one newly registered disk.
static void IopDiskArrived(uint8_t diskIndex, DEVICE_OBJECT* device) {
if (!device || IopDiskAlreadyDiscovered(diskIndex))
return;
IOP_PARTITION_RECORD* discovered = NULL;
KSTATUS status = IopDiscoverDiskPartitions(diskIndex, device, &discovered);
if (!KSUCCESS(status)) {
if (status != NTLSTATUS_NOT_FOUND) {
DbgPrint("partition: disk %u discovery failed status=0x%x\n", diskIndex, status);
} else {
DbgPrint("partition: disk %u has no partition table\n", diskIndex);
}
return;
}
if (!discovered) {
DbgPrint("partition: disk %u contains no supported partitions\n", diskIndex);
return;
}
status = IopPublishPartitions(diskIndex, discovered);
if (!KSUCCESS(status))
DbgPrint("partition: disk %u publish failed status=0x%x\n", diskIndex, status);
}
// Removes all partitions belonging to one disk and notifies consumers before freeing their snapshots.
static void IopDiskRemoved(uint8_t diskIndex, IO_PARTITION_CHANGE change) {
if (!IopAcquirePartitionMutex())
return;
IOP_PARTITION_RECORD** link = &gPartitions;
while (*link) {
IOP_PARTITION_RECORD* record = *link;
if (record->Info.DiskIndex != diskIndex) {
link = &record->Next;
continue;
}
*link = record->Next;
for (IOP_PARTITION_CALLBACK_ENTRY* entry = gPartitionCallbacks; entry; entry = entry->Next)
entry->Callback(&record->Info, change, entry->Context);
MmFreePool(record);
}
KeReleaseMutex(gPartitionMutex);
}
// Receives the underlying disk namespace and translates it into partition changes.
static void IopDiskChange(uint8_t diskIndex,
DEVICE_OBJECT* device,
IO_DEVICE_CHANGE change,
void* context) {
(void)context;
switch (change) {
case IoDeviceArrived:
IopDiskArrived(diskIndex, device);
break;
case IoDeviceSurpriseRemoved:
case IoDeviceRemoved:
IopDiskRemoved(diskIndex,
change == IoDeviceSurpriseRemoved
? IoPartitionSurpriseRemoved
: IoPartitionRemoved);
break;
case IoDeviceQueryRemove:
default:
break;
}
}
// Replays one disk that predates partition-manager initialization.
static void IopReplayDisk(uint8_t diskIndex, DEVICE_OBJECT* device, void* context) {
(void)context;
IopDiskArrived(diskIndex, device);
}
// Initializes dynamic partition discovery and the mandatory structural system-volume selector.
KSTATUS IoInitializePartitionManager(const char* systemVolumeSelector) {
if (!systemVolumeSelector || !*systemVolumeSelector)
return NTLSTATUS_INVALID_PARAMETER;
if (gPartitionManagerInitialized)
return NTLSTATUS_ALREADY_EXISTS;
BOOT_VOLUME_SELECTOR selector = {0};
if (!BootVolumeSelectorParse(systemVolumeSelector, strlen(systemVolumeSelector), &selector))
return NTLSTATUS_INVALID_PARAMETER;
MUTEX* mutex = NULL;
KSTATUS status = KeMutexCreate(&mutex);
if (!KSUCCESS(status))
return status;
gPartitionMutex = mutex;
gSystemVolumeSelector = selector;
status = IoRegisterDiskChangeCallback(IopDiskChange, NULL);
if (!KSUCCESS(status)) {
gPartitionMutex = NULL;
memset(&gSystemVolumeSelector, 0, sizeof(gSystemVolumeSelector));
ObDereferenceObject(&mutex->Header);
return status;
}
// The disk-change subscription is permanent, so the manager becomes live before replaying existing disks.
gPartitionManagerInitialized = true;
status = IoEnumerateDiskDevices(IopReplayDisk, NULL);
if (!KSUCCESS(status))
return status;
DbgPrint("partition: dynamic discovery initialized for system_volume=%s\n", systemVolumeSelector);
return NTLSTATUS_SUCCESS;
}
// Registers one partition-change consumer and replays every currently discovered partition before returning.
__declspec(dllexport) KSTATUS IoRegisterPartitionChangeCallback(IO_PARTITION_CHANGE_CALLBACK callback,
void* context) {
if (!callback || !gPartitionManagerInitialized)
return NTLSTATUS_INVALID_PARAMETER;
IOP_PARTITION_CALLBACK_ENTRY* entry = MmAllocatePool(sizeof(*entry));
if (!entry)
return NTLSTATUS_NO_MEMORY;
entry->Next = NULL;
entry->Callback = callback;
entry->Context = context;
if (!IopAcquirePartitionMutex()) {
MmFreePool(entry);
return NTLSTATUS_FAILURE;
}
IOP_PARTITION_CALLBACK_ENTRY** tail = &gPartitionCallbacks;
while (*tail)
tail = &(*tail)->Next;
*tail = entry;
for (IOP_PARTITION_RECORD* record = gPartitions; record; record = record->Next)
callback(&record->Info, IoPartitionArrived, context);
KeReleaseMutex(gPartitionMutex);
return NTLSTATUS_SUCCESS;
}
// Returns the uniquely selected system partition, or NOT_FOUND/COLLISION while discovery is incomplete or ambiguous.
KSTATUS IoQuerySystemPartition(IO_PARTITION_INFO* partitionOut) {
if (!partitionOut || !gPartitionManagerInitialized)
return NTLSTATUS_INVALID_PARAMETER;
if (!IopAcquirePartitionMutex())
return NTLSTATUS_FAILURE;
const IOP_PARTITION_RECORD* match = NULL;
uint32_t matchCount = 0;
for (IOP_PARTITION_RECORD* record = gPartitions; record; record = record->Next) {
if (!IopIdentityEquals(&record->Identity, &gSystemVolumeSelector))
continue;
match = record;
matchCount++;
}
KSTATUS status;
if (matchCount == 0) {
status = NTLSTATUS_NOT_FOUND;
} else if (matchCount != 1) {
status = NTLSTATUS_OBJECT_NAME_COLLISION;
} else {
*partitionOut = match->Info;
status = NTLSTATUS_SUCCESS;
}
KeReleaseMutex(gPartitionMutex);
return status;
}
code/kernel/io_volume.c
+634
View File
@@ -0,0 +1,634 @@
// Mounted-volume core. This file owns volume identity, provider lifetime, and dead-volume dispatch semantics.
#include <string.h>
#include <kernel/io_volume.h>
#include <kernel/ke_id.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
#include <kernel/mm_file_cache.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_event.h>
#include <kernel/ke_sched.h>
static KE_ID_ALLOCATOR gVolumeIds = KE_ID_ALLOCATOR_INIT(1);
static SPINLOCK gVolumeRegistryLock = SPINLOCK_INIT;
static LIST_ENTRY gVolumeRegistry;
static uint8_t gVolumeRegistryInitialized;
static SPINLOCK gVolumeStatsLock = SPINLOCK_INIT;
static IO_VOLUME_STATS gVolumeStats;
static IO_VOLUME_MANAGER_STATS gVolumeManagerStats;
static EVENT* gVolumeChangeEvent;
static SPINLOCK gVolumeChangeLock = SPINLOCK_INIT;
static uint64_t gVolumeGeneration;
static IO_VOLUME_MANAGER gVolumeManagerObject;
static uint8_t gVolumeManagerObjectInitialized;
static void KiIoVolumeAddStat(uint64_t* field, int64_t delta);
// Initializes the dynamic mounted-volume registry on first use.
static void KiIoEnsureVolumeRegistry(void) {
uintptr_t flags = KeAcquireSpinlock(&gVolumeRegistryLock);
if (!gVolumeRegistryInitialized) {
LIST_INIT(&gVolumeRegistry);
gVolumeRegistryInitialized = 1;
}
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
}
// Publishes one mounted-volume registry change to discovery waiters.
static void KiIoNotifyVolumeChange(void) {
uintptr_t flags = KeAcquireSpinlock(&gVolumeChangeLock);
gVolumeGeneration++;
KeReleaseSpinlock(&gVolumeChangeLock, flags);
if (gVolumeChangeEvent)
KeSignalEvent(gVolumeChangeEvent);
}
// Initializes the waitable mounted-volume registry notification before providers start.
KSTATUS IoInitializeVolumeManager(void) {
KiIoEnsureVolumeRegistry();
if (!gVolumeManagerObjectInitialized) {
ObInitializeObjectHeader(&gVolumeManagerObject.Header, ObjectTypeVolumeManager);
gVolumeManagerObjectInitialized = 1;
}
if (gVolumeChangeEvent)
return NTLSTATUS_SUCCESS;
return KeEventCreate(EventAutoReset, 0, &gVolumeChangeEvent);
}
// References the persistent manager object used to delegate mounted-volume control-plane authority.
KSTATUS IoReferenceVolumeManager(IO_VOLUME_MANAGER** managerOut) {
if (!managerOut)
return NTLSTATUS_INVALID_PARAMETER;
*managerOut = (IO_VOLUME_MANAGER*)0;
if (!gVolumeManagerObjectInitialized || !ObTryReferenceObject(&gVolumeManagerObject.Header))
return NTLSTATUS_OBJECT_DEAD;
*managerOut = &gVolumeManagerObject;
return NTLSTATUS_SUCCESS;
}
// Captures one stable value snapshot. A generation change during capture causes a retry instead of returning stale metadata as current.
KSTATUS IoSnapshotVolumes(IO_VOLUME_SNAPSHOT_ENTRY** entriesOut, uint32_t* countOut, uint64_t* generationOut) {
if (!entriesOut || !countOut || !generationOut)
return NTLSTATUS_INVALID_PARAMETER;
*entriesOut = (IO_VOLUME_SNAPSHOT_ENTRY*)0;
*countOut = 0;
*generationOut = 0;
KiIoEnsureVolumeRegistry();
for (;;) {
uint64_t generationBefore = IoGetVolumeGeneration();
uintptr_t flags = KeAcquireSpinlock(&gVolumeRegistryLock);
size_t count = 0;
LIST_ENTRY* head = &gVolumeRegistry;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd)
count++;
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
if (count > UINT32_MAX)
return NTLSTATUS_LIMIT_REACHED;
IO_VOLUME_SNAPSHOT_ENTRY* entries = (IO_VOLUME_SNAPSHOT_ENTRY*)0;
if (count) {
entries = (IO_VOLUME_SNAPSHOT_ENTRY*)MmAllocatePool(count * sizeof(*entries));
if (!entries)
return NTLSTATUS_NO_MEMORY;
memset(entries, 0, count * sizeof(*entries));
}
size_t used = 0;
bool overflowed = false;
flags = KeAcquireSpinlock(&gVolumeRegistryLock);
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
if (used >= count) {
overflowed = true;
break;
}
IO_VOLUME* volume = CONTAINING_RECORD(node, IO_VOLUME, RegistryLink);
entries[used].InstanceId = volume->InstanceId;
used++;
}
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
uint64_t generationAfter = IoGetVolumeGeneration();
if (!overflowed && generationBefore == generationAfter) {
*entriesOut = entries;
*countOut = (uint32_t)used;
*generationOut = generationAfter;
KiIoVolumeAddStat(&gVolumeManagerStats.Snapshots, 1);
return NTLSTATUS_SUCCESS;
}
if (entries)
MmFreePool(entries);
}
}
// References one currently mounted volume by its runtime instance identity.
KSTATUS IoReferenceVolumeByInstanceId(uint32_t instanceId, IO_VOLUME** volumeOut) {
if (!instanceId || !volumeOut)
return NTLSTATUS_INVALID_PARAMETER;
*volumeOut = (IO_VOLUME*)0;
KiIoEnsureVolumeRegistry();
uintptr_t flags = KeAcquireSpinlock(&gVolumeRegistryLock);
LIST_ENTRY* head = &gVolumeRegistry;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
IO_VOLUME* volume = CONTAINING_RECORD(node, IO_VOLUME, RegistryLink);
if (volume->InstanceId != instanceId)
continue;
if (!ObTryReferenceObject(&volume->Header)) {
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
return NTLSTATUS_OBJECT_DEAD;
}
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
if (!IoVolumeIsMounted(volume)) {
ObDereferenceObject(&volume->Header);
return NTLSTATUS_OBJECT_DEAD;
}
*volumeOut = volume;
KiIoVolumeAddStat(&gVolumeManagerStats.VolumeOpens, 1);
return NTLSTATUS_SUCCESS;
}
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
return NTLSTATUS_NOT_FOUND;
}
// Returns the current dynamic mounted-volume registry generation.
uint64_t IoGetVolumeGeneration(void) {
uintptr_t flags = KeAcquireSpinlock(&gVolumeChangeLock);
uint64_t generation = gVolumeGeneration;
KeReleaseSpinlock(&gVolumeChangeLock, flags);
return generation;
}
// Waits until the mounted-volume registry changes or the timeout expires.
KSTATUS IoWaitForVolumeChange(uint64_t observedGeneration, uint64_t timeoutMilliseconds) {
KiIoVolumeAddStat(&gVolumeManagerStats.Waits, 1);
if (IoGetVolumeGeneration() != observedGeneration) {
KiIoVolumeAddStat(&gVolumeManagerStats.Wakeups, 1);
return NTLSTATUS_SUCCESS;
}
if (!gVolumeChangeEvent)
return NTLSTATUS_OBJECT_DEAD;
EVENT* events[1] = { gVolumeChangeEvent };
uint32_t selected = 0;
KSTATUS status = KeWaitForMultipleEvents(events, 1, 0, timeoutMilliseconds, &selected);
if (KSUCCESS(status))
KiIoVolumeAddStat(&gVolumeManagerStats.Wakeups, 1);
return status;
}
// Adds one value to a mounted-volume statistic.
static void KiIoVolumeAddStat(uint64_t* field, int64_t delta) {
uintptr_t flags = KeAcquireSpinlock(&gVolumeStatsLock);
if (delta >= 0)
*field += (uint64_t)delta;
else {
uint64_t amount = (uint64_t)(-delta);
*field = *field > amount ? *field - amount : 0;
}
KeReleaseSpinlock(&gVolumeStatsLock, flags);
}
void IoQueryVolumeStats(IO_VOLUME_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gVolumeStatsLock);
*out = gVolumeStats;
KeReleaseSpinlock(&gVolumeStatsLock, flags);
}
void IoQueryVolumeManagerStats(IO_VOLUME_MANAGER_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gVolumeStatsLock);
*out = gVolumeManagerStats;
KeReleaseSpinlock(&gVolumeStatsLock, flags);
}
// References the unique mounted volume backed by one exact disk extent start.
KSTATUS IoReferenceVolumeByBacking(uint8_t diskIndex, uint64_t startLba, IO_VOLUME** volumeOut) {
if (!volumeOut)
return NTLSTATUS_INVALID_PARAMETER;
*volumeOut = (IO_VOLUME*)0;
KiIoEnsureVolumeRegistry();
IO_VOLUME* match = (IO_VOLUME*)0;
uint32_t matches = 0;
uintptr_t flags = KeAcquireSpinlock(&gVolumeRegistryLock);
LIST_ENTRY* head = &gVolumeRegistry;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
IO_VOLUME* volume = CONTAINING_RECORD(node, IO_VOLUME, RegistryLink);
if (volume->Source.DiskIndex != diskIndex || volume->Source.StartLba != startLba)
continue;
matches++;
if (!match && ObTryReferenceObject(&volume->Header))
match = volume;
}
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
if (matches == 0)
return NTLSTATUS_NOT_FOUND;
if (matches != 1) {
if (match)
ObDereferenceObject(&match->Header);
return NTLSTATUS_OBJECT_NAME_COLLISION;
}
if (!match)
return NTLSTATUS_OBJECT_DEAD;
if (!IoVolumeIsMounted(match)) {
ObDereferenceObject(&match->Header);
return NTLSTATUS_OBJECT_DEAD;
}
*volumeOut = match;
return NTLSTATUS_SUCCESS;
}
// Returns whether the volume still accepts provider operations.
bool IoVolumeIsMounted(const IO_VOLUME* volume) {
if (!volume)
return false;
IO_VOLUME* mutableVolume = (IO_VOLUME*)volume;
uintptr_t flags = KeAcquireSpinlock(&mutableVolume->Lock);
bool mounted = mutableVolume->State == IoVolumeMounted;
KeReleaseSpinlock(&mutableVolume->Lock, flags);
return mounted;
}
// Acquires one object reference only while the mounted-volume state remains live.
bool IoVolumeTryReferenceMounted(IO_VOLUME* volume) {
if (!volume || !ObTryReferenceObject(&volume->Header))
return false;
if (IoVolumeIsMounted(volume))
return true;
ObDereferenceObject(&volume->Header);
return false;
}
// Destroys provider state only after namespace, file, directory, and transient references drain.
static void KiIoVolumeDestroy(OBJECT* object) {
IO_VOLUME* volume = CONTAINING_RECORD(object, IO_VOLUME, Header);
if (volume->Provider.Destroy)
volume->Provider.Destroy(&volume->Provider);
if (volume->BackingDevice) {
IoDereferenceDeviceObject(volume->BackingDevice);
volume->BackingDevice = (DEVICE_OBJECT*)0;
}
if (volume->InstanceId)
KiIoVolumeAddStat(&gVolumeStats.ProviderDestroys, 1);
}
// Creates a dynamic mounted volume and inserts it into the mounted-volume registry.
__declspec(dllexport) KSTATUS IoCreateVolume(const IO_VOLUME_PROVIDER* provider,
const IO_VOLUME_SOURCE* source,
IO_VOLUME** volumeOut) {
if (!provider || !source || !volumeOut || !provider->Open || !provider->Read ||
!provider->GetSize || !provider->Close || source->SectorCount == 0 ||
(source->Type != IoVolumeSourcePartition && source->Type != IoVolumeSourceBlockDevice)) {
return NTLSTATUS_INVALID_PARAMETER;
}
if (source->Type == IoVolumeSourceBlockDevice && source->StartLba != 0)
return NTLSTATUS_INVALID_PARAMETER;
*volumeOut = (IO_VOLUME*)0;
DEVICE_OBJECT* backingDevice = IoReferenceDiskDevice(source->DiskIndex);
if (!backingDevice)
return NTLSTATUS_NOT_FOUND;
uint64_t diskSectors = backingDevice->u.Disk.SectorCount;
if (source->StartLba >= diskSectors || source->SectorCount > diskSectors - source->StartLba ||
(source->Type == IoVolumeSourceBlockDevice && source->SectorCount != diskSectors)) {
IoDereferenceDeviceObject(backingDevice);
return NTLSTATUS_INVALID_PARAMETER;
}
IO_VOLUME* volume = (IO_VOLUME*)ObAllocateObject(ObjectTypeVolume, sizeof(IO_VOLUME));
if (!volume) {
IoDereferenceDeviceObject(backingDevice);
return NTLSTATUS_NO_MEMORY;
}
uint32_t instanceId = 0;
KSTATUS status = KeAllocateId32(&gVolumeIds, &instanceId);
if (!KSUCCESS(status)) {
IoDereferenceDeviceObject(backingDevice);
ObDereferenceObject(&volume->Header);
return status;
}
volume->Header.Destroy = KiIoVolumeDestroy;
volume->Lock = (SPINLOCK)SPINLOCK_INIT;
volume->InstanceId = instanceId;
volume->State = IoVolumeMounted;
volume->RemovalReason = 0;
volume->Source = *source;
volume->BackingDevice = backingDevice;
volume->Provider = *provider;
LIST_INIT(&volume->RegistryLink);
KiIoEnsureVolumeRegistry();
uintptr_t flags = KeAcquireSpinlock(&gVolumeRegistryLock);
ListInsertTail(&gVolumeRegistry, &volume->RegistryLink);
volume->RegistryLinked = 1;
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
// The registry owns the allocation reference; the creator receives an independent reference.
ObReferenceObject(&volume->Header);
*volumeOut = volume;
KiIoVolumeAddStat(&gVolumeStats.CreatedVolumes, 1);
KiIoVolumeAddStat(&gVolumeStats.MountedVolumes, 1);
KiIoNotifyVolumeChange();
DbgPrint("io: volume %u mounted disk=%u start=%llu sectors=%llu\n",
instanceId,
source->DiskIndex,
(unsigned long long)source->StartLba,
(unsigned long long)source->SectorCount);
return NTLSTATUS_SUCCESS;
}
// Marks a mounted volume terminally dead while retaining provider state until references drain.
__declspec(dllexport) KSTATUS IoRemoveVolume(IO_VOLUME* volume, IO_VOLUME_REMOVAL_REASON reason) {
if (!volume || reason < IoVolumeUnmounted || reason > IoVolumeProviderDead)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&volume->Lock);
if (volume->State != IoVolumeMounted) {
KeReleaseSpinlock(&volume->Lock, flags);
return NTLSTATUS_OBJECT_DEAD;
}
volume->State = IoVolumeDead;
volume->RemovalReason = (uint32_t)reason;
KeReleaseSpinlock(&volume->Lock, flags);
// No future provider operation can start after this point. The callback may cancel
// provider-specific work, but Context remains valid until KiIoVolumeDestroy.
if (volume->Provider.VolumeDead)
volume->Provider.VolumeDead(&volume->Provider, reason);
MmFileCacheDiscardVolume(volume);
bool dropRegistryReference = false;
flags = KeAcquireSpinlock(&gVolumeRegistryLock);
if (volume->RegistryLinked) {
ListRemove(&volume->RegistryLink);
volume->RegistryLinked = 0;
dropRegistryReference = true;
}
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
KiIoVolumeAddStat(&gVolumeStats.MountedVolumes, -1);
KiIoVolumeAddStat(&gVolumeStats.RemovedVolumes, 1);
KiIoNotifyVolumeChange();
DbgPrint("io: volume %u removed reason=%u\n", volume->InstanceId, (unsigned)reason);
if (dropRegistryReference)
ObDereferenceObject(&volume->Header);
return NTLSTATUS_SUCCESS;
}
// Releases a creator-owned mounted-volume reference.
__declspec(dllexport) void IoReleaseVolume(IO_VOLUME* volume) {
if (volume)
ObDereferenceObject(&volume->Header);
}
// Returns the monotonic identity of a mounted-volume instance.
__declspec(dllexport) uint32_t IoGetVolumeInstanceId(const IO_VOLUME* volume) {
return volume ? volume->InstanceId : 0;
}
// Returns the immutable block extent backing this mounted-volume instance.
__declspec(dllexport) KSTATUS IoGetVolumeSource(const IO_VOLUME* volume, IO_VOLUME_SOURCE* sourceOut) {
if (!volume || !sourceOut)
return NTLSTATUS_INVALID_PARAMETER;
*sourceOut = volume->Source;
return NTLSTATUS_SUCCESS;
}
// Returns whether native provider async read is available while the volume remains mounted.
bool IoVolumeSupportsReadAsync(const IO_VOLUME* volume) {
return volume && IoVolumeIsMounted(volume) && volume->Provider.ReadAsync != (void*)0;
}
// Returns whether native provider async write is available while the volume remains mounted.
bool IoVolumeSupportsWriteAsync(const IO_VOLUME* volume) {
return volume && IoVolumeIsMounted(volume) && volume->Provider.WriteAsync != (void*)0;
}
// Validates mounted state before entering provider code.
static KSTATUS KiIoVolumeCheckMounted(IO_VOLUME* volume) {
if (IoVolumeIsMounted(volume))
return NTLSTATUS_SUCCESS;
KiIoVolumeAddStat(&gVolumeStats.DeadIoRejects, 1);
return NTLSTATUS_OBJECT_DEAD;
}
KSTATUS IoVolumeOpen(IO_VOLUME* volume, SYS_STRING_VIEW path, void** fileCtx) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
return volume->Provider.Open(&volume->Provider, path, fileCtx);
}
KSTATUS IoVolumeRead(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
void* buffer, size_t size, size_t* bytesRead) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Read) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.Read(&volume->Provider, fileCtx, offset, buffer, size, bytesRead);
}
KSTATUS IoVolumeWrite(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
const void* buffer, size_t size, size_t* bytesWritten) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Write) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.Write(&volume->Provider, fileCtx, offset, buffer, size, bytesWritten);
}
KSTATUS IoVolumeReadAsync(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
void* buffer, size_t size, IO_FS_ASYNC_COMPLETE complete,
void* completeContext) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.ReadAsync) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.ReadAsync(&volume->Provider, fileCtx, offset, buffer, size, complete, completeContext);
}
KSTATUS IoVolumeWriteAsync(IO_VOLUME* volume, void* fileCtx, uint64_t offset,
const void* buffer, size_t size, IO_FS_ASYNC_COMPLETE complete,
void* completeContext) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.WriteAsync) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.WriteAsync(&volume->Provider, fileCtx, offset, buffer, size, complete, completeContext);
}
KSTATUS IoVolumeCreate(IO_VOLUME* volume, SYS_STRING_VIEW path, void** fileCtx) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Create) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.Create(&volume->Provider, path, fileCtx);
}
KSTATUS IoVolumeDelete(IO_VOLUME* volume, SYS_STRING_VIEW path) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Delete) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.Delete(&volume->Provider, path);
}
KSTATUS IoVolumeRename(IO_VOLUME* volume, SYS_STRING_VIEW oldPath, SYS_STRING_VIEW newPath) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Rename) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.Rename(&volume->Provider, oldPath, newPath);
}
KSTATUS IoVolumeCreateDir(IO_VOLUME* volume, SYS_STRING_VIEW path) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.CreateDir) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.CreateDir(&volume->Provider, path);
}
KSTATUS IoVolumeRemoveDir(IO_VOLUME* volume, SYS_STRING_VIEW path) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.RemoveDir) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.RemoveDir(&volume->Provider, path);
}
KSTATUS IoVolumeTruncate(IO_VOLUME* volume, void* fileCtx, uint64_t newSize, uint32_t fileFlags) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Truncate) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.Truncate(&volume->Provider, fileCtx, newSize, fileFlags);
}
KSTATUS IoVolumeGetSize(IO_VOLUME* volume, void* fileCtx, uint64_t* sizeOut) {
if (!sizeOut) return NTLSTATUS_INVALID_PARAMETER;
*sizeOut = 0;
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.GetSize) return NTLSTATUS_NOT_SUPPORTED;
*sizeOut = volume->Provider.GetSize(&volume->Provider, fileCtx);
return NTLSTATUS_SUCCESS;
}
KSTATUS IoVolumeQueryInfo(IO_VOLUME* volume, void* fileCtx, SYS_FILE_STAT* out) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.QueryInfo) return NTLSTATUS_NOT_SUPPORTED;
s = volume->Provider.QueryInfo(&volume->Provider, fileCtx, out);
if (KSUCCESS(s) && out)
out->VolumeId = volume->InstanceId;
return s;
}
KSTATUS IoVolumeStat(IO_VOLUME* volume, SYS_STRING_VIEW path, SYS_FILE_STAT* out) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Stat) return NTLSTATUS_NOT_SUPPORTED;
s = volume->Provider.Stat(&volume->Provider, path, out);
if (KSUCCESS(s) && out)
out->VolumeId = volume->InstanceId;
return s;
}
KSTATUS IoVolumeQueryIdentity(IO_VOLUME* volume, void* fileCtx, IO_FILE_IDENTITY* out) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!out || !volume->Provider.QueryIdentity) return NTLSTATUS_NOT_SUPPORTED;
s = volume->Provider.QueryIdentity(&volume->Provider, fileCtx, out);
if (KSUCCESS(s))
out->VolumeId = volume->InstanceId;
return s;
}
KSTATUS IoVolumeFlush(IO_VOLUME* volume, void* fileCtx) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.Flush) return NTLSTATUS_SUCCESS;
return volume->Provider.Flush(&volume->Provider, fileCtx);
}
void IoVolumeClose(IO_VOLUME* volume, void* fileCtx) {
if (volume && volume->Provider.Close && fileCtx)
volume->Provider.Close(&volume->Provider, fileCtx);
}
void IoVolumeQueryStats(IO_VOLUME* volume, IO_FILESYSTEM_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
if (!volume || !IoVolumeIsMounted(volume) || !volume->Provider.QueryStats)
return;
volume->Provider.QueryStats(&volume->Provider, out);
}
// Aggregates filesystem traversal statistics across every dynamic mounted volume.
__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
KiIoEnsureVolumeRegistry();
uintptr_t flags = KeAcquireSpinlock(&gVolumeRegistryLock);
size_t count = 0;
LIST_ENTRY* head = &gVolumeRegistry;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd)
count++;
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
if (!count) return;
IO_VOLUME** snapshot = (IO_VOLUME**)MmAllocatePool(count * sizeof(IO_VOLUME*));
if (!snapshot) return;
memset(snapshot, 0, count * sizeof(IO_VOLUME*));
size_t used = 0;
flags = KeAcquireSpinlock(&gVolumeRegistryLock);
for (LIST_ENTRY* node = head->Fwd; node != head && used < count; node = node->Fwd) {
IO_VOLUME* volume = CONTAINING_RECORD(node, IO_VOLUME, RegistryLink);
if (ObTryReferenceObject(&volume->Header))
snapshot[used++] = volume;
}
KeReleaseSpinlock(&gVolumeRegistryLock, flags);
for (size_t i = 0; i < used; ++i) {
IO_FILESYSTEM_STATS one;
IoVolumeQueryStats(snapshot[i], &one);
ObDereferenceObject(&snapshot[i]->Header);
out->PathLookups += one.PathLookups;
out->PathCacheHits += one.PathCacheHits;
out->PathCacheMisses += one.PathCacheMisses;
out->PathCacheStores += one.PathCacheStores;
out->PathCacheInvalidations += one.PathCacheInvalidations;
if (one.PathCacheGeneration > out->PathCacheGeneration)
out->PathCacheGeneration = one.PathCacheGeneration;
out->DirectorySearches += one.DirectorySearches;
out->DirectoryBlocksRead += one.DirectoryBlocksRead;
out->DirectoryEntriesScanned += one.DirectoryEntriesScanned;
out->DataReadRuns += one.DataReadRuns;
out->DataReadSectors += one.DataReadSectors;
out->DataReadRunMerges += one.DataReadRunMerges;
if (one.MaxDataReadSpanSectors > out->MaxDataReadSpanSectors)
out->MaxDataReadSpanSectors = one.MaxDataReadSpanSectors;
}
MmFreePool(snapshot);
}
KSTATUS IoVolumeOpenDir(IO_VOLUME* volume, SYS_STRING_VIEW path, void** dirCtx) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.OpenDir) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.OpenDir(&volume->Provider, path, dirCtx);
}
KSTATUS IoVolumeReadDir(IO_VOLUME* volume, void* dirCtx, KDIR_ENTRY* out) {
KSTATUS s = KiIoVolumeCheckMounted(volume);
if (!KSUCCESS(s)) return s;
if (!volume->Provider.ReadDir) return NTLSTATUS_NOT_SUPPORTED;
return volume->Provider.ReadDir(&volume->Provider, dirCtx, out);
}
void IoVolumeCloseDir(IO_VOLUME* volume, void* dirCtx) {
if (volume && volume->Provider.CloseDir && dirCtx)
volume->Provider.CloseDir(&volume->Provider, dirCtx);
}
code/kernel/ke_audio.c
+23 -391
View File
@@ -8,6 +8,7 @@
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddk/audio.h>
#include <kernel/ke_audio.h>
#include <kernel/ke_id.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_event.h>
#include <kernel/ke_list.h>
@@ -22,7 +23,6 @@
#define KE_AUDIO_ENDPOINT_TAG MM_POOL_TAG('a', 'e', 'p', 't')
#define KE_AUDIO_RING_TAG MM_POOL_TAG('a', 'r', 'n', 'g')
#define KE_AUDIO_SELFTEST_TAG MM_POOL_TAG('a', 't', 's', 't')
#define KE_AUDIO_PERIOD_STATE_FREE 0u
#define KE_AUDIO_PERIOD_STATE_FILLING 1u
@@ -31,7 +31,6 @@
#define KE_AUDIO_PERIOD_STATE_REAPING 4u
#define KE_AUDIO_TICKS_PER_MILLISECOND 1000ull
#define KE_AUDIO_SELFTEST_WAIT_MILLISECONDS 5000ull
typedef struct KE_AUDIO_PERIOD_CONTRIBUTION {
KE_AUDIO_STREAM* Stream;
@@ -103,7 +102,7 @@ struct IO_AUDIO_ENDPOINT {
EVENT* MixerExitEvent;
THREAD* MixerThread;
SPINLOCK NotifyLock;
uint64_t EndpointId;
uint32_t EndpointId;
IO_AUDIO_ENDPOINT_KIND Kind;
KE_AUDIO_ENDPOINT_STATE State;
uint32_t Capabilities;
@@ -174,8 +173,8 @@ typedef struct KE_AUDIO_LIFECYCLE_SNAPSHOT {
static SPINLOCK gAudioRegistryLock = SPINLOCK_INIT;
static LIST_ENTRY gAudioEndpointList;
static uint32_t gAudioEndpointCount;
static uint64_t gAudioNextEndpointId = 1;
static uint64_t gAudioDefaultEndpointId;
static KE_ID_ALLOCATOR gAudioEndpointIds = KE_ID_ALLOCATOR_INIT(1u);
static uint32_t gAudioDefaultEndpointId;
static uint8_t gAudioInitialized;
static KE_AUDIO_CORE_STATS gAudioCoreStats;
@@ -433,7 +432,7 @@ static uint32_t KiAudioEndpointRank(IO_AUDIO_ENDPOINT_KIND kind) {
// Recomputes the default endpoint while the registry lock is held.
static void KiAudioSelectDefaultEndpointLocked(void) {
uint64_t selectedId = 0;
uint32_t selectedId = 0;
uint32_t selectedRank = UINT32_MAX;
for (LIST_ENTRY* link = gAudioEndpointList.Fwd;
link != &gAudioEndpointList;
@@ -452,7 +451,7 @@ static void KiAudioSelectDefaultEndpointLocked(void) {
}
// References one ready endpoint by stable identifier.
static IO_AUDIO_ENDPOINT* KiAudioReferenceEndpointById(uint64_t endpointId) {
static IO_AUDIO_ENDPOINT* KiAudioReferenceEndpointById(uint32_t endpointId) {
IO_AUDIO_ENDPOINT* result = (IO_AUDIO_ENDPOINT*)0;
uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock);
for (LIST_ENTRY* link = gAudioEndpointList.Fwd;
@@ -1421,7 +1420,18 @@ static KSTATUS KiAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* reg
ObDereferenceObject(&endpoint->Header);
return NTLSTATUS_NO_MEMORY;
}
endpoint->EndpointId = gAudioNextEndpointId++;
KSTATUS idStatus = KeAllocateId32(&gAudioEndpointIds, &endpoint->EndpointId);
if (!KSUCCESS(idStatus)) {
KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping);
endpoint->FailureStatus = idStatus;
KeReleaseSpinlock(&gAudioRegistryLock, flags);
KeSignalEvent(endpoint->WorkEvent);
(void)KeWaitForEvent(endpoint->MixerExitEvent, 0);
ObDereferenceObject(&endpoint->MixerThread->Header);
endpoint->MixerThread = (THREAD*)0;
ObDereferenceObject(&endpoint->Header);
return idStatus;
}
KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointReady);
endpoint->Registered = 1;
ObReferenceObject(&endpoint->Header);
@@ -1590,9 +1600,9 @@ const NTL_DDK_AUDIO* KeAudioGetDdk(void) {
}
// Returns the current default endpoint identifier.
uint64_t KeAudioGetDefaultEndpointId(void) {
uint32_t KeAudioGetDefaultEndpointId(void) {
uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock);
uint64_t endpointId = gAudioDefaultEndpointId;
uint32_t endpointId = gAudioDefaultEndpointId;
KeReleaseSpinlock(&gAudioRegistryLock, flags);
return endpointId;
}
@@ -1606,7 +1616,7 @@ KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints,
IO_AUDIO_ENDPOINT* references[KE_AUDIO_MAX_ENDPOINTS];
uint32_t count = 0;
uint64_t defaultId = 0;
uint32_t defaultId = 0;
uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock);
defaultId = gAudioDefaultEndpointId;
for (LIST_ENTRY* link = gAudioEndpointList.Fwd;
@@ -1668,7 +1678,7 @@ KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints,
}
// Creates one logical render stream with a fixed-capacity source ring.
KSTATUS KeAudioCreateStream(uint64_t endpointId,
KSTATUS KeAudioCreateStream(uint32_t endpointId,
const IO_AUDIO_FORMAT* format,
KE_AUDIO_STREAM** outStream) {
if (!outStream || !KiAudioValidateSourceFormat(format))
@@ -2010,7 +2020,7 @@ KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream,
}
// Updates endpoint master software gain and separate logical mute state.
KSTATUS KeAudioSetMasterVolume(uint64_t endpointId, float volume, bool muted) {
KSTATUS KeAudioSetMasterVolume(uint32_t endpointId, float volume, bool muted) {
if (!KiAudioFloatIsFinite(volume) || volume < 0.0f || volume > 1.0f)
return NTLSTATUS_INVALID_PARAMETER;
IO_AUDIO_ENDPOINT* endpoint = KiAudioReferenceEndpointById(endpointId);
@@ -2080,381 +2090,3 @@ KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOu
KiAudioReleaseMutex(stream->Lock);
return NTLSTATUS_SUCCESS;
}
// Copies one endpoint snapshot by stable identifier.
static KSTATUS KiAudioSelfTestQueryEndpoint(uint64_t endpointId,
KE_AUDIO_ENDPOINT_INFO* infoOut) {
KE_AUDIO_ENDPOINT_INFO infos[KE_AUDIO_MAX_ENDPOINTS];
uint32_t count = 0;
KSTATUS status = KeAudioQueryEndpoints(infos, KE_AUDIO_MAX_ENDPOINTS, &count);
if (KFAILED(status))
return status;
for (uint32_t index = 0; index < count; index++) {
if (infos[index].EndpointId == endpointId) {
*infoOut = infos[index];
return NTLSTATUS_SUCCESS;
}
}
return NTLSTATUS_NOT_FOUND;
}
// Generates one bounded run of a ramped 440 Hz stereo sine wave.
static void KiAudioSelfTestGenerateTone(int16_t* frames,
uint32_t frameCount,
uint64_t firstFrame,
double* sineInOut,
double* cosineInOut) {
const uint64_t totalFrames = 3ull * KE_AUDIO_OUTPUT_SAMPLE_RATE;
const uint64_t rampFrames = 384ull;
const double amplitude = 8231.0;
const double sineStep = 0.057564026959567284;
const double cosineStep = 0.9983418166140283;
double sine = *sineInOut;
double cosine = *cosineInOut;
for (uint32_t index = 0; index < frameCount; index++) {
uint64_t frame = firstFrame + index;
uint64_t framesFromEnd = totalFrames - 1ull - frame;
double envelope = 1.0;
if (frame < rampFrames)
envelope = (double)frame / (double)(rampFrames - 1ull);
if (framesFromEnd < rampFrames) {
double fadeOut = (double)framesFromEnd / (double)(rampFrames - 1ull);
if (fadeOut < envelope)
envelope = fadeOut;
}
double scaled = sine * amplitude * envelope;
int32_t sample = scaled >= 0.0 ? (int32_t)(scaled + 0.5)
: (int32_t)(scaled - 0.5);
frames[index * 2u] = (int16_t)sample;
frames[index * 2u + 1u] = (int16_t)sample;
double nextSine = sine * cosineStep + cosine * sineStep;
double nextCosine = cosine * cosineStep - sine * sineStep;
sine = nextSine;
cosine = nextCosine;
}
*sineInOut = sine;
*cosineInOut = cosine;
}
// Waits for one selftest stream condition with a finite boot-time deadline.
static KSTATUS KiAudioSelfTestWait(KE_AUDIO_STREAM* stream) {
EVENT* event = KeAudioStreamWaitEvent(stream);
if (!event)
return NTLSTATUS_OBJECT_DEAD;
uint32_t selected = 0;
return KeWaitForMultipleEvents(&event,
1,
0,
KE_AUDIO_SELFTEST_WAIT_MILLISECONDS,
&selected);
}
// Writes one generated run while using the stream event for bounded backpressure.
static KSTATUS KiAudioSelfTestWriteFrames(KE_AUDIO_STREAM* stream,
const int16_t* frames,
uint32_t frameCount,
uint64_t expectedDiscontinuity) {
uint32_t offset = 0;
while (offset < frameCount) {
uint32_t written = 0;
KSTATUS status = KeAudioWriteStream(stream,
frames + offset * 2u,
frameCount - offset,
&written);
if (KSUCCESS(status) && written) {
offset += written;
continue;
}
if (status != NTLSTATUS_WOULD_BLOCK)
return KFAILED(status) ? status : NTLSTATUS_FAILURE;
KE_AUDIO_STREAM_INFO info;
status = KeAudioQueryStream(stream, &info);
if (KFAILED(status))
return status;
if (info.State == KeAudioStreamFailed)
return KFAILED(info.FailureStatus) ? info.FailureStatus : NTLSTATUS_DEVICE_ERROR;
if (info.DiscontinuityGeneration != expectedDiscontinuity)
return NTLSTATUS_DEVICE_ERROR;
KE_AUDIO_STREAM_WAIT_CONDITION condition = {
.Mask = KE_AUDIO_WAIT_WRITABLE | KE_AUDIO_WAIT_FAILED | KE_AUDIO_WAIT_DISCONTINUITY,
.WritableFrames = 1,
.DiscontinuityGeneration = expectedDiscontinuity,
};
status = KeAudioArmStreamWait(stream, &condition);
if (KFAILED(status))
return status;
status = KiAudioSelfTestWait(stream);
if (KFAILED(status) || status == NTLSTATUS_TIMEOUT)
return status;
}
return NTLSTATUS_SUCCESS;
}
// Prints the common endpoint and stream counters for one selftest outcome.
static void KiAudioSelfTestPrintCounters(uint64_t endpointId,
const KE_AUDIO_ENDPOINT_INFO* endpointBefore,
const KE_AUDIO_ENDPOINT_INFO* endpointAfter,
const KE_AUDIO_STREAM_INFO* streamInfo,
uint64_t expectedDiscontinuity) {
uint64_t mixedPeriods = endpointAfter->PeriodsMixed - endpointBefore->PeriodsMixed;
uint64_t completedPeriods = endpointAfter->PeriodsCompleted - endpointBefore->PeriodsCompleted;
uint64_t interruptCount = endpointAfter->InterruptCount - endpointBefore->InterruptCount;
uint64_t endpointUnderruns = endpointAfter->Underruns - endpointBefore->Underruns;
uint64_t endpointDiscontinuities = endpointAfter->DiscontinuityGeneration -
endpointBefore->DiscontinuityGeneration;
uint64_t positionFrames = endpointAfter->HardwarePositionFrames -
endpointBefore->HardwarePositionFrames;
uint64_t streamDiscontinuities = streamInfo->DiscontinuityGeneration >= expectedDiscontinuity
? streamInfo->DiscontinuityGeneration - expectedDiscontinuity
: 0;
DbgPrint("AUDIO final selftest: endpoint=%llu submitted=%llu consumed=%llu presented=%llu\n",
(unsigned long long)endpointId,
(unsigned long long)streamInfo->FramesSubmitted,
(unsigned long long)streamInfo->FramesConsumed,
(unsigned long long)streamInfo->FramesPresented);
DbgPrint("AUDIO final selftest: periods mixed=%llu completed=%llu interrupts=%llu\n",
(unsigned long long)mixedPeriods,
(unsigned long long)completedPeriods,
(unsigned long long)interruptCount);
DbgPrint("AUDIO final selftest: underruns endpoint=%llu stream=%llu discontinuities endpoint=%llu stream=%llu\n",
(unsigned long long)endpointUnderruns,
(unsigned long long)streamInfo->Underruns,
(unsigned long long)endpointDiscontinuities,
(unsigned long long)streamDiscontinuities);
DbgPrint("AUDIO final selftest: position start=%llu end=%llu delta=%llu\n",
(unsigned long long)endpointBefore->HardwarePositionFrames,
(unsigned long long)endpointAfter->HardwarePositionFrames,
(unsigned long long)positionFrames);
}
// Plays the final-path sine test through the selected hardware endpoint and mixer.
void KiRunAudioFinalPathSelfTest(void) {
const uint64_t totalFrames = 3ull * KE_AUDIO_OUTPUT_SAMPLE_RATE;
const uint32_t chunkFrames = 1024u;
const char* failurePhase = (const char*)0;
KSTATUS failureStatus = NTLSTATUS_SUCCESS;
uint64_t failureValue = 0;
uint64_t endpointId = KeAudioGetDefaultEndpointId();
KE_AUDIO_ENDPOINT_INFO endpointBefore;
KE_AUDIO_ENDPOINT_INFO endpointAfter;
KE_AUDIO_STREAM_INFO startInfo;
KE_AUDIO_STREAM_INFO streamAfter;
KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)0;
int16_t* chunk = (int16_t*)0;
uint64_t expectedDiscontinuity = 0;
int haveEndpointBefore = 0;
int haveEndpointAfter = 0;
int haveStreamAfter = 0;
memset(&endpointBefore, 0, sizeof(endpointBefore));
memset(&endpointAfter, 0, sizeof(endpointAfter));
memset(&startInfo, 0, sizeof(startInfo));
memset(&streamAfter, 0, sizeof(streamAfter));
if (!endpointId) {
DbgPrint("AUDIO final selftest: SKIP no default endpoint\n");
return;
}
KSTATUS status = KiAudioSelfTestQueryEndpoint(endpointId, &endpointBefore);
if (KFAILED(status) || !endpointBefore.Available ||
endpointBefore.State != KeAudioEndpointReady) {
failurePhase = "endpoint-ready";
failureStatus = KFAILED(status) ? status : NTLSTATUS_DEVICE_ERROR;
goto cleanup;
}
haveEndpointBefore = 1;
if (endpointBefore.HardwareFormat.SampleRate != KE_AUDIO_OUTPUT_SAMPLE_RATE ||
endpointBefore.HardwareFormat.ChannelCount != KE_AUDIO_OUTPUT_CHANNELS ||
endpointBefore.HardwareFormat.SampleFormat != IoAudioSampleS16) {
failurePhase = "endpoint-format";
failureStatus = NTLSTATUS_NOT_SUPPORTED;
failureValue = endpointBefore.HardwareFormat.SampleFormat;
goto cleanup;
}
DbgPrint("AUDIO final selftest: begin endpoint=%llu kind=%u format=48000/S16/stereo\n",
(unsigned long long)endpointId,
(unsigned)endpointBefore.Kind);
IO_AUDIO_FORMAT format = {
.SampleRate = KE_AUDIO_OUTPUT_SAMPLE_RATE,
.ChannelCount = KE_AUDIO_OUTPUT_CHANNELS,
.SampleFormat = IoAudioSampleS16,
};
status = KeAudioCreateStream(endpointId, &format, &stream);
if (KFAILED(status)) {
failurePhase = "stream-create";
failureStatus = status;
goto cleanup;
}
chunk = (int16_t*)MmAllocatePoolTag(
(size_t)chunkFrames * KE_AUDIO_OUTPUT_BYTES_PER_FRAME,
KE_AUDIO_SELFTEST_TAG);
if (!chunk) {
failurePhase = "tone-allocation";
failureStatus = NTLSTATUS_NO_MEMORY;
goto cleanup;
}
double sine = 0.0;
double cosine = 1.0;
uint64_t submitted = 0;
while (submitted < KE_AUDIO_STREAM_RING_FRAMES) {
uint32_t count = (uint32_t)(KE_AUDIO_STREAM_RING_FRAMES - submitted);
if (count > chunkFrames)
count = chunkFrames;
KiAudioSelfTestGenerateTone(chunk, count, submitted, &sine, &cosine);
uint32_t written = 0;
status = KeAudioWriteStream(stream, chunk, count, &written);
if (KFAILED(status) || written != count) {
failurePhase = "initial-prefill";
failureStatus = KFAILED(status) ? status : NTLSTATUS_FAILURE;
failureValue = written;
goto cleanup;
}
submitted += written;
}
status = KeAudioStartStream(stream);
if (KFAILED(status)) {
failurePhase = "stream-start";
failureStatus = status;
goto cleanup;
}
status = KeAudioQueryStream(stream, &startInfo);
if (KFAILED(status)) {
failurePhase = "start-query";
failureStatus = status;
goto cleanup;
}
expectedDiscontinuity = startInfo.DiscontinuityGeneration;
while (submitted < totalFrames) {
uint64_t remaining = totalFrames - submitted;
uint32_t count = remaining < chunkFrames ? (uint32_t)remaining : chunkFrames;
KiAudioSelfTestGenerateTone(chunk, count, submitted, &sine, &cosine);
status = KiAudioSelfTestWriteFrames(stream,
chunk,
count,
expectedDiscontinuity);
if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) {
failurePhase = "stream-write";
failureStatus = status;
failureValue = submitted;
goto cleanup;
}
submitted += count;
}
status = KeAudioDrainStream(stream);
if (KFAILED(status)) {
failurePhase = "stream-drain";
failureStatus = status;
goto cleanup;
}
KE_AUDIO_STREAM_WAIT_CONDITION drainCondition = {
.Mask = KE_AUDIO_WAIT_DRAINED | KE_AUDIO_WAIT_FAILED | KE_AUDIO_WAIT_DISCONTINUITY,
.DrainedGeneration = startInfo.DrainedGeneration,
.DiscontinuityGeneration = expectedDiscontinuity,
};
status = KeAudioArmStreamWait(stream, &drainCondition);
if (KFAILED(status)) {
failurePhase = "drain-arm";
failureStatus = status;
goto cleanup;
}
status = KiAudioSelfTestWait(stream);
if (KFAILED(status) || status == NTLSTATUS_TIMEOUT) {
failurePhase = "drain-wait";
failureStatus = status;
goto cleanup;
}
status = KeAudioQueryStream(stream, &streamAfter);
if (KFAILED(status)) {
failurePhase = "final-stream-query";
failureStatus = status;
goto cleanup;
}
haveStreamAfter = 1;
if (streamAfter.State != KeAudioStreamStopped ||
streamAfter.DrainedGeneration != startInfo.DrainedGeneration + 1u ||
streamAfter.BufferedFrames != 0 || streamAfter.FramesSubmitted != totalFrames ||
streamAfter.FramesConsumed != totalFrames || streamAfter.FramesPresented != totalFrames ||
streamAfter.FramesDiscarded != 0 || streamAfter.Underruns != 0 ||
streamAfter.DiscontinuityGeneration != expectedDiscontinuity) {
failurePhase = "final-stream-counters";
failureStatus = NTLSTATUS_DEVICE_ERROR;
failureValue = streamAfter.FramesPresented;
goto cleanup;
}
status = KiAudioSelfTestQueryEndpoint(endpointId, &endpointAfter);
if (KFAILED(status)) {
failurePhase = "final-endpoint-query";
failureStatus = status;
goto cleanup;
}
haveEndpointAfter = 1;
uint64_t mixedPeriods = endpointAfter.PeriodsMixed - endpointBefore.PeriodsMixed;
uint64_t completedPeriods = endpointAfter.PeriodsCompleted - endpointBefore.PeriodsCompleted;
uint64_t interruptCount = endpointAfter.InterruptCount - endpointBefore.InterruptCount;
uint64_t endpointUnderruns = endpointAfter.Underruns - endpointBefore.Underruns;
uint64_t endpointDiscontinuities = endpointAfter.DiscontinuityGeneration -
endpointBefore.DiscontinuityGeneration;
uint64_t positionFrames = endpointAfter.HardwarePositionFrames -
endpointBefore.HardwarePositionFrames;
if (!mixedPeriods || !completedPeriods || !interruptCount ||
endpointUnderruns || endpointDiscontinuities ||
positionFrames + endpointAfter.FramesPerPeriod < totalFrames) {
failurePhase = "final-endpoint-counters";
failureStatus = NTLSTATUS_DEVICE_ERROR;
failureValue = positionFrames;
goto cleanup;
}
cleanup:
if (stream && !haveStreamAfter && KSUCCESS(KeAudioQueryStream(stream, &streamAfter)))
haveStreamAfter = 1;
if (haveEndpointBefore && !haveEndpointAfter &&
KSUCCESS(KiAudioSelfTestQueryEndpoint(endpointId, &endpointAfter)))
haveEndpointAfter = 1;
if (haveEndpointBefore && haveEndpointAfter && haveStreamAfter)
KiAudioSelfTestPrintCounters(endpointId,
&endpointBefore,
&endpointAfter,
&streamAfter,
expectedDiscontinuity);
if (stream) {
KE_AUDIO_STREAM_INFO cleanupInfo;
if (KSUCCESS(KeAudioQueryStream(stream, &cleanupInfo)) &&
cleanupInfo.State != KeAudioStreamStopped &&
cleanupInfo.State != KeAudioStreamFailed &&
cleanupInfo.State != KeAudioStreamClosed)
(void)KeAudioStopStream(stream);
}
if (chunk)
MmFreePool(chunk);
if (stream)
ObDereferenceObject(&stream->Header);
if (failurePhase) {
DbgPrint("AUDIO final selftest: FAIL phase=%s status=0x%x value=%llu\n",
failurePhase,
failureStatus,
(unsigned long long)failureValue);
return;
}
DbgPrint("AUDIO final selftest: PASS\n");
}
code/kernel/ke_boot_config.c
+154
View File
@@ -0,0 +1,154 @@
// Kernel boot configuration parser. This file intentionally implements only the boot.ini settings the kernel owns.
#include <kernel/ke_boot_config.h>
#include <ntlite/ddk/debug.h>
#include <stdbool.h>
#include <string.h>
// Returns whether one character is horizontal configuration whitespace.
static bool KiBootConfigSpace(char c) {
return c == ' ' || c == '\t' || c == '\r';
}
// Trims horizontal whitespace from one immutable line span.
static void KiBootConfigTrim(const char** begin, const char** end) {
while (*begin < *end && KiBootConfigSpace(**begin))
++*begin;
while (*end > *begin && KiBootConfigSpace((*end)[-1]))
--*end;
}
// Returns whether one immutable span exactly matches a NUL-terminated string.
static bool KiBootConfigSpanEquals(const char* begin, const char* end, const char* text) {
while (begin < end && *text) {
if (*begin++ != *text++)
return false;
}
return begin == end && *text == '\0';
}
// Parses one unsigned decimal or 0x-prefixed hexadecimal value without accepting trailing junk.
static KSTATUS KiBootConfigParseUInt32(const char* begin, const char* end, uint32_t* valueOut) {
KiBootConfigTrim(&begin, &end);
if (begin == end || !valueOut)
return NTLSTATUS_INVALID_PARAMETER;
uint32_t base = 10;
if (end - begin >= 2 && begin[0] == '0' && (begin[1] == 'x' || begin[1] == 'X')) {
base = 16;
begin += 2;
if (begin == end)
return NTLSTATUS_INVALID_PARAMETER;
}
uint32_t value = 0;
for (const char* p = begin; p < end; ++p) {
uint32_t digit;
if (*p >= '0' && *p <= '9') digit = (uint32_t)(*p - '0');
else if (*p >= 'a' && *p <= 'f') digit = (uint32_t)(10 + *p - 'a');
else if (*p >= 'A' && *p <= 'F') digit = (uint32_t)(10 + *p - 'A');
else return NTLSTATUS_INVALID_PARAMETER;
if (digit >= base || value > (UINT32_MAX - digit) / base)
return NTLSTATUS_INVALID_PARAMETER;
value = value * base + digit;
}
*valueOut = value;
return NTLSTATUS_SUCCESS;
}
// Copies one trimmed configuration value into a fixed kernel-owned string.
static KSTATUS KiBootConfigCopyString(const char* begin, const char* end, char* buffer, size_t bufferSize) {
KiBootConfigTrim(&begin, &end);
size_t length = (size_t)(end - begin);
if (!buffer || bufferSize == 0 || length >= bufferSize)
return NTLSTATUS_BUFFER_TOO_SMALL;
memcpy(buffer, begin, length);
buffer[length] = '\0';
return NTLSTATUS_SUCCESS;
}
// Parses the kernel-owned boot.ini settings from the raw bootloader string.
KSTATUS KeParseBootConfiguration(const char* text, size_t length, KE_BOOT_CONFIG* configOut) {
if (!text || !configOut)
return NTLSTATUS_INVALID_PARAMETER;
KE_BOOT_CONFIG config = {0};
bool systemVolumeSeen = false;
char section[32] = {0};
size_t offset = 0;
while (offset < length) {
size_t lineStart = offset;
while (offset < length && text[offset] != '\n')
++offset;
size_t lineEnd = offset;
if (offset < length)
++offset;
const char* begin = text + lineStart;
const char* end = text + lineEnd;
KiBootConfigTrim(&begin, &end);
if (begin == end || *begin == ';' || *begin == '#')
continue;
if (*begin == '[') {
if (end[-1] != ']')
return NTLSTATUS_INVALID_PARAMETER;
const char* sectionBegin = begin + 1;
const char* sectionEnd = end - 1;
KiBootConfigTrim(&sectionBegin, &sectionEnd);
size_t sectionLength = (size_t)(sectionEnd - sectionBegin);
if (sectionLength >= sizeof(section))
return NTLSTATUS_BUFFER_TOO_SMALL;
memcpy(section, sectionBegin, sectionLength);
section[sectionLength] = '\0';
continue;
}
const char* equals = NULL;
for (const char* p = begin; p < end; ++p) {
if (*p == '=') {
equals = p;
break;
}
}
if (!equals)
return NTLSTATUS_INVALID_PARAMETER;
const char* keyBegin = begin;
const char* keyEnd = equals;
const char* valueBegin = equals + 1;
const char* valueEnd = end;
KiBootConfigTrim(&keyBegin, &keyEnd);
if (strcmp(section, "boot") == 0 && KiBootConfigSpanEquals(keyBegin, keyEnd, "system_volume")) {
if (systemVolumeSeen)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = KiBootConfigCopyString(valueBegin, valueEnd,
config.SystemVolume, sizeof(config.SystemVolume));
if (!KSUCCESS(status) || config.SystemVolume[0] == '\0')
return KSUCCESS(status) ? NTLSTATUS_INVALID_PARAMETER : status;
systemVolumeSeen = true;
} else if (strcmp(section, "kernel") == 0 && KiBootConfigSpanEquals(keyBegin, keyEnd, "debug")) {
KSTATUS status = KiBootConfigParseUInt32(valueBegin, valueEnd, &config.Debug);
if (!KSUCCESS(status))
return status;
if (config.Debug & ~KE_DEBUG_VALID_MASK)
return NTLSTATUS_INVALID_PARAMETER;
} else if (strcmp(section, "init") == 0 && KiBootConfigSpanEquals(keyBegin, keyEnd, "commandline")) {
KSTATUS status = KiBootConfigCopyString(valueBegin, valueEnd,
config.CommandLine, sizeof(config.CommandLine));
if (!KSUCCESS(status))
return status;
}
}
if (!systemVolumeSeen)
return NTLSTATUS_NOT_FOUND;
*configOut = config;
return NTLSTATUS_SUCCESS;
}
code/kernel/ke_cpu.c
+10 -9
View File
@@ -3,9 +3,10 @@
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#include <kernel/ke_hal.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#define BOOTSTRAP_CPU_STACK_SIZE 0x4000u
#define IA32_GS_BASE 0xC0000101u
@@ -64,7 +65,7 @@ static uintptr_t KiDisableGlobalTranslations(void) {
// nt-lite does not enable PCID. Refuse inherited PCID state rather than
// pretending that a CR3 reload flushed every tagged translation.
if (cr4 & CR4_PCID_ENABLE)
HalStopProcessor();
KeStopProcessor();
if (cr4 & CR4_PAGE_GLOBAL_ENABLE)
__writecr4(cr4 & ~CR4_PAGE_GLOBAL_ENABLE);
@@ -98,7 +99,7 @@ static bool KiMemoryTypePolicyInstalled(uint32_t variableCount) {
// Programs the kernel-owned PAT policy over a uniform write-back MTRR baseline.
void KeInitializeMemoryTypes(void) {
if (!KiMemoryTypeFeaturesAvailable())
HalStopProcessor();
KeStopProcessor();
uint32_t variableCount = (uint32_t)(__readmsr(IA32_MTRRCAP) & MTRR_CAP_VARIABLE_COUNT);
if (KiMemoryTypePolicyInstalled(variableCount)) {
@@ -106,10 +107,10 @@ void KeInitializeMemoryTypes(void) {
return;
}
uintptr_t interruptFlags = HalDisableInterrupts();
uintptr_t interruptFlags = KeDisableInterrupts();
uintptr_t cr0 = __readcr0();
// Every CPU runs this before entering normal kernel execution. The BSP
// Every CPU runs this before entering normal kernel execution. The boot processor
// installs the policy while it is alone, and each AP installs it before
// publishing its started state.
__writecr0((cr0 | CR0_CACHE_DISABLE) & ~CR0_NOT_WRITE_THROUGH);
@@ -134,9 +135,9 @@ void KeInitializeMemoryTypes(void) {
__writecr4(cr4);
bool installed = KiMemoryTypePolicyInstalled(variableCount);
HalRestoreInterrupts(interruptFlags);
KeRestoreInterrupts(interruptFlags);
if (!installed)
HalStopProcessor();
KeStopProcessor();
_InterlockedExchange(&gWriteCombiningAvailable, 1);
}
@@ -176,9 +177,9 @@ void KeInitializeBootstrapCpu(void) {
KCPU* cpu = &gCpus[0];
cpu->Self = cpu;
cpu->CpuIndex = 0;
cpu->LapicId = 0;
cpu->HardwareId = 0;
cpu->State = KeCpuStarted;
cpu->IsBsp = 1;
cpu->IsBootProcessor = 1;
cpu->PackageId = 0;
cpu->CoreId = 0;
cpu->ThreadId = 0;
code/kernel/ke_debug.c
+20 -43
View File
@@ -4,8 +4,8 @@
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_hal.h>
#include <intrin.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_exception.h>
@@ -104,7 +104,7 @@ static volatile long gPanicStoppedCpuCount;
// Permanently stops a secondary processor that entered an active panic.
static _Noreturn void KiPanicSecondaryHalt(void) {
_InterlockedIncrement(&gPanicStoppedCpuCount);
HalStopProcessor();
KeStopProcessor();
}
// Returns a stable token for the current execution context.
@@ -131,7 +131,7 @@ static int KiPanicCurrentCpuOwnsReport(void) {
// Claims fatal-report ownership and masks interrupts on this processor.
static void KiBeginFatalReport(void) {
(void)HalDisableInterrupts();
(void)KeDisableInterrupts();
uintptr_t token = KiPanicCurrentToken();
if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) == 0) {
gPanicOwnerToken = token;
@@ -243,8 +243,8 @@ static KSTATUS KiDebugLogWriteAll(FILE_OBJECT* file,
}
// Persists the in-memory debug ring and flushes it through the storage stack.
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
KSTATUS KeFlushDebugLogToSystemRoot(const char* relativePath) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !relativePath || !relativePath[0])
return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait())
return NTLSTATUS_INVALID_IRQL;
@@ -259,10 +259,10 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
KeReleaseSpinlock(&gDebugPrintLock, flags);
(void)IoDeleteFile(path);
(void)KeDeleteSystemRootFile(relativePath);
FILE_OBJECT* file = (FILE_OBJECT*)0;
KSTATUS st = IoCreateFile(path, &file);
KSTATUS st = KeCreateSystemRootFile(relativePath, &file);
if (!KSUCCESS(st)) {
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
return st;
@@ -303,13 +303,9 @@ static void KiTryFlushPanicDebugLog(void) {
if (!KeCanWait() || !KeSystemRootReady())
return;
char path[1024];
if (!KeBuildSystemRootPath("kernel-panic.log", path, sizeof(path)))
return;
KSTATUS st = KeFlushDebugLogToFile(SysSvFromCstr(path));
KSTATUS st = KeFlushDebugLogToSystemRoot("kernel-panic.log");
if (!KSUCCESS(st))
DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
DbgPrint("debuglog: panic flush failed st=0x%x\n", st);
}
// Writes one formatted panic record through panic-safe endpoints.
@@ -327,7 +323,7 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
// Stop every other CPU before writing the final report. Panic-safe endpoints
// do not depend on scheduler locks held by processors stopped by this IPI.
HalApicSendPanicIpi();
KeBroadcastProcessorNmi();
static const char banner[] = "\n*** KERNEL PANIC ***\n";
KiOutputWrite(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
@@ -337,9 +333,9 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* thread = KeGetCurrentThread();
if (cpu) {
KiPanicPrintf("Panic CPU: %u LAPIC=%u",
KiPanicPrintf("Panic CPU: %u hardware=%u",
(unsigned)cpu->CpuIndex,
(unsigned)cpu->LapicId);
(unsigned)cpu->HardwareId);
if (thread) {
KiPanicPrintf(" thread=0x%016llX tid=%llu",
thread,
@@ -355,7 +351,7 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
va_end(args);
KiTryFlushPanicDebugLog();
HalStopProcessor();
KeStopProcessor();
}
// Writes user-visible console text through subscribed output endpoints.
@@ -377,9 +373,9 @@ static void KiFaultPrintf(const char* format, ...) {
va_end(args);
}
// Walk the process module list to find which image owns an address.
// Returns the module name string, or "unknown" if the address is outside all
// known image ranges. Base/size are optional and are useful for offset dumps.
// Resolves the main user image or a registered kernel image that owns an address.
// User-mode DLL bookkeeping belongs to the user-mode loader and is intentionally
// not duplicated in kernel diagnostics.
static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
uintptr_t* baseOut, size_t* sizeOut) {
if (baseOut) *baseOut = 0;
@@ -394,16 +390,6 @@ static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
return proc->Name;
}
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
uintptr_t base = proc->Modules[i].Base;
size_t size = proc->Modules[i].Size;
if (base && size && addr >= base && addr < base + size) {
if (baseOut) *baseOut = base;
if (sizeOut) *sizeOut = size;
return proc->Modules[i].Name;
}
}
uintptr_t kernelBase = 0;
size_t kernelSize = 0;
const char* kernelModule = LdrFindKernelModuleByAddress(addr, &kernelBase, &kernelSize);
@@ -595,15 +581,6 @@ static void KiDumpModules(PROCESS* proc) {
(unsigned long long)proc->ImageSize);
}
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
uintptr_t base = proc->Modules[i].Base;
size_t size = proc->Modules[i].Size;
KiFaultPrintf(" %-31s %016llx-%016llx size=%llx\n",
proc->Modules[i].Name,
(unsigned long long)base,
(unsigned long long)(base + size),
(unsigned long long)size);
}
}
#if 0
@@ -783,9 +760,9 @@ static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTA
// dead thread to the scheduler with interrupts enabled so waiters such as
// the foreground shell can observe the process exit event.
while (true) {
HalEnableInterrupts();
KeEnableInterrupts();
KeYield();
HalWaitForInterrupt();
KeWaitForInterrupt();
}
}
@@ -931,5 +908,5 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
frame.Rip = rip;
frame.Cs = cs;
(void)KeFaultDispatchFrame(&frame);
HalStopProcessor();
KeStopProcessor();
}
code/kernel/ke_event.c
+2 -2
View File
@@ -96,9 +96,9 @@ static void KiRundownMultiWait(THREAD* thread, void* context, KSTATUS status) {
}
// Creates event.
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT** out) {
if (!out) return NTLSTATUS_INVALID_PARAMETER;
EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
EVENT* e = (EVENT*)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
if (!e) return NTLSTATUS_NO_MEMORY;
e->Header.Destroy = KiEventDestroy;
e->Lock = (SPINLOCK)SPINLOCK_INIT;
code/kernel/ke_hal.c
+125
View File
@@ -0,0 +1,125 @@
// Kernel HAL binding and generic platform-operation wrappers.
#include <kernel/ke_hal.h>
static const HAL_API* gHal;
// Binds the bootstrap HAL table before kernel code begins using interrupt-state primitives.
KSTATUS KeBindBootstrapHal(void) {
const HAL_API* hal = HalGetApi();
if (!hal || hal->Size != sizeof(HAL_API) || hal->Version != HAL_API_VERSION ||
!hal->Processor.DisableInterrupts || !hal->Processor.RestoreInterrupts ||
!hal->Processor.EnableInterrupts || !hal->Processor.WaitForInterrupt ||
!hal->Processor.Halt || !hal->Processor.Stop) {
return NTLSTATUS_INVALID_PARAMETER;
}
gHal = hal;
return NTLSTATUS_SUCCESS;
}
// Returns whether the initialized table contains every mandatory kernel platform service.
static int KiHalRuntimeTableValid(const HAL_API* hal) {
return hal && hal->Size == sizeof(HAL_API) && hal->Version == HAL_API_VERSION &&
hal->Processor.DisableInterrupts && hal->Processor.RestoreInterrupts &&
hal->Processor.EnableInterrupts && hal->Processor.WaitForInterrupt &&
hal->Processor.Halt && hal->Processor.Stop && hal->Processor.QueryTopology &&
hal->Processor.Start &&
hal->Screen.GetFramebuffer &&
hal->Output.RegisterEndpoint && hal->Output.Write && hal->Output.QueryStats &&
hal->Timer.QueryTicks && hal->Timer.QueryTickFrequency && hal->Timer.Stall &&
hal->Timer.HandleInterrupt && hal->Timer.ConfigureCurrentProcessor &&
hal->Rtc.QueryTime && hal->Pci.Read32 && hal->Pci.Write32 &&
hal->InterruptController.RouteIrq && hal->InterruptController.MaskIrq &&
hal->InterruptController.UnmaskIrq && hal->InterruptController.SendEoi &&
hal->InterruptController.InitializeCurrentProcessor &&
hal->InterruptController.SendProcessorInterrupt &&
hal->InterruptController.SendProcessorNmi && hal->InterruptController.BroadcastNmi &&
hal->InterruptController.ComposeMsi && hal->Power.Shutdown;
}
// Initializes the HAL and validates every mandatory runtime service consumed by the kernel.
KSTATUS KeInitializeHal(const HAL_INIT_PARAMS* params) {
if (!gHal || !params)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = HalInitialize(params);
if (!KSUCCESS(status))
return status;
const HAL_API* hal = HalGetApi();
if (hal != gHal || !KiHalRuntimeTableValid(hal))
return NTLSTATUS_INVALID_PARAMETER;
return NTLSTATUS_SUCCESS;
}
// Returns the bound HAL table.
const HAL_API* KeGetHal(void) {
return gHal;
}
// Masks interrupts and returns the previous processor flags.
__declspec(dllexport) uintptr_t KeDisableInterrupts(void) {
return gHal->Processor.DisableInterrupts();
}
// Restores a previously saved processor interrupt state.
__declspec(dllexport) void KeRestoreInterrupts(uintptr_t flags) {
gHal->Processor.RestoreInterrupts(flags);
}
// Enables maskable interrupts on the current processor.
void KeEnableInterrupts(void) {
gHal->Processor.EnableInterrupts();
}
// Enables interrupts and waits for the next interrupt on the current processor.
void KeWaitForInterrupt(void) {
gHal->Processor.WaitForInterrupt();
}
// Halts the current processor once without changing interrupt state.
void KeHaltProcessor(void) {
gHal->Processor.Halt();
}
// Permanently stops the current processor.
_Noreturn void KeStopProcessor(void) {
gHal->Processor.Stop();
for (;;) {
}
}
// Queries platform processor topology without exposing interrupt-controller details.
KSTATUS KeQueryProcessorTopology(HAL_PROCESSOR_TOPOLOGY* topologyOut) {
return gHal->Processor.QueryTopology(topologyOut);
}
// Starts one processor at a physical bootstrap entry address.
KSTATUS KeStartProcessor(uint32_t hardwareId, uintptr_t startPhysical) {
return gHal->Processor.Start(hardwareId, startPhysical);
}
// Initializes the current processor's local interrupt-controller state.
void KeInitializeCurrentProcessorInterruptController(void) {
gHal->InterruptController.InitializeCurrentProcessor();
}
// Sends one fixed-vector interrupt to a processor hardware ID.
void KeSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector) {
gHal->InterruptController.SendProcessorInterrupt(hardwareId, vector);
}
// Sends one non-maskable interrupt to a processor hardware ID.
void KeSendProcessorNmi(uint32_t hardwareId) {
gHal->InterruptController.SendProcessorNmi(hardwareId);
}
// Sends a non-maskable interrupt to every other processor.
void KeBroadcastProcessorNmi(void) {
gHal->InterruptController.BroadcastNmi();
}
// Arms the calibrated periodic timer on the current processor.
void KeConfigureCurrentProcessorTimer(void) {
gHal->Timer.ConfigureCurrentProcessor();
}
code/kernel/ke_handle.c
+753
View File
@@ -0,0 +1,753 @@
// Kernel capability handles. This file owns process-local handle identity, grant delegation, and revocation.
#include <string.h>
#include <intrin.h>
#include <kernel/ke_handle.h>
#include <kernel/ke_process.h>
#include <kernel/ke_named_queue.h>
#include <kernel/ke_debug.h>
#include <kernel/io_file.h>
#include <kernel/ke_audio.h>
#include <kernel/mm_pool.h>
#include <ntlite/syscall.h>
#include <ntlite/security.h>
#include <ntlite/port.h>
typedef struct CAPABILITY_GRANT {
volatile long RefCount;
volatile long Revoked;
uint32_t Depth;
SYS_RIGHTS MaximumAccess;
OBJECT* Target;
struct CAPABILITY_GRANT* Parent;
} CAPABILITY_GRANT;
typedef struct KE_HANDLE_ENTRY {
struct KE_HANDLE_ENTRY* Next;
SYS_HANDLE Handle;
SYS_RIGHTS Access;
CAPABILITY_GRANT* Grant;
} KE_HANDLE_ENTRY;
typedef struct KE_CAPTURED_HANDLE {
CAPABILITY_GRANT* Grant;
SYS_RIGHTS Access;
} KE_CAPTURED_HANDLE;
_Static_assert((KE_HANDLE_BUCKET_COUNT & (KE_HANDLE_BUCKET_COUNT - 1u)) == 0,
"handle bucket count must be a power of two");
// Returns the process-local hash bucket for one opaque handle value.
static uint32_t KiHandleBucket(SYS_HANDLE handle) {
return (handle ^ (handle >> 16) ^ (handle >> 8)) & (KE_HANDLE_BUCKET_COUNT - 1u);
}
// Returns one entry while the handle-table lock is held.
static KE_HANDLE_ENTRY* KiFindHandleLocked(KE_HANDLE_TABLE* table, SYS_HANDLE handle, KE_HANDLE_ENTRY*** linkOut) {
if (linkOut) *linkOut = (KE_HANDLE_ENTRY**)0;
if (!table || handle == SYS_HANDLE_INVALID) return (KE_HANDLE_ENTRY*)0;
uint32_t bucket = KiHandleBucket(handle);
KE_HANDLE_ENTRY** link = &table->Buckets[bucket];
while (*link) {
if ((*link)->Handle == handle) {
if (linkOut) *linkOut = link;
return *link;
}
link = &(*link)->Next;
}
return (KE_HANDLE_ENTRY*)0;
}
// Acquires one grant reference while its owner still stabilizes the pointer.
static void KiReferenceGrant(CAPABILITY_GRANT* grant) {
if (!grant) return;
long refs = _InterlockedIncrement(&grant->RefCount);
if (refs <= 1) {
KePanic(KE_PANIC_CODE_GENERIC,
"capability: invalid grant reference grant=0x%016llX refs=%ld\n",
(uint64_t)(uintptr_t)grant,
refs);
}
}
// Releases one grant and recursively drops its immutable parent chain.
static void KiDereferenceGrant(CAPABILITY_GRANT* grant) {
while (grant) {
long refs = _InterlockedDecrement(&grant->RefCount);
if (refs > 0) return;
if (refs < 0) {
KePanic(KE_PANIC_CODE_GENERIC,
"capability: grant reference underflow grant=0x%016llX refs=%ld\n",
(uint64_t)(uintptr_t)grant,
refs);
}
CAPABILITY_GRANT* parent = grant->Parent;
if (!parent && grant->Target)
ObDereferenceObject(grant->Target);
MmFreePool(grant);
grant = parent;
}
}
// Returns whether a grant and every ancestor remain unrevoked.
static int KiGrantIsActive(const CAPABILITY_GRANT* grant) {
const CAPABILITY_GRANT* current = grant;
uint32_t depth = 0;
while (current) {
if (_InterlockedCompareExchange((volatile long*)&current->Revoked, 0, 0) != 0)
return 0;
current = current->Parent;
if (++depth > KE_CAPABILITY_MAX_DEPTH)
return 0;
}
return 1;
}
// Creates a root grant and takes the target object's lifetime reference.
static KSTATUS KiCreateRootGrant(OBJECT* object, SYS_RIGHTS maximumAccess, CAPABILITY_GRANT** grantOut) {
if (!object || !grantOut || maximumAccess == 0) return NTLSTATUS_INVALID_PARAMETER;
*grantOut = (CAPABILITY_GRANT*)0;
CAPABILITY_GRANT* grant = (CAPABILITY_GRANT*)MmAllocatePoolTag(sizeof(CAPABILITY_GRANT),
MM_POOL_TAG('c', 'a', 'p', 'g'));
if (!grant) return NTLSTATUS_NO_MEMORY;
memset(grant, 0, sizeof(*grant));
if (!ObTryReferenceObject(object)) {
MmFreePool(grant);
return NTLSTATUS_OBJECT_DEAD;
}
grant->RefCount = 1;
grant->Depth = 0;
grant->MaximumAccess = maximumAccess;
grant->Target = object;
grant->Parent = (CAPABILITY_GRANT*)0;
*grantOut = grant;
return NTLSTATUS_SUCCESS;
}
// Creates one attenuated child grant and retains its parent.
static KSTATUS KiCreateChildGrant(CAPABILITY_GRANT* parent,
SYS_RIGHTS maximumAccess,
CAPABILITY_GRANT** grantOut) {
if (!parent || !grantOut || maximumAccess == 0) return NTLSTATUS_INVALID_PARAMETER;
*grantOut = (CAPABILITY_GRANT*)0;
if (parent->Depth + 1u >= KE_CAPABILITY_MAX_DEPTH)
return NTLSTATUS_LIMIT_REACHED;
if ((maximumAccess & ~parent->MaximumAccess) != 0)
return NTLSTATUS_ACCESS_DENIED;
if (!KiGrantIsActive(parent))
return NTLSTATUS_CAPABILITY_REVOKED;
CAPABILITY_GRANT* grant = (CAPABILITY_GRANT*)MmAllocatePoolTag(sizeof(CAPABILITY_GRANT),
MM_POOL_TAG('c', 'a', 'p', 'g'));
if (!grant) return NTLSTATUS_NO_MEMORY;
memset(grant, 0, sizeof(*grant));
KiReferenceGrant(parent);
grant->RefCount = 1;
grant->Depth = parent->Depth + 1u;
grant->MaximumAccess = maximumAccess;
grant->Target = parent->Target;
grant->Parent = parent;
if (!KiGrantIsActive(parent)) {
KiDereferenceGrant(grant);
return NTLSTATUS_CAPABILITY_REVOKED;
}
*grantOut = grant;
return NTLSTATUS_SUCCESS;
}
// Records type-specific ownership after a process handle becomes visible.
static void KiHandleOpened(OBJECT* object) {
if (object && object->Type == ObjectTypeAudioStream)
KeAudioStreamHandleOpened((KE_AUDIO_STREAM*)object);
}
// Releases type-specific ownership before a process handle disappears.
static void KiHandleClosed(OBJECT* object) {
if (object && object->Type == ObjectTypeAudioStream)
KeAudioStreamHandleClosed((KE_AUDIO_STREAM*)object);
}
// Inserts one retained grant into a process and consumes that grant reference on success.
static KSTATUS KiInsertGrantHandle(PROCESS* process,
CAPABILITY_GRANT* grant,
SYS_RIGHTS access,
SYS_HANDLE* handleOut) {
if (!process || !grant || !handleOut || access == 0) return NTLSTATUS_INVALID_PARAMETER;
if ((access & ~grant->MaximumAccess) != 0) return NTLSTATUS_ACCESS_DENIED;
if (!KiGrantIsActive(grant)) return NTLSTATUS_CAPABILITY_REVOKED;
KE_HANDLE_ENTRY* entry = (KE_HANDLE_ENTRY*)MmAllocatePoolTag(sizeof(KE_HANDLE_ENTRY),
MM_POOL_TAG('h', 'n', 'd', 'l'));
if (!entry) return NTLSTATUS_NO_MEMORY;
memset(entry, 0, sizeof(*entry));
KE_HANDLE_TABLE* table = &process->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
if (table->Rundown) {
KeReleaseSpinlock(&table->Lock, flags);
MmFreePool(entry);
return NTLSTATUS_OBJECT_DEAD;
}
SYS_HANDLE handle = table->NextHandle;
if (handle == SYS_HANDLE_INVALID) {
KeReleaseSpinlock(&table->Lock, flags);
MmFreePool(entry);
return NTLSTATUS_ID_EXHAUSTED;
}
table->NextHandle = handle + 1u;
entry->Handle = handle;
entry->Access = access;
entry->Grant = grant;
uint32_t bucket = KiHandleBucket(handle);
entry->Next = table->Buckets[bucket];
table->Buckets[bucket] = entry;
table->Count++;
KiHandleOpened(grant->Target);
KeReleaseSpinlock(&table->Lock, flags);
*handleOut = handle;
return NTLSTATUS_SUCCESS;
}
// Initializes an empty process-local handle namespace.
void KeInitializeHandleTable(KE_HANDLE_TABLE* table) {
if (!table) return;
memset(table, 0, sizeof(*table));
table->NextHandle = 1u;
table->Lock = (SPINLOCK)SPINLOCK_INIT;
}
// Returns the object-type-specific rights mask accepted by the handle core.
SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type) {
const SYS_RIGHTS management = SYS_RIGHT_MANAGE;
switch (type) {
case ObjectTypeProcess:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT |
SYS_PROCESS_RIGHT_TERMINATE | SYS_PROCESS_RIGHT_INSERT_HANDLE |
SYS_PROCESS_RIGHT_BUILD;
case ObjectTypeThread:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_THREAD_RIGHT_RESUME;
case ObjectTypeMutex:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_MUTEX_RIGHT_RELEASE;
case ObjectTypeSemaphore:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_SEMAPHORE_RIGHT_SIGNAL;
case ObjectTypeEvent:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_EVENT_RIGHT_SIGNAL;
case ObjectTypeQueue:
return management | SYS_RIGHT_QUERY | SYS_QUEUE_RIGHT_SEND |
SYS_QUEUE_RIGHT_RECEIVE | SYS_QUEUE_RIGHT_PUBLISH;
case ObjectTypeFile:
return management | SYS_RIGHT_QUERY | SYS_FILE_RIGHT_READ_DATA |
SYS_FILE_RIGHT_WRITE_DATA | SYS_FILE_RIGHT_LOCK |
SYS_FILE_RIGHT_MAP_READ | SYS_FILE_RIGHT_MAP_WRITE;
case ObjectTypeDir:
return management | SYS_RIGHT_QUERY | SYS_DIR_RIGHT_ENUMERATE | SYS_DIR_RIGHT_OPEN |
SYS_DIR_RIGHT_MODIFY;
case ObjectTypeAsyncIo:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_ASYNC_IO_RIGHT_CANCEL;
case ObjectTypeSocket:
return management | SYS_RIGHT_QUERY | SYS_SOCKET_RIGHT_BIND |
SYS_SOCKET_RIGHT_CONNECT | SYS_SOCKET_RIGHT_LISTEN |
SYS_SOCKET_RIGHT_ACCEPT | SYS_SOCKET_RIGHT_SEND |
SYS_SOCKET_RIGHT_RECEIVE | SYS_SOCKET_RIGHT_SHUTDOWN |
SYS_SOCKET_RIGHT_POLL | SYS_SOCKET_RIGHT_SET_OPTION;
case ObjectTypeFileMapping:
return management | SYS_RIGHT_QUERY | SYS_FILE_MAPPING_RIGHT_MAP;
case ObjectTypeSurface:
return management | SYS_RIGHT_QUERY | SYS_SURFACE_RIGHT_MAP | SYS_SURFACE_RIGHT_PRESENT;
case ObjectTypeInputQueue:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_INPUT_QUEUE_RIGHT_READ;
case ObjectTypeTimer:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_TIMER_RIGHT_SET;
case ObjectTypeDisplayClock:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT;
case ObjectTypeAudioStream:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT |
SYS_AUDIO_STREAM_RIGHT_WRITE | SYS_AUDIO_STREAM_RIGHT_CONTROL;
case ObjectTypeSecurityContext:
return management | SYS_RIGHT_QUERY |
SYS_SECURITY_CONTEXT_RIGHT_DERIVE | SYS_SECURITY_CONTEXT_RIGHT_ASSIGN;
case ObjectTypePort:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT |
SYS_PORT_RIGHT_CONNECT | SYS_PORT_RIGHT_ACCEPT |
SYS_PORT_RIGHT_SEND | SYS_PORT_RIGHT_RECEIVE |
SYS_PORT_RIGHT_CALL | SYS_PORT_RIGHT_REPLY |
SYS_PORT_RIGHT_DISCONNECT;
case ObjectTypeVolume:
return management | SYS_RIGHT_QUERY | SYS_VOLUME_RIGHT_OPEN | SYS_VOLUME_RIGHT_MODIFY;
case ObjectTypeVolumeManager:
return management | SYS_RIGHT_QUERY |
SYS_VOLUME_MANAGER_RIGHT_ENUMERATE | SYS_VOLUME_MANAGER_RIGHT_OPEN |
SYS_VOLUME_MANAGER_RIGHT_WAIT;
case ObjectTypeDevice:
case ObjectTypeDriver:
case ObjectTypeUsbDevice:
case ObjectTypeUsbPipe:
case ObjectTypeImageSection:
case ObjectTypeAudioEndpoint:
return management | SYS_RIGHT_QUERY;
default:
return 0;
}
}
// Creates one root capability handle for an object.
KSTATUS KeCreateHandle(PROCESS* process, OBJECT* object, SYS_RIGHTS access, SYS_HANDLE* handleOut) {
if (!process || !object || !handleOut || access == 0) return NTLSTATUS_INVALID_PARAMETER;
SYS_RIGHTS valid = KeGetValidHandleRights(object->Type);
if (!valid || (access & ~valid) != 0) return NTLSTATUS_INVALID_PARAMETER;
CAPABILITY_GRANT* grant = (CAPABILITY_GRANT*)0;
KSTATUS status = KiCreateRootGrant(object, access, &grant);
if (!KSUCCESS(status)) return status;
status = KiInsertGrantHandle(process, grant, access, handleOut);
if (!KSUCCESS(status)) KiDereferenceGrant(grant);
return status;
}
// References the target object after validating type, revocation, and rights.
KSTATUS KeReferenceHandle(PROCESS* process,
SYS_HANDLE handle,
OBJECT_TYPE expectedType,
SYS_RIGHTS requiredRights,
OBJECT** objectOut) {
if (!process || !objectOut) return NTLSTATUS_INVALID_PARAMETER;
*objectOut = (OBJECT*)0;
if (handle == SYS_HANDLE_INVALID) return NTLSTATUS_INVALID_HANDLE;
KE_HANDLE_TABLE* table = &process->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
KE_HANDLE_ENTRY* entry = KiFindHandleLocked(table, handle, (KE_HANDLE_ENTRY***)0);
if (!entry) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_HANDLE;
}
if (requiredRights && (entry->Access & requiredRights) != requiredRights) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
if (!KiGrantIsActive(entry->Grant)) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_CAPABILITY_REVOKED;
}
OBJECT* object = entry->Grant->Target;
if (!object || (expectedType != 0 && object->Type != expectedType)) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_HANDLE;
}
if (!ObTryReferenceObject(object)) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_OBJECT_DEAD;
}
KeReleaseSpinlock(&table->Lock, flags);
*objectOut = object;
return NTLSTATUS_SUCCESS;
}
// Closes one process-local handle without reusing its numeric identity.
KSTATUS KeCloseHandle(PROCESS* process, SYS_HANDLE handle) {
if (!process || handle == SYS_HANDLE_INVALID) return NTLSTATUS_INVALID_HANDLE;
KE_HANDLE_TABLE* table = &process->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
KE_HANDLE_ENTRY** link = (KE_HANDLE_ENTRY**)0;
KE_HANDLE_ENTRY* entry = KiFindHandleLocked(table, handle, &link);
if (!entry || !link) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_HANDLE;
}
*link = entry->Next;
if (table->Count == 0) {
KeReleaseSpinlock(&table->Lock, flags);
KePanic(KE_PANIC_CODE_GENERIC, "capability: handle count underflow pid=%u\n", process->ProcessId);
}
table->Count--;
KeReleaseSpinlock(&table->Lock, flags);
KSTATUS closeStatus = NTLSTATUS_SUCCESS;
OBJECT* object = entry->Grant->Target;
if (object && object->Type == ObjectTypeFile)
closeStatus = IoFlushFile((FILE_OBJECT*)object);
KeNamedQueueUnpublishHandle(process, handle, object);
KiHandleClosed(object);
KiDereferenceGrant(entry->Grant);
MmFreePool(entry);
return closeStatus;
}
// Duplicates one alias of the same capability grant into another process.
KSTATUS KeDuplicateHandle(PROCESS* sourceProcess,
SYS_HANDLE sourceHandle,
PROCESS* targetProcess,
SYS_RIGHTS desiredRights,
uint32_t duplicateFlags,
SYS_HANDLE* targetHandleOut) {
if (!sourceProcess || !targetProcess || !targetHandleOut)
return NTLSTATUS_INVALID_PARAMETER;
if (sourceHandle == SYS_HANDLE_INVALID) return NTLSTATUS_INVALID_HANDLE;
const uint32_t validFlags = SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS;
if ((duplicateFlags & ~validFlags) != 0) return NTLSTATUS_INVALID_PARAMETER;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) && desiredRights != 0)
return NTLSTATUS_INVALID_PARAMETER;
if (!(duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) && desiredRights == 0)
return NTLSTATUS_INVALID_PARAMETER;
CAPABILITY_GRANT* grant = (CAPABILITY_GRANT*)0;
SYS_RIGHTS sourceAccess = 0;
KE_HANDLE_TABLE* table = &sourceProcess->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
KE_HANDLE_ENTRY* entry = KiFindHandleLocked(table, sourceHandle, (KE_HANDLE_ENTRY***)0);
if (!entry) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_HANDLE;
}
if (!KiGrantIsActive(entry->Grant)) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_CAPABILITY_REVOKED;
}
SYS_RIGHTS requiredManagement = SYS_RIGHT_DUPLICATE;
if (sourceProcess != targetProcess)
requiredManagement |= SYS_RIGHT_TRANSFER;
if ((entry->Access & requiredManagement) != requiredManagement) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
sourceAccess = entry->Access;
SYS_RIGHTS validRights = KeGetValidHandleRights(entry->Grant->Target->Type);
if (!(duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) &&
(desiredRights & ~validRights) != 0) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_PARAMETER;
}
if (!(duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) &&
(desiredRights & ~sourceAccess) != 0) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
grant = entry->Grant;
KiReferenceGrant(grant);
KeReleaseSpinlock(&table->Lock, flags);
SYS_RIGHTS targetAccess = (duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS)
? sourceAccess : desiredRights;
KSTATUS status = KiInsertGrantHandle(targetProcess, grant, targetAccess, targetHandleOut);
if (!KSUCCESS(status)) KiDereferenceGrant(grant);
return status;
}
// Derives one child grant whose maximum rights are an attenuated subset.
KSTATUS KeDeriveHandle(PROCESS* process,
SYS_HANDLE sourceHandle,
SYS_RIGHTS desiredRights,
SYS_HANDLE* derivedHandleOut) {
if (!process || !derivedHandleOut || desiredRights == 0)
return NTLSTATUS_INVALID_PARAMETER;
if (sourceHandle == SYS_HANDLE_INVALID) return NTLSTATUS_INVALID_HANDLE;
CAPABILITY_GRANT* parent = (CAPABILITY_GRANT*)0;
KE_HANDLE_TABLE* table = &process->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
KE_HANDLE_ENTRY* entry = KiFindHandleLocked(table, sourceHandle, (KE_HANDLE_ENTRY***)0);
if (!entry) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_HANDLE;
}
if (!KiGrantIsActive(entry->Grant)) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_CAPABILITY_REVOKED;
}
if ((entry->Access & SYS_RIGHT_DERIVE) == 0) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
SYS_RIGHTS validRights = KeGetValidHandleRights(entry->Grant->Target->Type);
if ((desiredRights & ~validRights) != 0) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_PARAMETER;
}
if ((desiredRights & ~entry->Access) != 0) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
parent = entry->Grant;
KiReferenceGrant(parent);
KeReleaseSpinlock(&table->Lock, flags);
CAPABILITY_GRANT* child = (CAPABILITY_GRANT*)0;
KSTATUS status = KiCreateChildGrant(parent, desiredRights, &child);
KiDereferenceGrant(parent);
if (!KSUCCESS(status)) return status;
status = KiInsertGrantHandle(process, child, desiredRights, derivedHandleOut);
if (!KSUCCESS(status)) KiDereferenceGrant(child);
return status;
}
// Revokes one grant and all grants derived from it.
KSTATUS KeRevokeHandle(PROCESS* process, SYS_HANDLE handle) {
if (!process) return NTLSTATUS_INVALID_PARAMETER;
if (handle == SYS_HANDLE_INVALID) return NTLSTATUS_INVALID_HANDLE;
KE_HANDLE_TABLE* table = &process->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
KE_HANDLE_ENTRY* entry = KiFindHandleLocked(table, handle, (KE_HANDLE_ENTRY***)0);
if (!entry) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_INVALID_HANDLE;
}
if ((entry->Access & SYS_RIGHT_REVOKE) == 0) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
if (!KiGrantIsActive(entry->Grant)) {
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_CAPABILITY_REVOKED;
}
_InterlockedExchange(&entry->Grant->Revoked, 1);
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_SUCCESS;
}
// Captures one capability so a later Port delivery does not depend on the source handle staying open.
KSTATUS KeCaptureHandleForTransfer(PROCESS* sourceProcess,
SYS_HANDLE sourceHandle,
SYS_RIGHTS desiredRights,
uint32_t flags,
KE_CAPTURED_HANDLE** capturedOut) {
if (!sourceProcess || !capturedOut)
return NTLSTATUS_INVALID_PARAMETER;
*capturedOut = (KE_CAPTURED_HANDLE*)0;
if (sourceHandle == SYS_HANDLE_INVALID)
return NTLSTATUS_INVALID_HANDLE;
const uint32_t validFlags = KE_HANDLE_TRANSFER_FLAG_SAME_RIGHTS |
KE_HANDLE_TRANSFER_FLAG_DERIVE;
if ((flags & ~validFlags) != 0)
return NTLSTATUS_INVALID_PARAMETER;
if ((flags & KE_HANDLE_TRANSFER_FLAG_SAME_RIGHTS) &&
(flags & KE_HANDLE_TRANSFER_FLAG_DERIVE))
return NTLSTATUS_INVALID_PARAMETER;
if ((flags & KE_HANDLE_TRANSFER_FLAG_SAME_RIGHTS) && desiredRights != 0)
return NTLSTATUS_INVALID_PARAMETER;
if (!(flags & KE_HANDLE_TRANSFER_FLAG_SAME_RIGHTS) && desiredRights == 0)
return NTLSTATUS_INVALID_PARAMETER;
CAPABILITY_GRANT* sourceGrant = (CAPABILITY_GRANT*)0;
SYS_RIGHTS sourceAccess = 0;
KE_HANDLE_TABLE* table = &sourceProcess->HandleTable;
uintptr_t lockFlags = KeAcquireSpinlock(&table->Lock);
KE_HANDLE_ENTRY* entry = KiFindHandleLocked(table, sourceHandle, (KE_HANDLE_ENTRY***)0);
if (!entry) {
KeReleaseSpinlock(&table->Lock, lockFlags);
return NTLSTATUS_INVALID_HANDLE;
}
if (!KiGrantIsActive(entry->Grant)) {
KeReleaseSpinlock(&table->Lock, lockFlags);
return NTLSTATUS_CAPABILITY_REVOKED;
}
SYS_RIGHTS required = SYS_RIGHT_TRANSFER;
if (flags & KE_HANDLE_TRANSFER_FLAG_DERIVE)
required |= SYS_RIGHT_DERIVE;
if ((entry->Access & required) != required) {
KeReleaseSpinlock(&table->Lock, lockFlags);
return NTLSTATUS_ACCESS_DENIED;
}
sourceAccess = entry->Access;
SYS_RIGHTS targetAccess = (flags & KE_HANDLE_TRANSFER_FLAG_SAME_RIGHTS)
? sourceAccess : desiredRights;
SYS_RIGHTS validRights = KeGetValidHandleRights(entry->Grant->Target->Type);
if ((targetAccess & ~validRights) != 0) {
KeReleaseSpinlock(&table->Lock, lockFlags);
return NTLSTATUS_INVALID_PARAMETER;
}
if ((targetAccess & ~sourceAccess) != 0) {
KeReleaseSpinlock(&table->Lock, lockFlags);
return NTLSTATUS_ACCESS_DENIED;
}
sourceGrant = entry->Grant;
KiReferenceGrant(sourceGrant);
KeReleaseSpinlock(&table->Lock, lockFlags);
CAPABILITY_GRANT* capturedGrant = sourceGrant;
if (flags & KE_HANDLE_TRANSFER_FLAG_DERIVE) {
KSTATUS status = KiCreateChildGrant(sourceGrant, targetAccess, &capturedGrant);
KiDereferenceGrant(sourceGrant);
if (!KSUCCESS(status))
return status;
}
KE_CAPTURED_HANDLE* captured = (KE_CAPTURED_HANDLE*)MmAllocatePoolTag(sizeof(*captured),
MM_POOL_TAG('c', 'a', 'p', 't'));
if (!captured) {
KiDereferenceGrant(capturedGrant);
return NTLSTATUS_NO_MEMORY;
}
captured->Grant = capturedGrant;
captured->Access = targetAccess;
*capturedOut = captured;
return NTLSTATUS_SUCCESS;
}
// Releases one captured capability that did not become a target-process handle.
void KeReleaseCapturedHandle(KE_CAPTURED_HANDLE* captured) {
if (!captured)
return;
if (captured->Grant)
KiDereferenceGrant(captured->Grant);
MmFreePool(captured);
}
// Materializes a complete transfer set atomically with respect to target-process handle visibility.
KSTATUS KeMaterializeCapturedHandles(PROCESS* targetProcess,
KE_CAPTURED_HANDLE** captured,
uint32_t count,
SYS_HANDLE* handlesOut) {
if (!targetProcess || (count && (!captured || !handlesOut)))
return NTLSTATUS_INVALID_PARAMETER;
if (count == 0)
return NTLSTATUS_SUCCESS;
KE_HANDLE_ENTRY** entries = (KE_HANDLE_ENTRY**)MmAllocatePoolTag(sizeof(KE_HANDLE_ENTRY*) * count,
MM_POOL_TAG('h', 'b', 'a', 't'));
if (!entries)
return NTLSTATUS_NO_MEMORY;
memset(entries, 0, sizeof(KE_HANDLE_ENTRY*) * count);
KSTATUS status = NTLSTATUS_SUCCESS;
for (uint32_t i = 0; i < count; ++i) {
if (!captured[i] || !captured[i]->Grant || captured[i]->Access == 0) {
status = NTLSTATUS_INVALID_PARAMETER;
break;
}
if (!KiGrantIsActive(captured[i]->Grant)) {
status = NTLSTATUS_CAPABILITY_REVOKED;
break;
}
entries[i] = (KE_HANDLE_ENTRY*)MmAllocatePoolTag(sizeof(KE_HANDLE_ENTRY),
MM_POOL_TAG('h', 'n', 'd', 'l'));
if (!entries[i]) {
status = NTLSTATUS_NO_MEMORY;
break;
}
memset(entries[i], 0, sizeof(*entries[i]));
}
if (!KSUCCESS(status)) {
for (uint32_t i = 0; i < count; ++i)
if (entries[i]) MmFreePool(entries[i]);
MmFreePool(entries);
return status;
}
KE_HANDLE_TABLE* table = &targetProcess->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
if (table->Rundown) {
status = NTLSTATUS_OBJECT_DEAD;
} else if (table->NextHandle == SYS_HANDLE_INVALID ||
(uint64_t)table->NextHandle + (uint64_t)count - 1ull > UINT32_MAX) {
status = NTLSTATUS_ID_EXHAUSTED;
} else {
for (uint32_t i = 0; i < count; ++i) {
if (!KiGrantIsActive(captured[i]->Grant)) {
status = NTLSTATUS_CAPABILITY_REVOKED;
break;
}
}
}
if (KSUCCESS(status)) {
for (uint32_t i = 0; i < count; ++i) {
SYS_HANDLE handle = table->NextHandle++;
entries[i]->Handle = handle;
entries[i]->Access = captured[i]->Access;
entries[i]->Grant = captured[i]->Grant;
uint32_t bucket = KiHandleBucket(handle);
entries[i]->Next = table->Buckets[bucket];
table->Buckets[bucket] = entries[i];
table->Count++;
KiHandleOpened(captured[i]->Grant->Target);
handlesOut[i] = handle;
captured[i]->Grant = (CAPABILITY_GRANT*)0;
}
}
KeReleaseSpinlock(&table->Lock, flags);
if (!KSUCCESS(status)) {
for (uint32_t i = 0; i < count; ++i)
MmFreePool(entries[i]);
MmFreePool(entries);
return status;
}
for (uint32_t i = 0; i < count; ++i) {
MmFreePool(captured[i]);
captured[i] = (KE_CAPTURED_HANDLE*)0;
}
MmFreePool(entries);
return NTLSTATUS_SUCCESS;
}
// Returns the number of currently live handles in one process.
uint32_t KeGetHandleCount(PROCESS* process) {
if (!process) return 0;
KE_HANDLE_TABLE* table = &process->HandleTable;
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
uint32_t count = table->Count;
KeReleaseSpinlock(&table->Lock, flags);
return count;
}
// Releases every handle and capability grant owned by one process.
void KeRundownHandleTable(PROCESS* process) {
if (!process) return;
KE_HANDLE_TABLE* table = &process->HandleTable;
uintptr_t rundownFlags = KeAcquireSpinlock(&table->Lock);
table->Rundown = 1u;
KeReleaseSpinlock(&table->Lock, rundownFlags);
for (uint32_t bucket = 0; bucket < KE_HANDLE_BUCKET_COUNT; ++bucket) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&table->Lock);
KE_HANDLE_ENTRY* entry = table->Buckets[bucket];
if (entry) {
table->Buckets[bucket] = entry->Next;
if (table->Count == 0) {
KeReleaseSpinlock(&table->Lock, flags);
KePanic(KE_PANIC_CODE_GENERIC,
"capability: rundown count underflow pid=%u\n",
process->ProcessId);
}
table->Count--;
}
KeReleaseSpinlock(&table->Lock, flags);
if (!entry) break;
OBJECT* object = entry->Grant->Target;
KeNamedQueueUnpublishHandle(process, entry->Handle, object);
KiHandleClosed(object);
KiDereferenceGrant(entry->Grant);
MmFreePool(entry);
}
}
}
code/kernel/ke_id.c
+22
View File
@@ -0,0 +1,22 @@
// Kernel monotonic identifier allocator. Public identities stay 32-bit while the internal cursor detects exhaustion without wrapping.
#include <intrin.h>
#include <kernel/ke_id.h>
// Allocates one 32-bit identifier and permanently advances the allocator.
KSTATUS KeAllocateId32(KE_ID_ALLOCATOR* allocator, uint32_t* idOut) {
if (!allocator || !idOut)
return NTLSTATUS_INVALID_PARAMETER;
while (1) {
long long next = _InterlockedCompareExchange64(&allocator->Next, 0, 0);
if ((unsigned long long)next > 0xFFFFFFFFULL)
return NTLSTATUS_ID_EXHAUSTED;
long long desired = next + 1;
if (_InterlockedCompareExchange64(&allocator->Next, desired, next) == next) {
*idOut = (uint32_t)next;
return NTLSTATUS_SUCCESS;
}
}
}
code/kernel/ke_idt.c
+1 -1
View File
@@ -221,7 +221,7 @@ void KeInitializeIdt(void) {
KiIdtSetGate(0x40, (uintptr_t)KeSyscallEntry, KI_IDT_TYPE_USER_INTERRUPT_GATE, 0);
static void *irqStubs[NUM_IRQS] = {
static void* irqStubs[NUM_IRQS] = {
(void*)KeIrqStub0, (void*)KeIrqStub1, (void*)KeIrqStub2,
(void*)KeIrqStub3, (void*)KeIrqStub4, (void*)KeIrqStub5,
(void*)KeIrqStub6, (void*)KeIrqStub7, (void*)KeIrqStub8,
code/kernel/ke_input.c
+5 -4
View File
@@ -7,6 +7,7 @@
#include <intrin.h>
#include <kernel/ke_input.h>
#include <kernel/ke_id.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_display.h>
#include <kernel/ke_event.h>
@@ -19,7 +20,6 @@
#define MOUSE_QUEUE_SIZE 128u
#define MOUSE_QUEUE_MASK (MOUSE_QUEUE_SIZE - 1u)
#define MAX_INPUT_KEYS 256u
#define MAX_INPUT_DEVICES 16u
#define KEY_REPEAT_DELAY_US 500000ull
#define KEY_REPEAT_PERIOD_US 33000ull
@@ -30,7 +30,7 @@ typedef struct KEY_CHAR_PAIR {
} KEY_CHAR_PAIR;
static KE_INPUT_CONTEXT gDefaultInputContext;
static volatile uint32_t gNextInputDeviceId = 1;
static KE_ID_ALLOCATOR gInputDeviceIds = KE_ID_ALLOCATOR_INIT(1u);
static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key);
static void KiInputServiceRepeat(KE_INPUT_CONTEXT* context);
@@ -246,8 +246,9 @@ void KeInputInitialize(const HAL_API* hal) {
// Implements the input Register Device operation.
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) {
(void)flags;
uint32_t id = gNextInputDeviceId++;
if (id >= MAX_INPUT_DEVICES) return NTLSTATUS_NO_MEMORY;
uint32_t id = 0;
KSTATUS status = KeAllocateId32(&gInputDeviceIds, &id);
if (!KSUCCESS(status)) return status;
if (outDeviceId) *outDeviceId = id;
DbgPrint("Input: registered device %u (%s)\n", id, name ? name : "unnamed");
return NTLSTATUS_SUCCESS;
code/kernel/ke_irq.c
+12 -5
View File
@@ -8,7 +8,7 @@
#include <kernel/ke_dpc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_debug.h>
#define MAX_IRQS 24
@@ -23,7 +23,7 @@ typedef struct IRQ_HANDLER_LIST {
static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
static bool gIrqSoftwareVector[MAX_IRQS];
static const HAL_API *gHal;
static const HAL_API* gHal;
// Implements the irq To Irql operation.
static KIRQL KiIrqToIrql(uint8_t irq) {
@@ -34,7 +34,9 @@ static KIRQL KiIrqToIrql(uint8_t irq) {
}
// Implements the irq Initialize operation.
void KeIrqInitialize(const HAL_API *hal) {
void KeIrqInitialize(const HAL_API* hal) {
if (!hal || !hal->InterruptController.RouteIrq || !hal->InterruptController.SendEoi)
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid interrupt-controller API during initialization\n");
gHal = hal;
}
@@ -59,6 +61,8 @@ __declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
// Implements the irq Register operation.
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
if (!gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: registration before interrupt-manager initialization\n");
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
for (uint8_t i = 0; i < list->Count; i++) {
@@ -73,11 +77,14 @@ __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
list->Handlers[list->Count++] = handler;
if (!gIrqSoftwareVector[irq])
gHal->Intc.RouteIrq(irq);
gHal->InterruptController.RouteIrq(irq);
}
// Implements the irq Dispatch operation.
void KeIrqDispatch(uint8_t irq) {
if (!gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: dispatch before interrupt-manager initialization, irq=%u\n", irq);
KIRQL targetIrql = KiIrqToIrql(irq);
KIRQL currentIrql = KeGetCurrentIrql();
@@ -104,7 +111,7 @@ void KeIrqDispatch(uint8_t irq) {
// only request rescheduling; KiExitInterrupt performs the switch after this
// function returns, so the EOI is guaranteed to be sent first. Keep the
// software level at DIRQL until after the controller has been acknowledged.
gHal->Intc.SendEoi();
gHal->InterruptController.SendEoi();
KeLowerIrql(oldIrql);
}
code/kernel/ke_port.c
+1636
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_process.c
+138 -332
View File
@@ -1,6 +1,8 @@
// Kernel process model. This file owns address-space metadata, handle tables, and process lifetime transitions.
// Kernel process model. This file owns process lifetime, thread membership, and address-space metadata.
#include <string.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_hal.h>
#include <kernel/ke_list.h>
#include <intrin.h>
#include <kernel/ke_process.h>
@@ -8,21 +10,19 @@
#include <kernel/ke_sched.h>
#include <kernel/ke_event.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_system_root.h>
#include <kernel/ke_userenv.h>
#include <kernel/io_file.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_named_queue.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_audio.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/ke_id.h>
#include <kernel/se_security.h>
#include <ntlite/syscall.h>
// Global process list.
static LIST_ENTRY gProcessList;
static SPINLOCK gProcessListLock;
static uint32_t gNextProcessId = 0;
static KE_ID_ALLOCATOR gProcessIds = KE_ID_ALLOCATOR_INIT(1u);
static PROCESS* gSystemProcess = (PROCESS*)0;
// Processes init.
@@ -64,7 +64,7 @@ KSTATUS KeCreateSystemProcess(PROCESS** out) {
}
PROCESS* proc = (PROCESS*)0;
KSTATUS s = KeCreateProcess("system", &proc);
KSTATUS s = KeCreateProcess("system", SeGetRootSecurityContext(), &proc);
if (!KSUCCESS(s)) {
*out = (PROCESS*)0;
return s;
@@ -76,17 +76,6 @@ KSTATUS KeCreateSystemProcess(PROCESS** out) {
return NTLSTATUS_SUCCESS;
}
// Implements the count Process Handles Locked operation.
static uint32_t KiCountProcessHandlesLocked(PROCESS* process) {
uint32_t count = 0;
if (!process) return 0;
for (uint32_t i = 1; i < HANDLE_TABLE_SIZE; i++) {
if (process->Handles[i].Object) count++;
}
return count;
}
// Copies truncated String.
static void KiCopyTruncatedString(char* dst, size_t dstSize, const char* src, size_t srcLen) {
if (!dst || dstSize == 0) return;
@@ -147,20 +136,17 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
memset(info, 0, sizeof(*info));
info->Size = sizeof(*info);
info->ProcessId = p->ProcessId;
info->SecurityId = p->SecurityContext ? p->SecurityContext->SecurityId : 0u;
info->ImageBase = p->ImageBase;
info->ImageSize = p->ImageSize;
info->PebBase = p->UserPebBase;
uintptr_t modFlags = KeAcquireSpinlock(&p->ModuleLock);
info->ModuleCount = p->ModuleCount;
KeReleaseSpinlock(&p->ModuleLock, modFlags);
uintptr_t handleFlags = KeAcquireSpinlock(&p->HandleLock);
uintptr_t threadFlags = KeAcquireSpinlock(&p->ThreadLock);
uint32_t threadCount = p->ThreadCount;
uint32_t handleCount = KiCountProcessHandlesLocked(p);
KeReleaseSpinlock(&p->ThreadLock, threadFlags);
uint32_t handleCount = KeGetHandleCount(p);
int32_t exitCode = p->ExitCode;
long terminating = p->Terminating;
long exited = p->Exited;
KeReleaseSpinlock(&p->HandleLock, handleFlags);
info->ThreadCount = threadCount;
info->HandleCount = handleCount;
@@ -184,6 +170,9 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
} else if (terminating) {
info->State = SYS_PROCESS_STATE_TERMINATING;
info->Flags |= SYS_PROCESS_FLAG_TERMINATING;
} else if (_InterlockedCompareExchange(&p->BuildState, 0, 0) != KE_PROCESS_BUILD_STARTED) {
info->State = SYS_PROCESS_STATE_CONSTRUCTING;
info->Flags |= SYS_PROCESS_FLAG_CONSTRUCTING;
} else {
info->State = SYS_PROCESS_STATE_RUNNING;
}
@@ -200,11 +189,6 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
KiCopyTruncatedString(info->Name, sizeof(info->Name), p->Name, sizeof(p->Name));
uintptr_t cwdFlags = KeAcquireSpinlock(&p->CwdLock);
info->CurrentDrive = p->CurrentDrive;
KiCopyTruncatedString(info->CurrentDirectory, sizeof(info->CurrentDirectory),
p->CurrentDirectory, p->CurrentDirectoryLength);
KeReleaseSpinlock(&p->CwdLock, cwdFlags);
}
count++;
}
@@ -260,7 +244,7 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
THREAD* current = KeGetCurrentThread();
size_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&process->ThreadLock);
LIST_ENTRY* head = &process->ThreadList;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, ProcessLink);
@@ -300,55 +284,13 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
}
count++;
}
KeReleaseSpinlock(&process->HandleLock, flags);
KeReleaseSpinlock(&process->ThreadLock, flags);
*countOut = count;
ObDereferenceObject(&process->Header);
return NTLSTATUS_SUCCESS;
}
// Records type-specific ownership after a handle slot becomes visible.
static void KiProcessHandleOpened(OBJECT* object) {
if (!object)
return;
if (object->Type == ObjectTypeAudioStream)
KeAudioStreamHandleOpened((KE_AUDIO_STREAM*)object);
}
// Releases type-specific ownership before the handle-table reference drops.
static void KiProcessReleaseHandleObject(OBJECT* object) {
if (!object)
return;
if (object->Type == ObjectTypeAudioStream)
KeAudioStreamHandleClosed((KE_AUDIO_STREAM*)object);
ObDereferenceObject(object);
}
// Releases every handle-table reference owned by one process.
static void KiProcessReleaseAllHandles(PROCESS* proc) {
if (!proc) return;
for (uint32_t i = 1; i < HANDLE_TABLE_SIZE; ++i) {
OBJECT* obj = (OBJECT*)0;
uintptr_t flags = KeAcquireSpinlock(&proc->HandleLock);
obj = proc->Handles[i].Object;
if (obj) {
proc->Handles[i].Object = (OBJECT*)0;
proc->Handles[i].Rights = 0;
proc->Handles[i].Flags = 0;
}
KeReleaseSpinlock(&proc->HandleLock, flags);
if (obj) {
KeNamedQueueUnpublishHandle(proc, (uint32_t)i, obj);
KiProcessReleaseHandleObject(obj);
}
}
}
// Releases the process-owned address space and per-process kernel resources.
// Idempotent because the normal exit path and the final OBJECT destructor can
// both observe a process whose threads are already gone.
@@ -358,7 +300,7 @@ static void KiProcessRundownResources(PROCESS* proc) {
return;
IoReleaseFileLocksForProcess(proc);
KiProcessReleaseAllHandles(proc);
KeRundownHandleTable(proc);
// Release every user VA region owned by this process. This includes the
// main image, DLLs, TEBs/stacks that survived thread teardown, process
@@ -366,17 +308,14 @@ static void KiProcessRundownResources(PROCESS* proc) {
MmVadDestroyAllForProcess(proc);
proc->ImageBase = 0;
proc->ImageSize = 0;
uintptr_t modFlags = KeAcquireSpinlock(&proc->ModuleLock);
proc->ModuleCount = 0;
KeReleaseSpinlock(&proc->ModuleLock, modFlags);
proc->UserPebBase = 0;
proc->UserPebSize = 0;
proc->UserImagePathBase = 0;
proc->UserImagePathSize = 0;
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
proc->UserStartupCapabilitiesBase = 0;
proc->UserStartupCapabilitiesSize = 0;
uintptr_t exceptionFlags = KeAcquireSpinlock(&proc->ExceptionLock);
proc->ExceptionDispatcher = 0;
proc->ExceptionDispatcherContext = 0;
@@ -397,11 +336,29 @@ static void KiProcessDestroy(OBJECT* obj) {
// ran, make sure its own handles and VADs are still reclaimed.
KiProcessRundownResources(proc);
if (proc->PortOutstandingMessages != 0 ||
proc->PortOutstandingBytes != 0 ||
proc->PortOutstandingHandles != 0 ||
proc->PortOutstandingCalls != 0) {
KePanic(KE_PANIC_CODE_GENERIC,
"process: destroying with live Port quota pid=%u messages=%u bytes=%u handles=%u calls=%u\n",
proc->ProcessId,
proc->PortOutstandingMessages,
proc->PortOutstandingBytes,
proc->PortOutstandingHandles,
proc->PortOutstandingCalls);
}
// Release the exit event only when the process object itself dies. External
// process handles may still wait/query an exited process after rundown.
if (proc->ExitEvent) {
ObDereferenceObject(&proc->ExitEvent->Header);
proc->ExitEvent = (EVENT *)0;
proc->ExitEvent = (EVENT*)0;
}
if (proc->SecurityContext) {
ObDereferenceObject(&proc->SecurityContext->Header);
proc->SecurityContext = (SECURITY_CONTEXT*)0;
}
// Remove from global list.
@@ -414,28 +371,36 @@ static void KiProcessDestroy(OBJECT* obj) {
}
// KeCreateProcess.
KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
KSTATUS KeCreateProcess(const char* name, SECURITY_CONTEXT* securityContext, PROCESS** out) {
KiProcessInit();
if (!name || !out) return NTLSTATUS_INVALID_PARAMETER;
if (!name || !securityContext || !out) return NTLSTATUS_INVALID_PARAMETER;
PROCESS* proc = (PROCESS *)ObAllocateObject(ObjectTypeProcess, sizeof(PROCESS));
uint32_t processId = 0;
KSTATUS idStatus = KeAllocateId32(&gProcessIds, &processId);
if (!KSUCCESS(idStatus))
return idStatus;
PROCESS* proc = (PROCESS*)ObAllocateObject(ObjectTypeProcess, sizeof(PROCESS));
if (!proc) return NTLSTATUS_NO_MEMORY;
proc->Header.Destroy = KiProcessDestroy;
proc->ProcessId = _InterlockedExchangeAdd((volatile long *)&gNextProcessId, 1);
proc->Flags = 0;
proc->ProcessId = processId;
proc->Flags = 0;
proc->SecurityContext = securityContext;
ObReferenceObject(&securityContext->Header);
strncpy(proc->Name, name, PROCESS_NAME_MAX - 1);
proc->Name[PROCESS_NAME_MAX - 1] = '\0';
LIST_INIT(&proc->ThreadList);
proc->ThreadCount = 0;
proc->HandleLock = (SPINLOCK)SPINLOCK_INIT;
proc->ThreadLock = (SPINLOCK)SPINLOCK_INIT;
proc->ExitCode = 0;
proc->Terminating = 0;
proc->Exited = 0;
proc->ResourcesReleased = 0;
proc->BuildState = KE_PROCESS_BUILD_STARTED;
proc->BuildOperations = 0;
proc->ImageBase = 0;
proc->ImageSize = 0;
proc->UserPebBase = 0;
@@ -444,10 +409,8 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->UserImagePathSize = 0;
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
proc->ModuleCount = 0;
proc->ModuleLock = (SPINLOCK)SPINLOCK_INIT;
proc->UserStartupCapabilitiesBase = 0;
proc->UserStartupCapabilitiesSize = 0;
proc->ExceptionDispatcher = 0;
proc->ExceptionDispatcherContext = 0;
proc->ExceptionStackBase = 0;
@@ -463,15 +426,15 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->LastExceptionAddress = 0;
proc->LastExceptionFaultAddress = 0;
proc->ExceptionLock = (SPINLOCK)SPINLOCK_INIT;
proc->CurrentDirectory[0] = '\0';
proc->CurrentDirectoryLength = 0;
proc->CurrentDrive = KeGetSystemRootDrive();
proc->CwdLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&proc->VadList);
proc->VadLock = (SPINLOCK)SPINLOCK_INIT;
// Slot 0 is the invalid handle - leave it NULL.
memset(proc->Handles, 0, sizeof(proc->Handles));
KeInitializeHandleTable(&proc->HandleTable);
proc->PortQuotaLock = (SPINLOCK)SPINLOCK_INIT;
proc->PortOutstandingMessages = 0;
proc->PortOutstandingBytes = 0;
proc->PortOutstandingHandles = 0;
proc->PortOutstandingCalls = 0;
LIST_INIT(&proc->GlobalLink);
@@ -514,32 +477,7 @@ static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
return NTLSTATUS_SUCCESS;
}
static KSTATUS KiCopyEnvironmentStringViewFixed(PROCESS* proc,
SYS_STRING_VIEW src,
size_t capacity,
uintptr_t* baseOut,
size_t* sizeOut) {
if (!baseOut || !sizeOut || capacity == 0) return NTLSTATUS_INVALID_PARAMETER;
*baseOut = 0;
*sizeOut = 0;
if (!SysSvIsValid(src)) return NTLSTATUS_INVALID_PARAMETER;
if (SysSvContainsByte(src, '\0')) return NTLSTATUS_INVALID_PARAMETER;
if (src.Length + 1 > capacity) return NTLSTATUS_BUFFER_TOO_SMALL;
char* dst = (char*)MmAllocateUserMemoryForProcess(proc, capacity);
if (!dst) return NTLSTATUS_NO_MEMORY;
memset(dst, 0, capacity);
if (src.Length) memcpy(dst, src.Data, src.Length);
dst[src.Length] = '\0';
*baseOut = (uintptr_t)dst;
*sizeOut = capacity;
return NTLSTATUS_SUCCESS;
}
// Releases partial Process Environment.
// Releases a partially constructed process environment.
static void KiFreePartialProcessEnvironment(PROCESS* proc) {
if (!proc) return;
@@ -558,78 +496,31 @@ static void KiFreePartialProcessEnvironment(PROCESS* proc) {
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
}
if (proc->UserCwdBase && proc->UserCwdSize) {
MmFreeUserMemoryForProcess(proc, (void*)proc->UserCwdBase);
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
if (proc->UserStartupCapabilitiesBase && proc->UserStartupCapabilitiesSize) {
MmFreeUserMemoryForProcess(proc, (void*)proc->UserStartupCapabilitiesBase);
proc->UserStartupCapabilitiesBase = 0;
proc->UserStartupCapabilitiesSize = 0;
}
}
static KSTATUS KiFormatNativeCwdV(const PROCESS* proc,
SYS_STRING_VIEW currentDirectory,
char* out,
size_t outSize,
size_t* lengthOut) {
if (lengthOut) *lengthOut = 0;
if (!out || outSize == 0) return NTLSTATUS_INVALID_PARAMETER;
out[0] = '\0';
if (!SysSvIsValid(currentDirectory)) return NTLSTATUS_INVALID_PARAMETER;
if (SysSvContainsByte(currentDirectory, '\0')) return NTLSTATUS_INVALID_PARAMETER;
uint8_t drive = proc ? proc->CurrentDrive : 0;
if (drive > 7) drive = 0;
if (currentDirectory.Length + 4 > outSize) return NTLSTATUS_BUFFER_TOO_SMALL;
out[0] = (char)('0' + drive);
out[1] = ':';
out[2] = '/';
if (currentDirectory.Length)
memcpy(out + 3, currentDirectory.Data, currentDirectory.Length);
out[3 + currentDirectory.Length] = '\0';
if (lengthOut) *lengthOut = 3 + currentDirectory.Length;
return NTLSTATUS_SUCCESS;
}
KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
SYS_STRING_VIEW imagePath,
SYS_STRING_VIEW commandLine,
SYS_STRING_VIEW currentDirectory) {
SYS_STRING_VIEW commandLine) {
if (!proc || imagePath.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
if (!SysSvIsValid(imagePath) || !SysSvIsValid(commandLine) || !SysSvIsValid(currentDirectory))
if (!SysSvIsValid(imagePath) || !SysSvIsValid(commandLine))
return NTLSTATUS_INVALID_PARAMETER;
if (SysSvContainsByte(imagePath, '\0') ||
SysSvContainsByte(commandLine, '\0') ||
SysSvContainsByte(currentDirectory, '\0'))
if (SysSvContainsByte(imagePath, '\0') || SysSvContainsByte(commandLine, '\0'))
return NTLSTATUS_INVALID_PARAMETER;
SYS_STRING_VIEW finalCommandLine = commandLine.Length ? commandLine : imagePath;
SYS_STRING_VIEW finalCwdRel = currentDirectory;
if (finalCwdRel.Length == 0 && proc->CurrentDirectoryLength)
finalCwdRel = SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength);
char finalCwd[SYS_CWD_MAX];
size_t finalCwdLength = 0;
KSTATUS s = KiFormatNativeCwdV(proc, finalCwdRel, finalCwd, sizeof(finalCwd), &finalCwdLength);
if (!KSUCCESS(s)) return s;
KiFreePartialProcessEnvironment(proc);
s = KiCopyEnvironmentStringView(proc, imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
KSTATUS s = KiCopyEnvironmentStringView(proc, imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
if (!KSUCCESS(s)) goto fail;
s = KiCopyEnvironmentStringView(proc, finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize);
if (!KSUCCESS(s)) goto fail;
s = KiCopyEnvironmentStringViewFixed(proc,
SysSvFromParts(finalCwd, finalCwdLength),
SYS_CWD_MAX,
&proc->UserCwdBase,
&proc->UserCwdSize);
if (!KSUCCESS(s)) goto fail;
SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemoryForProcess(proc, sizeof(SYS_PEB));
if (!peb) { s = NTLSTATUS_NO_MEMORY; goto fail; }
memset(peb, 0, sizeof(SYS_PEB));
@@ -639,7 +530,10 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
peb->ProcessId = proc->ProcessId;
peb->ImagePath = SysSvFromParts((const char*)proc->UserImagePathBase, imagePath.Length);
peb->CommandLine = SysSvFromParts((const char*)proc->UserCmdlineBase, finalCommandLine.Length);
peb->CurrentDirectory = SysSvFromParts((const char*)proc->UserCwdBase, finalCwdLength);
peb->StartupCapabilities = (const SYS_STARTUP_CAPABILITY*)0;
peb->StartupCapabilityCount = 0;
peb->Reserved = 0;
peb->LoaderData = (const void*)0;
proc->UserPebBase = (uintptr_t)peb;
proc->UserPebSize = sizeof(SYS_PEB);
@@ -650,13 +544,65 @@ fail:
return s;
}
// Copies the immutable startup-capability descriptor table into child-owned user memory.
KSTATUS KeSetProcessStartupCapabilities(PROCESS* proc,
const SYS_STARTUP_CAPABILITY* capabilities,
uint32_t count) {
if (!proc || !proc->UserPebBase) return NTLSTATUS_INVALID_PARAMETER;
if (count > SYS_STARTUP_CAPABILITY_MAX) return NTLSTATUS_LIMIT_REACHED;
if (count && !capabilities) return NTLSTATUS_INVALID_PARAMETER;
if (proc->UserStartupCapabilitiesBase && proc->UserStartupCapabilitiesSize) {
MmFreeUserMemoryForProcess(proc, (void*)proc->UserStartupCapabilitiesBase);
proc->UserStartupCapabilitiesBase = 0;
proc->UserStartupCapabilitiesSize = 0;
}
SYS_PEB* peb = (SYS_PEB*)proc->UserPebBase;
peb->StartupCapabilities = (const SYS_STARTUP_CAPABILITY*)0;
peb->StartupCapabilityCount = 0;
if (!count) return NTLSTATUS_SUCCESS;
size_t bytes = (size_t)count * sizeof(SYS_STARTUP_CAPABILITY);
SYS_STARTUP_CAPABILITY* dst = (SYS_STARTUP_CAPABILITY*)MmAllocateUserMemoryForProcess(proc, bytes);
if (!dst) return NTLSTATUS_NO_MEMORY;
memcpy(dst, capabilities, bytes);
KSTATUS protectStatus = MmVadProtect(proc, dst, bytes, PAGE_PRESENT | PAGE_USER | PAGE_NX);
if (!KSUCCESS(protectStatus)) {
MmFreeUserMemoryForProcess(proc, dst);
return protectStatus;
}
proc->UserStartupCapabilitiesBase = (uintptr_t)dst;
proc->UserStartupCapabilitiesSize = bytes;
peb->StartupCapabilities = dst;
peb->StartupCapabilityCount = count;
return NTLSTATUS_SUCCESS;
}
// Installs the opaque userspace loader-data pointer in the child PEB.
KSTATUS KeSetProcessLoaderData(PROCESS* proc, const void* loaderData) {
if (!proc || !proc->UserPebBase) return NTLSTATUS_INVALID_PARAMETER;
SYS_PEB* peb = (SYS_PEB*)proc->UserPebBase;
peb->LoaderData = loaderData;
return NTLSTATUS_SUCCESS;
}
// Stops new process-construction operations. Active builders defer final teardown until they leave.
void KeStopProcessConstruction(PROCESS* process) {
if (!process) return;
_InterlockedExchange(&process->BuildState, KE_PROCESS_BUILD_DEAD);
}
// Process exit / termination.
void KeCompleteProcessExitIfLastThread(PROCESS* process) {
if (!process) return;
if (_InterlockedCompareExchange(&process->BuildOperations, 0, 0) != 0) return;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&process->ThreadLock);
uint32_t threadCount = process->ThreadCount;
KeReleaseSpinlock(&process->HandleLock, flags);
KeReleaseSpinlock(&process->ThreadLock, flags);
if (threadCount == 0 && _InterlockedExchange(&process->Exited, 1) == 0) {
KiProcessRundownResources(process);
@@ -664,30 +610,24 @@ void KeCompleteProcessExitIfLastThread(PROCESS* process) {
}
}
// Implements the exit Process operation.
// Exits one process by terminating every thread in it.
// SysExitThread remains the operation for ending only the calling thread.
KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode) {
if (!process) return NTLSTATUS_INVALID_PARAMETER;
if (KeIsProcessTerminationProtected(process)) return NTLSTATUS_ACCESS_DENIED;
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
// The calling thread exits immediately after this in SysExitProcess. Do
// not drop the process object here: each thread owns one process reference,
// and KiThreadDestroy releases that reference after unlinking the thread.
KeCompleteProcessExitIfLastThread(process);
return NTLSTATUS_SUCCESS;
return KeTerminateProcess(process, exitCode);
}
// Implements the terminate Process operation.
// Begins termination of one process and wakes every sibling thread.
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
if (!process) return NTLSTATUS_INVALID_PARAMETER;
if (KeIsProcessTerminationProtected(process)) return NTLSTATUS_ACCESS_DENIED;
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
KeStopProcessConstruction(process);
if (_InterlockedCompareExchange(&process->BuildOperations, 0, 0) != 0)
return NTLSTATUS_SUCCESS;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&process->ThreadLock);
LIST_ENTRY* head = &process->ThreadList;
THREAD* self = KeGetCurrentThread();
KCPU* selfCpu = KeGetCurrentCpu();
@@ -714,11 +654,11 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
KCPU* cpu = &gCpus[t->RunningCpu];
if (cpu->State == KeCpuOnline) {
_InterlockedExchange(&cpu->ReschedulePending, 1);
HalApicSendRescheduleIpi(cpu->LapicId);
KeSendProcessorInterrupt(cpu->HardwareId, KE_VECTOR_RESCHEDULE_IPI);
}
}
}
KeReleaseSpinlock(&process->HandleLock, flags);
KeReleaseSpinlock(&process->ThreadLock, flags);
KeCompleteProcessExitIfLastThread(process);
return NTLSTATUS_SUCCESS;
@@ -741,140 +681,6 @@ KSTATUS KeGetProcessExitCode(PROCESS* process, int32_t* exitCodeOut) {
return NTLSTATUS_SUCCESS;
}
// Handle table.
KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t rights, uint32_t* handleOut) {
if (!process || !object || !handleOut || rights == 0) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
if (!process->Handles[i].Object) {
if (!ObTryReferenceObject(object)) {
KeReleaseSpinlock(&process->HandleLock, flags);
return NTLSTATUS_OBJECT_DEAD;
}
process->Handles[i].Object = object;
process->Handles[i].Rights = rights;
process->Handles[i].Flags = 0;
KiProcessHandleOpened(object);
*handleOut = (uint32_t)i;
KeReleaseSpinlock(&process->HandleLock, flags);
return NTLSTATUS_SUCCESS;
}
}
KeReleaseSpinlock(&process->HandleLock, flags);
return NTLSTATUS_NO_MEMORY;
}
// Creates handle.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut) {
return KeAllocateHandleWithRights(process, object, HANDLE_RIGHT_ALL, handleOut);
}
// Releases handle.
void KeCloseHandle(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
OBJECT* obj = process->Handles[handle].Object;
if (obj) {
process->Handles[handle].Object = (OBJECT*)0;
process->Handles[handle].Rights = 0;
process->Handles[handle].Flags = 0;
KeReleaseSpinlock(&process->HandleLock, flags);
if (obj->Type == ObjectTypeFile)
(void)IoFlushFile((FILE_OBJECT*)obj);
KeNamedQueueUnpublishHandle(process, handle, obj);
KiProcessReleaseHandleObject(obj);
return;
}
KeReleaseSpinlock(&process->HandleLock, flags);
}
// Returns handle With Rights.
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE)
return (OBJECT*)0;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
HANDLE_ENTRY* entry = &process->Handles[handle];
OBJECT* obj = entry->Object;
if (obj && requiredRights && ((entry->Rights & requiredRights) != requiredRights))
obj = (OBJECT*)0;
if (obj && !ObTryReferenceObject(obj))
obj = (OBJECT*)0;
KeReleaseSpinlock(&process->HandleLock, flags);
return obj;
}
// Returns handle.
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) {
return KeLookupHandleWithRights(process, handle, 0);
}
// Returns handle Rights.
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return 0;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
uint32_t rights = process->Handles[handle].Object ? process->Handles[handle].Rights : 0;
KeReleaseSpinlock(&process->HandleLock, flags);
return rights;
}
// Duplicates a handle from one process table into another.
KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle, PROCESS* targetProcess, uint32_t desiredRights, uint32_t duplicateFlags, uint32_t* targetHandleOut) {
if (!sourceProcess || !targetProcess || !targetHandleOut) return NTLSTATUS_INVALID_PARAMETER;
if (sourceHandle == 0 || sourceHandle >= HANDLE_TABLE_SIZE) return NTLSTATUS_INVALID_HANDLE;
const uint32_t validDuplicateFlags = SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS;
const uint32_t publicHandleRights = SYS_HANDLE_RIGHT_ALL;
if ((duplicateFlags & ~validDuplicateFlags) != 0) return NTLSTATUS_INVALID_PARAMETER;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) != 0 && desiredRights != 0)
return NTLSTATUS_INVALID_PARAMETER;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) == 0) {
if (desiredRights == 0) return NTLSTATUS_INVALID_PARAMETER;
if ((desiredRights & ~publicHandleRights) != 0) return NTLSTATUS_INVALID_PARAMETER;
}
OBJECT* obj = NULL;
uint32_t sourceRights = 0;
uintptr_t savedFlags = KeAcquireSpinlock(&sourceProcess->HandleLock);
HANDLE_ENTRY* entry = &sourceProcess->Handles[sourceHandle];
if (!entry->Object) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return NTLSTATUS_INVALID_HANDLE;
}
if ((entry->Rights & HANDLE_RIGHT_DUPLICATE) == 0) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return NTLSTATUS_ACCESS_DENIED;
}
obj = entry->Object;
sourceRights = entry->Rights;
if (!ObTryReferenceObject(obj)) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return NTLSTATUS_OBJECT_DEAD;
}
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
uint32_t targetRights = sourceRights;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) == 0) {
if ((desiredRights & ~sourceRights) != 0) {
ObDereferenceObject(obj);
return NTLSTATUS_ACCESS_DENIED;
}
targetRights = desiredRights;
}
KSTATUS s = KeAllocateHandleWithRights(targetProcess, obj, targetRights, targetHandleOut);
ObDereferenceObject(obj);
return s;
}
// Returns process By Id.
PROCESS* KeGetProcessById(uint32_t pid) {
KiProcessInit();
code/kernel/ke_queue.c
+15 -39
View File
@@ -4,6 +4,7 @@
#include <string.h>
#include <kernel/ke_queue.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_time.h>
#include <kernel/ke_irql.h>
#include <kernel/ob_object.h>
@@ -46,49 +47,24 @@ static void KiQueueNoteRemovedLocked(QUEUE* queue, uint32_t previousCount) {
KeClearEvent(queue->NotEmptyEvent);
}
// Returns the scheduler tick value used by queue finite-timeout accounting.
static uint64_t KiQueueReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Converts a millisecond timeout to at least one scheduler tick.
static uint64_t KiQueueTimeoutTicks(uint64_t timeoutMilliseconds) {
uint64_t ticks = timeoutMilliseconds / SCHED_TICK_MS;
if ((timeoutMilliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Computes an absolute scheduler-tick deadline for a finite timeout.
// Computes an absolute monotonic deadline for a finite timeout.
static uint64_t KiQueueTimeoutDeadline(uint64_t timeoutMilliseconds) {
uint64_t now = KiQueueReadSchedulerTick();
uint64_t ticks = KiQueueTimeoutTicks(timeoutMilliseconds);
uint64_t deadline = now + ticks;
return deadline < now ? UINT64_MAX : deadline;
return KeMonotonicDeadlineAfterMilliseconds(timeoutMilliseconds);
}
// Computes the remaining wait time for a finite timeout deadline.
static uint64_t KiQueueRemainingMilliseconds(uint64_t deadlineTick) {
uint64_t now = KiQueueReadSchedulerTick();
if (deadlineTick <= now)
return 0;
uint64_t ticks = deadlineTick - now;
if (ticks > UINT64_MAX / SCHED_TICK_MS)
return UINT64_MAX;
return ticks * SCHED_TICK_MS;
static uint64_t KiQueueRemainingMilliseconds(uint64_t deadlineMicroseconds) {
return KeMonotonicMillisecondsUntil(deadlineMicroseconds);
}
// Waits for a queue state event using either an infinite or finite timeout.
static KSTATUS KiQueueWaitForEvent(QUEUE* queue, EVENT* event, uint64_t timeoutMilliseconds, uint64_t deadlineTick) {
static KSTATUS KiQueueWaitForEvent(QUEUE* queue, EVENT* event, uint64_t timeoutMilliseconds, uint64_t deadlineMicroseconds) {
if (!queue || !event)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t waitMilliseconds = timeoutMilliseconds;
if (timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE) {
waitMilliseconds = KiQueueRemainingMilliseconds(deadlineTick);
waitMilliseconds = KiQueueRemainingMilliseconds(deadlineMicroseconds);
if (waitMilliseconds == 0)
return NTLSTATUS_TIMEOUT;
}
@@ -98,11 +74,11 @@ static KSTATUS KiQueueWaitForEvent(QUEUE* queue, EVENT* event, uint64_t timeoutM
}
// Creates a fixed-depth queue with waitable occupancy and capacity events.
KSTATUS KeQueueCreate(QUEUE **out) {
KSTATUS KeQueueCreate(QUEUE** out) {
if (!out) return NTLSTATUS_INVALID_PARAMETER;
*out = (QUEUE*)0;
QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
QUEUE* q = (QUEUE*)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
if (!q) return NTLSTATUS_NO_MEMORY;
q->Header.Destroy = KiQueueDestroy;
q->Lock = (SPINLOCK)SPINLOCK_INIT;
@@ -129,9 +105,9 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMi
if (!queue || !msg) return NTLSTATUS_INVALID_PARAMETER;
if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL;
uint64_t deadlineTick = 0;
uint64_t deadlineMicroseconds = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
deadlineMicroseconds = KiQueueTimeoutDeadline(timeoutMilliseconds);
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
@@ -150,7 +126,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMi
if (timeoutMilliseconds == 0)
return NTLSTATUS_WOULD_BLOCK;
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotFullEvent, timeoutMilliseconds, deadlineTick);
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotFullEvent, timeoutMilliseconds, deadlineMicroseconds);
if (s == NTLSTATUS_TIMEOUT || KFAILED(s))
return s;
}
@@ -161,9 +137,9 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilli
if (!queue || !msg) return NTLSTATUS_INVALID_PARAMETER;
if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL;
uint64_t deadlineTick = 0;
uint64_t deadlineMicroseconds = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
deadlineMicroseconds = KiQueueTimeoutDeadline(timeoutMilliseconds);
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
@@ -182,7 +158,7 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilli
if (timeoutMilliseconds == 0)
return NTLSTATUS_WOULD_BLOCK;
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotEmptyEvent, timeoutMilliseconds, deadlineTick);
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotEmptyEvent, timeoutMilliseconds, deadlineMicroseconds);
if (s == NTLSTATUS_TIMEOUT || KFAILED(s))
return s;
}
code/kernel/ke_sched.c
+49 -40
View File
@@ -3,10 +3,13 @@
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_hal.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/se_security.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
@@ -18,7 +21,6 @@
#include <kernel/ke_list.h>
#include <kernel/ke_work.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
static LIST_ENTRY gNormalRunQueue[PriorityCount];
static LIST_ENTRY gRealtimeRunQueue[KE_REALTIME_PRIORITY_COUNT];
@@ -129,7 +131,7 @@ int KiSchedulerRefreshMutexDonation(MUTEX* mutex) {
// Implements the default Thread Affinity Mask operation.
static uint64_t KiDefaultThreadAffinityMask(void) {
// Normal kernel, user, and driver-created threads are BSP-affine by
// Normal kernel, user, and driver-created threads are boot-processor-affine by
// default. Code that is explicitly audited for AP execution can widen its
// affinity before the thread enters the ready queue.
return 1ULL;
@@ -200,7 +202,7 @@ static void KiRequestRescheduleCpuLocked(KCPU* target) {
KCPU* self = KeGetCurrentCpu();
if (self && target->CpuIndex != self->CpuIndex) {
HalApicSendRescheduleIpi(target->LapicId);
KeSendProcessorInterrupt(target->HardwareId, KE_VECTOR_RESCHEDULE_IPI);
}
}
@@ -707,8 +709,8 @@ static void KiIdleLoop(void) {
KCPU* cpu = KeGetCurrentCpu();
cpu->SchedulerTicks++;
// AP idle threads sleep until an interrupt or reschedule IPI wakes the
// CPU. Ready paths set ReschedulePending and target the CPU's LAPIC;
// Secondary-processor idle threads sleep until an interrupt or reschedule IPI wakes
// the CPU. Ready paths set ReschedulePending and target that processor;
// KiExitInterrupt consumes the request after EOI.
if (KiAnyRunnableForCurrentCpu()) {
KeYield();
@@ -719,7 +721,7 @@ static void KiIdleLoop(void) {
// the idle thread through an unusual switch path must not sleep with
// interrupts disabled, otherwise reschedule/TLB shootdown IPIs can
// be delayed forever.
HalWaitForInterrupt();
KeWaitForInterrupt();
}
}
@@ -731,7 +733,7 @@ void KiInitializeSchedulerGlobal(void) {
KeInitializeWorkItem(&gTimeoutWorkItem, KiTimeoutWorkRoutine, (void*)0);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
KSTATUS s = KeCreateProcess("idle", SeGetRootSecurityContext(), &gIdleProcess);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create idle process, status=%u\n", s);
gIdleProcess->Flags |= KE_PROCESS_FLAG_IDLE | KE_PROCESS_FLAG_TERMINATION_PROTECTED;
@@ -750,12 +752,16 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
if (cpu->IdleThread)
return NTLSTATUS_SUCCESS;
THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
THREAD* idle = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!idle)
return NTLSTATUS_NO_MEMORY;
idle->Header.Destroy = (void *)0;
idle->ThreadId = cpu->CpuIndex ? (0x80000000u | cpu->CpuIndex) : 0;
idle->Header.Destroy = (void*)0;
KSTATUS idStatus = KeAllocateThreadId(&idle->ThreadId);
if (!KSUCCESS(idStatus)) {
ObDereferenceObject(&idle->Header);
return idStatus;
}
idle->State = ThreadRunning;
idle->BaseSchedulingClass = KeSchedulingNormal;
idle->BasePriority = PriorityLow;
@@ -776,7 +782,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
idle->LastCpu = cpu->CpuIndex;
idle->AffinityMask = (cpu->CpuIndex < 64) ? (1ULL << cpu->CpuIndex) : 0ULL;
idle->Owner = gIdleProcess;
idle->WaitObject = (OBJECT *)0;
idle->WaitObject = (OBJECT*)0;
idle->WaitStatus = NTLSTATUS_SUCCESS;
idle->WaitListHead = (LIST_ENTRY*)0;
idle->AlertableWait = 0;
@@ -822,13 +828,13 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
KeInitThreadFpuContext(idle);
// The hand-built CPU idle thread is not created through KeCreateThread(),
// so register it manually with the idle process. This keeps PID 0 visible
// as the real idle process while kernel worker threads live under PID 1.
// so register it manually with the dedicated idle process while kernel worker
// threads remain grouped under the system process.
ObReferenceObject(&gIdleProcess->Header);
uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock);
uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->ThreadLock);
KeListInsertTailChecked(&gIdleProcess->ThreadList, &idle->ProcessLink);
gIdleProcess->ThreadCount++;
KeReleaseSpinlock(&gIdleProcess->HandleLock, idleFlags);
KeReleaseSpinlock(&gIdleProcess->ThreadLock, idleFlags);
cpu->IdleThread = idle;
cpu->CurrentThread = idle;
@@ -1321,11 +1327,11 @@ static void KiTimeoutWorkRoutine(void* context) {
// Checks whether this CPU owns global scheduler tick duties.
static int KiCpuOwnsGlobalSchedulerTick(const KCPU* cpu) {
// LAPIC timer interrupts are per-CPU, but sleep deadlines, waitable timers,
// Periodic timer interrupts are per-CPU, but sleep deadlines, waitable timers,
// input repeat, and priority aging are global scheduler services. Keep the
// global clock single-sourced on the BSP so enabling AP timers does not
// global clock single-sourced on the boot processor so enabling AP timers does not
// make those services run once per online CPU.
return cpu && cpu->IsBsp;
return cpu && cpu->IsBootProcessor;
}
// Ages ready normal-class threads after enough global ticks have elapsed.
@@ -1451,6 +1457,29 @@ void KeSchedulerTick(void) {
KiRequestReschedule();
}
// Converts one requested delay to scheduler ticks while reserving one phase guard tick.
static uint64_t KiTimeoutTicksForMilliseconds(uint64_t milliseconds) {
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
// The scheduler tick count identifies completed quanta, not the current
// position inside one quantum. One additional tick guarantees that a
// timeout armed immediately before an interrupt cannot expire early.
if (ticks < UINT64_MAX)
ticks++;
return ticks ? ticks : 1;
}
// Returns the saturated scheduler tick at which one finite timeout may expire.
static uint64_t KiTimeoutWakeTickLocked(uint64_t milliseconds) {
uint64_t ticks = KiTimeoutTicksForMilliseconds(milliseconds);
uint64_t now = gSchedStats.Ticks;
if (ticks == UINT64_MAX || ticks > UINT64_MAX - now)
return UINT64_MAX;
return now + ticks;
}
// Arms a timeout for the current thread after a wait primitive has prepared it to block.
KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) {
if (milliseconds == 0)
@@ -1462,12 +1491,6 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus)
if (!self || self->IsIdleThread)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
if (ticks == 0)
ticks = 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (self->State != ThreadWaiting ||
self->RunningCpu != KeGetCurrentCpu()->CpuIndex ||
@@ -1478,12 +1501,7 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus)
return NTLSTATUS_INVALID_PARAMETER;
}
uint64_t now = gSchedStats.Ticks;
uint64_t wakeTick = now + ticks;
if (wakeTick < now)
wakeTick = UINT64_MAX;
self->TimeoutTick = wakeTick;
self->TimeoutTick = KiTimeoutWakeTickLocked(milliseconds);
self->TimeoutStatus = timeoutStatus;
self->TimeoutArmed = 1;
KeListInsertTailChecked(&gTimeoutList, &self->TimeoutLink);
@@ -1505,12 +1523,6 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
if (!self || self->IsIdleThread)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
if (ticks == 0)
ticks = 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (self->Queued || self->State != ThreadRunning ||
@@ -1522,10 +1534,7 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
return NTLSTATUS_INVALID_PARAMETER;
}
uint64_t now = gSchedStats.Ticks;
uint64_t wakeTick = now + ticks;
if (wakeTick < now)
wakeTick = UINT64_MAX;
uint64_t wakeTick = KiTimeoutWakeTickLocked(milliseconds);
self->State = ThreadWaiting;
self->WaitObject = (OBJECT*)0;
code/kernel/ke_selftest.c
+76 -16
View File
@@ -4,7 +4,7 @@
#include <stddef.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
@@ -18,6 +18,7 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <kernel/io_dma.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_trap.h>
@@ -1152,10 +1153,15 @@ static void KiRunStringViewSelfTest(void) {
if (SysSvCopyZ(copy, 4, emb))
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: undersized copy unexpectedly succeeded\n");
// Paths are string views at the ABI boundary, but FAT/VFS path names do not
// Paths are string views at the ABI boundary, but provider path names do not
// accept embedded NUL bytes. This must fail before reaching any filesystem.
IO_VOLUME* volume = (IO_VOLUME*)0;
KSTATUS s = KeReferenceSystemRootVolume(&volume);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: system volume reference status=%u\n", s);
FILE_OBJECT* f = (FILE_OBJECT*)0;
KSTATUS s = IoOpenFile(SysSvFromParts("system/init.exe\0junk", 20), &f);
s = IoOpenFileAtVolume(volume, SysSvFromParts("system/init.exe\0junk", 20), &f);
IoReleaseVolume(volume);
if (s != NTLSTATUS_INVALID_PARAMETER)
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: embedded-NUL path status=%u\n", s);
@@ -1164,6 +1170,59 @@ static void KiRunStringViewSelfTest(void) {
DbgPrint("STRINGVIEW selftest: PASS\n");
}
// Capability-relative native-path self-test.
static void KiRunVolumeRelativePathSelfTest(void) {
DbgPrint("VOLUME/PATH selftest: begin\n");
IO_VOLUME* volume = (IO_VOLUME*)0;
KSTATUS status = KeReferenceSystemRootVolume(&volume);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: system volume reference failed %u\n", status);
FILE_OBJECT* file = (FILE_OBJECT*)0;
status = IoOpenFileAtVolume(volume, SYS_SV_LITERAL("system/init.exe"), &file);
if (!KSUCCESS(status) || !file)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: direct open failed %u\n", status);
IoCloseFile(file);
const SYS_STRING_VIEW invalidPaths[] = {
SYS_SV_LITERAL("/system/init.exe"),
SYS_SV_LITERAL("system\\init.exe"),
SYS_SV_LITERAL("system//init.exe"),
SYS_SV_LITERAL("system/./init.exe"),
SYS_SV_LITERAL("system/../init.exe"),
SYS_SV_LITERAL("system/init.exe/"),
};
for (size_t i = 0; i < sizeof(invalidPaths) / sizeof(invalidPaths[0]); ++i) {
file = (FILE_OBJECT*)0;
status = IoOpenFileAtVolume(volume, invalidPaths[i], &file);
if (status != NTLSTATUS_INVALID_PARAMETER || file)
KePanic(KE_PANIC_CODE_SELFTEST,
"VOLUME/PATH selftest: invalid path %llu status=%u file=0x%016llX\n",
(uint64_t)i, status, (uint64_t)(uintptr_t)file);
}
DIR_OBJECT* directory = (DIR_OBJECT*)0;
status = IoOpenDirAtVolume(volume, SYS_SV_LITERAL("system"), &directory);
if (!KSUCCESS(status) || !directory)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: directory open failed %u\n", status);
file = (FILE_OBJECT*)0;
status = IoOpenFileAtDirectory(directory, SYS_SV_LITERAL("init.exe"), &file);
if (!KSUCCESS(status) || !file)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: directory-relative open failed %u\n", status);
IoCloseFile(file);
file = (FILE_OBJECT*)0;
status = IoOpenFileAtDirectory(directory, SYS_SV_LITERAL("../init.exe"), &file);
if (status != NTLSTATUS_INVALID_PARAMETER || file)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: parent traversal status=%u\n", status);
IoCloseDir(directory);
IoReleaseVolume(volume);
DbgPrint("VOLUME/PATH selftest: PASS\n");
}
// Async file I/O self-test.
typedef struct KI_FILE_ASYNC_SELFTEST_STATE {
@@ -1221,7 +1280,7 @@ static void KiRunAsyncFileIoSelfTest(void) {
memset(&st, 0, sizeof(st));
st.Iosb.Status = NTLSTATUS_FAILURE;
KSTATUS s = IoOpenFile(SYS_SV_LITERAL("system/init.exe"), &st.File);
KSTATUS s = KeOpenSystemRootFile("system/init.exe", &st.File);
if (!KSUCCESS(s)) {
DbgPrint("FILE/ASYNC selftest: SKIP, system/init.exe open failed %u\n", s);
return;
@@ -1303,8 +1362,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
wiosb.Status = NTLSTATUS_FAILURE;
wiosb.Information = 0;
(void)IoDeleteFile(SYS_SV_LITERAL("asyncio.tmp"));
s = IoCreateFile(SYS_SV_LITERAL("asyncio.tmp"), &wf);
(void)KeDeleteSystemRootFile("asyncio.tmp");
s = KeCreateSystemRootFile("asyncio.tmp", &wf);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: create write temp failed %u\n", s);
@@ -1343,7 +1402,7 @@ static void KiRunAsyncFileIoSelfTest(void) {
IoCloseFile(wf);
wf = (FILE_OBJECT*)0;
s = IoOpenFile(SYS_SV_LITERAL("asyncio.tmp"), &rf);
s = KeOpenSystemRootFile("asyncio.tmp", &rf);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: reopen write temp failed %u\n", s);
@@ -1357,7 +1416,7 @@ static void KiRunAsyncFileIoSelfTest(void) {
IoCloseFile(rf);
ObDereferenceObject(&wev->Header);
s = IoDeleteFile(SYS_SV_LITERAL("asyncio.tmp"));
s = KeDeleteSystemRootFile("asyncio.tmp");
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: delete temp failed %u\n", s);
@@ -1774,16 +1833,16 @@ void KiRunCoreHardeningSelfTests(void) {
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%016llX cpu=0x%016llX\n",
(uint64_t)cpu->Self, (uint64_t)cpu);
if (KeGetCurrentCpuIndex() != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n");
if (cpu->State != KeCpuOnline || !cpu->IsBsp)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle invalid state=%ld bsp=%u\n",
cpu->State, cpu->IsBsp);
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not boot processor/0\n");
if (cpu->State != KeCpuOnline || !cpu->IsBootProcessor)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: boot-processor lifecycle invalid state=%ld boot=%u\n",
cpu->State, cpu->IsBootProcessor);
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql());
if (KeGetCurrentTss() != &cpu->Tss)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n");
if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64))
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%016llX iomap=%u\n",
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid boot-processor TSS rsp0=0x%016llX iomap=%u\n",
cpu->Tss.RSP0, cpu->Tss.IoMapBase);
if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%016llX gdt=0x%016llX\n",
@@ -1793,10 +1852,10 @@ void KiRunCoreHardeningSelfTests(void) {
KCPU* ap = &gCpus[i];
if (ap->State == KeCpuAbsent)
continue;
if (ap->State != KeCpuOnline || ap->IsBsp)
if (ap->State != KeCpuOnline || ap->IsBootProcessor)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u lifecycle invalid state=%ld bsp=%u\n",
i, ap->State, ap->IsBsp);
"CORE selftest: AP%u lifecycle invalid state=%ld boot=%u\n",
i, ap->State, ap->IsBootProcessor);
if (!ap->KernelStack || !ap->KernelStackTop)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
@@ -1847,6 +1906,7 @@ void KiRunKernelSelfTests(const HAL_API* hal) {
KiRunVadCopyOnWriteSelfTest();
KiRunVadSwapSelfTest();
KiRunStringViewSelfTest();
KiRunVolumeRelativePathSelfTest();
KiRunIrpKernelApcSelfTest(hal);
KiRunAsyncFileIoSelfTest();
KiRunTrapFrameSelfTest();
code/kernel/ke_semaphore.c
+2 -2
View File
@@ -12,10 +12,10 @@
static void KiSemaphoreDestroy(OBJECT* obj) { (void)obj; }
// Implements the semaphore Create operation.
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out) {
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE** out) {
if (!out || initial < 0 || max < 1 || initial > max)
return NTLSTATUS_INVALID_PARAMETER;
SEMAPHORE* s = (SEMAPHORE *)ObAllocateObject(ObjectTypeSemaphore, sizeof(SEMAPHORE));
SEMAPHORE* s = (SEMAPHORE*)ObAllocateObject(ObjectTypeSemaphore, sizeof(SEMAPHORE));
if (!s) return NTLSTATUS_NO_MEMORY;
s->Header.Destroy = KiSemaphoreDestroy;
s->Lock = (SPINLOCK)SPINLOCK_INIT;
code/kernel/ke_settings.c
+100 -50
View File
@@ -1,7 +1,4 @@
// Kernel settings reader. This file loads simple runtime settings from the system configuration file.
// This intentionally exposes generic string settings only. Subsystems and
// drivers own the policy for interpreting their sections.
// Kernel settings reader. This file performs a direct lookup in the simple system settings file.
#include <stddef.h>
#include <stdint.h>
@@ -9,41 +6,102 @@
#include <kernel/ke_settings.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <kernel/mm_virt.h>
#include <common/ini_parser.h>
#define KE_SETTINGS_MAX_FILE_BYTES (64u * 1024u)
typedef struct KE_FIND_SETTING_CTX {
const char* Section;
const char* Key;
char* Buffer;
size_t BufferSize;
KSTATUS Status;
} KE_FIND_SETTING_CTX;
static void KeFindSettingCallback(const char* section, const char* key,
const char* value, void* userdata) {
KE_FIND_SETTING_CTX* ctx = (KE_FIND_SETTING_CTX*)userdata;
if (!ctx || ctx->Status != NTLSTATUS_NOT_FOUND)
return;
if (!section || !key || !value)
return;
if (strcmp(section, ctx->Section) != 0 || strcmp(key, ctx->Key) != 0)
return;
size_t len = strlen(value);
if (len + 1 > ctx->BufferSize) {
if (ctx->BufferSize)
ctx->Buffer[0] = '\0';
ctx->Status = NTLSTATUS_BUFFER_TOO_SMALL;
return;
}
memcpy(ctx->Buffer, value, len + 1);
ctx->Status = NTLSTATUS_SUCCESS;
// Returns whether one settings.ini character is horizontal whitespace.
static int KeSettingSpace(char c) {
return c == ' ' || c == '\t' || c == '\r';
}
// Trims horizontal whitespace from one immutable settings.ini span.
static void KeSettingTrim(const char** begin, const char** end) {
while (*begin < *end && KeSettingSpace(**begin))
++*begin;
while (*end > *begin && KeSettingSpace((*end)[-1]))
--*end;
}
// Returns whether one immutable span exactly matches a NUL-terminated string.
static int KeSettingSpanEquals(const char* begin, const char* end, const char* text) {
while (begin < end && *text) {
if (*begin++ != *text++)
return 0;
}
return begin == end && *text == '\0';
}
// Looks up one [section] key=value entry in a read-only settings buffer.
static KSTATUS KeFindSetting(const char* text, size_t length,
const char* wantedSection, const char* wantedKey,
char* buffer, size_t bufferSize) {
char section[64] = {0};
size_t offset = 0;
while (offset < length) {
size_t lineStart = offset;
while (offset < length && text[offset] != '\n')
++offset;
size_t lineEnd = offset;
if (offset < length)
++offset;
const char* begin = text + lineStart;
const char* end = text + lineEnd;
KeSettingTrim(&begin, &end);
if (begin == end || *begin == ';' || *begin == '#')
continue;
if (*begin == '[') {
if (end[-1] != ']')
return NTLSTATUS_INVALID_PARAMETER;
const char* sectionBegin = begin + 1;
const char* sectionEnd = end - 1;
KeSettingTrim(&sectionBegin, &sectionEnd);
size_t sectionLength = (size_t)(sectionEnd - sectionBegin);
if (sectionLength >= sizeof(section))
return NTLSTATUS_INVALID_PARAMETER;
memcpy(section, sectionBegin, sectionLength);
section[sectionLength] = '\0';
continue;
}
const char* equals = NULL;
for (const char* p = begin; p < end; ++p) {
if (*p == '=') {
equals = p;
break;
}
}
if (!equals)
continue;
const char* keyBegin = begin;
const char* keyEnd = equals;
const char* valueBegin = equals + 1;
const char* valueEnd = end;
KeSettingTrim(&keyBegin, &keyEnd);
KeSettingTrim(&valueBegin, &valueEnd);
if (strcmp(section, wantedSection) != 0 ||
!KeSettingSpanEquals(keyBegin, keyEnd, wantedKey)) {
continue;
}
size_t valueLength = (size_t)(valueEnd - valueBegin);
if (valueLength + 1u > bufferSize)
return NTLSTATUS_BUFFER_TOO_SMALL;
memcpy(buffer, valueBegin, valueLength);
buffer[valueLength] = '\0';
return NTLSTATUS_SUCCESS;
}
return NTLSTATUS_NOT_FOUND;
}
// Reads one string setting from the system settings file.
__declspec(dllexport) KSTATUS KeGetSettingString(const char* section,
const char* key,
char* buffer,
@@ -54,9 +112,9 @@ __declspec(dllexport) KSTATUS KeGetSettingString(const char* section,
buffer[0] = '\0';
FILE_OBJECT* file = NULL;
KSTATUS st = IoOpenFile(SYS_SV_LITERAL(KE_SETTINGS_PATH), &file);
if (!KSUCCESS(st))
return st;
KSTATUS status = KeOpenSystemRootFile(KE_SETTINGS_PATH, &file);
if (!KSUCCESS(status))
return status;
uint64_t fileSize64 = IoGetFileSize(file);
if (fileSize64 >= KE_SETTINGS_MAX_FILE_BYTES) {
@@ -65,30 +123,22 @@ __declspec(dllexport) KSTATUS KeGetSettingString(const char* section,
}
size_t fileSize = (size_t)fileSize64;
char* text = (char*)MmAllocatePool(fileSize + 1);
char* text = (char*)MmAllocatePool(fileSize + 1u);
if (!text) {
IoCloseFile(file);
return NTLSTATUS_NO_MEMORY;
}
size_t got = 0;
st = IoReadFile(file, text, fileSize, &got);
status = IoReadFile(file, text, fileSize, &got);
IoCloseFile(file);
if (!KSUCCESS(st)) {
if (!KSUCCESS(status) || got != fileSize) {
MmFreePool(text);
return st;
return KSUCCESS(status) ? NTLSTATUS_FAILURE : status;
}
text[got] = '\0';
KE_FIND_SETTING_CTX ctx;
ctx.Section = section;
ctx.Key = key;
ctx.Buffer = buffer;
ctx.BufferSize = bufferSize;
ctx.Status = NTLSTATUS_NOT_FOUND;
IniParse(text, KeFindSettingCallback, &ctx);
status = KeFindSetting(text, got, section, key, buffer, bufferSize);
MmFreePool(text);
return ctx.Status;
return status;
}
code/kernel/ke_smp.c
+64 -65
View File
@@ -1,15 +1,16 @@
// SMP bring-up and CPU topology management. This file owns AP startup, CPU admission, and topology-derived scheduling metadata.
// SMP bring-up and CPU topology management. This file owns secondary-processor admission and topology-derived scheduling metadata.
// The bootloader is intentionally not involved in AP ownership. The kernel
// asks the HAL for MADT-discovered LAPIC IDs, copies a small low-memory x86 AP
// startup blob, sends INIT/SIPI through the local APIC, and waits for each AP
// to enter KiApKernelEntry() and park in kernel-owned state.
// The bootloader is intentionally not involved in secondary-processor ownership.
// The kernel asks the HAL for platform processor identities, prepares the x86
// bootstrap state, asks the HAL to start each processor, and waits for it to
// enter KiApKernelEntry() and park in kernel-owned state.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/status.h>
#include <kernel/ke_hal.h>
#include <kernel/ke_smp.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
@@ -21,7 +22,6 @@
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
#define AP_TRAMPOLINE_MAX_PHYS 0x000A0000ULL
#define AP_TRAMPOLINE_PREFERRED_MIN 0x00070000ULL
@@ -276,13 +276,13 @@ static KSTATUS KiBuildApTrampoline(void) {
}
_Noreturn void KiApKernelEntry(KCPU* cpu) {
(void)HalDisableInterrupts();
(void)KeDisableInterrupts();
if (!cpu)
HalStopProcessor();
KeStopProcessor();
// APs start from the SIPI reset state, so do not assume the firmware/BSP
// CR4 feature bits are present. The scheduler and context-switch paths
// Secondary processors enter through the x86 bootstrap path, so do not assume
// firmware CR4 feature bits are present. The scheduler and context-switch paths
// use FXSAVE/FXRSTOR on every CPU; without OSFXSR, those instructions
// raise #UD as soon as an AP enters its idle thread.
uintptr_t cr4 = __readcr4();
@@ -302,7 +302,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
KeSetUserGsBase(0);
if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_GS;
HalApicEnableCurrentCpu();
KeInitializeCurrentProcessorInterruptController();
KeInitializeGdtForCpu(cpu);
KeLoadGdtForCurrentCpu(cpu);
@@ -332,41 +332,41 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
idle->State = ThreadRunning;
idle->RunningCpu = cpu->CpuIndex;
idle->Queued = 0;
// Do not arm the AP's periodic LAPIC timer while it is still in
// the raw park loop. Timer IRQs are scheduler-owned events;
// the AP may take them only after it has a current idle thread.
HalTimerConfigureCurrentCpu();
// Do not arm this processor's periodic timer while it is still in the raw
// park loop. Timer IRQs are scheduler-owned events and require
// a current idle thread before they may be delivered.
KeConfigureCurrentProcessorTimer();
_InterlockedExchange(&cpu->State, KeCpuSchedulerActive);
HalEnableInterrupts();
KeEnableInterrupts();
KeStartInitialThread(idle);
}
}
// The BSP admits parked APs by publishing AdmissionRequested and
// The boot processor admits parked secondary processors by publishing AdmissionRequested and
// sending a wake IPI. Avoid the classic lost wakeup where the state
// changes between the poll and HLT: check once with interrupts
// disabled, then use STI;HLT as a single sleep transition.
uintptr_t interruptFlags = HalDisableInterrupts();
uintptr_t interruptFlags = KeDisableInterrupts();
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
HalRestoreInterrupts(interruptFlags);
KeRestoreInterrupts(interruptFlags);
continue;
}
HalWaitForInterrupt();
KeWaitForInterrupt();
}
}
// Prepares cpu Record.
static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_PROCESSOR_INFO* info) {
KCPU* cpu = &gCpus[cpuIndex];
memset(cpu, 0, sizeof(*cpu));
cpu->Self = cpu;
cpu->CpuIndex = cpuIndex;
cpu->LapicId = info->LapicId;
cpu->AcpiProcessorId = info->ProcessorUid;
cpu->HardwareId = info->HardwareId;
cpu->FirmwareId = info->FirmwareId;
cpu->BootProcessorIndex = cpuIndex;
cpu->State = info->Enabled ? KeCpuPresent : KeCpuAbsent;
cpu->IsBsp = info->IsBsp ? 1 : 0;
cpu->IsBootProcessor = info->IsBootProcessor ? 1 : 0;
cpu->PackageId = 0;
cpu->CoreId = cpuIndex;
cpu->ThreadId = 0;
@@ -517,11 +517,11 @@ void KeBuildProcessorTopology(void) {
if (cpu->State == KeCpuAbsent)
continue;
uint32_t apic = cpu->LapicId;
cpu->ThreadId = smtMask ? (apic & smtMask) : 0u;
cpu->CoreId = coreBits ? ((apic >> topo.SmtShift) & coreMask) :
(apic >> topo.SmtShift);
cpu->PackageId = (topo.CoreShift < 32u) ? (apic >> topo.CoreShift) : 0u;
uint32_t hardwareId = cpu->HardwareId;
cpu->ThreadId = smtMask ? (hardwareId & smtMask) : 0u;
cpu->CoreId = coreBits ? ((hardwareId >> topo.SmtShift) & coreMask) :
(hardwareId >> topo.SmtShift);
cpu->PackageId = (topo.CoreShift < 32u) ? (hardwareId >> topo.CoreShift) : 0u;
cpu->PhysicalCoreIndex = KE_INVALID_CPU;
cpu->CoreSiblingMask = 0;
cpu->PackageMask = 0;
@@ -591,9 +591,9 @@ void KeBuildProcessorTopology(void) {
KCPU* cpu = &gCpus[i];
if (cpu->State == KeCpuAbsent)
continue;
DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
DbgPrint("SMP: cpu%u hardware=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
i,
cpu->LapicId,
cpu->HardwareId,
cpu->PackageId,
cpu->CoreId,
cpu->ThreadId,
@@ -656,40 +656,39 @@ static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) {
// Starts application Processors.
KSTATUS KeStartApplicationProcessors(void) {
HAL_CPU_TOPOLOGY topo;
KSTATUS st = HalGetProcessorTopology(&topo);
HAL_PROCESSOR_TOPOLOGY topo;
KSTATUS st = KeQueryProcessorTopology(&topo);
if (!KSUCCESS(st)) return st;
if (topo.CpuCount == 0 || topo.CpuCount > KE_MAX_CPUS)
if (topo.Count == 0 || topo.Count > KE_MAX_CPUS)
return NTLSTATUS_DEVICE_ERROR;
uint32_t bspLapic = HalGetBspLapicId();
uint32_t bootHardwareId = topo.BootProcessorHardwareId;
KCPU* bsp = KeGetCurrentCpu();
bsp->LapicId = bspLapic;
bsp->IsBsp = 1;
bsp->HardwareId = bootHardwareId;
bsp->IsBootProcessor = 1;
bsp->State = KeCpuStarted;
DbgPrint("SMP: APIC mode=%s BSP LAPIC=%u MADT CPUs=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
bspLapic, topo.CpuCount);
DbgPrint("SMP: platform processors boot-id=%u firmware-count=%u\n",
bootHardwareId, topo.Count);
uint32_t count = 1;
for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) {
HAL_CPU_INFO* info = &topo.Cpus[i];
for (uint32_t i = 0; i < topo.Count && count < KE_MAX_CPUS; i++) {
HAL_PROCESSOR_INFO* info = &topo.Processors[i];
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
i, info->ProcessorUid, info->LapicId, info->Flags,
info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
DbgPrint("SMP: topology[%u] firmware=%u hardware=%u flags=0x%x enabled=%u boot=%u startable=%u\n",
i, info->FirmwareId, info->HardwareId, info->Flags,
info->Enabled, info->IsBootProcessor, info->Startable);
}
if (!info->Enabled) continue;
if (info->LapicId == bspLapic) {
bsp->AcpiProcessorId = info->ProcessorUid;
if (info->HardwareId == bootHardwareId) {
bsp->FirmwareId = info->FirmwareId;
continue;
}
if (!HalApicCanAddressId(info->LapicId)) {
DbgPrint("SMP: skipping AP uid=%u lapic=%u; current APIC mode cannot address it\n",
info->ProcessorUid, info->LapicId);
if (!info->Startable) {
DbgPrint("SMP: skipping processor firmware=%u hardware=%u; HAL cannot start it\n",
info->FirmwareId, info->HardwareId);
continue;
}
KiPrepareCpuRecord(count, info);
@@ -697,7 +696,7 @@ KSTATUS KeStartApplicationProcessors(void) {
}
gCpuCount = count;
DbgPrint("SMP: discovered %u addressable CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic);
DbgPrint("SMP: discovered %u usable processor(s), boot hardware ID=%u\n", gCpuCount, bootHardwareId);
if (gCpuCount <= 1) {
DbgPrint("SMP: no application processors present\n");
return NTLSTATUS_SUCCESS;
@@ -721,7 +720,6 @@ KSTATUS KeStartApplicationProcessors(void) {
return NTLSTATUS_SUCCESS;
}
uint8_t sipiVector = (uint8_t)(gApTrampolinePhys >> 12);
uint32_t claimed = 0;
for (uint32_t i = 1; i < gCpuCount; i++) {
@@ -730,20 +728,21 @@ KSTATUS KeStartApplicationProcessors(void) {
st = KiAllocateApStack(cpu);
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
DbgPrint("SMP: AP%u hardware=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
i, cpu->HardwareId, st);
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
KiFillBootBlockForCpu(cpu, i);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
DbgPrint("SMP: starting AP%u hardware=%u entry=0x%016llX\n",
i, cpu->HardwareId, (uint64_t)gApTrampolinePhys);
st = HalStartApplicationProcessor(cpu->LapicId, sipiVector);
st = KeStartProcessor(cpu->HardwareId, gApTrampolinePhys);
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
DbgPrint("SMP: AP%u hardware=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
i, cpu->HardwareId, st);
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
@@ -763,8 +762,8 @@ KSTATUS KeStartApplicationProcessors(void) {
}
if (!parked) {
DbgPrint("SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
DbgPrint("SMP: AP%u failed to reach kernel park state hardware=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
i, cpu->HardwareId, gApBootBlock->State, gApBootBlock->Stage,
KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
@@ -781,7 +780,7 @@ KSTATUS KeStartApplicationProcessors(void) {
}
if (claimed == 0 && gCpuCount > 1)
DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n");
DbgPrint("SMP: no APs reached kernel-owned state; continuing boot-processor-only\n");
KeBuildProcessorTopology();
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
@@ -816,12 +815,12 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
_InterlockedExchange(&cpu->State, KeCpuAdmissionRequested);
KiSmpPoolCheckpoint("smp before AP wake", i);
// Parked APs sleep with HLT and have no LAPIC timer armed yet. The
// Parked secondary processors sleep with no periodic timer armed yet. The
// state transition alone is not sufficient; send a fixed-vector IPI
// only to wake the parked loop. ReschedulePending is intentionally left
// clear, so the interrupt exit path will not try to schedule before
// the AP has installed its idle thread.
HalApicSendRescheduleIpi(cpu->LapicId);
KeSendProcessorInterrupt(cpu->HardwareId, KE_VECTOR_RESCHEDULE_IPI);
int active = 0;
for (uint32_t spin = 0; spin < AP_SCHED_TIMEOUT_SPINS; spin++) {
@@ -834,8 +833,8 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
if (!active) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u failed scheduler admission lapic=%u state=%ld ticks=%llu\n",
i, cpu->LapicId, cpu->State, cpu->SchedulerTicks);
"SMP: AP%u failed scheduler admission hardware=%u state=%ld ticks=%llu\n",
i, cpu->HardwareId, cpu->State, cpu->SchedulerTicks);
}
_InterlockedExchange(&cpu->State, KeCpuOnline);
code/kernel/ke_switch.asm
+1 -1
View File
@@ -38,7 +38,7 @@ KeSwitchContext PROC
mov [rcx + THREAD_SavedContext], rsp
; Update this CPU's TSS.RSP0 before switching to the incoming stack.
; The helper hides whether current-CPU storage is still CPU0-only, LAPIC-ID
; The helper hides whether current-CPU storage is still CPU0-only, hardware-ID
; indexed, or eventually reached through a SWAPGS-backed per-CPU base.
mov rbx, rdx
mov rcx, [rbx + THREAD_TssRsp0]
code/kernel/ke_syscall.asm
+61 -27
View File
@@ -112,22 +112,18 @@ KeSyscallEntry PROC
call KiEnterTrapFrame
add rsp, 32
; Re-order original user arguments into KeSyscallDispatch(nr,a0..a5).
; Re-order original user arguments into KeSyscallDispatch(nr,a0..a3).
; Original registers were saved in the trap frame before any C calls.
sub rsp, 64
sub rsp, 48
mov rax, [rbx + TF_R9]
mov [rsp + 32], rax ; a3
mov rax, [rbx + TF_R10]
mov [rsp + 40], rax ; a4
mov rax, [rbx + TF_R11]
mov [rsp + 48], rax ; a5
mov r9, [rbx + TF_R8] ; a2
mov r8, [rbx + TF_Rdx] ; a1
mov rdx, [rbx + TF_Rcx] ; a0
mov rcx, [rbx + TF_Rax] ; nr
call KeSyscallDispatch
add rsp, 64
add rsp, 48
mov [rbx + TF_Rax], rax
@@ -199,7 +195,22 @@ KF_R13 EQU 0A8h
KF_R14 EQU 0B0h
KF_R15 EQU 0B8h
FAULT_CALL_SHADOW EQU 28h
; Asynchronous kernel entry must preserve the full interrupted x87/SSE state.
; MS x64 calls may freely clobber XMM0-XMM5, and an interrupt or fault can
; arrive between any two user or kernel instructions rather than at a call ABI.
; Reserve one 512-byte FXSAVE image plus alignment slack on the interrupted stack.
ASYNC_FPU_SCRATCH EQU 528
ASYNC_FPU_SAVE MACRO
sub rsp, ASYNC_FPU_SCRATCH
lea r12, [rsp + 15]
and r12, -16
fxsave64 [r12]
ENDM
ASYNC_FPU_RESTORE MACRO
fxrstor64 [r12]
ENDM
FAULT_RESUME_NOERR MACRO
LOCAL noerr_skip_user_slots, noerr_no_swapgs, noerr_unhandled_halt
@@ -372,11 +383,13 @@ de_kernel_frame:
mov qword ptr [rax + KF_Ss], 0
de_call:
and rsp, -16
sub rsp, FAULT_CALL_SHADOW
mov rbx, rax
mov rcx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_NOERR
KeFaultDE ENDP
@@ -440,11 +453,13 @@ gp_kernel_frame:
mov qword ptr [rax + KF_Ss], 0
gp_call:
and rsp, -16
sub rsp, FAULT_CALL_SHADOW
mov rbx, rax
mov rcx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_ERR
KeFaultGP ENDP
@@ -509,11 +524,13 @@ pf_kernel_frame:
mov qword ptr [rax + KF_Ss], 0
pf_call:
and rsp, -16
sub rsp, FAULT_CALL_SHADOW
mov rbx, rax ; Keep the KFAULT_FRAME pointer across C.
mov rcx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_ERR
KeFaultPF ENDP
@@ -579,11 +596,13 @@ noerr_kernel_frame:
mov qword ptr [rax + KF_Ss], 0
noerr_call:
and rsp, -16
sub rsp, FAULT_CALL_SHADOW
mov rbx, rax
mov rcx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_NOERR
StubName ENDP
ENDM
@@ -646,11 +665,13 @@ err_kernel_frame:
mov qword ptr [rax + KF_Ss], 0
err_call:
and rsp, -16
sub rsp, FAULT_CALL_SHADOW
mov rbx, rax
mov rcx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_ERR
StubName ENDP
ENDM
@@ -669,14 +690,22 @@ KeNmiIpiStub PROC
push r9
push r10
push r11
push r12
push r13
mov r13, rsp
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
call KeHandleNmiIpi
add rsp, 32
ASYNC_FPU_RESTORE
mov rsp, r13
test eax, eax
jz short NmiUnhandled
pop r13
pop r12
pop r11
pop r10
pop r9
@@ -687,6 +716,8 @@ KeNmiIpiStub PROC
iretq
NmiUnhandled:
pop r13
pop r12
pop r11
pop r10
pop r9
@@ -857,7 +888,9 @@ IrqKernelModeFrame:
mov qword ptr [rbx + TF_SsSlot], 0
IrqFrameReady:
sub rsp, 40 ; 32-byte shadow space + alignment/local slack
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KiEnterTrapFrame
call KiEnterInterrupt
@@ -866,14 +899,15 @@ IrqFrameReady:
call KiExitInterrupt
mov rcx, rbx
call KiExitTrapFrame
add rsp, 40
ASYNC_FPU_RESTORE
mov eax, [rbx + TF_PreviousMode]
cmp eax, KTRAP_MODE_USER
jne short IrqNoSwapGsOut
swapgs ; returning to ring 3: restore user GS/TEB
IrqNoSwapGsOut:
add rsp, KTRAP_FRAME_SIZE
mov r12, [rbx + TF_R12]
lea rsp, [rbx + KTRAP_FRAME_SIZE]
pop rbx
pop r11
pop r10
code/kernel/ke_syscall.c
+2564 -1917
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_system_root.c
+80 -87
View File
@@ -1,60 +1,21 @@
// System root volume selection. This file keeps boot-volume identity in one place so loaders do not guess.
// System root volume selection. This file retains the structurally selected mounted volume for bootstrap capability-relative I/O.
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <kernel/ke_system_root.h>
#include <kernel/ke_debug.h>
#include <kernel/io_file.h>
#include <kernel/io_volume.h>
#include <kernel/io_partition.h>
#include <kernel/ke_sched.h>
static uint8_t gSystemRootDrive;
static IO_VOLUME* gSystemRootVolume;
static bool gSystemRootReady;
#define KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS 30000ull
#define KE_SYSTEM_ROOT_WAIT_SLICE_MS 1000ull
#define KE_SYSTEM_ROOT_LOG_INTERVAL_MS 5000ull
// Builds a drive-qualified path without depending on any process CWD.
static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) {
if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES)
return false;
size_t relLen = strlen(relative);
if (relLen + 4 > outSize)
return false;
out[0] = (char)('0' + drive);
out[1] = ':';
out[2] = '/';
memcpy(out + 3, relative, relLen + 1);
return true;
}
// Returns true when the candidate volume looks like the volume that booted the kernel.
static bool KiDriveLooksLikeSystemRoot(uint8_t drive) {
static const char* required[] = {
"system/colonel.exe",
"system/hal.dll",
"drivers/drivers.ini",
};
char path[1024];
for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) {
if (!KiBuildDrivePath(drive, required[i], path, sizeof(path)))
return false;
FILE_OBJECT* file = NULL;
KSTATUS s = IoOpenFile(SysSvFromCstr(path), &file);
if (!KSUCCESS(s))
return false;
IoCloseFile(file);
}
return true;
}
// Returns the current scheduler time in milliseconds.
static uint64_t KiSystemRootNowMilliseconds(void) {
KE_SCHED_STATS stats;
@@ -62,54 +23,68 @@ static uint64_t KiSystemRootNowMilliseconds(void) {
return stats.Ticks * SCHED_TICK_MS;
}
// Scans the currently mounted volumes for the system tree.
static KSTATUS KiTrySelectSystemRootVolume(uint32_t* mountedOut) {
uint32_t mounted = 0;
for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) {
if (!IoFilesystemPresent(drive))
continue;
mounted++;
if (!KiDriveLooksLikeSystemRoot(drive))
continue;
gSystemRootDrive = drive;
gSystemRootReady = true;
if (mountedOut)
*mountedOut = mounted;
DbgPrint("System root: drive=%u\n", drive);
return NTLSTATUS_SUCCESS;
}
// Resolves the unique configured partition identity to its mounted filesystem volume.
static KSTATUS KiTrySelectSystemRootVolume(uint64_t* mountedOut) {
IO_VOLUME_STATS stats;
IoQueryVolumeStats(&stats);
if (mountedOut)
*mountedOut = mounted;
return NTLSTATUS_NOT_FOUND;
*mountedOut = stats.MountedVolumes;
IO_PARTITION_INFO systemPartition;
KSTATUS status = IoQuerySystemPartition(&systemPartition);
if (!KSUCCESS(status))
return status;
IO_VOLUME* volume = (IO_VOLUME*)0;
status = IoReferenceVolumeByBacking(systemPartition.DiskIndex,
systemPartition.StartLba,
&volume);
if (!KSUCCESS(status))
return status;
gSystemRootVolume = volume; // Retain the mounted-volume reference for the system lifetime.
gSystemRootReady = true;
DbgPrint("System root: volume=%u disk=%u start_lba=%llu sectors=%llu\n",
IoGetVolumeInstanceId(volume), systemPartition.DiskIndex,
(unsigned long long)systemPartition.StartLba,
(unsigned long long)systemPartition.SectorCount);
return NTLSTATUS_SUCCESS;
}
// Waits for the mounted boot volume instead of racing asynchronous bus discovery.
// Waits for the configured structural system partition to be discovered and mounted.
KSTATUS KeInitializeSystemRootVolume(void) {
if (gSystemRootReady)
return NTLSTATUS_SUCCESS;
uint64_t start = KiSystemRootNowMilliseconds();
uint64_t deadline = start + KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS;
uint64_t nextLog = start;
for (;;) {
uint64_t generation = IoGetVolumeNamespaceGeneration();
uint32_t mounted = 0;
uint64_t generation = IoGetVolumeGeneration();
uint64_t mounted = 0;
KSTATUS status = KiTrySelectSystemRootVolume(&mounted);
if (KSUCCESS(status))
return status;
if (status == NTLSTATUS_OBJECT_NAME_COLLISION) {
DbgPrint("System root: configured system_volume matches multiple mounted volumes\n");
return status;
}
if (status != NTLSTATUS_NOT_FOUND)
return status;
uint64_t now = KiSystemRootNowMilliseconds();
if (now >= deadline) {
DbgPrint("System root: discovery timed out after %llu ms; mounted=%u generation=%llu\n",
DbgPrint("System root: discovery timed out after %llu ms; mounted=%llu generation=%llu\n",
(unsigned long long)KE_SYSTEM_ROOT_DISCOVERY_TIMEOUT_MS,
mounted,
(unsigned long long)mounted,
(unsigned long long)generation);
return NTLSTATUS_TIMEOUT;
}
if (now >= nextLog) {
DbgPrint("System root: waiting for boot volume; mounted=%u generation=%llu remaining=%llu ms\n",
mounted,
DbgPrint("System root: waiting for configured partition; mounted=%llu generation=%llu remaining=%llu ms\n",
(unsigned long long)mounted,
(unsigned long long)generation,
(unsigned long long)(deadline - now));
nextLog = now + KE_SYSTEM_ROOT_LOG_INTERVAL_MS;
@@ -119,7 +94,7 @@ KSTATUS KeInitializeSystemRootVolume(void) {
uint64_t wait = remaining < KE_SYSTEM_ROOT_WAIT_SLICE_MS
? remaining
: KE_SYSTEM_ROOT_WAIT_SLICE_MS;
KSTATUS waitStatus = IoWaitForVolumeNamespaceChange(generation, wait);
KSTATUS waitStatus = IoWaitForVolumeChange(generation, wait);
if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT)
return waitStatus;
}
@@ -130,26 +105,44 @@ bool KeSystemRootReady(void) {
return gSystemRootReady;
}
// Returns the selected system root drive, or zero before discovery.
uint8_t KeGetSystemRootDrive(void) {
return gSystemRootReady ? gSystemRootDrive : 0;
// References the selected system volume for capability transfer or kernel-relative I/O.
KSTATUS KeReferenceSystemRootVolume(IO_VOLUME** volumeOut) {
if (!volumeOut)
return NTLSTATUS_INVALID_PARAMETER;
*volumeOut = (IO_VOLUME*)0;
if (!gSystemRootReady || !gSystemRootVolume)
return NTLSTATUS_NOT_FOUND;
if (!IoVolumeTryReferenceMounted(gSystemRootVolume))
return NTLSTATUS_OBJECT_DEAD;
*volumeOut = gSystemRootVolume;
return NTLSTATUS_SUCCESS;
}
// Builds an absolute path rooted at the selected system volume.
bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize) {
if (!gSystemRootReady)
return false;
return KiBuildDrivePath(gSystemRootDrive, relative, out, outSize);
}
// Opens a file rooted at the selected system volume.
// Opens a canonical native path rooted directly at the selected system volume.
KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out) {
if (!out)
if (!relative || !out)
return NTLSTATUS_INVALID_PARAMETER;
*out = NULL;
char path[1024];
if (!KeBuildSystemRootPath(relative, path, sizeof(path)))
if (!gSystemRootReady || !gSystemRootVolume)
return NTLSTATUS_NOT_FOUND;
return IoOpenFile(SysSvFromCstr(path), out);
return IoOpenFileAtVolume(gSystemRootVolume, SysSvFromCstr(relative), out);
}
// Creates a canonical native path rooted directly at the selected system volume.
KSTATUS KeCreateSystemRootFile(const char* relative, FILE_OBJECT** out) {
if (!relative || !out)
return NTLSTATUS_INVALID_PARAMETER;
*out = NULL;
if (!gSystemRootReady || !gSystemRootVolume)
return NTLSTATUS_NOT_FOUND;
return IoCreateFileAtVolume(gSystemRootVolume, SysSvFromCstr(relative), out);
}
// Deletes a canonical native path rooted directly at the selected system volume.
KSTATUS KeDeleteSystemRootFile(const char* relative) {
if (!relative)
return NTLSTATUS_INVALID_PARAMETER;
if (!gSystemRootReady || !gSystemRootVolume)
return NTLSTATUS_NOT_FOUND;
return IoDeleteFileAtRoot(&gSystemRootVolume->Header, SysSvFromCstr(relative));
}
code/kernel/ke_system_service.c
+92
View File
@@ -0,0 +1,92 @@
// Kernel well-known system-service registry. This file intentionally owns only numeric service discovery.
#include <kernel/ke_system_service.h>
#include <kernel/ke_port.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ob_object.h>
#include <ntlite/security.h>
static SPINLOCK gSystemServiceLock = SPINLOCK_INIT;
static PORT* gSystemServices[SYS_SERVICE_MAX + 1u];
// Returns whether one numeric service identifier is valid for the current ABI.
static bool KiSystemServiceIdValid(SYS_SERVICE_ID serviceId) {
return serviceId > SYS_SERVICE_INVALID && serviceId <= SYS_SERVICE_MAX;
}
// Returns the security claim that authorizes publication of one well-known service.
SYS_SERVICE_CLAIMS KeSystemServiceRequiredClaim(SYS_SERVICE_ID serviceId) {
switch (serviceId) {
case SYS_SERVICE_RESOURCE:
return SYS_SERVICE_CLAIM_RESOURCE;
default:
return 0;
}
}
// Registers one weak listener reference without creating a string or pathname namespace.
KSTATUS KeSystemServiceRegister(SYS_SERVICE_ID serviceId, PORT* listener) {
if (!KiSystemServiceIdValid(serviceId) || !listener || !KePortIsListener(listener))
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gSystemServiceLock);
if (gSystemServices[serviceId]) {
KeReleaseSpinlock(&gSystemServiceLock, flags);
return NTLSTATUS_ALREADY_EXISTS;
}
gSystemServices[serviceId] = listener;
KeReleaseSpinlock(&gSystemServiceLock, flags);
return NTLSTATUS_SUCCESS;
}
// Removes one registration only when the caller still names the registered listener object.
KSTATUS KeSystemServiceUnregister(SYS_SERVICE_ID serviceId, PORT* listener) {
if (!KiSystemServiceIdValid(serviceId) || !listener || !KePortIsListener(listener))
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gSystemServiceLock);
if (!gSystemServices[serviceId]) {
KeReleaseSpinlock(&gSystemServiceLock, flags);
return NTLSTATUS_NOT_FOUND;
}
if (gSystemServices[serviceId] != listener) {
KeReleaseSpinlock(&gSystemServiceLock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
gSystemServices[serviceId] = (PORT*)0;
KeReleaseSpinlock(&gSystemServiceLock, flags);
return NTLSTATUS_SUCCESS;
}
// References one currently registered listener for a connect operation.
KSTATUS KeSystemServiceReference(SYS_SERVICE_ID serviceId, PORT** listenerOut) {
if (!listenerOut)
return NTLSTATUS_INVALID_PARAMETER;
*listenerOut = (PORT*)0;
if (!KiSystemServiceIdValid(serviceId))
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gSystemServiceLock);
PORT* listener = gSystemServices[serviceId];
if (!listener || !ObTryReferenceObject((OBJECT*)listener)) {
KeReleaseSpinlock(&gSystemServiceLock, flags);
return NTLSTATUS_NOT_FOUND;
}
KeReleaseSpinlock(&gSystemServiceLock, flags);
*listenerOut = listener;
return NTLSTATUS_SUCCESS;
}
// Removes any weak registration whose listener is entering terminal object destruction.
void KeSystemServiceUnregisterListener(PORT* listener) {
if (!listener)
return;
uintptr_t flags = KeAcquireSpinlock(&gSystemServiceLock);
for (SYS_SERVICE_ID id = 1; id <= SYS_SERVICE_MAX; ++id) {
if (gSystemServices[id] == listener)
gSystemServices[id] = (PORT*)0;
}
KeReleaseSpinlock(&gSystemServiceLock, flags);
}
code/kernel/ke_thread.c
+34 -19
View File
@@ -16,6 +16,7 @@
#include <kernel/ke_userenv.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_id.h>
#define FPU_FCW_DEFAULT 0x037Fu
#define FPU_MXCSR_DEFAULT 0x1F80u
@@ -32,9 +33,14 @@ STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 128);
STATIC_ASSERT_OFFSET(THREAD, Teb, 136);
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 192);
static uint32_t gNextThreadId = 1;
static KE_ID_ALLOCATOR gThreadIds = KE_ID_ALLOCATOR_INIT(1u);
extern void KeThreadEntry(void);
// Allocates one boot-unique thread identifier for normal and hand-built idle threads.
KSTATUS KeAllocateThreadId(uint32_t* threadIdOut) {
return KeAllocateId32(&gThreadIds, threadIdOut);
}
// Prevents a newly scheduled user thread from entering a terminating process.
void KePrepareUserThreadStart(void) {
THREAD* thread = KeGetCurrentThread();
@@ -85,11 +91,11 @@ void KeFinalizeThreadExit(THREAD* thread) {
uint32_t remaining = 0;
int detached = 0;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
int fwdDetached = thread->ProcessLink.Fwd == &thread->ProcessLink;
int bwdDetached = thread->ProcessLink.Bwd == &thread->ProcessLink;
if (fwdDetached != bwdDetached) {
KeReleaseSpinlock(&owner->HandleLock, flags);
KeReleaseSpinlock(&owner->ThreadLock, flags);
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread process linkage corrupt: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
(uint64_t)thread,
@@ -100,7 +106,7 @@ void KeFinalizeThreadExit(THREAD* thread) {
if (!fwdDetached) {
if (owner->ThreadCount == 0) {
KeReleaseSpinlock(&owner->HandleLock, flags);
KeReleaseSpinlock(&owner->ThreadLock, flags);
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread process count underflow: thread=0x%016llX tid=%u pid=%u\n",
(uint64_t)thread,
@@ -113,7 +119,7 @@ void KeFinalizeThreadExit(THREAD* thread) {
detached = 1;
}
remaining = owner->ThreadCount;
KeReleaseSpinlock(&owner->HandleLock, flags);
KeReleaseSpinlock(&owner->ThreadLock, flags);
if (!detached)
return;
@@ -122,6 +128,7 @@ void KeFinalizeThreadExit(THREAD* thread) {
// ThreadCount represents live process membership, not retained thread
// handles. Dead THREAD objects may remain waitable after process exit.
_InterlockedExchange(&owner->Terminating, 1);
KeStopProcessConstruction(owner);
KeCompleteProcessExitIfLastThread(owner);
}
}
@@ -184,12 +191,12 @@ static void KiThreadDestroy(OBJECT* obj) {
if (thread->Owner) {
ObDereferenceObject(&thread->Owner->Header);
thread->Owner = (PROCESS *)0;
thread->Owner = (PROCESS*)0;
}
if (thread->KernelStackBase) {
MmFreePool(thread->KernelStackBase);
thread->KernelStackBase = (void *)0;
thread->KernelStackBase = (void*)0;
}
}
@@ -204,11 +211,15 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
uint8_t initiallySuspended =
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
uint32_t threadId = 0;
KSTATUS idStatus = KeAllocateThreadId(&threadId);
if (!KSUCCESS(idStatus)) return idStatus;
THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!thread) return NTLSTATUS_NO_MEMORY;
thread->Header.Destroy = KiThreadDestroy;
thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
thread->ThreadId = threadId;
thread->State = initiallySuspended ? ThreadSuspended : ThreadReady;
thread->BaseSchedulingClass = parameters->SchedulingClass;
thread->BasePriority = parameters->Priority;
@@ -232,7 +243,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
: KeGetDefaultThreadAffinityMask();
LIST_INIT(&thread->ProcessLink);
LIST_INIT(&thread->SchedulerLink);
thread->WaitObject = (OBJECT *)0;
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = NTLSTATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
@@ -310,7 +321,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->UserEntry = entry;
thread->UserArg = arg;
uintptr_t* ksp = (uintptr_t *)thread->KernelStackTop;
uintptr_t* ksp = (uintptr_t*)thread->KernelStackTop;
*(--ksp) = userStackTop;
*(--ksp) = 0x23ULL;
*(--ksp) = userStackTop;
@@ -320,7 +331,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
*(--ksp) = arg;
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
KCONTEXT* ctx = (KCONTEXT*)ksp;
memset(ctx, 0, sizeof(KCONTEXT));
ctx->Rip = (uint64_t)KeThreadEntry;
thread->SavedContext = ctx;
@@ -347,10 +358,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
}
thread->ExitEvent = exitEvent;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
owner->ThreadCount++;
KeReleaseSpinlock(&owner->HandleLock, flags);
KeReleaseSpinlock(&owner->ThreadLock, flags);
KeInitThreadFpuContext(thread);
@@ -432,8 +443,8 @@ static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
const KE_THREAD_CREATE_PARAMETERS* parameters,
THREAD** out) {
// Kernel worker threads are owned by the system process. The scheduler's
// hand-built CPU idle thread stays attached to PID 0 (idle), while driver,
// async I/O, and init worker threads are grouped under PID 1 (system).
// hand-built CPU idle thread stays attached to the dedicated idle process, while
// driver, async I/O, and init worker threads use the system process.
PROCESS* owner = KeGetSystemProcess();
if (!owner) {
THREAD* currentThread = KeGetCurrentThread();
@@ -455,11 +466,15 @@ static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
uint8_t initiallySuspended =
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
uint32_t threadId = 0;
KSTATUS idStatus = KeAllocateThreadId(&threadId);
if (!KSUCCESS(idStatus)) return idStatus;
THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!thread) return NTLSTATUS_NO_MEMORY;
thread->Header.Destroy = KiThreadDestroy;
thread->ThreadId = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1);
thread->ThreadId = threadId;
thread->State = initiallySuspended ? ThreadSuspended : ThreadReady;
thread->BaseSchedulingClass = parameters->SchedulingClass;
thread->BasePriority = parameters->Priority;
@@ -528,10 +543,10 @@ static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
ObReferenceObject(&owner->Header);
thread->Owner = owner;
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
owner->ThreadCount++;
KeReleaseSpinlock(&owner->HandleLock, flags);
KeReleaseSpinlock(&owner->ThreadLock, flags);
// Hand the creator a short-lived reference before making the kernel
// thread runnable. Kernel workers can run, signal their completion event,
code/kernel/ke_time.c
+22 -1
View File
@@ -5,7 +5,7 @@
#include <string.h>
#include <ntlite/status.h>
#include <ntlite/boot_info.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_time.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_settings.h>
@@ -526,6 +526,27 @@ uint64_t KeQueryMonotonicTicks(void) {
: 0;
}
// Returns a saturated monotonic deadline for a relative millisecond interval.
uint64_t KeMonotonicDeadlineAfterMilliseconds(uint64_t milliseconds) {
uint64_t now = KeQueryMonotonicTicks();
if (milliseconds > (UINT64_MAX - now) / 1000ull)
return UINT64_MAX;
return now + (milliseconds * 1000ull);
}
// Returns the remaining whole milliseconds, rounded up, until a monotonic deadline.
uint64_t KeMonotonicMillisecondsUntil(uint64_t deadlineMicroseconds) {
uint64_t now = KeQueryMonotonicTicks();
if (deadlineMicroseconds <= now)
return 0;
uint64_t remaining = deadlineMicroseconds - now;
uint64_t milliseconds = remaining / 1000ull;
if ((remaining % 1000ull) != 0)
milliseconds++;
return milliseconds;
}
// Returns unix Time Seconds.
__declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void) {
SYS_SYSTEM_TIME t;
code/kernel/ke_timer.c
+28 -38
View File
@@ -4,7 +4,7 @@
#include <kernel/ke_list.h>
#include <string.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_time.h>
#include <kernel/ob_object.h>
#define TIMER_SIGNAL_BATCH 32u
@@ -23,29 +23,19 @@ static void KiTimerEnsureInitialized(void) {
gTimerInitialized = 1;
}
// Reads the scheduler tick value used by timer deadlines.
static uint64_t KiTimerReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
// Converts a millisecond interval to microseconds with saturation.
static uint64_t KiTimerMillisecondsToMicroseconds(uint64_t milliseconds) {
if (milliseconds > UINT64_MAX / 1000ull)
return UINT64_MAX;
return milliseconds * 1000ull;
}
// Converts a millisecond interval to at least one scheduler tick.
static uint64_t KiTimerMillisecondsToTicks(uint64_t milliseconds) {
if (milliseconds == 0)
return 0;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Adds a tick interval to the current scheduler tick with saturation.
static uint64_t KiTimerDeadlineFromNow(uint64_t intervalTicks) {
uint64_t now = KiTimerReadSchedulerTick();
uint64_t due = now + intervalTicks;
return due < now ? UINT64_MAX : due;
// Adds a monotonic interval to the current time with saturation.
static uint64_t KiTimerDeadlineFromNow(uint64_t intervalMicroseconds) {
uint64_t now = KeQueryMonotonicTicks();
if (intervalMicroseconds > UINT64_MAX - now)
return UINT64_MAX;
return now + intervalMicroseconds;
}
// Removes an active timer from the global deadline list while the timer lock is held.
@@ -56,8 +46,8 @@ static uint8_t KiTimerRemoveLocked(KTIMER* timer) {
KeListRemoveChecked(&timer->Link);
timer->Linked = 0;
timer->Active = 0;
timer->DueTick = 0;
timer->PeriodTicks = 0;
timer->DueMicroseconds = 0;
timer->PeriodMicroseconds = 0;
return 1;
}
@@ -110,27 +100,27 @@ KSTATUS KeTimerSet(KTIMER* timer, uint64_t dueTimeMilliseconds, uint64_t periodM
KiTimerEnsureInitialized();
uint64_t dueTicks = KiTimerMillisecondsToTicks(dueTimeMilliseconds);
uint64_t periodTicks = KiTimerMillisecondsToTicks(periodMilliseconds);
uint64_t dueMicroseconds = KiTimerMillisecondsToMicroseconds(dueTimeMilliseconds);
uint64_t periodMicroseconds = KiTimerMillisecondsToMicroseconds(periodMilliseconds);
uint8_t dropListReference = 0;
uint8_t addListReference = 0;
uint8_t signalNow = dueTicks == 0;
uint8_t signalNow = dueMicroseconds == 0;
EVENT* event = timer->Event;
ObReferenceObject(&event->Header);
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
if (timer->Linked) {
if (signalNow && periodTicks == 0) {
if (signalNow && periodMicroseconds == 0) {
dropListReference = KiTimerRemoveLocked(timer);
} else {
timer->DueTick = KiTimerDeadlineFromNow(signalNow ? periodTicks : dueTicks);
timer->PeriodTicks = periodTicks;
timer->DueMicroseconds = KiTimerDeadlineFromNow(signalNow ? periodMicroseconds : dueMicroseconds);
timer->PeriodMicroseconds = periodMicroseconds;
timer->Active = 1;
}
} else if (!signalNow || periodTicks != 0) {
timer->DueTick = KiTimerDeadlineFromNow(signalNow ? periodTicks : dueTicks);
timer->PeriodTicks = periodTicks;
} else if (!signalNow || periodMicroseconds != 0) {
timer->DueMicroseconds = KiTimerDeadlineFromNow(signalNow ? periodMicroseconds : dueMicroseconds);
timer->PeriodMicroseconds = periodMicroseconds;
timer->Active = 1;
timer->Linked = 1;
KeListInsertTailChecked(&gTimerList, &timer->Link);
@@ -183,7 +173,7 @@ void KeTimerTick(void) {
KTIMER* listReferencesToDrop[TIMER_SIGNAL_BATCH];
uint32_t count = 0;
uint32_t dropCount = 0;
uint64_t now = KiTimerReadSchedulerTick();
uint64_t now = KeQueryMonotonicTicks();
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
LIST_ENTRY* node = gTimerList.Fwd;
@@ -191,15 +181,15 @@ void KeTimerTick(void) {
LIST_ENTRY* next = node->Fwd;
KTIMER* timer = CONTAINING_RECORD(node, KTIMER, Link);
if (timer->Active && timer->DueTick <= now) {
if (timer->Active && timer->DueMicroseconds <= now) {
if (timer->Event) {
ObReferenceObject(&timer->Event->Header);
events[count++] = timer->Event;
}
if (timer->PeriodTicks != 0) {
uint64_t due = now + timer->PeriodTicks;
timer->DueTick = due < now ? UINT64_MAX : due;
if (timer->PeriodMicroseconds != 0) {
uint64_t due = now + timer->PeriodMicroseconds;
timer->DueMicroseconds = due < now ? UINT64_MAX : due;
} else {
if (KiTimerRemoveLocked(timer))
listReferencesToDrop[dropCount++] = timer;
code/kernel/ke_trap.c
+5 -7
View File
@@ -97,8 +97,7 @@ void KiExitTrapFrame(KTRAP_FRAME* frame) {
}
void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4,
uint64_t a5) {
uint64_t a2, uint64_t a3) {
KTRAP_FRAME* f = KeGetCurrentTrapFrame();
if (!f)
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: no current trap frame for nr=%llu\n", nr);
@@ -110,11 +109,10 @@ void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: syscall did not come from user cs=0x%016llX mode=%u\n",
f->Cs, f->PreviousMode);
if (f->Rax != nr || f->Rcx != a0 || f->Rdx != a1 || f->R8 != a2 ||
f->R9 != a3 || f->R10 != a4 || f->R11 != a5) {
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: arg mismatch nr=%llu tf=%llu/%llX/%llX/%llX/%llX/%llX/%llX call=%llu/%llX/%llX/%llX/%llX/%llX/%llX\n",
nr,
f->Rax, f->Rcx, f->Rdx, f->R8, f->R9, f->R10, f->R11,
nr, a0, a1, a2, a3, a4, a5);
f->R9 != a3) {
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: arg mismatch nr=%llu tf=%llu/%llX/%llX/%llX/%llX call=%llu/%llX/%llX/%llX/%llX\n",
nr, f->Rax, f->Rcx, f->Rdx, f->R8, f->R9,
nr, a0, a1, a2, a3);
}
}
code/kernel/kmain.c
+418 -213
View File
@@ -3,7 +3,9 @@
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <ntlite/boot_info.h>
#include <kernel/ke_hal.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
@@ -19,10 +21,11 @@
#include <kernel/ke_syscall.h>
#include <ntlite/ddi/net_device.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_util.h>
#include <kernel/ldr_image.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_api.h>
#include <kernel/io_file.h>
#include <kernel/io_volume.h>
#include <kernel/io_partition.h>
#include <kernel/io_ddk.h>
#include <kernel/ke_selftest.h>
#include <kernel/ke_smp.h>
@@ -33,11 +36,13 @@
#include <kernel/ke_settings.h>
#include <kernel/ke_work.h>
#include <kernel/ke_system_root.h>
#include <common/ini_parser.h>
#include <kernel/se_security.h>
#include <kernel/ke_boot_config.h>
typedef struct {
BOOT_INFO* bootInfo;
const HAL_API* hal;
BOOT_INFO* BootInfo;
const HAL_API* Hal;
KE_BOOT_CONFIG BootConfig;
} SYSTEM_INIT_ARGS;
#define KI_SWAP_MIN_MB 4ull
@@ -107,19 +112,12 @@ static void KiInitializeSwapFromSettings(void) {
(unsigned long long)KI_SWAP_MAX_MB);
}
char pagefilePath[1024];
if (!KeBuildSystemRootPath("pagefile.sys", pagefilePath, sizeof(pagefilePath))) {
KePanic(KE_PANIC_CODE_INIT,
"swap: failed to build system-root pagefile path\n");
}
uint64_t swapBytes = swapMb * 1024ull * 1024ull;
KSTATUS s = MmSwapInitialize(SysSvFromCstr(pagefilePath), swapBytes);
KSTATUS s = MmSwapInitialize("pagefile.sys", swapBytes);
if (!KSUCCESS(s)) {
KePanic(KE_PANIC_CODE_INIT,
"swap: failed to initialize %llu MB pagefile at %s, status=0x%x\n",
"swap: failed to initialize %llu MB system-root pagefile, status=0x%x\n",
(unsigned long long)swapMb,
pagefilePath,
s);
}
}
@@ -144,50 +142,162 @@ static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS
return 0;
}
// Implements the reclaim Bootloader Memory operation.
// Reclaims one exact page-backed blob after its boot consumer has copied or materialized it.
static void KiReclaimBootBlob(const BOOT_INFO* bootInfo, uintptr_t physicalBase, uint32_t pageCount) {
for (uint32_t index = 0; index < pageCount; ++index) {
PHYSICAL_ADDRESS page = (PHYSICAL_ADDRESS)(physicalBase + (uintptr_t)index * PAGE_SIZE);
if (KiPageInBootloaderRegion(bootInfo, page))
MmPfnFree(page);
}
}
// Reclaims raw boot-driver/configuration pages while leaving mapped kernel/HAL/page-table memory owned by bootstrap state.
static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
// Reclaim only the raw buffers backing boot-loaded .sys modules.
// The kernel/HAL images, page tables, and bootloader code remain live.
for (uint32_t i = 2; i < bootInfo->ModuleCount; i++) {
const BOOT_MODULE* mod = &bootInfo->Modules[i];
if (!mod->PhysBase || !mod->ImageSize) continue;
const BOOT_FILE_LIST* list = &bootInfo->BootDrivers;
const BOOT_FILE* files = list->FilesPhysical
? (const BOOT_FILE*)(bootInfo->PhysicalMapBase + list->FilesPhysical)
: NULL;
uintptr_t base = (uintptr_t)(mod->PhysBase & PAGE_MASK);
uintptr_t end = (uintptr_t)((mod->PhysBase + mod->ImageSize + PAGE_SIZE - 1) & PAGE_MASK);
for (uintptr_t page = base; page < end; page += PAGE_SIZE) {
if (!KiPageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue;
MmPfnFree((PHYSICAL_ADDRESS)page);
}
for (uint32_t index = 0; files && index < list->Count; ++index) {
if (files[index].PhysicalBase && files[index].PageCount)
KiReclaimBootBlob(bootInfo, files[index].PhysicalBase, files[index].PageCount);
}
if (list->FilesPhysical && list->PageCount)
KiReclaimBootBlob(bootInfo, list->FilesPhysical, list->PageCount);
if (bootInfo->BootIni.PhysicalBase && bootInfo->BootIni.PageCount)
KiReclaimBootBlob(bootInfo, bootInfo->BootIni.PhysicalBase, bootInfo->BootIni.PageCount);
}
// Phase Helper: Resolve Imports for core modules.
static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
// Only resolve imports for kernel.exe and hal.dll (typically indices 0 and 1).
// The bootloader maps these, but drivers are now passed as raw physical files.
uint32_t coreModuleCount = (bootInfo->ModuleCount > 2) ? 2 : bootInfo->ModuleCount;
for (uint32_t i = 0; i < coreModuleCount; i++) {
if (bootInfo->Modules[i].VirtBase != 0) {
LdrResolveImports(bootInfo->Modules[i].VirtBase);
}
}
// Returns whether one drivers.ini line character is horizontal whitespace.
static int KiDriverConfigSpace(char c) {
return c == ' ' || c == '\t' || c == '\r';
}
// Phase Helper: Initialize dynamically loaded .sys drivers.
static void KiDriversIniCb(const char* section, const char* key,
const char* val, void* ud) {
(void)val;
if (!strcmp(section, "runtime")) {
char*** pArray = (char***)ud;
size_t len = strlen(key) + 1;
char* keyCopy = MmAllocatePool(len);
if (keyCopy) {
memcpy(keyCopy, key, len);
arr_push(*pArray, keyCopy);
}
// Trims horizontal whitespace from one immutable drivers.ini line span.
static void KiDriverConfigTrim(const char** begin, const char** end) {
while (*begin < *end && KiDriverConfigSpace(**begin))
++*begin;
while (*end > *begin && KiDriverConfigSpace((*end)[-1]))
--*end;
}
// Returns whether one immutable drivers.ini span exactly matches an ASCII string.
static int KiDriverConfigSpanEquals(const char* begin, const char* end, const char* text) {
while (begin < end && *text) {
if (*begin++ != *text++)
return 0;
}
return begin == end && *text == '\0';
}
// Loads one runtime driver filename from the system-root drivers directory.
static KSTATUS KiLoadRuntimeDriverByName(const char* name, const NTL_DDK* ddk) {
char relative[512];
size_t length = strlen(name);
if (length + 9u > sizeof(relative))
return NTLSTATUS_BUFFER_TOO_SMALL;
memcpy(relative, "drivers/", 8);
memcpy(relative + 8, name, length + 1);
FILE_OBJECT* file = NULL;
KSTATUS status = KeOpenSystemRootFile(relative, &file);
if (!KSUCCESS(status))
return status;
uint64_t driverSize = IoGetFileSize(file);
if (driverSize == 0 || driverSize > SIZE_MAX) {
IoCloseFile(file);
return NTLSTATUS_INVALID_PARAMETER;
}
void* driverMemory = MmAllocatePool((size_t)driverSize);
if (!driverMemory) {
IoCloseFile(file);
return NTLSTATUS_NO_MEMORY;
}
size_t got = 0;
status = IoReadFile(file, driverMemory, (size_t)driverSize, &got);
IoCloseFile(file);
if (!KSUCCESS(status) || got != (size_t)driverSize) {
MmFreePool(driverMemory);
return KSUCCESS(status) ? NTLSTATUS_FAILURE : status;
}
uintptr_t mappedBase = 0;
status = LdrLoadKernelImage(driverMemory, (size_t)driverSize, &mappedBase);
MmFreePool(driverMemory);
if (!KSUCCESS(status))
return status;
status = LdrResolveImports(mappedBase);
if (!KSUCCESS(status))
return status;
status = LdrRegisterModule(name, mappedBase);
if (!KSUCCESS(status))
return status;
DRIVER_ENTRY_FN entry = (DRIVER_ENTRY_FN)LdrGetEntryPoint(mappedBase);
if (!entry)
return NTLSTATUS_FAILURE;
NTL_DRIVER_INIT init = {0};
return entry(ddk, &init);
}
// Scans only the [runtime] section of drivers.ini and loads each filename immediately.
static KSTATUS KiLoadRuntimeDriversFromConfig(const char* text, size_t length, const NTL_DDK* ddk) {
int inRuntime = 0;
size_t offset = 0;
while (offset < length) {
size_t lineStart = offset;
while (offset < length && text[offset] != '\n')
++offset;
size_t lineEnd = offset;
if (offset < length)
++offset;
const char* begin = text + lineStart;
const char* end = text + lineEnd;
KiDriverConfigTrim(&begin, &end);
if (begin == end || *begin == ';' || *begin == '#')
continue;
if (*begin == '[') {
if (end[-1] != ']')
return NTLSTATUS_INVALID_PARAMETER;
const char* sectionBegin = begin + 1;
const char* sectionEnd = end - 1;
KiDriverConfigTrim(&sectionBegin, &sectionEnd);
inRuntime = KiDriverConfigSpanEquals(sectionBegin, sectionEnd, "runtime");
continue;
}
if (!inRuntime)
continue;
size_t nameLength = (size_t)(end - begin);
if (nameLength == 0 || nameLength >= BOOT_FILE_NAME_MAX)
return NTLSTATUS_INVALID_PARAMETER;
for (const char* p = begin; p < end; ++p) {
if (*p == '=' || *p == '/' || *p == '\\')
return NTLSTATUS_INVALID_PARAMETER;
}
char name[BOOT_FILE_NAME_MAX];
memcpy(name, begin, nameLength);
name[nameLength] = '\0';
DbgPrint("Found runtime driver: %s\n", name);
KSTATUS status = KiLoadRuntimeDriverByName(name, ddk);
if (!KSUCCESS(status))
DbgPrint("KiInitializeDrivers: runtime driver %s failed, status=0x%x\n", name, status);
}
return NTLSTATUS_SUCCESS;
}
// Initializes drivers.
@@ -196,49 +306,45 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
const NTL_DDK* ddk = IoGetDdk();
DbgPrint("KiInitializeDrivers: Initializing drivers...\n");
// boot drivers (provided by the bootloader).
for (uint32_t i = 0; i < bootInfo->ModuleCount; i++) {
BOOT_MODULE* mod = &bootInfo->Modules[i];
// Skip modules that are already mapped (Kernel/HAL)
// or have no physical data (invalid).
if (mod->VirtBase != 0 || mod->PhysBase == 0) continue;
// Boot drivers are page-backed raw files supplied by the bootloader.
const BOOT_FILE* bootFiles = bootInfo->BootDrivers.FilesPhysical
? (const BOOT_FILE*)(bootInfo->PhysicalMapBase + bootInfo->BootDrivers.FilesPhysical)
: NULL;
DbgPrint("KiInitializeDrivers: Loading boot driver %s\n", mod->Name);
for (uint32_t i = 0; bootFiles && i < bootInfo->BootDrivers.Count; ++i) {
const BOOT_FILE* file = &bootFiles[i];
if (!file->PhysicalBase || !file->Length) {
DbgPrint("KiInitializeDrivers: invalid boot driver descriptor at index %u\n", i);
continue;
}
// Calculate virtual address of the raw file in the physical window.
void* rawPeBuffer = (void*)(bootInfo->PhysicalMapBase + mod->PhysBase);
DbgPrint("KiInitializeDrivers: Loading boot driver %s\n", file->Name);
void* rawPeBuffer = (void*)(bootInfo->PhysicalMapBase + file->PhysicalBase);
// Map the PE image into kernel space.
uintptr_t mappedBase = 0;
KSTATUS st = LdrLoadKernelImage(rawPeBuffer, mod->ImageSize, &mappedBase);
KSTATUS st = LdrLoadKernelImage(rawPeBuffer, file->Length, &mappedBase);
if (!KSUCCESS(st)) {
DbgPrint("ERROR: Failed to map driver %s (0x%X)\n", mod->Name, st);
DbgPrint("ERROR: Failed to map driver %s (0x%X)\n", file->Name, st);
continue;
}
// Update the struct so other modules can find it.
mod->VirtBase = mappedBase;
// Resolve imports while the image is still writable. Kernel driver images
// stay writable until the loader grows real section-backed mappings; the
// loader still clears NX for the whole image before returning it.
if (!KSUCCESS(LdrResolveImports(mappedBase))) {
DbgPrint("ERROR: Failed to resolve imports for %s\n", mod->Name);
DbgPrint("ERROR: Failed to resolve imports for %s\n", file->Name);
continue;
}
// Register with the global module list so subsequent drivers can import from this one.
LdrRegisterModule(mod->Name, mod->VirtBase);
KSTATUS registerStatus = LdrRegisterModule(file->Name, mappedBase);
if (!KSUCCESS(registerStatus)) {
DbgPrint("ERROR: Failed to register driver %s (0x%X)\n", file->Name, registerStatus);
continue;
}
// Execute Entry Point.
DRIVER_ENTRY_FN entry = (DRIVER_ENTRY_FN)LdrGetEntryPoint(mappedBase);
if (entry) {
NTL_DRIVER_INIT init = {0};
KSTATUS ds = entry(ddk, &init);
if (!KSUCCESS(ds)) {
DbgPrint("WARNING: Driver %s returned failure status 0x%X\n", mod->Name, ds);
}
if (!KSUCCESS(ds))
DbgPrint("WARNING: Driver %s returned failure status 0x%X\n", file->Name, ds);
}
}
@@ -251,106 +357,41 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to find system root volume (status %d)\n", s);
}
char root[4];
root[0] = (char)('0' + KeGetSystemRootDrive());
root[1] = ':';
root[2] = '/';
root[3] = '\0';
s = IoSetCwd(SysSvFromCstr(root));
if (s != NTLSTATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to set system CWD to %s (status %d)\n", root, s);
}
s = KeOpenSystemRootFile("drivers\\drivers.ini", &f);
s = KeOpenSystemRootFile("drivers/drivers.ini", &f);
if (s != NTLSTATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini on system root (status %d)\n", s);
}
uint64_t size = IoGetFileSize(f) + 1;
char* mem = MmAllocatePool(size);
size_t got;
if (!KSUCCESS(IoReadFile(f, mem, size, &got))) {
uint64_t fileSize = IoGetFileSize(f);
if (fileSize == 0 || fileSize >= SIZE_MAX) {
IoCloseFile(f);
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to read drivers.ini\n");
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: drivers.ini has an invalid size.\n");
}
size_t size = (size_t)fileSize;
char* mem = (char*)MmAllocatePool(size + 1u);
if (!mem) {
IoCloseFile(f);
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to allocate drivers.ini buffer.\n");
}
size_t got = 0;
KSTATUS readStatus = IoReadFile(f, mem, size, &got);
IoCloseFile(f);
if (!KSUCCESS(readStatus) || got != size) {
MmFreePool(mem);
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to read drivers.ini.\n");
}
mem[got] = '\0';
IoCloseFile(f);
char** runtimeKeys = NULL;
IniParse(mem, KiDriversIniCb, &runtimeKeys);
for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]);
char path[1024];
char relative[512];
size_t len = strlen(runtimeKeys[i]);
if (len + 9 > sizeof(relative)) {
DbgPrint("KiInitializeDrivers: runtime driver path too long: %s\n", runtimeKeys[i]);
continue;
}
memcpy(relative, "drivers\\", 8);
memcpy(relative + 8, runtimeKeys[i], len + 1);
if (!KeBuildSystemRootPath(relative, path, sizeof(path))) {
DbgPrint("KiInitializeDrivers: failed to build runtime driver path: %s\n", runtimeKeys[i]);
continue;
}
s = IoOpenFile(SysSvFromCstr(path), &f);
if (s != NTLSTATUS_SUCCESS) {
DbgPrint("KiInitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path);
continue;
}
uint64_t drvSize = IoGetFileSize(f);
void* drvMem = MmAllocatePool(drvSize);
if (!KSUCCESS(IoReadFile(f, drvMem, drvSize, &got))) {
DbgPrint("KiInitializeDrivers: Failed to read runtime driver: %s\n", runtimeKeys[i]);
IoCloseFile(f);
continue;
}
IoCloseFile(f);
uintptr_t mappedBase = 0;
if (!KSUCCESS(LdrLoadKernelImage(drvMem, drvSize, &mappedBase))) { // Load into kernel space
DbgPrint("KiInitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]);
MmFreePool(drvMem);
continue;
}
MmFreePool(drvMem);
if (!KSUCCESS(LdrResolveImports(mappedBase))) {
DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
continue;
}
// Runtime modules may also export DDIs to drivers loaded later. Boot
// drivers were already registered above; keep the module model uniform.
LdrRegisterModule(runtimeKeys[i], mappedBase);
DRIVER_ENTRY_FN entry = (DRIVER_ENTRY_FN)(LdrGetEntryPoint(mappedBase));
NTL_DRIVER_INIT init = {0};
KSTATUS ds = entry(ddk, &init);
if (!KSUCCESS(ds)) {
DbgPrint("KiInitializeDrivers: Driver %s failed to init!\n", runtimeKeys[i]);
}
}
// Free the individual strings allocated via strdup in the callback.
if (runtimeKeys) {
for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
if (runtimeKeys[i]) {
MmFreePool(runtimeKeys[i]);
}
}
// Free the dynamic array structure itself.
arr_free(runtimeKeys);
}
// Free the file buffer.
KSTATUS runtimeStatus = KiLoadRuntimeDriversFromConfig(mem, got, ddk);
MmFreePool(mem);
if (!KSUCCESS(runtimeStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"KiInitializeDrivers: drivers.ini [runtime] is malformed, status=0x%x\n",
runtimeStatus);
}
}
// Phase Helper: Load and launch a process from the filesystem.
@@ -374,7 +415,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
PROCESS* proc;
THREAD* thread;
s = KeCreateProcess(filePath, &proc);
s = KeCreateProcess(filePath, SeGetRootSecurityContext(), &proc);
if (!KSUCCESS(s)) {
DbgPrint("Failed to create process. System halted.\n");
IoCloseFile(f);
@@ -384,8 +425,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
s = KeSetupUserProcessEnvironment(proc,
SysSvFromCstr(filePath),
SysSvFromCstr(args),
SysSvFromCstr(proc->CurrentDirectory));
SysSvFromCstr(args));
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s);
IoCloseFile(f);
@@ -393,6 +433,56 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
return 0;
}
// Bootstrap init receives read-only system-volume authority, separate namespace-seeding
// authority, and the volume-manager capability used to supervise RSSRV publication.
IO_VOLUME* systemVolume = (IO_VOLUME*)0;
SYS_HANDLE systemVolumeHandle = SYS_HANDLE_INVALID;
SYS_HANDLE namespaceVolumeHandle = SYS_HANDLE_INVALID;
SYS_HANDLE volumeManagerHandle = SYS_HANDLE_INVALID;
s = KeReferenceSystemRootVolume(&systemVolume);
if (KSUCCESS(s)) {
s = KeCreateHandle(proc, &systemVolume->Header,
SYS_RIGHT_QUERY | SYS_VOLUME_RIGHT_OPEN,
&systemVolumeHandle);
if (KSUCCESS(s)) {
s = KeCreateHandle(proc, &systemVolume->Header,
SYS_RIGHT_DUPLICATE | SYS_RIGHT_TRANSFER | SYS_RIGHT_QUERY |
SYS_VOLUME_RIGHT_OPEN | SYS_VOLUME_RIGHT_MODIFY,
&namespaceVolumeHandle);
}
IoReleaseVolume(systemVolume);
}
if (KSUCCESS(s)) {
IO_VOLUME_MANAGER* volumeManager = (IO_VOLUME_MANAGER*)0;
s = IoReferenceVolumeManager(&volumeManager);
if (KSUCCESS(s)) {
s = KeCreateHandle(proc, &volumeManager->Header,
SYS_RIGHT_DUPLICATE | SYS_RIGHT_TRANSFER | SYS_RIGHT_QUERY |
SYS_VOLUME_MANAGER_RIGHT_ENUMERATE | SYS_VOLUME_MANAGER_RIGHT_OPEN |
SYS_VOLUME_MANAGER_RIGHT_WAIT,
&volumeManagerHandle);
ObDereferenceObject(&volumeManager->Header);
}
}
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to create bootstrap startup capabilities. Status: %d\n", s);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
}
SYS_STARTUP_CAPABILITY startupCapabilities[3] = {
{ SYS_STARTUP_CAP_SYSTEM_VOLUME, systemVolumeHandle },
{ SYS_STARTUP_CAP_SYSTEM_VOLUME_NAMESPACE, namespaceVolumeHandle },
{ SYS_STARTUP_CAP_VOLUME_MANAGER, volumeManagerHandle },
};
s = KeSetProcessStartupCapabilities(proc, startupCapabilities, 3);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to install startup capabilities. Status: %d\n", s);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
}
// Map the executable through an image section backed by the open file.
uintptr_t imageBase = 0;
s = LdrLoadImageFileForProcess(proc, f, &imageBase);
@@ -403,9 +493,9 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
return 0;
}
s = LdrResolveImportsForProcess(proc, imageBase);
s = LdrRequireNoImports(imageBase);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s);
DbgPrint("KiLaunchUserProcess: bootstrap image %s must be import-free (status %d)\n", filePath, s);
ObDereferenceObject(&proc->Header);
return 0;
}
@@ -448,22 +538,19 @@ void SystemInitThread(void* arg) {
KiRunEarlyKernelSelfTests();
KiInitializeDrivers(args->bootInfo, args->hal);
KiInitializeDrivers(args->BootInfo, args->Hal);
if (args->bootInfo->Debug & KE_DEBUG_AUDIO_SELFTEST)
KiRunAudioFinalPathSelfTest();
KeTimeReloadSettings();
KiInitializeSwapFromSettings();
KiResolveSystemImports(args->bootInfo);
KiReclaimBootloaderMemory(args->bootInfo);
KiReclaimBootloaderMemory(args->BootInfo);
if (args->bootInfo->Debug & KE_DEBUG_SELFTEST)
KiRunKernelSelfTests(args->hal);
if (args->BootConfig.Debug & KE_DEBUG_SELFTEST)
KiRunKernelSelfTests(args->Hal);
// Launch User-Mode init.exe.
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->BootConfig.CommandLine);
if (!shellProc) {
DbgPrint("SystemInitThread: Failed to start system/init.exe.\n");
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n");
@@ -477,28 +564,114 @@ void SystemInitThread(void* arg) {
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
ObDereferenceObject(&shellProc->Header);
if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
char logPath[1024];
KSTATUS logStatus = NTLSTATUS_NOT_FOUND;
if (KeBuildSystemRootPath("kernel.log", logPath, sizeof(logPath))) {
logStatus = KeFlushDebugLogToFile(SysSvFromCstr(logPath));
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush to %s failed st=0x%x\n", logPath, logStatus);
} else {
DbgPrint("debuglog: shutdown flush skipped; system root path unavailable\n");
}
if (args->BootConfig.Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
KSTATUS logStatus = KeFlushDebugLogToSystemRoot("kernel.log");
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush failed st=0x%x\n", logStatus);
}
gKmainHal->Power.Shutdown();
MmFreePool(args);
}
// Returns whether one page-backed boot allocation is aligned, nonempty, and does not overflow.
static int KiValidateBootPageRange(uintptr_t physicalBase, uint32_t pageCount) {
if (!physicalBase || (physicalBase & (PAGE_SIZE - 1u)) != 0 || pageCount == 0)
return 0;
uint64_t byteCount = (uint64_t)pageCount * PAGE_SIZE;
return byteCount != 0 && physicalBase <= UINTPTR_MAX - byteCount;
}
// Returns whether one page-backed raw boot file is fully contained in its recorded allocation.
static int KiValidateBootFile(const BOOT_FILE* file) {
if (!file || !KiValidateBootPageRange(file->PhysicalBase, file->PageCount) || file->Length == 0)
return 0;
uint64_t capacity = (uint64_t)file->PageCount * PAGE_SIZE;
if ((uint64_t)file->Length + 1u > capacity || file->Name[0] == '\0')
return 0;
return memchr(file->Name, '\0', sizeof(file->Name)) != NULL;
}
// Returns whether one fixed core-image descriptor fits inside its page-backed physical allocation.
static int KiValidateBootImage(const BOOT_IMAGE* image, int requireEntry) {
if (!image || !image->VirtualBase || (image->VirtualBase & (PAGE_SIZE - 1u)) != 0 ||
!KiValidateBootPageRange(image->PhysicalBase, image->PageCount) || !image->ImageSize) {
return 0;
}
uint64_t capacity = (uint64_t)image->PageCount * PAGE_SIZE;
if (image->ImageSize > capacity || image->VirtualBase > UINTPTR_MAX - image->ImageSize)
return 0;
if (requireEntry && image->EntryRva >= image->ImageSize)
return 0;
return 1;
}
// Validates the exact bootloader/kernel handoff contract before dereferencing variable physical blobs.
static int KiValidateBootInfo(const BOOT_INFO* bootInfo) {
if (!bootInfo || bootInfo->Magic != BOOT_INFO_MAGIC ||
bootInfo->Version != BOOT_INFO_VERSION || bootInfo->Size != sizeof(BOOT_INFO) ||
bootInfo->MemoryMap.Magic != MEMORY_MAP_MAGIC ||
bootInfo->MemoryMap.RegionCount > MEMORY_MAX_REGIONS ||
bootInfo->PhysicalMapBase == 0) {
return 0;
}
uint64_t bootIniCapacity = (uint64_t)bootInfo->BootIni.PageCount * PAGE_SIZE;
if (!KiValidateBootPageRange(bootInfo->BootIni.PhysicalBase, bootInfo->BootIni.PageCount) ||
bootInfo->BootIni.Length == 0 || (uint64_t)bootInfo->BootIni.Length + 1u > bootIniCapacity ||
bootInfo->BootIni.PhysicalBase > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
return 0;
}
uint64_t driverListCapacity = (uint64_t)bootInfo->BootDrivers.PageCount * PAGE_SIZE;
uint64_t driverListBytes = (uint64_t)bootInfo->BootDrivers.Count * sizeof(BOOT_FILE);
if (!KiValidateBootPageRange(bootInfo->BootDrivers.FilesPhysical, bootInfo->BootDrivers.PageCount) ||
bootInfo->BootDrivers.Count == 0 || driverListBytes > driverListCapacity ||
bootInfo->BootDrivers.FilesPhysical > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
return 0;
}
const BOOT_FILE* bootFiles =
(const BOOT_FILE*)(bootInfo->PhysicalMapBase + bootInfo->BootDrivers.FilesPhysical);
for (uint32_t index = 0; index < bootInfo->BootDrivers.Count; ++index) {
if (!KiValidateBootFile(&bootFiles[index]) ||
bootFiles[index].PhysicalBase > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
return 0;
}
}
if (!KiValidateBootImage(&bootInfo->KernelImage, 1) ||
!KiValidateBootImage(&bootInfo->HalImage, 0)) {
return 0;
}
return 1;
}
// Main Kernel Entry.
extern void KernelMain(BOOT_INFO* bootInfo) {
// Firmware is not required to transfer control with IF clear. Keep the
// bootstrap CPU masked until the interrupt manager and scheduler are ready.
_disable();
if (!KiValidateBootInfo(bootInfo)) {
for (;;)
__halt();
}
if (!KSUCCESS(KeBindBootstrapHal())) {
for (;;)
__halt();
}
const HAL_API* hal = KeGetHal();
// Physical & Virtual Memory initialization.
if (!MmInitializePhysical(&bootInfo->MemoryMap, bootInfo->PhysicalMapBase))
HalStopProcessor();
KeStopProcessor();
PHYSICAL_ADDRESS pml4 = (PHYSICAL_ADDRESS)__readcr3();
// The early virtual allocator starts small because it only needs bootstrap page tables before the PMM is online.
@@ -513,38 +686,56 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Register boot modules (Allows LdrGetExport to work for drivers and the HAL)
// Note: Kept to a single call here to avoid the duplicate in your original code.
LdrRegisterModulesFromBootInfo(bootInfo);
KSTATUS moduleRegistryStatus = LdrRegisterModulesFromBootInfo(bootInfo);
if (!KSUCCESS(moduleRegistryStatus))
KeStopProcessor();
// Parse the raw mandatory boot.ini handoff once before HAL policy consumes debug flags.
KE_BOOT_CONFIG bootConfig = {0};
const char* bootIniText = (const char*)(bootInfo->PhysicalMapBase + bootInfo->BootIni.PhysicalBase);
KSTATUS bootConfigStatus = KeParseBootConfiguration(bootIniText, bootInfo->BootIni.Length, &bootConfig);
// HAL Initialization.
HAL_INIT_PARAMS halParams = {
.Size = sizeof(HAL_INIT_PARAMS),
.Flags = 0,
.PhysicalWindowBase = bootInfo->PhysicalMapBase,
.FramebufferBase = bootInfo->FramebufferBase,
.FramebufferWidth = bootInfo->FramebufferWidth,
.FramebufferHeight = bootInfo->FramebufferHeight,
.FramebufferStride = bootInfo->FramebufferStride,
.RsdpPhysical = bootInfo->RsdpPhysical,
.Debug = bootInfo->Debug & 1, // bitmask 1 debug prints
.Debug = KSUCCESS(bootConfigStatus) ? (int)(bootConfig.Debug & 1u) : 0,
.AllocatePool = MmAllocatePool
};
KSTATUS s = HalInitialize(&halParams);
KSTATUS s = KeInitializeHal(&halParams);
if (!KSUCCESS(s))
HalStopProcessor();
KeStopProcessor();
const HAL_API* hal = HalGetApi();
IoInitializeDdk(hal);
IoInitializeDdk(hal, bootInfo->PhysicalMapBase);
gKmainHal = hal;
KeInitDebug(hal, bootInfo->Debug);
KeInitDebug(hal, KSUCCESS(bootConfigStatus) ? bootConfig.Debug : 0);
if (!KSUCCESS(bootConfigStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"boot.ini: malformed or missing mandatory configuration, status=0x%x\n",
bootConfigStatus);
}
DbgPrint("Boot: system volume selector=%s\n", bootConfig.SystemVolume);
// Bind IRQ dispatch while IF remains clear. Interrupt sources are armed
// later, after their kernel consumers have registered.
KeIrqInitialize(hal);
DbgPrint("CPU: PAT owns WB/WC/UC policy; MTRR default=WB fixed=off variable=off.\n");
KeTimeInitialize(bootInfo, hal);
KeDisplayInitialize(hal);
// CPU startup intentionally lives after HAL initialization but before IRQs
// and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI;
// the kernel owns KCPU records, per-CPU stacks, GDT/TSS/IDT state, and the
// AP park loop. External device IRQs remain routed to the BSP during this
// and scheduler startup. HAL owns firmware processor discovery and platform
// startup signaling; the kernel owns KCPU records, per-CPU stacks, GDT/TSS/IDT state, and the
// AP park loop. External device IRQs remain routed to the boot processor during this
// step; APs only become runnable after scheduler admission below.
KeSetApplicationProcessorTrampoline((uint64_t)bootInfo->ApTrampolinePhys);
KSTATUS smpStatus = KeStartApplicationProcessors();
@@ -552,31 +743,42 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to start application processors, status=0x%x\n", smpStatus);
}
KeIrqInitialize(hal);
KeInitializeDpcSystem();
KeInputInitialize(hal);
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr);
HalRegisterScheduler(KeSchedulerTick);
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.HandleInterrupt);
KeSyscallInit(hal);
KSTATUS securityStatus = SeInitializeSecurity();
if (!KSUCCESS(securityStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"security: failed to initialize root context, status=0x%x\n",
securityStatus);
}
// Dynamic Linking & Driver Execution.
KiResolveSystemImports(bootInfo);
//KiInitializeDrivers(bootInfo, hal);.
// Scheduler Prep & Status Update.
KeInitializeScheduler();
KeIrqRegister(hal->Timer.IrqNumber, KeSchedulerTick);
KeConfigureCurrentProcessorTimer();
KSTATUS volumeNamespaceStatus = IoInitializeVolumeNamespace();
if (!KSUCCESS(volumeNamespaceStatus)) {
KSTATUS volumeManagerStatus = IoInitializeVolumeManager();
if (!KSUCCESS(volumeManagerStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"I/O: failed to initialize volume namespace notifications, status=0x%x\n",
volumeNamespaceStatus);
"I/O: failed to initialize volume manager notifications, status=0x%x\n",
volumeManagerStatus);
}
KSTATUS partitionStatus = IoInitializePartitionManager(bootConfig.SystemVolume);
if (!KSUCCESS(partitionStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"I/O: failed to initialize partition discovery, status=0x%x\n",
partitionStatus);
}
KeAudioInitialize();
@@ -595,8 +797,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Pack the arguments for the init thread.
SYSTEM_INIT_ARGS* initArgs = MmAllocatePool(sizeof(SYSTEM_INIT_ARGS));
initArgs->bootInfo = bootInfo;
initArgs->hal = hal;
if (!initArgs)
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: failed to allocate initialization arguments.\n");
initArgs->BootInfo = bootInfo;
initArgs->Hal = hal;
initArgs->BootConfig = bootConfig;
// Spawn the Initialization Thread. No creator reference is needed; the
// scheduler owns the live thread lifetime.
@@ -613,5 +818,5 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// The bootstrap CPU becomes the scheduler idle context once interrupts are enabled.
while (true)
HalWaitForInterrupt();
KeWaitForInterrupt();
}
code/kernel/ldr_image.c
+119 -470
View File
@@ -9,7 +9,6 @@
#include <kernel/io_file.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_system_root.h>
#include <ntlite/syscall.h>
#ifndef LDR_DEBUG_VERBOSE
@@ -164,17 +163,17 @@ typedef struct {
#define IMAGE_SCN_MEM_READ 0x40000000u
#define IMAGE_SCN_MEM_WRITE 0x80000000u
// Module registry.
#define MAX_MODULES 32
// Global kernel-module registry. Entries are allocated as modules arrive so boot-driver count is not an ABI limit.
#define LDR_KERNEL_MODULE_NAME_MAX 64u
typedef struct {
char Name[32];
typedef struct MODULE_ENTRY {
char Name[LDR_KERNEL_MODULE_NAME_MAX];
uintptr_t Base;
size_t Size;
size_t Size;
struct MODULE_ENTRY* Next;
} MODULE_ENTRY;
static MODULE_ENTRY gModules[MAX_MODULES];
static uint32_t gModuleCount = 0;
static MODULE_ENTRY* gModules;
// Ensures a user image page is privately writable before the kernel patches it.
static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t address, size_t size) {
@@ -182,62 +181,50 @@ static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t add
return MmVadMakeImageRangePrivate(process, (void*)address, size);
}
// Implements the canonical Module Name operation.
static const char* KiCanonicalModuleName(const char* name) {
if (!name) return name;
// Allocates and inserts one global kernel-module registry entry.
static KSTATUS KiRegisterGlobalModule(const char* name, uintptr_t base, size_t size) {
if (!name || !*name || !base || !size)
return NTLSTATUS_INVALID_PARAMETER;
if (!_stricmp(name, "kernel32.dll") ||
!_stricmp(name, "user32.dll") ||
!_stricmp(name, "gdi32.dll") ||
!_stricmp(name, "ole32.dll") ||
!_stricmp(name, "winmm.dll")) {
return "win32.dll";
}
size_t nameLength = 0;
while (nameLength < LDR_KERNEL_MODULE_NAME_MAX && name[nameLength]) nameLength++;
if (nameLength == 0 || nameLength >= LDR_KERNEL_MODULE_NAME_MAX)
return NTLSTATUS_INVALID_PARAMETER;
if (!_stricmp(name, "msvcrt.dll")) {
return "posix.dll";
}
MODULE_ENTRY* entry = (MODULE_ENTRY*)MmAllocatePool(sizeof(MODULE_ENTRY));
if (!entry)
return NTLSTATUS_NO_MEMORY;
return name;
}
// Returns module Base In Table.
static uintptr_t KiFindModuleBaseInTable(const MODULE_ENTRY* modules, uint32_t count, const char* name) {
for (uint32_t i = 0; i < count; i++) {
if (_stricmp(modules[i].Name, name) == 0) return modules[i].Base;
}
return 0;
}
// Registers module Entry.
static void KiRegisterModuleEntry(MODULE_ENTRY* entry, const char* name, uintptr_t base, size_t size) {
strncpy(entry->Name, name, 31);
entry->Name[31] = '\0';
memset(entry, 0, sizeof(*entry));
memcpy(entry->Name, name, nameLength);
entry->Name[nameLength] = '\0';
entry->Base = base;
entry->Size = size;
entry->Next = gModules;
gModules = entry;
return NTLSTATUS_SUCCESS;
}
// Implements the ldr Register Modules From Boot Info operation.
void LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
for (uint32_t i = 0; i < bootInfo->ModuleCount && gModuleCount < MAX_MODULES; i++) {
// Only register modules that have a virtual base (Kernel, HAL).
if (bootInfo->Modules[i].VirtBase != 0) {
KiRegisterModuleEntry(&gModules[gModuleCount],
bootInfo->Modules[i].Name,
bootInfo->Modules[i].VirtBase,
bootInfo->Modules[i].ImageSize);
gModuleCount++;
}
}
// Registers the two fixed bootstrap images described explicitly by BOOT_INFO.
KSTATUS LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
if (!bootInfo)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = KiRegisterGlobalModule("kernel.exe",
bootInfo->KernelImage.VirtualBase,
bootInfo->KernelImage.ImageSize);
if (!KSUCCESS(status))
return status;
return KiRegisterGlobalModule("hal.dll",
bootInfo->HalImage.VirtualBase,
bootInfo->HalImage.ImageSize);
}
// Implements the ldr Register Module operation.
void LdrRegisterModule(const char* name, uintptr_t base) {
if (gModuleCount < MAX_MODULES) {
size_t size = base ? LdrGetImageSize(base) : 0;
KiRegisterModuleEntry(&gModules[gModuleCount], name, base, size);
gModuleCount++;
}
// Registers one loaded kernel module without imposing a fixed registry capacity.
KSTATUS LdrRegisterModule(const char* name, uintptr_t base) {
size_t size = base ? LdrGetImageSize(base) : 0;
return KiRegisterGlobalModule(name, base, size);
}
// Finds the loaded kernel module that owns an address.
@@ -245,111 +232,70 @@ const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut,
if (baseOut) *baseOut = 0;
if (sizeOut) *sizeOut = 0;
for (uint32_t i = 0; i < gModuleCount; i++) {
uintptr_t base = gModules[i].Base;
size_t size = gModules[i].Size;
if (!base || !size)
for (MODULE_ENTRY* module = gModules; module; module = module->Next) {
uintptr_t base = module->Base;
size_t size = module->Size;
if (!base || !size || base > UINTPTR_MAX - size)
continue;
if (address >= base && address < base + size) {
if (baseOut) *baseOut = base;
if (sizeOut) *sizeOut = size;
return gModules[i].Name;
return module->Name;
}
}
return "unknown";
}
// Returns global Module Base.
// Returns the loaded global kernel module base for one exact module name.
static uintptr_t KiFindGlobalModuleBase(const char* name) {
return KiFindModuleBaseInTable(gModules, gModuleCount, KiCanonicalModuleName(name));
}
static uintptr_t KiFindProcessModuleBaseInTable(
const PROCESS_MODULE* modules,
uint32_t count,
const char* name
) {
for (uint32_t i = 0; i < count; i++) {
if (_stricmp(modules[i].Name, KiCanonicalModuleName(name)) == 0) {
return modules[i].Base;
}
if (!name) return 0;
for (MODULE_ENTRY* module = gModules; module; module = module->Next) {
if (_stricmp(module->Name, name) == 0)
return module->Base;
}
return 0;
}
// Returns process Or Global Module Base.
static uintptr_t KiFindProcessOrGlobalModuleBase(PROCESS* process, const char* name) {
if (process) {
uintptr_t base = KiFindProcessModuleBaseInTable(process->Modules, process->ModuleCount, name);
if (base) return base;
}
return KiFindGlobalModuleBase(name);
}
// Returns export At Base.
static void* KiGetExportAtBase(uintptr_t base, const char* functionName) {
if (!base) return (void *)0;
if (!base) return (void*)0;
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
if (pe->Signature != PE_SIGNATURE) return (void *)0;
uint32_t peOffset = *(uint32_t*)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER*)(base + peOffset);
if (pe->Signature != PE_SIGNATURE) return (void*)0;
OPT_HEADER64* opt = &pe->OptionalHeader;
if (opt->NumberOfRvaAndSizes < 1) return (void *)0;
if (opt->NumberOfRvaAndSizes < 1) return (void*)0;
uint32_t expRva = opt->DataDirectory[0].VirtualAddress;
if (!expRva) return (void *)0;
if (!expRva) return (void*)0;
EXPORT_DIR* exp = (EXPORT_DIR *)(base + expRva);
uint32_t* names = (uint32_t *)(base + exp->AddressOfNames);
uint16_t* ordinals = (uint16_t *)(base + exp->AddressOfNameOrdinals);
uint32_t* functions = (uint32_t *)(base + exp->AddressOfFunctions);
EXPORT_DIR* exp = (EXPORT_DIR*)(base + expRva);
uint32_t* names = (uint32_t*)(base + exp->AddressOfNames);
uint16_t* ordinals = (uint16_t*)(base + exp->AddressOfNameOrdinals);
uint32_t* functions = (uint32_t*)(base + exp->AddressOfFunctions);
for (uint32_t i = 0; i < exp->NumberOfNames; i++) {
const char* name = (const char *)(base + names[i]);
const char* name = (const char*)(base + names[i]);
if (strcmp(name, functionName) == 0) {
uint16_t ord = ordinals[i];
return (void *)(base + functions[ord]);
return (void*)(base + functions[ord]);
}
}
return (void *)0;
return (void*)0;
}
// Export lookup.
void* LdrGetExport(const char* moduleName, const char* functionName) {
uintptr_t base = KiFindGlobalModuleBase(KiCanonicalModuleName(moduleName));
uintptr_t base = KiFindGlobalModuleBase(moduleName);
return KiGetExportAtBase(base, functionName);
}
// Implements the ldr Get Export For Process operation.
void* LdrGetExportForProcess(PROCESS* process, const char* moduleName, const char* functionName) {
uintptr_t base = KiFindProcessOrGlobalModuleBase(process, KiCanonicalModuleName(moduleName));
return KiGetExportAtBase(base, functionName);
}
// Looks up an export in a specific process module mapping.
void* LdrGetModuleExportForProcess(PROCESS* process, uintptr_t imageBase, const char* functionName) {
if (!process || !imageBase || !functionName) return (void*)0;
uintptr_t flags = KeAcquireSpinlock(&process->ModuleLock);
int found = 0;
for (uint32_t i = 0; i < process->ModuleCount; i++) {
if (process->Modules[i].Base == imageBase) {
found = 1;
break;
}
}
KeReleaseSpinlock(&process->ModuleLock, flags);
if (!found) return (void*)0;
return KiGetExportAtBase(imageBase, functionName);
}
// Implements the ldr Get Image Size operation.
size_t LdrGetImageSize(uintptr_t base) {
if (!base) return 0;
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
uint32_t peOffset = *(uint32_t*)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER*)(base + peOffset);
if (pe->Signature != PE_SIGNATURE) return 0;
return (size_t)pe->OptionalHeader.SizeOfImage;
}
@@ -401,11 +347,11 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
size_t mappedSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase);
if (!mappedSize) return NTLSTATUS_FAILURE;
uint32_t peOffset = *(uint32_t *)(imageBase + 0x3C);
uint32_t peOffset = *(uint32_t*)(imageBase + 0x3C);
if ((size_t)peOffset > mappedSize || mappedSize - (size_t)peOffset < sizeof(uint32_t) + 20)
return NTLSTATUS_FAILURE;
PE_HEADER* pe = (PE_HEADER *)(imageBase + peOffset);
PE_HEADER* pe = (PE_HEADER*)(imageBase + peOffset);
if (pe->Signature != PE_SIGNATURE) return NTLSTATUS_FAILURE;
size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader;
@@ -421,7 +367,7 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
for (size_t i = 0; i < pageCount; ++i)
pageProt[i] = headerProt;
SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset);
SECTION_HEADER* sec = (SECTION_HEADER*)(imageBase + sectionTableOffset);
for (int i = 0; i < pe->NumberOfSections; i++) {
size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData;
if (!protectSize) continue;
@@ -497,8 +443,8 @@ uintptr_t LdrGetEntryPoint(uintptr_t base) {
// Base Relocation.
static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, uintptr_t delta, bool user) {
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
uint32_t peOffset = *(uint32_t*)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER*)(base + peOffset);
OPT_HEADER64* opt = &pe->OptionalHeader;
if (opt->NumberOfRvaAndSizes < 6) return NTLSTATUS_SUCCESS;
@@ -506,12 +452,12 @@ static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, u
uint32_t relocSize = opt->DataDirectory[5].Size;
if (!relocRva || !relocSize) return NTLSTATUS_SUCCESS;
BASE_RELOC* block = (BASE_RELOC *)(base + relocRva);
BASE_RELOC* block = (BASE_RELOC*)(base + relocRva);
uintptr_t end = (uintptr_t)block + relocSize;
while ((uintptr_t)block < end && block->SizeOfBlock > 0) {
uint32_t count = (block->SizeOfBlock - sizeof(BASE_RELOC)) / sizeof(uint16_t);
uint16_t* entries = (uint16_t *)(block + 1);
uint16_t* entries = (uint16_t*)(block + 1);
for (uint32_t i = 0; i < count; i++) {
int type = entries[i] >> 12;
@@ -519,39 +465,21 @@ static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, u
// For 64-bit PEs, we mostly care about DIR64 (Type 10).
if (type == IMAGE_REL_BASED_DIR64) {
uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset);
uint64_t* ptr = (uint64_t*)(base + block->VirtualAddress + offset);
KSTATUS ws = KiEnsureUserImageWrite(process, user, (uintptr_t)ptr, sizeof(*ptr));
if (!KSUCCESS(ws)) return ws;
*ptr += delta;
}
}
block = (BASE_RELOC *)((uint8_t *)block + block->SizeOfBlock);
block = (BASE_RELOC*)((uint8_t*)block + block->SizeOfBlock);
}
return NTLSTATUS_SUCCESS;
}
// Import resolution.
typedef void* (*EXPORT_LOOKUP_FN)(void* ctx, const char* moduleName, const char* functionName);
// Implements the global Export Lookup operation.
static void* KiGlobalExportLookup(void* ctx, const char* moduleName, const char* functionName) {
(void)ctx;
return LdrGetExport(moduleName, functionName);
}
// Processes export Lookup.
static void* KiProcessExportLookup(void* ctx, const char* moduleName, const char* functionName) {
return LdrGetExportForProcess((PROCESS*)ctx, moduleName, functionName);
}
static KSTATUS LdrResolveImportsCommon(
PROCESS* process,
uintptr_t imageBase,
EXPORT_LOOKUP_FN lookup,
void* ctx
) {
uint32_t peOffset = *(uint32_t *)(imageBase + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(imageBase + peOffset);
// Resolves imports for kernel images against the kernel module registry.
KSTATUS LdrResolveImports(uintptr_t imageBase) {
uint32_t peOffset = *(uint32_t*)(imageBase + 0x3C);
PE_HEADER* pe = (PE_HEADER*)(imageBase + peOffset);
OPT_HEADER64* opt = &pe->OptionalHeader;
if (opt->NumberOfRvaAndSizes < 2)
@@ -561,101 +489,57 @@ static KSTATUS LdrResolveImportsCommon(
if (!impRva)
return NTLSTATUS_SUCCESS;
IMPORT_DESC* imp = (IMPORT_DESC *)(imageBase + impRva);
IMPORT_DESC* imp = (IMPORT_DESC*)(imageBase + impRva);
while (imp->Name) {
const char* dllName = (const char *)(imageBase + imp->Name);
const char* dllName = (const char*)(imageBase + imp->Name);
LdrDbg("LDR: resolve kernel import dll=%s image=%016llx\n",
dllName, (unsigned long long)imageBase);
//Compatibility remap.
dllName = KiCanonicalModuleName(dllName);
LdrDbg("LDR: resolve import dll=%s image=%016llx pid=%u\n", dllName, (unsigned long long)imageBase, process ? process->ProcessId : 0);
// Ensure dependency is loaded into the process.
if (process) {
uintptr_t modBase =
KiFindProcessOrGlobalModuleBase(process, dllName);
if (!modBase) {
KSTATUS s =
LdrLoadModuleForProcess(process, dllName, LDR_MODULE_LOAD_IMPORT, &modBase);
if (!KSUCCESS(s)) {
DbgPrint(
"LdrResolveImports: Failed to load dependency %s (%d)\n",
dllName,
s
);
return s;
}
}
}
uint64_t* thunk =
(uint64_t *)(imageBase + imp->FirstThunk);
uint64_t* origThunk =
(uint64_t *)(imageBase +
(imp->OriginalFirstThunk
? imp->OriginalFirstThunk
: imp->FirstThunk));
uint64_t* thunk = (uint64_t*)(imageBase + imp->FirstThunk);
uint64_t* origThunk = (uint64_t*)(imageBase +
(imp->OriginalFirstThunk ? imp->OriginalFirstThunk : imp->FirstThunk));
while (*origThunk) {
if (*origThunk & (1ULL << 63)) {
DbgPrint(
"LdrResolveImports: Ordinal imports not supported! (%s)\n",
dllName
);
DbgPrint("LdrResolveImports: Ordinal imports not supported! (%s)\n", dllName);
return NTLSTATUS_NOT_SUPPORTED;
}
const char* funcName =
(const char *)(imageBase +
(*origThunk & 0xFFFFFFFF) + 2);
void* addr = lookup(ctx, dllName, funcName);
(const char*)(imageBase + (*origThunk & 0xFFFFFFFF) + 2);
void* addr = LdrGetExport(dllName, funcName);
if (!addr) {
DbgPrint(
"LdrResolveImports: Function not found! (%s!%s)\n",
dllName,
funcName
);
DbgPrint("LdrResolveImports: Function not found! (%s!%s)\n", dllName, funcName);
return NTLSTATUS_NOT_FOUND;
}
KSTATUS ws = KiEnsureUserImageWrite(process, process != (PROCESS*)0, (uintptr_t)thunk, sizeof(*thunk));
if (!KSUCCESS(ws)) return ws;
*thunk = (uint64_t)addr;
thunk++;
origThunk++;
}
imp++;
}
return NTLSTATUS_SUCCESS;
}
// Implements the ldr Resolve Imports operation.
KSTATUS LdrResolveImports(uintptr_t imageBase) {
return LdrResolveImportsCommon(
(PROCESS*)0,
imageBase,
KiGlobalExportLookup,
(void*)0
);
}
// Implements the ldr Resolve Imports For Process operation.
KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
return LdrResolveImportsCommon(
process,
imageBase,
KiProcessExportLookup,
process
);
// Requires an already-mapped bootstrap image to have no import directory.
KSTATUS LdrRequireNoImports(uintptr_t imageBase) {
if (!imageBase) return NTLSTATUS_INVALID_PARAMETER;
uint32_t peOffset = *(uint32_t*)(imageBase + 0x3C);
PE_HEADER* pe = (PE_HEADER*)(imageBase + peOffset);
if (pe->Signature != PE_SIGNATURE ||
pe->Machine != IMAGE_FILE_MACHINE_AMD64 ||
pe->OptionalHeader.Magic != IMAGE_NT_OPTIONAL_HDR64_MAGIC)
return NTLSTATUS_FAILURE;
if (pe->OptionalHeader.NumberOfRvaAndSizes >= 2 &&
(pe->OptionalHeader.DataDirectory[1].VirtualAddress != 0 ||
pe->OptionalHeader.DataDirectory[1].Size != 0))
return NTLSTATUS_NOT_SUPPORTED;
if (pe->OptionalHeader.NumberOfRvaAndSizes >= 14 &&
(pe->OptionalHeader.DataDirectory[13].VirtualAddress != 0 ||
pe->OptionalHeader.DataDirectory[13].Size != 0))
return NTLSTATUS_NOT_SUPPORTED;
return NTLSTATUS_SUCCESS;
}
// Loads a user image from an open file through the image-section object model.
@@ -693,261 +577,26 @@ KSTATUS LdrLoadImageFileForProcess(PROCESS* process, FILE_OBJECT* file, uintptr_
return NTLSTATUS_SUCCESS;
}
// Converts a process module entry into the public module ABI.
static void KiFillProcessModuleInfo(const PROCESS_MODULE* mod, SYS_MODULE_INFO* info) {
memset(info, 0, sizeof(*info));
info->Size = sizeof(*info);
info->Version = SYS_MODULE_INFO_VERSION;
info->Flags = mod->Flags;
if (mod->PublicRefCount)
info->Flags |= SYS_MODULE_FLAG_PUBLIC_REF;
if (mod->ImportRefCount)
info->Flags |= SYS_MODULE_FLAG_IMPORT_PINNED;
if (mod->PublicRefCount && mod->ImportRefCount == 0)
info->Flags |= SYS_MODULE_FLAG_UNLOADABLE;
info->Base = mod->Base;
info->ImageSize = mod->Size;
info->EntryPoint = mod->EntryPoint;
info->PublicRefCount = mod->PublicRefCount;
info->ImportRefCount = mod->ImportRefCount;
info->TotalRefCount = mod->PublicRefCount + mod->ImportRefCount;
strncpy(info->Name, mod->Name, sizeof(info->Name) - 1);
info->Name[sizeof(info->Name) - 1] = '\0';
}
// Adds one reference of the requested class to an existing process module.
static void KiReferenceProcessModule(PROCESS_MODULE* mod, uint32_t loadFlags) {
if (loadFlags & LDR_MODULE_LOAD_PUBLIC)
mod->PublicRefCount++;
if (loadFlags & LDR_MODULE_LOAD_IMPORT)
mod->ImportRefCount++;
}
// Finds a process module by canonical name while the caller owns ModuleLock.
static PROCESS_MODULE* KiFindProcessModuleByNameLocked(PROCESS* process, const char* name) {
for (uint32_t i = 0; i < process->ModuleCount; i++) {
if (_stricmp(process->Modules[i].Name, name) == 0)
return &process->Modules[i];
}
return (PROCESS_MODULE*)0;
}
// Finds a process module by image base while the caller owns ModuleLock.
static PROCESS_MODULE* KiFindProcessModuleByBaseLocked(PROCESS* process, uintptr_t base) {
for (uint32_t i = 0; i < process->ModuleCount; i++) {
if (process->Modules[i].Base == base)
return &process->Modules[i];
}
return (PROCESS_MODULE*)0;
}
// Registers a mapped process module or records a reference to an existing one.
static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base, uint32_t loadFlags, uintptr_t* registeredBaseOut) {
if (!process || !name || !base || !loadFlags) return NTLSTATUS_INVALID_PARAMETER;
if (registeredBaseOut) *registeredBaseOut = 0;
uintptr_t flags = KeAcquireSpinlock(&process->ModuleLock);
PROCESS_MODULE* existing = KiFindProcessModuleByNameLocked(process, name);
if (existing) {
KiReferenceProcessModule(existing, loadFlags);
if (registeredBaseOut) *registeredBaseOut = existing->Base;
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_ALREADY_EXISTS;
}
if (process->ModuleCount >= PROCESS_MODULE_MAX) {
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_NO_MEMORY;
}
PROCESS_MODULE* mod = &process->Modules[process->ModuleCount++];
memset(mod, 0, sizeof(*mod));
strncpy(mod->Name, name, sizeof(mod->Name) - 1);
mod->Name[sizeof(mod->Name) - 1] = '\0';
mod->Base = base;
mod->Size = LdrGetMappedImageSize(process, base);
mod->EntryPoint = LdrGetEntryPoint(base);
KiReferenceProcessModule(mod, loadFlags);
if (registeredBaseOut) *registeredBaseOut = mod->Base;
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_SUCCESS;
}
// Loads a DLL-like user module into a process and records the requested reference.
KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uint32_t loadFlags, uintptr_t* baseOut) {
if (!process || !filename || !loadFlags) return NTLSTATUS_INVALID_PARAMETER;
filename = KiCanonicalModuleName(filename);
uintptr_t flags = KeAcquireSpinlock(&process->ModuleLock);
PROCESS_MODULE* existing = KiFindProcessModuleByNameLocked(process, filename);
if (existing) {
KiReferenceProcessModule(existing, loadFlags);
if (baseOut) *baseOut = existing->Base;
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_SUCCESS;
}
KeReleaseSpinlock(&process->ModuleLock, flags);
FILE_OBJECT* f;
KSTATUS s = IoOpenFile(SysSvFromCstr(filename), &f);
if (!KSUCCESS(s)) {
char relative[260];
size_t nameLen = strnlen(filename, sizeof(relative) - 8);
if (nameLen + 8 >= sizeof(relative))
return NTLSTATUS_BUFFER_TOO_SMALL;
memcpy(relative, "system/", 7);
memcpy(relative + 7, filename, nameLen);
relative[7 + nameLen] = '\0';
s = KeOpenSystemRootFile(relative, &f);
if (!KSUCCESS(s)) {
DbgPrint("LdrLoadModuleForProcess: Failed to open %s on system root (%d)\n", filename, s);
return s;
}
}
uint64_t fileSize = IoGetFileSize(f);
LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize);
uintptr_t imageBase = 0;
s = LdrLoadImageFileForProcess(process, f, &imageBase);
IoCloseFile(f);
if (!KSUCCESS(s)) {
DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s);
return s;
}
LdrDbg("LDR: module mapped pid=%u file=%s base=%016llx size=%llx\n",
process->ProcessId, filename, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(process, imageBase));
s = LdrResolveImportsForProcess(process, imageBase);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
return s;
}
s = LdrProtectImageForProcess(process, imageBase, true);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
return s;
}
uintptr_t registeredBase = 0;
s = LdrRegisterProcessModule(process, filename, imageBase, loadFlags, &registeredBase);
if (s == NTLSTATUS_ALREADY_EXISTS) {
KiFreeLoadedImageForProcess(process, imageBase, true);
imageBase = registeredBase;
s = NTLSTATUS_SUCCESS;
}
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
return s;
}
LdrDbg("LDR: module registered pid=%u name=%s base=%016llx modules=%u\n",
process->ProcessId, filename, (unsigned long long)imageBase, process->ModuleCount);
if (baseOut) *baseOut = imageBase;
return NTLSTATUS_SUCCESS;
}
// Releases one public module load reference and unmaps the image when no references remain.
KSTATUS LdrUnloadModuleForProcess(PROCESS* process, uintptr_t imageBase) {
if (!process || !imageBase) return NTLSTATUS_INVALID_PARAMETER;
char name[PROCESS_MODULE_NAME_MAX];
uintptr_t base = 0;
int shouldUnmap = 0;
uintptr_t flags = KeAcquireSpinlock(&process->ModuleLock);
PROCESS_MODULE* mod = KiFindProcessModuleByBaseLocked(process, imageBase);
if (!mod) {
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_NOT_FOUND;
}
if (mod->PublicRefCount == 0) {
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_INVALID_PARAMETER;
}
mod->PublicRefCount--;
if (mod->PublicRefCount == 0 && mod->ImportRefCount == 0) {
base = mod->Base;
strncpy(name, mod->Name, sizeof(name) - 1);
name[sizeof(name) - 1] = '\0';
uint32_t index = (uint32_t)(mod - process->Modules);
if (index + 1 < process->ModuleCount) {
memmove(&process->Modules[index],
&process->Modules[index + 1],
(process->ModuleCount - index - 1) * sizeof(process->Modules[0]));
}
process->ModuleCount--;
shouldUnmap = 1;
}
KeReleaseSpinlock(&process->ModuleLock, flags);
if (shouldUnmap) {
LdrDbg("LDR: unload module pid=%u name=%s base=%016llx\n",
process->ProcessId, name, (unsigned long long)base);
KiFreeLoadedImageForProcess(process, base, true);
}
return NTLSTATUS_SUCCESS;
}
// Queries one loaded process module by image base.
KSTATUS LdrQueryModuleForProcess(PROCESS* process, uintptr_t imageBase, SYS_MODULE_INFO* infoOut) {
if (!process || !imageBase || !infoOut) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&process->ModuleLock);
PROCESS_MODULE* mod = KiFindProcessModuleByBaseLocked(process, imageBase);
if (!mod) {
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_NOT_FOUND;
}
KiFillProcessModuleInfo(mod, infoOut);
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_SUCCESS;
}
// Copies a bounded snapshot of the process module table.
KSTATUS LdrCopyProcessModuleInfo(PROCESS* process, SYS_MODULE_INFO* infoOut, size_t maxCount, size_t* countOut) {
if (!process || !countOut) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&process->ModuleLock);
size_t count = process->ModuleCount;
*countOut = count;
size_t toCopy = count;
if (toCopy > maxCount) toCopy = maxCount;
if (toCopy && !infoOut) {
KeReleaseSpinlock(&process->ModuleLock, flags);
return NTLSTATUS_INVALID_PARAMETER;
}
for (size_t i = 0; i < toCopy; i++)
KiFillProcessModuleInfo(&process->Modules[i], &infoOut[i]);
KeReleaseSpinlock(&process->ModuleLock, flags);
return (maxCount < count) ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS;
}
// Loads a boot/kernel PE image from a raw memory buffer into kernel space.
static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, uintptr_t* baseOut) {
if (!buffer || !size || !baseOut) return NTLSTATUS_INVALID_PARAMETER;
if (size < 0x40) return NTLSTATUS_FAILURE;
const uint8_t* bytes = (const uint8_t*)buffer;
uint32_t peOffset = *(const uint32_t *)(bytes + 0x3C);
uint32_t peOffset = *(const uint32_t*)(bytes + 0x3C);
if ((size_t)peOffset > size || size - (size_t)peOffset < 24) return NTLSTATUS_FAILURE;
uint32_t signature = *(const uint32_t *)(bytes + peOffset);
uint32_t signature = *(const uint32_t*)(bytes + peOffset);
if (signature != PE_SIGNATURE) return NTLSTATUS_FAILURE;
uint16_t machine = *(const uint16_t *)(bytes + peOffset + 4);
uint16_t optionalSize = *(const uint16_t *)(bytes + peOffset + 20);
uint16_t machine = *(const uint16_t*)(bytes + peOffset + 4);
uint16_t optionalSize = *(const uint16_t*)(bytes + peOffset + 20);
if (optionalSize < sizeof(OPT_HEADER64) ||
(size_t)peOffset + 24 + optionalSize > size) {
return NTLSTATUS_FAILURE;
}
uint16_t optionalMagic = *(const uint16_t *)(bytes + peOffset + 24);
uint16_t optionalMagic = *(const uint16_t*)(bytes + peOffset + 24);
if (machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
DbgPrint("LdrLoadKernelImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
machine, optionalMagic);
@@ -960,7 +609,7 @@ static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, u
return NTLSTATUS_NOT_SUPPORTED;
}
PE_HEADER* pe = (PE_HEADER *)(bytes + peOffset);
PE_HEADER* pe = (PE_HEADER*)(bytes + peOffset);
OPT_HEADER64* opt = &pe->OptionalHeader;
size_t imageSize = 0;
KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize);
@@ -981,10 +630,10 @@ static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, u
memset(mem, 0, imageSize);
memcpy(mem, buffer, opt->SizeOfHeaders);
SECTION_HEADER* sec = (SECTION_HEADER *)((uint8_t *)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
SECTION_HEADER* sec = (SECTION_HEADER*)((uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
for (int i = 0; i < pe->NumberOfSections; i++) {
void* dst = (uint8_t *)mem + sec[i].VirtualAddress;
const void* src = (const uint8_t *)buffer + sec[i].PointerToRawData;
void* dst = (uint8_t*)mem + sec[i].VirtualAddress;
const void* src = (const uint8_t*)buffer + sec[i].PointerToRawData;
if (sec[i].SizeOfRawData) {
size_t copySize = sec[i].SizeOfRawData;
if (sec[i].VirtualSize && sec[i].VirtualSize < copySize) copySize = sec[i].VirtualSize;
code/kernel/ldr_section.c
+9 -8
View File
@@ -10,6 +10,7 @@
#include <kernel/mm_pte.h>
#include <kernel/ob_object.h>
#include <kernel/io_file.h>
#include <kernel/io_volume.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_debug.h>
@@ -329,16 +330,16 @@ static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section,
// Image prototype pages are the canonical cache for executable image bytes.
// Read the backing filesystem directly so the fault path has one cache owner.
if (!section->BackingFile->Fs || !section->BackingFile->Fs->Read)
return NTLSTATUS_NOT_SUPPORTED;
if (!IoVolumeIsMounted(section->BackingFile->Volume))
return NTLSTATUS_OBJECT_DEAD;
size_t got = 0;
KSTATUS s = section->BackingFile->Fs->Read(section->BackingFile->Fs,
section->BackingFile->FileCtx,
fileOffset,
dst,
size,
&got);
KSTATUS s = IoVolumeRead(section->BackingFile->Volume,
section->BackingFile->FileCtx,
fileOffset,
dst,
size,
&got);
if (!KSUCCESS(s))
return s;
if (got != size)
code/kernel/mm_file_cache.c
+58 -9
View File
@@ -11,6 +11,7 @@
#include <intrin.h>
#include <kernel/mm_file_cache.h>
#include <kernel/io_file.h>
#include <kernel/io_volume.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
@@ -52,6 +53,8 @@ static uint64_t gFileCacheMaxResidentPages;
static MM_FILE_CACHE_STATS gFileCacheStats;
static int KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry);
static SPINLOCK gWritebackLock = SPINLOCK_INIT;
static SEMAPHORE* gWritebackSemaphore;
static int gWritebackStartState;
@@ -291,27 +294,35 @@ static KSTATUS KiFileCacheFlushOneDirtyPage(FILE_OBJECT* file, int* didWork) {
void* page = MmPfnToVirtual(frame);
size_t written = 0;
KSTATUS s = NTLSTATUS_FAILURE;
if (page && target->Fs && target->Fs->Write)
s = target->Fs->Write(target->Fs, target->FileCtx, offset, page, validBytes, &written);
if (page)
s = IoVolumeWrite(target->Volume, target->FileCtx, offset, page, validBytes, &written);
if (KSUCCESS(s) && written != validBytes)
s = NTLSTATUS_FAILURE;
MM_FILE_CACHE_ENTRY* invalidatedVictim = (MM_FILE_CACHE_ENTRY*)0;
flags = KeAcquireSpinlock(&gFileCacheLock);
uint32_t bucket = KiFileCacheHash(target, offset);
MM_FILE_CACHE_ENTRY* cur = KiFileCacheFindLocked(target, offset, bucket);
if (cur && cur->Frame == frame) {
if (KSUCCESS(s) && cur->Dirty && cur->DirtySequence == sequence) {
cur->Writeback = 0;
if (cur->Invalidated) {
if (cur->Dirty)
KiFileCacheSetDirtyLocked(cur, 0);
if (KiFileCacheUnlinkEntryLocked(cur))
invalidatedVictim = cur;
} else if (KSUCCESS(s) && cur->Dirty && cur->DirtySequence == sequence) {
KiFileCacheSetDirtyLocked(cur, 0);
gFileCacheStats.WritebackPages++;
gFileCacheStats.WritebackBytes += validBytes;
} else if (!KSUCCESS(s)) {
gFileCacheStats.WritebackFailures++;
}
cur->Writeback = 0;
}
KeReleaseSpinlock(&gFileCacheLock, flags);
(void)MmPfnRelease(frame);
if (invalidatedVictim)
KiFileCacheFreeEntry(invalidatedVictim);
ObDereferenceObject(&target->Header);
if (didWork) *didWork = 1;
return s;
@@ -619,9 +630,7 @@ static KSTATUS KiFileCacheLoadPage(FILE_OBJECT* file,
toRead = (size_t)remaining;
size_t got = 0;
s = NTLSTATUS_NOT_SUPPORTED;
if (file->Fs && file->Fs->Read)
s = file->Fs->Read(file->Fs, file->FileCtx, aligned, page, toRead, &got);
s = IoVolumeRead(file->Volume, file->FileCtx, aligned, page, toRead, &got);
if (!KSUCCESS(s)) {
(void)MmPfnRelease(frame);
KiFileCacheCancelLoadingPage(entry);
@@ -747,8 +756,8 @@ KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf,
return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait())
return NTLSTATUS_INVALID_IRQL;
if (!file->Fs || !file->Fs->Write)
return NTLSTATUS_NOT_SUPPORTED;
if (!IoVolumeIsMounted(file->Volume))
return NTLSTATUS_OBJECT_DEAD;
if (size == 0)
return NTLSTATUS_SUCCESS;
@@ -925,6 +934,46 @@ void MmFileCacheInvalidateAll(FILE_OBJECT* file) {
}
}
// Discards all cache state owned by one dead volume without attempting writeback.
void MmFileCacheDiscardVolume(IO_VOLUME* volume) {
if (!volume) return;
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
while (true) {
MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
bool pending = false;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) {
if (!e->File || e->File->Volume != volume)
continue;
if (e->Loading || e->Writeback) {
e->Invalidated = 1;
if (e->Dirty)
KiFileCacheSetDirtyLocked(e, 0);
pending = true;
continue;
}
victim = e;
KiFileCacheRemoveEntryLocked(b, prev, e);
gFileCacheStats.Invalidations++;
break;
}
KeReleaseSpinlock(&gFileCacheLock, flags);
if (victim) {
KiFileCacheFreeEntry(victim);
continue;
}
if (!pending)
break;
// Loading/writeback entries unlink themselves when their in-flight operation completes.
break;
}
}
}
KSTATUS MmFileCacheDropOnePageForPressure(void) {
return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0);
}
code/kernel/mm_pool.c
+18 -18
View File
@@ -68,19 +68,19 @@ _Static_assert((sizeof(BLOCK_FTR) % POOL_ALIGNMENT) == 0, "pool footer must pres
// Implements the hdr To Ftr operation.
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
return (BLOCK_FTR*)((char*)h + HDR_SIZE + h->Size);
}
// Implements the ftr To Hdr operation.
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
return (BLOCK_HDR*)((char*)f - HDR_SIZE - f->Size);
}
// Implements the next Block operation.
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
return (BLOCK_HDR*)((char*)h + HDR_SIZE + h->Size + FTR_SIZE);
}
// Implements the prev Ftr operation.
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
return (BLOCK_FTR*)((char*)h - FTR_SIZE);
}
typedef struct HEAP HEAP;
@@ -268,7 +268,7 @@ static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
return 0;
}
mappedEnd = va + PAGE_SIZE;
memset((void *)va, 0, PAGE_SIZE);
memset((void*)va, 0, PAGE_SIZE);
}
// The grow path owns the only transition from reserved heap VA to mapped heap
@@ -300,12 +300,12 @@ static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
}
// Make the new pages a single free block.
BLOCK_HDR* blk = (BLOCK_HDR *)oldBrk;
BLOCK_HDR* blk = (BLOCK_HDR*)oldBrk;
blk->Magic = BLOCK_MAGIC;
blk->Free = BLOCK_STATE_FREE;
blk->Size = grow - OVERHEAD;
blk->PrevFree = (BLOCK_HDR *)0;
blk->NextFree = (BLOCK_HDR *)0;
blk->PrevFree = (BLOCK_HDR*)0;
blk->NextFree = (BLOCK_HDR*)0;
blk->RequestedSize = 0;
blk->Tag = 0;
blk->Flags = 0;
@@ -381,7 +381,7 @@ static void KiFreeListRemoveLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
else h->FreeList = blk->NextFree;
if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
blk->PrevFree = blk->NextFree = (BLOCK_HDR*)0;
}
// Implements the heap Try Decommit Tail operation.
@@ -548,7 +548,7 @@ static void KiHeapQuarantineBlockLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t calle
// Allocates a block from the heap.
static void* KiHeapAlloc(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t caller) {
if (!requestedSize) return (void *)0;
if (!requestedSize) return (void*)0;
uint32_t dbg = KiPoolDebugFlags();
size_t size = requestedSize;
@@ -564,7 +564,7 @@ static void* KiHeapAlloc(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
// Best-fit search.
BLOCK_HDR* best = (BLOCK_HDR *)0;
BLOCK_HDR* best = (BLOCK_HDR*)0;
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
KiPoolValidateFooterLocked(h, cur, "free list block corrupt");
if (cur->Free != BLOCK_STATE_FREE)
@@ -589,14 +589,14 @@ static void* KiHeapAlloc(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t
if (!best) {
if (!KiHeapGrow(h, size + OVERHEAD, caller)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
return (void*)0;
}
// Re-search; KiHeapGrow inserted at the front.
best = h->FreeList;
while (best && best->Size < size) best = best->NextFree;
if (!best) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
return (void*)0;
}
KiPoolValidateFooterLocked(h, best, "grown free block corrupt");
}
@@ -605,12 +605,12 @@ static void* KiHeapAlloc(HEAP* h, size_t requestedSize, uint32_t tag, uintptr_t
// Split the block if the remainder is big enough.
if (best->Size >= size + OVERHEAD + POOL_ALIGNMENT) {
BLOCK_HDR* rem = (BLOCK_HDR *)((char *)best + HDR_SIZE + size + FTR_SIZE);
BLOCK_HDR* rem = (BLOCK_HDR*)((char*)best + HDR_SIZE + size + FTR_SIZE);
rem->Magic = BLOCK_MAGIC;
rem->Free = BLOCK_STATE_FREE;
rem->Size = best->Size - size - OVERHEAD;
rem->PrevFree = (BLOCK_HDR *)0;
rem->NextFree = (BLOCK_HDR *)0;
rem->PrevFree = (BLOCK_HDR*)0;
rem->NextFree = (BLOCK_HDR*)0;
rem->RequestedSize = 0;
rem->Tag = 0;
rem->Flags = 0;
@@ -661,7 +661,7 @@ static void KiHeapFree(HEAP* h, void* ptr, uintptr_t caller) {
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid free pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
BLOCK_HDR* blk = (BLOCK_HDR*)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
@@ -708,7 +708,7 @@ static void* KiHeapRealloc(HEAP* h, void* ptr, size_t newSize, uintptr_t caller)
if (!h || u < h->Start + HDR_SIZE || u >= h->Brk)
KiPoolPanic("invalid realloc pointer outside heap", h, (BLOCK_HDR*)0, ptr);
BLOCK_HDR* blk = (BLOCK_HDR *)(u - HDR_SIZE);
BLOCK_HDR* blk = (BLOCK_HDR*)(u - HDR_SIZE);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (blk->Magic != BLOCK_MAGIC)
code/kernel/mm_pte.c
+4 -4
View File
@@ -4,23 +4,23 @@
#include <string.h>
#include <intrin.h>
#include <kernel/ke_hal.h>
#include <kernel/mm_pte.h>
#include <kernel/mm_pfn.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <kernel/hal/hal_apic.h>
// Page-table frames are reached through the physical direct map while the kernel still owns one shared CR3.
static uint64_t gPhysBase = PHYS_MAP_BASE;
// Converts a physical address to the kernel physical-window alias used for page-table editing.
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
return (void *)(p + gPhysBase);
return (void*)(p + gPhysBase);
}
// Returns the entry pointer for a page-table frame that is already mapped in the physical window.
static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
return (uint64_t*)KiPhysToVirt(tablePhys) + idx;
}
// The bootloader builds the initial PML4, and this layer owns all later edits to it.
@@ -154,7 +154,7 @@ void MmPteFlushKernelAddress(uintptr_t virt) {
continue;
targets++;
KiPteStatInc(&gPteStats.ShootdownTargetCpus);
HalApicSendTlbShootdownIpi(cpu->LapicId);
KeSendProcessorNmi(cpu->HardwareId);
}
if (!targets) {
code/kernel/mm_swap.c
+8 -7
View File
@@ -1,5 +1,5 @@
// Kernel swap manager.
// The first backend is a fixed-size pagefile accessed through VFS offset I/O.
// The first backend is a fixed-size pagefile accessed through system-volume file I/O.
// Kernel memory and filesystem code remain non-pageable, so this layer is safe
// to use before the VMM has direct block-extents for pageout.
@@ -12,6 +12,7 @@
#include <kernel/io_file.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_system_root.h>
typedef struct MM_SWAP_STATE {
SPINLOCK Lock;
@@ -70,8 +71,8 @@ static KSTATUS KiSwapAllocateSlot(uint32_t* slotOut) {
}
// Initializes the fixed pagefile backend.
KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
if (!path.Data || !path.Length || sizeBytes < PAGE_SIZE)
KSTATUS MmSwapInitialize(const char* relativePath, uint64_t sizeBytes) {
if (!relativePath || !relativePath[0] || sizeBytes < PAGE_SIZE)
return NTLSTATUS_INVALID_PARAMETER;
if (sizeBytes & (PAGE_SIZE - 1))
sizeBytes = (sizeBytes + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1);
@@ -84,9 +85,9 @@ KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
KeReleaseSpinlock(&gSwap.Lock, flags);
FILE_OBJECT* file = (FILE_OBJECT*)0;
KSTATUS s = IoOpenFile(path, &file);
KSTATUS s = KeOpenSystemRootFile(relativePath, &file);
if (!KSUCCESS(s)) {
s = IoCreateFile(path, &file);
s = KeCreateSystemRootFile(relativePath, &file);
if (!KSUCCESS(s))
return s;
}
@@ -137,10 +138,10 @@ KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
gSwap.Stats.FreeSlots = slotCount;
KeReleaseSpinlock(&gSwap.Lock, flags);
DbgPrint("swap: initialized %u slots (%llu KiB) at %v\n",
DbgPrint("swap: initialized %u slots (%llu KiB) at system-root/%s\n",
slotCount,
(unsigned long long)(sizeBytes / 1024),
path);
relativePath);
return NTLSTATUS_SUCCESS;
}
code/kernel/mm_virt.c
+61 -3
View File
@@ -443,7 +443,12 @@ static void KiVadSyncSharedFilePageBeforeRelease(MM_VAD* vad, size_t index) {
if ((page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE) == 0)
return;
uintptr_t va = vad->Base + index * PAGE_SIZE;
if (page->Resident && page->Frame && MmPteTestAndClearDirty(va))
if (!page->Resident || !page->Frame)
return;
int wasDirty = MmPteTestAndClearDirty(va);
int wasWriteEnabled = (page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0;
if (wasDirty || wasWriteEnabled)
(void)KiVadMarkSharedFilePageDirty(vad, index, 0);
}
@@ -1361,6 +1366,58 @@ static KSTATUS KiCopyToUserPaged(void* userDst, const void* kernelSrc, size_t si
return KiCopyUserPaged(userDst, kernelSrc, size, 1);
}
// Copies between process-owned user ranges without bouncing through a kernel payload buffer.
__declspec(dllexport) KSTATUS MmCopyUserToUser(PROCESS* sourceProcess, const void* source,
PROCESS* targetProcess, void* target, size_t size) {
if (size == 0)
return NTLSTATUS_SUCCESS;
if (!sourceProcess || !targetProcess || !source || !target)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t sourceAddress = (uintptr_t)source;
uintptr_t targetAddress = (uintptr_t)target;
if (!KiUserRangeValid(sourceAddress, size) || !KiUserRangeValid(targetAddress, size))
return NTLSTATUS_ACCESS_DENIED;
size_t done = 0;
while (done < size) {
MM_PINNED_USER_PAGE sourcePage;
MM_PINNED_USER_PAGE targetPage;
memset(&sourcePage, 0, sizeof(sourcePage));
memset(&targetPage, 0, sizeof(targetPage));
KSTATUS status = MmPinUserPageForRead(sourceProcess,
(const void*)(sourceAddress + done),
&sourcePage);
if (!KSUCCESS(status))
return status;
status = MmPinUserPageForWrite(targetProcess,
(void*)(targetAddress + done),
&targetPage);
if (!KSUCCESS(status)) {
MmUnpinUserPage(&sourcePage);
return status;
}
size_t sourceRemaining = PAGE_SIZE - sourcePage.PageOffset;
size_t targetRemaining = PAGE_SIZE - targetPage.PageOffset;
size_t chunk = size - done;
if (chunk > sourceRemaining) chunk = sourceRemaining;
if (chunk > targetRemaining) chunk = targetRemaining;
memcpy((uint8_t*)targetPage.KernelAddress + targetPage.PageOffset,
(const uint8_t*)sourcePage.KernelAddress + sourcePage.PageOffset,
chunk);
MmUnpinUserPage(&targetPage);
MmUnpinUserPage(&sourcePage);
done += chunk;
}
return NTLSTATUS_SUCCESS;
}
// Reserves a VAD in the global user VA arena without committing physical pages.
KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t vadFlags,
@@ -2158,10 +2215,11 @@ KSTATUS MmVadSyncFileMappings(FILE_OBJECT* file) {
uintptr_t va = vad->Base + i * PAGE_SIZE;
int wasDirty = MmPteTestAndClearDirty(va);
if (wasDirty)
int wasWriteEnabled = (page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0;
if (wasDirty || wasWriteEnabled)
(void)KiVadMarkSharedFilePageDirty(vad, i, 0);
if ((page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0) {
if (wasWriteEnabled) {
page->Flags |= MM_VAD_PAGE_FILE_DIRTY_TRACK;
(void)MmPteProtectRange(va, PAGE_SIZE, KiVadHardwarePageFlags(page));
}
code/kernel/net_socket.c
+6 -6
View File
@@ -106,7 +106,7 @@ struct NET_ASYNC_SOCKET_REQUEST_OBJECT {
size_t OptionValueSize;
size_t* OptionActualSizeOut;
uint32_t* HandleOut;
SYS_HANDLE* HandleOut;
uint32_t* PollReturnedEventsOut;
MM_LOCKED_USER_BUFFER* LockedAddressOut;
@@ -277,11 +277,11 @@ static void NetAsyncSocketExecute(NET_ASYNC_SOCKET_REQUEST_OBJECT* req,
memset(&remote, 0, sizeof(remote));
status = NetSocketAccept(req->Socket, &accepted, req->AddressOut ? &remote : (SYS_SOCKET_ADDRESS_K*)0, &req->Async);
if (KSUCCESS(status)) {
uint32_t handle = 0;
status = KeAllocateHandleWithRights(req->OwnerProcess,
&accepted->Header,
HANDLE_RIGHT_READ | HANDLE_RIGHT_WRITE | HANDLE_RIGHT_QUERY,
&handle);
SYS_HANDLE handle = SYS_HANDLE_INVALID;
status = KeCreateHandle(req->OwnerProcess,
&accepted->Header,
KeGetValidHandleRights(ObjectTypeSocket),
&handle);
ObDereferenceObject(&accepted->Header);
if (KSUCCESS(status)) {
if (req->HandleOut)
code/kernel/ob_object.c
+4 -1
View File
@@ -29,10 +29,13 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
case ObjectTypeInputQueue: return MM_POOL_TAG('i', 'n', 'p', 'q');
case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' ');
case ObjectTypeModule: return MM_POOL_TAG('m', 'o', 'd', 'u');
case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k');
case ObjectTypeAudioEndpoint: return MM_POOL_TAG('a', 'e', 'p', 't');
case ObjectTypeAudioStream: return MM_POOL_TAG('a', 's', 't', 'r');
case ObjectTypeSecurityContext: return MM_POOL_TAG('s', 'e', 'c', 'x');
case ObjectTypePort: return MM_POOL_TAG('p', 'o', 'r', 't');
case ObjectTypeVolume: return MM_POOL_TAG('v', 'o', 'l', 'm');
case ObjectTypeVolumeManager: return MM_POOL_TAG('v', 'm', 'g', 'r');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}
code/kernel/se_security.c
+130
View File
@@ -0,0 +1,130 @@
// Kernel security contexts. Contexts are immutable authority sets carried by processes and delegated only by attenuation.
#include <string.h>
#include <kernel/se_security.h>
#include <kernel/ke_id.h>
#include <kernel/ke_process.h>
#include <kernel/ob_object.h>
static KE_ID_ALLOCATOR gSecurityIds = KE_ID_ALLOCATOR_INIT(1u);
static SECURITY_CONTEXT* gRootSecurityContext;
// Destroys one non-root security context after its last object reference disappears.
static void KiSecurityContextDestroy(OBJECT* object) {
(void)object;
}
// Allocates and initializes one immutable security context object.
static KSTATUS KiCreateSecurityContext(SYS_PRIVILEGES effectivePrivileges,
SYS_PRIVILEGES delegablePrivileges,
SYS_SERVICE_CLAIMS serviceClaims,
SYS_SERVICE_CLAIMS delegableServiceClaims,
SECURITY_CONTEXT** contextOut) {
if (!contextOut)
return NTLSTATUS_INVALID_PARAMETER;
*contextOut = (SECURITY_CONTEXT*)0;
if ((effectivePrivileges & ~SYS_PRIV_ALL) != 0 ||
(delegablePrivileges & ~SYS_PRIV_ALL) != 0 ||
(serviceClaims & ~SYS_SERVICE_CLAIM_ALL) != 0 ||
(delegableServiceClaims & ~SYS_SERVICE_CLAIM_ALL) != 0)
return NTLSTATUS_INVALID_PARAMETER;
if ((delegablePrivileges & ~effectivePrivileges) != 0 ||
(delegableServiceClaims & ~serviceClaims) != 0)
return NTLSTATUS_ACCESS_DENIED;
SECURITY_CONTEXT* context = (SECURITY_CONTEXT*)ObAllocateObject(ObjectTypeSecurityContext,
sizeof(SECURITY_CONTEXT));
if (!context)
return NTLSTATUS_NO_MEMORY;
context->Header.Destroy = KiSecurityContextDestroy;
KSTATUS status = KeAllocateId32(&gSecurityIds, &context->SecurityId);
if (!KSUCCESS(status)) {
ObDereferenceObject(&context->Header);
return status;
}
context->EffectivePrivileges = effectivePrivileges;
context->DelegablePrivileges = delegablePrivileges;
context->ServiceClaims = serviceClaims;
context->DelegableServiceClaims = delegableServiceClaims;
context->Flags = SYS_SECURITY_CONTEXT_FLAG_NONE;
*contextOut = context;
return NTLSTATUS_SUCCESS;
}
// Initializes the root context exactly once before any process is created.
KSTATUS SeInitializeSecurity(void) {
if (gRootSecurityContext)
return NTLSTATUS_SUCCESS;
SECURITY_CONTEXT* context = (SECURITY_CONTEXT*)0;
KSTATUS status = KiCreateSecurityContext(SYS_PRIV_ALL,
SYS_PRIV_ALL,
SYS_SERVICE_CLAIM_ALL,
SYS_SERVICE_CLAIM_ALL,
&context);
if (!KSUCCESS(status))
return status;
gRootSecurityContext = context;
return NTLSTATUS_SUCCESS;
}
// Returns the permanently referenced bootstrap root context.
SECURITY_CONTEXT* SeGetRootSecurityContext(void) {
return gRootSecurityContext;
}
// Derives one child context whose effective and delegable authority are subsets of the source delegable sets.
KSTATUS SeDeriveSecurityContext(SECURITY_CONTEXT* source,
SYS_PRIVILEGES effectivePrivileges,
SYS_PRIVILEGES delegablePrivileges,
SYS_SERVICE_CLAIMS serviceClaims,
SYS_SERVICE_CLAIMS delegableServiceClaims,
SECURITY_CONTEXT** contextOut) {
if (!source || !contextOut)
return NTLSTATUS_INVALID_PARAMETER;
if ((effectivePrivileges & ~source->DelegablePrivileges) != 0 ||
(delegablePrivileges & ~source->DelegablePrivileges) != 0 ||
(serviceClaims & ~source->DelegableServiceClaims) != 0 ||
(delegableServiceClaims & ~source->DelegableServiceClaims) != 0)
return NTLSTATUS_ACCESS_DENIED;
return KiCreateSecurityContext(effectivePrivileges,
delegablePrivileges,
serviceClaims,
delegableServiceClaims,
contextOut);
}
// Tests one process's immutable effective privilege mask.
bool SeProcessHasPrivilege(const struct PROCESS* process, SYS_PRIVILEGES privileges) {
if (!process || !process->SecurityContext)
return false;
return (process->SecurityContext->EffectivePrivileges & privileges) == privileges;
}
// Tests one security context's effective service-claim mask.
bool SeContextHasServiceClaims(const SECURITY_CONTEXT* context, SYS_SERVICE_CLAIMS claims) {
if (!context)
return false;
return (context->ServiceClaims & claims) == claims;
}
// Exports immutable context metadata to one public structure.
void SeQuerySecurityContext(const SECURITY_CONTEXT* context, SYS_SECURITY_CONTEXT_INFO* infoOut) {
if (!context || !infoOut)
return;
memset(infoOut, 0, sizeof(*infoOut));
infoOut->Size = sizeof(*infoOut);
infoOut->SecurityId = context->SecurityId;
infoOut->EffectivePrivileges = context->EffectivePrivileges;
infoOut->DelegablePrivileges = context->DelegablePrivileges;
infoOut->ServiceClaims = context->ServiceClaims;
infoOut->DelegableServiceClaims = context->DelegableServiceClaims;
infoOut->Flags = context->Flags;
}