maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

rewrite comments into one, unified style

Browse Source
This commit is contained in:
maksw committed 2026-06-15 10:17:48 +02:00
1 parent 6260de0085
commit 467a58c9c6
163 files changed
+3560 -2556
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/hal/hal_apic.c
+24 -3
View File
@@ -1,4 +1,5 @@
// hal_apic.c - Local APIC + I/O APIC
// HAL APIC support. This file owns local APIC, I/O APIC, and interrupt-controller setup.
#include <kernel/hal/hal_apic.h>
#include <ntlite/ddi/hal_api.h>
#include <stdint.h>
@@ -6,7 +7,7 @@
extern HAL_API gApi;
// ---- Local APIC ----
// ---- Local APIC ----.
static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
#define LAPIC_ID 0x020
@@ -28,7 +29,7 @@ static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define APIC_VECTOR_RESCHEDULE 0xF0u
// ---- I/O APIC ----
// ---- I/O APIC ----.
static volatile uint32_t* gIoapic = (volatile uint32_t *)0;
static uint32_t gBspLapicId = 0;
static uint8_t gX2ApicSupported = 0;
@@ -36,15 +37,18 @@ static uint8_t gX2ApicEnabled = 0;
#define IOAPIC_IDX 0x00
#define IOAPIC_DATA 0x10
// Implements the ioapic Read operation.
static uint32_t IoapicRead(uint8_t reg) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg;
return *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA);
}
// Implements the ioapic Write operation.
static void IoapicWrite(uint8_t reg, uint32_t val) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg;
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val;
}
// Implements the apic Detect Mode operation.
static void HalApicDetectMode(void) {
int regs[4] = {0, 0, 0, 0};
__cpuid(regs, 1);
@@ -54,17 +58,20 @@ static void HalApicDetectMode(void) {
gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u;
}
// Implements the apic Stall Fallback operation.
static void HalApicStallFallback(uint32_t loops) {
for (volatile uint32_t i = 0; i < loops; i++) {
_mm_pause();
}
}
// Implements the apic Stall Us operation.
static void HalApicStallUs(uint32_t us) {
if (gApi.Timer.Stall) gApi.Timer.Stall(us);
else HalApicStallFallback(us * 128u + 1u);
}
// Implements the apic Wait Icr Idle operation.
static void HalApicWaitIcrIdle(void) {
if (gX2ApicEnabled)
return;
@@ -76,6 +83,7 @@ static void HalApicWaitIcrIdle(void) {
}
}
// Implements the apic Send Ipi operation.
static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low);
@@ -94,20 +102,24 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
#define IRQ_VECTOR_BASE 0x20
// Returns bsp Lapic Id.
__declspec(dllexport) uint32_t HalGetBspLapicId(void) {
return gBspLapicId;
}
// Implements the apic Using X2 Apic operation.
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void) {
return gX2ApicEnabled;
}
// Implements the apic Can Address Id operation.
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId) {
if (gX2ApicEnabled)
return 1u;
return (lapicId < 256u) ? 1u : 0u;
}
// Implements the apic Enable Current Cpu operation.
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE;
@@ -123,6 +135,7 @@ __declspec(dllexport) void HalApicEnableCurrentCpu(void) {
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
}
// Implements the apic Send Panic Ipi operation.
__declspec(dllexport) void HalApicSendPanicIpi(void) {
if (!gLapic && !gX2ApicEnabled) return;
@@ -133,11 +146,13 @@ __declspec(dllexport) void HalApicSendPanicIpi(void) {
HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI
}
// Implements the apic Send Reschedule Ipi operation.
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) {
if (!gLapic && !gX2ApicEnabled) return;
HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination
}
// Implements the apic Send Tlb Shootdown Ipi operation.
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
if (!gLapic && !gX2ApicEnabled) return;
@@ -148,6 +163,7 @@ __declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination
}
// Starts application Processor.
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) {
if ((!gLapic && !gX2ApicEnabled) || !startupVector || lapicId == gBspLapicId)
return STATUS_INVALID_PARAMETER;
@@ -166,6 +182,7 @@ __declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uin
return STATUS_SUCCESS;
}
// Implements the apic Route Irq operation.
void HalApicRouteIrq(uint8_t irq) {
// Full initial setup of the redirection entry.
// Sets the vector and clears the mask bit in one write.
@@ -174,12 +191,14 @@ void HalApicRouteIrq(uint8_t irq) {
IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed
}
// Implements the apic Mask Irq operation.
void HalApicMaskIrq(uint8_t irq) {
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
lo |= (1 << 16);
IoapicWrite(IOREDTBL_LO(irq), lo);
}
// Implements the apic Unmask Irq operation.
void HalApicUnmaskIrq(uint8_t irq) {
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
lo &= ~(1 << 16); // clear mask bit
@@ -188,6 +207,7 @@ void HalApicUnmaskIrq(uint8_t irq) {
IoapicWrite(IOREDTBL_LO(irq), lo);
}
// Implements the apic Send Eoi operation.
void HalApicSendEoi(void) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_EOI, 0);
@@ -196,6 +216,7 @@ void HalApicSendEoi(void) {
LAPIC_REG(LAPIC_EOI) = 0;
}
// Implements the apic Initialize operation.
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
HalApicDetectMode();
code/kernel/hal/hal_disk.c
+2 -2
View File
@@ -1,2 +1,2 @@
// hal_disk.c intentionally left empty.
// Disk registration and dispatch live in kernel/io_device.c.
// HAL disk stub support. This file preserves the old HAL disk API while storage lives in drivers.
code/kernel/hal/hal_keyboard.c
+2 -1
View File
@@ -1 +1,2 @@
// hal_keyboard.c - input moved out of HAL into kernel input core and ps2.sys.
// HAL keyboard stub support. Keyboard input is handled by the kernel input core and ps2.sys.
code/kernel/hal/hal_main.c
+34 -23
View File
@@ -1,4 +1,5 @@
// hal_main.c - HAL initialisation and HalGetApi
// HAL initialization entry point. This file discovers platform tables and publishes the HAL API to the kernel.
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <string.h>
@@ -23,15 +24,13 @@ HAL_API gApi;
static int gInitialised = 0;
extern void (*gTimerCallback)(void);
// ---------------------------------------------------------------------------
// ACPI table walker - minimal, just enough to find APIC and HPET addresses.
// ---------------------------------------------------------------------------
#define ACPI_SIG(a,b,c,d) ((uint32_t)(a)|((uint32_t)(b)<<8)|((uint32_t)(c)<<16)|((uint32_t)(d)<<24))
#pragma pack(push, 1)
#pragma warning(disable : 4200)
// ACPI RSDP (Root System Description Pointer)
// ACPI RSDP (Root System Description Pointer).
typedef struct {
char Signature[8];
uint8_t Checksum;
@@ -44,7 +43,7 @@ typedef struct {
uint8_t Padding[3];
} RSDP;
// ACPI SDT Header
// ACPI SDT Header.
typedef struct {
char Signature[4];
uint32_t Length;
@@ -57,19 +56,19 @@ typedef struct {
uint32_t CreatorRevision;
} ACPISDTHeader;
// RSDT (Root System Description Table)
// RSDT (Root System Description Table).
typedef struct {
ACPISDTHeader Header;
uint32_t PointerToOtherSDT[];
} RSDT;
// XSDT (Extended System Description Table)
// XSDT (Extended System Description Table).
typedef struct {
ACPISDTHeader Header;
uint64_t PointerToOtherSDT[];
} XSDT;
// MADT (Multiple APIC Description Table)
// MADT (Multiple APIC Description Table).
typedef struct {
ACPISDTHeader Header;
uint32_t LocalAPICAddress;
@@ -94,9 +93,11 @@ static int gS5Found = 0;
int debugEnabled = 0;
int _fltused; // MSVC floating-point helper anchor
// Implements the hal putchar operation.
static void hal_putchar(char c) {
gApi.Screen.PrintChar(c, 0xCCCCCC, 0x012456);
}
// Implements the dbg Print operation.
void HalDbgPrint(const char* fmt, ...) {
if (!debugEnabled) return;
@@ -131,6 +132,7 @@ typedef struct {
} ACPI_GAS;
#pragma pack(pop)
// Implements the acpi Read U32 operation.
static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) {
uint32_t value = 0;
if (off + sizeof(value) <= tableLen)
@@ -138,6 +140,7 @@ static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off)
return value;
}
// Implements the acpi Read U64 operation.
static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off) {
uint64_t value = 0;
if (off + sizeof(value) <= tableLen)
@@ -145,6 +148,7 @@ static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off)
return value;
}
// Implements the acpi Read Gas Io Port operation.
static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t off) {
ACPI_GAS gas;
if (off + sizeof(gas) > tableLen) return 0;
@@ -214,6 +218,7 @@ static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
return p;
}
// Implements the aml Read Integer operation.
static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
const uint8_t* p = *pp;
if (p >= end) return 0;
@@ -290,6 +295,7 @@ static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
HalDbgPrint("HAL: _S5_ not found in DSDT; ACPI shutdown disabled\n");
}
// Validates checksum.
static int validate_checksum(void* table, uint32_t length) {
uint8_t sum = 0;
uint8_t* bytes = (uint8_t*)table;
@@ -299,6 +305,7 @@ static int validate_checksum(void* table, uint32_t length) {
return sum == 0;
}
// Parses fadt.
static void ParseFadt(const ACPISDTHeader* header) {
const void* fadt = header;
uint32_t len = header->Length;
@@ -326,6 +333,7 @@ static void ParseFadt(const ACPISDTHeader* header) {
if (dsdtPhys) {
const ACPISDTHeader* dsdt = (const ACPISDTHeader*)(dsdtPhys + gPhysBase);
if (memcmp(dsdt->Signature, "DSDT", 4) == 0 &&
// Validates checksum.
validate_checksum((void*)dsdt, dsdt->Length)) {
const uint8_t* aml = (const uint8_t*)(dsdt + 1);
ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader));
@@ -336,6 +344,7 @@ static void ParseFadt(const ACPISDTHeader* header) {
}
// Implements the add Cpu Record operation.
static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
if (gCpuTopology.CpuCount >= HAL_MAX_CPUS)
return;
@@ -353,6 +362,7 @@ static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
cpu->Enabled = (flags & 1u) ? 1u : 0u;
}
// Parses madt.
static void ParseMadt(MADT* madt) {
gLapicPhys = madt->LocalAPICAddress;
uint8_t* p = (uint8_t *)(madt + 1);
@@ -383,10 +393,11 @@ static void ParseMadt(MADT* madt) {
}
}
// Implements the walk Acpi operation.
static void WalkAcpi(uintptr_t rsdpPhys) {
if (!rsdpPhys) return;
// Map RSDP into virtual memory
// Map RSDP into virtual memory.
RSDP* rsdp = (RSDP*)(rsdpPhys + gPhysBase);
if (memcmp(rsdp->Signature, "RSD PTR ", 8) != 0) {
@@ -398,11 +409,12 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
uint32_t count = 0;
int isXsdt = 0;
// Safely find and validate the XSDT (ACPI 2.0+)
// Safely find and validate the XSDT (ACPI 2.0+).
if (rsdp->Revision >= 2 && rsdp->XsdtAddress != 0) {
XSDT* xsdt = (XSDT*)(rsdp->XsdtAddress + gPhysBase);
if (validate_checksum(xsdt, xsdt->Header.Length) &&
// Implements the memcmp operation.
memcmp(xsdt->Header.Signature, "XSDT", 4) == 0) {
sdtHeader = &xsdt->Header;
@@ -411,11 +423,12 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
}
}
// Fallback to safely finding and validating the RSDT (ACPI 1.0)
// Fallback to safely finding and validating the RSDT (ACPI 1.0).
if (sdtHeader == NULL && rsdp->RsdtAddress != 0) {
RSDT* rsdt = (RSDT*)((uintptr_t)rsdp->RsdtAddress + gPhysBase);
if (validate_checksum(rsdt, rsdt->Header.Length) &&
// Implements the memcmp operation.
memcmp(rsdt->Header.Signature, "RSDT", 4) == 0) {
sdtHeader = &rsdt->Header;
@@ -429,7 +442,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
return;
}
// Walk the tables
// Walk the tables.
for (uint32_t i = 0; i < count; i++) {
uintptr_t tablePhys = 0;
@@ -439,34 +452,32 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
tablePhys = ((RSDT*)sdtHeader)->PointerToOtherSDT[i];
}
// Paranoid alignment check to reject firmware garbage pointers
// Paranoid alignment check to reject firmware garbage pointers.
if (!tablePhys || (tablePhys & 3) != 0) continue;
ACPISDTHeader* header = (ACPISDTHeader*)(tablePhys + gPhysBase);
// Ensure the table isn't corrupt before parsing
// Ensure the table isn't corrupt before parsing.
if (!validate_checksum(header, header->Length)) continue;
// Parse MADT
// Parse MADT.
if (memcmp(header->Signature, "APIC", 4) == 0) {
ParseMadt((MADT*)header);
}
// Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer
// Parse FADT (signature "FACP") for PM1 control ports and DSDT pointer.
else if (memcmp(header->Signature, "FACP", 4) == 0) {
ParseFadt(header);
}
// Parse HPET
// Parse HPET.
else if (memcmp(header->Signature, "HPET", 4) == 0) {
// HPET base address block is located at offset 44
// (which is 8 bytes past the 36-byte ACPISDTHeader)
// (which is 8 bytes past the 36-byte ACPISDTHeader).
gHpetPhys = *(uint64_t*)((uint8_t*)header + 44) & ~0xFFFULL;
}
}
}
// ---------------------------------------------------------------------------
// HalShutdown - ACPI S5 soft power-off. Does not return.
// ---------------------------------------------------------------------------
static void HalShutdown(void) {
// Silence! Stop the world!
_disable();
@@ -477,7 +488,7 @@ static void HalShutdown(void) {
while (true) __halt();
}
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10)
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10).
uint16_t valA = (uint16_t)(((uint16_t)gSlpTypA << 10) | (1 << 13));
uint16_t valB = (uint16_t)(((uint16_t)gSlpTypB << 10) | (1 << 13));
@@ -488,7 +499,6 @@ static void HalShutdown(void) {
while (true) __halt();
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
if (!out) return STATUS_INVALID_PARAMETER;
@@ -526,7 +536,6 @@ __declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
if (gInitialised) return STATUS_ALREADY_EXISTS;
gPhysBase = initParams->PhysicalWindowBase;
@@ -585,8 +594,10 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
return STATUS_SUCCESS;
}
// Registers scheduler.
__declspec(dllexport) void HalRegisterScheduler(void (*pKeSchedulerTick)(void)) {
gTimerCallback = pKeSchedulerTick;
}
// Returns api.
__declspec(dllexport) const HAL_API* HalGetApi(void) { return &gApi; }
code/kernel/hal/hal_mouse.c
+2 -1
View File
@@ -1 +1,2 @@
// hal_mouse.c - input moved out of HAL into kernel input core and ps2.sys.
// HAL mouse stub support. Mouse input is handled by the kernel input core and ps2.sys.
code/kernel/hal/hal_pci.c
+4 -1
View File
@@ -1,10 +1,12 @@
// hal_pci.c - raw PCI configuration access via legacy Mechanism 1 port I/O
// HAL PCI configuration access. This file owns legacy mechanism-1 PCI config port I/O.
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#define PCI_CONFIG_ADDR 0xCF8
#define PCI_CONFIG_DATA 0xCFC
// Creates addr.
static uint32_t MakeAddr(uint8_t bus, uint8_t dev, uint8_t fn, uint16_t offset) {
return 0x80000000UL
| ((uint32_t)bus << 16)
@@ -38,6 +40,7 @@ static void PciWrite32(uint16_t segment,
__outdword(PCI_CONFIG_DATA, value);
}
// Implements the pci Initialize operation.
void HalPciInitialize(HAL_PCI* out) {
out->Read32 = PciRead32;
out->Write32 = PciWrite32;
code/kernel/hal/hal_rtc.c
+12 -1
View File
@@ -1,4 +1,5 @@
// hal_rtc.c - legacy CMOS RTC fallback clock
// HAL RTC support. This file reads the CMOS clock as the fallback wall-clock source.
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
@@ -22,19 +23,23 @@
static uint8_t gCenturyRegister;
// Implements the rtc Read operation.
static uint8_t RtcRead(uint8_t reg) {
__outbyte(CMOS_INDEX_PORT, (uint8_t)(reg | 0x80u)); // keep NMI masked while selecting.
return __inbyte(CMOS_DATA_PORT);
}
// Implements the rtc Update In Progress operation.
static bool RtcUpdateInProgress(void) {
return (RtcRead(RTC_REG_STATUS_A) & 0x80u) != 0;
}
// Implements the rtc Bcd To Binary operation.
static uint8_t RtcBcdToBinary(uint8_t value) {
return (uint8_t)(((value >> 4) * 10u) + (value & 0x0Fu));
}
// Implements the rtc Read Stable operation.
static bool RtcReadStable(HAL_RTC_TIME* out) {
if (!out) return false;
@@ -111,10 +116,12 @@ static bool RtcReadStable(HAL_RTC_TIME* out) {
return true;
}
// Implements the rtc Leap Year operation.
static bool RtcLeapYear(uint16_t year) {
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
}
// Implements the rtc Days In Month operation.
static uint8_t RtcDaysInMonth(uint16_t year, uint8_t month) {
static const uint8_t days[12] = { 31,28,31,30,31,30,31,31,30,31,30,31 };
if (month < 1u || month > 12u) return 0;
@@ -122,6 +129,7 @@ static uint8_t RtcDaysInMonth(uint16_t year, uint8_t month) {
return days[month - 1u];
}
// Implements the rtc Valid operation.
static bool RtcValid(const HAL_RTC_TIME* t) {
if (!t) return false;
if (t->Year < 1970u || t->Year > 9999u) return false;
@@ -132,6 +140,7 @@ static bool RtcValid(const HAL_RTC_TIME* t) {
return true;
}
// Implements the rtc Query Time operation.
static KSTATUS HalRtcQueryTime(HAL_RTC_TIME* outTime) {
if (!outTime) return STATUS_INVALID_PARAMETER;
@@ -144,10 +153,12 @@ static KSTATUS HalRtcQueryTime(HAL_RTC_TIME* outTime) {
return STATUS_SUCCESS;
}
// Implements the rtc Set Century Register operation.
void HalRtcSetCenturyRegister(uint8_t centuryRegister) {
gCenturyRegister = centuryRegister;
}
// Implements the rtc Initialize operation.
void HalRtcInitialize(HAL_RTC* out) {
if (!out) return;
out->QueryTime = HalRtcQueryTime;
code/kernel/hal/hal_screen.c
+20 -21
View File
@@ -1,5 +1,5 @@
// hal_screen.c - GOP framebuffer screen driver with embedded 8x16 font
//
// HAL screen support. This file owns GOP framebuffer drawing before a runtime display provider takes over.
// The 8x16 bitmap font is a standard VGA/PC font. We embed just the
// printable ASCII range (0x20–0x7E). Each glyph is 16 bytes; bit 7 of
// each byte is the leftmost pixel.
@@ -22,18 +22,15 @@ static int gCursorX = 0, gCursorY = 0;
#define SCREEN_DEFAULT_FG 0xCCCCCCu
#define SCREEN_DEFAULT_BG 0x012456u
// ---------------------------------------------------------------------------
// 8x16 VGA font - printable ASCII only (glyph 0 = space = 0x20)
// ---------------------------------------------------------------------------
// 8x16 VGA font - printable ASCII only (glyph 0 = space = 0x20).
#include <kernel/hal/font8x16.h> // see font8x16.h
// ---------------------------------------------------------------------------
// Internals
// ---------------------------------------------------------------------------
// Internals.
static inline uint32_t* ScreenBackbufferPixel(int x, int y) {
return gBackbuffer ? (gBackbuffer + y * gStride + x) : (gFb + y * gStride + x);
}
// Updates pixel Raw.
static inline void PutPixelRaw(int x, int y, uint32_t color) {
if ((unsigned)x >= (unsigned)gWidth || (unsigned)y >= (unsigned)gHeight)
return;
@@ -42,6 +39,7 @@ static inline void PutPixelRaw(int x, int y, uint32_t color) {
gFb[y * gStride + x] = color;
}
// Implements the screen Flush Rows operation.
static void ScreenFlushRows(int startY, int rows) {
if (!gBackbuffer || rows <= 0)
return;
@@ -60,9 +58,7 @@ static void ScreenFlushRows(int startY, int rows) {
(size_t)rows * rowBytes);
}
// ---------------------------------------------------------------------------
// HAL_SCREEN implementations
// ---------------------------------------------------------------------------
// HAL_SCREEN implementations.
static void ScreenClear(uint32_t color) {
for (int y = 0; y < gHeight; y++)
for (int x = 0; x < gWidth; x++)
@@ -77,12 +73,14 @@ static void ScreenClear(uint32_t color) {
gCursorX = 0; gCursorY = 0;
}
// Implements the screen Put Pixel operation.
static void ScreenPutPixel(int x, int y, uint32_t color) {
PutPixelRaw(x, y, color);
}
// Implements the screen Blit operation.
static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
// Calculate clipping bounds
// Calculate clipping bounds.
int startX = (x < 0) ? 0 : x;
int startY = (y < 0) ? 0 : y;
int endX = (x + w > gWidth) ? gWidth : (x + w);
@@ -91,16 +89,16 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
int drawW = endX - startX;
int drawH = endY - startY;
// If the image is entirely off-screen, do nothing
// If the image is entirely off-screen, do nothing.
if (drawW <= 0 || drawH <= 0) {
return;
}
// Adjust the source pointer if the image was clipped on the top or left
// Adjust the source pointer if the image was clipped on the top or left.
int srcOffsetX = startX - x;
int srcOffsetY = startY - y;
// Calculate bytes to copy per row
// Calculate bytes to copy per row.
size_t bytesPerRow = (size_t)drawW * sizeof(uint32_t);
for (int row = 0; row < drawH; row++) {
@@ -113,8 +111,10 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
}
}
// Implements the screen Get Size operation.
static void ScreenGetSize(int* w, int* h) { *w = gWidth; *h = gHeight; }
// Implements the screen Get Framebuffer operation.
static KSTATUS ScreenGetFramebuffer(HAL_FRAMEBUFFER_INFO* outInfo) {
if (!outInfo || !gFb || gWidth <= 0 || gHeight <= 0 || gStride <= 0)
return STATUS_INVALID_PARAMETER;
@@ -127,6 +127,7 @@ static KSTATUS ScreenGetFramebuffer(HAL_FRAMEBUFFER_INFO* outInfo) {
return STATUS_SUCCESS;
}
// Implements the screen Scroll operation.
static void ScreenScroll(int lines) {
if (lines <= 0)
return;
@@ -152,6 +153,7 @@ static void ScreenScroll(int lines) {
ScreenFlushRows(0, gHeight);
}
// Implements the screen Put Char operation.
static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16);
for (int row = 0; row < 16; row++) {
@@ -163,11 +165,12 @@ static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
}
}
// Implements the screen Print Char operation.
static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
int cols = gWidth / FONT_W;
int rows = gHeight / FONT_H;
// Mirror to serial
// Mirror to serial.
while ((__inbyte(0x3FD) & 0x20) == 0);
if (c == '\n') __outbyte(0x3F8, '\r');
__outbyte(0x3F8, c);
@@ -194,11 +197,9 @@ static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
}
}
// ---------------------------------------------------------------------------
// Mouse cursor – 12×19 monochrome arrow sprite
// 0 = transparent 1 = black outline 2 = white fill
// The hotspot is at (0,0) (top-left pixel = tip of the arrow).
// ---------------------------------------------------------------------------
#define CURSOR_W 16
#define CURSOR_H 25
@@ -317,9 +318,7 @@ void HalScreenEraseCursor(void) {
gCursorDrawY = -1;
}
// ---------------------------------------------------------------------------
// Populate the HAL_SCREEN vtable
// ---------------------------------------------------------------------------
// Populate the HAL_SCREEN vtable.
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out) {
gFb = (uint32_t *)fb;
code/kernel/hal/hal_timer.c
+24 -16
View File
@@ -1,4 +1,5 @@
// hal_timer.c - Local APIC timer
// HAL timer support. This file owns APIC timer calibration and early tick delivery.
#include <stdint.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
@@ -30,15 +31,19 @@ static volatile uint64_t* gHpet = NULL;
static volatile uint64_t gTicks = 0;
void (* volatile gTimerCallback)(void) = NULL;
// Implements the outb operation.
static inline void outb(uint16_t port, uint8_t val) { __outbyte(port, val); }
// Implements the inb operation.
static inline uint8_t inb(uint16_t port) { return __inbyte(port); }
// Implements the timer Isr operation.
static void TimerIsr(void) {
gTicks++;
if (gTimerCallback) gTimerCallback();
// EOI is sent by KeIrqDispatch after this returns
// EOI is sent by KeIrqDispatch after this returns.
}
// Returns ticks.
static uint64_t QueryTicks(void) {
// Prefer HPET when present: it is already a monotonically increasing main
// counter and avoids races between the LAPIC periodic tick and the current
@@ -76,8 +81,9 @@ static uint64_t QueryTicks(void) {
return (baseTicks1 * TIMER_PERIOD_MS * 1000ull) + subQuantumUs;
}
// Implements the stall operation.
static void Stall(uint32_t us) {
// Priority 1: Use HPET (Highest Precision)
// Priority 1: Use HPET (Highest Precision).
if (gHpet && gHpetTicksPerUs > 0) {
uint64_t start = gHpet[30]; // Offset 0xF0 is the Main Counter
uint64_t delta = (uint64_t)us * gHpetTicksPerUs;
@@ -87,7 +93,7 @@ static void Stall(uint32_t us) {
return;
}
// Priority 2: Use APIC Timer (High Precision, but handles wrap-around)
// Priority 2: Use APIC Timer (High Precision, but handles wrap-around).
if (gApicTicksPerUs > 0) {
uint32_t total_ticks = us * gApicTicksPerUs;
uint32_t last_val = LAPIC_REG(LAPIC_TIMER_CUR);
@@ -98,7 +104,7 @@ static void Stall(uint32_t us) {
if (current < last_val) {
elapsed += (last_val - current);
} else if (current > last_val) {
// Handled the reset/wrap at the end of TIMER_PERIOD_MS
// Handled the reset/wrap at the end of TIMER_PERIOD_MS.
uint32_t init_val = LAPIC_REG(LAPIC_TIMER_INIT);
elapsed += (last_val + (init_val - current));
}
@@ -108,7 +114,7 @@ static void Stall(uint32_t us) {
return;
}
// Priority 3: Fallback to Port 0x80
// Priority 3: Fallback to Port 0x80.
for (uint32_t i = 0; i < us; i++) {
__outbyte(0x80, 0);
}
@@ -116,16 +122,16 @@ static void Stall(uint32_t us) {
// Returns HPET ticks/second, or 0 if HPET is unavailable.
static uint64_t HpetInit(uintptr_t hpetPhys) {
// HPET register block, accessed through physical window
// HPET register block, accessed through physical window.
volatile uint64_t *hpet = (volatile uint64_t *)gApi.PhysToVirt(hpetPhys);
// General Capabilities register (offset 0):
// Bits 63:32 = counter clock period in femtoseconds
// Bits 63:32 = counter clock period in femtoseconds.
uint64_t caps = hpet[0];
uint32_t periodFs = (uint32_t)(caps >> 32);
if (!periodFs) return 0;
// Enable the HPET main counter (General Configuration, offset 0x10, bit 0)
// Enable the HPET main counter (General Configuration, offset 0x10, bit 0).
hpet[2] |= 1; // offset 0x10 / 8 = index 2
return (uint64_t)1e15 / periodFs; // ticks per second
@@ -133,21 +139,21 @@ static uint64_t HpetInit(uintptr_t hpetPhys) {
// Use PIT channel 2 one-shot to measure how many APIC ticks elapse in 10 ms, then return ticks/ms.
static uint32_t CalibrateApicTimerViaPit(void) {
// Divide by 16
// Divide by 16.
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
// Mask, set vector - leave masked during calibration
// Mask, set vector - leave masked during calibration.
LAPIC_REG(LAPIC_TIMER_LVT) = TIMER_MASKED | TIMER_VECTOR;
// Load max count
// Load max count.
LAPIC_REG(LAPIC_TIMER_INIT) = 0xFFFFFFFF;
// PIT channel 2: one-shot, ~10 ms
// PIT channel 2: one-shot, ~10 ms.
uint16_t pit_count = (uint16_t)(PIT_HZ / 100); // 11931 ≈ 10 ms
outb(0x61, (inb(0x61) & ~0x02) | 0x01); // gate on, speaker off
outb(0x43, 0xB0); // ch2, lo/hi, mode 0, binary
outb(0x42, (uint8_t)(pit_count & 0xFF));
outb(0x42, (uint8_t)(pit_count >> 8));
// Wait for OUT2 (bit 5 of port 0x61) to go high
// Wait for OUT2 (bit 5 of port 0x61) to go high.
while (!(inb(0x61) & 0x20)) _mm_pause();
uint32_t elapsed = 0xFFFFFFFF - LAPIC_REG(LAPIC_TIMER_CUR);
@@ -160,14 +166,14 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
uint64_t hpetHz = HpetInit(hpetPhys);
if (!hpetHz) return 0;
// Ticks in 10 ms = hpetHz / 100
// Ticks in 10 ms = hpetHz / 100.
uint64_t ticksFor10ms = hpetHz / 100;
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
LAPIC_REG(LAPIC_TIMER_LVT) = TIMER_MASKED | TIMER_VECTOR;
LAPIC_REG(LAPIC_TIMER_INIT) = 0xFFFFFFFF;
// HPET main counter is at offset 0xF0, index 0xF0/8 = 30
// HPET main counter is at offset 0xF0, index 0xF0/8 = 30.
uint64_t hpetStart = hpet[30];
while ((hpet[30] - hpetStart) < ticksFor10ms) _mm_pause();
@@ -175,6 +181,7 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
return elapsed / 10; // APIC ticks per ms
}
// Implements the timer Configure Current Cpu operation.
__declspec(dllexport) void HalTimerConfigureCurrentCpu(void) {
if (!gLapic || !gApicTicksPerQuantum) return;
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
@@ -182,6 +189,7 @@ __declspec(dllexport) void HalTimerConfigureCurrentCpu(void) {
LAPIC_REG(LAPIC_TIMER_INIT) = gApicTicksPerQuantum;
}
// Implements the timer Initialize operation.
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out) {
gLapic = (volatile uint32_t *)gApi.PhysToVirt(lapicPhys);
HalDbgPrint("HAL: lapicPhys=0x%016llX, lapicVirt=0x%016llX, hpetPhys=0x%016llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
code/kernel/include/kernel/display_cursor.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel display cursor declarations shared by implementation files and kernel clients.
#pragma once
// display_cursor.h - shared default software cursor image
// display_cursor.h - shared default software cursor image.
#include <stdint.h>
code/kernel/include/kernel/hal/font8x16.h
+523 -521
View File
File diff suppressed because it is too large. Load diff
code/kernel/include/kernel/hal/hal_apic.h
+2 -1
View File
@@ -1,4 +1,5 @@
// hal_apic.h
// Kernel hal apic declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
code/kernel/include/kernel/hal/hal_debug.h
+5 -3
View File
@@ -1,4 +1,6 @@
#pragma once
// hal_debug.h - HAL debug output function
// Kernel hal debug declarations shared by implementation files and kernel clients.
#pragma once
// hal_debug.h - HAL debug output function.
void HalDbgPrint(const char* fmt, ...);
code/kernel/include/kernel/io_file.h
+9 -25
View File
@@ -1,3 +1,5 @@
// Kernel io file declarations shared by implementation files and kernel clients.
#pragma once
// io_file.h - VFS layer
// Filesystem drivers register an IO_FILESYSTEM vtable via IoRegisterFilesystem.
@@ -23,9 +25,7 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
KSTATUS status,
size_t information);
// ---------------------------------------------------------------------------
// Filesystem driver vtable - filled in by a filesystem driver.
// ---------------------------------------------------------------------------
typedef struct IO_FILESYSTEM {
// Open a file by absolute path (e.g. "apps\\shell.exe").
// Returns an opaque per-file driver cookie in* fileCtx.
@@ -80,9 +80,7 @@ typedef struct IO_FILESYSTEM {
// Optional driver-private data.
void* DriverCtx;
// ---------------------------------------------------------------------------
// Directory enumeration (optional - may be NULL if not supported)
// ---------------------------------------------------------------------------
// Directory enumeration (optional - may be NULL if not supported).
// Open a directory iterator for 'dirPath' (relative or drive-prefixed).
// Returns an opaque per-iteration driver cookie in *dirCtx.
@@ -100,9 +98,7 @@ typedef struct IO_FILESYSTEM {
} IO_FILESYSTEM;
// ---------------------------------------------------------------------------
// IO_STATUS_BLOCK - completion status visible to async callers
// ---------------------------------------------------------------------------
// IO_STATUS_BLOCK - completion status visible to async callers.
typedef struct IO_STATUS_BLOCK {
volatile KSTATUS Status;
volatile size_t Information;
@@ -112,9 +108,7 @@ struct EVENT;
struct KAPC;
typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
// ---------------------------------------------------------------------------
// FILE_OBJECT - the kernel object returned to callers
// ---------------------------------------------------------------------------
// FILE_OBJECT - the kernel object returned to callers.
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
@@ -124,9 +118,7 @@ typedef struct FILE_OBJECT {
uint64_t Offset; // current position for sequential reads
} FILE_OBJECT;
// ---------------------------------------------------------------------------
// DIR_OBJECT - kernel object for an open directory iterator
// ---------------------------------------------------------------------------
// DIR_OBJECT - kernel object for an open directory iterator.
typedef struct DIR_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
@@ -139,18 +131,14 @@ typedef struct IO_VOLUME_PARAMS {
uint64_t PartitionSize; // sectors, for bounds checking
} IO_VOLUME_PARAMS;
// ---------------------------------------------------------------------------
// Registration - called by fat32.sys from DriverEntry
// ---------------------------------------------------------------------------
// driveIndex matches the index passed to IoRegisterDiskDevice.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol);
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
__declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex);
// ---------------------------------------------------------------------------
// Kernel-side API
// ---------------------------------------------------------------------------
// Kernel-side API.
__declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out);
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead);
@@ -183,15 +171,11 @@ __declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// ---------------------------------------------------------------------------
// Directory enumeration API
// ---------------------------------------------------------------------------
// Directory enumeration API.
__declspec(dllexport) KSTATUS IoOpenDir (SYS_STRING_VIEW dirPath, DIR_OBJECT** out);
__declspec(dllexport) KSTATUS IoReadDir (DIR_OBJECT* dir, KDIR_ENTRY* entryOut);
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir);
// ---------------------------------------------------------------------------
// Current Working Directory
// ---------------------------------------------------------------------------
// Current Working Directory.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size);
code/kernel/include/kernel/ke_apc.h
+4 -2
View File
@@ -1,6 +1,8 @@
// Kernel ke apc declarations shared by implementation files and kernel clients.
#pragma once
// ke_apc.h - kernel APC infrastructure
//
// Kernel APCs are per-thread callbacks that run in the target thread's
// context when that thread reaches PASSIVE_LEVEL and APC delivery is enabled.
// They are the first APC layer; user APCs will build on this once trap-frame
@@ -43,7 +45,7 @@ typedef struct SYS_USER_APC_FRAME_K {
uint64_t SystemArgument2;
} SYS_USER_APC_FRAME_K;
#define SYS_USER_APC_FRAME_MAGIC 0x3143504152455355ULL /* "USERAPC1" little-ish */
#define SYS_USER_APC_FRAME_MAGIC 0x3143504152455355ULL // "USERAPC1" little-ish
struct KUSER_APC {
LIST_ENTRY Link;
code/kernel/include/kernel/ke_cpu.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke cpu declarations shared by implementation files and kernel clients.
#pragma once
// ke_cpu.h - per-CPU execution state
//
// Every processor that reaches kernel-owned state has a KCPU record. The record
// owns CPU-local scheduler, IRQL, GDT, and TSS state.
code/kernel/include/kernel/ke_debug.h
+3 -3
View File
@@ -1,3 +1,5 @@
// Kernel ke debug declarations shared by implementation files and kernel clients.
#ifndef KE_DEBUG_H
#define KE_DEBUG_H
@@ -31,15 +33,13 @@ typedef enum KE_PANIC_CODE {
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...);
__declspec(dllexport) int KeHandleNmiIpi(void);
// ---------------------------------------------------------------------------
// Fault dispatch - called from the IDT fault stubs (ke_syscall.asm).
//
// The old scalar entry only reported RIP/error/CR2, which is not enough for
// debugging null indirect calls. The frame entry snapshots user GPRs plus the
// CPU exception frame, including user RSP. For a CALL through a null function
// pointer, RIP is 0 and [RSP] is usually the return address after the bad call;
// that is the useful caller site.
// ---------------------------------------------------------------------------
typedef struct KFAULT_FRAME {
uint64_t Magic;
uint64_t Vector;
code/kernel/include/kernel/ke_display.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke display declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <stdbool.h>
code/kernel/include/kernel/ke_dpc.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke dpc declarations shared by implementation files and kernel clients.
#pragma once
// ke_dpc.h - deferred procedure calls
//
// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC
// after acknowledging/capturing device state; the DPC then runs later at
// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not
code/kernel/include/kernel/ke_event.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel ke event declarations shared by implementation files and kernel clients.
#pragma once
// ke_event.h - auto-reset and manual-reset events
// ke_event.h - auto-reset and manual-reset events.
#include <stdint.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
code/kernel/include/kernel/ke_idt.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel ke idt declarations shared by implementation files and kernel clients.
#pragma once
// ke_idt.h - IDT, GDT, and TSS setup
// ke_idt.h - IDT, GDT, and TSS setup.
#include <stdint.h>
// 64-bit TSS - minimal; we only care about RSP0.
code/kernel/include/kernel/ke_input.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke input declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <stdbool.h>
code/kernel/include/kernel/ke_irp.h
+11 -25
View File
@@ -1,6 +1,8 @@
// Kernel ke irp declarations shared by implementation files and kernel clients.
#pragma once
// ke_irp.h - I/O Request Packet and remove-lock primitives
//
// IRPs are stack-based. A caller builds an IRP with one or more
// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O
// manager/driver stack advances CurrentLocation as the IRP is forwarded.
@@ -18,9 +20,7 @@ struct SEMAPHORE;
struct KAPC;
typedef struct DEVICE_OBJECT DEVICE_OBJECT;
// -------------------------------------------------------------------------
// IRP major/minor functions
// -------------------------------------------------------------------------
// IRP major/minor functions.
typedef enum IO_MAJOR_FUNCTION {
IRP_MJ_CREATE = 0,
@@ -50,17 +50,13 @@ typedef enum IO_DEVICE_RELATION_TYPE {
IoRemovalRelations = 1,
} IO_DEVICE_RELATION_TYPE;
// -------------------------------------------------------------------------
// IRP flags
// -------------------------------------------------------------------------
// IRP flags.
#define IRP_FLAG_PRIORITY (1u << 0) // insert at head of a driver queue
#define IRP_FLAG_CANCELLED (1u << 1)
#define IRP_FLAG_COMPLETED (1u << 2)
// -------------------------------------------------------------------------
// IRP stack parameters
// -------------------------------------------------------------------------
// IRP stack parameters.
typedef struct IO_STACK_LOCATION {
IO_MAJOR_FUNCTION MajorFunction;
@@ -91,18 +87,14 @@ typedef struct IO_STACK_LOCATION {
} Parameters;
} IO_STACK_LOCATION;
// -------------------------------------------------------------------------
// Forward declaration / callbacks
// -------------------------------------------------------------------------
// Forward declaration / callbacks.
typedef struct IRP IRP;
typedef void (*IRP_COMPLETION_FN)(IRP* irp, void* ctx);
typedef void (*IRP_CANCEL_ROUTINE)(DEVICE_OBJECT* device, IRP* irp);
// -------------------------------------------------------------------------
// I/O Request Packet
// -------------------------------------------------------------------------
// I/O Request Packet.
typedef struct IRP {
LIST_ENTRY Link;
@@ -135,9 +127,7 @@ typedef struct IRP {
IO_STACK_LOCATION Stack[1];
} IRP;
// -------------------------------------------------------------------------
// Remove lock
// -------------------------------------------------------------------------
// Remove lock.
typedef struct IO_REMOVE_LOCK {
SPINLOCK Lock;
@@ -152,9 +142,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp);
// -------------------------------------------------------------------------
// IRP_QUEUE (per-device, driver-internal)
// -------------------------------------------------------------------------
// IRP_QUEUE (per-device, driver-internal).
typedef struct IRP_QUEUE {
SPINLOCK Lock;
@@ -172,9 +160,7 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
void* context);
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status);
// -------------------------------------------------------------------------
// IRP allocation / stack helpers
// -------------------------------------------------------------------------
// IRP allocation / stack helpers.
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp);
__declspec(dllexport) void IoFreeIrp(IRP* irp);
code/kernel/include/kernel/ke_irq.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke irq declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
code/kernel/include/kernel/ke_irql.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke irql declarations shared by implementation files and kernel clients.
#pragma once
// ke_irql.h - software IRQL skeleton
//
// This is intentionally small for the first IRQL/APC bring-up step. It does
// not yet program interrupt hardware or replace the spinlock interrupt-disable
// path. It gives the kernel one authoritative execution-level state plus
code/kernel/include/kernel/ke_list.h
+49 -44
View File
@@ -1,44 +1,49 @@
#pragma once
// ke_list.h - intrusive doubly-linked list
// Used throughout the kernel for run queues, wait lists, thread lists, etc.
#include <stddef.h>
typedef struct LIST_ENTRY {
struct LIST_ENTRY* Fwd;
struct LIST_ENTRY* Bwd;
} LIST_ENTRY;
#define LIST_INIT(head) ((head)->Fwd = (head)->Bwd = (head))
#define LIST_IS_EMPTY(head) ((head)->Fwd == (head))
// Recover the containing struct pointer from an embedded LIST_ENTRY.
#define CONTAINING_RECORD(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
static inline void ListInsertTail(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Bwd = head->Bwd;
node->Fwd = head;
head->Bwd->Fwd = node;
head->Bwd = node;
}
static inline void ListInsertHead(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Fwd = head->Fwd;
node->Bwd = head;
head->Fwd->Bwd = node;
head->Fwd = node;
}
static inline void ListRemove(LIST_ENTRY* node) {
node->Bwd->Fwd = node->Fwd;
node->Fwd->Bwd = node->Bwd;
node->Fwd = node->Bwd = node; /* self-loop: poison against double-remove */
}
// Pop the first entry off the list, or NULL if empty.
static inline LIST_ENTRY* ListPopHead(LIST_ENTRY* head) {
if (LIST_IS_EMPTY(head)) return (LIST_ENTRY *)0;
LIST_ENTRY* node = head->Fwd;
ListRemove(node);
return node;
}
// Kernel ke list declarations shared by implementation files and kernel clients.
#pragma once
// ke_list.h - intrusive doubly-linked list
// Used throughout the kernel for run queues, wait lists, thread lists, etc.
#include <stddef.h>
typedef struct LIST_ENTRY {
struct LIST_ENTRY* Fwd;
struct LIST_ENTRY* Bwd;
} LIST_ENTRY;
#define LIST_INIT(head) ((head)->Fwd = (head)->Bwd = (head))
#define LIST_IS_EMPTY(head) ((head)->Fwd == (head))
// Recover the containing struct pointer from an embedded LIST_ENTRY.
#define CONTAINING_RECORD(ptr, type, member) \
((type *)((char *)(ptr) - offsetof(type, member)))
// Implements the list Insert Tail operation.
static inline void ListInsertTail(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Bwd = head->Bwd;
node->Fwd = head;
head->Bwd->Fwd = node;
head->Bwd = node;
}
// Implements the list Insert Head operation.
static inline void ListInsertHead(LIST_ENTRY* head, LIST_ENTRY* node) {
node->Fwd = head->Fwd;
node->Bwd = head;
head->Fwd->Bwd = node;
head->Fwd = node;
}
// Implements the list Remove operation.
static inline void ListRemove(LIST_ENTRY* node) {
node->Bwd->Fwd = node->Fwd;
node->Fwd->Bwd = node->Bwd;
node->Fwd = node->Bwd = node; // self-loop: poison against double-remove
}
// Pop the first entry off the list, or NULL if empty.
static inline LIST_ENTRY* ListPopHead(LIST_ENTRY* head) {
if (LIST_IS_EMPTY(head)) return (LIST_ENTRY *)0;
LIST_ENTRY* node = head->Fwd;
ListRemove(node);
return node;
}
code/kernel/include/kernel/ke_mutex.h
+4 -2
View File
@@ -1,5 +1,7 @@
// Kernel ke mutex declarations shared by implementation files and kernel clients.
#pragma once
// ke_mutex.h - binary ownership mutex
// ke_mutex.h - binary ownership mutex.
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_list.h>
@@ -8,7 +10,7 @@
typedef struct MUTEX {
OBJECT Header;
SPINLOCK Lock;
struct THREAD* Owner; /* NULL if unowned */
struct THREAD* Owner; // NULL if unowned
LIST_ENTRY WaitList;
} MUTEX;
code/kernel/include/kernel/ke_process.h
+22 -20
View File
@@ -1,3 +1,5 @@
// Kernel ke process declarations shared by implementation files and kernel clients.
#pragma once
// ke_process.h - process model
// A process owns a handle table and a list of threads.
@@ -37,23 +39,23 @@ typedef struct PROCESS_MODULE {
} PROCESS_MODULE;
typedef struct PROCESS {
OBJECT Header; /* must be first */
OBJECT Header; // must be first
uint32_t ProcessId;
char Name[PROCESS_NAME_MAX]; /* UTF-8 */
char Name[PROCESS_NAME_MAX]; // UTF-8
char CurrentDirectory[PROCESS_CWD_MAX];
size_t CurrentDirectoryLength;
uint8_t CurrentDrive;
SPINLOCK CwdLock;
/* Thread list. */
LIST_ENTRY ThreadList; /* links THREAD.ProcessLink */
// Thread list.
LIST_ENTRY ThreadList; // links THREAD.ProcessLink
uint32_t ThreadCount;
/* Handle table.
Index 0 is always empty (the invalid handle).
Each live entry holds one object reference plus rights metadata. */
// Handle table.
// Index 0 is always empty (the invalid handle).
// Each live entry holds one object reference plus rights metadata.
HANDLE_ENTRY Handles[HANDLE_TABLE_SIZE];
SPINLOCK HandleLock;
@@ -61,11 +63,11 @@ typedef struct PROCESS {
volatile long Terminating;
volatile long Exited;
/* Main executable image. Used for cleanup. */
// Main executable image. Used for cleanup.
uintptr_t ImageBase;
size_t ImageSize;
/* User-mode process environment. The initial thread sees this through GS->TEB->PEB. */
// User-mode process environment. The initial thread sees this through GS->TEB->PEB.
uintptr_t UserPebBase;
size_t UserPebSize;
uintptr_t UserImagePathBase;
@@ -75,21 +77,21 @@ typedef struct PROCESS {
uintptr_t UserCwdBase;
size_t UserCwdSize;
/* Per-process user virtual regions. The actual VA arena is global because
NT-Lite currently uses one shared address space; this list tracks
ownership so leaked user mappings can be reclaimed on process exit. */
// Per-process user virtual regions. The actual VA arena is global because
// NT-Lite currently uses one shared address space; this list tracks
// ownership so leaked user mappings can be reclaimed on process exit.
LIST_ENTRY UserRegions;
SPINLOCK UserRegionLock;
/* Per-process loaded modules (user processes only). */
// Per-process loaded modules (user processes only).
PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
uint32_t ModuleCount;
/* Signaled (manual-reset) when the process exits.
Waiters block on this via KeWaitForEvent. */
// Signaled (manual-reset) when the process exits.
// Waiters block on this via KeWaitForEvent.
EVENT* ExitEvent;
LIST_ENTRY GlobalLink; /* node in the global process list */
LIST_ENTRY GlobalLink; // node in the global process list
} PROCESS;
KSTATUS KeCreateProcess(const char* name, PROCESS **out);
@@ -103,7 +105,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode);
int KeIsProcessTerminating(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut);
/* Handle table operations. */
// Handle table operations.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut);
KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t rights, uint32_t* handleOut);
void KeCloseHandle(PROCESS* process, uint32_t handle);
@@ -111,13 +113,13 @@ OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights);
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle);
/* Return the global process list head (kernel-internal). */
// Return the global process list head (kernel-internal).
LIST_ENTRY* KeGetProcessList(void);
PROCESS* KeGetCurrentProcess(void);
PROCESS* KeGetSystemProcess(void);
struct SYS_PROCESS_INFO;
/* Copy a snapshot of rich process information into caller-provided storage. */
// Copy a snapshot of rich process information into caller-provided storage.
size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount);
/* Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject. */
// Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject.
PROCESS* KeGetProcessById(uint32_t pid);
code/kernel/include/kernel/ke_queue.h
+5 -3
View File
@@ -1,3 +1,5 @@
// Kernel ke queue declarations shared by implementation files and kernel clients.
#pragma once
// ke_queue.h - fixed-depth message queue
// Processes share an address space, so a pointer in Param[] is valid
@@ -9,14 +11,14 @@
#include <kernel/ke_list.h>
#include <ntlite/status.h>
#define QUEUE_MESSAGE_SIZE 64 /* bytes, fixed */
#define QUEUE_MESSAGE_SIZE 64 // bytes, fixed
#define QUEUE_DEPTH 256
typedef struct QUEUE_MESSAGE {
uint32_t Type;
uint32_t Flags;
uint64_t Param[6]; /* 48 bytes of payload */
} QUEUE_MESSAGE; /* total: 64 bytes */
uint64_t Param[6]; // 48 bytes of payload
} QUEUE_MESSAGE; // total: 64 bytes
typedef struct QUEUE {
OBJECT Header;
code/kernel/include/kernel/ke_sched.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke sched declarations shared by implementation files and kernel clients.
#pragma once
// ke_sched.h - Multi-Level Feedback Queue scheduler
// Three run queues (Low / Normal / High). IRQ 0 drives KeSchedulerTick.
code/kernel/include/kernel/ke_selftest.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke selftest declarations shared by implementation files and kernel clients.
#pragma once
// ke_selftest.h - kernel bring-up and regression self-tests.
code/kernel/include/kernel/ke_semaphore.h
+3 -1
View File
@@ -1,5 +1,7 @@
// Kernel ke semaphore declarations shared by implementation files and kernel clients.
#pragma once
// ke_semaphore.h - counting semaphore
// ke_semaphore.h - counting semaphore.
#include <stdint.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
code/kernel/include/kernel/ke_settings.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke settings declarations shared by implementation files and kernel clients.
#pragma once
#include <stddef.h>
code/kernel/include/kernel/ke_smp.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke smp declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
code/kernel/include/kernel/ke_spinlock.h
+9 -2
View File
@@ -1,12 +1,14 @@
// Kernel ke spinlock declarations shared by implementation files and kernel clients.
#pragma once
// ke_spinlock.h - spinlock for short, non-sleeping critical sections
//
// Step 3 IRQL bring-up:
// * Acquiring a spinlock disables maskable interrupts and raises the software
// IRQL to at least DISPATCH_LEVEL.
// * Releasing restores both the previous IRQL and the previous RFLAGS.IF
// state through the returned cookie.
//
// The public type remains uintptr_t so existing callers do not need to change.
// Never hold a spinlock across a wait/blocking operation.
#include <stdint.h>
@@ -23,15 +25,18 @@ typedef struct SPINLOCK {
#define KI_SPINLOCK_IRQL_SHIFT ((unsigned)(sizeof(uintptr_t) * 8u - 8u))
#define KI_SPINLOCK_FLAGS_MASK (~(((uintptr_t)0xFFu) << KI_SPINLOCK_IRQL_SHIFT))
// Implements the pack Spinlock Cookie operation.
static __forceinline uintptr_t KiPackSpinlockCookie(uintptr_t flags, KIRQL oldIrql) {
return (flags & KI_SPINLOCK_FLAGS_MASK) |
(((uintptr_t)oldIrql) << KI_SPINLOCK_IRQL_SHIFT);
}
// Implements the spinlock Cookie Flags operation.
static __forceinline uintptr_t KiSpinlockCookieFlags(uintptr_t cookie) {
return cookie & KI_SPINLOCK_FLAGS_MASK;
}
// Implements the spinlock Cookie Irql operation.
static __forceinline KIRQL KiSpinlockCookieIrql(uintptr_t cookie) {
return (KIRQL)(cookie >> KI_SPINLOCK_IRQL_SHIFT);
}
@@ -62,6 +67,7 @@ static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t cookie) {
__writeeflags(flags);
}
// Implements the try Acquire Spinlock operation.
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie) {
uintptr_t flags = __readeflags();
_disable();
@@ -85,6 +91,7 @@ static __forceinline uintptr_t KeAcquireSpinlockRaiseToDpc(SPINLOCK* sl) {
return KeAcquireSpinlock(sl);
}
// Releases spinlock From Dpc.
static __forceinline void KeReleaseSpinlockFromDpc(SPINLOCK* sl, uintptr_t cookie) {
KeReleaseSpinlock(sl, cookie);
}
code/kernel/include/kernel/ke_syscall.h
+4 -2
View File
@@ -1,11 +1,13 @@
// Kernel ke syscall declarations shared by implementation files and kernel clients.
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
// Set the HAL API pointer for syscalls
// Set the HAL API pointer for syscalls.
void KeSyscallInit(const HAL_API* hal);
// Syscall dispatcher, called from assembly entry point
// Syscall dispatcher, called from assembly entry point.
uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5);
code/kernel/include/kernel/ke_thread.h
+10 -8
View File
@@ -1,3 +1,5 @@
// Kernel ke thread declarations shared by implementation files and kernel clients.
#pragma once
// ke_thread.h - thread model
// A THREAD is the unit of scheduling. Every thread belongs to exactly one process.
@@ -9,8 +11,8 @@
#include <kernel/ke_spinlock.h>
#include <kernel/ke_trap.h>
#define KSTACK_SIZE (64 * 1024) /* 64 KB kernel stack */
#define USTACK_SIZE (512 * 1024) /* 512 KB user stack */
#define KSTACK_SIZE (64 * 1024) // 64 KB kernel stack
#define USTACK_SIZE (512 * 1024) // 512 KB user stack
typedef enum THREAD_PRIORITY {
PriorityLow = 0,
@@ -31,11 +33,11 @@ typedef enum THREAD_STATE {
typedef struct KCONTEXT {
uint64_t R15, R14, R13, R12;
uint64_t Rbp, Rbx;
uint64_t Rip; /* return address back into KeSwitchContext */
uint64_t Rip; // return address back into KeSwitchContext
} KCONTEXT;
typedef struct THREAD {
OBJECT Header; /* must be first */
OBJECT Header; // must be first
uint32_t ThreadId;
THREAD_STATE State;
@@ -46,8 +48,8 @@ typedef struct THREAD {
uint32_t QuantaConsumed;
PROCESS* Owner;
LIST_ENTRY ProcessLink; /* node in Owner->ThreadList */
LIST_ENTRY SchedulerLink; /* node in run queue or wait list */
LIST_ENTRY ProcessLink; // node in Owner->ThreadList
LIST_ENTRY SchedulerLink; // node in run queue or wait list
// Kernel stack - never PAGE_USER, pointed to by TssRsp0.
void* KernelStackBase;
@@ -152,14 +154,14 @@ __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uin
__declspec(dllexport) void KeExitThread(THREAD* thread, int code);
// Block / unblock primitives used by sync objects.
//
// Correct wait pattern:
// object lock held
// KePrepareBlockThread(self, object, alertable);
// insert self into object's wait list
// release object lock
// KeCommitBlockThread(self, alertable);
//
// The split form prevents the classic lost wakeup where an ISR/DPC signals
// the object after the waiter is queued but before the waiter marks itself
// waiting. Alertable waits are explicit at every call site.
code/kernel/include/kernel/ke_time.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ke time declarations shared by implementation files and kernel clients.
#pragma once
#include <ntlite/status.h>
#include <ntlite/time.h>
code/kernel/include/kernel/ke_trap.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel ke trap declarations shared by implementation files and kernel clients.
#pragma once
// ke_trap.h - trap-frame metadata for syscall/interrupt return paths
//
// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally
// an editable return frame: user APC delivery can rewrite Rip/Rsp in the frame,
// and KiExitTrapFrame commits those values back to the hardware IRET slots.
code/kernel/include/kernel/ke_userenv.h
+4 -2
View File
@@ -1,3 +1,5 @@
// Kernel ke userenv declarations shared by implementation files and kernel clients.
#pragma once
// ke_userenv.h - kernel view of the user-mode process/thread environment.
// This is this OS' ABI, not a Windows compatibility structure.
@@ -6,8 +8,8 @@
#include <stddef.h>
#include <ntlite/string_view.h>
#define SYS_PEB_MAGIC 0x30424550u /* "PEB0" */
#define SYS_TEB_MAGIC 0x30424554u /* "TEB0" */
#define SYS_PEB_MAGIC 0x30424550u // "PEB0"
#define SYS_TEB_MAGIC 0x30424554u // "TEB0"
#define SYS_ENV_VERSION 1u
#define SYS_CWD_MAX 520u
code/kernel/include/kernel/ke_util.h
+4 -5
View File
@@ -1,23 +1,22 @@
// ke_util.h - kernel utility macros (dynamic arrays, etc.)
// Kernel ke util declarations shared by implementation files and kernel clients.
#pragma once
#include <string.h>
#include <kernel/mm_virt.h>
// ---------------------------------------------------------------------------
// Dynamic array (stretchy-buffer style)
//
// T *buf = NULL;
// arr_push(buf, value); // appends; grows automatically
// arr_push_front(buf, value); // prepends
// arr_remove(buf, i); // removes index i, shifts tail left
// size_t n = arr_len(buf); // element count (0 if buf == NULL)
// arr_free(buf); // frees header + data
//
// The allocation layout is:
// [ Header (count, capacity) ][ T[0] ][ T[1] ] ...
// buf always points at T[0]; the Header lives one slot before it.
// ---------------------------------------------------------------------------
typedef struct {
size_t count;
code/kernel/include/kernel/ldr_image.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ldr image declarations shared by implementation files and kernel clients.
#pragma once
// ldr_image.h - minimal PE32+ loader
// Loads an already-mapped PE image and resolves its imports.
code/kernel/include/kernel/mm_phys.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel mm phys declarations shared by implementation files and kernel clients.
#pragma once
// mm_phys.h - physical memory manager (PMM)
// Bitmap allocator over 4 KB frames, seeded from MEMORY_MAP.
code/kernel/include/kernel/mm_virt.h
+5 -17
View File
@@ -1,3 +1,5 @@
// Kernel mm virt declarations shared by implementation files and kernel clients.
#pragma once
// mm_virt.h - virtual memory manager (VMM)
// Manages the single shared PML4, kernel pool, and page-granular user VM regions.
@@ -30,9 +32,7 @@ typedef enum USER_REGION_TYPE {
#define USER_REGION_GUARD_HIGH 0x00000002u
#define USER_REGION_ZERO 0x00000004u
// ---------------------------------------------------------------------------
// Page flags (x86-64 PTE bits)
// ---------------------------------------------------------------------------
// Page flags (x86-64 PTE bits).
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
@@ -40,9 +40,7 @@ typedef enum USER_REGION_TYPE {
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_NX (1ULL << 63)
// ---------------------------------------------------------------------------
// Virtual address space constants
// ---------------------------------------------------------------------------
// Virtual address space constants.
#define USER_BASE 0x0000000000001000ULL
#define USER_LIMIT 0x00007FFFFFFFF000ULL
@@ -51,9 +49,7 @@ typedef enum USER_REGION_TYPE {
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
// ---------------------------------------------------------------------------
// Initialisation
// ---------------------------------------------------------------------------
// Called by KernelMain after MmInitializePhysical.
// bootPml4Phys: the PML4 set up by the bootloader.
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
@@ -62,9 +58,7 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
void MmActivate(void);
// ---------------------------------------------------------------------------
// Page mapping
// ---------------------------------------------------------------------------
// Page mapping.
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode);
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt);
@@ -86,9 +80,7 @@ void MmFlushTlbForKernelMapping(uintptr_t virt);
int MmHandleTlbShootdownNmi(void);
// ---------------------------------------------------------------------------
// User/kernel copy boundary helpers
// ---------------------------------------------------------------------------
// MmQueryPage returns the physical address and effective PAGE_* bits for a
// mapped virtual address. The reported PAGE_USER/PAGE_WRITE bits account for
// every page-table level, not just the leaf entry.
@@ -105,20 +97,16 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size);
__declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size);
__declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size);
// ---------------------------------------------------------------------------
// Kernel pool - ring 0 only, never PAGE_USER.
// Best-fit free-list allocator, thread-safe via spinlock.
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocatePool(size_t size);
__declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// ---------------------------------------------------------------------------
// User virtual regions.
// The kernel has one shared user VA arena, but every user mapping is owned by
// a PROCESS so process teardown can reclaim leaked memory. Guard pages are
// reserved VA with no present PTE.
// ---------------------------------------------------------------------------
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t regionFlags,
USER_REGION_TYPE type, void** outBase);
code/kernel/include/kernel/net_socket.h
+3 -1
View File
@@ -1,6 +1,8 @@
// Kernel net socket declarations shared by implementation files and kernel clients.
#pragma once
// net_socket.h - process-owned, provider-neutral kernel socket objects.
//
// The native networking ABI lives in net_abi.h. Protocol implementation lives
// in a network stack driver registered through net_stack.h.
code/kernel/include/kernel/ob_object.h
+2
View File
@@ -1,3 +1,5 @@
// Kernel ob object declarations shared by implementation files and kernel clients.
#pragma once
// ob_object.h - executive object model
// Every resource the kernel tracks is a typed, reference-counted OBJECT.
code/kernel/include/kernel/syscall_nr.h
+3 -1
View File
@@ -1,10 +1,12 @@
// Kernel syscall nr declarations shared by implementation files and kernel clients.
#pragma once
// syscall_nr.h
// INT 0x40 calling convention, MS x64:
// RAX = syscall number
// RCX, RDX, R8, R9 = args 0-3
// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6)
// RAX = return value
// RAX = return value.
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
code/kernel/io_device.c
+29 -19
View File
@@ -1,4 +1,5 @@
// io_device.c - I/O manager: driver/device objects, stacks, PnP skeleton, disk namespace
// I/O manager. This file owns driver objects, device stacks, PnP dispatch, and the disk namespace.
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
@@ -10,10 +11,12 @@
#define MAX_DRIVER_OBJECTS 64
#define MAX_PDO_OBJECTS 128
// Implements the sv Or Unnamed operation.
static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) {
return SysSvIsEmpty(name) ? SYS_SV_LITERAL("<unnamed>") : name;
}
// Implements the bus Type Name operation.
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
switch (busType) {
case IoBusPci: return "pci";
@@ -22,15 +25,14 @@ static const char* IoBusTypeName(IO_BUS_TYPE busType) {
}
}
// -------------------------------------------------------------------------
// Driver object registry / PDO AddDevice binding
// -------------------------------------------------------------------------
// Driver object registry / PDO AddDevice binding.
static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS];
static uint32_t gDriverObjectCount;
static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS];
static uint32_t gPhysicalDeviceCount;
// Implements the driver Matches Pdo operation.
static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical)
return false;
@@ -47,6 +49,7 @@ static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT*
return false;
}
// Implements the try Add Device For Driver operation.
static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) {
if (!driver || !pdo || !driver->DriverAddDevice)
return false;
@@ -81,6 +84,7 @@ __declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
driver->Name = name;
}
// Registers driver Object.
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
if (!driver) return STATUS_INVALID_PARAMETER;
if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
@@ -100,9 +104,7 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
return STATUS_SUCCESS;
}
// -------------------------------------------------------------------------
// Device object lifetime
// -------------------------------------------------------------------------
// Device object lifetime.
static void IoDeviceDestroy(OBJECT* object) {
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
@@ -221,6 +223,7 @@ __declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
return STATUS_SUCCESS;
}
// Registers physical Device.
__declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
if (!pdo || pdo->Type != DeviceObjectPhysical)
return STATUS_INVALID_PARAMETER;
@@ -264,14 +267,17 @@ __declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
return STATUS_SUCCESS;
}
// Implements the reference Device Object operation.
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObReferenceObject(&device->Header);
}
// Implements the dereference Device Object operation.
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObDereferenceObject(&device->Header);
}
// Releases physical Device Registry Entry.
static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
if (!device || device->Type != DeviceObjectPhysical)
return;
@@ -288,6 +294,7 @@ static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
}
}
// Implements the unlink Device From Parent operation.
static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
if (!device || !device->Parent)
return;
@@ -309,6 +316,7 @@ static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
IoDereferenceDeviceObject(parent);
}
// Releases device.
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
if (device->DeletePending)
@@ -321,9 +329,7 @@ __declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
IoDereferenceDeviceObject(device);
}
// -------------------------------------------------------------------------
// Device stack management
// -------------------------------------------------------------------------
// Device stack management.
__declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT* source,
DEVICE_OBJECT* target) {
@@ -339,6 +345,7 @@ __declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT*
return top;
}
// Implements the detach Device operation.
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
if (!source || !source->LowerDevice) return;
@@ -349,6 +356,7 @@ __declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
IoDereferenceDeviceObject(lower);
}
// Returns attached Device Reference.
__declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT* device) {
if (!device) return (DEVICE_OBJECT*)0;
DEVICE_OBJECT* top = device;
@@ -358,9 +366,7 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetAttachedDeviceReference(DEVICE_OBJECT*
return top;
}
// -------------------------------------------------------------------------
// Device-stack helpers / PnP dispatch
// -------------------------------------------------------------------------
// Device-stack helpers / PnP dispatch.
__declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
uint8_t count = 0;
@@ -369,6 +375,7 @@ __declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
return count ? count : 1;
}
// Implements the forward Irp To Lower operation.
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->LowerDevice)
return STATUS_NOT_SUPPORTED;
@@ -413,9 +420,7 @@ __declspec(dllexport) KSTATUS IoSendPnpIrpSync(DEVICE_OBJECT* device,
return st;
}
// -------------------------------------------------------------------------
// Disk namespace
// -------------------------------------------------------------------------
// Disk namespace.
#define MAX_DISK_DEVICES 8
#define MAX_DISK_CALLBACKS 16
@@ -438,6 +443,7 @@ static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
// Implements the notify Disk Change operation.
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i];
@@ -477,6 +483,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
return STATUS_SUCCESS;
}
// Implements the notify Disk Surprise Removed operation.
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
@@ -489,6 +496,7 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
return STATUS_SUCCESS;
}
// Implements the unregister Disk Device operation.
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return STATUS_INVALID_PARAMETER;
@@ -562,29 +570,31 @@ __declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK ca
return STATUS_SUCCESS;
}
// Returns disk Device.
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES) return (DEVICE_OBJECT*)0;
return gDiskDevices[driveIndex];
}
// Implements the reference Disk Device operation.
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
if (dev) IoReferenceDeviceObject(dev);
return dev;
}
// Implements the disk Device Present operation.
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
}
// Implements the disk Sector Count operation.
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
return dev ? dev->u.Disk.SectorCount : 0;
}
// -------------------------------------------------------------------------
// Dispatch
// -------------------------------------------------------------------------
// Dispatch.
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DriverObject || !irp)
code/kernel/io_file.c
+36 -37
View File
@@ -1,4 +1,5 @@
// io_file.c - VFS dispatch layer
// Kernel file I/O layer. This file owns file handles, filesystem dispatch, and asynchronous request plumbing.
#include <string.h>
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
@@ -22,9 +23,7 @@ typedef struct {
static VOLUME_ENTRY gVolumes[MAX_DRIVES];
// ---------------------------------------------------------------------------
// Async file I/O worker
// ---------------------------------------------------------------------------
// Async file I/O worker.
typedef enum IO_ASYNC_FILE_OP {
IoAsyncFileRead = 0,
IoAsyncFileWrite = 1,
@@ -61,6 +60,7 @@ static SPINLOCK gAsyncFileQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gAsyncFileQueue;
static SEMAPHORE* gAsyncFileSemaphore;
// Implements the io Async File Request Destroy operation.
static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header);
@@ -101,6 +101,7 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
KeSignalEvent(req->CompletionEvent);
}
// Implements the io Async File Dequeue operation.
static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
LIST_ENTRY* node = ListPopHead(&gAsyncFileQueue);
@@ -121,6 +122,7 @@ static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
return req;
}
// Implements the io Async File Worker operation.
static void KiIoAsyncFileWorker(void* arg) {
(void)arg;
@@ -162,6 +164,7 @@ static void KiIoAsyncFileWorker(void* arg) {
}
}
// Implements the io Ensure Async File Io Started operation.
static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileInitLock);
if (gAsyncFileInitState == 2) {
@@ -337,10 +340,12 @@ __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const vo
requestOut);
}
// Implements the async File Request Object operation.
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? &req->Header : (OBJECT*)0;
}
// Cancels async File Request.
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
if (!req) return STATUS_INVALID_PARAMETER;
@@ -382,11 +387,9 @@ __declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJE
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Current Working Directory
// Stored as (CurrentDrive, drive-relative path with no trailing backslash).
// Exported as native "N:/path" so personalities can round-trip volumes.
// ---------------------------------------------------------------------------
#define CWD_MAX PROCESS_CWD_MAX
typedef struct KRESOLVED_PATH {
@@ -395,6 +398,7 @@ typedef struct KRESOLVED_PATH {
SYS_STRING_VIEW Path;
} KRESOLVED_PATH;
// Implements the format Cwd operation.
static size_t KiFormatCwd(uint8_t drive, SYS_STRING_VIEW rel, char* buf, size_t size) {
if (!buf || size == 0 || !SysSvIsValid(rel)) return 0;
if (drive > 7) drive = 0;
@@ -410,6 +414,7 @@ static size_t KiFormatCwd(uint8_t drive, SYS_STRING_VIEW rel, char* buf, size_t
return needed;
}
// Returns cwd.
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
if (!buf || size == 0) return STATUS_INVALID_PARAMETER;
@@ -423,6 +428,7 @@ __declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
return needed && needed <= size ? STATUS_SUCCESS : STATUS_BUFFER_TOO_SMALL;
}
// Implements the sv Has Drive Root operation.
static int KiSvHasDriveRoot(SYS_STRING_VIEW path) {
return path.Length >= 3 &&
path.Data[0] >= '0' && path.Data[0] <= '7' &&
@@ -449,6 +455,7 @@ static int KiAppendComponent(char* work, size_t* workLen, size_t cap,
return KiAppendBytes(work, workLen, cap, data, len);
}
// Implements the pop Component operation.
static void KiPopComponent(char* work, size_t* workLen) {
if (!work || !workLen || *workLen == 0) return;
size_t i = *workLen;
@@ -529,6 +536,7 @@ static KSTATUS KiResolvePathV(SYS_STRING_VIEW input, KRESOLVED_PATH* out) {
return STATUS_SUCCESS;
}
// Updates cwd.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path) {
if (path.Length && !path.Data) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -571,9 +579,7 @@ __declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Registration
// ---------------------------------------------------------------------------
// Registration.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
IO_FILESYSTEM *fs,
const IO_VOLUME_PARAMS *vol) {
@@ -586,6 +592,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
return STATUS_SUCCESS;
}
// Implements the unregister Filesystem operation.
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return STATUS_INVALID_PARAMETER;
@@ -596,6 +603,7 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
return STATUS_SUCCESS;
}
// Implements the unregister Filesystems For Disk operation.
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
KSTATUS last = STATUS_NOT_FOUND;
for (uint8_t i = 0; i < MAX_DRIVES; i++) {
@@ -607,23 +615,20 @@ __declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex)
return last;
}
// Implements the filesystem Present operation.
__declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
if (volumeIndex >= MAX_DRIVES)
return false;
return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
}
// ---------------------------------------------------------------------------
// Destructor
// ---------------------------------------------------------------------------
// Destructor.
static void KiFileDestroy(OBJECT* obj) {
FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx);
}
// ---------------------------------------------------------------------------
// IoOpenFile
// ---------------------------------------------------------------------------
// IoOpenFile.
__declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out) {
if ((!path.Data && path.Length) || !out) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -640,7 +645,8 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
s = fs->Open(fs, resolved.Path, &fileCtx);
if (!KSUCCESS(s)) return s;
FILE_OBJECT* f = (FILE_OBJECT *)ObAllocateObject(ObjectTypeFile /* reuse slot */,
// File objects reuse the generic object slot because the handle table only needs the shared header contract here.
FILE_OBJECT* f = (FILE_OBJECT *)ObAllocateObject(ObjectTypeFile,
sizeof(FILE_OBJECT));
if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; }
f->Header.Destroy = KiFileDestroy;
@@ -654,10 +660,8 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
void* buf, size_t size,
size_t* bytesRead) {
@@ -700,9 +704,7 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
return s;
}
// ---------------------------------------------------------------------------
// IoReadFile - sequential read at current offset
// ---------------------------------------------------------------------------
// IoReadFile - sequential read at current offset.
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead) {
if (bytesRead) *bytesRead = 0;
@@ -721,9 +723,7 @@ __declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
return s;
}
// ---------------------------------------------------------------------------
// IoWriteFile - sequential write at current offset
// ---------------------------------------------------------------------------
// IoWriteFile - sequential write at current offset.
__declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
size_t size, size_t* bytesWritten) {
if (bytesWritten) *bytesWritten = 0;
@@ -742,9 +742,7 @@ __declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
return s;
}
// ---------------------------------------------------------------------------
// IoCreateFile - create a new file and return it open for writing
// ---------------------------------------------------------------------------
// IoCreateFile - create a new file and return it open for writing.
__declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** out) {
if ((!path.Data && path.Length) || !out || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -775,9 +773,7 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// IoDeleteFile - delete a file by path (does not require an open handle)
// ---------------------------------------------------------------------------
// IoDeleteFile - delete a file by path (does not require an open handle).
__declspec(dllexport) KSTATUS IoDeleteFile(SYS_STRING_VIEW path) {
if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -793,6 +789,7 @@ __declspec(dllexport) KSTATUS IoDeleteFile(SYS_STRING_VIEW path) {
return fs->Delete(fs, resolved.Path);
}
// Implements the rename File operation.
__declspec(dllexport) KSTATUS IoRenameFile(SYS_STRING_VIEW oldPath, SYS_STRING_VIEW newPath) {
if ((!oldPath.Data && oldPath.Length) || (!newPath.Data && newPath.Length) ||
oldPath.Length == 0 || newPath.Length == 0) return STATUS_INVALID_PARAMETER;
@@ -813,6 +810,7 @@ __declspec(dllexport) KSTATUS IoRenameFile(SYS_STRING_VIEW oldPath, SYS_STRING_V
return fs->Rename(fs, oldResolved.Path, newResolved.Path);
}
// Creates dir.
__declspec(dllexport) KSTATUS IoCreateDir(SYS_STRING_VIEW path) {
if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -828,6 +826,7 @@ __declspec(dllexport) KSTATUS IoCreateDir(SYS_STRING_VIEW path) {
return fs->CreateDir(fs, resolved.Path);
}
// Releases dir.
__declspec(dllexport) KSTATUS IoRemoveDir(SYS_STRING_VIEW path) {
if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -843,9 +842,7 @@ __declspec(dllexport) KSTATUS IoRemoveDir(SYS_STRING_VIEW path) {
return fs->RemoveDir(fs, resolved.Path);
}
// ---------------------------------------------------------------------------
// IoTruncateFile - resize the file; updates the cached Size on success
// ---------------------------------------------------------------------------
// IoTruncateFile - resize the file; updates the cached Size on success.
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize) {
if (!file) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -861,7 +858,6 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize
return s;
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) {
if (!file) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
@@ -870,6 +866,7 @@ __declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) {
return STATUS_SUCCESS;
}
// Returns file Size.
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
if (!file) return 0;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
@@ -878,19 +875,19 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
return size;
}
// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
if (file) ObDereferenceObject(&file->Header);
}
// ---------------------------------------------------------------------------
// Directory enumeration
// ---------------------------------------------------------------------------
// Directory enumeration.
static void KiDirDestroy(OBJECT* obj) {
DIR_OBJECT* d = (DIR_OBJECT*)obj; // Header is first member
if (d->Fs && d->DirCtx) d->Fs->CloseDir(d->Fs, d->DirCtx);
}
// Opens dir.
__declspec(dllexport) KSTATUS IoOpenDir(SYS_STRING_VIEW dirPath, DIR_OBJECT** out) {
if (!out) return STATUS_INVALID_PARAMETER;
if (dirPath.Length && !dirPath.Data) return STATUS_INVALID_PARAMETER;
@@ -937,6 +934,7 @@ __declspec(dllexport) KSTATUS IoOpenDir(SYS_STRING_VIEW dirPath, DIR_OBJECT** ou
return STATUS_SUCCESS;
}
// Returns dir.
__declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) {
if (!dir || !entryOut) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -944,6 +942,7 @@ __declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) {
return dir->Fs->ReadDir(dir->Fs, dir->DirCtx, entryOut);
}
// Releases dir.
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir) {
if (dir) ObDereferenceObject(&dir->Header);
}
code/kernel/ke_apc.c
+13 -1
View File
@@ -1,4 +1,5 @@
// ke_apc.c - kernel APC infrastructure
// Kernel APC infrastructure. This file owns APC queuing, delivery, and user-mode return transitions.
#include <kernel/ke_apc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
@@ -9,6 +10,7 @@
#include <string.h>
#include <intrin.h>
// Initializes apc State.
void KiInitializeApcState(THREAD* thread) {
if (!thread) return;
LIST_INIT(&thread->ApcQueue);
@@ -88,6 +90,7 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
return true;
}
// Implements the enter Critical Region operation.
__declspec(dllexport) void KeEnterCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
@@ -97,6 +100,7 @@ __declspec(dllexport) void KeEnterCriticalRegion(void) {
KeReleaseSpinlock(&thread->ApcLock, flags);
}
// Implements the leave Critical Region operation.
__declspec(dllexport) void KeLeaveCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
@@ -111,12 +115,14 @@ __declspec(dllexport) void KeLeaveCriticalRegion(void) {
KiDeliverKernelApcs();
}
// Implements the are Kernel Apcs Disabled operation.
__declspec(dllexport) bool KeAreKernelApcsDisabled(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return true;
return thread->KernelApcDisable != 0;
}
// Implements the pop Kernel Apc operation.
static KAPC* KiPopKernelApc(THREAD* thread) {
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
LIST_ENTRY* node = ListPopHead(&thread->ApcQueue);
@@ -129,6 +135,7 @@ static KAPC* KiPopKernelApc(THREAD* thread) {
return CONTAINING_RECORD(node, KAPC, Link);
}
// Implements the deliver Kernel Apcs operation.
void KiDeliverKernelApcs(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread)
@@ -178,6 +185,7 @@ void KiDeliverKernelApcs(void) {
}
// Updates user Apc Dispatcher.
__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return STATUS_FAILURE;
@@ -221,6 +229,7 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
return STATUS_SUCCESS;
}
// Implements the pop User Apc operation.
static KUSER_APC* KiPopUserApc(THREAD* thread) {
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue);
@@ -233,6 +242,7 @@ static KUSER_APC* KiPopUserApc(THREAD* thread) {
return CONTAINING_RECORD(node, KUSER_APC, Link);
}
// Implements the check For User Apc Delivery operation.
void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
if (!frame || !KeTrapFrameFromUser(frame))
return;
@@ -288,6 +298,7 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
MmFreePool(apc);
}
// Implements the rundown Apc State operation.
void KiRundownApcState(THREAD* thread) {
if (!thread) return;
@@ -322,6 +333,7 @@ void KiRundownApcState(THREAD* thread) {
thread->UserApcDeliveryActive = 0;
}
// Implements the user Apc Return operation.
__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* userFrame) {
SYS_USER_APC_FRAME_K frameCopy;
KSTATUS s = CopyFromUser(&frameCopy, userFrame, sizeof(frameCopy));
code/kernel/ke_cpu.c
+9 -1
View File
@@ -1,4 +1,5 @@
// ke_cpu.c - bootstrap/per-CPU state plumbing
// CPU-local kernel state. This file owns bootstrap and per-CPU bookkeeping shared by scheduler and SMP code.
#include <stdint.h>
#include <string.h>
#include <intrin.h>
@@ -16,6 +17,7 @@ static volatile long gKernelGsReady = 0;
extern KCPU* KeReadCurrentCpuFromGs(void);
// Loads kernel Gs Base.
void KeLoadKernelGsBase(KCPU* cpu) {
if (!cpu)
cpu = &gCpus[0];
@@ -25,6 +27,7 @@ void KeLoadKernelGsBase(KCPU* cpu) {
gKernelGsReady = 1;
}
// Updates user Gs Base.
void KeSetUserGsBase(uintptr_t userGsBase) {
// While running in the kernel, GS_BASE is the current KCPU. The user TEB
// base is parked in IA32_KERNEL_GS_BASE so swapgs can exchange it with the
@@ -32,10 +35,12 @@ void KeSetUserGsBase(uintptr_t userGsBase) {
__writemsr(IA32_KERNEL_GS_BASE, (uint64_t)userGsBase);
}
// Returns user Gs Base.
uintptr_t KeGetUserGsBase(void) {
return (uintptr_t)__readmsr(IA32_KERNEL_GS_BASE);
}
// Initializes bootstrap Cpu.
void KeInitializeBootstrapCpu(void) {
memset(gCpus, 0, sizeof(gCpus));
@@ -73,6 +78,7 @@ void KeInitializeBootstrapCpu(void) {
KeSetUserGsBase(0);
}
// Returns current Cpu.
KCPU* KeGetCurrentCpu(void) {
if (gKernelGsReady) {
KCPU* cpu = KeReadCurrentCpuFromGs();
@@ -84,12 +90,14 @@ KCPU* KeGetCurrentCpu(void) {
return &gCpus[0];
}
// Returns cpu By Index.
KCPU* KeGetCpuByIndex(uint32_t index) {
if (index >= KE_MAX_CPUS)
return (KCPU*)0;
return &gCpus[index];
}
// Returns current Cpu Index.
uint32_t KeGetCurrentCpuIndex(void) {
return KeGetCurrentCpu()->CpuIndex;
}
code/kernel/ke_debug.c
+34 -8
View File
@@ -1,4 +1,5 @@
// ke_debug.c - debugging utilities
// Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.
#include <stdarg.h>
#include <stdint.h>
#include <string.h>
@@ -31,6 +32,7 @@ static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile long gDebugLogFlushing;
// Implements the debug Log Append Locked operation.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
return;
@@ -53,6 +55,7 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
gDebugLogUsed += len;
}
// Implements the debug Console Write operation.
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
return;
@@ -64,12 +67,14 @@ static volatile long gPanicInProgress = 0;
static volatile uintptr_t gPanicOwnerToken = 0;
static volatile long gPanicStoppedCpuCount = 0;
// Implements the panic Secondary Halt operation.
static _Noreturn void KiPanicSecondaryHalt(void) {
_disable();
_InterlockedIncrement(&gPanicStoppedCpuCount);
for (;;) __halt();
}
// Implements the panic Current Token operation.
static uintptr_t KiPanicCurrentToken(void) {
THREAD* thread = KeGetCurrentThread();
if (thread)
@@ -80,15 +85,18 @@ static uintptr_t KiPanicCurrentToken(void) {
return 1u;
}
// Implements the panic Is Active operation.
static int KiPanicIsActive(void) {
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
}
// Implements the panic Current Cpu Owns Report operation.
static int KiPanicCurrentCpuOwnsReport(void) {
uintptr_t token = KiPanicCurrentToken();
return token != 0 && gPanicOwnerToken == token;
}
// Implements the begin Fatal Report operation.
static void KiBeginFatalReport(void) {
_disable();
uintptr_t token = KiPanicCurrentToken();
@@ -114,11 +122,13 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
return 0;
}
// Implements the kernel Put Char operation.
void KiKernelPutChar(char c) {
if (PrintChar)
PrintChar(c, 0xCCCCCC, 0x012456);
}
// Initializes debug.
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
gDebugFlags = flags;
gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
@@ -136,10 +146,12 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
}
}
// Returns debug Flags.
uint32_t KeGetDebugFlags(void) {
return gDebugFlags;
}
// Updates debug Console Writer.
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
if (!PrintPtr)
return;
@@ -147,6 +159,7 @@ void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
printf_set_putchar(KiKernelPutChar);
}
// Implements the dbg Print operation.
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled || !fmt) return;
@@ -179,6 +192,7 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
KeReleaseSpinlock(&gDebugPrintLock, flags);
}
// Flushes debug Log To File.
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
return STATUS_INVALID_PARAMETER;
@@ -228,7 +242,7 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// KePanic
// KePanic.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
KiBeginFatalReport();
@@ -276,13 +290,11 @@ __declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
PrintChar(buf[i], 0xCCCCCC, 0x012456);
}
// ---------------------------------------------------------------------------
// KeBugCheckProcess - user-mode fault handler
//
// Resolves the faulting RIP to a module name, prints a yellow crash banner,
// kills the offending process, then exits the current thread so the scheduler
// can continue running everything else.
// ---------------------------------------------------------------------------
// Return the short exception name for a CPU exception vector.
// Keep this table complete for vectors 0..31 so faults never collapse into
@@ -326,6 +338,7 @@ static const char* KiFaultVectorName(int vector) {
}
}
// Implements the fault Has Error Code operation.
static int KiFaultHasErrorCode(int vector) {
switch (vector) {
case 8: // #DF
@@ -374,10 +387,12 @@ static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
return "unknown";
}
// Implements the resolve Module operation.
static const char* KiResolveModule(PROCESS* proc, uintptr_t addr) {
return KiResolveModuleEx(proc, addr, (uintptr_t*)0, (size_t*)0);
}
// Prints address.
static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
uintptr_t base = 0;
size_t size = 0;
@@ -396,6 +411,7 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
}
}
// Implements the user Read U64 operation.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
if (!out) return 0;
if (!addr) return 0;
@@ -405,6 +421,7 @@ static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
return 1;
}
// Implements the user Read U8 operation.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
if (!out) return 0;
if (!addr) return 0;
@@ -414,6 +431,7 @@ static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
return 1;
}
// Dumps bytes.
static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
if (!center) {
printf("%s: <null>\n", label);
@@ -438,6 +456,7 @@ static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
}
}
// Implements the user Read I32 operation.
static int KiUserReadI32(uintptr_t addr, int32_t* out) {
if (!out) return 0;
if (!addr) return 0;
@@ -447,6 +466,7 @@ static int KiUserReadI32(uintptr_t addr, int32_t* out) {
return 1;
}
// Dumps indirect Call Slot.
static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
if (!ret) return;
@@ -461,12 +481,14 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
if (KiUserReadU8(ret - 6, &b0) &&
KiUserReadU8(ret - 5, &b1) &&
b0 == 0xff && b1 == 0x15 &&
// Implements the user Read I32 operation.
KiUserReadI32(ret - 4, &disp)) {
insn = ret - 6;
} else if (KiUserReadU8(ret - 7, &b0) &&
KiUserReadU8(ret - 6, &b1) &&
KiUserReadU8(ret - 5, &b2) &&
b0 == 0x48 && b1 == 0xff && b2 == 0x15 &&
// Implements the user Read I32 operation.
KiUserReadI32(ret - 4, &disp)) {
insn = ret - 7;
} else {
@@ -490,6 +512,7 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
KiDumpBytes(proc, "Bytes near indirect slot", slot);
}
// Dumps modules.
static void KiDumpModules(PROCESS* proc) {
if (!proc) return;
@@ -513,6 +536,7 @@ static void KiDumpModules(PROCESS* proc) {
}
}
// Dumps user Stack.
static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
printf("User stack @ %016llx:\n", (unsigned long long)rsp);
if (!rsp) {
@@ -548,6 +572,7 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
}
}
// Dumps fault Frame.
static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
if (!f) return;
@@ -586,6 +611,7 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
KiDumpUserStack(proc, (uintptr_t)f->Rsp);
}
// Prints crash Char.
static void KiPrintCrashChar(char c) {
if (PrintChar)
PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic
@@ -621,6 +647,7 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
(unsigned long long)errCode);
}
// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread) {
printf_set_putchar(KiKernelPutChar);
@@ -640,6 +667,7 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
KiFinishFaultedProcess(proc, thread);
}
// Implements the fault Dispatch Frame operation.
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
// the interrupted context on a secondary CPU once another CPU owns panic.
@@ -718,13 +746,11 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
KiFinishFaultedProcess(proc, thread);
}
// ---------------------------------------------------------------------------
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
//
// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and
// call here. We check the CS RPL to decide: kernel fault → panic and halt,
// user fault → KeBugCheckProcess (kill process, resume scheduler).
// ---------------------------------------------------------------------------
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
uintptr_t errCode, uintptr_t cr2,
uintptr_t cs) {
code/kernel/ke_display.c
+48 -6
View File
@@ -1,9 +1,4 @@
// ke_display.c - kernel display manager
//
// The display manager owns the primary display device, cursor policy, dirty
// rectangles, and the kernel/user screen API. The first provider is a boot
// framebuffer backend over the HAL GOP framebuffer; later GPU drivers register
// DISPLAY_DEVICE providers here.
// Kernel display manager. This file owns provider registration, framebuffer routing, dirty rectangles, text console state, and cursor delegation.
#include <stdint.h>
#include <stdbool.h>
@@ -96,28 +91,33 @@ static const DISPLAY_DEVICE_OPS gBootFbOps = {
};
// Checks whether a primary display provider is available.
static bool KiDisplayHasDevice(void) {
return gPrimaryDisplay && gPrimaryDisplay->Ops;
}
// Checks whether the active provider expects the manager to draw the cursor into the framebuffer.
static bool KiDisplayUsesSoftwareCursor(void) {
return KiDisplayHasDevice() &&
(gPrimaryDisplay->Caps & DISPLAY_CAP_SOFTWARE_CURSOR) &&
gPrimaryDisplay->Ops->ShowCursor;
}
// Checks whether the active provider owns cursor presentation in hardware or device firmware.
static bool KiDisplayUsesHardwareCursor(void) {
return KiDisplayHasDevice() &&
(gPrimaryDisplay->Caps & DISPLAY_CAP_HARDWARE_CURSOR) &&
gPrimaryDisplay->Ops->ShowCursor;
}
// Checks whether the active provider exposes any cursor-control path.
static bool KiDisplayHasCursorOps(void) {
return KiDisplayHasDevice() &&
(gPrimaryDisplay->Caps & (DISPLAY_CAP_SOFTWARE_CURSOR | DISPLAY_CAP_HARDWARE_CURSOR)) &&
gPrimaryDisplay->Ops->ShowCursor;
}
// Clips a rectangle to a display mode so providers never see negative or out-of-bounds regions.
static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) {
if (!mode || !rect || rect->W <= 0 || rect->H <= 0)
return false;
@@ -141,12 +141,14 @@ static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) {
return true;
}
// Implements the display Dirty All operation.
static void KiDisplayDirtyAll(void) {
gDisplayStats.DirtyAllCount++;
gDirtyAll = true;
gDirtyRectCount = 0;
}
// Implements the display Add Dirty operation.
static void KiDisplayAddDirty(int x, int y, int w, int h) {
if (!KiDisplayHasDevice() || gDirtyAll || w <= 0 || h <= 0)
return;
@@ -174,6 +176,7 @@ static void KiDisplayAddDirty(int x, int y, int w, int h) {
gDirtyRects[gDirtyRectCount++] = rect;
}
// Implements the display Flush Dirty operation.
static KSTATUS KiDisplayFlushDirty(void) {
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Present)
return STATUS_SUCCESS;
@@ -207,6 +210,7 @@ static KSTATUS KiDisplayFlushDirty(void) {
return s;
}
// Implements the display Hide Cursor For Draw operation.
static void KiDisplayHideCursorForDraw(void) {
if (!gCursorDrawn || !KiDisplayUsesSoftwareCursor())
return;
@@ -215,6 +219,7 @@ static void KiDisplayHideCursorForDraw(void) {
gCursorDrawn = false;
}
// Implements the display Restore Cursor After Draw operation.
static void KiDisplayRestoreCursorAfterDraw(void) {
if (!gCursorVisible || gCursorDrawn || !KiDisplayHasCursorOps())
return;
@@ -226,11 +231,13 @@ static void KiDisplayRestoreCursorAfterDraw(void) {
gCursorDrawn = true;
}
// Implements the display Begin Draw operation.
static void KiDisplayBeginDraw(void) {
if (gDrawDepth++ == 0)
KiDisplayHideCursorForDraw();
}
// Implements the display End Draw operation.
static KSTATUS KiDisplayEndDraw(void) {
KSTATUS s = STATUS_SUCCESS;
if (gDrawDepth > 0)
@@ -243,6 +250,7 @@ static KSTATUS KiDisplayEndDraw(void) {
return s;
}
// Implements the display Draw Blit No Cursor operation.
static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint32_t* pixels) {
if (!pixels || w <= 0 || h <= 0)
return STATUS_INVALID_PARAMETER;
@@ -258,6 +266,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3
return s;
}
// Implements the display Fill Rect No Cursor operation.
static KSTATUS KiDisplayFillRectNoCursor(int x, int y, int w, int h, uint32_t color) {
if (w <= 0 || h <= 0)
return STATUS_INVALID_PARAMETER;
@@ -273,6 +282,7 @@ static KSTATUS KiDisplayFillRectNoCursor(int x, int y, int w, int h, uint32_t co
return s;
}
// Implements the display Copy Rect No Cursor operation.
static KSTATUS KiDisplayCopyRectNoCursor(int dstX, int dstY, int srcX, int srcY, int w, int h) {
if (w <= 0 || h <= 0)
return STATUS_INVALID_PARAMETER;
@@ -288,10 +298,12 @@ static KSTATUS KiDisplayCopyRectNoCursor(int dstX, int dstY, int srcX, int srcY,
return s;
}
// Implements the display Console Index operation.
static size_t KiDisplayConsoleIndex(int col, int row) {
return (size_t)row * DISPLAY_CONSOLE_MAX_COLS + (size_t)col;
}
// Implements the display Console Color operation.
static uint32_t KiDisplayConsoleColor(uint8_t index) {
static const uint32_t colors[16] = {
0x00000000u, 0x000000AAu, 0x0000AA00u, 0x0000AAAAu,
@@ -302,14 +314,17 @@ static uint32_t KiDisplayConsoleColor(uint8_t index) {
return colors[index & 0x0Fu];
}
// Implements the display Console Fg operation.
static uint32_t KiDisplayConsoleFg(uint16_t attr) {
return KiDisplayConsoleColor((uint8_t)(attr & 0x0Fu));
}
// Implements the display Console Bg operation.
static uint32_t KiDisplayConsoleBg(uint16_t attr) {
return KiDisplayConsoleColor((uint8_t)((attr >> 4) & 0x0Fu));
}
// Implements the display Get Console Grid Locked operation.
static bool KiDisplayGetConsoleGridLocked(int* outCols, int* outRows) {
DISPLAY_MODE mode;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode)
@@ -329,6 +344,7 @@ static bool KiDisplayGetConsoleGridLocked(int* outCols, int* outRows) {
return true;
}
// Implements the display Console Reset Cells Locked operation.
static void KiDisplayConsoleResetCellsLocked(void) {
for (size_t i = 0; i < DISPLAY_CONSOLE_MAX_CELLS; i++) {
gConsoleCells[i] = ' ';
@@ -337,6 +353,7 @@ static void KiDisplayConsoleResetCellsLocked(void) {
gConsoleAttribute = DISPLAY_CONSOLE_DEFAULT_ATTR;
}
// Implements the display Console Scroll Cells Locked operation.
static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines) {
if (cols <= 0 || rows <= 0 || cellLines <= 0)
return;
@@ -370,6 +387,7 @@ static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines)
}
}
// Implements the display Draw Char Pixels Locked operation.
static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int x, int y, uint32_t fg, uint32_t bg) {
uint32_t glyphPixels[FONT_W * FONT_H];
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u);
@@ -383,6 +401,7 @@ static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int x, int y, uint32_t fg,
return KiDisplayDrawBlitNoCursor(x, y, FONT_W, FONT_H, glyphPixels);
}
// Implements the display Draw Console Cell Locked operation.
static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t attr, bool updateCell) {
if (updateCell) {
size_t i = KiDisplayConsoleIndex(col, row);
@@ -395,6 +414,7 @@ static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t
KiDisplayConsoleBg(attr));
}
// Implements the display Serial Write Char operation.
static void KiDisplaySerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n')
@@ -402,6 +422,7 @@ static void KiDisplaySerialWriteChar(char c) {
__outbyte(0x3F8, (uint8_t)c);
}
// Implements the display Initialize operation.
void KeDisplayInitialize(const HAL_API* hal) {
gDisplayHal = hal;
gDisplayList = NULL;
@@ -498,6 +519,7 @@ DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
return result;
}
// Implements the display Seed New Primary operation.
static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE* newDevice) {
if (!oldDevice || !newDevice || oldDevice == newDevice)
return STATUS_SUCCESS;
@@ -540,6 +562,7 @@ DISPLAY_API void KeDisplaySuspendConsole(bool suspend) {
_InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, suspend ? 1 : 0);
}
// Implements the display Output Suspended operation.
static bool KiDisplayOutputSuspended(void) {
return _InterlockedCompareExchange((volatile long*)&gDisplayConsoleSuspended, 0, 0) != 0;
}
@@ -644,6 +667,7 @@ DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color
return KSUCCESS(s) ? ps : s;
}
// Implements the display Clear Locked operation.
static KSTATUS KiDisplayClearLocked(uint32_t color) {
DISPLAY_MODE mode;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode)
@@ -675,6 +699,7 @@ DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, in
return KSUCCESS(s) ? ps : s;
}
// Implements the display Scroll Locked operation.
static KSTATUS KiDisplayScrollLocked(int lines) {
if (lines <= 0)
return STATUS_SUCCESS;
@@ -705,6 +730,7 @@ DISPLAY_API KSTATUS KeDisplayScroll(int lines) {
return s;
}
// Implements the display Put Char Locked operation.
static KSTATUS KiDisplayPutCharLocked(char c, int x, int y, uint32_t fg, uint32_t bg) {
KiDisplayBeginDraw();
KSTATUS s = KiDisplayDrawCharPixelsLocked(c, x, y, fg, bg);
@@ -789,6 +815,7 @@ DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
return STATUS_SUCCESS;
}
// Implements the display Debug Print Char operation.
void KeDisplayDebugPrintChar(char c, uint32_t fg, uint32_t bg) {
(void)KeDisplayPrintChar(c, fg, bg);
}
@@ -1069,10 +1096,12 @@ DISPLAY_API KSTATUS KeDisplayShowCursor(bool show) {
return s;
}
// Implements the boot Fb Context operation.
static BOOTFB_CONTEXT* BootFbContext(DISPLAY_DEVICE* device) {
return device ? (BOOTFB_CONTEXT*)device->DriverContext : NULL;
}
// Implements the boot Fb Get Mode operation.
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !outMode)
@@ -1085,6 +1114,7 @@ static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
return STATUS_SUCCESS;
}
// Implements the boot Fb Blit operation.
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0)
@@ -1114,6 +1144,7 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co
return STATUS_SUCCESS;
}
// Implements the boot Fb Fill Rect operation.
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
@@ -1139,6 +1170,7 @@ static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h
return STATUS_SUCCESS;
}
// Implements the boot Fb Copy Rect operation.
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
@@ -1177,6 +1209,7 @@ static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int sr
return STATUS_SUCCESS;
}
// Implements the boot Fb Read Rect operation.
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
@@ -1201,6 +1234,7 @@ static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h
return STATUS_SUCCESS;
}
// Implements the boot Fb Present operation.
static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer)
@@ -1231,12 +1265,14 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects,
return STATUS_SUCCESS;
}
// Implements the boot Fb Sync operation.
static KSTATUS BootFbSync(DISPLAY_DEVICE* device) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (fb) fb->Stats.SyncCount++;
return STATUS_SUCCESS;
}
// Implements the boot Fb Set Cursor Image operation.
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
(void)device;
(void)width;
@@ -1247,6 +1283,7 @@ static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int heigh
return STATUS_NOT_SUPPORTED;
}
// Implements the boot Fb Set Cursor Position operation.
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (fb) fb->Stats.CursorMoveCount++;
@@ -1255,6 +1292,7 @@ static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
return STATUS_SUCCESS;
}
// Implements the display Blend Pixel operation.
static inline uint32_t KiDisplayBlendPixel(uint32_t dst, uint32_t src) {
uint8_t a = (uint8_t)(src >> 24);
if (a == 0) return dst;
@@ -1273,6 +1311,7 @@ static inline uint32_t KiDisplayBlendPixel(uint32_t dst, uint32_t src) {
return ((uint32_t)rr << 16) | ((uint32_t)gg << 8) | bb;
}
// Implements the boot Fb Restore Cursor operation.
static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
if (!fb || !fb->Framebuffer || !fb->Backbuffer || gCursorDrawX < 0)
return;
@@ -1291,6 +1330,7 @@ static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
gCursorDrawY = -1;
}
// Implements the boot Fb Draw Cursor operation.
static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int x, int y) {
if (!fb || !fb->Framebuffer || !fb->Backbuffer)
return;
@@ -1315,6 +1355,7 @@ static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int x, int y) {
}
}
// Implements the boot Fb Show Cursor operation.
static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !fb->Backbuffer)
@@ -1330,6 +1371,7 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
return STATUS_SUCCESS;
}
// Implements the boot Fb Query Stats operation.
static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !outStats)
code/kernel/ke_dpc.c
+5 -1
View File
@@ -1,4 +1,5 @@
// ke_dpc.c - deferred procedure calls
// Deferred procedure call infrastructure. This file owns DPC queuing and execution outside hard IRQ context.
#include <kernel/ke_dpc.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_debug.h>
@@ -9,6 +10,7 @@ static SPINLOCK gDpcLock;
static volatile long gDpcInitialized = 0;
static volatile long gDpcActive = 0;
// Initializes dpc System.
void KeInitializeDpcSystem(void) {
LIST_INIT(&gDpcQueue);
gDpcLock = (SPINLOCK)SPINLOCK_INIT;
@@ -16,6 +18,7 @@ void KeInitializeDpcSystem(void) {
gDpcInitialized = 1;
}
// Implements the ensure Dpc Initialized operation.
static void KiEnsureDpcInitialized(void) {
if (gDpcInitialized)
return;
@@ -60,6 +63,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
return true;
}
// Implements the drain Dpc Queue operation.
void KiDrainDpcQueue(void) {
if (!gDpcInitialized)
return;
code/kernel/ke_event.c
+7 -1
View File
@@ -1,4 +1,5 @@
// ke_event.c - synchronisation events
// Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
#include <stdbool.h>
#include <kernel/ke_event.h>
#include <kernel/ke_thread.h>
@@ -7,8 +8,10 @@
#include <kernel/ke_apc.h>
#include <kernel/ob_object.h>
// Implements the event Destroy operation.
static void KiEventDestroy(OBJECT* obj) { (void)obj; }
// Implements the event Create operation.
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
if (!out) return STATUS_INVALID_PARAMETER;
EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
@@ -22,6 +25,7 @@ __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialSta
return STATUS_SUCCESS;
}
// Signals event.
__declspec(dllexport) void KeSignalEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
@@ -52,6 +56,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
KeReleaseSpinlock(&event->Lock, flags);
}
// Clears event.
__declspec(dllexport) void KeClearEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
@@ -59,6 +64,7 @@ __declspec(dllexport) void KeClearEvent(EVENT* event) {
KeReleaseSpinlock(&event->Lock, flags);
}
// Waits for for Event.
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
if (!event) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
code/kernel/ke_idt.c
+18 -15
View File
@@ -1,5 +1,5 @@
// ke_idt.c - GDT, IDT, and TSS initialisation
//
// Kernel descriptor-table setup. This file owns GDT, IDT, and TSS initialization.
// GDT layout (matches segment selectors in ke_switch.asm / ke_syscall.asm):
// 0x00 null
// 0x08 ring-0 code (64-bit)
@@ -7,13 +7,13 @@
// 0x18 ring-3 code (64-bit, DPL=3)
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x28 TSS low
// 0x30 TSS high
// 0x30 TSS high.
#include <stdint.h>
#include <string.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_cpu.h>
// External assembly procedures
// External assembly procedures.
extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
extern void KeFlushIdt(void* pIdtr);
@@ -76,9 +76,7 @@ extern void KeFaultRSV31(void);
#define NUM_IRQS 24
// ---------------------------------------------------------------------------
// GDT
// ---------------------------------------------------------------------------
// GDT.
static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
uint8_t access, uint8_t flags) {
gdt[idx] =
@@ -90,6 +88,7 @@ static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
((uint64_t)(base >> 24 & 0xFF)<< 56);
}
// Implements the gdt Set Tss operation.
static void KiGdtSetTss(uint64_t* gdt, int idx, uintptr_t base, uint32_t limit) {
// TSS descriptor is 16 bytes (two GDT slots).
gdt[idx] = 0;
@@ -105,6 +104,7 @@ static void KiGdtSetTss(uint64_t* gdt, int idx, uintptr_t base, uint32_t limit)
gdt[idx+1] = (base >> 32);
}
// Initializes gdt For Cpu.
void KeInitializeGdtForCpu(KCPU* cpu) {
if (!cpu) return;
@@ -113,15 +113,15 @@ void KeInitializeGdtForCpu(KCPU* cpu) {
memset(gdt, 0, sizeof(cpu->Gdt));
// 0x00: Null
// 0x00: Null.
KiGdtSetEntry(gdt, 0, 0, 0, 0, 0);
// 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit)
// 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit).
KiGdtSetEntry(gdt, 1, 0, 0xFFFFF, 0x9A, 0xA);
// 0x10: Ring-0 Data
// 0x10: Ring-0 Data.
KiGdtSetEntry(gdt, 2, 0, 0xFFFFF, 0x92, 0xC);
// 0x18: Ring-3 Code
// 0x18: Ring-3 Code.
KiGdtSetEntry(gdt, 3, 0, 0xFFFFF, 0xFA, 0xA);
// 0x20: Ring-3 Data
// 0x20: Ring-3 Data.
KiGdtSetEntry(gdt, 4, 0, 0xFFFFF, 0xF2, 0xC);
// 0x28: CPU-local TSS (occupies slots 0x28 and 0x30).
@@ -135,6 +135,7 @@ void KeInitializeGdtForCpu(KCPU* cpu) {
cpu->Gdtr.Base = (uint64_t)cpu->Gdt;
}
// Loads gdt For Current Cpu.
void KeLoadGdtForCurrentCpu(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
KeFlushGdt(&cpu->Gdtr, 0x08, 0x10, 0x28);
@@ -145,21 +146,21 @@ void KeLoadGdtForCurrentCpu(KCPU* cpu) {
KeLoadKernelGsBase(cpu);
}
// Initializes gdt.
void KeInitializeGdt(void) {
KCPU* cpu = KeGetCurrentCpu();
KeInitializeGdtForCpu(cpu);
KeLoadGdtForCurrentCpu(cpu);
}
// Updates current Tss Rsp0.
void KeSetCurrentTssRsp0(uintptr_t rsp0) {
TSS64* tss = KeGetCurrentTss();
if (tss)
tss->RSP0 = rsp0;
}
// ---------------------------------------------------------------------------
// IDT
// ---------------------------------------------------------------------------
// IDT.
#define IDT_ENTRIES 256
#pragma pack(push, 1)
@@ -191,6 +192,7 @@ static void KiIdtSetGate(int vec, uintptr_t handler,
gIdt[vec].MustBeZero = 0;
}
// Initializes idt.
void KeInitializeIdt(void) {
memset(gIdt, 0, sizeof(gIdt));
@@ -231,6 +233,7 @@ void KeInitializeIdt(void) {
KeLoadIdtForCurrentCpu();
}
// Loads idt For Current Cpu.
void KeLoadIdtForCurrentCpu(void) {
IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt };
KeFlushIdt(&idtr);
code/kernel/ke_input.c
+26 -1
View File
@@ -1,4 +1,5 @@
// ke_input.c - kernel-owned canonical input core
// Kernel input core. This file owns canonical keyboard and mouse state shared by input drivers and syscalls.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
@@ -76,6 +77,7 @@ static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Implements the input Queue Key operation.
static void KiInputQueueKey(INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
@@ -85,11 +87,13 @@ static void KiInputQueueKey(INPUT_KEY key, char ch) {
gKeyTail = next;
}
// Implements the input Queue Has Space operation.
static bool KiInputQueueHasSpace(void) {
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
return next != gKeyHead;
}
// Implements the input Key Is Repeatable operation.
static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return false;
@@ -110,18 +114,21 @@ static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
}
}
// Implements the input Now Ticks operation.
static uint64_t KiInputNowTicks(void) {
if (gInputHal && gInputHal->Timer.QueryTicks)
return gInputHal->Timer.QueryTicks();
return 0;
}
// Implements the input Start Repeat operation.
static void KiInputStartRepeat(INPUT_KEY key, uint64_t now) {
if (!KiInputKeyIsRepeatable(key)) return;
gRepeatKey = key;
gRepeatNextTick = now + KEY_REPEAT_DELAY_US;
}
// Implements the input Stop Repeat operation.
static void KiInputStopRepeat(INPUT_KEY key) {
if (gRepeatKey == key) {
gRepeatKey = INPUT_KEY_NONE;
@@ -129,6 +136,7 @@ static void KiInputStopRepeat(INPUT_KEY key) {
}
}
// Implements the input Service Repeat operation.
static void KiInputServiceRepeat(void) {
INPUT_KEY key = gRepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key]) {
@@ -150,6 +158,7 @@ static void KiInputServiceRepeat(void) {
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
}
// Implements the input Set Modifier operation.
static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
uint32_t bit = 0;
switch (key) {
@@ -173,6 +182,7 @@ static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
if (pressed && key == INPUT_KEY_NUM_LOCK) gModifiers ^= INPUT_MOD_NUM_LOCK;
}
// Implements the input Key To Char operation.
static char KiInputKeyToChar(INPUT_KEY key) {
if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) {
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
@@ -190,6 +200,7 @@ static char KiInputKeyToChar(INPUT_KEY key) {
return 0;
}
// Implements the input Initialize operation.
void KeInputInitialize(const HAL_API* hal) {
gInputHal = hal;
memset((void*)gKeyDown, 0, sizeof(gKeyDown));
@@ -214,6 +225,7 @@ void KeInputInitialize(const HAL_API* hal) {
DbgPrint("Input: kernel input core initialized (%dx%d)\n", w, h);
}
// Implements the input Register Device operation.
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) {
(void)flags;
uint32_t id = gNextInputDeviceId++;
@@ -223,6 +235,7 @@ __declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t f
return STATUS_SUCCESS;
}
// Implements the input Submit Key operation.
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
if (!event) return;
INPUT_KEY key = event->Key;
@@ -251,6 +264,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
}
}
// Implements the input Commit Mouse Position operation.
static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
@@ -268,6 +282,7 @@ static void KiInputCommitMousePosition(int nx, int ny, uint8_t buttons) {
KeDisplaySetCursorPosition(nx, ny);
}
// Implements the input Submit Mouse operation.
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
if (!event) return;
@@ -285,6 +300,7 @@ __declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
}
}
// Implements the input Submit Mouse Absolute operation.
__declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE_EVENT* event) {
if (!event) return;
@@ -305,6 +321,7 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
}
}
// Implements the input Submit Mouse Absolute Scaled operation.
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
@@ -324,6 +341,7 @@ INPUT_KEY KeInputKbPoll(void) {
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
return gKeyDown[key] != 0;
@@ -337,6 +355,7 @@ INPUT_KEY KeInputReadKey(void) {
return key;
}
// Implements the input Read Char operation.
char KeInputReadChar(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return 0;
@@ -345,14 +364,17 @@ char KeInputReadChar(void) {
return ch;
}
// Implements the input Get Modifiers operation.
uint32_t KeInputGetModifiers(void) {
return gModifiers;
}
// Implements the input Mouse Poll operation.
bool KeInputMousePoll(void) {
return gMouseHead != gMouseTail;
}
// Implements the input Mouse Read operation.
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
if (!dx || !dy || !buttons) return false;
if (gMouseHead == gMouseTail) return false;
@@ -365,11 +387,13 @@ bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
return true;
}
// Implements the input Mouse Get Position operation.
void KeInputMouseGetPosition(int* x, int* y) {
if (x) *x = gMouseX;
if (y) *y = gMouseY;
}
// Implements the input Mouse Set Position operation.
void KeInputMouseSetPosition(int x, int y) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
@@ -384,6 +408,7 @@ void KeInputMouseSetPosition(int x, int y) {
KeDisplaySetCursorPosition(x, y);
}
// Implements the input Mouse Set Bounds operation.
void KeInputMouseSetBounds(int w, int h) {
if (w < 1) w = 1;
if (h < 1) h = 1;
code/kernel/ke_irp.c
+24 -13
View File
@@ -1,4 +1,4 @@
// ke_irp.c - I/O Request Packet dispatch, queue, cancellation, remove-locks
// I/O request packet infrastructure. This file owns IRP dispatch, queuing, cancellation, and remove-lock helpers.
#include <string.h>
#include <intrin.h>
@@ -10,9 +10,7 @@
#include <kernel/mm_virt.h>
#include <ntlite/ddi/io_driver.h>
// -------------------------------------------------------------------------
// IRP_QUEUE operations
// -------------------------------------------------------------------------
// IRP_QUEUE operations.
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
q->Lock = (SPINLOCK)SPINLOCK_INIT;
@@ -20,6 +18,7 @@ __declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
LIST_INIT(&q->Head);
}
// Implements the irp Queue Enqueue operation.
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
if (irp->Flags & IRP_FLAG_PRIORITY)
@@ -30,6 +29,7 @@ __declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
KeSignalSemaphore((SEMAPHORE*)q->Semaphore);
}
// Implements the irp Queue Dequeue operation.
__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY* node = ListPopHead(&q->Head);
@@ -58,13 +58,12 @@ __declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
}
}
// Implements the irp Queue Cancel All operation.
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
}
// -------------------------------------------------------------------------
// Remove locks
// -------------------------------------------------------------------------
// Remove locks.
__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return STATUS_INVALID_PARAMETER;
@@ -76,6 +75,7 @@ __declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
return STATUS_SUCCESS;
}
// Releases remove Lock.
__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
if (!lock) return;
if (lock->RemovedEvent) {
@@ -84,6 +84,7 @@ __declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
}
}
// Implements the acquire Remove Lock operation.
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return STATUS_INVALID_PARAMETER;
@@ -97,6 +98,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
return STATUS_SUCCESS;
}
// Releases remove Lock.
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return;
@@ -105,6 +107,7 @@ __declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
KeSignalEvent(lock->RemovedEvent);
}
// Releases remove Lock And Wait.
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp) {
(void)removeIrp;
if (!lock) return;
@@ -124,15 +127,14 @@ __declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP*
KeWaitForEvent(lock->RemovedEvent, 0);
}
// -------------------------------------------------------------------------
// IRP helpers
// -------------------------------------------------------------------------
// IRP helpers.
static size_t IoIrpSize(uint8_t stackCount) {
if (stackCount == 0) stackCount = 1;
return sizeof(IRP) + ((size_t)stackCount - 1u) * sizeof(IO_STACK_LOCATION);
}
// Creates irp.
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
if (!outIrp) return STATUS_INVALID_PARAMETER;
if (stackCount == 0) stackCount = 1;
@@ -143,10 +145,12 @@ __declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
return STATUS_SUCCESS;
}
// Releases irp.
__declspec(dllexport) void IoFreeIrp(IRP* irp) {
if (irp) MmFreePool(irp);
}
// Initializes irp.
__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
if (!irp) return;
if (stackCount == 0) stackCount = 1;
@@ -159,22 +163,26 @@ __declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
irp->CurrentLocation = 0;
}
// Returns current Irp Stack Location.
__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation];
}
// Returns next Irp Stack Location.
__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) {
if (!irp || irp->CurrentLocation + 1u >= irp->StackCount) return (IO_STACK_LOCATION*)0;
return &irp->Stack[irp->CurrentLocation + 1u];
}
// Implements the skip Current Irp Stack Location operation.
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) {
if (!irp) return;
if (irp->CurrentLocation + 1u < irp->StackCount)
irp->CurrentLocation++;
}
// Copies current Irp Stack Location To Next.
__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
@@ -183,12 +191,14 @@ __declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
irp->CurrentLocation++;
}
// Updates completion Routine.
__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx) {
if (!irp) return;
irp->CompletionFn = fn;
irp->CompletionCtx = ctx;
}
// Updates cancel Routine.
__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine) {
if (!irp) return (IRP_CANCEL_ROUTINE)0;
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
@@ -198,6 +208,7 @@ __declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL
return old;
}
// Cancels irp.
__declspec(dllexport) void IoCancelIrp(IRP* irp) {
if (!irp) return;
@@ -262,9 +273,7 @@ __declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
ios->Parameters.DeviceIoControl.OutputLength = outputLength;
}
// -------------------------------------------------------------------------
// Completion/submission
// -------------------------------------------------------------------------
// Completion/submission.
__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information) {
if (!irp) return STATUS_INVALID_PARAMETER;
@@ -293,6 +302,7 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t
return STATUS_SUCCESS;
}
// Submits irp.
__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
irp->Status = STATUS_FAILURE;
@@ -301,6 +311,7 @@ __declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
return IoCallDriver(device, irp);
}
// Submits irp Sync.
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp) return STATUS_INVALID_PARAMETER;
code/kernel/ke_irq.c
+9 -1
View File
@@ -1,4 +1,5 @@
// ke_irq.c - IRQ dispatch, kernel-side
// Interrupt manager. This file owns IRQ routing, handler dispatch, and post-interrupt scheduling hooks.
#include <kernel/ke_irq.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
@@ -24,6 +25,7 @@ static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
static bool gIrqSoftwareVector[MAX_IRQS];
static const HAL_API *gHal;
// Implements the irq To Irql operation.
static KIRQL KiIrqToIrql(uint8_t irq) {
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)irq;
if (level >= HIGH_LEVEL)
@@ -31,10 +33,12 @@ static KIRQL KiIrqToIrql(uint8_t irq) {
return (KIRQL)level;
}
// Implements the irq Initialize operation.
void KeIrqInitialize(const HAL_API *hal) {
gHal = hal;
}
// Implements the irq Allocate Msi Vector operation.
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
if (!outIrq) return STATUS_INVALID_PARAMETER;
@@ -52,6 +56,7 @@ __declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
return STATUS_NO_MEMORY;
}
// Implements the irq Register operation.
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
@@ -71,6 +76,7 @@ __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
gHal->Intc.RouteIrq(irq);
}
// Implements the irq Dispatch operation.
void KeIrqDispatch(uint8_t irq) {
KIRQL targetIrql = KiIrqToIrql(irq);
KIRQL currentIrql = KeGetCurrentIrql();
@@ -103,10 +109,12 @@ void KeIrqDispatch(uint8_t irq) {
KeLowerIrql(oldIrql);
}
// Implements the enter Interrupt operation.
void KiEnterInterrupt(void) {
KeGetCurrentCpu()->InterruptDepth++;
}
// Implements the exit Interrupt operation.
void KiExitInterrupt(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu->InterruptDepth)
code/kernel/ke_irql.c
+9 -1
View File
@@ -1,9 +1,11 @@
// ke_irql.c - software IRQL skeleton
// Kernel IRQL model. This file owns the software interrupt-priority state used by IRQ and scheduler code.
#include <kernel/ke_irql.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_apc.h>
// Implements the irql Name operation.
__declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
switch (irql) {
case PASSIVE_LEVEL: return "PASSIVE_LEVEL";
@@ -17,10 +19,12 @@ __declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
}
}
// Returns current Irql.
__declspec(dllexport) KIRQL KeGetCurrentIrql(void) {
return KeGetCurrentCpu()->CurrentIrql;
}
// Implements the raise Irql operation.
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
KCPU* cpu = KeGetCurrentCpu();
KIRQL oldIrql = cpu->CurrentIrql;
@@ -39,6 +43,7 @@ __declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
return oldIrql;
}
// Implements the lower Irql operation.
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KCPU* cpu = KeGetCurrentCpu();
KIRQL currentIrql = cpu->CurrentIrql;
@@ -59,6 +64,7 @@ __declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KiDeliverKernelApcs();
}
// Implements the assert Irql operation.
__declspec(dllexport) void KeAssertIrql(KIRQL expected) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql != expected) {
@@ -68,6 +74,7 @@ __declspec(dllexport) void KeAssertIrql(KIRQL expected) {
}
}
// Implements the assert Irql At Or Below operation.
__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql > maxIrql) {
@@ -77,6 +84,7 @@ __declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
}
}
// Checks whether wait.
__declspec(dllexport) int KeCanWait(void) {
return KeGetCurrentIrql() == PASSIVE_LEVEL;
}
code/kernel/ke_mutex.c
+7 -1
View File
@@ -1,4 +1,5 @@
// ke_mutex.c - binary ownership mutex
// Kernel mutex primitive. This file owns exclusive ownership, wait queues, and priority-aware wakeups.
#include <stdbool.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_thread.h>
@@ -7,8 +8,10 @@
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
// Implements the mutex Destroy operation.
static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
// Implements the mutex Create operation.
KSTATUS KeMutexCreate(MUTEX **out) {
if (!out) return STATUS_INVALID_PARAMETER;
MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
@@ -21,6 +24,7 @@ KSTATUS KeMutexCreate(MUTEX **out) {
return STATUS_SUCCESS;
}
// Implements the acquire Mutex operation.
KSTATUS KeAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -40,6 +44,7 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) {
return KeCommitBlockThread(self, 0);
}
// Implements the try Acquire Mutex operation.
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
@@ -53,6 +58,7 @@ KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
return STATUS_FAILURE;
}
// Releases mutex.
void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
code/kernel/ke_process.c
+27 -22
View File
@@ -1,4 +1,5 @@
// ke_process.c - process model
// Kernel process model. This file owns address-space metadata, handle tables, and process lifetime transitions.
#include <string.h>
#include <intrin.h>
#include <kernel/ke_process.h>
@@ -12,14 +13,13 @@
#include <kernel/hal/hal_apic.h>
#include <ntlite/syscall.h>
// ---------------------------------------------------------------------------
// Global process list
// ---------------------------------------------------------------------------
// Global process list.
static LIST_ENTRY gProcessList;
static SPINLOCK gProcessListLock;
static uint32_t gNextProcessId = 0;
static PROCESS* gSystemProcess = (PROCESS*)0;
// Processes init.
static void KiProcessInit(void) {
static int inited = 0;
if (!inited) {
@@ -29,11 +29,13 @@ static void KiProcessInit(void) {
}
}
// Returns process List.
LIST_ENTRY* KeGetProcessList(void) {
KiProcessInit();
return &gProcessList;
}
// Returns current Process.
PROCESS* KeGetCurrentProcess(void) {
THREAD* currentThread = KeGetCurrentThread();
if (currentThread) {
@@ -42,10 +44,12 @@ PROCESS* KeGetCurrentProcess(void) {
return NULL;
}
// Returns system Process.
PROCESS* KeGetSystemProcess(void) {
return gSystemProcess;
}
// Creates system Process.
KSTATUS KeCreateSystemProcess(PROCESS** out) {
if (!out) return STATUS_INVALID_PARAMETER;
if (gSystemProcess) {
@@ -65,6 +69,7 @@ KSTATUS KeCreateSystemProcess(PROCESS** out) {
return STATUS_SUCCESS;
}
// Implements the count Process Handles Locked operation.
static uint32_t KiCountProcessHandlesLocked(PROCESS* process) {
uint32_t count = 0;
if (!process) return 0;
@@ -75,6 +80,7 @@ static uint32_t KiCountProcessHandlesLocked(PROCESS* process) {
return count;
}
// Copies truncated String.
static void KiCopyTruncatedString(char* dst, size_t dstSize, const char* src, size_t srcLen) {
if (!dst || dstSize == 0) return;
size_t n = 0;
@@ -87,6 +93,7 @@ static void KiCopyTruncatedString(char* dst, size_t dstSize, const char* src, si
dst[n] = '\0';
}
// Copies process Info.
size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
KiProcessInit();
@@ -151,9 +158,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
return count;
}
// ---------------------------------------------------------------------------
// Destructor (called by ObDereferenceObject when RefCount hits zero)
// ---------------------------------------------------------------------------
// Destructor (called by ObDereferenceObject when RefCount hits zero).
static void KiProcessDestroy(OBJECT* obj) {
PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
@@ -201,9 +206,7 @@ static void KiProcessDestroy(OBJECT* obj) {
// Destroy returns.
}
// ---------------------------------------------------------------------------
// KeCreateProcess
// ---------------------------------------------------------------------------
// KeCreateProcess.
KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
KiProcessInit();
@@ -262,9 +265,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
}
// ---------------------------------------------------------------------------
// KeSetupUserProcessEnvironment
// ---------------------------------------------------------------------------
// KeSetupUserProcessEnvironment.
static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
SYS_STRING_VIEW src,
uintptr_t* baseOut,
@@ -314,6 +315,7 @@ static KSTATUS KiCopyEnvironmentStringViewFixed(PROCESS* proc,
return STATUS_SUCCESS;
}
// Releases partial Process Environment.
static void KiFreePartialProcessEnvironment(PROCESS* proc) {
if (!proc) return;
@@ -424,9 +426,7 @@ fail:
return s;
}
// ---------------------------------------------------------------------------
// Process exit / termination
// ---------------------------------------------------------------------------
// Process exit / termination.
static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
if (!process) return;
@@ -439,6 +439,7 @@ static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
}
}
// Implements the exit Process operation.
void KeExitProcess(PROCESS* process, int exitCode) {
if (!process) return;
process->ExitCode = exitCode;
@@ -450,6 +451,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
KiCompleteProcessExitIfLastThread(process);
}
// Implements the terminate Process operation.
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
if (!process) return STATUS_INVALID_PARAMETER;
@@ -483,22 +485,19 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
return STATUS_SUCCESS;
}
// Checks whether process Terminating.
int KeIsProcessTerminating(PROCESS* process) {
return process && process->Terminating != 0;
}
// ---------------------------------------------------------------------------
// KeGetProcessExitCode
// ---------------------------------------------------------------------------
// KeGetProcessExitCode.
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut) {
if (!process || !exitCodeOut) return STATUS_INVALID_PARAMETER;
*exitCodeOut = process->ExitCode;
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Handle table
// ---------------------------------------------------------------------------
// Handle table.
KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t rights, uint32_t* handleOut) {
if (!process || !object || !handleOut || rights == 0) return STATUS_INVALID_PARAMETER;
@@ -520,10 +519,12 @@ KSTATUS KeAllocateHandleWithRights(PROCESS* process, OBJECT* object, uint32_t ri
return STATUS_NO_MEMORY;
}
// Creates handle.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut) {
return KeAllocateHandleWithRights(process, object, HANDLE_RIGHT_ALL, handleOut);
}
// Releases handle.
void KeCloseHandle(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return;
@@ -540,6 +541,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) {
KeReleaseSpinlock(&process->HandleLock, flags);
}
// Returns handle With Rights.
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE)
return (OBJECT*)0;
@@ -554,10 +556,12 @@ OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t req
return obj;
}
// Returns handle.
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) {
return KeLookupHandleWithRights(process, handle, 0);
}
// Returns handle Rights.
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return 0;
@@ -567,6 +571,7 @@ uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle) {
return rights;
}
// Returns process By Id.
PROCESS* KeGetProcessById(uint32_t pid) {
KiProcessInit();
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
code/kernel/ke_queue.c
+6 -1
View File
@@ -1,4 +1,5 @@
// ke_queue.c - ring-buffer message queue
// Kernel message queue primitive. This file owns bounded queue storage and blocking send/receive behavior.
#include <string.h>
#include <kernel/ke_queue.h>
#include <kernel/ke_thread.h>
@@ -6,8 +7,10 @@
#include <kernel/ke_irql.h>
#include <kernel/ob_object.h>
// Implements the queue Destroy operation.
static void KiQueueDestroy(OBJECT* obj) { (void)obj; }
// Implements the queue Create operation.
KSTATUS KeQueueCreate(QUEUE **out) {
if (!out) return STATUS_INVALID_PARAMETER;
QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
@@ -21,6 +24,7 @@ KSTATUS KeQueueCreate(QUEUE **out) {
return STATUS_SUCCESS;
}
// Implements the send Message operation.
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
@@ -56,6 +60,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
}
}
// Implements the receive Message operation.
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
code/kernel/ke_sched.c
+43 -2
View File
@@ -1,4 +1,5 @@
// ke_sched.c - MLFQ scheduler
// Kernel scheduler. This file owns run queues, dispatch decisions, preemption, and CPU-local scheduling state.
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
@@ -22,6 +23,7 @@ static uint32_t gAgingCounter = 0;
static KE_SCHED_STATS gSchedStats;
static PROCESS* gIdleProcess;
// Implements the default Thread Affinity Mask operation.
static uint64_t KiDefaultThreadAffinityMask(void) {
// Normal kernel, user, and driver-created threads are BSP-affine by
// default. Code that is explicitly audited for AP execution can widen its
@@ -29,10 +31,12 @@ static uint64_t KiDefaultThreadAffinityMask(void) {
return 1ULL;
}
// Implements the thread Is Killed Locked operation.
static int KiThreadIsKilledLocked(const THREAD* thread) {
return thread && !thread->IsIdleThread && thread->Owner && KeIsProcessTerminating(thread->Owner);
}
// Implements the thread Can Run On Cpu Locked operation.
static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
if (!thread || !cpu) return 0;
if (thread->IsIdleThread) return thread == cpu->IdleThread;
@@ -40,6 +44,7 @@ static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
}
// Implements the cpu Is Idle Locked operation.
static int KiCpuIsIdleLocked(const KCPU* cpu) {
if (!cpu || !cpu->Online)
return 0;
@@ -47,6 +52,7 @@ static int KiCpuIsIdleLocked(const KCPU* cpu) {
return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
}
// Implements the any Idle Primary For Thread Locked operation.
static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
if (!thread || thread->IsIdleThread)
return 0;
@@ -65,6 +71,7 @@ static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
return 0;
}
// Checks whether defer To Idle Primary Locked.
static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
if (!cpu || !thread || thread->IsIdleThread)
return 0;
@@ -78,6 +85,7 @@ static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
return KiAnyIdlePrimaryForThreadLocked(thread);
}
// Implements the request Reschedule Cpu Locked operation.
static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || !target->Online)
return;
@@ -90,6 +98,7 @@ static void KiRequestRescheduleCpuLocked(KCPU* target) {
}
}
// Implements the poke Runnable Cpus Locked operation.
static void KiPokeRunnableCpusLocked(THREAD* thread) {
if (!thread || thread->IsIdleThread)
return;
@@ -137,6 +146,7 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
}
}
// Implements the base Quantum For Priority operation.
static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
switch (priority) {
case PriorityHigh: return SCHED_HIGH_QUANTUM_TICKS;
@@ -146,6 +156,7 @@ static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
}
}
// Implements the compute Quantum Locked operation.
static uint32_t KiComputeQuantumLocked(THREAD* thread) {
if (!thread || thread == KeGetCurrentCpu()->IdleThread)
return 1;
@@ -164,15 +175,19 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) {
return quantum ? quantum : 1;
}
// Implements the refill Quantum Locked operation.
static void KiRefillQuantumLocked(THREAD* thread) {
if (!thread) return;
thread->QuantumTicks = KiComputeQuantumLocked(thread);
thread->QuantumRemaining = thread->QuantumTicks;
}
// Returns current Thread.
THREAD* KeGetCurrentThread(void) { return KeGetCurrentCpu()->CurrentThread; }
// Updates current Thread.
void KeSetCurrentThread(THREAD* t) { KeGetCurrentCpu()->CurrentThread = t; }
// Returns scheduler Stats.
void KeGetSchedulerStats(KE_SCHED_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -181,6 +196,7 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
}
// Implements the arm Post Switch Unlock operation.
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
// The scheduler lock is intentionally held across KeSwitchContext(). The
// unlock cookie belongs to the thread continuation that will resume after
@@ -191,11 +207,13 @@ static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
thread->SwitchLockCookie = cookie;
}
// Implements the post Switch Unlock Cookie operation.
static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
return thread ? thread->SwitchLockCookie : 0x202ULL;
}
// Implements the scheduler Bug operation.
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"scheduler invariant failed: %s: thread=0x%016llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
@@ -209,6 +227,7 @@ static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
why ? why : "?");
}
// Implements the assert Ready Queue Candidate Locked operation.
static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null thread");
@@ -222,6 +241,7 @@ static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread)
KiSchedulerBug(where, thread, "thread still reserved/running on a CPU");
}
// Implements the assert Picked Thread Locked operation.
static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null picked thread");
@@ -235,6 +255,7 @@ static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) {
KiSchedulerBug(where, thread, "picked thread was not ready");
}
// Inserts ready Locked.
static void KiInsertReadyLocked(THREAD* thread) {
KiAssertReadyQueueCandidateLocked("KiInsertReadyLocked", thread);
@@ -249,6 +270,7 @@ static void KiInsertReadyLocked(THREAD* thread) {
static int KiAnyRunnableForCpuLocked(KCPU* cpu);
// Implements the any Runnable For Current Cpu operation.
static int KiAnyRunnableForCurrentCpu(void) {
KCPU* cpu = KeGetCurrentCpu();
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -257,6 +279,7 @@ static int KiAnyRunnableForCurrentCpu(void) {
return any;
}
// Implements the idle Loop operation.
static void KiIdleLoop(void) {
for (;;) {
KCPU* cpu = KeGetCurrentCpu();
@@ -279,6 +302,7 @@ static void KiIdleLoop(void) {
}
}
// Initializes scheduler Global.
void KiInitializeSchedulerGlobal(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
@@ -294,6 +318,7 @@ void KiInitializeSchedulerGlobal(void) {
(void)systemProc;
}
// Creates idle Thread For Cpu.
KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
if (!cpu || !gIdleProcess)
return STATUS_INVALID_PARAMETER;
@@ -366,6 +391,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
return STATUS_SUCCESS;
}
// Initializes scheduler.
void KeInitializeScheduler(void) {
KiInitializeSchedulerGlobal();
@@ -376,7 +402,7 @@ void KeInitializeScheduler(void) {
(unsigned)cpu->CpuIndex, s);
}
// Safely release the lock and clean up any dead threads natively post-switch
// Safely release the lock and clean up any dead threads natively post-switch.
void KeSchedulerPostSwitch(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -392,6 +418,7 @@ void KeSchedulerPostSwitch(void) {
}
}
// Updates thread Affinity Mask.
void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask) {
if (!thread || thread->IsIdleThread) return;
@@ -405,10 +432,12 @@ void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Returns default Thread Affinity Mask.
uint64_t KeGetDefaultThreadAffinityMask(void) {
return KiDefaultThreadAffinityMask();
}
// Implements the ready Thread operation.
void KeReadyThread(THREAD* thread) {
if (!thread) return;
@@ -419,6 +448,7 @@ void KeReadyThread(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the unready Thread operation.
void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -430,6 +460,7 @@ void KeUnreadyThread(THREAD* thread) {
}
// Implements the any Runnable For Cpu Locked operation.
static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -447,6 +478,7 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
return 0;
}
// Implements the pick Next For Cpu operation.
static THREAD* KiPickNextForCpu(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -472,17 +504,20 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
}
// Implements the request Reschedule operation.
void KiRequestReschedule(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu)
_InterlockedExchange(&cpu->ReschedulePending, 1);
}
// Checks whether reschedule Pending.
int KiIsReschedulePending(void) {
KCPU* cpu = KeGetCurrentCpu();
return cpu && _InterlockedCompareExchange(&cpu->ReschedulePending, 0, 0) != 0;
}
// Implements the schedule operation.
static void KiSchedule(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -547,12 +582,14 @@ static void KiSchedule(void) {
KeSchedulerPostSwitch();
}
// Implements the schedule From Interrupt operation.
void KiScheduleFromInterrupt(void) {
if (!KiIsReschedulePending()) return;
KiSchedule();
}
// Implements the scheduler Mark Thread Waiting operation.
void KiSchedulerMarkThreadWaiting(THREAD* thread) {
if (!thread) return;
@@ -574,6 +611,7 @@ void KiSchedulerMarkThreadWaiting(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the scheduler Note Voluntary Block operation.
void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -583,6 +621,7 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
}
// Implements the scheduler Wake Thread operation.
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
if (!thread) return;
@@ -627,6 +666,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the scheduler Tick operation.
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -705,6 +745,7 @@ void KeSchedulerTick(void) {
KiRequestReschedule();
}
// Implements the yield operation.
__declspec(dllexport) void KeYield(void) {
KeAssertIrql(PASSIVE_LEVEL);
KiSchedule();
code/kernel/ke_selftest.c
+17 -16
View File
@@ -1,4 +1,5 @@
// ke_selftest.c - kernel bring-up and regression self-tests.
// Kernel self-tests. This file runs bring-up regression checks before normal user workloads start.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
@@ -20,9 +21,7 @@
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>
// -------------------------------------------------------------------------
// Kernel APC self-test
// -------------------------------------------------------------------------
// Kernel APC self-test.
typedef struct KI_APC_SELFTEST_STATE {
EVENT* ReadyEvent;
@@ -85,6 +84,7 @@ static void KiApcSelfTestCrossThreadRoutine(void* normalContext,
KeSignalEvent(st->DoneEvent);
}
// Implements the apc Self Test Worker operation.
static void KiApcSelfTestWorker(void* arg) {
KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)arg;
@@ -135,6 +135,7 @@ static void KiApcAlertableSelfTestNormalRoutine(void* normalContext,
st->ApcDelivered++;
}
// Implements the apc Alertable Self Test Worker operation.
static void KiApcAlertableSelfTestWorker(void* arg) {
KI_APC_ALERTABLE_SELFTEST_STATE* st =
(KI_APC_ALERTABLE_SELFTEST_STATE*)arg;
@@ -159,6 +160,7 @@ static void KiApcAlertableSelfTestWorker(void* arg) {
KeExitThread(KeGetCurrentThread(), 0);
}
// Runs kernel Apc Self Test.
static void KiRunKernelApcSelfTest(void) {
DbgPrint("APC selftest: begin\n");
@@ -346,9 +348,7 @@ static void KiRunKernelApcSelfTest(void) {
DbgPrint("APC selftest: PASS\n");
}
// -------------------------------------------------------------------------
// IRP + kernel APC self-test
// -------------------------------------------------------------------------
// IRP + kernel APC self-test.
typedef struct KI_IRP_APC_SELFTEST_STATE {
EVENT* CompletionEvent;
@@ -401,6 +401,7 @@ static void KiIrpApcSelfTestNormalRoutine(void* normalContext,
st->ApcInformation = irp->Information;
}
// Runs irp Kernel Apc Self Test.
static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
DbgPrint("IRP/APC selftest: begin\n");
@@ -477,9 +478,7 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
// -------------------------------------------------------------------------
// Virtual memory mapping invariants
// -------------------------------------------------------------------------
// Virtual memory mapping invariants.
static void KiRunMemoryMappingSelfTest(void) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
@@ -494,9 +493,7 @@ static void KiRunMemoryMappingSelfTest(void) {
DbgPrint("MM selftest: page zero unmapped\n");
}
// -------------------------------------------------------------------------
// String-view ABI self-test
// -------------------------------------------------------------------------
// String-view ABI self-test.
static void KiRunStringViewSelfTest(void) {
DbgPrint("STRINGVIEW selftest: begin\n");
@@ -530,9 +527,7 @@ static void KiRunStringViewSelfTest(void) {
DbgPrint("STRINGVIEW selftest: PASS\n");
}
// -------------------------------------------------------------------------
// Async file I/O self-test
// -------------------------------------------------------------------------
// Async file I/O self-test.
typedef struct KI_FILE_ASYNC_SELFTEST_STATE {
EVENT* CompletionEvent;
@@ -581,6 +576,7 @@ static void KiFileAsyncSelfTestNormalRoutine(void* normalContext,
st->ApcStatus = status;
}
// Runs async File Io Self Test.
static void KiRunAsyncFileIoSelfTest(void) {
DbgPrint("FILE/ASYNC selftest: begin\n");
@@ -711,6 +707,7 @@ static void KiRunAsyncFileIoSelfTest(void) {
s = IoReadFileAt(rf, 0, verify, sizeof(payload) - 1, &got);
if (!KSUCCESS(s) || got != sizeof(payload) - 1 ||
// Implements the memcmp operation.
memcmp(verify, payload, sizeof(payload) - 1) != 0) {
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: verify failed status=%u got=%llu\n",
s, (uint64_t)got);
@@ -729,6 +726,7 @@ static void KiRunAsyncFileIoSelfTest(void) {
}
// Runs trap Frame Self Test.
static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: begin\n");
@@ -748,6 +746,7 @@ static void KiRunTrapFrameSelfTest(void) {
// Runs core Hardening Self Tests.
void KiRunCoreHardeningSelfTests(void) {
DbgPrint("CORE selftest: begin\n");
@@ -819,6 +818,7 @@ void KiRunCoreHardeningSelfTests(void) {
}
// Runs early Kernel Self Tests.
void KiRunEarlyKernelSelfTests(void) {
KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests();
if (!KSUCCESS(s)) {
@@ -828,6 +828,7 @@ void KiRunEarlyKernelSelfTests(void) {
}
}
// Runs kernel Self Tests.
void KiRunKernelSelfTests(const HAL_API* hal) {
DbgPrint("Kernel selftests: begin\n");
KiRunKernelApcSelfTest();
code/kernel/ke_semaphore.c
+7 -1
View File
@@ -1,4 +1,5 @@
// ke_semaphore.c - counting semaphore
// Kernel semaphore primitive. This file owns counting wakeups and bounded resource acquisition.
#include <stdbool.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
@@ -6,8 +7,10 @@
#include <kernel/ke_irql.h>
#include <kernel/ob_object.h>
// Implements the semaphore Destroy operation.
static void KiSemaphoreDestroy(OBJECT* obj) { (void)obj; }
// Implements the semaphore Create operation.
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out) {
if (!out || initial < 0 || max < 1 || initial > max)
return STATUS_INVALID_PARAMETER;
@@ -22,6 +25,7 @@ __declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SE
return STATUS_SUCCESS;
}
// Waits for semaphore.
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
if (!sem) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -41,6 +45,7 @@ __declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
return KeCommitBlockThread(self, 0);
}
// Implements the try Semaphore operation.
__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
if (!sem) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
@@ -53,6 +58,7 @@ __declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
return STATUS_FAILURE;
}
// Signals semaphore.
__declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
if (!sem) return;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
code/kernel/ke_settings.c
+2 -2
View File
@@ -1,5 +1,5 @@
// ke_settings.c - small system/settings.ini reader.
//
// Kernel settings reader. This file loads simple runtime settings from the system configuration file.
// This intentionally exposes generic string settings only. Subsystems and
// drivers own the policy for interpreting their sections.
code/kernel/ke_smp.c
+37 -13
View File
@@ -1,5 +1,5 @@
// ke_smp.c - SMP application-processor startup and scheduler admission
//
// SMP bring-up and CPU topology management. This file owns AP startup, CPU admission, and topology-derived scheduling metadata.
// The bootloader is intentionally not involved in AP ownership. The kernel
// asks the HAL for MADT-discovered LAPIC IDs, copies a small low-memory x86 AP
// startup blob, sends INIT/SIPI through the local APIC, and waits for each AP
@@ -42,7 +42,7 @@
#define AP_CR0_NE 0x00000020u
#define AP_CR0_PG 0x80000000u
#define AP_BOOT_MAGIC 0x3436544F4F425041ULL /* 'APBOOT64' */
#define AP_BOOT_MAGIC 0x3436544F4F425041ULL // 'APBOOT64'
#define AP_STATE_EMPTY 0u
#define AP_STATE_READY 1u
#define AP_STATE_LONG_MODE 2u
@@ -51,12 +51,12 @@
#define AP_STATE_FAILED 0xFFFFFFFFu
#define AP_STAGE_NONE 0x00000000u
#define AP_STAGE_LONG64 0x4C4D3634u /* 'LM64' */
#define AP_STAGE_ENTRY 0x454E5452u /* 'ENTR' */
#define AP_STAGE_GS 0x47534241u /* 'GSBA' */
#define AP_STAGE_GDT 0x47445421u /* 'GDT!' */
#define AP_STAGE_IDT 0x49445421u /* 'IDT!' */
#define AP_STAGE_PARK 0x5041524Bu /* 'PARK' */
#define AP_STAGE_LONG64 0x4C4D3634u // 'LM64'
#define AP_STAGE_ENTRY 0x454E5452u // 'ENTR'
#define AP_STAGE_GS 0x47534241u // 'GSBA'
#define AP_STAGE_GDT 0x47445421u // 'GDT!'
#define AP_STAGE_IDT 0x49445421u // 'IDT!'
#define AP_STAGE_PARK 0x5041524Bu // 'PARK'
#pragma pack(push, 1)
typedef struct AP_BOOT_BLOCK {
@@ -81,15 +81,23 @@ static PHYSICAL_ADDRESS gApTrampolinePhys;
static PHYSICAL_ADDRESS gReservedApTrampolinePhys;
static AP_BOOT_BLOCK* gApBootBlock;
// Implements the emit8 operation.
static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
// Implements the emit16 operation.
static void KiEmit16(uint8_t** p, uint16_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
// Implements the emit32 operation.
static void KiEmit32(uint8_t** p, uint32_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
// Implements the emit64 operation.
static void KiEmit64(uint8_t** p, uint64_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
// Implements the patch16 operation.
static void KiPatch16(uint8_t* base, size_t off, uint16_t v) { memcpy(base + off, &v, sizeof(v)); }
// Implements the patch32 operation.
static void KiPatch32(uint8_t* base, size_t off, uint32_t v) { memcpy(base + off, &v, sizeof(v)); }
// Implements the patch64 operation.
static void KiPatch64(uint8_t* base, size_t off, uint64_t v) { memcpy(base + off, &v, sizeof(v)); }
// Implements the gdt Descriptor operation.
static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) {
return ((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
@@ -99,6 +107,7 @@ static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, u
((uint64_t)((base >> 24) & 0xFF) << 56);
}
// Updates application Processor Trampoline.
void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys) {
if (trampolinePhys >= PAGE_SIZE &&
trampolinePhys < AP_TRAMPOLINE_MAX_PHYS &&
@@ -109,6 +118,7 @@ void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys) {
}
}
// Builds ap Trampoline.
static KSTATUS KiBuildApTrampoline(void) {
if (gApTrampolinePhys)
return STATUS_SUCCESS;
@@ -155,7 +165,7 @@ static KSTATUS KiBuildApTrampoline(void) {
uint32_t physBase = (uint32_t)frame;
uint8_t* p = page;
// --- 16-bit real-mode SIPI entry ---
// --- 16-bit real-mode SIPI entry ---.
KiEmit8(&p, 0xFA); // cli
KiEmit8(&p, 0x8C); KiEmit8(&p, 0xC8); // mov ax, cs
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD8); // mov ds, ax
@@ -171,7 +181,7 @@ static KSTATUS KiBuildApTrampoline(void) {
while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90);
uint32_t prot32Off = (uint32_t)(p - page);
// --- 32-bit protected-mode transition to long mode ---
// --- 32-bit protected-mode transition to long mode ---.
KiEmit8(&p, 0x66); KiEmit8(&p, 0xB8); KiEmit16(&p, 0x0010); // mov ax, 0x10
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD8); // mov ds, ax
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xC0); // mov es, ax
@@ -197,7 +207,7 @@ static KSTATUS KiBuildApTrampoline(void) {
while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90);
uint32_t long64Off = (uint32_t)(p - page);
// --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). ---
// --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). ---.
KiEmit8(&p, 0x48); KiEmit8(&p, 0xBB); size_t bootBlockPatch = (size_t)(p - page); KiEmit64(&p, 0); // mov rbx, bootblock
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Stage)); KiEmit32(&p, AP_STAGE_LONG64);
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, State)); KiEmit32(&p, AP_STATE_LONG_MODE);
@@ -324,6 +334,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
}
}
// Prepares cpu Record.
static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
KCPU* cpu = &gCpus[cpuIndex];
memset(cpu, 0, sizeof(*cpu));
@@ -354,6 +365,7 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
}
// Implements the ceil Log2 U32 operation.
static uint32_t KiCeilLog2U32(uint32_t value) {
if (value <= 1u)
return 0u;
@@ -366,6 +378,7 @@ static uint32_t KiCeilLog2U32(uint32_t value) {
return shift;
}
// Implements the mask Width operation.
static uint32_t KiMaskWidth(uint32_t shift) {
if (shift >= 32u)
return 0xFFFFFFFFu;
@@ -382,6 +395,7 @@ typedef struct KI_CPUID_TOPOLOGY_PARAMS {
const char* Source;
} KI_CPUID_TOPOLOGY_PARAMS;
// Returns cpuid Topology Leaf.
static void KiReadCpuidTopologyLeaf(uint32_t leaf, KI_CPUID_TOPOLOGY_PARAMS* out) {
if (!out)
return;
@@ -408,6 +422,7 @@ static void KiReadCpuidTopologyLeaf(uint32_t leaf, KI_CPUID_TOPOLOGY_PARAMS* out
}
}
// Implements the detect Cpuid Topology Params operation.
static KI_CPUID_TOPOLOGY_PARAMS KiDetectCpuidTopologyParams(void) {
KI_CPUID_TOPOLOGY_PARAMS topo;
memset(&topo, 0, sizeof(topo));
@@ -472,6 +487,7 @@ static KI_CPUID_TOPOLOGY_PARAMS KiDetectCpuidTopologyParams(void) {
return topo;
}
// Builds processor Topology.
void KeBuildProcessorTopology(void) {
KI_CPUID_TOPOLOGY_PARAMS topo = KiDetectCpuidTopologyParams();
@@ -570,6 +586,7 @@ void KeBuildProcessorTopology(void) {
}
}
// Creates ap Stack.
static KSTATUS KiAllocateApStack(KCPU* cpu) {
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
@@ -593,6 +610,7 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
}
// Implements the ap Stage Name operation.
static const char* KiApStageName(uint32_t stage) {
switch (stage) {
case AP_STAGE_NONE: return "none";
@@ -606,6 +624,7 @@ static const char* KiApStageName(uint32_t stage) {
}
}
// Fills boot Block For Cpu.
static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) {
memset(gApBootBlock, 0, sizeof(*gApBootBlock));
gApBootBlock->Magic = AP_BOOT_MAGIC;
@@ -618,6 +637,7 @@ static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) {
gApBootBlock->ApIndex = apIndex;
}
// Starts application Processors.
KSTATUS KeStartApplicationProcessors(void) {
HAL_CPU_TOPOLOGY topo;
KSTATUS st = HalGetProcessorTopology(&topo);
@@ -757,6 +777,7 @@ KSTATUS KeStartApplicationProcessors(void) {
return STATUS_SUCCESS;
}
// Implements the admit Application Processors To Scheduler operation.
KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
if (gCpuCount <= 1) {
DbgPrint("SMP: no AP scheduler admission needed\n");
@@ -815,7 +836,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
#define SMP_AP_WORKER_ROUNDS 64u
#define SMP_AP_WORKER_TIMEOUT_SPINS 4000000u
#define SMP_AP_TEST_MAGIC 0x524B575041504D53ULL /* 'SMPAPWKR' */
#define SMP_AP_TEST_MAGIC 0x524B575041504D53ULL // 'SMPAPWKR'
typedef struct SMP_AP_WORK_TEST SMP_AP_WORK_TEST;
@@ -844,6 +865,7 @@ struct SMP_AP_WORK_TEST {
SMP_AP_WORKER_ARG Workers[KE_MAX_CPUS];
};
// Implements the smp Ap Controlled Worker operation.
static void KiSmpApControlledWorker(void* context) {
SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context;
if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) {
@@ -895,6 +917,7 @@ static void KiSmpApControlledWorker(void* context) {
_InterlockedIncrement(&arg->Test->DoneCount);
}
// Waits for for Worker Counter.
static int KiWaitForWorkerCounter(volatile long* counter, long expected, const char* what) {
for (uint32_t spin = 0; spin < SMP_AP_WORKER_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(counter, 0, 0) >= expected)
@@ -909,6 +932,7 @@ static int KiWaitForWorkerCounter(volatile long* counter, long expected, const c
return 0;
}
// Runs application Processor Scheduler Self Tests.
KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
if (gCpuCount <= 1) {
DbgPrint("SMP scheduler selftest: SKIP, no APs\n");
code/kernel/ke_syscall.c
+213 -402
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_thread.c
+18 -6
View File
@@ -1,4 +1,5 @@
// ke_thread.c - thread model
// Thread manager. This file owns thread allocation, kernel/user startup contexts, and lifecycle transitions.
#include <string.h>
#include <stddef.h>
#include <intrin.h>
@@ -31,6 +32,7 @@ STATIC_ASSERT_OFFSET(THREAD, FpuContext, 168);
static uint32_t gNextThreadId = 1;
extern void KeThreadEntry(void);
// Implements the thread Destroy operation.
static void KiThreadDestroy(OBJECT* obj) {
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
@@ -65,6 +67,7 @@ static void KiThreadDestroy(OBJECT* obj) {
KeReleaseSpinlock(&owner->HandleLock, flags);
if (remaining == 0 && owner->Terminating &&
// Implements the interlocked Exchange operation.
_InterlockedExchange(&owner->Exited, 1) == 0) {
if (owner->ExitEvent) KeSignalEvent(owner->ExitEvent);
}
@@ -213,6 +216,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
return STATUS_SUCCESS;
}
// Implements the exit Thread operation.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
(void)code;
if (!thread) return;
@@ -229,18 +233,15 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
KeYield();
}
// ---------------------------------------------------------------------------
// KeCreateKernelThread
// ---------------------------------------------------------------------------
// Kernel threads never go to ring 3: they have no user stack, no TEB, and
// their entry stub (KeKernelThreadEntry) calls the function directly then
// calls KeExitThread instead of IRETQ-ing.
//
// Stack layout built here (from KernelStackTop downward):
// [ksp - 8 ] arg <-- popped by KeKernelThreadEntry as r12
// [ksp - 16] fn <-- popped by KeKernelThreadEntry as rbx
// [ksp - 16 - sizeof(KCONTEXT)] KCONTEXT { .Rip = KeKernelThreadEntry }
// ---------------------------------------------------------------------------
// [ksp - 16 - sizeof(KCONTEXT)] KCONTEXT { .Rip = KeKernelThreadEntry }.
extern void KeKernelThreadEntry(void);
static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
@@ -338,6 +339,7 @@ __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uin
return KiCreateKernelThreadWithAffinity(fn, arg, priority, affinityMask, out);
}
// Prepares block Thread.
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
// Called with the synchronization object's spinlock held, so the caller is
// normally at DISPATCH_LEVEL here. This function only marks intent to
@@ -353,12 +355,14 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable)
}
// Updates thread Wait List.
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
thread->WaitListHead = waitListHead;
}
// Implements the finish Wait operation.
static void KiFinishWait(THREAD* thread, KSTATUS status) {
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = status;
@@ -366,6 +370,7 @@ static void KiFinishWait(THREAD* thread, KSTATUS status) {
thread->AlertableWait = 0;
}
// Implements the commit Block Thread operation.
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
KeAssertIrql(PASSIVE_LEVEL);
if (!thread) return STATUS_INVALID_PARAMETER;
@@ -405,21 +410,25 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
}
// Implements the block Thread operation.
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
KePrepareBlockThread(thread, waitObject, alertable);
return KeCommitBlockThread(thread, alertable);
}
// Implements the unblock Thread With Status operation.
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
KiSchedulerWakeThread(thread, status);
}
// Implements the unblock Thread operation.
void KeUnblockThread(THREAD* thread) {
KeUnblockThreadWithStatus(thread, STATUS_SUCCESS);
}
// Implements the alert Thread With Status operation.
static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return STATUS_INVALID_PARAMETER;
@@ -433,14 +442,17 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
return STATUS_SUCCESS;
}
// Implements the alert Thread operation.
KSTATUS KeAlertThread(THREAD* thread) {
return KiAlertThreadWithStatus(thread, STATUS_ALERTED);
}
// Implements the wake Thread For Kernel Apc operation.
void KiWakeThreadForKernelApc(THREAD* thread) {
(void)KiAlertThreadWithStatus(thread, STATUS_KERNEL_APC);
}
// Implements the wake Thread For User Apc operation.
void KiWakeThreadForUserApc(THREAD* thread) {
(void)KiAlertThreadWithStatus(thread, STATUS_USER_APC);
}
code/kernel/ke_time.c
+29 -1
View File
@@ -1,4 +1,5 @@
// ke_time.c - kernel wall-clock time service
// Kernel wall-clock service. This file owns boot-time conversion, timezone policy, and current-time queries.
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
@@ -50,10 +51,12 @@ static KE_TIME_ZONE_POLICY gTimeZonePolicy = {
};
static KE_HARDWARE_CLOCK_MODE gHardwareClockMode = KeHardwareClockUtc;
// Checks whether leap Year.
static bool KiIsLeapYear(uint32_t year) {
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
}
// Implements the days In Month operation.
static uint32_t KiDaysInMonth(uint32_t year, uint32_t month) {
static const uint8_t days[12] = { 31,28,31,30,31,30,31,31,30,31,30,31 };
if (month == 2u && KiIsLeapYear(year)) return 29u;
@@ -73,6 +76,7 @@ static bool KiValidCalendar(uint32_t year, uint32_t month, uint32_t day,
return true;
}
// Implements the days Before Year operation.
static uint64_t KiDaysBeforeYear(uint32_t year) {
uint64_t days = 0;
for (uint32_t y = 1970u; y < year; y++)
@@ -89,6 +93,7 @@ static uint64_t KiCalendarToEpochSeconds(uint32_t year, uint32_t month, uint32_t
return (((days * 24u) + hour) * 60u + minute) * 60u + second;
}
// Implements the epoch Seconds To Calendar operation.
static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out) {
uint64_t days = epochSeconds / 86400u;
uint32_t sod = (uint32_t)(epochSeconds % 86400u);
@@ -119,17 +124,20 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out
out->DayOfWeek = (uint8_t)((totalDays + 4u) % 7u); // 1970-01-01 was Thursday.
}
// Implements the day Of Week operation.
static uint32_t KiDayOfWeek(uint32_t year, uint32_t month, uint32_t day) {
uint64_t epoch = KiCalendarToEpochSeconds(year, month, day, 0, 0, 0);
return (uint32_t)(((epoch / 86400u) + 4u) % 7u);
}
// Implements the last Sunday Of Month operation.
static uint32_t KiLastSundayOfMonth(uint32_t year, uint32_t month) {
uint32_t last = KiDaysInMonth(year, month);
uint32_t dow = KiDayOfWeek(year, month, last); // Sunday == 0.
return last - dow;
}
// Implements the central Europe Dst Active Utc operation.
static bool KiCentralEuropeDstActiveUtc(uint64_t utcEpochSeconds) {
SYS_SYSTEM_TIME utc;
memset(&utc, 0, sizeof(utc));
@@ -142,6 +150,7 @@ static bool KiCentralEuropeDstActiveUtc(uint64_t utcEpochSeconds) {
return utcEpochSeconds >= startUtc && utcEpochSeconds < endUtc;
}
// Implements the offset For Utc Epoch operation.
static int16_t KiOffsetForUtcEpoch(uint64_t utcEpochSeconds, bool* daylight) {
int32_t offset = gTimeZonePolicy.StandardOffsetMinutes;
bool dst = false;
@@ -157,6 +166,7 @@ static int16_t KiOffsetForUtcEpoch(uint64_t utcEpochSeconds, bool* daylight) {
return (int16_t)offset;
}
// Implements the local Epoch To Utc operation.
static uint64_t KiLocalEpochToUtc(uint64_t localEpochSeconds, int16_t offsetMinutes) {
uint64_t offsetSeconds = (uint64_t)((offsetMinutes < 0) ? -offsetMinutes : offsetMinutes) * 60u;
if (offsetMinutes >= 0)
@@ -164,6 +174,7 @@ static uint64_t KiLocalEpochToUtc(uint64_t localEpochSeconds, int16_t offsetMinu
return localEpochSeconds + offsetSeconds;
}
// Implements the rebase Utc From Source operation.
static void KiRebaseUtcFromSource(void) {
if (gSourceTimeZoneKnown) {
gBaseUtcEpochSeconds = KiLocalEpochToUtc(gSourceRawEpochSeconds, gSourceTimeZoneMinutes);
@@ -183,11 +194,13 @@ static void KiRebaseUtcFromSource(void) {
gBaseTickMicroseconds = gSourceTickMicroseconds;
}
// Implements the ascii Lower operation.
static int KiAsciiLower(int c) {
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
return c;
}
// Implements the string Equals No Case operation.
static bool KiStringEqualsNoCase(const char* a, const char* b) {
if (!a || !b) return false;
while (*a && *b) {
@@ -199,6 +212,7 @@ static bool KiStringEqualsNoCase(const char* a, const char* b) {
return *a == '\0' && *b == '\0';
}
// Parses int32.
static bool KiParseInt32(const char* s, int32_t* out) {
if (!s || !*s || !out) return false;
@@ -223,6 +237,7 @@ static bool KiParseInt32(const char* s, int32_t* out) {
return true;
}
// Implements the starts With No Case operation.
static bool KiStartsWithNoCase(const char* s, const char* prefix) {
if (!s || !prefix) return false;
while (*prefix) {
@@ -234,10 +249,12 @@ static bool KiStartsWithNoCase(const char* s, const char* prefix) {
return true;
}
// Parses utc Offset.
static bool KiParseUtcOffset(const char* s, int16_t* out) {
if (!s || !out) return false;
if (KiStringEqualsNoCase(s, "utc") || KiStringEqualsNoCase(s, "gmt") ||
// Implements the string Equals No Case operation.
KiStringEqualsNoCase(s, "z") || KiStringEqualsNoCase(s, "utcz")) {
*out = 0;
return true;
@@ -290,6 +307,7 @@ static bool KiParseUtcOffset(const char* s, int16_t* out) {
return true;
}
// Implements the apply Default Time Zone Policy operation.
static void KiApplyDefaultTimeZonePolicy(void) {
gTimeZonePolicy.StandardOffsetMinutes = KE_CE_STANDARD_OFFSET;
gTimeZonePolicy.DstRule = KeDstRuleCentralEurope;
@@ -297,6 +315,7 @@ static void KiApplyDefaultTimeZonePolicy(void) {
gHardwareClockMode = KeHardwareClockUtc;
}
// Implements the time Initialize operation.
void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal) {
gTimeHal = hal;
KiApplyDefaultTimeZonePolicy();
@@ -350,6 +369,7 @@ void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal) {
}
}
// Implements the time Reload Settings operation.
KSTATUS KeTimeReloadSettings(void) {
KE_TIME_ZONE_POLICY policy = gTimeZonePolicy;
char value[64];
@@ -359,6 +379,7 @@ KSTATUS KeTimeReloadSettings(void) {
KSTATUS st = KeGetSettingString("time", "timezone", value, sizeof(value));
if (KSUCCESS(st)) {
if (KiStringEqualsNoCase(value, "central-europe") ||
// Implements the string Equals No Case operation.
KiStringEqualsNoCase(value, "europe/warsaw")) {
policy.StandardOffsetMinutes = KE_CE_STANDARD_OFFSET;
policy.DstRule = KeDstRuleCentralEurope;
@@ -374,6 +395,7 @@ KSTATUS KeTimeReloadSettings(void) {
policy.TimeZoneKnown = true;
fixedTimezone = true;
} else if (KiStringEqualsNoCase(value, "custom") ||
// Implements the string Equals No Case operation.
KiStringEqualsNoCase(value, "manual")) {
policy.DstRule = KeDstRuleNone;
policy.TimeZoneKnown = true;
@@ -381,6 +403,7 @@ KSTATUS KeTimeReloadSettings(void) {
allowStandardOffsetSetting = true;
} else if (KiStringEqualsNoCase(value, "utc") ||
KiStringEqualsNoCase(value, "gmt") ||
// Implements the string Equals No Case operation.
KiStringEqualsNoCase(value, "z")) {
policy.StandardOffsetMinutes = 0;
policy.DstRule = KeDstRuleNone;
@@ -415,9 +438,11 @@ KSTATUS KeTimeReloadSettings(void) {
} else if (KiStringEqualsNoCase(value, "central-europe")) {
policy.DstRule = KeDstRuleCentralEurope;
} else if (KiStringEqualsNoCase(value, "on") || KiStringEqualsNoCase(value, "yes") ||
// Implements the string Equals No Case operation.
KiStringEqualsNoCase(value, "true") || KiStringEqualsNoCase(value, "1")) {
policy.DstRule = KeDstRuleAlways;
} else if (KiStringEqualsNoCase(value, "off") || KiStringEqualsNoCase(value, "no") ||
// Implements the string Equals No Case operation.
KiStringEqualsNoCase(value, "false") || KiStringEqualsNoCase(value, "0")) {
policy.DstRule = KeDstRuleNone;
}
@@ -430,6 +455,7 @@ KSTATUS KeTimeReloadSettings(void) {
hardwareClockMode = KeHardwareClockUtc;
} else if (KiStringEqualsNoCase(value, "local") ||
KiStringEqualsNoCase(value, "localtime") ||
// Implements the string Equals No Case operation.
KiStringEqualsNoCase(value, "wall-clock")) {
hardwareClockMode = KeHardwareClockLocal;
}
@@ -451,6 +477,7 @@ KSTATUS KeTimeReloadSettings(void) {
return STATUS_SUCCESS;
}
// Returns system Time.
__declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime) {
if (!outTime) return STATUS_INVALID_PARAMETER;
@@ -492,6 +519,7 @@ __declspec(dllexport) KSTATUS KeQuerySystemTime(SYS_SYSTEM_TIME* outTime) {
return STATUS_SUCCESS;
}
// Returns unix Time Seconds.
__declspec(dllexport) uint64_t KeQueryUnixTimeSeconds(void) {
SYS_SYSTEM_TIME t;
if (KeQuerySystemTime(&t) != STATUS_SUCCESS)
code/kernel/ke_trap.c
+7 -1
View File
@@ -1,4 +1,5 @@
// ke_trap.c - current-thread trap-frame tracking and validation
// Trap-frame tracking. This file owns current-thread trap snapshots and validation for user diagnostics.
#include <stddef.h>
#include <string.h>
#include <kernel/ke_trap.h>
@@ -43,17 +44,20 @@ STATIC_ASSERT_OFFSET(KTRAP_FRAME, SsSlot, 216);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousFrame, 224);
_Static_assert(sizeof(KTRAP_FRAME) == 240, "KTRAP_FRAME size must match ke_syscall.asm");
// Implements the trap Frame From User operation.
int KeTrapFrameFromUser(const KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) return 0;
return (frame->Cs & 3u) == 3u;
}
// Returns current Trap Frame.
KTRAP_FRAME* KeGetCurrentTrapFrame(void) {
THREAD* t = KeGetCurrentThread();
if (!t) return (KTRAP_FRAME*)0;
return t->TrapFrame;
}
// Implements the enter Trap Frame operation.
void KiEnterTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
@@ -66,6 +70,7 @@ void KiEnterTrapFrame(KTRAP_FRAME* frame) {
t->TrapFrame = frame;
}
// Implements the exit Trap Frame operation.
void KiExitTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
@@ -113,6 +118,7 @@ void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
}
}
// Returns current Trap Frame.
KSTATUS KeQueryCurrentTrapFrame(SYS_TRAP_FRAME_INFO_K* out) {
if (!out) return STATUS_INVALID_PARAMETER;
code/kernel/kmain.c
+41 -48
View File
@@ -1,4 +1,5 @@
// kmain.c - kernel entry point
// Kernel entry point. This file connects boot information to core subsystem initialization and scheduler start.
#include <stdint.h>
#include <string.h>
#include <intrin.h>
@@ -39,6 +40,7 @@ typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *);
// HAL pointer cached at boot for subsystems that still need platform services.
static const HAL_API* gKmainHal;
// Implements the page In Bootloader Region operation.
static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) {
const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i];
@@ -53,6 +55,7 @@ static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS
return 0;
}
// Implements the reclaim Bootloader Memory operation.
static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
// Reclaim only the raw buffers backing boot-loaded .sys modules.
// The kernel/HAL images, page tables, and bootloader code remain live.
@@ -70,9 +73,7 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
}
}
// ---------------------------------------------------------------------------
// Phase Helper: Resolve Imports for core modules
// ---------------------------------------------------------------------------
// Phase Helper: Resolve Imports for core modules.
static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
// Only resolve imports for kernel.exe and hal.dll (typically indices 0 and 1).
// The bootloader maps these, but drivers are now passed as raw physical files.
@@ -85,9 +86,7 @@ static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
}
}
// ---------------------------------------------------------------------------
// Phase Helper: Initialize dynamically loaded .sys drivers
// ---------------------------------------------------------------------------
// Phase Helper: Initialize dynamically loaded .sys drivers.
static void KiDriversIniCb(const char* section, const char* key,
const char* val, void* ud) {
(void)val;
@@ -102,23 +101,24 @@ static void KiDriversIniCb(const char* section, const char* key,
}
}
// Initializes drivers.
static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
DbgPrint("KiInitializeDrivers: Initializing drivers...\n");
// boot drivers (provided by the bootloader)
// boot drivers (provided by the bootloader).
for (uint32_t i = 0; i < bootInfo->ModuleCount; i++) {
BOOT_MODULE* mod = &bootInfo->Modules[i];
// Skip modules that are already mapped (Kernel/HAL)
// or have no physical data (invalid)
// or have no physical data (invalid).
if (mod->VirtBase != 0 || mod->PhysBase == 0) continue;
DbgPrint("KiInitializeDrivers: Loading boot driver %s\n", mod->Name);
// Calculate virtual address of the raw file in the physical window
// Calculate virtual address of the raw file in the physical window.
void* rawPeBuffer = (void*)(bootInfo->PhysicalMapBase + mod->PhysBase);
// Map the PE image into kernel space
// Map the PE image into kernel space.
uintptr_t mappedBase = 0;
KSTATUS st = LdrLoadImage(rawPeBuffer, mod->ImageSize, &mappedBase, false);
if (!KSUCCESS(st)) {
@@ -126,19 +126,19 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
continue;
}
// Update the struct so other modules can find it
// Update the struct so other modules can find it.
mod->VirtBase = mappedBase;
// Resolve imports (can now see Kernel, HAL, and previous drivers)
// Resolve imports (can now see Kernel, HAL, and previous drivers).
if (!KSUCCESS(LdrResolveImports(mappedBase))) {
DbgPrint("ERROR: Failed to resolve imports for %s\n", mod->Name);
continue;
}
// Register with the global module list so subsequent drivers can import from this one
// Register with the global module list so subsequent drivers can import from this one.
LdrRegisterModule(mod->Name, mod->VirtBase);
// Execute Entry Point
// Execute Entry Point.
DRIVER_ENTRY_FN entry = (DRIVER_ENTRY_FN)LdrGetEntryPoint(mappedBase);
if (entry) {
KSTATUS ds = entry(hal);
@@ -148,7 +148,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
}
}
// runtime drivers
// runtime drivers.
KSTATUS s;
FILE_OBJECT* f;
@@ -174,7 +174,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]);
// Load the driver
// Load the driver.
size_t len = strlen(runtimeKeys[i]);
char* path = MmAllocatePool(len + 10);
if (!path) {
@@ -182,7 +182,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
continue;
}
// memcpy maneuvers
// memcpy maneuvers.
memcpy(path, "drivers\\", 8);
memcpy(path + 8, runtimeKeys[i], len + 1);
@@ -228,26 +228,24 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
}
}
// Free the individual strings allocated via strdup in the callback
// Free the individual strings allocated via strdup in the callback.
if (runtimeKeys) {
for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
if (runtimeKeys[i]) {
MmFreePool(runtimeKeys[i]);
}
}
// Free the dynamic array structure itself
// Free the dynamic array structure itself.
arr_free(runtimeKeys);
}
// Free the file buffer
// Free the file buffer.
MmFreePool(mem);
}
// ---------------------------------------------------------------------------
// Phase Helper: Load and launch a process from the filesystem.
// Returns the new PROCESS* with one reference owned by the caller.
// Returns NULL on failure.
// ---------------------------------------------------------------------------
static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
KSTATUS s;
FILE_OBJECT* f;
@@ -335,6 +333,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
return proc;
}
// Implements the system Init Thread operation.
void SystemInitThread(void* arg) {
SYSTEM_INIT_ARGS* args = (SYSTEM_INIT_ARGS*)arg;
@@ -379,36 +378,32 @@ void SystemInitThread(void* arg) {
MmFreePool(args);
}
// ---------------------------------------------------------------------------
// Main Kernel Entry
// ---------------------------------------------------------------------------
// Main Kernel Entry.
extern void KernelMain(BOOT_INFO* bootInfo) {
//
// Physical & Virtual Memory initialization
//
// Physical & Virtual Memory initialization.
if (!MmInitializePhysical(&bootInfo->MemoryMap, bootInfo->PhysicalMapBase)) {
_disable(); while (true) __halt(); // there's nothing we can do
}
PHYSICAL_ADDRESS pml4 = (PHYSICAL_ADDRESS)__readcr3();
MmInitializeVirtual(pml4, 1 * 1024 * 1024 /* 1 MB initial pool */);
// The early virtual allocator starts small because it only needs bootstrap page tables before the PMM is online.
MmInitializeVirtual(pml4, 1 * 1024 * 1024);
// Bootstrap CPU and architecture setup (GDT/IDT/Faults).
//
// Bootstrap CPU and architecture setup (GDT/IDT/Faults)
//
KeInitializeBootstrapCpu();
KeInitializeGdt();
KeInitializeIdt();
//
// Register boot modules (Allows LdrGetExport to work for drivers and the HAL)
// Note: Kept to a single call here to avoid the duplicate in your original code.
//
LdrRegisterModulesFromBootInfo(bootInfo);
//
// HAL Initialization
//
// HAL Initialization.
HAL_INIT_PARAMS halParams = {
.PhysicalWindowBase = bootInfo->PhysicalMapBase,
.FramebufferBase = bootInfo->FramebufferBase,
@@ -455,15 +450,13 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeSyscallInit(hal);
//
// Dynamic Linking & Driver Execution
//
KiResolveSystemImports(bootInfo);
//KiInitializeDrivers(bootInfo, hal);
// Dynamic Linking & Driver Execution.
KiResolveSystemImports(bootInfo);
//KiInitializeDrivers(bootInfo, hal);.
// Scheduler Prep & Status Update.
//
// Scheduler Prep & Status Update
//
KeInitializeScheduler();
KSTATUS smpSchedStatus = KeAdmitApplicationProcessorsToScheduler();
@@ -473,12 +466,12 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KiRunCoreHardeningSelfTests();
// Pack the arguments for the init thread
// Pack the arguments for the init thread.
SYSTEM_INIT_ARGS* initArgs = MmAllocatePool(sizeof(SYSTEM_INIT_ARGS));
initArgs->bootInfo = bootInfo;
initArgs->hal = hal;
// Spawn the Initialization Thread
// Spawn the Initialization Thread.
THREAD* initThread = (THREAD*)0;
KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
(uintptr_t)initArgs,
code/kernel/ldr_image.c
+36 -24
View File
@@ -1,4 +1,5 @@
// ldr_image.c - PE32+ loader and export resolver
// PE image loader. This file owns image mapping, relocation, import resolution, and export lookup.
#include <stdint.h>
#include <string.h>
#include <kernel/ldr_image.h>
@@ -20,9 +21,7 @@
int _stricmp(const char* a, const char* b);
// ---------------------------------------------------------------------------
// PE32+ structures (just what we need)
// ---------------------------------------------------------------------------
// PE32+ structures (just what we need).
#define PE_SIGNATURE 0x00004550UL // 'PE\0\0'
#define IMAGE_FILE_MACHINE_AMD64 0x8664
#define IMAGE_FILE_MACHINE_I386 0x014c
@@ -88,10 +87,12 @@ typedef struct {
uint32_t Characteristics;
} SECTION_HEADER;
// Implements the align Pe Size operation.
static size_t KiAlignPeSize(size_t value) {
return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
}
// Implements the compute Required Image Size operation.
static KSTATUS KiComputeRequiredImageSize(const PE_HEADER* pe, size_t fileSize, size_t* outSize) {
if (!pe || !outSize) return STATUS_INVALID_PARAMETER;
@@ -158,9 +159,7 @@ typedef struct {
#define IMAGE_SCN_MEM_READ 0x40000000u
#define IMAGE_SCN_MEM_WRITE 0x80000000u
// ---------------------------------------------------------------------------
// Module registry
// ---------------------------------------------------------------------------
// Module registry.
#define MAX_MODULES 32
typedef struct {
@@ -173,6 +172,7 @@ static MODULE_ENTRY gModules[MAX_MODULES];
static uint32_t gModuleCount = 0;
// Implements the canonical Module Name operation.
static const char* KiCanonicalModuleName(const char* name) {
if (!name) return name;
@@ -191,6 +191,7 @@ static const char* KiCanonicalModuleName(const char* name) {
return name;
}
// Returns module Base In Table.
static uintptr_t KiFindModuleBaseInTable(const MODULE_ENTRY* modules, uint32_t count, const char* name) {
for (uint32_t i = 0; i < count; i++) {
if (_stricmp(modules[i].Name, name) == 0) return modules[i].Base;
@@ -198,6 +199,7 @@ static uintptr_t KiFindModuleBaseInTable(const MODULE_ENTRY* modules, uint32_t c
return 0;
}
// Registers module Entry.
static void KiRegisterModuleEntry(MODULE_ENTRY* entry, const char* name, uintptr_t base, size_t size) {
strncpy(entry->Name, name, 31);
entry->Name[31] = '\0';
@@ -205,9 +207,10 @@ static void KiRegisterModuleEntry(MODULE_ENTRY* entry, const char* name, uintptr
entry->Size = size;
}
// Implements the ldr Register Modules From Boot Info operation.
void LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
for (uint32_t i = 0; i < bootInfo->ModuleCount && gModuleCount < MAX_MODULES; i++) {
// Only register modules that have a virtual base (Kernel, HAL)
// Only register modules that have a virtual base (Kernel, HAL).
if (bootInfo->Modules[i].VirtBase != 0) {
KiRegisterModuleEntry(&gModules[gModuleCount],
bootInfo->Modules[i].Name,
@@ -218,6 +221,7 @@ void LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
}
}
// Implements the ldr Register Module operation.
void LdrRegisterModule(const char* name, uintptr_t base) {
if (gModuleCount < MAX_MODULES) {
KiRegisterModuleEntry(&gModules[gModuleCount], name, base, 0);
@@ -225,6 +229,7 @@ void LdrRegisterModule(const char* name, uintptr_t base) {
}
}
// Returns global Module Base.
static uintptr_t KiFindGlobalModuleBase(const char* name) {
return KiFindModuleBaseInTable(gModules, gModuleCount, KiCanonicalModuleName(name));
}
@@ -242,6 +247,7 @@ static uintptr_t KiFindProcessModuleBaseInTable(
return 0;
}
// Returns process Or Global Module Base.
static uintptr_t KiFindProcessOrGlobalModuleBase(PROCESS* process, const char* name) {
if (process) {
uintptr_t base = KiFindProcessModuleBaseInTable(process->Modules, process->ModuleCount, name);
@@ -250,6 +256,7 @@ static uintptr_t KiFindProcessOrGlobalModuleBase(PROCESS* process, const char* n
return KiFindGlobalModuleBase(name);
}
// Returns export At Base.
static void* KiGetExportAtBase(uintptr_t base, const char* functionName) {
if (!base) return (void *)0;
@@ -278,19 +285,19 @@ static void* KiGetExportAtBase(uintptr_t base, const char* functionName) {
return (void *)0;
}
// ---------------------------------------------------------------------------
// Export lookup
// ---------------------------------------------------------------------------
// Export lookup.
void* LdrGetExport(const char* moduleName, const char* functionName) {
uintptr_t base = KiFindGlobalModuleBase(KiCanonicalModuleName(moduleName));
return KiGetExportAtBase(base, functionName);
}
// Implements the ldr Get Export For Process operation.
void* LdrGetExportForProcess(PROCESS* process, const char* moduleName, const char* functionName) {
uintptr_t base = KiFindProcessOrGlobalModuleBase(process, KiCanonicalModuleName(moduleName));
return KiGetExportAtBase(base, functionName);
}
// Implements the ldr Get Image Size operation.
size_t LdrGetImageSize(uintptr_t base) {
if (!base) return 0;
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
@@ -299,6 +306,7 @@ size_t LdrGetImageSize(uintptr_t base) {
return (size_t)pe->OptionalHeader.SizeOfImage;
}
// Implements the ldr Get Mapped Image Size operation.
size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
if (!base) return 0;
if (process) {
@@ -311,6 +319,7 @@ size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
}
// Releases loaded Image For Process.
static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return;
size_t imageSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase);
@@ -318,11 +327,13 @@ static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, b
else MmFreePool((void*)imageBase);
}
// Releases loaded Image.
static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user);
}
// Implements the ldr Protect Image For Process operation.
KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return STATUS_INVALID_PARAMETER;
if (user && !process) return STATUS_INVALID_PARAMETER;
@@ -389,18 +400,14 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
}
// ---------------------------------------------------------------------------
// Entry point lookup
// ---------------------------------------------------------------------------
// Entry point lookup.
uintptr_t LdrGetEntryPoint(uintptr_t base) {
uint32_t pe_offset = *(uint32_t*)(base + 0x3C);
uint32_t entry_rva = *(uint32_t*)(base + pe_offset + 0x28);
return base + entry_rva;
}
// ---------------------------------------------------------------------------
// Base Relocation
// ---------------------------------------------------------------------------
// Base Relocation.
static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
@@ -422,7 +429,7 @@ static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
int type = entries[i] >> 12;
int offset = entries[i] & 0xFFF;
// For 64-bit PEs, we mostly care about DIR64 (Type 10)
// For 64-bit PEs, we mostly care about DIR64 (Type 10).
if (type == IMAGE_REL_BASED_DIR64) {
uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset);
*ptr += delta;
@@ -433,16 +440,16 @@ static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Import resolution
// ---------------------------------------------------------------------------
// Import resolution.
typedef void* (*EXPORT_LOOKUP_FN)(void* ctx, const char* moduleName, const char* functionName);
// Implements the global Export Lookup operation.
static void* KiGlobalExportLookup(void* ctx, const char* moduleName, const char* functionName) {
(void)ctx;
return LdrGetExport(moduleName, functionName);
}
// Processes export Lookup.
static void* KiProcessExportLookup(void* ctx, const char* moduleName, const char* functionName) {
return LdrGetExportForProcess((PROCESS*)ctx, moduleName, functionName);
}
@@ -473,9 +480,8 @@ static KSTATUS LdrResolveImportsCommon(
dllName = KiCanonicalModuleName(dllName);
LdrDbg("LDR: resolve import dll=%s image=%016llx pid=%u\n", dllName, (unsigned long long)imageBase, process ? process->ProcessId : 0);
//
// Ensure dependency is loaded into the process
//
// Ensure dependency is loaded into the process.
if (process) {
uintptr_t modBase =
KiFindProcessOrGlobalModuleBase(process, dllName);
@@ -540,6 +546,7 @@ static KSTATUS LdrResolveImportsCommon(
return STATUS_SUCCESS;
}
// Implements the ldr Resolve Imports operation.
KSTATUS LdrResolveImports(uintptr_t imageBase) {
return LdrResolveImportsCommon(
(PROCESS*)0,
@@ -549,6 +556,7 @@ KSTATUS LdrResolveImports(uintptr_t imageBase) {
);
}
// Implements the ldr Resolve Imports For Process operation.
KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
return LdrResolveImportsCommon(
process,
@@ -558,6 +566,7 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
);
}
// Implements the ldr Register Process Module operation.
static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) {
if (!process || !name || !base) return STATUS_INVALID_PARAMETER;
@@ -577,6 +586,7 @@ static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uint
return STATUS_SUCCESS;
}
// Implements the ldr Load Module For Process operation.
KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_t* baseOut) {
if (!process || !filename) return STATUS_INVALID_PARAMETER;
@@ -735,10 +745,12 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
return STATUS_SUCCESS;
}
// Implements the ldr Load Image For Process operation.
KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) {
return KiLdrLoadImageCommon(process, buffer, size, baseOut, true);
}
// Implements the ldr Load Image operation.
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user);
}
code/kernel/mm_phys.c
+21 -13
View File
@@ -1,5 +1,5 @@
// mm_phys.c - physical memory manager
//
// Physical-memory manager. This file owns frame accounting and boot memory-map consumption.
// Bitmap allocator over 4 KB physical frames. One bitmap records which PFNs
// exist in usable RAM, and one bitmap records which usable PFNs are currently
// free. This keeps single-frame allocation simple while making contiguous
@@ -8,18 +8,15 @@
#include <kernel/ke_spinlock.h>
#include <string.h>
// ---------------------------------------------------------------------------
// Physical-to-virtual translation via the direct map window.
// ---------------------------------------------------------------------------
static uint64_t gPhysBase = 0; // set by MmInitializePhysical
// Implements the phys To Virt operation.
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS phys) {
return (void *)(phys + gPhysBase);
}
// ---------------------------------------------------------------------------
// Frame bitmap
// ---------------------------------------------------------------------------
// Frame bitmap.
typedef struct {
uint8_t* Usable; // bit = 1 when PFN belongs to usable RAM
@@ -33,34 +30,42 @@ typedef struct {
static FRAME_BITMAP gFrameBitmap;
// Implements the align Up operation.
static inline PHYSICAL_ADDRESS KiAlignUp(PHYSICAL_ADDRESS value) {
return (value + PAGE_SIZE - 1) & PAGE_MASK;
}
// Implements the align Down operation.
static inline PHYSICAL_ADDRESS KiAlignDown(PHYSICAL_ADDRESS value) {
return value & PAGE_MASK;
}
// Implements the bit Test operation.
static inline int KiBitTest(const uint8_t* bits, size_t index) {
return (bits[index >> 3] & (uint8_t)(1u << (index & 7))) != 0;
}
// Implements the bit Set operation.
static inline void KiBitSet(uint8_t* bits, size_t index) {
bits[index >> 3] |= (uint8_t)(1u << (index & 7));
}
// Implements the bit Clear operation.
static inline void KiBitClear(uint8_t* bits, size_t index) {
bits[index >> 3] &= (uint8_t)~(1u << (index & 7));
}
// Implements the pfn Usable Locked operation.
static inline int KiPfnUsableLocked(size_t pfn) {
return pfn < gFrameBitmap.MaxPfn && KiBitTest(gFrameBitmap.Usable, pfn);
}
// Implements the pfn Free Locked operation.
static inline int KiPfnFreeLocked(size_t pfn) {
return KiPfnUsableLocked(pfn) && KiBitTest(gFrameBitmap.Free, pfn);
}
// Implements the mark Usable Free Range operation.
static void KiMarkUsableFreeRange(size_t firstPfn, size_t count) {
for (size_t i = 0; i < count; i++) {
size_t pfn = firstPfn + i;
@@ -76,6 +81,7 @@ static void KiMarkUsableFreeRange(size_t firstPfn, size_t count) {
}
}
// Releases usable Pfn.
static void KiRemoveUsablePfn(size_t pfn) {
if (pfn >= gFrameBitmap.MaxPfn) return;
if (KiBitTest(gFrameBitmap.Free, pfn)) {
@@ -88,9 +94,7 @@ static void KiRemoveUsablePfn(size_t pfn) {
}
}
// ---------------------------------------------------------------------------
// Initialisation
// ---------------------------------------------------------------------------
// Initialisation.
bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
if (!map) return false;
gPhysBase = physicalWindowBase;
@@ -183,9 +187,7 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
return true;
}
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
// Public API.
PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive) {
minInclusive = KiAlignUp(minInclusive);
@@ -217,10 +219,12 @@ PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) {
return MmAllocateFrameInRange(PAGE_SIZE, maxExclusive);
}
// Creates frame.
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void) {
return MmAllocateFrameInRange(PAGE_SIZE, (PHYSICAL_ADDRESS)gFrameBitmap.MaxPfn * PAGE_SIZE);
}
// Releases frame.
void MmFreeFrame(PHYSICAL_ADDRESS frame) {
if (!frame || (frame & (PAGE_SIZE - 1))) return;
size_t pfn = (size_t)(frame / PAGE_SIZE);
@@ -233,6 +237,7 @@ void MmFreeFrame(PHYSICAL_ADDRESS frame) {
KeReleaseSpinlock(&gFrameBitmap.Lock, flags);
}
// Creates contiguous.
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
if (!count || count > gFrameBitmap.MaxPfn) return (PHYSICAL_ADDRESS)0;
@@ -261,6 +266,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
return (PHYSICAL_ADDRESS)0;
}
// Releases frame Count.
size_t MmFreeFrameCount(void) {
uintptr_t flags = KeAcquireSpinlock(&gFrameBitmap.Lock);
size_t count = gFrameBitmap.FreeFrames;
@@ -268,9 +274,11 @@ size_t MmFreeFrameCount(void) {
return count;
}
// Implements the total Frame Count operation.
size_t MmTotalFrameCount(void) {
return gFrameBitmap.TotalFrames;
}
// Returns physical Stats.
void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out) {
if (!out) return;
code/kernel/mm_virt.c
+81 -46
View File
@@ -1,14 +1,14 @@
// mm_virt.c - virtual memory manager
//
// Kernel virtual-memory manager. This file owns kernel page-table construction, mapping attributes, and direct-map maintenance.
// Page tables:
// One PML4 shared by all processes.
// Upper half (bit 47 set) has U/S = 0 - ring-3 access always faults.
// Lower half pages are PAGE_USER.
//
// Kernel pool:
// A boundary-tag (header + footer) free-list allocator starting at POOL_BASE.
// Pages are committed on demand from the PMM.
//
// User heap:
// A single best-fit free list in the lower half.
// Page-granular lower-half regions owned by PROCESS metadata. Sub-page
@@ -24,21 +24,19 @@
#include <kernel/ke_irql.h>
#include <kernel/hal/hal_apic.h>
// ---------------------------------------------------------------------------
// Helpers - physical window access
// ---------------------------------------------------------------------------
// Helpers - physical window access.
static uint64_t gPhysBase = PHYS_MAP_BASE;
// Implements the phys To Virt operation.
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
return (void *)(p + gPhysBase);
}
// Implements the pte Ptr operation.
static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
}
// ---------------------------------------------------------------------------
// PML4 / page-table helpers
// ---------------------------------------------------------------------------
// PML4 / page-table helpers.
static PHYSICAL_ADDRESS gPml4Phys = 0;
static SPINLOCK gPageTableLock = SPINLOCK_INIT;
@@ -78,13 +76,12 @@ static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
return childPhys;
}
// ---------------------------------------------------------------------------
// TLB/cache coherency hooks
// ---------------------------------------------------------------------------
// TLB/cache coherency hooks.
void MmFlushTlbLocal(uintptr_t virt) {
__invlpg((void*)virt);
}
// Implements the acquire Tlb Shootdown Lock operation.
static void KiAcquireTlbShootdownLock(void) {
// Do not use the normal spinlock here: it masks interrupts while spinning.
// A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
@@ -95,11 +92,13 @@ static void KiAcquireTlbShootdownLock(void) {
_ReadWriteBarrier();
}
// Releases tlb Shootdown Lock.
static void KiReleaseTlbShootdownLock(void) {
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownLock, 0);
}
// Flushes tlb For Kernel Mapping.
void MmFlushTlbForKernelMapping(uintptr_t virt) {
MmFlushTlbLocal(virt);
@@ -146,6 +145,7 @@ void MmFlushTlbForKernelMapping(uintptr_t virt) {
}
}
// Handles tlb Shootdown.
static int KiHandleTlbShootdown(void) {
if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
return 0;
@@ -158,13 +158,12 @@ static int KiHandleTlbShootdown(void) {
return 1;
}
// Handles tlb Shootdown Nmi.
int MmHandleTlbShootdownNmi(void) {
return KiHandleTlbShootdown();
}
// ---------------------------------------------------------------------------
// Public mapping
// ---------------------------------------------------------------------------
// Public mapping.
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode) {
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
@@ -199,6 +198,7 @@ __declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, u
}
// Releases page.
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) {
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
@@ -226,6 +226,7 @@ __declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) {
}
// Changes page attributes for 4 KiB mappings and refuses huge-page leaves that cannot be edited in place.
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags) {
if (!size) return STATUS_SUCCESS;
@@ -282,6 +283,7 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint
}
// Returns physical Address.
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
if (!gPml4Phys) return 0; // Not initialized
@@ -299,7 +301,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK);
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out;
// Check for 1GB huge pages (bit 7 - PS)
// Check for 1GB huge pages (bit 7 - PS).
if (pdpt[pdpt_idx] & PAGE_PS) {
result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF);
goto out;
@@ -308,7 +310,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK);
if (!(pd[pd_idx] & PAGE_PRESENT)) goto out;
// Check for 2MB huge pages (bit 7 - PS)
// Check for 2MB huge pages (bit 7 - PS).
if (pd[pd_idx] & PAGE_PS) {
result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF);
goto out;
@@ -317,7 +319,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK);
if (!(pt[pt_idx] & PAGE_PRESENT)) goto out;
// Extract the physical frame and add the offset within the page
// Extract the physical frame and add the offset within the page.
result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
out:
@@ -327,9 +329,7 @@ out:
// ---------------------------------------------------------------------------
// Page query and user/kernel copy boundary helpers
// ---------------------------------------------------------------------------
// Page query and user/kernel copy boundary helpers.
__declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
PHYSICAL_ADDRESS* physOut,
uint64_t* flagsOut) {
@@ -409,6 +409,7 @@ out:
}
// Implements the user Range Valid operation.
static int KiUserRangeValid(uintptr_t addr, size_t size) {
if (size == 0) return 1;
if (addr < USER_BASE) return 0;
@@ -417,6 +418,7 @@ static int KiUserRangeValid(uintptr_t addr, size_t size) {
return 1;
}
// Probes user Range.
static KSTATUS KiProbeUserRange(uintptr_t addr, size_t size, uint64_t requiredFlags) {
if (!KiUserRangeValid(addr, size))
return STATUS_ACCESS_DENIED;
@@ -442,14 +444,17 @@ static KSTATUS KiProbeUserRange(uintptr_t addr, size_t size, uint64_t requiredFl
return STATUS_SUCCESS;
}
// Probes user Read.
__declspec(dllexport) KSTATUS MmProbeUserRead(const void* ptr, size_t size) {
return KiProbeUserRange((uintptr_t)ptr, size, PAGE_PRESENT | PAGE_USER);
}
// Probes user Write.
__declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size) {
return KiProbeUserRange((uintptr_t)ptr, size, PAGE_PRESENT | PAGE_USER | PAGE_WRITE);
}
// Copies from User.
__declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size) {
if (!kernelDst && size) return STATUS_INVALID_PARAMETER;
@@ -460,6 +465,7 @@ __declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc,
return STATUS_SUCCESS;
}
// Copies to User.
__declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size) {
if (!kernelSrc && size) return STATUS_INVALID_PARAMETER;
@@ -470,9 +476,7 @@ __declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, s
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Boundary-tag allocator (used for both pool and user heap)
// ---------------------------------------------------------------------------
// Each allocation has a BLOCK_HDR immediately before the user data, and a
// BLOCK_FTR immediately after. Both carry the block size and a 'free' flag.
// This allows O(1) coalescing of adjacent free blocks.
@@ -498,15 +502,19 @@ typedef struct BLOCK_FTR {
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
// Implements the hdr To Ftr operation.
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
}
// Implements the ftr To Hdr operation.
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
}
// Implements the next Block operation.
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
}
// Implements the prev Ftr operation.
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
}
@@ -520,6 +528,7 @@ typedef struct HEAP {
SPINLOCK Lock;
} HEAP;
// Implements the heap Ensure Mapped operation.
static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) {
if (!h || !size) return 1;
if (start < h->Start) return 0;
@@ -592,6 +601,7 @@ static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
}
// Implements the heap Try Decommit Tail operation.
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
(void)h;
if (!blk || blk->Free == 0) return;
@@ -607,6 +617,7 @@ static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
KiHdrToFtr(blk)->Free = 1;
}
// Implements the heap Alloc operation.
static void* KiHeapAlloc(HEAP* h, size_t size) {
if (!size) return (void *)0;
size = (size + 7) & ~7ULL; // 8-byte align
@@ -670,6 +681,7 @@ static void* KiHeapAlloc(HEAP* h, size_t size) {
return (char *)best + HDR_SIZE;
}
// Implements the heap Free operation.
static void KiHeapFree(HEAP* h, void* ptr) {
if (!ptr) return;
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
@@ -717,27 +729,28 @@ static void KiHeapFree(HEAP* h, void* ptr) {
KeReleaseSpinlock(&h->Lock, flags);
}
// Implements the heap Realloc operation.
static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
// If ptr is NULL, realloc acts like malloc
// If ptr is NULL, realloc acts like malloc.
if (!ptr) {
return KiHeapAlloc(h, new_size);
}
// If size is 0, realloc acts like free
// If size is 0, realloc acts like free.
if (new_size == 0) {
KiHeapFree(h, ptr);
return (void*)0;
}
// Retrieve the header of the existing block
// Retrieve the header of the existing block.
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
// Sanity checks to prevent operating on corrupted or already-freed pointers
// Sanity checks to prevent operating on corrupted or already-freed pointers.
if (blk->Magic != BLOCK_MAGIC || blk->Free) {
return (void*)0;
}
// Align the requested size (matches KiHeapAlloc 8-byte alignment)
// Align the requested size (matches KiHeapAlloc 8-byte alignment).
size_t aligned_size = (new_size + 7) & ~7ULL;
// If the current block is already big enough, we can reuse it in-place.
@@ -758,20 +771,16 @@ static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
// We only copy up to the old block's usable size.
memcpy(new_ptr, ptr, blk->Size);
// Free the old block
// Free the old block.
KiHeapFree(h, ptr);
return new_ptr;
}
// ---------------------------------------------------------------------------
// Kernel pool
// ---------------------------------------------------------------------------
// Kernel pool.
static HEAP gPoolHeap;
// ---------------------------------------------------------------------------
// User virtual region arena
// ---------------------------------------------------------------------------
// User virtual region arena.
typedef struct USER_VA_REGION {
LIST_ENTRY ProcessLink;
LIST_ENTRY GlobalLink;
@@ -794,9 +803,7 @@ static int gUserRegionListReady = 0;
#define USER_REGION_ARENA_BASE 0x0000100000000000ULL
// ---------------------------------------------------------------------------
// Module init
// ---------------------------------------------------------------------------
// Module init.
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
gPml4Phys = bootPml4Phys;
gPageTableLock = (SPINLOCK)SPINLOCK_INIT;
@@ -822,38 +829,40 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
gUserRegionListReady = 1;
}
// Implements the activate operation.
void MmActivate(void) {
__writecr3(gPml4Phys);
}
// ---------------------------------------------------------------------------
// Public pool API
// ---------------------------------------------------------------------------
// Public pool API.
__declspec(dllexport) void* MmAllocatePool(size_t size) { return KiHeapAlloc(&gPoolHeap, size); }
// Releases pool.
__declspec(dllexport) void MmFreePool(void* ptr) { KiHeapFree(&gPoolHeap, ptr); }
// Implements the reallocate Pool operation.
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); }
// ---------------------------------------------------------------------------
// User virtual region API
// ---------------------------------------------------------------------------
// User virtual region API.
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
if (align & (align - 1)) return 0;
return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1);
}
// Implements the align Up Size operation.
static inline size_t KiAlignUpSize(size_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
if (align & (align - 1)) return 0;
return (v + align - 1) & ~(align - 1);
}
// Implements the ranges Overlap operation.
static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) {
uintptr_t aEnd = aBase + aSize;
uintptr_t bEnd = bBase + bSize;
return aBase < bEnd && bBase < aEnd;
}
// Returns user Va Range.
static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) {
if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER;
if (!gUserRegionListReady) return STATUS_FAILURE;
@@ -887,6 +896,7 @@ static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintp
}
}
// Returns user Region Locked.
static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) {
if (!process || !base) return (USER_VA_REGION*)0;
uintptr_t b = (uintptr_t)base;
@@ -897,6 +907,7 @@ static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base
return (USER_VA_REGION*)0;
}
// Returns user Region Containing Locked.
static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) {
if (!process || !base || !size) return (USER_VA_REGION*)0;
if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0;
@@ -910,6 +921,7 @@ static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintpt
return (USER_VA_REGION*)0;
}
// Releases user Region Payload.
static void KiUnmapUserRegionPayload(USER_VA_REGION* r) {
if (!r) return;
uintptr_t end = r->Base + r->CommittedSize;
@@ -1007,6 +1019,7 @@ KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
return STATUS_SUCCESS;
}
// Implements the ensure User Region Mapped operation.
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
@@ -1061,6 +1074,7 @@ KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint
return STATUS_SUCCESS;
}
// Implements the protect User Region operation.
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
@@ -1143,6 +1157,7 @@ KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t
return STATUS_SUCCESS;
}
// Releases user Region.
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base) return STATUS_INVALID_PARAMETER;
@@ -1165,6 +1180,7 @@ KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) {
return STATUS_SUCCESS;
}
// Releases all User Regions.
void MmDestroyAllUserRegions(PROCESS* process) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process) return;
@@ -1213,6 +1229,7 @@ KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut,
}
// Returns process User Memory.
void MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t* reservedBytesOut, size_t* committedBytesOut) {
size_t regions = 0;
size_t reserved = 0;
@@ -1234,6 +1251,7 @@ void MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t*
if (committedBytesOut) *committedBytesOut = committed;
}
// Returns user Region Size.
size_t MmGetUserRegionSize(PROCESS* process, const void* base) {
if (!process || !base) return 0;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
@@ -1243,6 +1261,7 @@ size_t MmGetUserRegionSize(PROCESS* process, const void* base) {
return size;
}
// Returns user Region Size By Base.
size_t MmGetUserRegionSizeByBase(const void* base) {
if (!base) return 0;
uintptr_t b = (uintptr_t)base;
@@ -1260,6 +1279,7 @@ size_t MmGetUserRegionSizeByBase(const void* base) {
}
// Implements the user Region Type Name operation.
static const char* KiUserRegionTypeName(USER_REGION_TYPE type) {
switch (type) {
case UserRegionGeneric: return "generic";
@@ -1275,6 +1295,7 @@ static const char* KiUserRegionTypeName(USER_REGION_TYPE type) {
}
}
// Implements the debug Dump User Regions operation.
void MmDebugDumpUserRegions(PROCESS* process) {
if (!process) {
DbgPrint("MM: user regions: <no process>\n");
@@ -1302,6 +1323,7 @@ void MmDebugDumpUserRegions(PROCESS* process) {
if (!count) DbgPrint(" <none>\n");
}
// Implements the debug Describe Pool Address operation.
static void KiDebugDescribePoolAddress(uintptr_t addr) {
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
(unsigned long long)gPoolHeap.Start,
@@ -1347,6 +1369,7 @@ static void KiDebugDescribePoolAddress(uintptr_t addr) {
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
}
// Implements the debug Describe Address operation.
void MmDebugDescribeAddress(uintptr_t addr) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
@@ -1385,6 +1408,7 @@ void MmDebugDescribeAddress(uintptr_t addr) {
}
}
// Creates user Memory For Process.
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) {
void* base = (void*)0;
if (!process || !size) return (void*)0;
@@ -1396,11 +1420,13 @@ __declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, siz
return KSUCCESS(s) ? base : (void*)0;
}
// Releases user Memory For Process.
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) {
if (!process || !ptr) return;
(void)MmDestroyUserRegion(process, ptr);
}
// Implements the reallocate User Memory For Process operation.
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) {
if (!process) return (void*)0;
if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize);
@@ -1420,21 +1446,25 @@ __declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, v
return newPtr;
}
// Creates user Memory.
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
PROCESS* process = KeGetCurrentProcess();
return MmAllocateUserMemoryForProcess(process, size);
}
// Releases user Memory.
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
PROCESS* process = KeGetCurrentProcess();
MmFreeUserMemoryForProcess(process, ptr);
}
// Implements the reallocate User Memory operation.
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
PROCESS* process = KeGetCurrentProcess();
return MmReallocateUserMemoryForProcess(process, ptr, newSize);
}
// Creates user Image For Process.
__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) {
void* base = (void*)0;
if (!process || !size) return (void*)0;
@@ -1446,27 +1476,32 @@ __declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size
return KSUCCESS(s) ? base : (void*)0;
}
// Releases user Image For Process.
__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) {
(void)size;
if (!process || !ptr) return;
(void)MmDestroyUserRegion(process, ptr);
}
// Creates user Image.
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
PROCESS* process = KeGetCurrentProcess();
return MmAllocateUserImageForProcess(process, size);
}
// Releases user Image.
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
PROCESS* process = KeGetCurrentProcess();
MmFreeUserImageForProcess(process, ptr, size);
}
// Returns kernel Pool Usage.
size_t MmGetKernelPoolUsage(void) {
// Returns bytes committed to the kernel pool
// Returns bytes committed to the kernel pool.
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
}
// Returns user Heap Usage.
size_t MmGetUserHeapUsage(void) {
size_t total = 0;
if (!gUserRegionListReady) return 0;
code/kernel/net_device.c
+9 -1
View File
@@ -1,4 +1,5 @@
// net_device.c - minimal network-device registry and stack-provider RX handoff.
// Network device registry. This file owns net-device registration and RX handoff to the active stack provider.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
@@ -14,6 +15,7 @@ static NET_DEVICE* gNetDevices[NET_MAX_DEVICES];
static uint32_t gNetDeviceCount = 0;
static SPINLOCK gNetLock = SPINLOCK_INIT;
// Implements the net Make Default Name operation.
static void NetMakeDefaultName(NET_DEVICE* device, uint32_t index) {
if (!device || device->Name[0])
return;
@@ -25,6 +27,7 @@ static void NetMakeDefaultName(NET_DEVICE* device, uint32_t index) {
device->Name[4] = '\0';
}
// Implements the net Is Device Registered Locked operation.
static bool NetIsDeviceRegisteredLocked(const NET_DEVICE* device) {
for (uint32_t i = 0; i < gNetDeviceCount; ++i) {
if (gNetDevices[i] == device)
@@ -33,10 +36,12 @@ static bool NetIsDeviceRegisteredLocked(const NET_DEVICE* device) {
return false;
}
// Implements the net Device Has Ethernet Address operation.
static bool NetDeviceHasEthernetAddress(const NET_DEVICE* device) {
return device && device->HardwareAddressLength == NET_ETHERNET_ADDRESS_LENGTH;
}
// Implements the net Log Hardware Address operation.
static void NetLogHardwareAddress(const NET_DEVICE* device) {
if (NetDeviceHasEthernetAddress(device)) {
DbgPrint("%02x:%02x:%02x:%02x:%02x:%02x",
@@ -48,6 +53,7 @@ static void NetLogHardwareAddress(const NET_DEVICE* device) {
}
}
// Implements the net Register Device operation.
__declspec(dllexport) KSTATUS NetRegisterDevice(NET_DEVICE* device) {
if (!device || !device->SendFrame)
return STATUS_INVALID_PARAMETER;
@@ -90,6 +96,7 @@ __declspec(dllexport) KSTATUS NetRegisterDevice(NET_DEVICE* device) {
return STATUS_SUCCESS;
}
// Implements the net Get Device Count operation.
__declspec(dllexport) uint32_t NetGetDeviceCount(void) {
uintptr_t flags = KeAcquireSpinlock(&gNetLock);
uint32_t count = gNetDeviceCount;
@@ -97,6 +104,7 @@ __declspec(dllexport) uint32_t NetGetDeviceCount(void) {
return count;
}
// Implements the net Get Device operation.
__declspec(dllexport) NET_DEVICE* NetGetDevice(uint32_t index) {
uintptr_t flags = KeAcquireSpinlock(&gNetLock);
NET_DEVICE* device = (index < gNetDeviceCount) ? gNetDevices[index] : (NET_DEVICE*)0;
code/kernel/net_socket.c
+14 -1
View File
@@ -1,4 +1,4 @@
// net_socket.c - process-owned socket objects with provider-neutral dispatch.
// Kernel socket layer. This file owns process socket objects and provider-neutral socket dispatch.
#include <stdint.h>
#include <stddef.h>
@@ -12,6 +12,7 @@
#define NET_SOCKET_UDP_MAX_BYTES 65507u
#define NET_SOCKET_RAW_MAX_BYTES 65535u
// Implements the net Socket Destroy operation.
static void NetSocketDestroy(OBJECT* object) {
SOCKET_OBJECT* socket = (SOCKET_OBJECT*)object;
if (socket->Provider && socket->ProviderSocket) {
@@ -56,6 +57,7 @@ static KSTATUS NetSocketGetProviderSocket(SOCKET_OBJECT* socket,
return STATUS_SUCCESS;
}
// Implements the net Socket Create operation.
KSTATUS NetSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, SOCKET_OBJECT** out) {
if (!out)
return STATUS_INVALID_PARAMETER;
@@ -78,6 +80,7 @@ KSTATUS NetSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, SOCKE
return st;
}
// Implements the net Socket Bind operation.
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
@@ -85,6 +88,7 @@ KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* ad
return KSUCCESS(st) ? provider->Bind(providerSocket, address, addressSize) : st;
}
// Implements the net Socket Listen operation.
KSTATUS NetSocketListen(SOCKET_OBJECT* socket, uint32_t backlog) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
@@ -92,6 +96,7 @@ KSTATUS NetSocketListen(SOCKET_OBJECT* socket, uint32_t backlog) {
return KSUCCESS(st) ? provider->Listen(providerSocket, backlog) : st;
}
// Implements the net Socket Accept operation.
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut) {
if (!acceptedOut)
return STATUS_INVALID_PARAMETER;
@@ -121,6 +126,7 @@ KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_
return st;
}
// Implements the net Socket Connect operation.
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
@@ -128,6 +134,7 @@ KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K*
return KSUCCESS(st) ? provider->Connect(providerSocket, address, addressSize) : st;
}
// Implements the net Socket Send operation.
KSTATUS NetSocketSend(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags, size_t* sentOut) {
if (sentOut)
*sentOut = 0;
@@ -142,6 +149,7 @@ KSTATUS NetSocketSend(SOCKET_OBJECT* socket, const void* buffer, size_t length,
return KSUCCESS(st) ? provider->Send(providerSocket, buffer, length, flags, sentOut) : st;
}
// Implements the net Socket Receive operation.
KSTATUS NetSocketReceive(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags, size_t* receivedOut) {
if (receivedOut)
*receivedOut = 0;
@@ -194,6 +202,7 @@ KSTATUS NetSocketReceiveFrom(SOCKET_OBJECT* socket, void* buffer, size_t length,
return KSUCCESS(st) ? provider->ReceiveFrom(providerSocket, buffer, length, flags, remoteOut, receivedOut) : st;
}
// Implements the net Socket Shutdown operation.
KSTATUS NetSocketShutdown(SOCKET_OBJECT* socket, uint32_t how) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
@@ -201,6 +210,7 @@ KSTATUS NetSocketShutdown(SOCKET_OBJECT* socket, uint32_t how) {
return KSUCCESS(st) ? provider->Shutdown(providerSocket, how) : st;
}
// Implements the net Socket Set Option operation.
KSTATUS NetSocketSetOption(SOCKET_OBJECT* socket, uint32_t option, const void* value, size_t valueSize) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
@@ -208,6 +218,7 @@ KSTATUS NetSocketSetOption(SOCKET_OBJECT* socket, uint32_t option, const void* v
return KSUCCESS(st) ? provider->SetOption(providerSocket, option, value, valueSize) : st;
}
// Implements the net Socket Get Option operation.
KSTATUS NetSocketGetOption(SOCKET_OBJECT* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
@@ -215,6 +226,7 @@ KSTATUS NetSocketGetOption(SOCKET_OBJECT* socket, uint32_t option, void* value,
return KSUCCESS(st) ? provider->GetOption(providerSocket, option, value, valueSize, actualSizeOut) : st;
}
// Implements the net Socket Poll operation.
KSTATUS NetSocketPoll(SOCKET_OBJECT* socket, uint32_t events, uint32_t* returnedEventsOut) {
if (!returnedEventsOut)
return STATUS_INVALID_PARAMETER;
@@ -226,6 +238,7 @@ KSTATUS NetSocketPoll(SOCKET_OBJECT* socket, uint32_t events, uint32_t* returned
return KSUCCESS(st) ? provider->Poll(providerSocket, events, returnedEventsOut) : st;
}
// Implements the net Resolve Host Ipv4 operation.
KSTATUS NetResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut) {
if (!name || !addressOut)
return STATUS_INVALID_PARAMETER;
code/kernel/net_stack.c
+4 -1
View File
@@ -1,4 +1,4 @@
// net_stack.c - single network stack provider registry.
// Network stack registry. This file owns the single active stack provider binding.
#include <stdint.h>
#include <stddef.h>
@@ -10,6 +10,7 @@
static const NET_STACK_PROVIDER* gNetStackProvider = 0;
static SPINLOCK gNetStackLock = SPINLOCK_INIT;
// Implements the net Validate Stack Provider operation.
static KSTATUS NetValidateStackProvider(const NET_STACK_PROVIDER* p) {
if (!p || p->Version != NET_STACK_PROVIDER_VERSION || !p->Name)
return STATUS_INVALID_PARAMETER;
@@ -25,6 +26,7 @@ static KSTATUS NetValidateStackProvider(const NET_STACK_PROVIDER* p) {
return STATUS_SUCCESS;
}
// Implements the net Register Stack Provider operation.
__declspec(dllexport) KSTATUS NetRegisterStackProvider(const NET_STACK_PROVIDER* provider) {
KSTATUS st = NetValidateStackProvider(provider);
if (!KSUCCESS(st))
@@ -57,6 +59,7 @@ __declspec(dllexport) KSTATUS NetRegisterStackProvider(const NET_STACK_PROVIDER*
return STATUS_SUCCESS;
}
// Implements the net Get Stack Provider operation.
__declspec(dllexport) const NET_STACK_PROVIDER* NetGetStackProvider(void) {
uintptr_t flags = KeAcquireSpinlock(&gNetStackLock);
const NET_STACK_PROVIDER* p = gNetStackProvider;
code/kernel/ob_object.c
+5 -1
View File
@@ -1,9 +1,11 @@
// ob_object.c - executive object manager
// Object manager. This file owns reference-counted kernel objects and handle-table lifetime rules.
#include <string.h>
#include <intrin.h>
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
// Creates object.
__declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) {
if (size < sizeof(OBJECT)) size = sizeof(OBJECT);
@@ -17,11 +19,13 @@ __declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) {
return obj;
}
// Implements the reference Object operation.
__declspec(dllexport) void ObReferenceObject(OBJECT* object) {
if (!object) return;
_InterlockedIncrement(&object->RefCount);
}
// Implements the dereference Object operation.
__declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
if (!object) return;
long prev = _InterlockedDecrement(&object->RefCount);