maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parallel compile, SMP, swapgs, cleanup, remove useless Reserved fields

Browse Source
This commit is contained in:
maksw committed 2026-05-30 11:18:46 +02:00
1 parent b17c9ac854
commit bfe2b55142
42 files changed
+2131 -426
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/hal/hal_apic.c
+95 -3
View File
@@ -2,18 +2,25 @@
#include <kernel/hal/hal_apic.h>
#include <ntlite/ddi/hal_api.h>
#include <stdint.h>
#include <intrin.h>
extern HAL_API gApi;
// ---- Local APIC ----
static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
#define LAPIC_ID 0x020
#define LAPIC_EOI 0x0B0
#define LAPIC_SPURIOUS 0x0F0
#define LAPIC_ICR_LO 0x300
#define LAPIC_ICR_HI 0x310
#define LAPIC_ENABLE 0x100 // bit in spurious vector register
#define APIC_VECTOR_RESCHEDULE 0xF0u
// ---- I/O APIC ----
static volatile uint32_t* gIoapic = (volatile uint32_t *)0;
static uint32_t gBspLapicId = 0;
#define IOAPIC_IDX 0x00
#define IOAPIC_DATA 0x10
@@ -26,18 +33,99 @@ static void IoapicWrite(uint8_t reg, uint32_t val) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val;
}
static void HalApicStallFallback(uint32_t loops) {
for (volatile uint32_t i = 0; i < loops; i++) {
_mm_pause();
}
}
static void HalApicStallUs(uint32_t us) {
if (gApi.Timer.Stall) gApi.Timer.Stall(us);
else HalApicStallFallback(us * 128u + 1u);
}
static void HalApicWaitIcrIdle(void) {
for (uint32_t i = 0; i < 1000000u; i++) {
if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12)))
return;
_mm_pause();
}
}
static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
HalApicWaitIcrIdle();
LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24;
LAPIC_REG(LAPIC_ICR_LO) = low;
HalApicWaitIcrIdle();
}
// IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high).
#define IOREDTBL_LO(n) (0x10 + 2*(n))
#define IOREDTBL_HI(n) (0x11 + 2*(n))
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> 0x21, ...
#define IRQ_VECTOR_BASE 0x20
__declspec(dllexport) uint32_t HalGetBspLapicId(void) {
return gBspLapicId;
}
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
if (!gLapic) return;
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
}
__declspec(dllexport) void HalApicSendPanicIpi(void) {
if (!gLapic) return;
// Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed
// vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still
// enter the panic path. The NMI vector is already installed in the IDT on
// every CPU that has reached kernel-owned state.
HalApicWaitIcrIdle();
LAPIC_REG(LAPIC_ICR_HI) = 0;
LAPIC_REG(LAPIC_ICR_LO) = (3u << 18) | (4u << 8); // all-excluding-self, NMI
HalApicWaitIcrIdle();
}
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) {
if (!gLapic) return;
HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination
}
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
if (!gLapic) return;
// TLB shootdown must reach CPUs that may be spinning with IF=0 inside
// high-IRQL/spinlock regions, so use NMI delivery rather than a fixed
// vector. The NMI stub handles active shootdowns and returns; unexpected
// NMIs still go through the normal fault/panic path.
HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination
}
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) {
if (!gLapic || !startupVector || lapicId == gBspLapicId)
return STATUS_INVALID_PARAMETER;
// INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12.
HalApicSendIpi(lapicId, 0x0000C500u); // INIT, level assert, level-triggered
HalApicStallUs(10000);
HalApicSendIpi(lapicId, 0x00008500u); // INIT, level deassert
HalApicStallUs(200);
HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI
HalApicStallUs(200);
HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI retry
HalApicStallUs(200);
return STATUS_SUCCESS;
}
void HalApicRouteIrq(uint8_t irq) {
// Full initial setup of the redirection entry.
// Sets the vector and clears the mask bit in one write.
uint32_t vector = IRQ_VECTOR_BASE + irq; // e.g. IRQ 1 -> vector 0x21
IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24); // external IRQs target the BSP
IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed
IoapicWrite(IOREDTBL_HI(irq), 0); // deliver to LAPIC id 0
}
void HalApicMaskIrq(uint8_t irq) {
@@ -50,8 +138,8 @@ void HalApicUnmaskIrq(uint8_t irq) {
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
lo &= ~(1 << 16); // clear mask bit
lo = (lo & ~0xFF) | (IRQ_VECTOR_BASE + irq);
IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24);
IoapicWrite(IOREDTBL_LO(irq), lo);
IoapicWrite(IOREDTBL_HI(irq), 0); // LAPIC id 0
}
void HalApicSendEoi(void) {
@@ -62,6 +150,10 @@ void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
gLapic = (volatile uint32_t *)(gApi.PhysToVirt(lapicPhys));
gIoapic = (volatile uint32_t *)(gApi.PhysToVirt(ioapicPhys));
// The BSP is the CPU executing HAL init. Keep all external/device IRQs
// routed to this LAPIC; CPU-local IPIs use the local APIC directly.
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
// Enable Local APIC, set spurious vector to 0xFF.
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
@@ -69,6 +161,6 @@ void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
for (uint8_t i = 0; i <= maxRedir; i++) {
IoapicWrite(IOREDTBL_LO(i), (1 << 16) | (IRQ_VECTOR_BASE + i));
IoapicWrite(IOREDTBL_HI(i), 0);
IoapicWrite(IOREDTBL_HI(i), gBspLapicId << 24);
}
}
code/kernel/hal/hal_main.c
+84 -12
View File
@@ -40,7 +40,7 @@ typedef struct {
uint32_t Length;
uint64_t XsdtAddress;
uint8_t ExtendedChecksum;
uint8_t Reserved[3];
uint8_t Padding[3];
} RSDP;
// ACPI SDT Header
@@ -81,6 +81,7 @@ typedef struct {
static uintptr_t gLapicPhys = 0xFEE00000;
static uintptr_t gIoapicPhys = 0xFEC00000;
static uintptr_t gHpetPhys = 0xFED00000;
static HAL_CPU_TOPOLOGY gCpuTopology;
// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5)
static uint16_t gPm1aCntBlk = 0;
@@ -89,7 +90,7 @@ static uint8_t gSlpTypA = 5; // S5 SLP_TYP for PM1a; 5 is correct for QEMU
static uint8_t gSlpTypB = 5; // S5 SLP_TYP for PM1b
int debugEnabled = 0;
int _fltused; // fuckass
int _fltused; // MSVC floating-point helper anchor
static void hal_putchar(char c) {
gApi.Screen.PrintChar(c, 0x00FF00, 0x000000);
}
@@ -114,7 +115,7 @@ typedef struct {
ACPISDTHeader Header; // 0 (36 bytes)
uint32_t FirmwareCtrl; // 36
uint32_t Dsdt; // 40 - physical addr of DSDT
uint8_t Reserved1; // 44
uint8_t Ignored44; // 44
uint8_t PreferredPmProfile; // 45
uint16_t SciInt; // 46
uint32_t SmiCmd; // 48
@@ -201,16 +202,51 @@ static void ParseFadt(const FADT* fadt) {
}
}
static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
if (gCpuTopology.CpuCount >= HAL_MAX_CPUS)
return;
for (uint32_t i = 0; i < gCpuTopology.CpuCount; i++) {
if (gCpuTopology.Cpus[i].LapicId == lapicId)
return;
}
HAL_CPU_INFO* cpu = &gCpuTopology.Cpus[gCpuTopology.CpuCount++];
memset(cpu, 0, sizeof(*cpu));
cpu->ProcessorUid = uid;
cpu->LapicId = lapicId;
cpu->Flags = flags;
cpu->Enabled = (flags & 1u) ? 1u : 0u;
}
static void ParseMadt(MADT* madt) {
gLapicPhys = madt->LocalAPICAddress;
uint8_t* p = (uint8_t *)(madt + 1);
uint8_t* end = (uint8_t *)madt + madt->Header.Length;
while (p < end) {
while (p + 2 <= end) {
uint8_t type = p[0], len = p[1];
if (type == 1) { // I/O APIC entry
if (!len || p + len > end) break;
if (type == 0 && len >= 8) { // Processor Local APIC
uint32_t uid = p[2];
uint32_t lapicId = p[3];
uint32_t flags = *(uint32_t *)(p + 4);
if (flags & 1u)
HalAddCpuRecord(uid, lapicId, flags);
} else if (type == 1 && len >= 12) { // I/O APIC entry
gIoapicPhys = *(uint32_t *)(p + 4);
} else if (type == 5 && len >= 12) { // Local APIC Address Override
gLapicPhys = (uintptr_t)*(uint64_t *)(p + 4);
} else if (type == 9 && len >= 16) { // Processor Local x2APIC
uint32_t lapicId = *(uint32_t *)(p + 4);
uint32_t flags = *(uint32_t *)(p + 8);
uint32_t uid = *(uint32_t *)(p + 12);
if ((flags & 1u) && lapicId < 256u)
HalAddCpuRecord(uid, lapicId, flags);
}
p += len ? len : 1;
p += len;
}
}
@@ -313,6 +349,44 @@ static void HalShutdown(void) {
while (true) __halt();
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
if (!out) return STATUS_INVALID_PARAMETER;
memset(out, 0, sizeof(*out));
*out = gCpuTopology;
if (!out->CpuCount) {
out->CpuCount = 1;
out->BspLapicId = HalGetBspLapicId();
out->Cpus[0].LapicId = out->BspLapicId;
out->Cpus[0].ProcessorUid = 0;
out->Cpus[0].Flags = 1;
out->Cpus[0].Enabled = 1;
out->Cpus[0].IsBsp = 1;
return STATUS_SUCCESS;
}
uint32_t bsp = HalGetBspLapicId();
out->BspLapicId = bsp;
int foundBsp = 0;
for (uint32_t i = 0; i < out->CpuCount; i++) {
out->Cpus[i].IsBsp = (out->Cpus[i].LapicId == bsp) ? 1u : 0u;
if (out->Cpus[i].IsBsp) foundBsp = 1;
}
if (!foundBsp && out->CpuCount < HAL_MAX_CPUS) {
HAL_CPU_INFO* cpu = &out->Cpus[out->CpuCount++];
memset(cpu, 0, sizeof(*cpu));
cpu->LapicId = bsp;
cpu->ProcessorUid = bsp;
cpu->Flags = 1;
cpu->Enabled = 1;
cpu->IsBsp = 1;
}
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
if (gInitialised) return STATUS_ALREADY_EXISTS;
@@ -357,12 +431,10 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
gApi.Version = HAL_API_VERSION;
gInitialised = 1;
// This shouldn't be here but i have no other way.
// After i send 0xF4 and move the mouse the controller sends an interrupts that doesn't fire
// and because the CPU missed the IRQ 12 edge transition (cli) later when we expect it to fire, it doesn't
// leaving us with no mouse and no keyboard. The dreaded 8042 will hold the CLOCK line low forever.
// To combat this we just go about our day executing the rest of the init and just before the end
// of init we just iterate 256 times to grab whatever the controller left for us ("Drain and Pray").
// The 8042 can leave output bytes pending while the keyboard/mouse drivers
// are being brought up with interrupts temporarily disabled. Drain the
// controller before returning from HAL init so stale status/data bytes do
// not suppress the next keyboard or mouse interrupt edge.
for (int i = 0; i < 256; i++) {
if (__inbyte(0x64) & 0x01) {
__inbyte(0x60); // Consume
code/kernel/hal/hal_mouse.c
+1 -2
View File
@@ -271,8 +271,7 @@ void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH) {
ack = MouseRecvByte();
} while (ack != 0xFAu && --retries > 0);
// Thanks to the increased Ps2WaitOutput timeout, we will actually
// wait long enough for the physical hardware self-test to finish here.
// The output timeout covers the physical hardware self-test latency.
MouseRecvByte(); // 0xAA - BAT passed
MouseRecvByte(); // 0x00 - device ID
code/kernel/include/kernel/hal/hal_apic.h
+35 -9
View File
@@ -1,9 +1,35 @@
// hal_apic.h
#pragma once
#include <stdint.h>
void HalApicRouteIrq(uint8_t irq);
void HalApicMaskIrq(uint8_t irq);
void HalApicUnmaskIrq(uint8_t irq);
void HalApicSendEoi(void);
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys);
// hal_apic.h
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
void HalApicRouteIrq(uint8_t irq);
void HalApicMaskIrq(uint8_t irq);
void HalApicUnmaskIrq(uint8_t irq);
void HalApicSendEoi(void);
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys);
#define HAL_MAX_CPUS 32u
typedef struct HAL_CPU_INFO {
uint32_t ProcessorUid;
uint32_t LapicId;
uint32_t Flags;
uint8_t Enabled;
uint8_t IsBsp;
} HAL_CPU_INFO;
typedef struct HAL_CPU_TOPOLOGY {
uint32_t CpuCount;
uint32_t BspLapicId;
HAL_CPU_INFO Cpus[HAL_MAX_CPUS];
} HAL_CPU_TOPOLOGY;
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out);
__declspec(dllexport) uint32_t HalGetBspLapicId(void);
__declspec(dllexport) void HalApicEnableCurrentCpu(void);
__declspec(dllexport) void HalApicSendPanicIpi(void);
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId);
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId);
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector);
code/kernel/include/kernel/io_file.h
+1 -1
View File
@@ -23,7 +23,7 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
size_t information);
// ---------------------------------------------------------------------------
// Filesystem driver vtable - filled in by fat32.sys (or any future fs driver)
// Filesystem driver vtable - filled in by a filesystem driver.
// ---------------------------------------------------------------------------
typedef struct IO_FILESYSTEM {
// Open a file by absolute path (e.g. "apps\\shell.exe").
code/kernel/include/kernel/ke_cpu.h
+25 -4
View File
@@ -1,9 +1,8 @@
#pragma once
// ke_cpu.h - per-CPU execution state
//
// SMP phase 1 still runs only the bootstrap processor, but the kernel now
// represents CPU-local state explicitly. AP startup can fill additional KCPU
// entries later without first untangling scheduler/IRQL/TSS globals.
// Every processor that reaches kernel-owned state has a KCPU record. The record
// owns CPU-local scheduler, IRQL, GDT, and TSS state.
#include <stdint.h>
#include <kernel/ke_irql.h>
@@ -15,23 +14,42 @@
struct THREAD;
typedef struct KCPU {
// GS:0 points back at the owning KCPU while executing in ring 0.
// User mode keeps its TEB in GS_BASE; swapgs restores this pointer on
// entry from user mode and parks the user TEB in IA32_KERNEL_GS_BASE.
struct KCPU* Self;
uint32_t CpuIndex;
uint32_t LapicId;
uint32_t AcpiProcessorId;
uint32_t BootProcessorIndex;
uint8_t Present;
uint8_t Started;
uint8_t Parked;
uint8_t Online;
uint8_t IsBsp;
uint8_t Reserved0;
struct THREAD* CurrentThread;
struct THREAD* IdleThread;
struct THREAD* OutgoingThread;
// Do not store the scheduler post-switch unlock cookie here. It belongs
// to the suspended THREAD continuation that resumes after KeSwitchContext().
KIRQL CurrentIrql;
uint32_t InterruptDepth;
volatile long ReschedulePending;
// AP scheduler admission handshake. APs set SchedulerActive just before
// they leave the raw kernel park loop and enter their per-CPU idle thread.
volatile long SchedulerCommand;
volatile long SchedulerActive;
TSS64 Tss;
uint64_t Gdt[KE_GDT_ENTRIES];
KGDTR Gdtr;
void* KernelStack;
uintptr_t KernelStackTop;
@@ -43,6 +61,9 @@ extern KCPU gCpus[KE_MAX_CPUS];
extern uint32_t gCpuCount;
void KeInitializeBootstrapCpu(void);
void KeLoadKernelGsBase(KCPU* cpu);
void KeSetUserGsBase(uintptr_t userGsBase);
uintptr_t KeGetUserGsBase(void);
KCPU* KeGetCurrentCpu(void);
KCPU* KeGetCpuByIndex(uint32_t index);
uint32_t KeGetCurrentCpuIndex(void);
code/kernel/include/kernel/ke_debug.h
+2
View File
@@ -16,9 +16,11 @@ typedef enum KE_PANIC_CODE {
KE_PANIC_CODE_TRAP = 4,
KE_PANIC_CODE_DRIVER = 5,
KE_PANIC_CODE_INIT = 6,
KE_PANIC_CODE_SCHEDULER= 7,
} KE_PANIC_CODE;
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...);
__declspec(dllexport) int KeHandleNmiIpi(void);
// ---------------------------------------------------------------------------
// Fault dispatch - called from the IDT fault stubs (ke_syscall.asm).
code/kernel/include/kernel/ke_idt.h
+24 -8
View File
@@ -5,25 +5,36 @@
// 64-bit TSS - minimal; we only care about RSP0.
#pragma pack(push, 1)
typedef struct TSS64 {
uint32_t Reserved0;
uint32_t MustBeZero0;
uint64_t RSP0; // loaded into RSP on ring-0 entry
uint64_t RSP1;
uint64_t RSP2;
uint64_t Reserved1;
uint64_t MustBeZero1;
uint64_t IST[7];
uint64_t Reserved2;
uint16_t Reserved3;
uint64_t MustBeZero2;
uint16_t MustBeZero3;
uint16_t IoMapBase;
} TSS64;
#pragma pack(pop)
// Compatibility pointer used by ke_switch.asm.
// It points at CPU0's TSS until the context-switch path grows a true
// per-CPU base lookup.
extern TSS64* gTss;
#define KE_GDT_ENTRIES 8u
#pragma pack(push, 1)
typedef struct KGDTR {
uint16_t Limit;
uint64_t Base;
} KGDTR;
#pragma pack(pop)
struct KCPU;
void KeInitializeGdt(void);
void KeInitializeGdtForCpu(struct KCPU* cpu);
void KeLoadGdtForCurrentCpu(struct KCPU* cpu);
void KeSetCurrentTssRsp0(uintptr_t rsp0);
void KeInitializeIdt(void);
void KeLoadIdtForCurrentCpu(void);
// Called from hal_timer.c after HPET/APIC timer fires.
void KeTimerInterrupt(void);
@@ -36,3 +47,8 @@ extern void KeFaultDE(void); // #DE divide error
extern void KeFaultGP(void); // #GP general protection
extern void KeFaultPF(void); // #PF page fault
extern void KeFaultUD(void); // #UD invalid opcode
extern void KeNmiIpiStub(void); // returning NMI SMP-control stub
// CPU-local IPI vector. This uses the common interrupt entry machinery but is
// not an external/device IRQ and is never routed through the I/O APIC.
#define KE_VECTOR_RESCHEDULE_IPI 0xF0u
code/kernel/include/kernel/ke_irp.h
+2 -2
View File
@@ -37,8 +37,8 @@ struct KAPC;
typedef struct IRP IRP;
// Completion callback: invoked by IoCompleteRequest() in the completing
// context. Today that is the driver worker or sync fallback path; future
// DPC/APC work will keep all completion policy centralized here.
// context. Driver worker, synchronous fallback, DPC, and APC completion paths
// all centralize policy here.
typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx);
// -------------------------------------------------------------------------
code/kernel/include/kernel/ke_sched.h
+9
View File
@@ -3,6 +3,8 @@
// Three run queues (Low / Normal / High). IRQ 0 drives KeSchedulerTick.
#include <kernel/ke_thread.h>
struct KCPU;
#define SCHED_TICK_MS 10u
#define SCHED_HIGH_QUANTUM_TICKS 2u
#define SCHED_NORMAL_QUANTUM_TICKS 4u
@@ -26,6 +28,11 @@ typedef struct KE_SCHED_STATS {
// Called once by KernelMain before any threads are created.
void KeInitializeScheduler(void);
// Split global scheduler setup from per-CPU idle-thread creation so AP bring-up
// can add one idle thread per CPU without reinitializing the global queues.
void KiInitializeSchedulerGlobal(void);
KSTATUS KiCreateIdleThreadForCpu(struct KCPU* cpu);
// Add / remove a thread from the run queues.
void KeReadyThread(THREAD* thread);
void KeUnreadyThread(THREAD* thread);
@@ -44,6 +51,8 @@ void KiRequestReschedule(void);
int KiIsReschedulePending(void);
void KiScheduleFromInterrupt(void);
void KiSchedulerNoteVoluntaryBlock(THREAD* thread);
void KiSchedulerMarkThreadWaiting(THREAD* thread);
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status);
// Voluntarily yield the CPU. Picks the next runnable thread immediately.
__declspec(dllexport) void KeYield(void);
code/kernel/include/kernel/ke_selftest.h
+3 -4
View File
@@ -1,9 +1,8 @@
#pragma once
// ke_selftest.h - temporary kernel bring-up self-tests.
//
// These tests intentionally live behind one call site so they can be walled
// off or compiled out later without touching kmain.c again.
// ke_selftest.h - kernel bring-up and regression self-tests.
#include <ntlite/ddi/hal_api.h>
void KiRunCoreHardeningSelfTests(void);
void KiRunEarlyKernelSelfTests(void);
void KiRunKernelSelfTests(const HAL_API* hal);
code/kernel/include/kernel/ke_smp.h
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
// Start application processors with hardware INIT/SIPI and park them in
// kernel-owned state. APs are not runnable until scheduler admission succeeds.
KSTATUS KeStartApplicationProcessors(void);
// Create one scheduler idle thread per AP and release APs from the raw kernel
// park loop into their idle threads.
KSTATUS KeAdmitApplicationProcessorsToScheduler(void);
// Run explicit AP-affine kernel workers that exercise the global scheduler,
// reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver
// threads remain BSP-affine unless their affinity is explicitly widened.
KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void);
code/kernel/include/kernel/ke_thread.h
+21 -1
View File
@@ -70,7 +70,12 @@ typedef struct THREAD {
uintptr_t UserEntry;
uintptr_t UserArg;
uintptr_t SchedLockFlags; // Stores RFLAGS while thread is context switched out
// Saved scheduler-lock cookie for this thread's suspended switch
// continuation. This is deliberately thread-owned, not CPU-owned:
// after KeSwitchContext() resumes a saved KiSchedule() continuation, the
// resumed thread must restore the IF/IRQL state captured when that same
// thread switched out.
uintptr_t SwitchLockCookie;
void* UserStackBase; // Tracks user stack allocation to prevent memory leaks
// Signaled (manual-reset) when the thread exits.
@@ -90,6 +95,15 @@ typedef struct THREAD {
uint32_t LastRunTicks;
uint32_t RunningCpu;
// Scheduler hardening state. Queued means membership in a scheduler ready
// queue only; wait-object membership is tracked by WaitListHead. Idle
// threads are CPU-local and never enter a global ready queue.
uint8_t Queued;
uint8_t IsIdleThread;
uint16_t SchedulerFlags;
uint32_t LastCpu;
uint64_t AffinityMask;
// Set by wait primitives before blocking and completed by the object
// that wakes the thread. Keep this field after the assembly-visible
// context-switch fields: ke_switch.asm hard-codes offsets through
@@ -130,6 +144,9 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
// given priority.
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, uint64_t affinityMask,
THREAD** out);
// Mark a thread dead and deschedule it permanently.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code);
@@ -158,5 +175,8 @@ void KiWakeThreadForUserApc(THREAD* thread);
// Assembly stub - implemented in ke_switch.asm.
void KeSwitchContext(THREAD* from, THREAD* to);
__declspec(noreturn) void KeStartInitialThread(THREAD* thread);
void KeInitThreadFpuContext(THREAD* thread);
void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask);
uint64_t KeGetDefaultThreadAffinityMask(void);
code/kernel/include/kernel/ke_trap.h
+4 -7
View File
@@ -1,10 +1,9 @@
#pragma once
// ke_trap.h - trap-frame metadata for syscall/interrupt return paths
//
// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally
// an editable return frame: future user APC delivery can rewrite Rip/Rsp in the
// frame, and KiExitTrapFrame will commit those values back to the hardware IRET
// slots before returning to user mode.
// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally
// an editable return frame: user APC delivery can rewrite Rip/Rsp in the frame,
// and KiExitTrapFrame commits those values back to the hardware IRET slots.
#include <stdint.h>
#include <ntlite/status.h>
@@ -18,12 +17,11 @@
#define KTRAP_MODE_KERNEL 0u
#define KTRAP_MODE_USER 1u
typedef struct KTRAP_FRAME {
typedef __declspec(align(16)) struct KTRAP_FRAME {
uint64_t Magic;
uint32_t Kind;
uint32_t Vector;
uint32_t PreviousMode;
uint32_t Reserved;
uint64_t Rax;
uint64_t Rcx;
@@ -54,7 +52,6 @@ typedef struct KTRAP_FRAME {
uint64_t* SsSlot;
struct KTRAP_FRAME* PreviousFrame;
uint64_t Reserved2;
} KTRAP_FRAME;
typedef struct SYS_TRAP_FRAME_INFO_K {
code/kernel/include/kernel/ldr_image.h
+1 -1
View File
@@ -36,6 +36,6 @@ KSTATUS LdrLoadModuleForProcess(struct PROCESS* process, const char* filename, u
// Called by KernelMain to register all modules from BOOT_INFO so that
// LdrGetExport can find them.
struct BOOT_INFO; // forward declare for the compiler to shut the fuck up
struct BOOT_INFO; // forward declaration
void LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
void LdrRegisterModule(const char* name, uintptr_t base);
code/kernel/include/kernel/mm_phys.h
+1
View File
@@ -18,6 +18,7 @@ bool MmInitializePhysical(const MEMORY_MAP* map,
// Allocate / free a single 4 KB frame.
// MmAllocateFrame returns 0 on failure.
PHYSICAL_ADDRESS MmAllocateFrame(void);
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive);
void MmFreeFrame(PHYSICAL_ADDRESS frame);
// Allocate 'count' physically contiguous 4 KB frames.
code/kernel/include/kernel/mm_virt.h
+7
View File
@@ -53,6 +53,13 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint
// Returns 0 if the address is not mapped.
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt);
// TLB/cache coherency hooks for the shared kernel address space. Local flushes
// use INVLPG directly; kernel mapping changes broadcast an NMI shootdown to
// other online CPUs and wait for acknowledgements.
void MmFlushTlbLocal(uintptr_t virt);
void MmFlushTlbForKernelMapping(uintptr_t virt);
int MmHandleTlbShootdownNmi(void);
// ---------------------------------------------------------------------------
// User/kernel copy boundary helpers
code/kernel/ke_cpu.c
+41 -2
View File
@@ -1,26 +1,58 @@
// ke_cpu.c - bootstrap/per-CPU state plumbing
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#include <kernel/ke_cpu.h>
#define BOOTSTRAP_CPU_STACK_SIZE 0x4000u
#define IA32_GS_BASE 0xC0000101u
#define IA32_KERNEL_GS_BASE 0xC0000102u
KCPU gCpus[KE_MAX_CPUS];
uint32_t gCpuCount = 0;
static uint8_t gBootstrapCpuStack[BOOTSTRAP_CPU_STACK_SIZE];
static volatile long gKernelGsReady = 0;
extern KCPU* KeReadCurrentCpuFromGs(void);
void KeLoadKernelGsBase(KCPU* cpu) {
if (!cpu)
cpu = &gCpus[0];
cpu->Self = cpu;
__writemsr(IA32_GS_BASE, (uint64_t)(uintptr_t)cpu);
gKernelGsReady = 1;
}
void KeSetUserGsBase(uintptr_t userGsBase) {
// While running in the kernel, GS_BASE is the current KCPU. The user TEB
// base is parked in IA32_KERNEL_GS_BASE so swapgs can exchange it with the
// KCPU pointer on return to ring 3.
__writemsr(IA32_KERNEL_GS_BASE, (uint64_t)userGsBase);
}
uintptr_t KeGetUserGsBase(void) {
return (uintptr_t)__readmsr(IA32_KERNEL_GS_BASE);
}
void KeInitializeBootstrapCpu(void) {
memset(gCpus, 0, sizeof(gCpus));
KCPU* cpu = &gCpus[0];
cpu->Self = cpu;
cpu->CpuIndex = 0;
cpu->LapicId = 0;
cpu->Present = 1;
cpu->Started = 1;
cpu->Parked = 0;
cpu->Online = 1;
cpu->IsBsp = 1;
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
cpu->SchedulerCommand = 0;
cpu->SchedulerActive = 0;
cpu->KernelStack = gBootstrapCpuStack;
cpu->KernelStackTop = (uintptr_t)(gBootstrapCpuStack + sizeof(gBootstrapCpuStack));
memset(&cpu->Tss, 0, sizeof(cpu->Tss));
@@ -28,11 +60,18 @@ void KeInitializeBootstrapCpu(void) {
cpu->Tss.RSP0 = cpu->KernelStackTop;
gCpuCount = 1;
KeLoadKernelGsBase(cpu);
KeSetUserGsBase(0);
}
KCPU* KeGetCurrentCpu(void) {
// SMP phase 1: only CPU0 is online. Later this becomes LAPIC-ID or
// per-CPU-base based lookup once the bootloader parks APs for us.
if (gKernelGsReady) {
KCPU* cpu = KeReadCurrentCpuFromGs();
if (cpu >= &gCpus[0] && cpu < &gCpus[KE_MAX_CPUS] && cpu->Self == cpu)
return cpu;
}
// Early bootstrap fallback before GS_BASE has been installed.
return &gCpus[0];
}
code/kernel/ke_debug.c
+89 -20
View File
@@ -3,18 +3,48 @@
#include <stdint.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <intrin.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/mm_virt.h>
#include <common/printf.h>
int _fltused; // fuckass
int _fltused; // MSVC floating-point helper anchor
int gDebugEnabled = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
static volatile long gPanicInProgress = 0;
static volatile long gPanicStoppedCpuCount = 0;
static _Noreturn void KiPanicSecondaryHalt(void) {
_disable();
_InterlockedIncrement(&gPanicStoppedCpuCount);
for (;;) __halt();
}
static int KiPanicIsActive(void) {
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
}
// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
// unexpected NMI is not handled here and will fall through to the normal fault
// path in the assembly stub.
__declspec(dllexport) int KeHandleNmiIpi(void) {
if (KiPanicIsActive())
KiPanicSecondaryHalt();
if (MmHandleTlbShootdownNmi())
return 1;
return 0;
}
void KiKernelPutChar(char c) {
PrintChar(c, 0x00FF00, 0x000000);
}
@@ -39,26 +69,48 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
// KePanic
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
// Silence! Stop the world!
// Silence this CPU first.
_disable();
// Print a bright-red banner so the framebuffer is unmistakable.
// We call PrintChar directly so we can set a red foreground.
// (DbgPrint uses green; we want this visually distinct.)
static const char banner[] = "\n*** KERNEL PANIC ***\n";
for (const char* p = banner; *p; p++)
PrintChar(*p, 0xFF2222, 0x000000);
// Only one CPU owns the visible panic report. CPUs that fault while the
// system is already panicking immediately join the stopped set.
if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) != 0) {
KiPanicSecondaryHalt();
}
// Print the panic code first, then the caller's message through the normal vprintf path.
printf("Panic code: %d\n", code);
// Stop every other CPU as early as possible. This uses an NMI IPI so it
// still reaches CPUs with IF cleared inside spinlock/high-IRQL regions.
HalApicSendPanicIpi();
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
if (PrintChar) {
// Print a bright-red banner so the framebuffer is unmistakable.
static const char banner[] = "\n*** KERNEL PANIC ***\n";
const char* p;
KCPU* cpu = KeGetCurrentCpu();
THREAD* thread = KeGetCurrentThread();
// Die!
while (true) __halt();
for (p = banner; *p; p++)
PrintChar(*p, 0xFF2222, 0x000000);
printf("Panic code: %d\n", code);
if (cpu) {
printf("Panic CPU: %u LAPIC=%u",
(unsigned)cpu->CpuIndex,
(unsigned)cpu->LapicId);
if (thread)
printf(" thread=%p tid=%llu",
thread,
(unsigned long long)thread->ThreadId);
printf("\n");
}
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
}
for (;;) __halt();
}
// KePrintConsole - write user-visible console text in white-on-black.
@@ -434,6 +486,11 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
}
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
// the interrupted context on a secondary CPU once another CPU owns panic.
if (KiPanicIsActive())
KiPanicSecondaryHalt();
THREAD* thread = KeGetCurrentThread();
PROCESS* proc = thread ? thread->Owner : (PROCESS*)0;
@@ -453,14 +510,26 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
(unsigned long long)frame->ErrorCode,
(unsigned long long)frame->Cr2);
} else if (KiFaultHasErrorCode(vector)) {
KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | Code: %04llx\n",
const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip;
KePanic(KE_PANIC_CODE_FAULT,
"PANIC: %s | RIP: %016llx | Code: %04llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n",
name,
(unsigned long long)frame->Rip,
(unsigned long long)frame->ErrorCode);
(unsigned long long)frame->ErrorCode,
(unsigned long long)frame->Cs,
(unsigned long long)frame->Rflags,
(unsigned long long)frame->Rsp,
ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]);
} else {
KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx\n",
const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip;
KePanic(KE_PANIC_CODE_FAULT,
"PANIC: %s | RIP: %016llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n",
name,
(unsigned long long)frame->Rip);
(unsigned long long)frame->Rip,
(unsigned long long)frame->Cs,
(unsigned long long)frame->Rflags,
(unsigned long long)frame->Rsp,
ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]);
}
}
code/kernel/ke_dpc.c
+2 -2
View File
@@ -64,8 +64,8 @@ void KiDrainDpcQueue(void) {
if (!gDpcInitialized)
return;
// Never run DPC routines above DISPATCH_LEVEL. If a future nested IRQ path
// reaches here at DIRQL, leave the work pending for the outermost exit.
// Never run DPC routines above DISPATCH_LEVEL. If a nested IRQ reaches here
// at DIRQL, leave the work pending for the outermost exit.
if (KeGetCurrentIrql() > DISPATCH_LEVEL)
return;
code/kernel/ke_gdt_reload.asm
+52 -45
View File
@@ -1,45 +1,52 @@
; ke_helpers.asm
.code
; void KeFlushGdt(GDTR* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
KeFlushGdt PROC
; RCX = pGdtr (pointer to limit/base struct)
; RDX = codeSel
; R8 = dataSel
; R9 = tssSel
; 1. Load the GDT
lgdt fword ptr [rcx]
; 2. Reload Data Segments
mov ax, r8w
mov ds, ax
mov es, ax
mov ss, ax
mov fs, ax
mov gs, ax
; 3. Load Task Register
mov ax, r9w
ltr ax
; 4. Reload Code Segment (Far Return)
; In MASM64, use OFFSET to get the address of the label.
; We push the selector, then the offset, then issue retff / retfq.
push rdx ; Push new Code Selector (RDX)
lea rax, [reload_cs] ; RIP-relative address of the label
push rax ; Push "return" address
db 048h, 0CBh ; rex_w retf (effectively retfq)
reload_cs:
ret
KeFlushGdt ENDP
; void KeFlushIdt(IDTR* pIdtr);
KeFlushIdt PROC
lidt fword ptr [rcx]
ret
KeFlushIdt ENDP
END
; ke_helpers.asm
.code
; void KeFlushGdt(GDTR* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
KeFlushGdt PROC
; RCX = pGdtr (pointer to limit/base struct)
; RDX = codeSel
; R8 = dataSel
; R9 = tssSel
; 1. Load the GDT
lgdt fword ptr [rcx]
; 2. Reload Data Segments
mov ax, r8w
mov ds, ax
mov es, ax
mov ss, ax
mov fs, ax
mov gs, ax
; 3. Load Task Register
mov ax, r9w
ltr ax
; 4. Reload Code Segment (Far Return)
; In MASM64, use OFFSET to get the address of the label.
; We push the selector, then the offset, then issue retff / retfq.
push rdx ; Push new Code Selector (RDX)
lea rax, [reload_cs] ; RIP-relative address of the label
push rax ; Push "return" address
db 048h, 0CBh ; rex_w retf (effectively retfq)
reload_cs:
ret
KeFlushGdt ENDP
; void KeFlushIdt(IDTR* pIdtr);
KeFlushIdt PROC
lidt fword ptr [rcx]
ret
KeFlushIdt ENDP
; KCPU* KeReadCurrentCpuFromGs(void);
; GS:0 contains KCPU.Self while executing in the kernel.
KeReadCurrentCpuFromGs PROC
mov rax, qword ptr gs:[0]
ret
KeReadCurrentCpuFromGs ENDP
END
code/kernel/ke_idt.c
+53 -37
View File
@@ -41,9 +41,11 @@ extern void KeIrqStub20(void);
extern void KeIrqStub21(void);
extern void KeIrqStub22(void);
extern void KeIrqStub23(void);
extern void KeRescheduleIpiStub(void);
extern void KeFaultDB(void);
extern void KeFaultNMI(void);
extern void KeNmiIpiStub(void);
extern void KeFaultBP(void);
extern void KeFaultOF(void);
extern void KeFaultBR(void);
@@ -77,16 +79,9 @@ extern void KeFaultRSV31(void);
// ---------------------------------------------------------------------------
// GDT
// ---------------------------------------------------------------------------
#define GDT_ENTRIES 8
static uint64_t gGdt[GDT_ENTRIES];
#pragma pack(push, 1)
typedef struct { uint16_t Limit; uint64_t Base; } GDTR;
#pragma pack(pop)
static void KiGdtSetEntry(int idx, uint32_t base, uint32_t limit,
static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
uint8_t access, uint8_t flags) {
gGdt[idx] =
gdt[idx] =
((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
((uint64_t)access << 40) |
@@ -95,54 +90,71 @@ static void KiGdtSetEntry(int idx, uint32_t base, uint32_t limit,
((uint64_t)(base >> 24 & 0xFF)<< 56);
}
static void KiGdtSetTss(int idx, uintptr_t base, uint32_t limit) {
static void KiGdtSetTss(uint64_t* gdt, int idx, uintptr_t base, uint32_t limit) {
// TSS descriptor is 16 bytes (two GDT slots).
gGdt[idx] = 0;
gGdt[idx+1] = 0;
gdt[idx] = 0;
gdt[idx+1] = 0;
gGdt[idx] =
gdt[idx] =
((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
(0x89ULL << 40) | // P=1,DPL=0,type=TSS64 avail
((uint64_t)((limit>>16)&0xF) << 48) |
((uint64_t)(base>>24 & 0xFF) << 56);
gGdt[idx+1] = (base >> 32);
gdt[idx+1] = (base >> 32);
}
TSS64* gTss = (TSS64*)0;
void KeInitializeGdtForCpu(KCPU* cpu) {
if (!cpu) return;
void KeInitializeGdt(void) {
KCPU* cpu = KeGetCurrentCpu();
TSS64* tss = &cpu->Tss;
gTss = tss;
uint64_t* gdt = cpu->Gdt;
memset(gGdt, 0, sizeof(gGdt));
memset(gdt, 0, sizeof(cpu->Gdt));
// 0x00: Null
KiGdtSetEntry(0, 0, 0, 0, 0);
KiGdtSetEntry(gdt, 0, 0, 0, 0, 0);
// 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit)
KiGdtSetEntry(1, 0, 0xFFFFF, 0x9A, 0xA);
KiGdtSetEntry(gdt, 1, 0, 0xFFFFF, 0x9A, 0xA);
// 0x10: Ring-0 Data
KiGdtSetEntry(2, 0, 0xFFFFF, 0x92, 0xC);
KiGdtSetEntry(gdt, 2, 0, 0xFFFFF, 0x92, 0xC);
// 0x18: Ring-3 Code
KiGdtSetEntry(3, 0, 0xFFFFF, 0xFA, 0xA);
KiGdtSetEntry(gdt, 3, 0, 0xFFFFF, 0xFA, 0xA);
// 0x20: Ring-3 Data
KiGdtSetEntry(4, 0, 0xFFFFF, 0xF2, 0xC);
// 0x28: TSS (Occupies two slots: 0x28 and 0x30).
// The TSS storage is per-CPU; phase 1 initializes CPU0 only.
KiGdtSetEntry(gdt, 4, 0, 0xFFFFF, 0xF2, 0xC);
// 0x28: CPU-local TSS (occupies slots 0x28 and 0x30).
tss->IoMapBase = sizeof(TSS64);
if (!tss->RSP0)
tss->RSP0 = cpu->KernelStackTop;
KiGdtSetTss(5, (uintptr_t)tss, sizeof(TSS64) - 1);
KiGdtSetTss(gdt, 5, (uintptr_t)tss, sizeof(TSS64) - 1);
GDTR gdtr = { sizeof(gGdt) - 1, (uint64_t)gGdt };
cpu->Gdtr.Limit = sizeof(cpu->Gdt) - 1;
cpu->Gdtr.Base = (uint64_t)cpu->Gdt;
}
// Call the MASM64 proc to reload everything
// Selectors: CS=0x08, Data=0x10, TSS=0x28
KeFlushGdt(&gdtr, 0x08, 0x10, 0x28);
void KeLoadGdtForCurrentCpu(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
KeFlushGdt(&cpu->Gdtr, 0x08, 0x10, 0x28);
// Reloading GS as a segment register during GDT reload can disturb the
// hidden GS base. Reinstall the kernel per-CPU base immediately after
// the descriptor reload so KeGetCurrentCpu() remains GS-backed.
KeLoadKernelGsBase(cpu);
}
void KeInitializeGdt(void) {
KCPU* cpu = KeGetCurrentCpu();
KeInitializeGdtForCpu(cpu);
KeLoadGdtForCurrentCpu(cpu);
}
void KeSetCurrentTssRsp0(uintptr_t rsp0) {
TSS64* tss = KeGetCurrentTss();
if (tss)
tss->RSP0 = rsp0;
}
// ---------------------------------------------------------------------------
@@ -158,7 +170,7 @@ typedef struct {
uint8_t TypeAttr;
uint16_t OffsetMid;
uint32_t OffsetHigh;
uint32_t Reserved;
uint32_t MustBeZero;
} IDT_ENTRY;
#pragma pack(pop)
@@ -176,14 +188,14 @@ static void KiIdtSetGate(int vec, uintptr_t handler,
gIdt[vec].TypeAttr = typeAttr;
gIdt[vec].OffsetMid = (uint16_t)((handler >> 16) & 0xFFFF);
gIdt[vec].OffsetHigh = (uint32_t)((handler >> 32) & 0xFFFFFFFF);
gIdt[vec].Reserved = 0;
gIdt[vec].MustBeZero = 0;
}
void KeInitializeIdt(void) {
memset(gIdt, 0, sizeof(gIdt));
static void* faultStubs[32] = {
(void*)KeFaultDE, (void*)KeFaultDB, (void*)KeFaultNMI,
(void*)KeFaultDE, (void*)KeFaultDB, (void*)KeNmiIpiStub,
(void*)KeFaultBP, (void*)KeFaultOF, (void*)KeFaultBR,
(void*)KeFaultUD, (void*)KeFaultNM, (void*)KeFaultDF,
(void*)KeFaultCSO, (void*)KeFaultTS, (void*)KeFaultNP,
@@ -214,8 +226,12 @@ void KeInitializeIdt(void) {
for (int i = 0; i < NUM_IRQS; i++)
KiIdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], 0x8E, 0);
KiIdtSetGate(KE_VECTOR_RESCHEDULE_IPI, (uintptr_t)KeRescheduleIpiStub, 0x8E, 0);
KeLoadIdtForCurrentCpu();
}
void KeLoadIdtForCurrentCpu(void) {
IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt };
// Call the MASM64 proc to load IDT
KeFlushIdt(&idtr);
}
code/kernel/ke_irq.c
+16 -12
View File
@@ -2,6 +2,7 @@
#include <kernel/ke_irq.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_dpc.h>
#include <ntlite/ddi/hal_api.h>
@@ -19,10 +20,10 @@ static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
static const HAL_API *gHal;
static KIRQL KiIrqToIrql(uint8_t irq) {
KIRQL irql = (KIRQL)(DIRQL_BASE + irq);
if (irql >= HIGH_LEVEL)
irql = (KIRQL)(HIGH_LEVEL - 1);
return irql;
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)irq;
if (level >= HIGH_LEVEL)
level = (uint32_t)(HIGH_LEVEL - 1);
return (KIRQL)level;
}
void KeIrqInitialize(const HAL_API *hal) {
@@ -51,15 +52,18 @@ void KeIrqDispatch(uint8_t irq) {
KIRQL targetIrql = KiIrqToIrql(irq);
KIRQL currentIrql = KeGetCurrentIrql();
// Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected
// while IF is clear, but this keeps the software state sane if that changes
// later.
// Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected
// while IF is clear, but preserving monotonic IRQL keeps nesting safe.
if (targetIrql < currentIrql)
targetIrql = currentIrql;
KIRQL oldIrql = KeRaiseIrql(targetIrql);
if (irq < MAX_IRQS) {
if (irq == KE_VECTOR_RESCHEDULE_IPI) {
// Nothing to do here. The sender has already set this CPU's
// ReschedulePending flag. KiExitInterrupt consumes it after EOI and
// after the interrupted frame unwinds to PASSIVE_LEVEL.
} else if (irq < MAX_IRQS) {
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
for (uint8_t i = 0; i < list->Count; i++) {
if (list->Handlers[i])
@@ -85,10 +89,10 @@ void KiExitInterrupt(void) {
if (cpu->InterruptDepth)
cpu->InterruptDepth--;
// A reschedule is only legal after the interrupt unwound back to the
// interrupted thread's original PASSIVE context. If a future path ever
// permits nested interrupting of DISPATCH/DIRQL code, leave the request
// pending instead of switching while a spinlock/high-IRQL region is active.
// A reschedule is only legal after the interrupt unwinds back to the
// interrupted thread's PASSIVE context. If an interrupt hit DISPATCH/DIRQL
// code, leave the request pending instead of switching out of a spinlock or
// high-IRQL region.
if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
KiDrainDpcQueue();
if (KeGetCurrentIrql() == PASSIVE_LEVEL)
code/kernel/ke_sched.c
+332 -60
View File
@@ -1,6 +1,7 @@
// ke_sched.c - MLFQ scheduler
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_process.h>
@@ -8,14 +9,64 @@
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_list.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
static LIST_ENTRY gRunQueue[PriorityCount];
static SPINLOCK gSchedLock;
static uint32_t gAgingCounter = 0;
static KE_SCHED_STATS gSchedStats;
static PROCESS* gIdleProcess;
static uint64_t KiDefaultThreadAffinityMask(void) {
// Normal kernel, user, and driver-created threads are BSP-affine by
// default. Code that is explicitly audited for AP execution can widen its
// affinity before the thread enters the ready queue.
return 1ULL;
}
static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
if (!thread || !cpu) return 0;
if (thread->IsIdleThread) return thread == cpu->IdleThread;
if (cpu->CpuIndex >= 64) return 0;
return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
}
static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || !target->Online)
return;
_InterlockedExchange(&target->ReschedulePending, 1);
KCPU* self = KeGetCurrentCpu();
if (self && target->CpuIndex != self->CpuIndex) {
HalApicSendRescheduleIpi(target->LapicId);
}
}
static void KiPokeRunnableCpusLocked(THREAD* thread) {
if (!thread || thread->IsIdleThread)
return;
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
THREAD* cur = cpu->CurrentThread;
if (!cur || cur == cpu->IdleThread) {
KiRequestRescheduleCpuLocked(cpu);
} else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) {
gSchedStats.PriorityPreemptions++;
KiRequestRescheduleCpuLocked(cpu);
}
}
}
static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
switch (priority) {
@@ -60,29 +111,133 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
KeReleaseSpinlock(&gSchedLock, flags);
}
static void KiIdleLoop(void) {
while (true) { __halt(); }
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
// The scheduler lock is intentionally held across KeSwitchContext(). The
// unlock cookie belongs to the thread continuation that will resume after
// the switch, not to the physical CPU. Another dispatch may resume this
// thread from an unrelated interrupt, so using a KCPU slot here would
// restore the outgoing CPU context's IF/IRQL into the incoming thread.
if (thread)
thread->SwitchLockCookie = cookie;
}
void KeInitializeScheduler(void) {
KCPU* cpu = KeGetCurrentCpu();
static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
return thread ? thread->SwitchLockCookie : 0x202ULL;
}
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"scheduler invariant failed: %s: thread=0x%llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
where,
(uint64_t)thread,
thread ? thread->ThreadId : 0,
thread ? (unsigned)thread->State : 0,
thread ? (unsigned)thread->Queued : 0,
thread ? (unsigned)thread->RunningCpu : 0,
thread ? (unsigned)thread->IsIdleThread : 0,
why ? why : "?");
}
static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null thread");
if (thread->IsIdleThread)
KiSchedulerBug(where, thread, "idle thread cannot enter ready queue");
if (thread->State == ThreadDead)
KiSchedulerBug(where, thread, "dead thread cannot enter ready queue");
if (thread->Queued)
KiSchedulerBug(where, thread, "thread already in ready queue");
if (thread->RunningCpu != KE_INVALID_CPU)
KiSchedulerBug(where, thread, "thread still reserved/running on a CPU");
}
static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null picked thread");
if (thread->IsIdleThread)
return;
if (thread->Queued)
KiSchedulerBug(where, thread, "picked thread still marked queued");
if (thread->RunningCpu != KE_INVALID_CPU)
KiSchedulerBug(where, thread, "picked thread already running on a CPU");
if (thread->State != ThreadReady)
KiSchedulerBug(where, thread, "picked thread was not ready");
}
static void KiInsertReadyLocked(THREAD* thread) {
KiAssertReadyQueueCandidateLocked("KiInsertReadyLocked", thread);
int p = (int)thread->Priority;
if (p < 0) p = 0;
if (p >= PriorityCount) p = PriorityCount - 1;
thread->State = ThreadReady;
thread->Queued = 1;
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
}
static int KiAnyRunnableForCpuLocked(KCPU* cpu);
static int KiAnyRunnableForCurrentCpu(void) {
KCPU* cpu = KeGetCurrentCpu();
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
int any = KiAnyRunnableForCpuLocked(cpu);
KeReleaseSpinlock(&gSchedLock, flags);
return any;
}
static void KiIdleLoop(void) {
for (;;) {
KCPU* cpu = KeGetCurrentCpu();
cpu->SchedulerTicks++;
// AP idle threads sleep until an interrupt or reschedule IPI wakes the
// CPU. Ready paths set ReschedulePending and target the CPU's LAPIC;
// KiExitInterrupt consumes the request after EOI.
if (KiAnyRunnableForCurrentCpu()) {
KeYield();
continue;
}
// HLT only wakes for maskable IPIs when IF=1. A CPU that reaches
// the idle thread through an unusual switch path must not sleep with
// interrupts disabled, otherwise reschedule/TLB shootdown IPIs can
// be delayed forever.
_enable();
__halt();
}
}
void KiInitializeSchedulerGlobal(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
PROCESS* idleProc;
KSTATUS s = KeCreateProcess("idle", &idleProc);
if (!KSUCCESS(s)) return;
KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create idle process, status=%u\n", s);
PROCESS* systemProc;
s = KeCreateSystemProcess(&systemProc);
if (!KSUCCESS(s)) return;
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create system process, status=%u\n", s);
(void)systemProc;
}
KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
if (!cpu || !gIdleProcess)
return STATUS_INVALID_PARAMETER;
if (cpu->IdleThread)
return STATUS_SUCCESS;
THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!idle)
return STATUS_NO_MEMORY;
idle->Header.Destroy = (void *)0;
idle->ThreadId = 0;
idle->State = ThreadReady;
idle->ThreadId = cpu->CpuIndex ? (0x80000000u | cpu->CpuIndex) : 0;
idle->State = ThreadRunning;
idle->Priority = PriorityLow;
idle->QuantaConsumed = 0;
idle->QuantumTicks = 1;
@@ -90,12 +245,16 @@ void KeInitializeScheduler(void) {
idle->CpuBurstTicks = 0;
idle->LastRunTicks = 0;
idle->RunningCpu = cpu->CpuIndex;
idle->Owner = idleProc;
idle->Queued = 0;
idle->IsIdleThread = 1;
idle->SchedulerFlags = 0;
idle->LastCpu = cpu->CpuIndex;
idle->AffinityMask = (cpu->CpuIndex < 64) ? (1ULL << cpu->CpuIndex) : 0ULL;
idle->Owner = gIdleProcess;
idle->WaitObject = (OBJECT *)0;
idle->WaitStatus = STATUS_SUCCESS;
idle->WaitListHead = (LIST_ENTRY*)0;
idle->AlertableWait = 0;
idle->SchedLockFlags = 0x202ULL; // IF=1
idle->UserStackBase = (void*)0;
idle->Teb = (void*)0;
idle->TebSize = 0;
@@ -106,6 +265,10 @@ void KeInitializeScheduler(void) {
LIST_INIT(&idle->SchedulerLink);
void* kstack = MmAllocatePool(KSTACK_SIZE);
if (!kstack) {
ObDereferenceObject(&idle->Header);
return STATUS_NO_MEMORY;
}
idle->KernelStackBase = kstack;
idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
idle->TssRsp0 = idle->KernelStackTop;
@@ -121,22 +284,34 @@ void KeInitializeScheduler(void) {
// The hand-built CPU idle thread is not created through KeCreateThread(),
// so register it manually with the idle process. This keeps PID 0 visible
// as the real idle process while kernel worker threads live under PID 1.
ObReferenceObject(&idleProc->Header);
uintptr_t idleFlags = KeAcquireSpinlock(&idleProc->HandleLock);
ListInsertTail(&idleProc->ThreadList, &idle->ProcessLink);
idleProc->ThreadCount++;
KeReleaseSpinlock(&idleProc->HandleLock, idleFlags);
ObReferenceObject(&gIdleProcess->Header);
uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock);
ListInsertTail(&gIdleProcess->ThreadList, &idle->ProcessLink);
gIdleProcess->ThreadCount++;
KeReleaseSpinlock(&gIdleProcess->HandleLock, idleFlags);
cpu->IdleThread = idle;
cpu->IdleThread = idle;
cpu->CurrentThread = idle;
cpu->OutgoingThread = (THREAD*)0;
idle->SwitchLockCookie = 0x202ULL;
return STATUS_SUCCESS;
}
void KeInitializeScheduler(void) {
KiInitializeSchedulerGlobal();
KCPU* cpu = KeGetCurrentCpu();
KSTATUS s = KiCreateIdleThreadForCpu(cpu);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create CPU%u idle thread, status=%u\n",
(unsigned)cpu->CpuIndex, s);
}
// Safely release the lock and clean up any dead threads natively post-switch
void KeSchedulerPostSwitch(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags);
KeReleaseSpinlock(&gSchedLock, KiPostSwitchUnlockCookie(cur));
KiDeliverKernelApcs();
@@ -148,24 +323,29 @@ void KeSchedulerPostSwitch(void) {
}
}
void KeReadyThread(THREAD* thread) {
if (!thread) return;
thread->State = ThreadReady;
int p = (int)thread->Priority;
if (p < 0) p = 0;
if (p >= PriorityCount) p = PriorityCount - 1;
void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask) {
if (!thread || thread->IsIdleThread) return;
if (!affinityMask)
affinityMask = 1ULL;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
thread->AffinityMask = affinityMask;
if (thread->Queued)
KiPokeRunnableCpusLocked(thread);
KeReleaseSpinlock(&gSchedLock, flags);
}
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
if (!cur || cur == cpu->IdleThread) {
KiRequestReschedule();
} else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) {
gSchedStats.PriorityPreemptions++;
KiRequestReschedule();
}
uint64_t KeGetDefaultThreadAffinityMask(void) {
return KiDefaultThreadAffinityMask();
}
void KeReadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiInsertReadyLocked(thread);
KiPokeRunnableCpusLocked(thread);
KeReleaseSpinlock(&gSchedLock, flags);
}
@@ -173,35 +353,58 @@ void KeReadyThread(THREAD* thread) {
void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListRemove(&thread->SchedulerLink);
if (thread->Queued) {
ListRemove(&thread->SchedulerLink);
thread->Queued = 0;
}
KeReleaseSpinlock(&gSchedLock, flags);
}
static int KiAnyRunnableLocked(void) {
static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
for (int p = 0; p < PriorityCount; p++) {
if (!LIST_IS_EMPTY(&gRunQueue[p]))
return 1;
LIST_ENTRY* head = &gRunQueue[p];
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (KiThreadCanRunOnCpuLocked(thread, cpu))
return 1;
}
}
return 0;
}
static THREAD* KiPickNext(void) {
static THREAD* KiPickNextForCpu(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
for (int p = PriorityCount - 1; p >= 0; p--) {
if (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = gRunQueue[p].Fwd;
LIST_ENTRY* head = &gRunQueue[p];
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
ListRemove(node);
return CONTAINING_RECORD(node, THREAD, SchedulerLink);
thread->Queued = 0;
KiAssertPickedThreadLocked("KiPickNextForCpu", thread);
return thread;
}
}
return KeGetCurrentCpu()->IdleThread;
return cpu->IdleThread;
}
void KiRequestReschedule(void) {
KeGetCurrentCpu()->ReschedulePending = 1;
KCPU* cpu = KeGetCurrentCpu();
if (cpu)
_InterlockedExchange(&cpu->ReschedulePending, 1);
}
int KiIsReschedulePending(void) {
return KeGetCurrentCpu()->ReschedulePending != 0;
KCPU* cpu = KeGetCurrentCpu();
return cpu && _InterlockedCompareExchange(&cpu->ReschedulePending, 0, 0) != 0;
}
static void KiSchedule(void) {
@@ -210,16 +413,16 @@ static void KiSchedule(void) {
THREAD* idle = cpu->IdleThread;
if (!cur) return;
cpu->ReschedulePending = 0;
_InterlockedExchange(&cpu->ReschedulePending, 0);
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags;
KiArmPostSwitchUnlock(cur, flags);
// The currently running thread is not present in a run queue. If it is
// still runnable and nobody else is ready, keep executing it. Otherwise a
// lone CPU-bound thread gets alternated with the idle thread on every timer
// tick, effectively turning a periodic preemption check into a 50% throttle.
if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableLocked()) {
if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableForCpuLocked(cpu)) {
if (cur->QuantumRemaining == 0)
KiRefillQuantumLocked(cur);
gSchedStats.KeepCurrentDecisions++;
@@ -227,22 +430,21 @@ static void KiSchedule(void) {
return;
}
THREAD* next = KiPickNext();
THREAD* next = KiPickNextForCpu(cpu);
if (next == idle) gSchedStats.IdleSelections++;
if (next == cur) {
// Only re-queue if still running normally; a ThreadWaiting/ThreadDead
// caller has already placed SchedulerLink elsewhere (or is discarding
// the thread).
if (cur != idle && cur->State == ThreadRunning)
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
if (cur != idle)
KiSchedulerBug("KiSchedule", cur, "current thread unexpectedly appeared in ready queue");
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
KiAssertPickedThreadLocked("KiSchedule", next);
next->State = ThreadRunning;
KiRefillQuantumLocked(next);
cpu->CurrentThread = next;
next->RunningCpu = cpu->CpuIndex;
next->LastCpu = cpu->CpuIndex;
// Only re-queue the outgoing thread if it is still in the running state.
// If KeBlockThread set State = ThreadWaiting, SchedulerLink was already
@@ -250,11 +452,14 @@ static void KiSchedule(void) {
// corrupt both lists simultaneously (same intrusive node in two lists).
// If KeExitThread set State = ThreadDead, the thread must not be re-queued.
if (cur != idle && cur->State == ThreadRunning) {
cur->RunningCpu = KE_INVALID_CPU;
cur->LastCpu = cpu->CpuIndex;
cur->State = ThreadReady;
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
KiInsertReadyLocked(cur);
} else if (cur) {
cur->RunningCpu = KE_INVALID_CPU;
cur->LastCpu = cpu->CpuIndex;
}
if (cur) cur->RunningCpu = KE_INVALID_CPU;
cpu->OutgoingThread = cur;
cpu->ContextSwitches++;
gSchedStats.ContextSwitches++;
@@ -267,6 +472,28 @@ void KiScheduleFromInterrupt(void) {
KiSchedule();
}
void KiSchedulerMarkThreadWaiting(THREAD* thread) {
if (!thread) return;
KCPU* cpu = KeGetCurrentCpu();
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (thread->IsIdleThread)
KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "idle thread cannot block");
if (thread->Queued)
KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "queued thread cannot block");
if (thread != cpu->CurrentThread)
KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "only the current thread can prepare a blocking wait");
if (thread->RunningCpu != cpu->CpuIndex)
KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread running CPU does not match current CPU");
if (thread->State != ThreadRunning)
KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread was not running before wait");
thread->State = ThreadWaiting;
KeReleaseSpinlock(&gSchedLock, flags);
}
void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -275,6 +502,51 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
// Complete the wait while holding the scheduler lock. On SMP a wake can
// race the prepare->commit window on another CPU: the target thread is
// already on the object's wait list and has State=ThreadWaiting, but it is
// still executing with RunningCpu set until it actually calls KeYield().
// That case must complete in-place, not enqueue the still-running thread.
if (thread->State != ThreadWaiting) {
thread->WaitStatus = status;
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = status;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->CpuBurstTicks = 0;
thread->QuantaConsumed = 0;
thread->QuantumRemaining = 0;
if (thread->RunningCpu != KE_INVALID_CPU) {
// Prepare->commit race: the waiter is still running somewhere. Its
// KeCommitBlockThread() will see State != ThreadWaiting and return the
// completed wait status without ever entering the ready queue.
thread->State = ThreadRunning;
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
if (thread->Priority < PriorityHigh) {
thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1);
}
KiInsertReadyLocked(thread);
KiPokeRunnableCpusLocked(thread);
KeReleaseSpinlock(&gSchedLock, flags);
}
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -282,14 +554,14 @@ void KeSchedulerTick(void) {
if (!cur) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags;
KiArmPostSwitchUnlock(cur, flags);
gSchedStats.Ticks++;
int requestReschedule = 0;
if (cur == idle) {
if (KiAnyRunnableLocked())
if (KiAnyRunnableForCpuLocked(cpu))
requestReschedule = 1;
} else {
cur->CpuBurstTicks++;
@@ -312,7 +584,7 @@ void KeSchedulerTick(void) {
cur->QuantumRemaining--;
if (cur->QuantumRemaining == 0) {
if (KiAnyRunnableLocked()) {
if (KiAnyRunnableForCpuLocked(cpu)) {
gSchedStats.QuantumExpirations++;
requestReschedule = 1;
} else {
code/kernel/ke_selftest.c
+83 -1
View File
@@ -1,10 +1,11 @@
// ke_selftest.c - temporary kernel bring-up self-tests for execution/I/O model.
// ke_selftest.c - kernel bring-up and regression self-tests.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_event.h>
#include <kernel/ke_irp.h>
@@ -17,6 +18,7 @@
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>
// -------------------------------------------------------------------------
// Kernel APC self-test
@@ -747,6 +749,86 @@ static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: PASS\n");
}
void KiRunCoreHardeningSelfTests(void) {
DbgPrint("CORE selftest: begin\n");
if (gCpuCount == 0 || gCpuCount > KE_MAX_CPUS)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid CPU count %u\n", gCpuCount);
KCPU* cpu = KeGetCurrentCpu();
if (cpu != &gCpus[0])
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%llX cpu0=0x%llX\n",
(uint64_t)cpu, (uint64_t)&gCpus[0]);
if (cpu->Self != cpu)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%llX cpu=0x%llX\n",
(uint64_t)cpu->Self, (uint64_t)cpu);
if (KeGetCurrentCpuIndex() != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n");
if (!cpu->Present || !cpu->Started || !cpu->Online || !cpu->IsBsp)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle flags invalid P/S/O/B=%u/%u/%u/%u\n",
cpu->Present, cpu->Started, cpu->Online, cpu->IsBsp);
if (cpu->Parked)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP is marked parked\n");
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql());
if (KeGetCurrentTss() != &cpu->Tss)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n");
if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64))
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%llX iomap=%u\n",
cpu->Tss.RSP0, cpu->Tss.IoMapBase);
if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%llX gdt=0x%llX\n",
cpu->Gdtr.Base, (uint64_t)cpu->Gdt);
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* ap = &gCpus[i];
if (!ap->Present || !ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n",
i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp);
if (!ap->SchedulerCommand || !ap->SchedulerActive)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u scheduler admission invalid command=%ld active=%ld\n",
i, ap->SchedulerCommand, ap->SchedulerActive);
if (!ap->KernelStack || !ap->KernelStackTop)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%llX iomap=%u\n",
i, ap->Tss.RSP0, ap->Tss.IoMapBase);
if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i);
if (!ap->IdleThread || !ap->CurrentThread || ap->CurrentThread != ap->IdleThread)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u idle/current thread missing\n", i);
if (!ap->IdleThread->IsIdleThread || ap->IdleThread->Queued || ap->IdleThread->RunningCpu != i)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u idle invariant failed queued=%u idle=%u runningCpu=%u\n",
i, ap->IdleThread->Queued, ap->IdleThread->IsIdleThread, ap->IdleThread->RunningCpu);
}
THREAD* cur = KeGetCurrentThread();
if (!cur || cur != cpu->CurrentThread)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current thread mismatch\n");
if (!cpu->IdleThread || cur != cpu->IdleThread)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: scheduler did not start on CPU0 idle thread\n");
if (!cur->IsIdleThread || cur->Queued || cur->RunningCpu != cpu->CpuIndex)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: idle invariant failed queued=%u idle=%u runningCpu=%u\n",
cur->Queued, cur->IsIdleThread, cur->RunningCpu);
DbgPrint("CORE selftest: PASS\n");
}
void KiRunEarlyKernelSelfTests(void) {
KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests();
if (!KSUCCESS(s)) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest failed status=0x%x\n",
s);
}
}
void KiRunKernelSelfTests(const HAL_API* hal) {
DbgPrint("Kernel selftests: begin\n");
KiRunKernelApcSelfTest();
code/kernel/ke_smp.c
+660
View File
@@ -0,0 +1,660 @@
// ke_smp.c - SMP application-processor startup and scheduler admission
//
// The bootloader is intentionally not involved in AP ownership. The kernel
// asks the HAL for MADT-discovered LAPIC IDs, copies a small low-memory x86 AP
// startup blob, sends INIT/SIPI through the local APIC, and waits for each AP
// to enter KiApKernelEntry() and park in kernel-owned state.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/status.h>
#include <kernel/ke_smp.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_event.h>
#include <kernel/ob_object.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
#define AP_TRAMPOLINE_MAX_PHYS 0x000A0000ULL
#define AP_TRAMPOLINE_BOOT_OFF 0x0800u
#define AP_STACK_PAGES 8u
#define AP_START_TIMEOUT_SPINS 20000000u
#define AP_SCHED_TIMEOUT_SPINS 20000000u
#define AP_CR4_PAE 0x00000020u
#define AP_CR4_OSFXSR 0x00000200u
#define AP_CR4_OSXMMEXCPT 0x00000400u
#define AP_CR4_REQUIRED (AP_CR4_PAE | AP_CR4_OSFXSR | AP_CR4_OSXMMEXCPT)
#define AP_CR0_PE 0x00000001u
#define AP_CR0_MP 0x00000002u
#define AP_CR0_EM 0x00000004u
#define AP_CR0_TS 0x00000008u
#define AP_CR0_NE 0x00000020u
#define AP_CR0_PG 0x80000000u
#define AP_BOOT_MAGIC 0x3436544F4F425041ULL /* 'APBOOT64' */
#define AP_STATE_EMPTY 0u
#define AP_STATE_READY 1u
#define AP_STATE_LONG_MODE 2u
#define AP_STATE_KERNEL_ENTRY 3u
#define AP_STATE_KERNEL_PARKED 4u
#define AP_STATE_FAILED 0xFFFFFFFFu
#define AP_STAGE_NONE 0x00000000u
#define AP_STAGE_LONG64 0x4C4D3634u /* 'LM64' */
#define AP_STAGE_ENTRY 0x454E5452u /* 'ENTR' */
#define AP_STAGE_GS 0x47534241u /* 'GSBA' */
#define AP_STAGE_GDT 0x47445421u /* 'GDT!' */
#define AP_STAGE_IDT 0x49445421u /* 'IDT!' */
#define AP_STAGE_PARK 0x5041524Bu /* 'PARK' */
#pragma pack(push, 1)
typedef struct AP_BOOT_BLOCK {
uint64_t Magic;
uint64_t KernelCr3;
uint64_t LongModeEntry;
uint64_t Kcpu;
uint64_t StackTop;
volatile uint64_t Heartbeat;
volatile uint32_t State;
volatile uint32_t Stage;
volatile uint32_t ApIndex;
} AP_BOOT_BLOCK;
#pragma pack(pop)
_Static_assert(offsetof(AP_BOOT_BLOCK, KernelCr3) == 8, "AP_BOOT_BLOCK.KernelCr3 offset");
_Static_assert(offsetof(AP_BOOT_BLOCK, StackTop) == 32, "AP_BOOT_BLOCK.StackTop offset");
_Static_assert(offsetof(AP_BOOT_BLOCK, State) == 48, "AP_BOOT_BLOCK.State offset");
_Static_assert(offsetof(AP_BOOT_BLOCK, Stage) == 52, "AP_BOOT_BLOCK.Stage offset");
static PHYSICAL_ADDRESS gApTrampolinePhys;
static AP_BOOT_BLOCK* gApBootBlock;
static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
static void KiEmit16(uint8_t** p, uint16_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
static void KiEmit32(uint8_t** p, uint32_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
static void KiEmit64(uint8_t** p, uint64_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
static void KiPatch16(uint8_t* base, size_t off, uint16_t v) { memcpy(base + off, &v, sizeof(v)); }
static void KiPatch32(uint8_t* base, size_t off, uint32_t v) { memcpy(base + off, &v, sizeof(v)); }
static void KiPatch64(uint8_t* base, size_t off, uint64_t v) { memcpy(base + off, &v, sizeof(v)); }
static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) {
return ((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
((uint64_t)access << 40) |
((uint64_t)((limit >> 16) & 0xF) << 48) |
((uint64_t)(flags & 0xF) << 52) |
((uint64_t)((base >> 24) & 0xFF) << 56);
}
static KSTATUS KiBuildApTrampoline(void) {
if (gApTrampolinePhys)
return STATUS_SUCCESS;
PHYSICAL_ADDRESS frame = MmAllocateFrameBelow(AP_TRAMPOLINE_MAX_PHYS);
if (!frame || frame < PAGE_SIZE || (frame & (PAGE_SIZE - 1)))
return STATUS_NO_MEMORY;
uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame);
memset(page, 0, PAGE_SIZE);
uint32_t physBase = (uint32_t)frame;
uint8_t* p = page;
// --- 16-bit real-mode SIPI entry ---
KiEmit8(&p, 0xFA); // cli
KiEmit8(&p, 0x8C); KiEmit8(&p, 0xC8); // mov ax, cs
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD8); // mov ds, ax
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x01); KiEmit8(&p, 0x16);
size_t gdtr16Patch = (size_t)(p - page); KiEmit16(&p, 0); // lgdt [gdtr]
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x20); KiEmit8(&p, 0xC0); // mov eax, cr0
KiEmit8(&p, 0x66); KiEmit8(&p, 0x83); KiEmit8(&p, 0xC8); KiEmit8(&p, 0x01); // or eax, 1
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xC0); // mov cr0, eax
KiEmit8(&p, 0x66); KiEmit8(&p, 0xEA); // jmp far 32-bit offset
size_t prot32Patch = (size_t)(p - page); KiEmit32(&p, 0);
KiEmit16(&p, 0x0008); // selector 32-bit code
while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90);
uint32_t prot32Off = (uint32_t)(p - page);
// --- 32-bit protected-mode transition to long mode ---
KiEmit8(&p, 0x66); KiEmit8(&p, 0xB8); KiEmit16(&p, 0x0010); // mov ax, 0x10
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD8); // mov ds, ax
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xC0); // mov es, ax
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD0); // mov ss, ax
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xE0); // mov fs, ax
KiEmit8(&p, 0x8E); KiEmit8(&p, 0xE8); // mov gs, ax
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x20); KiEmit8(&p, 0xE0); // mov eax, cr4
KiEmit8(&p, 0x0D); KiEmit32(&p, AP_CR4_REQUIRED); // or eax, PAE|OSFXSR|OSXMMEXCPT
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xE0); // mov cr4, eax
KiEmit8(&p, 0xB9); KiEmit32(&p, 0xC0000080u); // mov ecx, IA32_EFER
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x32); // rdmsr
KiEmit8(&p, 0x0D); KiEmit32(&p, 0x00000900u); // or eax, LME|NXE
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x30); // wrmsr
KiEmit8(&p, 0xA1); size_t cr3Patch = (size_t)(p - page); KiEmit32(&p, 0); // mov eax,[cr3]
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xD8); // mov cr3, eax
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x20); KiEmit8(&p, 0xC0); // mov eax, cr0
KiEmit8(&p, 0x25); KiEmit32(&p, ~(AP_CR0_EM | AP_CR0_TS)); // and eax, ~(EM|TS)
KiEmit8(&p, 0x0D); KiEmit32(&p, AP_CR0_PG | AP_CR0_PE | AP_CR0_MP | AP_CR0_NE); // or eax, PG|PE|MP|NE
KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xC0); // mov cr0, eax
KiEmit8(&p, 0xEA); size_t long64Patch = (size_t)(p - page); KiEmit32(&p, 0);
KiEmit16(&p, 0x0018); // selector 64-bit code
while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90);
uint32_t long64Off = (uint32_t)(p - page);
// --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). ---
KiEmit8(&p, 0x48); KiEmit8(&p, 0xBB); size_t bootBlockPatch = (size_t)(p - page); KiEmit64(&p, 0); // mov rbx, bootblock
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Stage)); KiEmit32(&p, AP_STAGE_LONG64);
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, State)); KiEmit32(&p, AP_STATE_LONG_MODE);
KiEmit8(&p, 0x48); KiEmit8(&p, 0x8B); KiEmit8(&p, 0xA3); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, StackTop)); // mov rsp,[rbx+StackTop]
KiEmit8(&p, 0x48); KiEmit8(&p, 0x83); KiEmit8(&p, 0xE4); KiEmit8(&p, 0xF0); // and rsp,-16
KiEmit8(&p, 0x48); KiEmit8(&p, 0x83); KiEmit8(&p, 0xEC); KiEmit8(&p, 0x28); // sub rsp,28h
KiEmit8(&p, 0x48); KiEmit8(&p, 0x8B); KiEmit8(&p, 0x8B); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Kcpu)); // mov rcx,[rbx+Kcpu]
KiEmit8(&p, 0x48); KiEmit8(&p, 0x8B); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, LongModeEntry)); // mov rax,[rbx+LongModeEntry]
KiEmit8(&p, 0xFF); KiEmit8(&p, 0xE0); // jmp rax
while (((uintptr_t)(p - page)) & 0x07) KiEmit8(&p, 0);
uint32_t gdtOff = (uint32_t)(p - page);
uint64_t* gdt = (uint64_t*)p;
gdt[0] = 0;
gdt[1] = KiGdtDescriptor(0, 0xFFFFF, 0x9A, 0xC); // 32-bit code
gdt[2] = KiGdtDescriptor(0, 0xFFFFF, 0x92, 0xC); // 32-bit data
gdt[3] = KiGdtDescriptor(0, 0xFFFFF, 0x9A, 0xA); // 64-bit code
gdt[4] = KiGdtDescriptor(0, 0xFFFFF, 0x92, 0xC); // 64-bit data
p += 5 * sizeof(uint64_t);
uint32_t gdtrOff = (uint32_t)(p - page);
KiEmit16(&p, (uint16_t)(5 * sizeof(uint64_t) - 1));
KiEmit32(&p, physBase + gdtOff);
if ((uintptr_t)(p - page) >= AP_TRAMPOLINE_BOOT_OFF)
return STATUS_DEVICE_ERROR;
AP_BOOT_BLOCK* boot = (AP_BOOT_BLOCK*)(page + AP_TRAMPOLINE_BOOT_OFF);
memset(boot, 0, sizeof(*boot));
boot->Magic = AP_BOOT_MAGIC;
KiPatch16(page, gdtr16Patch, (uint16_t)gdtrOff);
KiPatch32(page, prot32Patch, physBase + prot32Off);
KiPatch32(page, cr3Patch, physBase + AP_TRAMPOLINE_BOOT_OFF + (uint32_t)offsetof(AP_BOOT_BLOCK, KernelCr3));
KiPatch32(page, long64Patch, physBase + long64Off);
KiPatch64(page, bootBlockPatch, (uint64_t)(physBase + AP_TRAMPOLINE_BOOT_OFF));
gApTrampolinePhys = frame;
gApBootBlock = boot;
return STATUS_SUCCESS;
}
_Noreturn void KiApKernelEntry(KCPU* cpu) {
_disable();
if (!cpu) {
while (1) __halt();
}
// APs start from the SIPI reset state, so do not assume the firmware/BSP
// CR4 feature bits are present. The scheduler and context-switch paths
// use FXSAVE/FXRSTOR on every CPU; without OSFXSR, those instructions
// raise #UD as soon as an AP enters its idle thread.
uintptr_t cr4 = __readcr4();
if ((cr4 & AP_CR4_REQUIRED) != AP_CR4_REQUIRED)
__writecr4(cr4 | AP_CR4_REQUIRED);
if (gApBootBlock) {
gApBootBlock->Stage = AP_STAGE_ENTRY;
gApBootBlock->State = AP_STATE_KERNEL_ENTRY;
gApBootBlock->Heartbeat++;
}
cpu->Self = cpu;
KeLoadKernelGsBase(cpu);
KeSetUserGsBase(0);
if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_GS;
HalApicEnableCurrentCpu();
KeInitializeGdtForCpu(cpu);
KeLoadGdtForCurrentCpu(cpu);
if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_GDT;
KeLoadIdtForCurrentCpu();
if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_IDT;
cpu->Started = 1;
cpu->Parked = 1;
cpu->Online = 0;
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
if (gApBootBlock) {
gApBootBlock->Stage = AP_STAGE_PARK;
gApBootBlock->State = AP_STATE_KERNEL_PARKED;
}
for (;;) {
cpu->SchedulerTicks++;
if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
THREAD* idle = cpu->IdleThread;
if (idle) {
cpu->CurrentThread = idle;
idle->State = ThreadRunning;
idle->RunningCpu = cpu->CpuIndex;
idle->Queued = 0;
cpu->Parked = 0;
_InterlockedExchange(&cpu->SchedulerActive, 1);
_enable();
KeStartInitialThread(idle);
}
}
_mm_pause();
}
}
static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
KCPU* cpu = &gCpus[cpuIndex];
memset(cpu, 0, sizeof(*cpu));
cpu->Self = cpu;
cpu->CpuIndex = cpuIndex;
cpu->LapicId = info->LapicId;
cpu->AcpiProcessorId = info->ProcessorUid;
cpu->BootProcessorIndex = cpuIndex;
cpu->Present = info->Enabled ? 1 : 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
cpu->IsBsp = info->IsBsp ? 1 : 0;
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
cpu->SchedulerCommand = 0;
cpu->SchedulerActive = 0;
}
static KSTATUS KiAllocateApStack(KCPU* cpu) {
PHYSICAL_ADDRESS stackPhys = MmAllocateContiguous(AP_STACK_PAGES);
if (!stackPhys) return STATUS_NO_MEMORY;
cpu->KernelStack = (void*)(PHYS_MAP_BASE + stackPhys);
cpu->KernelStackTop = (uintptr_t)cpu->KernelStack + AP_STACK_PAGES * PAGE_SIZE;
memset(cpu->KernelStack, 0, AP_STACK_PAGES * PAGE_SIZE);
memset(&cpu->Tss, 0, sizeof(cpu->Tss));
cpu->Tss.IoMapBase = sizeof(TSS64);
cpu->Tss.RSP0 = cpu->KernelStackTop;
KeInitializeGdtForCpu(cpu);
return STATUS_SUCCESS;
}
static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) {
memset(gApBootBlock, 0, sizeof(*gApBootBlock));
gApBootBlock->Magic = AP_BOOT_MAGIC;
gApBootBlock->KernelCr3 = (uint64_t)__readcr3();
gApBootBlock->LongModeEntry = (uint64_t)(uintptr_t)KiApKernelEntry;
gApBootBlock->Kcpu = (uint64_t)(uintptr_t)cpu;
gApBootBlock->StackTop = cpu->KernelStackTop;
gApBootBlock->State = AP_STATE_READY;
gApBootBlock->Stage = AP_STAGE_NONE;
gApBootBlock->ApIndex = apIndex;
}
KSTATUS KeStartApplicationProcessors(void) {
HAL_CPU_TOPOLOGY topo;
KSTATUS st = HalGetProcessorTopology(&topo);
if (!KSUCCESS(st)) return st;
if (topo.CpuCount == 0 || topo.CpuCount > KE_MAX_CPUS)
return STATUS_DEVICE_ERROR;
uint32_t bspLapic = HalGetBspLapicId();
KCPU* bsp = KeGetCurrentCpu();
bsp->LapicId = bspLapic;
bsp->IsBsp = 1;
bsp->Present = 1;
bsp->Started = 1;
bsp->Online = 1;
bsp->Parked = 0;
bsp->SchedulerCommand = 0;
bsp->SchedulerActive = 1;
uint32_t count = 1;
for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) {
HAL_CPU_INFO* info = &topo.Cpus[i];
if (!info->Enabled) continue;
if (info->LapicId == bspLapic) {
bsp->AcpiProcessorId = info->ProcessorUid;
continue;
}
KiPrepareCpuRecord(count, info);
count++;
}
gCpuCount = count;
DbgPrint("SMP: discovered %u CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic);
if (gCpuCount <= 1) {
DbgPrint("SMP: no application processors present\n");
return STATUS_SUCCESS;
}
if ((__readcr3() >> 32) != 0)
return STATUS_NOT_SUPPORTED;
st = KiBuildApTrampoline();
if (!KSUCCESS(st)) return st;
uint8_t sipiVector = (uint8_t)(gApTrampolinePhys >> 12);
uint32_t claimed = 0;
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present) continue;
st = KiAllocateApStack(cpu);
if (!KSUCCESS(st)) return st;
KiFillBootBlockForCpu(cpu, i);
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
st = HalStartApplicationProcessor(cpu->LapicId, sipiVector);
if (!KSUCCESS(st)) return st;
uint64_t lastHeartbeat = gApBootBlock->Heartbeat;
int parked = 0;
for (uint32_t spin = 0; spin < AP_START_TIMEOUT_SPINS; spin++) {
if (gApBootBlock->State == AP_STATE_KERNEL_PARKED) {
parked = 1;
break;
}
if ((spin & 0xFFFFu) == 0) {
uint64_t hb = gApBootBlock->Heartbeat;
lastHeartbeat = hb;
}
_mm_pause();
}
if (!parked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x heartbeat=%llu last=%llu\n",
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
gApBootBlock->Heartbeat, lastHeartbeat);
}
cpu->Started = 1;
cpu->Parked = 1;
cpu->Online = 0;
claimed++;
DbgPrint("SMP: AP%u reached kernel park\n", i);
}
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
return STATUS_SUCCESS;
}
KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
if (gCpuCount <= 1) {
DbgPrint("SMP: no AP scheduler admission needed\n");
return STATUS_SUCCESS;
}
uint32_t admitted = 0;
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present) continue;
if (!cpu->Started || !cpu->Parked || cpu->Online) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u cannot enter scheduler P/S/Park/O=%u/%u/%u/%u\n",
i, cpu->Present, cpu->Started, cpu->Parked, cpu->Online);
}
KSTATUS st = KiCreateIdleThreadForCpu(cpu);
if (!KSUCCESS(st))
return st;
cpu->Online = 1;
_ReadWriteBarrier();
_InterlockedExchange(&cpu->SchedulerCommand, 1);
int active = 0;
for (uint32_t spin = 0; spin < AP_SCHED_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&cpu->SchedulerActive, 0, 0) != 0) {
active = 1;
break;
}
_mm_pause();
}
if (!active) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u failed scheduler admission lapic=%u parked=%u online=%u command=%ld active=%ld ticks=%llu\n",
i, cpu->LapicId, cpu->Parked, cpu->Online,
cpu->SchedulerCommand, cpu->SchedulerActive, cpu->SchedulerTicks);
}
admitted++;
DbgPrint("SMP: AP%u entered scheduler idle thread\n", i);
}
DbgPrint("SMP: scheduler admitted %u APs\n", admitted);
return STATUS_SUCCESS;
}
#define SMP_AP_WORKER_ROUNDS 64u
#define SMP_AP_WORKER_TIMEOUT_SPINS 4000000u
#define SMP_AP_TEST_MAGIC 0x524B575041504D53ULL /* 'SMPAPWKR' */
typedef struct SMP_AP_WORK_TEST SMP_AP_WORK_TEST;
typedef struct SMP_AP_WORKER_ARG {
uint64_t Magic;
uint32_t CpuIndex;
uint32_t WorkerIndex;
struct SMP_AP_WORK_TEST* Test;
EVENT* StepEvent;
volatile long Ready;
volatile long WaitArmed;
volatile long Progress;
volatile long Done;
volatile long Failure;
uint32_t WrongCpu;
uint32_t WrongRunningCpu;
} SMP_AP_WORKER_ARG;
struct SMP_AP_WORK_TEST {
uint64_t Magic;
uint32_t WorkerCount;
volatile long ReadyCount;
volatile long DoneCount;
volatile long FailureCount;
volatile long CpuHits[KE_MAX_CPUS];
SMP_AP_WORKER_ARG Workers[KE_MAX_CPUS];
};
static void KiSmpApControlledWorker(void* context) {
SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context;
if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%llX\n", (uint64_t)arg);
}
_InterlockedExchange(&arg->Ready, 1);
_InterlockedIncrement(&arg->Test->ReadyCount);
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
_InterlockedIncrement(&arg->WaitArmed);
KSTATUS waitStatus = KeWaitForEvent(arg->StepEvent, 0);
if (!KSUCCESS(waitStatus)) {
_InterlockedExchange(&arg->Failure, 1);
break;
}
KCPU* cpu = KeGetCurrentCpu();
THREAD* self = KeGetCurrentThread();
if (!cpu || cpu->CpuIndex != arg->CpuIndex) {
arg->WrongCpu = cpu ? cpu->CpuIndex : KE_INVALID_CPU;
_InterlockedExchange(&arg->Failure, 1);
break;
}
if (!self || self->RunningCpu != cpu->CpuIndex) {
arg->WrongRunningCpu = self ? self->RunningCpu : KE_INVALID_CPU;
_InterlockedExchange(&arg->Failure, 1);
break;
}
if (self && (self->AffinityMask & (1ULL << cpu->CpuIndex)) == 0) {
arg->WrongCpu = cpu->CpuIndex;
_InterlockedExchange(&arg->Failure, 1);
break;
}
if (cpu->CpuIndex < KE_MAX_CPUS)
_InterlockedIncrement(&arg->Test->CpuHits[cpu->CpuIndex]);
_InterlockedIncrement(&arg->Progress);
if ((round & 3u) == 0)
KeYield();
}
if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) {
_InterlockedIncrement(&arg->Test->FailureCount);
}
_InterlockedExchange(&arg->Done, 1);
_InterlockedIncrement(&arg->Test->DoneCount);
}
static int KiWaitForWorkerCounter(volatile long* counter, long expected, const char* what) {
for (uint32_t spin = 0; spin < SMP_AP_WORKER_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(counter, 0, 0) >= expected)
return 1;
if ((spin & 0x3FFu) == 0)
KeYield();
else
_mm_pause();
}
DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%ld got=%ld\n",
what ? what : "counter", expected, _InterlockedCompareExchange(counter, 0, 0));
return 0;
}
KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
if (gCpuCount <= 1) {
DbgPrint("SMP scheduler selftest: SKIP, no APs\n");
return STATUS_SUCCESS;
}
DbgPrint("SMP scheduler selftest: begin AP-affine workers\n");
SMP_AP_WORK_TEST* test = (SMP_AP_WORK_TEST*)MmAllocatePool(sizeof(SMP_AP_WORK_TEST));
if (!test) return STATUS_NO_MEMORY;
memset(test, 0, sizeof(*test));
test->Magic = SMP_AP_TEST_MAGIC;
uint32_t workerCount = 0;
for (uint32_t cpuIndex = 1; cpuIndex < gCpuCount; cpuIndex++) {
KCPU* cpu = &gCpus[cpuIndex];
if (!cpu->Present || !cpu->Started || !cpu->Online || cpu->Parked)
continue;
if (cpuIndex >= 64)
continue;
SMP_AP_WORKER_ARG* arg = &test->Workers[workerCount];
arg->Magic = SMP_AP_TEST_MAGIC;
arg->CpuIndex = cpuIndex;
arg->WorkerIndex = workerCount;
arg->Test = test;
KSTATUS st = KeEventCreate(EventAutoReset, 0, &arg->StepEvent);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u step event create failed status=0x%x\n",
cpuIndex, st);
}
THREAD* thread = (THREAD*)0;
st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker,
(uintptr_t)arg,
PriorityNormal,
(1ULL << cpuIndex),
&thread);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u worker create failed status=0x%x\n",
cpuIndex, st);
}
workerCount++;
}
test->WorkerCount = workerCount;
if (workerCount == 0) {
MmFreePool(test);
DbgPrint("SMP scheduler selftest: SKIP, no online AP workers\n");
return STATUS_SUCCESS;
}
if (!KiWaitForWorkerCounter(&test->ReadyCount, (long)workerCount, "AP worker ready")) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers failed to become ready ready=%ld workers=%u\n",
test->ReadyCount, workerCount);
}
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
for (uint32_t w = 0; w < workerCount; w++) {
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
if (!KiWaitForWorkerCounter(&arg->WaitArmed, (long)(round + 1), "AP worker wait-armed")) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u did not arm wait for round %u armed=%ld progress=%ld\n",
arg->CpuIndex, round, arg->WaitArmed, arg->Progress);
}
KeSignalEvent(arg->StepEvent);
}
for (uint32_t w = 0; w < workerCount; w++) {
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
if (!KiWaitForWorkerCounter(&arg->Progress, (long)(round + 1), "AP worker progress")) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u no progress round=%u progress=%ld failure=%ld wrongCpu=%u wrongRunningCpu=%u\n",
arg->CpuIndex, round, arg->Progress, arg->Failure,
arg->WrongCpu, arg->WrongRunningCpu);
}
if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u worker failed round=%u wrongCpu=%u wrongRunningCpu=%u progress=%ld\n",
arg->CpuIndex, round, arg->WrongCpu, arg->WrongRunningCpu, arg->Progress);
}
}
}
if (!KiWaitForWorkerCounter(&test->DoneCount, (long)workerCount, "AP worker done")) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%ld workers=%u\n",
test->DoneCount, workerCount);
}
if (_InterlockedCompareExchange(&test->FailureCount, 0, 0) != 0) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%ld\n",
test->FailureCount);
}
for (uint32_t w = 0; w < workerCount; w++) {
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
long hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0;
if (hits <= 0) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u recorded no CPU hits\n", arg->CpuIndex);
}
if (arg->StepEvent)
ObDereferenceObject(&arg->StepEvent->Header);
}
DbgPrint("SMP scheduler selftest: PASS workers=%u rounds=%u\n",
workerCount, SMP_AP_WORKER_ROUNDS);
MmFreePool(test);
return STATUS_SUCCESS;
}
code/kernel/ke_switch.asm
+58 -10
View File
@@ -14,7 +14,7 @@ THREAD_Teb EQU 112
THREAD_FpuContext EQU 168
TSS_RSP0_OFFSET EQU 4
EXTERN gTss:QWORD ; phase-1 pointer to current CPU TSS, exported from ke_idt.c
EXTERN KeSetCurrentTssRsp0 : PROC
.CODE
@@ -37,12 +37,24 @@ KeSwitchContext PROC
; from->SavedContext = rsp
mov [rcx + THREAD_SavedContext], rsp
; Update this CPU's TSS.RSP0 before switching to the incoming stack.
; The helper hides whether current-CPU storage is still CPU0-only, LAPIC-ID
; indexed, or eventually reached through a SWAPGS-backed per-CPU base.
mov rbx, rdx
mov rcx, [rbx + THREAD_TssRsp0]
sub rsp, 40
call KeSetCurrentTssRsp0
add rsp, 40
mov rdx, rbx
; rsp = to->SavedContext
mov rsp, [rdx + THREAD_SavedContext]
; Update MSR_GS_BASE (0xC0000101) with to->Teb
mov rax, [rdx + THREAD_Teb] ; Fetch the incoming TEB pointer
mov ecx, 0C0000101h ; IA32_GS_BASE MSR
; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the
; incoming thread's user TEB in IA32_KERNEL_GS_BASE; user-mode entry/exit
; swaps it with the kernel GS base using swapgs.
mov rax, [rdx + THREAD_Teb] ; Fetch the incoming user GS/TEB pointer
mov ecx, 0C0000102h ; IA32_KERNEL_GS_BASE MSR
mov r8, rdx ; Save RDX ('to' thread) as WRMSR clobbers EDX
mov rdx, rax ; Copy TEB pointer to RDX
shr rdx, 32 ; EDX = High 32 bits of TEB
@@ -50,12 +62,6 @@ KeSwitchContext PROC
wrmsr ; Write to MSR
mov rdx, r8 ; Restore RDX
; Update TSS.RSP0 so that the next interrupt/syscall uses the
; incoming thread's kernel stack top.
mov rax, [rdx + THREAD_TssRsp0]
mov r8, gTss
mov qword ptr [r8 + TSS_RSP0_OFFSET], rax
mov rax, [rdx + THREAD_FpuContext]
fxrstor64 [rax]
@@ -69,6 +75,47 @@ KeSwitchContext PROC
ret
KeSwitchContext ENDP
; -------------------------------------------------------------------------
; void KeStartInitialThread(THREAD* to)
; RCX = to
;
; Used by AP scheduler admission. The AP is already in kernel long mode on a
; AP bootstrap stack; this helper installs the idle thread's stack/TSS
; state and returns directly into that thread's saved context. There is no
; outgoing THREAD and no scheduler lock to release.
; -------------------------------------------------------------------------
KeStartInitialThread PROC
mov rbx, rcx
; Update this CPU's TSS.RSP0 for the incoming thread.
mov rcx, [rbx + THREAD_TssRsp0]
sub rsp, 40
call KeSetCurrentTssRsp0
add rsp, 40
; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel
; threads have a NULL TEB, which is fine.
mov rax, [rbx + THREAD_Teb]
mov ecx, 0C0000102h
mov rdx, rax
shr rdx, 32
wrmsr
; Load the thread's saved kernel context and jump into it.
mov rsp, [rbx + THREAD_SavedContext]
mov rax, [rbx + THREAD_FpuContext]
fxrstor64 [rax]
pop r15
pop r14
pop r13
pop r12
pop rbp
pop rbx
ret
KeStartInitialThread ENDP
; -------------------------------------------------------------------------
; KeThreadEntry - landing pad for brand-new threads.
;
@@ -93,6 +140,7 @@ KeThreadEntry PROC
call KeSchedulerPostSwitch
add rsp, 32
pop rcx ; load user arg
swapgs ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU
iretq ; -> ring 3
KeThreadEntry ENDP
code/kernel/ke_syscall.asm
+63 -4
View File
@@ -2,7 +2,7 @@
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
; slots, which is the foundation for future user APC return redirection.
; slots used by user APC return redirection.
EXTERN KeSyscallDispatch:PROC
EXTERN KiEnterTrapFrame:PROC
@@ -20,7 +20,6 @@ TF_Magic EQU 0
TF_Kind EQU 8
TF_Vector EQU 12
TF_PreviousMode EQU 16
TF_Reserved EQU 20
TF_Rax EQU 24
TF_Rcx EQU 32
TF_Rdx EQU 40
@@ -50,6 +49,7 @@ TF_SsSlot EQU 216
.CODE
KeSyscallEntry PROC
swapgs ; GS_BASE: user TEB -> KCPU, KERNEL_GS_BASE: KCPU -> user TEB
; CPU frame at entry, ring 3 -> ring 0:
; [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS,
; [rsp+18] user RSP, [rsp+20] SS
@@ -65,7 +65,6 @@ KeSyscallEntry PROC
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
mov dword ptr [rbx + TF_Vector], 40h
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
mov dword ptr [rbx + TF_Reserved], 0
mov [rbx + TF_Rcx], rcx
mov [rbx + TF_Rdx], rdx
@@ -140,6 +139,7 @@ KeSyscallEntry PROC
mov rax, [rbx + TF_Rax]
add rsp, KTRAP_FRAME_SIZE
pop rbx
swapgs ; GS_BASE: KCPU -> user TEB before returning to ring 3
iretq
KeSyscallEntry ENDP
@@ -235,6 +235,7 @@ KeFaultDE PROC
mov [rax + KF_Rsp], r10
mov r10, [rdx+32]
mov [rax + KF_Ss], r10
swapgs
jmp de_call
de_kernel_frame:
@@ -299,6 +300,7 @@ KeFaultGP PROC
mov [rax + KF_Rsp], r10
mov r10, [rdx+40]
mov [rax + KF_Ss], r10
swapgs
jmp gp_call
gp_kernel_frame:
@@ -364,6 +366,7 @@ KeFaultPF PROC
mov [rax + KF_Rsp], r10
mov r10, [rdx+40]
mov [rax + KF_Ss], r10
swapgs
jmp pf_call
pf_kernel_frame:
@@ -431,6 +434,7 @@ StubName PROC
mov [rax + KF_Rsp], r10
mov r10, [rdx+32]
mov [rax + KF_Ss], r10
swapgs
jmp noerr_call
noerr_kernel_frame:
@@ -494,6 +498,7 @@ StubName PROC
mov [rax + KF_Rsp], r10
mov r10, [rdx+40]
mov [rax + KF_Ss], r10
swapgs
jmp err_call
err_kernel_frame:
@@ -508,6 +513,48 @@ err_call:
StubName ENDP
ENDM
EXTERN KeHandleNmiIpi:PROC
PUBLIC KeNmiIpiStub
KeNmiIpiStub PROC
; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
; the NMI was a panic/TLB-control event and is fully handled. If it
; returns 0, restore the interrupted context and fall through to the normal
; #NMI fault stub so unexpected NMIs still panic with a useful frame.
push rax
push rcx
push rdx
push r8
push r9
push r10
push r11
sub rsp, 32
call KeHandleNmiIpi
add rsp, 32
test eax, eax
jz short NmiUnhandled
pop r11
pop r10
pop r9
pop r8
pop rdx
pop rcx
pop rax
iretq
NmiUnhandled:
pop r11
pop r10
pop r9
pop r8
pop rdx
pop rcx
pop rax
jmp KeFaultNMI
KeNmiIpiStub ENDP
FAULT_NOERR KeFaultDB, 1
FAULT_NOERR KeFaultNMI, 2
FAULT_NOERR KeFaultBP, 3
@@ -576,6 +623,13 @@ IrqStub 21
IrqStub 22
IrqStub 23
PUBLIC KeRescheduleIpiStub
KeRescheduleIpiStub PROC
push rax
mov al, 0F0h
jmp KeIrqCommon
KeRescheduleIpiStub ENDP
KeIrqCommon PROC
; al = irq number, original rax saved by the per-IRQ stub.
push rcx
@@ -594,7 +648,6 @@ KeIrqCommon PROC
mov [rbx + TF_Magic], rax
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT
mov [rbx + TF_Vector], edx
mov dword ptr [rbx + TF_Reserved], 0
; Saved volatile registers live immediately above the trap frame.
mov rax, [rbx + KTRAP_FRAME_SIZE + 56]
@@ -651,6 +704,7 @@ KeIrqCommon PROC
mov [rbx + TF_SsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Ss], rdx
swapgs ; interrupted ring 3: install kernel GS/KCPU
jmp short IrqFrameReady
IrqKernelModeFrame:
@@ -672,6 +726,11 @@ IrqFrameReady:
call KiExitTrapFrame
add rsp, 40
mov eax, [rbx + TF_PreviousMode]
cmp eax, KTRAP_MODE_USER
jne short IrqNoSwapGsOut
swapgs ; returning to ring 3: restore user GS/TEB
IrqNoSwapGsOut:
add rsp, KTRAP_FRAME_SIZE
pop rbx
pop r11
code/kernel/ke_syscall.c
+1 -7
View File
@@ -69,7 +69,6 @@ typedef struct SYS_ASYNC_FILE_REQUEST_K {
uint32_t FileHandle;
uint32_t EventHandle;
uint32_t Flags;
uint32_t Reserved;
uint64_t Offset;
void* Buffer;
size_t Size;
@@ -805,7 +804,7 @@ static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) {
KSTATUS s = CopyFromUser(&req, (const void*)(uintptr_t)a0, sizeof(req));
if (!KSUCCESS(s)) return s;
if (req.Flags || req.Reserved) return STATUS_INVALID_PARAMETER;
if (req.Flags) return STATUS_INVALID_PARAMETER;
if (!req.Iosb) return STATUS_INVALID_PARAMETER;
if (!req.Buffer && req.Size) return STATUS_INVALID_PARAMETER;
if (req.Size > SYSCALL_IO_MAX_BYTES) return STATUS_INVALID_PARAMETER;
@@ -1989,7 +1988,6 @@ typedef struct SYS_SOCKET_SEND_TO_REQUEST_K {
const void* Buffer;
size_t Length;
uint32_t Flags;
uint32_t Reserved;
const SYS_SOCKET_ADDRESS_IPV4_K* RemoteAddress;
size_t RemoteAddressSize;
size_t* SentOut;
@@ -1999,7 +1997,6 @@ typedef struct SYS_SOCKET_RECV_FROM_REQUEST_K {
void* Buffer;
size_t Length;
uint32_t Flags;
uint32_t Reserved;
SYS_SOCKET_ADDRESS_IPV4_K* RemoteAddressOut;
size_t* RemoteAddressSizeInOut;
size_t* ReceivedOut;
@@ -2015,7 +2012,6 @@ typedef struct SYS_RESOLVE_HOST_REQUEST_K {
typedef struct SYS_SOCKET_OPTION_REQUEST_K {
uint32_t Option;
uint32_t Reserved;
void* Value;
size_t ValueSize;
size_t* ActualSizeOut;
@@ -2269,7 +2265,6 @@ static uint64_t SyscallSendTo(uint64_t a0, uint64_t a1, uint64_t a2,
SYS_SOCKET_SEND_TO_REQUEST_K req;
KSTATUS s = CopyFromUser(&req, (const void*)(uintptr_t)a1, sizeof(req));
if (!KSUCCESS(s)) return s;
if (req.Reserved != 0) return STATUS_INVALID_PARAMETER;
if (req.Length && !req.Buffer) return STATUS_INVALID_PARAMETER;
if (!req.RemoteAddress) return STATUS_INVALID_PARAMETER;
if (req.RemoteAddressSize < sizeof(SYS_SOCKET_ADDRESS_IPV4_K)) return STATUS_BUFFER_TOO_SMALL;
@@ -2313,7 +2308,6 @@ static uint64_t SyscallRecvFrom(uint64_t a0, uint64_t a1, uint64_t a2,
SYS_SOCKET_RECV_FROM_REQUEST_K req;
KSTATUS s = CopyFromUser(&req, (const void*)(uintptr_t)a1, sizeof(req));
if (!KSUCCESS(s)) return s;
if (req.Reserved != 0) return STATUS_INVALID_PARAMETER;
if (req.Length && !req.Buffer) return STATUS_INVALID_PARAMETER;
if (req.Length > SYSCALL_IO_MAX_BYTES) return STATUS_INVALID_PARAMETER;
code/kernel/ke_thread.c
+36 -40
View File
@@ -8,6 +8,7 @@
#include <kernel/ke_irql.h>
#include <kernel/ke_event.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_userenv.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
@@ -16,7 +17,7 @@
#define FPU_MXCSR_DEFAULT 0x1F80u
// These offsets are consumed directly by ke_switch.asm. Keep the asserts
// here so future THREAD layout edits fail at build time instead of corrupting
// here so THREAD layout edits fail at build time instead of corrupting
// context switches at runtime.
#define STATIC_ASSERT_OFFSET(type, field, expected) \
_Static_assert(offsetof(type, field) == (expected), \
@@ -93,11 +94,18 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->CpuBurstTicks = 0;
thread->LastRunTicks = 0;
thread->RunningCpu = KE_INVALID_CPU;
thread->Queued = 0;
thread->IsIdleThread = 0;
thread->SchedulerFlags = 0;
thread->LastCpu = KE_INVALID_CPU;
thread->AffinityMask = KeGetDefaultThreadAffinityMask();
LIST_INIT(&thread->ProcessLink);
LIST_INIT(&thread->SchedulerLink);
thread->WaitObject = (OBJECT *)0;
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SchedLockFlags = 0x202ULL;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
@@ -226,8 +234,10 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
// ---------------------------------------------------------------------------
extern void KeKernelThreadEntry(void);
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out) {
static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority,
uint64_t affinityMask,
THREAD** out) {
// Kernel worker threads are owned by the system process. The scheduler's
// hand-built CPU idle thread stays attached to PID 0 (idle), while driver,
// async I/O, and init worker threads are grouped under PID 1 (system).
@@ -252,11 +262,18 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
thread->CpuBurstTicks = 0;
thread->LastRunTicks = 0;
thread->RunningCpu = KE_INVALID_CPU;
thread->Queued = 0;
thread->IsIdleThread = 0;
thread->SchedulerFlags = 0;
thread->LastCpu = KE_INVALID_CPU;
thread->AffinityMask = affinityMask ? affinityMask : KeGetDefaultThreadAffinityMask();
LIST_INIT(&thread->ProcessLink);
LIST_INIT(&thread->SchedulerLink);
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SchedLockFlags = 0x202ULL;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
thread->UserStackBase = (void*)0; // no user stack
@@ -300,14 +317,26 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out) {
return KiCreateKernelThreadWithAffinity(fn, arg, priority,
KeGetDefaultThreadAffinityMask(), out);
}
__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, uint64_t affinityMask,
THREAD** out) {
return KiCreateKernelThreadWithAffinity(fn, arg, priority, affinityMask, out);
}
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
// Called with the synchronization object's spinlock held, so the caller is
// normally at DISPATCH_LEVEL here. This function only marks intent to
// block; the actual scheduling point is KeCommitBlockThread at PASSIVE.
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
KiSchedulerMarkThreadWaiting(thread);
thread->State = ThreadWaiting;
thread->WaitObject = waitObject;
thread->WaitStatus = STATUS_PENDING;
thread->WaitListHead = (LIST_ENTRY*)0;
@@ -375,40 +404,7 @@ KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
// If the selected waiter is still the currently executing thread, the
// signal landed in the prepare->commit window. Complete the wait in-place
// instead of inserting the running thread into the ready queue.
if (thread == KeGetCurrentThread()) {
KiFinishWait(thread, status);
thread->CpuBurstTicks = 0;
thread->QuantaConsumed = 0;
thread->QuantumRemaining = 0;
if (thread->State == ThreadWaiting)
thread->State = ThreadRunning;
return;
}
// The same object can be signaled after a timeout/cancel path exists, or a
// buggy caller can try to wake a non-waiting thread. Do not enqueue such a
// thread twice.
if (thread->State != ThreadWaiting) {
thread->WaitStatus = status;
return;
}
KiFinishWait(thread, status);
thread->State = ThreadReady;
thread->CpuBurstTicks = 0;
thread->QuantaConsumed = 0;
thread->QuantumRemaining = 0;
if (thread->Priority < PriorityHigh) {
thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1);
}
KeReadyThread(thread);
KiSchedulerWakeThread(thread, status);
}
void KeUnblockThread(THREAD* thread) {
code/kernel/ke_trap.c
-1
View File
@@ -41,7 +41,6 @@ STATIC_ASSERT_OFFSET(KTRAP_FRAME, RflagsSlot, 200);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, RspSlot, 208);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, SsSlot, 216);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousFrame, 224);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Reserved2, 232);
_Static_assert(sizeof(KTRAP_FRAME) == 240, "KTRAP_FRAME size must match ke_syscall.asm");
int KeTrapFrameFromUser(const KTRAP_FRAME* frame) {
code/kernel/kmain.c
+22 -2
View File
@@ -21,6 +21,7 @@
#include <ntlite/ddi/hal_api.h>
#include <kernel/io_file.h>
#include <kernel/ke_selftest.h>
#include <kernel/ke_smp.h>
#include <common/ini_parser.h>
typedef struct {
@@ -380,6 +381,8 @@ void SystemInitThread(void* arg) {
DbgPrint("SystemInitThread: Thread started. Igniting the system...\n");
KiRunEarlyKernelSelfTests();
KiInitializeDrivers(args->bootInfo, args->hal);
KiResolveSystemImports(args->bootInfo);
@@ -436,7 +439,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
}
PHYSICAL_ADDRESS pml4 = (PHYSICAL_ADDRESS)__readcr3();
MmInitializeVirtual(pml4, 4 * 1024 * 1024 /* 4 MB initial pool */);
MmInitializeVirtual(pml4, 1 * 1024 * 1024 /* 1 MB initial pool */);
//
// Bootstrap CPU and architecture setup (GDT/IDT/Faults)
@@ -472,7 +475,17 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
const HAL_API* hal = HalGetApi();
gKmainHal = hal;
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1); // Enable debug
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1);
// CPU startup intentionally lives after HAL initialization but before IRQs
// and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI;
// the kernel owns KCPU records, per-CPU stacks, GDT/TSS/IDT state, and the
// AP park loop. External device IRQs remain routed to the BSP during this
// step; APs only become runnable after scheduler admission below.
KSTATUS smpStatus = KeStartApplicationProcessors();
if (!KSUCCESS(smpStatus)) {
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to start application processors, status=0x%x\n", smpStatus);
}
KeIrqInitialize(hal);
KeInitializeDpcSystem();
@@ -501,6 +514,13 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
//
KeInitializeScheduler();
KSTATUS smpSchedStatus = KeAdmitApplicationProcessorsToScheduler();
if (!KSUCCESS(smpSchedStatus)) {
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to admit APs to scheduler, status=0x%x\n", smpSchedStatus);
}
KiRunCoreHardeningSelfTests();
// Pack the arguments for the init thread
SYSTEM_INIT_ARGS* initArgs = MmAllocatePool(sizeof(SYSTEM_INIT_ARGS));
initArgs->bootInfo = bootInfo;
code/kernel/ldr_image.c
-24
View File
@@ -454,28 +454,6 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
);
}
static void KiDebugLibtccAllocatorSlot(const char* phase, const char* filename, uintptr_t imageBase) {
if (!filename || !imageBase) return;
if (_stricmp(filename, "libtcc.dll") != 0) return;
uint64_t slot = *(uint64_t*)(imageBase + 0x4A010);
uint64_t expectedDefault = (uint64_t)(imageBase + 0x25F0);
DbgPrint("LDR libtcc %s: base=%p slot[+4A010]=%p expected-default=%p\n",
phase,
(void*)imageBase,
(void*)(uintptr_t)slot,
(void*)(uintptr_t)expectedDefault);
DbgPrint("LDR libtcc data[+4A000..+4A028]: %016llx %016llx %016llx %016llx %016llx %016llx\n",
(unsigned long long)*(uint64_t*)(imageBase + 0x4A000),
(unsigned long long)*(uint64_t*)(imageBase + 0x4A008),
(unsigned long long)*(uint64_t*)(imageBase + 0x4A010),
(unsigned long long)*(uint64_t*)(imageBase + 0x4A018),
(unsigned long long)*(uint64_t*)(imageBase + 0x4A020),
(unsigned long long)*(uint64_t*)(imageBase + 0x4A028));
}
static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) {
if (!process || !name || !base) return STATUS_INVALID_PARAMETER;
@@ -543,7 +521,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
return s;
}
KiDebugLibtccAllocatorSlot("after load/reloc before imports", filename, imageBase);
s = LdrResolveImportsForProcess(process, imageBase);
if (!KSUCCESS(s)) {
@@ -557,7 +534,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
return s;
}
KiDebugLibtccAllocatorSlot("after imports", filename, imageBase);
s = LdrRegisterProcessModule(process, filename, imageBase);
if (!KSUCCESS(s)) {
code/kernel/mm_phys.c
+20
View File
@@ -101,6 +101,26 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) {
if (maxExclusive <= PAGE_SIZE) return (PHYSICAL_ADDRESS)0;
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
for (size_t i = gFrameStack.Top; i > 0; i--) {
size_t idx = i - 1;
PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx];
if (frame && frame < maxExclusive && !(frame & (PAGE_SIZE - 1))) {
size_t tail = gFrameStack.Top - idx - 1;
for (size_t j = 0; j < tail; j++)
gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j];
gFrameStack.Top--;
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return frame;
}
}
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return (PHYSICAL_ADDRESS)0;
}
PHYSICAL_ADDRESS MmAllocateFrame(void) {
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
if (gFrameStack.Top == 0) {
code/kernel/mm_virt.c
+197 -30
View File
@@ -14,9 +14,13 @@
// Backed on demand from the PMM. No per-process heaps.
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <kernel/hal/hal_apic.h>
// ---------------------------------------------------------------------------
// Helpers - physical window access
@@ -34,6 +38,14 @@ static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
// PML4 / page-table helpers
// ---------------------------------------------------------------------------
static PHYSICAL_ADDRESS gPml4Phys = 0;
static SPINLOCK gPageTableLock = SPINLOCK_INIT;
static volatile long gTlbShootdownLock = 0;
static volatile long gTlbShootdownActive = 0;
static volatile long gTlbShootdownAcks = 0;
static uintptr_t gTlbShootdownVirt = 0;
#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u
// Extract the page-table index for each level from a virtual address.
#define PML4_IDX(va) (((va) >> 39) & 0x1FF)
@@ -64,38 +76,150 @@ static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
return childPhys;
}
// ---------------------------------------------------------------------------
// TLB/cache coherency hooks
// ---------------------------------------------------------------------------
void MmFlushTlbLocal(uintptr_t virt) {
__invlpg((void*)virt);
}
static void KiAcquireTlbShootdownLock(void) {
// Do not use the normal spinlock here: it masks interrupts while spinning.
// A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
// concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is
// meant to avoid.
while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0)
_mm_pause();
_ReadWriteBarrier();
}
static void KiReleaseTlbShootdownLock(void) {
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownLock, 0);
}
void MmFlushTlbForKernelMapping(uintptr_t virt) {
MmFlushTlbLocal(virt);
if (gCpuCount <= 1)
return;
KCPU* self = KeGetCurrentCpu();
uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU;
KiAcquireTlbShootdownLock();
long targets = 0;
gTlbShootdownVirt = virt;
_InterlockedExchange(&gTlbShootdownAcks, 0);
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownActive, 1);
_ReadWriteBarrier();
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (i == selfIndex)
continue;
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
targets++;
HalApicSendTlbShootdownIpi(cpu->LapicId);
}
for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets)
break;
_mm_pause();
}
long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0);
_InterlockedExchange(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
if (acks < targets) {
KePanic(KE_PANIC_CODE_GENERIC,
"TLB shootdown timeout: va=0x%llX targets=%ld acks=%ld selfCpu=%u\n",
(uint64_t)virt, targets, acks, selfIndex);
}
}
static int KiHandleTlbShootdown(void) {
if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
return 0;
uintptr_t va = gTlbShootdownVirt;
_ReadWriteBarrier();
MmFlushTlbLocal(va);
_ReadWriteBarrier();
_InterlockedIncrement(&gTlbShootdownAcks);
return 1;
}
int MmHandleTlbShootdownNmi(void) {
return KiHandleTlbShootdown();
}
// ---------------------------------------------------------------------------
// Public mapping
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags) {
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
if (!pdpt) return STATUS_NO_MEMORY;
if (!pdpt) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
if (!pdt) return STATUS_NO_MEMORY;
if (!pdt) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag);
if (!pt) return STATUS_NO_MEMORY;
if (!pt) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
*pte = phys | flags;
__invlpg((void *)virt);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
MmFlushTlbForKernelMapping(virt);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) {
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e4 & PAGE_PRESENT)) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e3 & PAGE_PRESENT)) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e2 & PAGE_PRESENT)) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt));
*pte = 0;
__invlpg((void *)virt);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
MmFlushTlbForKernelMapping(virt);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags) {
if (!size) return STATUS_SUCCESS;
@@ -103,60 +227,92 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint
uintptr_t base = virt & PAGE_MASK;
uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
KSTATUS status = STATUS_SUCCESS;
uintptr_t flushEnd = base;
for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e4 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e3 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e2 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*pte & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
// Preserve the physical address, replace everything else.
PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK;
*pte = phys | flags;
__invlpg((void *)va);
flushEnd = va + PAGE_SIZE;
}
return STATUS_SUCCESS;
KeReleaseSpinlock(&gPageTableLock, lockFlags);
for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE)
MmFlushTlbForKernelMapping(va);
return status;
}
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
if (!gPml4Phys) return 0; // Not initialized
PHYSICAL_ADDRESS result = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t pml4_idx = (virt >> 39) & 0x1FF;
uint64_t pdpt_idx = (virt >> 30) & 0x1FF;
uint64_t pd_idx = (virt >> 21) & 0x1FF;
uint64_t pt_idx = (virt >> 12) & 0x1FF;
uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
if (!(pml4[pml4_idx] & PAGE_PRESENT)) return 0;
if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out;
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK);
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) return 0;
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out;
// Check for 1GB huge pages (bit 7 - PS)
if (pdpt[pdpt_idx] & 0x80) {
return (pdpt[pdpt_idx] & 0x000FFFFFC0000000ULL) + (virt & 0x3FFFFFFF);
if (pdpt[pdpt_idx] & PAGE_PS) {
result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF);
goto out;
}
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK);
if (!(pd[pd_idx] & PAGE_PRESENT)) return 0;
if (!(pd[pd_idx] & PAGE_PRESENT)) goto out;
// Check for 2MB huge pages (bit 7 - PS)
if (pd[pd_idx] & 0x80) {
return (pd[pd_idx] & 0x000FFFFFFFE00000ULL) + (virt & 0x1FFFFF);
if (pd[pd_idx] & PAGE_PS) {
result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF);
goto out;
}
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK);
if (!(pt[pt_idx] & PAGE_PRESENT)) return 0;
if (!(pt[pt_idx] & PAGE_PRESENT)) goto out;
// Extract the physical frame and add the offset within the page
return (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return result;
}
// ---------------------------------------------------------------------------
// Page query and user/kernel copy boundary helpers
// ---------------------------------------------------------------------------
@@ -165,9 +321,12 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
uint64_t* flagsOut) {
if (!gPml4Phys) return STATUS_NOT_FOUND;
KSTATUS status = STATUS_NOT_FOUND;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys);
uint64_t e4 = pml4[PML4_IDX(virt)];
if (!(e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e4 & PAGE_PRESENT)) goto out;
uint64_t effectiveWrite = e4 & PAGE_WRITE;
uint64_t effectiveUser = e4 & PAGE_USER;
@@ -175,7 +334,7 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK);
uint64_t e3 = pdpt[PDPT_IDX(virt)];
if (!(e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e3 & PAGE_PRESENT)) goto out;
effectiveWrite &= e3 & PAGE_WRITE;
effectiveUser &= e3 & PAGE_USER;
effectiveNx |= e3 & PAGE_NX;
@@ -189,12 +348,13 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
return STATUS_SUCCESS;
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK);
uint64_t e2 = pd[PDT_IDX(virt)];
if (!(e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e2 & PAGE_PRESENT)) goto out;
effectiveWrite &= e2 & PAGE_WRITE;
effectiveUser &= e2 & PAGE_USER;
effectiveNx |= e2 & PAGE_NX;
@@ -208,12 +368,13 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
return STATUS_SUCCESS;
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK);
uint64_t e1 = pt[PT_IDX(virt)];
if (!(e1 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e1 & PAGE_PRESENT)) goto out;
effectiveWrite &= e1 & PAGE_WRITE;
effectiveUser &= e1 & PAGE_USER;
effectiveNx |= e1 & PAGE_NX;
@@ -226,9 +387,14 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
return STATUS_SUCCESS;
status = STATUS_SUCCESS;
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return status;
}
static int KiUserRangeValid(uintptr_t addr, size_t size) {
if (size == 0) return 1;
if (addr < USER_BASE) return 0;
@@ -581,6 +747,7 @@ static uintptr_t gUserImageNext = USER_LIMIT;
// ---------------------------------------------------------------------------
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
gPml4Phys = bootPml4Phys;
gPageTableLock = (SPINLOCK)SPINLOCK_INIT;
// Kernel pool heap.
gPoolHeap.FreeList = (BLOCK_HDR *)0;