maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace the VHD/diskpart disk image pipeline with a directory-based vvfat root, we'll see when it breaks, fix /O2, add an all_tests, generalize the fat32 driver into a fat 16+32 driver, change policy on volume drivers, collapse KCPU's fields into a single atomic KE_CPU_STATE state machine, move scheduler ticking to per-CPU LAPIC timers, gate verbose SMP topology logging behind selftest diagnostics and add SMP pool-checkpoint validation around AP admission, add checked intrusive list helpers behind a new KE_DEBUG_LIST_CHECKS flag, add ObInitializeObjectHeader/ObTryReferenceObject, add waitable display clock objects and a SYS_OPEN_DISPLAY_CLOCK syscall for vsync-style waits, refactor kernel input into a per-queue KE_INPUT_CONTEXT with its own key/mouse ring buffers and DPC-driven key repeat, add KeInputQueuePeekEvent/ReadEvent/WaitKey and a SYS_QUERY_GLOBAL_INPUT_STATE syscall, split the kernel pool allocator out of mm_virt, new video files!

Browse Source
This commit is contained in:
maksw committed 2026-07-12 00:10:21 +02:00
1 parent 485a6bbfe7
commit 8b254f84f4
74 files changed
+4488 -2503
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/ke_smp.c
+107 -91
View File
@@ -6,6 +6,7 @@
// to enter KiApKernelEntry() and park in kernel-owned state.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/status.h>
@@ -80,6 +81,20 @@ static PHYSICAL_ADDRESS gApTrampolinePhys;
static PHYSICAL_ADDRESS gReservedApTrampolinePhys;
static AP_BOOT_BLOCK* gApBootBlock;
// Returns whether investigation-only SMP diagnostics are enabled.
static bool KiSmpDiagnosticsEnabled(void) {
return (KeGetDebugFlags() & KE_DEBUG_SELFTEST) != 0;
}
// Validates pool state around AP admission so SMP races fail near their trigger.
static void KiSmpPoolCheckpoint(const char* phase, uint32_t cpuIndex) {
if (!KiSmpDiagnosticsEnabled())
return;
DbgPrint("SMP: pool checkpoint %s AP%u\n", phase ? phase : "unknown", cpuIndex);
(void)MmValidateKernelPool(phase ? phase : "smp");
}
// Implements the emit8 operation.
static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
// Implements the emit16 operation.
@@ -127,27 +142,35 @@ static KSTATUS KiBuildApTrampoline(void) {
if (frame < AP_TRAMPOLINE_FALLBACK_MIN ||
frame >= AP_TRAMPOLINE_MAX_PHYS ||
(frame & (PAGE_SIZE - 1))) {
DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
}
frame = 0;
} else {
DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
}
}
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
}
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_PREFERRED_MIN,
AP_TRAMPOLINE_MAX_PHYS,
MmPfnUseKernel);
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
}
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_FALLBACK_MIN,
AP_TRAMPOLINE_PREFERRED_MIN,
MmPfnUseKernel);
@@ -158,7 +181,8 @@ static KSTATUS KiBuildApTrampoline(void) {
return NTLSTATUS_NO_MEMORY;
}
DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame);
memset(page, 0, PAGE_SIZE);
@@ -285,13 +309,12 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
KeLoadIdtForCurrentCpu();
if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_IDT;
cpu->Started = 1;
cpu->Parked = 1;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuStarted);
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
_InterlockedExchange(&cpu->State, KeCpuParked);
if (gApBootBlock) {
gApBootBlock->Stage = AP_STAGE_PARK;
gApBootBlock->State = AP_STATE_KERNEL_PARKED;
@@ -300,32 +323,30 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
for (;;) {
cpu->SchedulerTicks++;
if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
THREAD* idle = cpu->IdleThread;
if (idle) {
cpu->CurrentThread = idle;
idle->State = ThreadRunning;
idle->RunningCpu = cpu->CpuIndex;
idle->Queued = 0;
cpu->Parked = 0;
// Do not arm the AP's periodic LAPIC timer while it is still in
// the raw park loop. Timer IRQs are scheduler-owned events;
// the AP may take them only after it has a current idle thread.
HalTimerConfigureCurrentCpu();
_InterlockedExchange(&cpu->SchedulerActive, 1);
_InterlockedExchange(&cpu->State, KeCpuSchedulerActive);
HalEnableInterrupts();
KeStartInitialThread(idle);
}
}
// The BSP admits parked APs by storing SchedulerCommand and sending a
// wake IPI. Avoid the classic lost wakeup where the command appears
// between the poll and HLT: check once with interrupts disabled, then
// STI;HLT as a single sleep transition.
// The BSP admits parked APs by publishing AdmissionRequested and
// sending a wake IPI. Avoid the classic lost wakeup where the state
// changes between the poll and HLT: check once with interrupts
// disabled, then use STI;HLT as a single sleep transition.
uintptr_t interruptFlags = HalDisableInterrupts();
if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
HalRestoreInterrupts(interruptFlags);
continue;
}
@@ -342,10 +363,7 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->LapicId = info->LapicId;
cpu->AcpiProcessorId = info->ProcessorUid;
cpu->BootProcessorIndex = cpuIndex;
cpu->Present = info->Enabled ? 1 : 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
cpu->State = info->Enabled ? KeCpuPresent : KeCpuAbsent;
cpu->IsBsp = info->IsBsp ? 1 : 0;
cpu->PackageId = 0;
cpu->CoreId = cpuIndex;
@@ -359,8 +377,6 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
cpu->SchedulerCommand = 0;
cpu->SchedulerActive = 0;
}
// Implements the ceil Log2 U32 operation.
@@ -496,7 +512,7 @@ void KeBuildProcessorTopology(void) {
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
if (cpu->State == KeCpuAbsent)
continue;
uint32_t apic = cpu->LapicId;
@@ -515,7 +531,7 @@ void KeBuildProcessorTopology(void) {
uint32_t nextCoreIndex = 0;
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
if (cpu->State == KeCpuAbsent)
continue;
if (cpu->PhysicalCoreIndex != KE_INVALID_CPU)
continue;
@@ -528,7 +544,7 @@ void KeBuildProcessorTopology(void) {
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
if (peer->State == KeCpuAbsent)
continue;
if (peer->PackageId == cpu->PackageId) {
if (j < 64)
@@ -549,7 +565,7 @@ void KeBuildProcessorTopology(void) {
for (uint32_t j = 0; j < gCpuCount && j < KE_MAX_CPUS; j++) {
KCPU* peer = &gCpus[j];
if (!peer->Present)
if (peer->State == KeCpuAbsent)
continue;
if (peer->PackageId == cpu->PackageId && peer->CoreId == cpu->CoreId) {
peer->CoreSiblingMask = coreMaskBits;
@@ -568,19 +584,21 @@ void KeBuildProcessorTopology(void) {
topo.LogicalPerPackage,
nextCoreIndex);
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present)
continue;
DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
i,
cpu->LapicId,
cpu->PackageId,
cpu->CoreId,
cpu->ThreadId,
cpu->IsPrimaryThread,
(unsigned long long)cpu->CoreSiblingMask,
(unsigned long long)cpu->PackageMask);
if (KiSmpDiagnosticsEnabled()) {
for (uint32_t i = 0; i < gCpuCount && i < KE_MAX_CPUS; i++) {
KCPU* cpu = &gCpus[i];
if (cpu->State == KeCpuAbsent)
continue;
DbgPrint("SMP: cpu%u lapic=%u pkg=%u core=%u thread=%u primary=%u siblings=0x%llx package=0x%llx\n",
i,
cpu->LapicId,
cpu->PackageId,
cpu->CoreId,
cpu->ThreadId,
cpu->IsPrimaryThread,
(unsigned long long)cpu->CoreSiblingMask,
(unsigned long long)cpu->PackageMask);
}
}
}
@@ -598,7 +616,7 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
}
cpu->KernelStack = stack;
cpu->KernelStackTop = (uintptr_t)stack + stackBytes;
cpu->KernelStackTop = ((uintptr_t)stack + stackBytes) & ~(uintptr_t)15;
memset(cpu->KernelStack, 0, stackBytes);
memset(&cpu->Tss, 0, sizeof(cpu->Tss));
cpu->Tss.IoMapBase = sizeof(TSS64);
@@ -647,12 +665,7 @@ KSTATUS KeStartApplicationProcessors(void) {
KCPU* bsp = KeGetCurrentCpu();
bsp->LapicId = bspLapic;
bsp->IsBsp = 1;
bsp->Present = 1;
bsp->Started = 1;
bsp->Online = 1;
bsp->Parked = 0;
bsp->SchedulerCommand = 0;
bsp->SchedulerActive = 1;
bsp->State = KeCpuStarted;
DbgPrint("SMP: APIC mode=%s BSP LAPIC=%u MADT CPUs=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
@@ -661,9 +674,11 @@ KSTATUS KeStartApplicationProcessors(void) {
uint32_t count = 1;
for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) {
HAL_CPU_INFO* info = &topo.Cpus[i];
DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
i, info->ProcessorUid, info->LapicId, info->Flags,
info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
i, info->ProcessorUid, info->LapicId, info->Flags,
info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
}
if (!info->Enabled) continue;
if (info->LapicId == bspLapic) {
@@ -693,8 +708,10 @@ KSTATUS KeStartApplicationProcessors(void) {
return NTLSTATUS_SUCCESS;
}
DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
(uint64_t)gReservedApTrampolinePhys);
if (KiSmpDiagnosticsEnabled()) {
DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
(uint64_t)gReservedApTrampolinePhys);
}
st = KiBuildApTrampoline();
if (!KSUCCESS(st)) {
DbgPrint("SMP: disabled; AP trampoline allocation/build failed status=0x%x\n", st);
@@ -707,30 +724,25 @@ KSTATUS KeStartApplicationProcessors(void) {
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present) continue;
if (cpu->State == KeCpuAbsent) continue;
st = KiAllocateApStack(cpu);
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
KiFillBootBlockForCpu(cpu, i);
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
st = HalStartApplicationProcessor(cpu->LapicId, sipiVector);
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
@@ -752,18 +764,18 @@ KSTATUS KeStartApplicationProcessors(void) {
DbgPrint("SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
_InterlockedExchange(&cpu->State, KeCpuAbsent);
continue;
}
cpu->Started = 1;
cpu->Parked = 1;
cpu->Online = 0;
if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u boot block parked without CPU state transition state=%ld\n",
i, cpu->State);
}
claimed++;
DbgPrint("SMP: AP%u reached kernel park\n", i);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: AP%u reached kernel park\n", i);
}
if (claimed == 0 && gCpuCount > 1)
@@ -771,6 +783,7 @@ KSTATUS KeStartApplicationProcessors(void) {
KeBuildProcessorTopology();
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
KiSmpPoolCheckpoint("smp after AP park", 0);
return NTLSTATUS_SUCCESS;
}
@@ -784,32 +797,33 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
uint32_t admitted = 0;
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (!cpu->Present) continue;
if (cpu->State == KeCpuAbsent) continue;
if (!cpu->Started || !cpu->Parked || cpu->Online) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u cannot enter scheduler P/S/Park/O=%u/%u/%u/%u\n",
i, cpu->Present, cpu->Started, cpu->Parked, cpu->Online);
"SMP: AP%u cannot enter scheduler from state=%ld\n",
i, cpu->State);
}
KiSmpPoolCheckpoint("smp before AP idle create", i);
KSTATUS st = KiCreateIdleThreadForCpu(cpu);
if (!KSUCCESS(st))
return st;
KiSmpPoolCheckpoint("smp after AP idle create", i);
cpu->Online = 1;
_ReadWriteBarrier();
_InterlockedExchange(&cpu->SchedulerCommand, 1);
_InterlockedExchange(&cpu->State, KeCpuAdmissionRequested);
KiSmpPoolCheckpoint("smp before AP wake", i);
// Parked APs sleep with HLT and have no LAPIC timer armed yet. The
// command store alone is not sufficient; send a fixed-vector IPI only
// to wake the parked loop. ReschedulePending is intentionally left
// state transition alone is not sufficient; send a fixed-vector IPI
// only to wake the parked loop. ReschedulePending is intentionally left
// clear, so the interrupt exit path will not try to schedule before
// the AP has installed its idle thread.
HalApicSendRescheduleIpi(cpu->LapicId);
int active = 0;
for (uint32_t spin = 0; spin < AP_SCHED_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&cpu->SchedulerActive, 0, 0) != 0) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuSchedulerActive) {
active = 1;
break;
}
@@ -818,13 +832,15 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
if (!active) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u failed scheduler admission lapic=%u parked=%u online=%u command=%ld active=%ld ticks=%llu\n",
i, cpu->LapicId, cpu->Parked, cpu->Online,
cpu->SchedulerCommand, cpu->SchedulerActive, cpu->SchedulerTicks);
"SMP: AP%u failed scheduler admission lapic=%u state=%ld ticks=%llu\n",
i, cpu->LapicId, cpu->State, cpu->SchedulerTicks);
}
_InterlockedExchange(&cpu->State, KeCpuOnline);
KiSmpPoolCheckpoint("smp after AP scheduler active", i);
admitted++;
DbgPrint("SMP: AP%u entered scheduler idle thread\n", i);
if (KiSmpDiagnosticsEnabled())
DbgPrint("SMP: AP%u entered scheduler idle thread\n", i);
}
DbgPrint("SMP: scheduler admitted %u APs\n", admitted);
@@ -946,7 +962,7 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
uint32_t workerCount = 0;
for (uint32_t cpuIndex = 1; cpuIndex < gCpuCount; cpuIndex++) {
KCPU* cpu = &gCpus[cpuIndex];
if (!cpu->Present || !cpu->Started || !cpu->Online || cpu->Parked)
if (cpu->State != KeCpuOnline)
continue;
if (cpuIndex >= 64)
continue;