maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

temp

Browse Source
This commit is contained in:
maksw committed 2026-10-27 09:28:31 +01:00
1 parent d36f19509c
commit 30d20457f0
309 files changed
+121746 -1963
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/hal/hal_apic.c
+160 -103
View File
@@ -4,6 +4,8 @@
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_timer.h>
#include <kernel/hal/hal_processor.h>
#include <kernel/ke_vectors.h>
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>
@@ -14,6 +16,9 @@ static volatile uint32_t* gLapic = (volatile uint32_t*)0;
#define LAPIC_REG(off) (*(volatile uint32_t*)((uint8_t*)gLapic + (off)))
#define LAPIC_ID 0x020
#define LAPIC_EOI 0x0B0
#define LAPIC_TPR 0x080
#define LAPIC_ISR_BASE 0x100
#define LAPIC_IRR_BASE 0x200
#define LAPIC_SPURIOUS 0x0F0
#define LAPIC_ICR_LO 0x300
#define LAPIC_ICR_HI 0x310
@@ -24,35 +29,68 @@ static volatile uint32_t* gLapic = (volatile uint32_t*)0;
#define IA32_APIC_BASE_X2APIC (1ULL << 10)
#define IA32_X2APIC_ID 0x00000802u
#define IA32_X2APIC_EOI 0x0000080Bu
#define IA32_X2APIC_TPR 0x00000808u
#define IA32_X2APIC_ISR_BASE 0x00000810u
#define IA32_X2APIC_IRR_BASE 0x00000820u
#define IA32_X2APIC_SVR 0x0000080Fu
#define IA32_X2APIC_ICR 0x00000830u
#define CPUID_FEATURE_ECX_X2APIC (1u << 21)
// ---- I/O APIC ----.
static volatile uint32_t* gIoapic = (volatile uint32_t*)0;
static uint32_t gBspLapicId = 0;
static uint8_t gX2ApicSupported = 0;
static uint8_t gX2ApicEnabled = 0;
static uint32_t gIoapicGsiBase = 0;
static uint32_t gIoapicRedirectionCount = 0;
// ---- I/O APIC interrupt domains ----.
#define HAL_IOAPIC_MAX 8u
#define IOAPIC_IDX 0x00u
#define IOAPIC_DATA 0x10u
typedef struct HAL_IOAPIC {
volatile uint32_t* Registers;
uintptr_t Physical;
uint32_t GsiBase;
uint32_t Pins;
volatile long RegisterLock;
} HAL_IOAPIC;
static HAL_IOAPIC gIoapics[HAL_IOAPIC_MAX];
static uint32_t gIoapicCount;
static uint32_t gBspLapicId;
static uint8_t gX2ApicSupported;
static uint8_t gX2ApicEnabled;
static uint32_t gIsaGsi[16];
static uint16_t gIsaFlags[16];
static uint16_t gSciIrq = UINT16_MAX;
static bool gIsaOverridesInitialized = false;
#define IOAPIC_IDX 0x00
#define IOAPIC_DATA 0x10
static bool gIsaOverridesInitialized;
// Implements the ioapic Read operation.
static uint32_t IoapicRead(uint8_t reg) {
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
return *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA);
// I/O APIC register selectors are shared by every CPU accessing that APIC.
static uint32_t IoapicRead(HAL_IOAPIC* apic, uint8_t reg) {
uintptr_t interruptFlags = HalDisableInterrupts();
while (_InterlockedCompareExchange(&apic->RegisterLock, 1, 0))
_mm_pause();
apic->Registers[IOAPIC_IDX / 4u] = reg;
uint32_t value = apic->Registers[IOAPIC_DATA / 4u];
_InterlockedExchange(&apic->RegisterLock, 0);
HalRestoreInterrupts(interruptFlags);
return value;
}
// Implements the ioapic Write operation.
static void IoapicWrite(uint8_t reg, uint32_t val) {
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA) = val;
static void IoapicWrite(HAL_IOAPIC* apic, uint8_t reg, uint32_t value) {
uintptr_t interruptFlags = HalDisableInterrupts();
while (_InterlockedCompareExchange(&apic->RegisterLock, 1, 0))
_mm_pause();
apic->Registers[IOAPIC_IDX / 4u] = reg;
apic->Registers[IOAPIC_DATA / 4u] = value;
_InterlockedExchange(&apic->RegisterLock, 0);
HalRestoreInterrupts(interruptFlags);
}
static HAL_IOAPIC* HalApicFindGsi(uint32_t gsi, uint32_t* pinOut) {
for (uint32_t i = 0; i < gIoapicCount; i++) {
HAL_IOAPIC* apic = &gIoapics[i];
if (gsi >= apic->GsiBase && gsi - apic->GsiBase < apic->Pins) {
*pinOut = gsi - apic->GsiBase;
return apic;
}
}
return (HAL_IOAPIC*)0;
}
// Implements the apic Detect Mode operation.
@@ -98,8 +136,6 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
// IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high).
#define IOREDTBL_LO(n) (0x10 + 2*(n))
#define IOREDTBL_HI(n) (0x11 + 2*(n))
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
#define IRQ_VECTOR_BASE 0x20
// Returns the local interrupt-controller hardware ID of the boot processor.
uint32_t HalApicGetBootProcessorId(void) {
@@ -126,12 +162,46 @@ void HalApicInitializeCurrentProcessor(void) {
apicBase |= IA32_APIC_BASE_X2APIC;
__writemsr(IA32_APIC_BASE, apicBase);
__writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu);
HalApicSetTaskPriority(0);
return;
}
__writemsr(IA32_APIC_BASE, apicBase);
if (gLapic)
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
HalApicSetTaskPriority(0);
}
// Controls the current CPU's interrupt priority without changing its interrupt-enable flag.
void HalApicSetTaskPriority(uint8_t priority) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_TPR, priority);
} else if (gLapic) {
LAPIC_REG(LAPIC_TPR) = priority;
}
}
uint8_t HalApicQueryTaskPriority(void) {
if (gX2ApicEnabled)
return (uint8_t)__readmsr(IA32_X2APIC_TPR);
return gLapic ? (uint8_t)LAPIC_REG(LAPIC_TPR) : 0u;
}
// Reads the current processor's local delivery state after a source is masked.
bool HalApicVectorPending(uint8_t vector) {
const uint32_t index = (uint32_t)vector >> 5u;
const uint32_t mask = 1u << (vector & 31u);
uint32_t irr, isr;
if (gX2ApicEnabled) {
irr = (uint32_t)__readmsr(IA32_X2APIC_IRR_BASE + index);
isr = (uint32_t)__readmsr(IA32_X2APIC_ISR_BASE + index);
} else if (gLapic) {
irr = LAPIC_REG(LAPIC_IRR_BASE + 0x10u * index);
isr = LAPIC_REG(LAPIC_ISR_BASE + 0x10u * index);
} else {
return true;
}
return ((irr | isr) & mask) != 0u;
}
// Broadcasts an NMI to every processor except the caller.
@@ -182,99 +252,83 @@ KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical) {
return NTLSTATUS_SUCCESS;
}
// Composes an MSI message targeting the boot processor without exposing APIC mode.
KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut) {
if (!messageOut || vector < 0x20u)
// MSI destination encoding without interrupt remapping supports an 8-bit APIC ID.
KSTATUS HalApicComposeMsi(uint32_t hardwareId, uint8_t vector, HAL_MSI_MESSAGE* out) {
if (!out || vector < KE_VECTOR_DEVICE_FIRST || vector > KE_VECTOR_DEVICE_LAST)
return NTLSTATUS_INVALID_PARAMETER;
if (gBspLapicId >= 256u)
if (hardwareId > 255u)
return NTLSTATUS_NOT_SUPPORTED;
messageOut->Address = 0xFEE00000ull | ((uint64_t)(gBspLapicId & 0xFFu) << 12);
messageOut->Data = vector;
messageOut->Reserved = 0;
out->Address = 0xFEE00000ull | ((uint64_t)hardwareId << 12);
out->Data = vector;
out->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
// Initializes the identity ISA map before processing MADT source overrides.
static void HalApicInitializeIsaMap(void) {
if (gIsaOverridesInitialized) return;
for (uint32_t irq = 0; irq < 16; ++irq) {
for (uint32_t irq = 0; irq < 16; irq++) {
gIsaGsi[irq] = irq;
gIsaFlags[irq] = 0;
}
gIsaOverridesInitialized = true;
}
// Records the global-system-interrupt base of the currently supported IOAPIC.
void HalApicSetGsiBase(uint32_t gsiBase) { gIoapicGsiBase = gsiBase; }
void HalApicAddIoapic(uintptr_t physical, uint32_t gsiBase) {
if (!physical || gIoapicCount >= HAL_IOAPIC_MAX)
return;
for (uint32_t i = 0; i < gIoapicCount; i++)
if (gIoapics[i].Physical == physical)
return;
HAL_IOAPIC* apic = &gIoapics[gIoapicCount++];
apic->Physical = physical;
apic->GsiBase = gsiBase;
}
// Preserves MADT interrupt-source overrides for legacy ISA devices.
void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags) {
if (irq >= 16) return;
if (irq >= 16u) return;
HalApicInitializeIsaMap();
gIsaGsi[irq] = gsi;
gIsaFlags[irq] = flags;
}
// Marks the FADT SCI as a low-active, level-triggered interrupt by default.
void HalApicSetSciIrq(uint16_t irq) { gSciIrq = irq; }
// Resolves one logical IRQ to a physical pin on the selected IOAPIC.
static bool HalApicResolveIrq(uint8_t irq, uint32_t* pinOut, uint32_t* flagsOut) {
KSTATUS HalApicResolveIsa(uint8_t irq, uint32_t* gsiOut, uint16_t* flagsOut) {
if (irq >= 16u || !gsiOut || !flagsOut)
return NTLSTATUS_INVALID_PARAMETER;
HalApicInitializeIsaMap();
uint32_t gsi = irq < 16 ? gIsaGsi[irq] : irq;
if (gsi < gIoapicGsiBase || gsi - gIoapicGsiBase >= gIoapicRedirectionCount)
return false;
uint32_t flags = 0;
if (irq == gSciIrq) flags = (1u << 13) | (1u << 15);
if (irq < 16) {
uint16_t override = gIsaFlags[irq];
// MADT 00 is bus-conformant; 11 selects active low or level triggered.
if ((override & 3u) == 3u) flags |= (1u << 13);
else if ((override & 3u) == 1u) flags &= ~(1u << 13);
if (((override >> 2) & 3u) == 3u) flags |= (1u << 15);
else if (((override >> 2) & 3u) == 1u) flags &= ~(1u << 15);
}
*pinOut = gsi - gIoapicGsiBase;
*flagsOut = flags;
return true;
uint32_t pin;
if (!HalApicFindGsi(gIsaGsi[irq], &pin))
return NTLSTATUS_NOT_SUPPORTED;
*gsiOut = gIsaGsi[irq];
*flagsOut = gIsaFlags[irq];
return NTLSTATUS_SUCCESS;
}
// Reports whether a platform IRQ can be routed by the current IOAPIC.
bool HalApicCanRouteIrq(uint8_t irq) {
uint32_t pin, flags;
return irq < 24 && HalApicResolveIrq(irq, &pin, &flags);
// ACPI polarity bits 1:0 and trigger bits 3:2 encode bus-defined defaults.
KSTATUS HalApicRouteGsi(uint32_t gsi, uint8_t vector, uint32_t hardwareId,
uint16_t flags, bool masked) {
if (vector < KE_VECTOR_DEVICE_FIRST || vector > KE_VECTOR_DEVICE_LAST || hardwareId > 255u)
return NTLSTATUS_INVALID_PARAMETER;
uint32_t pin;
HAL_IOAPIC* apic = HalApicFindGsi(gsi, &pin);
if (!apic) return NTLSTATUS_NOT_SUPPORTED;
uint32_t mode = 0;
if ((flags & 3u) == 3u) mode |= (1u << 13);
if (((flags >> 2) & 3u) == 3u) mode |= (1u << 15);
IoapicWrite(apic, IOREDTBL_LO(pin), (1u << 16) | vector | mode);
IoapicWrite(apic, IOREDTBL_HI(pin), (hardwareId & 255u) << 24);
if (!masked) IoapicWrite(apic, IOREDTBL_LO(pin), vector | mode);
return NTLSTATUS_SUCCESS;
}
// Implements the apic Route Irq operation.
void HalApicRouteIrq(uint8_t irq) {
// Full initial setup of the redirection entry.
// Sets the vector and clears the mask bit in one write.
uint32_t pin, flags;
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
uint32_t vector = IRQ_VECTOR_BASE + irq;
IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
IoapicWrite(IOREDTBL_LO(pin), vector | flags);
}
// Implements the apic Mask Irq operation.
void HalApicMaskIrq(uint8_t irq) {
uint32_t pin, flags;
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
lo |= (1 << 16);
IoapicWrite(IOREDTBL_LO(pin), lo);
}
// Implements the apic Unmask Irq operation.
void HalApicUnmaskIrq(uint8_t irq) {
uint32_t pin, flags;
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
lo = (lo & ~(0xFFu | (1u << 16) | (1u << 13) | (1u << 15))) |
(IRQ_VECTOR_BASE + irq) | flags;
IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
IoapicWrite(IOREDTBL_LO(pin), lo);
KSTATUS HalApicMaskGsi(uint32_t gsi, bool masked) {
uint32_t pin;
HAL_IOAPIC* apic = HalApicFindGsi(gsi, &pin);
if (!apic) return NTLSTATUS_NOT_SUPPORTED;
uint32_t lo = IoapicRead(apic, IOREDTBL_LO(pin));
if (masked) lo |= 1u << 16;
else lo &= ~(1u << 16);
IoapicWrite(apic, IOREDTBL_LO(pin), lo);
return NTLSTATUS_SUCCESS;
}
// Implements the apic Send Eoi operation.
@@ -287,26 +341,29 @@ void HalApicSendEoi(void) {
}
// Implements the apic Initialize operation.
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
void HalApicInitialize(uintptr_t lapicPhys) {
// The PIC remains masked: legacy input arrives through programmed I/O APICs.
__outbyte(0x21, 0xFF);
__outbyte(0xA1, 0xFF);
HalApicDetectMode();
gLapic = (volatile uint32_t*)(HalPhysicalToVirtual(lapicPhys));
gIoapic = (volatile uint32_t*)(HalPhysicalToVirtual(ioapicPhys));
// The BSP is the CPU executing HAL init. Keep all external/device IRQs
// routed to this LAPIC; CPU-local IPIs use the local APIC directly.
gLapic = (volatile uint32_t*)HalPhysicalToVirtual(lapicPhys);
if (gX2ApicEnabled)
gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID);
else
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFFu;
HalApicInitializeCurrentProcessor();
if (gIoapicCount == 0)
HalApicAddIoapic(0xFEC00000u, 0);
// Mask all IOAPIC lines initially.
uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
gIoapicRedirectionCount = maxRedir + 1u;
for (uint32_t i = 0; i <= maxRedir; i++) {
IoapicWrite(IOREDTBL_LO(i), (1u << 16) | IRQ_VECTOR_BASE);
IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24);
for (uint32_t index = 0; index < gIoapicCount; index++) {
HAL_IOAPIC* apic = &gIoapics[index];
apic->Registers = (volatile uint32_t*)HalPhysicalToVirtual(apic->Physical);
uint32_t version = IoapicRead(apic, 1u);
apic->Pins = ((version >> 16) & 0xFFu) + 1u;
for (uint32_t pin = 0; pin < apic->Pins; pin++) {
IoapicWrite(apic, IOREDTBL_LO(pin), (1u << 16) | KE_VECTOR_DEVICE_FIRST);
IoapicWrite(apic, IOREDTBL_HI(pin), (gBspLapicId & 255u) << 24);
}
}
}
code/kernel/hal/hal_main.c
+12 -108
View File
@@ -39,10 +39,13 @@ static HAL_API gApi = {
.Start = HalApicStartProcessor,
},
.InterruptController = {
.RouteIrq = HalApicRouteIrq,
.MaskIrq = HalApicMaskIrq,
.UnmaskIrq = HalApicUnmaskIrq,
.RouteGsi = HalApicRouteGsi,
.MaskGsi = HalApicMaskGsi,
.ResolveIsa = HalApicResolveIsa,
.SendEoi = HalApicSendEoi,
.SetTaskPriority = HalApicSetTaskPriority,
.QueryTaskPriority = HalApicQueryTaskPriority,
.VectorPending = HalApicVectorPending,
.InitializeCurrentProcessor = HalApicInitializeCurrentProcessor,
.SendProcessorInterrupt = HalApicSendProcessorInterrupt,
.SendProcessorNmi = HalApicSendProcessorNmi,
@@ -122,12 +125,7 @@ typedef struct ACPI_FIXED_REGISTER {
static ACPI_FIXED_REGISTER gPm1aCnt;
static ACPI_FIXED_REGISTER gPm1bCnt;
static ACPI_FIXED_REGISTER gPm1aStatus;
static ACPI_FIXED_REGISTER gPm1bStatus;
static ACPI_FIXED_REGISTER gPm1aEnable;
static ACPI_FIXED_REGISTER gPm1bEnable;
static uint16_t gSciIrq = 0;
static bool gFixedPowerButtonArmed = false;
static ACPI_FIXED_REGISTER gSleepControl;
static uint32_t gLegacyPm1aPort = 0;
static uint32_t gLegacyPm1bPort = 0;
@@ -179,13 +177,6 @@ void* HalPhysicalToVirtual(uint64_t phys) {
// systems may put the real DSDT and PM1 control registers in the extended
// X_* fields, so do not rely only on the legacy 32-bit fields.
#define FADT_SCI_INT_OFFSET 46u
#define FADT_PM1A_EVT_BLK_OFFSET 56u
#define FADT_PM1B_EVT_BLK_OFFSET 60u
#define FADT_PM1_EVT_LEN_OFFSET 88u
#define FADT_X_PM1A_EVT_BLK_OFFSET 148u
#define FADT_X_PM1B_EVT_BLK_OFFSET 160u
#define FADT_FLAG_POWER_BUTTON (1u << 4)
#define ACPI_PM1_POWER_BUTTON_BIT (1u << 8)
#define FADT_DSDT_OFFSET 40u
#define FADT_SMI_CMD_OFFSET 48u
#define FADT_ACPI_ENABLE_OFFSET 52u
@@ -310,77 +301,6 @@ static void AcpiWriteFixedRegister(const ACPI_FIXED_REGISTER* reg, uint32_t valu
else *(volatile uint32_t*)address = value;
}
// Selects a PM1 event block as two adjacent equal-width status and enable registers.
static void AcpiConfigurePm1EventBlock(const void* fadt, uint32_t len,
uint32_t legacyOffset, uint32_t gasOffset,
uint8_t halfLength, ACPI_FIXED_REGISTER* status,
ACPI_FIXED_REGISTER* enable) {
ACPI_FIXED_REGISTER block = {0};
uint32_t legacy = AcpiReadU32(fadt, len, legacyOffset);
if (legacy && legacy <= 0xFFFFu) {
block.AddressSpaceId = 1;
block.Address = legacy;
}
ACPI_FIXED_REGISTER extended = {0};
if (AcpiReadGasRegister(fadt, len, gasOffset, &extended)) block = extended;
if (!block.Address || (block.AddressSpaceId == 1 &&
block.Address + halfLength * 2u - 1u > 0xFFFFu)) return;
block.WidthBytes = halfLength;
*status = block;
block.Address += halfLength;
*enable = block;
}
// Reports the fixed-feature power-button interrupt supported by this HAL.
static KSTATUS AcpiQueryFixedButtonIrq(uint8_t* irqOut) {
if (!irqOut) return NTLSTATUS_INVALID_PARAMETER;
// A FADT control-method power button requires AML evaluation, not PM1 polling.
if (gHwReducedAcpi || (gFadtFlags & FADT_FLAG_POWER_BUTTON) ||
!gSciIrq || gSciIrq >= 16u || !gPm1aStatus.Address || !gPm1aEnable.Address ||
!HalApicCanRouteIrq((uint8_t)gSciIrq))
return NTLSTATUS_NOT_SUPPORTED;
*irqOut = (uint8_t)gSciIrq;
return NTLSTATUS_SUCCESS;
}
// Discards any stale press then enables the PM1 fixed power-button SCI.
static KSTATUS AcpiArmFixedButton(void) {
uint8_t irq;
KSTATUS status = AcpiQueryFixedButtonIrq(&irq);
if (!KSUCCESS(status)) return status;
// No SCI_EN means firmware still owns ACPI events. Never steal ownership
// through SMI_CMD just to install a power-button handler during boot.
if (!(AcpiReadFixedRegister(&gPm1aCnt) & 1u))
return NTLSTATUS_NOT_SUPPORTED;
AcpiWriteFixedRegister(&gPm1aStatus, ACPI_PM1_POWER_BUTTON_BIT);
if (gPm1bStatus.Address)
AcpiWriteFixedRegister(&gPm1bStatus, ACPI_PM1_POWER_BUTTON_BIT);
uint32_t enabled = AcpiReadFixedRegister(&gPm1aEnable);
AcpiWriteFixedRegister(&gPm1aEnable, enabled | ACPI_PM1_POWER_BUTTON_BIT);
if (gPm1bEnable.Address) {
enabled = AcpiReadFixedRegister(&gPm1bEnable);
AcpiWriteFixedRegister(&gPm1bEnable, enabled | ACPI_PM1_POWER_BUTTON_BIT);
}
gFixedPowerButtonArmed = true;
return NTLSTATUS_SUCCESS;
}
// Acknowledges the fixed-button source only; the SCI may carry unrelated events.
static bool AcpiConsumeFixedButton(void) {
if (!gFixedPowerButtonArmed) return false;
bool pressed = false;
if (AcpiReadFixedRegister(&gPm1aStatus) & ACPI_PM1_POWER_BUTTON_BIT) {
AcpiWriteFixedRegister(&gPm1aStatus, ACPI_PM1_POWER_BUTTON_BIT);
pressed = true;
}
if (gPm1bStatus.Address &&
(AcpiReadFixedRegister(&gPm1bStatus) & ACPI_PM1_POWER_BUTTON_BIT)) {
AcpiWriteFixedRegister(&gPm1bStatus, ACPI_PM1_POWER_BUTTON_BIT);
pressed = true;
}
return pressed;
}
// Detects whether PM1 control is using the standard Intel PCH PMBASE layout.
static void AcpiDetectIntelPchPmBase(void) {
gIntelPchPmBase = 0;
@@ -580,10 +500,6 @@ static void ParseFadt(const ACPISDTHeader* header) {
memset(&gPm1aCnt, 0, sizeof(gPm1aCnt));
memset(&gPm1bCnt, 0, sizeof(gPm1bCnt));
memset(&gPm1aStatus, 0, sizeof(gPm1aStatus));
memset(&gPm1bStatus, 0, sizeof(gPm1bStatus));
memset(&gPm1aEnable, 0, sizeof(gPm1aEnable));
memset(&gPm1bEnable, 0, sizeof(gPm1bEnable));
gSciIrq = 0;
if (len >= FADT_SCI_INT_OFFSET + sizeof(gSciIrq))
memcpy(&gSciIrq, (const uint8_t*)fadt + FADT_SCI_INT_OFFSET, sizeof(gSciIrq));
@@ -628,16 +544,6 @@ static void ParseFadt(const ACPISDTHeader* header) {
if (gHwReducedAcpi)
(void)AcpiReadByteGasRegister(fadt, len, FADT_SLEEP_CONTROL_OFFSET, &gSleepControl);
uint8_t eventLength = FADT_PM1_EVT_LEN_OFFSET < len ?
*((const uint8_t*)fadt + FADT_PM1_EVT_LEN_OFFSET) : 0u;
if (!gHwReducedAcpi && (eventLength == 4u || eventLength == 8u)) {
uint8_t half = eventLength / 2u;
AcpiConfigurePm1EventBlock(fadt, len, FADT_PM1A_EVT_BLK_OFFSET,
FADT_X_PM1A_EVT_BLK_OFFSET, half, &gPm1aStatus, &gPm1aEnable);
AcpiConfigurePm1EventBlock(fadt, len, FADT_PM1B_EVT_BLK_OFFSET,
FADT_X_PM1B_EVT_BLK_OFFSET, half, &gPm1bStatus, &gPm1bEnable);
}
gSmiCommandPort = AcpiReadU32(fadt, len, FADT_SMI_CMD_OFFSET);
gAcpiEnableValue = FADT_ACPI_ENABLE_OFFSET < len
? *((const uint8_t*)fadt + FADT_ACPI_ENABLE_OFFSET)
@@ -652,10 +558,12 @@ static void ParseFadt(const ACPISDTHeader* header) {
centuryRegister = *((const uint8_t*)fadt + FADT_CENTURY_OFFSET);
HalRtcSetCenturyRegister(centuryRegister);
HalDbgPrint("HAL: FADT flags=0x%08x hw_reduced=%u PM1a_CNT=%u:0x%016llx legacy=0x%04x "
HalDbgPrint("HAL: FADT flags=0x%08x sci_irq=%u power_button=%s hw_reduced=%u PM1a_CNT=%u:0x%016llx legacy=0x%04x "
"PM1b_CNT=%u:0x%016llx legacy=0x%04x SLEEP_CTL=%u:0x%016llx "
"SMI=0x%08x DSDT=0x%016llx RTC_CENTURY=0x%02x\n",
gFadtFlags, gHwReducedAcpi ? 1u : 0u,
gFadtFlags, (unsigned)gSciIrq,
(gFadtFlags & (1u << 4)) ? "control-method" : "fixed",
gHwReducedAcpi ? 1u : 0u,
gPm1aCnt.AddressSpaceId, gPm1aCnt.Address, gLegacyPm1aPort,
gPm1bCnt.AddressSpaceId, gPm1bCnt.Address, gLegacyPm1bPort,
gSleepControl.AddressSpaceId, gSleepControl.Address,
@@ -711,7 +619,7 @@ static void ParseMadt(MADT* madt) {
gIoapicPhys = *(uint32_t*)(p + 4);
uint32_t base;
memcpy(&base, p + 8, sizeof(base));
HalApicSetGsiBase(base);
HalApicAddIoapic(*(uint32_t*)(p + 4), base);
} else if (type == 2 && len >= 10 && p[2] == 0) { // ISA source override
uint32_t gsi;
uint16_t flags;
@@ -995,8 +903,7 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
HalDbgPrint("HAL: LAPIC=%016llx IOAPIC=%016llx HPET=%016llx\n",
gLapicPhys, gIoapicPhys, gHpetPhys);
HalApicSetSciIrq(gSciIrq);
HalApicInitialize(gLapicPhys, gIoapicPhys);
HalApicInitialize(gLapicPhys);
HalDbgPrint("HAL: local APIC mode=%s BSP_ID=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
HalApicGetBootProcessorId());
@@ -1008,9 +915,6 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
HalPciInitialize(&gApi.Pci);
gApi.Power.Shutdown = HalShutdown;
gApi.Power.Reset = HalReset;
gApi.Power.QueryFixedButtonIrq = AcpiQueryFixedButtonIrq;
gApi.Power.ArmFixedButton = AcpiArmFixedButton;
gApi.Power.ConsumeFixedButton = AcpiConsumeFixedButton;
AcpiDetectIntelPchPmBase();
gInitialised = 1;
code/kernel/hal/hal_timer.c
+4 -4
View File
@@ -6,13 +6,14 @@
#include <kernel/hal/hal_timer.h>
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_debug.h>
#include <kernel/ke_vectors.h>
#define LAPIC_TIMER_LVT 0x320
#define LAPIC_TIMER_INIT 0x380
#define LAPIC_TIMER_CUR 0x390
#define LAPIC_TIMER_DIV 0x3E0
#define TIMER_VECTOR 0x20 // vector 0x20 = IRQ 0 slot in IDT
#define TIMER_VECTOR KE_VECTOR_CPU_TIMER
#define TIMER_PERIODIC (1 << 17)
#define TIMER_MASKED (1 << 16)
#define TIMER_DIV16 0x3
@@ -226,14 +227,13 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* o
HalDbgPrint("HAL: APIC timer: %u ticks/ms\n", ticksPerMs);
// Leave the BSP timer masked until the kernel registers its scheduler
// callback and IRQ route. APs arm their local timer after per-CPU setup.
// Leave the BSP timer masked until the kernel installs its timer handlers.
// APs arm their local timer after per-CPU setup.
out->QueryTicks = HalTimerQueryTicks;
out->QueryTickFrequency = QueryTickFrequency;
out->Stall = HalTimerStall;
out->HandleInterrupt = TimerInterrupt;
out->ConfigureCurrentProcessor = HalTimerConfigureCurrentProcessor;
out->IrqNumber = 0;
return NTLSTATUS_SUCCESS;
}
code/kernel/include/kernel/hal/hal_api.h
+14 -12
View File
@@ -132,7 +132,6 @@ typedef struct HAL_TIMER {
void (*Stall)(uint32_t us);
void (*HandleInterrupt)(void);
void (*ConfigureCurrentProcessor)(void);
uint8_t IrqNumber;
} HAL_TIMER;
#define HAL_RTC_TIME_FLAG_VALID 0x0001u
@@ -173,24 +172,27 @@ typedef struct HAL_PCI {
} HAL_PCI;
typedef struct HAL_INTERRUPT_CONTROLLER {
void (*RouteIrq)(uint8_t irq);
void (*MaskIrq)(uint8_t irq);
void (*UnmaskIrq)(uint8_t irq);
KSTATUS (*RouteGsi)(uint32_t gsi, uint8_t vector, uint32_t hardwareId,
uint16_t flags, bool masked);
KSTATUS (*MaskGsi)(uint32_t gsi, bool masked);
KSTATUS (*ResolveIsa)(uint8_t isaIrq, uint32_t* gsiOut, uint16_t* flagsOut);
void (*SendEoi)(void);
// The APIC task priority must follow the kernel's per-CPU IRQL.
void (*SetTaskPriority)(uint8_t priority);
uint8_t (*QueryTaskPriority)(void);
// Queries the current CPU's pending and in-service APIC bits for a vector.
bool (*VectorPending)(uint8_t vector);
void (*InitializeCurrentProcessor)(void);
void (*SendProcessorInterrupt)(uint32_t hardwareId, uint8_t vector);
void (*SendProcessorNmi)(uint32_t hardwareId);
void (*BroadcastNmi)(void);
KSTATUS (*ComposeMsi)(uint8_t vector, HAL_MSI_MESSAGE* messageOut);
KSTATUS (*ComposeMsi)(uint32_t hardwareId, uint8_t vector,
HAL_MSI_MESSAGE* messageOut);
} HAL_INTERRUPT_CONTROLLER;
typedef struct HAL_POWER {
// ACPI fixed-feature button events. Control-method devices require an AML owner.
KSTATUS (*QueryFixedButtonIrq)(uint8_t* irqOut);
KSTATUS (*ArmFixedButton)(void);
bool (*ConsumeFixedButton)(void);
// ACPI S5 soft-off. Disables interrupts, writes SLP_EN to PM1a/b_CNT,
// and halts if firmware doesn't respond. Does not return.
// Basic platform shutdown and reset, available without a firmware driver.
// The HAL halts if its final hardware transition returns.
void (*Shutdown)(void);
void (*Reset)(void);
} HAL_POWER;
@@ -231,7 +233,7 @@ typedef struct HAL_INIT_PARAMS {
} HAL_INIT_PARAMS;
// Increment whenever the kernel/HAL table ABI changes.
#define HAL_API_VERSION 4u
#define HAL_API_VERSION 8u
// Returns the HAL table. Processor interrupt-state operations are valid before HalInitialize().
HAL_ENTRY const HAL_API* HalGetApi(void);
code/kernel/include/kernel/hal/hal_apic.h
+10 -8
View File
@@ -6,15 +6,17 @@
#include <ntlite/status.h>
#include <kernel/hal/hal_api.h>
void HalApicRouteIrq(uint8_t irq);
void HalApicSetGsiBase(uint32_t gsiBase);
KSTATUS HalApicRouteGsi(uint32_t gsi, uint8_t vector, uint32_t hardwareId,
uint16_t flags, bool masked);
KSTATUS HalApicMaskGsi(uint32_t gsi, bool masked);
KSTATUS HalApicResolveIsa(uint8_t irq, uint32_t* gsiOut, uint16_t* flagsOut);
void HalApicAddIoapic(uintptr_t physical, uint32_t gsiBase);
void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags);
void HalApicSetSciIrq(uint16_t irq);
bool HalApicCanRouteIrq(uint8_t irq);
void HalApicMaskIrq(uint8_t irq);
void HalApicUnmaskIrq(uint8_t irq);
void HalApicSendEoi(void);
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys);
void HalApicSetTaskPriority(uint8_t priority);
uint8_t HalApicQueryTaskPriority(void);
bool HalApicVectorPending(uint8_t vector);
void HalApicInitialize(uintptr_t lapicPhys);
uint32_t HalApicGetBootProcessorId(void);
uint8_t HalApicUsingX2Apic(void);
uint8_t HalApicCanAddressId(uint32_t hardwareId);
@@ -23,4 +25,4 @@ void HalApicBroadcastNmi(void);
void HalApicSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector);
void HalApicSendProcessorNmi(uint32_t hardwareId);
KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical);
KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut);
KSTATUS HalApicComposeMsi(uint32_t hardwareId, uint8_t vector, HAL_MSI_MESSAGE* messageOut);
code/kernel/include/kernel/ke_firmware.h
+8
View File
@@ -0,0 +1,8 @@
// Firmware roots retained from the verified bootloader handoff.
#pragma once
#include <ntlite/boot_info.h>
void KeFirmwareInitialize(const BOOT_INFO* bootInfo);
void KeFirmwareLog(void);
code/kernel/include/kernel/ke_idt.h
+1 -3
View File
@@ -51,6 +51,4 @@ extern void KeFaultPF(void); // #PF page fault
extern void KeFaultUD(void); // #UD invalid opcode
extern void KeNmiIpiStub(void); // returning NMI SMP-control stub
// CPU-local IPI vector. This uses the common interrupt entry machinery but is
// not an external/device IRQ and is never routed through the external interrupt controller.
#define KE_VECTOR_RESCHEDULE_IPI 0xF0u
#include <kernel/ke_vectors.h>
code/kernel/include/kernel/ke_irq.h
+7 -12
View File
@@ -1,18 +1,13 @@
// Kernel ke irq declarations shared by implementation files and kernel clients.
// Kernel interrupt-manager declarations.
#pragma once
#include <stdint.h>
#include <ntlite/ddk/interrupt.h>
#include <kernel/hal/hal_api.h>
#include <ntlite/status.h>
void KeIrqInitialize(const HAL_API* hal);
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq);
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void));
void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm
// Low-level interrupt entry/exit hooks. These are called by the common IRQ
// assembly stub around KeIrqDispatch. The exit hook is where deferred
// interrupt-time scheduling now happens, so IRQ handlers never switch away
// before KeIrqDispatch has returned and sent EOI.
void KeIrqRegisterLocalTimerHandler(void (*handler)(void));
void KeIrqDispatch(uint8_t vector);
void KiEnterInterrupt(void);
void KiExitInterrupt(void);
// CPU offlining requires the interrupt manager to evacuate every route first.
KSTATUS KiInterruptPrepareCpuOffline(uint32_t cpuIndex);
code/kernel/include/kernel/ke_irql.h
+1 -3
View File
@@ -2,9 +2,7 @@
#pragma once
// ke_irql.h - software IRQL skeleton.
// The level constants are DDK-visible because current synchronization and IRP
// layouts expose KIRQL. Kernel routines remain declared here.
// IRQL levels are DDK-visible; the kernel synchronizes them with APIC TPR.
#include <stdbool.h>
#include <ntlite/ddk/irql.h>
code/kernel/include/kernel/ke_power.h
+3 -2
View File
@@ -5,9 +5,10 @@
struct PROCESS;
struct EVENT;
struct HAL_API;
KSTATUS KePowerInitialize(void);
void KePowerInitializeButton(const struct HAL_API* hal);
void KePowerInitializePolicy(void);
KSTATUS KePowerPrepareFirmware(uint32_t action);
KSTATUS KePowerEnterFirmware(uint32_t action);
void KePowerSetInitProcess(struct PROCESS* init);
KSTATUS KePowerRequest(uint32_t action);
KSTATUS KePowerRequestHardware(uint32_t action);
code/kernel/include/kernel/ke_vectors.h
+12
View File
@@ -0,0 +1,12 @@
// x86-64 interrupt-vector assignments shared by the kernel and HAL.
#pragma once
// CPU exception gates occupy vectors 0x00 through 0x1f.
// Software gates occupy 0x20 through 0x2f; only the syscall gate is active.
#define KE_VECTOR_SYSCALL 0x2Fu
#define KE_VECTOR_DEVICE_FIRST 0x30u
#define KE_VECTOR_DEVICE_LAST 0xCFu
#define KE_VECTOR_CPU_TIMER 0xE0u
#define KE_VECTOR_RESCHEDULE_IPI 0xF0u
#define KE_VECTOR_SPURIOUS 0xFFu
code/kernel/include/kernel/syscall_nr.h
+1 -1
View File
@@ -2,7 +2,7 @@
#pragma once
// INT 0x40 calling convention, MS x64.
// INT 0x2F calling convention, MS x64.
// RAX selects the syscall number. RCX, RDX, R8, and R9 carry args 0-3.
// Syscalls use at most four register arguments; larger operations use request structures.
// RAX carries the return value.
code/kernel/io_ddk.c
+119 -12
View File
@@ -4,6 +4,9 @@
#include <kernel/io_dma.h>
#include <kernel/ke_audio.h>
#include <kernel/ke_irq.h>
#include <kernel/ke_vectors.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/mm_pte.h>
static const HAL_API* gHal;
@@ -14,6 +17,10 @@ void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase) {
gPhysicalWindowBase = physicalWindowBase;
}
__declspec(dllexport) uintptr_t KeCurrentThreadIdentity(void) {
return (uintptr_t)KeGetCurrentThread();
}
__declspec(dllexport) uint64_t KeQueryTicks(void) {
return gHal && gHal->Timer.QueryTicks ? gHal->Timer.QueryTicks() : 0;
}
@@ -30,18 +37,6 @@ __declspec(dllexport) void KePciWriteConfig32(uint16_t segment, uint8_t bus, uin
if (gHal && gHal->Pci.Write32) gHal->Pci.Write32(segment, bus, device, function, offset, value);
}
__declspec(dllexport) KSTATUS KeIrqComposeMsiMessage(uint8_t irq, NTL_DDK_MSI_MESSAGE* out) {
if (!gHal || !out || irq >= 24u || !gHal->InterruptController.ComposeMsi)
return NTLSTATUS_INVALID_PARAMETER;
HAL_MSI_MESSAGE message;
KSTATUS status = gHal->InterruptController.ComposeMsi((uint8_t)(0x20u + irq), &message);
if (!KSUCCESS(status)) return status;
out->Address = message.Address;
out->Data = message.Data;
out->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) void* MmDdkPhysToVirt(uint64_t physicalAddress) {
if (!gPhysicalWindowBase || physicalAddress > UINTPTR_MAX - gPhysicalWindowBase)
return (void*)0;
@@ -53,6 +48,118 @@ __declspec(dllexport) KSTATUS MmDdkMapPage(uintptr_t virtualAddress, uint64_t ph
mode == NtlDdkMapReplaceExisting ? MmPteMapReplaceExisting : MmPteMapFailIfPresent);
}
// Each slot is a recyclable 2 MiB VA reservation; no pool allocation is needed
// when reserving a device mapping. This aperture stays separate from PCI BARs.
#define DDK_IO_VA_BASE 0xFFFF980000000000ULL
#define DDK_IO_SLOT_SIZE (2ULL * 1024ULL * 1024ULL)
#define DDK_IO_SLOT_COUNT 256u
typedef struct DDK_IO_MAPPING {
uint8_t State;
uint32_t Pages;
uintptr_t Offset;
size_t Length;
} DDK_IO_MAPPING;
static SPINLOCK gDdkIoMapLock = SPINLOCK_INIT;
static DDK_IO_MAPPING gDdkIoMaps[DDK_IO_SLOT_COUNT];
// The caller owns the mapping until an exact-length unmap completes.
__declspec(dllexport) KSTATUS MmDdkMapIoMemory(uint64_t physicalAddress, size_t length, void** outVirtual) {
if (!outVirtual) return NTLSTATUS_INVALID_PARAMETER;
*outVirtual = NULL;
if (!KeCanWait()) return NTLSTATUS_NOT_SUPPORTED;
if (!length || physicalAddress > UINT64_MAX - length) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t offset = (uintptr_t)(physicalAddress & (PAGE_SIZE - 1u));
if (length > DDK_IO_SLOT_SIZE - offset) return NTLSTATUS_INVALID_PARAMETER;
uint32_t pages = (uint32_t)((offset + length + PAGE_SIZE - 1u) / PAGE_SIZE);
if (pages == 0 || pages > DDK_IO_SLOT_SIZE / PAGE_SIZE) return NTLSTATUS_INVALID_PARAMETER;
// Reject a UC alias of RAM already covered by the firmware's WB/WC
// direct map. Real device registers have no direct mapping or an UC one.
for (uint32_t i = 0; i < pages; i++) {
uint64_t frame = (physicalAddress - offset) + (uint64_t)i * PAGE_SIZE;
if (gPhysicalWindowBase && frame <= UINTPTR_MAX - gPhysicalWindowBase) {
PHYSICAL_ADDRESS existing = 0;
uint64_t existingFlags = 0;
if (KSUCCESS(MmPteQueryPage(gPhysicalWindowBase + (uintptr_t)frame,
&existing, &existingFlags))) {
if ((existing & PAGE_MASK) != frame ||
(existingFlags & PAGE_CACHE_MASK) != PAGE_CACHE_UC)
return NTLSTATUS_NOT_SUPPORTED;
}
}
}
uintptr_t flags = KeAcquireSpinlock(&gDdkIoMapLock);
uint32_t slot;
for (slot = 0; slot < DDK_IO_SLOT_COUNT; slot++) {
if (!gDdkIoMaps[slot].State) {
gDdkIoMaps[slot].State = 1;
gDdkIoMaps[slot].Length = length;
gDdkIoMaps[slot].Offset = offset;
gDdkIoMaps[slot].Pages = pages;
break;
}
}
KeReleaseSpinlock(&gDdkIoMapLock, flags);
if (slot == DDK_IO_SLOT_COUNT) return NTLSTATUS_NO_MEMORY;
uintptr_t base = (uintptr_t)(DDK_IO_VA_BASE + (uint64_t)slot * DDK_IO_SLOT_SIZE);
uint64_t physBase = physicalAddress - offset;
uint32_t mapped = 0;
KSTATUS status = NTLSTATUS_SUCCESS;
for (; mapped < pages; mapped++) {
status = MmPteMapPage(base + (uintptr_t)mapped * PAGE_SIZE,
physBase + (uint64_t)mapped * PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC | PAGE_NX,
MmPteMapFailIfPresent);
if (!KSUCCESS(status)) break;
}
if (mapped != pages) {
while (mapped) (void)MmPteUnmapPage(base + (uintptr_t)(--mapped) * PAGE_SIZE);
flags = KeAcquireSpinlock(&gDdkIoMapLock);
gDdkIoMaps[slot].State = 0;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
return status;
}
flags = KeAcquireSpinlock(&gDdkIoMapLock);
gDdkIoMaps[slot].State = 2;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
*outVirtual = (void*)(base + offset);
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmDdkUnmapIoMemory(void* virtualAddress, size_t length) {
if (!KeCanWait()) return NTLSTATUS_NOT_SUPPORTED;
if (!virtualAddress || !length) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t address = (uintptr_t)virtualAddress;
if (address < DDK_IO_VA_BASE ||
(uint64_t)(address - DDK_IO_VA_BASE) >= DDK_IO_SLOT_SIZE * DDK_IO_SLOT_COUNT)
return NTLSTATUS_INVALID_PARAMETER;
uint32_t slot = (uint32_t)((address - DDK_IO_VA_BASE) / DDK_IO_SLOT_SIZE);
uintptr_t base = (uintptr_t)(DDK_IO_VA_BASE + (uint64_t)slot * DDK_IO_SLOT_SIZE);
uintptr_t flags = KeAcquireSpinlock(&gDdkIoMapLock);
DDK_IO_MAPPING* map = &gDdkIoMaps[slot];
if (map->State != 2 || map->Length != length || address != base + map->Offset) {
KeReleaseSpinlock(&gDdkIoMapLock, flags);
return NTLSTATUS_INVALID_PARAMETER;
}
map->State = 3;
uint32_t pages = map->Pages;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
KSTATUS result = NTLSTATUS_SUCCESS;
for (uint32_t i = 0; i < pages; i++) {
KSTATUS status = MmPteUnmapPage(base + (uintptr_t)i * PAGE_SIZE);
if (!KSUCCESS(status)) result = status;
}
flags = KeAcquireSpinlock(&gDdkIoMapLock);
// Preserve the slot after a failed unmap to prevent stale PTE reuse.
if (KSUCCESS(result)) map->State = 0;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
return result;
}
__declspec(dllexport) KSTATUS IoDmaGetAdapter(struct DEVICE_OBJECT* device, const IO_DMA_REQUIREMENTS* requirements, IO_DMA_ADAPTER** outAdapter) {
return IoGetDmaDdk()->GetAdapter(device, requirements, outAdapter);
}
code/kernel/io_device.c
+11 -8
View File
@@ -429,7 +429,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
return NTLSTATUS_SUCCESS;
}
// Implements the notify Disk Surprise Removed operation.
// Invalidates mounted partitions when a disk disappears unexpectedly.
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return NTLSTATUS_INVALID_PARAMETER;
@@ -442,29 +442,32 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
return NTLSTATUS_SUCCESS;
}
// Implements the unregister Disk Device operation.
// Releases volume consumers before the backing device completes orderly teardown.
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return NTLSTATUS_INVALID_PARAMETER;
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return NTLSTATUS_NOT_FOUND;
if (dev->PnpState != DevicePnpSurpriseRemoved) {
bool surpriseRemoved = dev->PnpState == DevicePnpSurpriseRemoved;
if (!surpriseRemoved) {
IoNotifyDiskChange(driveIndex, dev, IoDeviceQueryRemove);
KSTATUS queryStatus = IoSendPnpIrpSync(dev, IRP_MN_QUERY_REMOVE_DEVICE);
if (!KSUCCESS(queryStatus) && queryStatus != NTLSTATUS_NOT_SUPPORTED)
if (!KSUCCESS(queryStatus) && queryStatus != NTLSTATUS_NOT_SUPPORTED) {
(void)IoSendPnpIrpSync(dev, IRP_MN_CANCEL_REMOVE_DEVICE);
return queryStatus;
}
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
// Notify volume owners while the backing disk can still service teardown I/O.
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
}
dev->PnpState = DevicePnpRemovePending;
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
if (surpriseRemoved)
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return NTLSTATUS_SUCCESS;
}
code/kernel/io_volume.c
+3 -1
View File
@@ -379,7 +379,9 @@ __declspec(dllexport) KSTATUS IoRemoveVolume(IO_VOLUME* volume, IO_VOLUME_REMOVA
KiIoVolumeAddStat(&gVolumeStats.MountedVolumes, -1);
KiIoVolumeAddStat(&gVolumeStats.RemovedVolumes, 1);
KiIoNotifyVolumeChange();
DbgPrint("io: volume %u removed reason=%u\n", volume->InstanceId, (unsigned)reason);
const char* reasonName = reason == IoVolumeUnmounted ? "unmounted" :
reason == IoVolumeBackingRemoved ? "backing-removed" : "provider-dead";
DbgPrint("io: volume %u removed reason=%s\n", volume->InstanceId, reasonName);
if (dropRegistryReference)
ObDereferenceObject(&volume->Header);
return NTLSTATUS_SUCCESS;
code/kernel/ke_firmware.c
+106
View File
@@ -0,0 +1,106 @@
// Firmware root discovery for optional platform drivers.
#include <kernel/ke_firmware.h>
#include <ntlite/ddk/platform.h>
#include <ntlite/memory_map.h>
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <kernel/ke_debug.h>
#include <string.h>
#define KI_RSDP_V1_SIZE 20u
#define KI_RSDP_V2_SIZE 36u
#define KI_RSDP_MAX_SIZE 4096u
static uint64_t gAcpiRsdpPhysical;
static const char* gAcpiRootDiagnostic = "absent";
static uint8_t gAcpiRsdpRevision;
// Firmware roots must remain in reserved memory after ExitBootServices.
static bool KiFirmwareReservedSpan(const MEMORY_MAP* map, uint64_t start, uint32_t length) {
if (!map || map->Magic != MEMORY_MAP_MAGIC ||
map->RegionCount > MEMORY_MAX_REGIONS || !length || start > UINT64_MAX - length)
return false;
uint64_t end = start + length;
for (uint32_t i = 0; i < map->RegionCount; i++) {
const MEMORY_REGION* region = &map->Regions[i];
if (region->Type != MemoryReserved ||
region->Base > UINT64_MAX - region->Length)
continue;
if (start >= region->Base && end <= region->Base + region->Length)
return true;
}
return false;
}
static bool KiFirmwareChecksumValid(const uint8_t* bytes, uint32_t length) {
uint8_t sum = 0;
for (uint32_t i = 0; i < length; i++) sum = (uint8_t)(sum + bytes[i]);
return sum == 0;
}
void KeFirmwareInitialize(const BOOT_INFO* bootInfo) {
gAcpiRsdpPhysical = 0;
gAcpiRsdpRevision = 0;
gAcpiRootDiagnostic = "absent";
if (!bootInfo || !bootInfo->RsdpPhysical) return;
if (!bootInfo->PhysicalMapBase ||
!KiFirmwareReservedSpan(&bootInfo->MemoryMap,
bootInfo->RsdpPhysical, KI_RSDP_V1_SIZE) ||
bootInfo->RsdpPhysical > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
gAcpiRootDiagnostic = "invalid physical range";
return;
}
const uint8_t* rsdp = (const uint8_t*)(bootInfo->PhysicalMapBase + bootInfo->RsdpPhysical);
if (memcmp(rsdp, "RSD PTR ", 8) != 0) {
gAcpiRootDiagnostic = "invalid signature";
return;
}
if (!KiFirmwareChecksumValid(rsdp, KI_RSDP_V1_SIZE)) {
gAcpiRootDiagnostic = "invalid legacy checksum";
return;
}
if (rsdp[15] >= 2u) {
if (!KiFirmwareReservedSpan(&bootInfo->MemoryMap,
bootInfo->RsdpPhysical, KI_RSDP_V2_SIZE)) {
gAcpiRootDiagnostic = "truncated extended root";
return;
}
uint32_t length;
memcpy(&length, rsdp + 20, sizeof(length));
if (length < KI_RSDP_V2_SIZE || length > KI_RSDP_MAX_SIZE ||
!KiFirmwareReservedSpan(&bootInfo->MemoryMap,
bootInfo->RsdpPhysical, length)) {
gAcpiRootDiagnostic = "invalid extended length";
return;
}
if (!KiFirmwareChecksumValid(rsdp, length)) {
gAcpiRootDiagnostic = "invalid extended checksum";
return;
}
}
gAcpiRsdpPhysical = (uint64_t)bootInfo->RsdpPhysical;
gAcpiRsdpRevision = rsdp[15];
gAcpiRootDiagnostic = "valid";
}
// A driver chooses its own ACPI interpreter and maps tables through the DDK.
__declspec(dllexport) KSTATUS KeFirmwareQueryAcpiRsdp(uint64_t* physicalAddressOut) {
if (!physicalAddressOut) return NTLSTATUS_INVALID_PARAMETER;
*physicalAddressOut = 0;
if (!gAcpiRsdpPhysical) return NTLSTATUS_NOT_SUPPORTED;
*physicalAddressOut = gAcpiRsdpPhysical;
return NTLSTATUS_SUCCESS;
}
// Reports the firmware handoff state through the normal kernel diagnostic stream.
void KeFirmwareLog(void) {
DbgPrint("firmware: ACPI RSDP %s phys=0x%016llX revision=%u\n",
gAcpiRootDiagnostic,
(unsigned long long)gAcpiRsdpPhysical,
(unsigned)gAcpiRsdpRevision);
}
code/kernel/ke_hal.c
+4 -2
View File
@@ -30,8 +30,10 @@ static bool KiHalRuntimeTableValid(const HAL_API* hal) {
hal->Timer.QueryTicks && hal->Timer.QueryTickFrequency && hal->Timer.Stall &&
hal->Timer.HandleInterrupt && hal->Timer.ConfigureCurrentProcessor &&
hal->Rtc.QueryTime && hal->Pci.Read32 && hal->Pci.Write32 &&
hal->InterruptController.RouteIrq && hal->InterruptController.MaskIrq &&
hal->InterruptController.UnmaskIrq && hal->InterruptController.SendEoi &&
hal->InterruptController.RouteGsi && hal->InterruptController.MaskGsi &&
hal->InterruptController.ResolveIsa && hal->InterruptController.SendEoi &&
hal->InterruptController.SetTaskPriority && hal->InterruptController.QueryTaskPriority &&
hal->InterruptController.VectorPending &&
hal->InterruptController.InitializeCurrentProcessor &&
hal->InterruptController.SendProcessorInterrupt &&
hal->InterruptController.SendProcessorNmi && hal->InterruptController.BroadcastNmi &&
code/kernel/ke_idt.c
+309 -13
View File
@@ -12,6 +12,7 @@
#include <string.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_vectors.h>
// External assembly procedures.
extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
@@ -41,7 +42,145 @@ extern void KeIrqStub20(void);
extern void KeIrqStub21(void);
extern void KeIrqStub22(void);
extern void KeIrqStub23(void);
extern void KeIrqStub24(void);
extern void KeIrqStub25(void);
extern void KeIrqStub26(void);
extern void KeIrqStub27(void);
extern void KeIrqStub28(void);
extern void KeIrqStub29(void);
extern void KeIrqStub30(void);
extern void KeIrqStub31(void);
extern void KeIrqStub32(void);
extern void KeIrqStub33(void);
extern void KeIrqStub34(void);
extern void KeIrqStub35(void);
extern void KeIrqStub36(void);
extern void KeIrqStub37(void);
extern void KeIrqStub38(void);
extern void KeIrqStub39(void);
extern void KeIrqStub40(void);
extern void KeIrqStub41(void);
extern void KeIrqStub42(void);
extern void KeIrqStub43(void);
extern void KeIrqStub44(void);
extern void KeIrqStub45(void);
extern void KeIrqStub46(void);
extern void KeIrqStub47(void);
extern void KeIrqStub48(void);
extern void KeIrqStub49(void);
extern void KeIrqStub50(void);
extern void KeIrqStub51(void);
extern void KeIrqStub52(void);
extern void KeIrqStub53(void);
extern void KeIrqStub54(void);
extern void KeIrqStub55(void);
extern void KeIrqStub56(void);
extern void KeIrqStub57(void);
extern void KeIrqStub58(void);
extern void KeIrqStub59(void);
extern void KeIrqStub60(void);
extern void KeIrqStub61(void);
extern void KeIrqStub62(void);
extern void KeIrqStub63(void);
extern void KeIrqStub64(void);
extern void KeIrqStub65(void);
extern void KeIrqStub66(void);
extern void KeIrqStub67(void);
extern void KeIrqStub68(void);
extern void KeIrqStub69(void);
extern void KeIrqStub70(void);
extern void KeIrqStub71(void);
extern void KeIrqStub72(void);
extern void KeIrqStub73(void);
extern void KeIrqStub74(void);
extern void KeIrqStub75(void);
extern void KeIrqStub76(void);
extern void KeIrqStub77(void);
extern void KeIrqStub78(void);
extern void KeIrqStub79(void);
extern void KeIrqStub80(void);
extern void KeIrqStub81(void);
extern void KeIrqStub82(void);
extern void KeIrqStub83(void);
extern void KeIrqStub84(void);
extern void KeIrqStub85(void);
extern void KeIrqStub86(void);
extern void KeIrqStub87(void);
extern void KeIrqStub88(void);
extern void KeIrqStub89(void);
extern void KeIrqStub90(void);
extern void KeIrqStub91(void);
extern void KeIrqStub92(void);
extern void KeIrqStub93(void);
extern void KeIrqStub94(void);
extern void KeIrqStub95(void);
extern void KeIrqStub96(void);
extern void KeIrqStub97(void);
extern void KeIrqStub98(void);
extern void KeIrqStub99(void);
extern void KeIrqStub100(void);
extern void KeIrqStub101(void);
extern void KeIrqStub102(void);
extern void KeIrqStub103(void);
extern void KeIrqStub104(void);
extern void KeIrqStub105(void);
extern void KeIrqStub106(void);
extern void KeIrqStub107(void);
extern void KeIrqStub108(void);
extern void KeIrqStub109(void);
extern void KeIrqStub110(void);
extern void KeIrqStub111(void);
extern void KeIrqStub112(void);
extern void KeIrqStub113(void);
extern void KeIrqStub114(void);
extern void KeIrqStub115(void);
extern void KeIrqStub116(void);
extern void KeIrqStub117(void);
extern void KeIrqStub118(void);
extern void KeIrqStub119(void);
extern void KeIrqStub120(void);
extern void KeIrqStub121(void);
extern void KeIrqStub122(void);
extern void KeIrqStub123(void);
extern void KeIrqStub124(void);
extern void KeIrqStub125(void);
extern void KeIrqStub126(void);
extern void KeIrqStub127(void);
extern void KeIrqStub128(void);
extern void KeIrqStub129(void);
extern void KeIrqStub130(void);
extern void KeIrqStub131(void);
extern void KeIrqStub132(void);
extern void KeIrqStub133(void);
extern void KeIrqStub134(void);
extern void KeIrqStub135(void);
extern void KeIrqStub136(void);
extern void KeIrqStub137(void);
extern void KeIrqStub138(void);
extern void KeIrqStub139(void);
extern void KeIrqStub140(void);
extern void KeIrqStub141(void);
extern void KeIrqStub142(void);
extern void KeIrqStub143(void);
extern void KeIrqStub144(void);
extern void KeIrqStub145(void);
extern void KeIrqStub146(void);
extern void KeIrqStub147(void);
extern void KeIrqStub148(void);
extern void KeIrqStub149(void);
extern void KeIrqStub150(void);
extern void KeIrqStub151(void);
extern void KeIrqStub152(void);
extern void KeIrqStub153(void);
extern void KeIrqStub154(void);
extern void KeIrqStub155(void);
extern void KeIrqStub156(void);
extern void KeIrqStub157(void);
extern void KeIrqStub158(void);
extern void KeIrqStub159(void);
extern void KeRescheduleIpiStub(void);
extern void KeLocalTimerStub(void);
extern void KeSpuriousStub(void);
extern void KeFaultDB(void);
extern void KeFaultNMI(void);
@@ -74,7 +213,7 @@ extern void KeFaultVC(void);
extern void KeFaultSX(void);
extern void KeFaultRSV31(void);
#define NUM_IRQS 24
#define NUM_DEVICE_VECTORS (KE_VECTOR_DEVICE_LAST - KE_VECTOR_DEVICE_FIRST + 1u)
// GDT.
static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
@@ -219,23 +358,180 @@ void KeInitializeIdt(void) {
// User-mode INT3 must enter #BP, not fail the gate-DPL check and become #GP.
KiIdtSetGate(3, (uintptr_t)KeFaultBP, KI_IDT_TYPE_USER_INTERRUPT_GATE, 0);
KiIdtSetGate(0x40, (uintptr_t)KeSyscallEntry, KI_IDT_TYPE_USER_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_SYSCALL, (uintptr_t)KeSyscallEntry, KI_IDT_TYPE_USER_INTERRUPT_GATE, 0);
static void* irqStubs[NUM_IRQS] = {
(void*)KeIrqStub0, (void*)KeIrqStub1, (void*)KeIrqStub2,
(void*)KeIrqStub3, (void*)KeIrqStub4, (void*)KeIrqStub5,
(void*)KeIrqStub6, (void*)KeIrqStub7, (void*)KeIrqStub8,
(void*)KeIrqStub9, (void*)KeIrqStub10, (void*)KeIrqStub11,
(void*)KeIrqStub12, (void*)KeIrqStub13, (void*)KeIrqStub14,
(void*)KeIrqStub15, (void*)KeIrqStub16, (void*)KeIrqStub17,
(void*)KeIrqStub18, (void*)KeIrqStub19, (void*)KeIrqStub20,
(void*)KeIrqStub21, (void*)KeIrqStub22, (void*)KeIrqStub23,
static void* irqStubs[NUM_DEVICE_VECTORS] = {
(void*)KeIrqStub0,
(void*)KeIrqStub1,
(void*)KeIrqStub2,
(void*)KeIrqStub3,
(void*)KeIrqStub4,
(void*)KeIrqStub5,
(void*)KeIrqStub6,
(void*)KeIrqStub7,
(void*)KeIrqStub8,
(void*)KeIrqStub9,
(void*)KeIrqStub10,
(void*)KeIrqStub11,
(void*)KeIrqStub12,
(void*)KeIrqStub13,
(void*)KeIrqStub14,
(void*)KeIrqStub15,
(void*)KeIrqStub16,
(void*)KeIrqStub17,
(void*)KeIrqStub18,
(void*)KeIrqStub19,
(void*)KeIrqStub20,
(void*)KeIrqStub21,
(void*)KeIrqStub22,
(void*)KeIrqStub23,
(void*)KeIrqStub24,
(void*)KeIrqStub25,
(void*)KeIrqStub26,
(void*)KeIrqStub27,
(void*)KeIrqStub28,
(void*)KeIrqStub29,
(void*)KeIrqStub30,
(void*)KeIrqStub31,
(void*)KeIrqStub32,
(void*)KeIrqStub33,
(void*)KeIrqStub34,
(void*)KeIrqStub35,
(void*)KeIrqStub36,
(void*)KeIrqStub37,
(void*)KeIrqStub38,
(void*)KeIrqStub39,
(void*)KeIrqStub40,
(void*)KeIrqStub41,
(void*)KeIrqStub42,
(void*)KeIrqStub43,
(void*)KeIrqStub44,
(void*)KeIrqStub45,
(void*)KeIrqStub46,
(void*)KeIrqStub47,
(void*)KeIrqStub48,
(void*)KeIrqStub49,
(void*)KeIrqStub50,
(void*)KeIrqStub51,
(void*)KeIrqStub52,
(void*)KeIrqStub53,
(void*)KeIrqStub54,
(void*)KeIrqStub55,
(void*)KeIrqStub56,
(void*)KeIrqStub57,
(void*)KeIrqStub58,
(void*)KeIrqStub59,
(void*)KeIrqStub60,
(void*)KeIrqStub61,
(void*)KeIrqStub62,
(void*)KeIrqStub63,
(void*)KeIrqStub64,
(void*)KeIrqStub65,
(void*)KeIrqStub66,
(void*)KeIrqStub67,
(void*)KeIrqStub68,
(void*)KeIrqStub69,
(void*)KeIrqStub70,
(void*)KeIrqStub71,
(void*)KeIrqStub72,
(void*)KeIrqStub73,
(void*)KeIrqStub74,
(void*)KeIrqStub75,
(void*)KeIrqStub76,
(void*)KeIrqStub77,
(void*)KeIrqStub78,
(void*)KeIrqStub79,
(void*)KeIrqStub80,
(void*)KeIrqStub81,
(void*)KeIrqStub82,
(void*)KeIrqStub83,
(void*)KeIrqStub84,
(void*)KeIrqStub85,
(void*)KeIrqStub86,
(void*)KeIrqStub87,
(void*)KeIrqStub88,
(void*)KeIrqStub89,
(void*)KeIrqStub90,
(void*)KeIrqStub91,
(void*)KeIrqStub92,
(void*)KeIrqStub93,
(void*)KeIrqStub94,
(void*)KeIrqStub95,
(void*)KeIrqStub96,
(void*)KeIrqStub97,
(void*)KeIrqStub98,
(void*)KeIrqStub99,
(void*)KeIrqStub100,
(void*)KeIrqStub101,
(void*)KeIrqStub102,
(void*)KeIrqStub103,
(void*)KeIrqStub104,
(void*)KeIrqStub105,
(void*)KeIrqStub106,
(void*)KeIrqStub107,
(void*)KeIrqStub108,
(void*)KeIrqStub109,
(void*)KeIrqStub110,
(void*)KeIrqStub111,
(void*)KeIrqStub112,
(void*)KeIrqStub113,
(void*)KeIrqStub114,
(void*)KeIrqStub115,
(void*)KeIrqStub116,
(void*)KeIrqStub117,
(void*)KeIrqStub118,
(void*)KeIrqStub119,
(void*)KeIrqStub120,
(void*)KeIrqStub121,
(void*)KeIrqStub122,
(void*)KeIrqStub123,
(void*)KeIrqStub124,
(void*)KeIrqStub125,
(void*)KeIrqStub126,
(void*)KeIrqStub127,
(void*)KeIrqStub128,
(void*)KeIrqStub129,
(void*)KeIrqStub130,
(void*)KeIrqStub131,
(void*)KeIrqStub132,
(void*)KeIrqStub133,
(void*)KeIrqStub134,
(void*)KeIrqStub135,
(void*)KeIrqStub136,
(void*)KeIrqStub137,
(void*)KeIrqStub138,
(void*)KeIrqStub139,
(void*)KeIrqStub140,
(void*)KeIrqStub141,
(void*)KeIrqStub142,
(void*)KeIrqStub143,
(void*)KeIrqStub144,
(void*)KeIrqStub145,
(void*)KeIrqStub146,
(void*)KeIrqStub147,
(void*)KeIrqStub148,
(void*)KeIrqStub149,
(void*)KeIrqStub150,
(void*)KeIrqStub151,
(void*)KeIrqStub152,
(void*)KeIrqStub153,
(void*)KeIrqStub154,
(void*)KeIrqStub155,
(void*)KeIrqStub156,
(void*)KeIrqStub157,
(void*)KeIrqStub158,
(void*)KeIrqStub159,
};
for (int i = 0; i < NUM_IRQS; i++)
KiIdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], KI_IDT_TYPE_INTERRUPT_GATE, 0);
for (uint32_t i = 0; i < NUM_DEVICE_VECTORS; i++)
KiIdtSetGate(KE_VECTOR_DEVICE_FIRST + i, (uintptr_t)irqStubs[i],
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_CPU_TIMER, (uintptr_t)KeLocalTimerStub,
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_RESCHEDULE_IPI, (uintptr_t)KeRescheduleIpiStub,
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_SPURIOUS, (uintptr_t)KeSpuriousStub,
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KeLoadIdtForCurrentCpu();
}
code/kernel/ke_irq.c
+599 -98
View File
@@ -1,136 +1,643 @@
// Interrupt manager. This file owns IRQ routing, handler dispatch, and post-interrupt scheduling hooks.
// Interrupt manager. Per-CPU vector ownership is independent of physical sources.
#include <kernel/ke_irq.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_vectors.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <intrin.h>
#define MAX_IRQS 24
#define MAX_IRQ_HANDLERS 8
#define MSI_IRQ_FIRST 16u
#define MSI_IRQ_LAST 23u
#define KI_INTERRUPT_ROUTES 256u
#define KI_INTERRUPT_HANDLERS 512u
typedef struct IRQ_HANDLER_LIST {
void (*Handlers[MAX_IRQ_HANDLERS])(void);
uint8_t Count;
} IRQ_HANDLER_LIST;
typedef struct KI_INTERRUPT_ROUTE KI_INTERRUPT_ROUTE;
struct KE_INTERRUPT {
KE_INTERRUPT_ROUTINE Routine;
void* Context;
KI_INTERRUPT_ROUTE* Route;
KE_INTERRUPT_MESSAGE_PROGRAM Program;
void* ProgramContext;
uint64_t Affinity;
struct KE_INTERRUPT* Next;
bool Used;
};
static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
static bool gIrqSoftwareVector[MAX_IRQS];
struct KI_INTERRUPT_ROUTE {
KE_INTERRUPT* Handlers;
uint32_t Gsi;
uint32_t Cpu;
uint8_t Vector;
uint16_t Flags;
bool Used;
bool Message;
bool Draining;
bool Closing;
bool MessageActive;
volatile long Active;
volatile long Enabled;
volatile uint64_t Delivered;
};
static KI_INTERRUPT_ROUTE gRoutes[KI_INTERRUPT_ROUTES];
static KE_INTERRUPT gHandlers[KI_INTERRUPT_HANDLERS];
static KI_INTERRUPT_ROUTE* volatile gCpuVectors[KE_MAX_CPUS][256];
// A reader is registered before it resolves the route pointer. A retired route
// waits for all readers on its former CPU/vector before releasing handler state.
static volatile long gCpuVectorReaders[KE_MAX_CPUS][256];
// Vector reservations outlive publication while a route is draining.
static bool gCpuVectorReserved[KE_MAX_CPUS][256];
static bool gCpuVectorRetired[KE_MAX_CPUS][256];
static uint8_t gCpuVectorQuietTicks[KE_MAX_CPUS][256];
static SPINLOCK gInterruptLock = SPINLOCK_INIT;
static const HAL_API* gHal;
static void (*gLocalTimerHandlers[2])(void);
static uint8_t gLocalTimerHandlerCount;
static uint32_t gPlacementCursor;
// Implements the irq To Irql operation.
static KIRQL KiIrqToIrql(uint8_t irq) {
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)irq;
if (level >= HIGH_LEVEL)
level = (uint32_t)(HIGH_LEVEL - 1);
static KIRQL KiVectorIrql(uint8_t vector) {
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)((vector >> 4u) - 3u);
if (level >= HIGH_LEVEL) level = HIGH_LEVEL - 1u;
return (KIRQL)level;
}
// Implements the irq Initialize operation.
void KeIrqInitialize(const HAL_API* hal) {
if (!hal || !hal->InterruptController.RouteIrq || !hal->InterruptController.SendEoi)
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid interrupt-controller API during initialization\n");
if (!hal || !hal->InterruptController.SendEoi ||
!hal->InterruptController.RouteGsi || !hal->InterruptController.ResolveIsa ||
!hal->InterruptController.ComposeMsi)
KePanic(KE_PANIC_CODE_INIT, "IRQ: incomplete interrupt-controller API\n");
gHal = hal;
}
// Implements the irq Allocate Msi Vector operation.
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
if (!outIrq) return NTLSTATUS_INVALID_PARAMETER;
for (uint8_t irq = MSI_IRQ_FIRST; irq <= MSI_IRQ_LAST; irq++) {
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
if (!gIrqSoftwareVector[irq] && list->Count == 0) {
gIrqSoftwareVector[irq] = true;
*outIrq = irq;
DbgPrint("KeIrq: allocated MSI vector slot IRQ %u vector 0x%02x\n",
irq, 0x20u + irq);
return NTLSTATUS_SUCCESS;
}
}
return NTLSTATUS_NO_MEMORY;
static bool KiCpuAvailable(uint32_t index, uint64_t affinity) {
if (index >= gCpuCount || index >= KE_MAX_CPUS ||
(affinity && !(affinity & (1ull << index)))) return false;
KCPU* cpu = KeGetCpuByIndex(index);
return cpu && (cpu->IsBootProcessor || cpu->State >= KeCpuSchedulerActive) &&
cpu->HardwareId <= 255u;
}
// Implements the irq Register operation.
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
if (!gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: registration before interrupt-manager initialization\n");
static bool KiPickVector(uint64_t affinity, uint32_t* cpuOut, uint8_t* vectorOut) {
uint32_t count = gCpuCount;
if (!count || count > KE_MAX_CPUS) count = 1;
// Rotate the first candidate CPU, and allocate the least occupied vector.
uint32_t start = gPlacementCursor++ % count;
uint32_t bestLoad = UINT32_MAX;
bool found = false;
for (uint32_t i = 0; i < count; i++) {
uint32_t cpu = (start + i) % count;
if (!KiCpuAvailable(cpu, affinity)) continue;
uint32_t occupied = 0;
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++)
if (gCpuVectorReserved[cpu][v]) occupied++;
if (occupied >= KE_VECTOR_DEVICE_LAST - KE_VECTOR_DEVICE_FIRST + 1u || occupied >= bestLoad)
continue;
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
if (!gCpuVectorReserved[cpu][v]) {
bestLoad = occupied;
*cpuOut = cpu;
*vectorOut = (uint8_t)v;
found = true;
break;
}
}
}
return found;
}
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
for (uint8_t i = 0; i < list->Count; i++) {
if (list->Handlers[i] == handler)
return;
static KSTATUS KiReserveInterrupt(bool message, uint32_t gsi, uint16_t flags,
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context,
KE_INTERRUPT** out) {
if (!out || !routine || !gHal) return NTLSTATUS_INVALID_PARAMETER;
*out = (KE_INTERRUPT*)0;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)0;
if (!message) {
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
if (gRoutes[i].Used && !gRoutes[i].Message && gRoutes[i].Gsi == gsi) {
if (gRoutes[i].Draining || gRoutes[i].Closing) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
if (gRoutes[i].Flags != flags ||
(affinity && !(affinity & (1ull << gRoutes[i].Cpu)))) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
route = &gRoutes[i];
break;
}
}
}
bool newRoute = route == 0;
uint32_t cpu = 0;
uint8_t vector = 0;
if (newRoute) {
if (!KiPickVector(affinity, &cpu, &vector)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++)
if (!gRoutes[i].Used) { route = &gRoutes[i]; break; }
if (!route) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
}
KE_INTERRUPT* handler = 0;
for (uint32_t i = 0; i < KI_INTERRUPT_HANDLERS; i++)
if (!gHandlers[i].Used) { handler = &gHandlers[i]; break; }
if (!handler) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
if (newRoute) {
route->Used = true;
route->Message = message;
route->Draining = false;
route->Closing = false;
route->MessageActive = false;
route->Gsi = gsi;
route->Cpu = cpu;
route->Vector = vector;
route->Flags = flags;
route->Handlers = 0;
route->Active = 0;
route->Enabled = 0;
route->Delivered = 0;
gCpuVectorReserved[cpu][vector] = true;
gCpuVectorRetired[cpu][vector] = false;
gCpuVectorQuietTicks[cpu][vector] = 0;
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
}
handler->Used = true;
handler->Routine = routine;
handler->Context = context;
handler->Route = route;
handler->Program = 0;
handler->ProgramContext = 0;
handler->Affinity = affinity;
handler->Next = route->Handlers;
route->Handlers = handler;
KeReleaseSpinlock(&gInterruptLock, cookie);
if (newRoute && !message) {
KCPU* target = KeGetCpuByIndex(cpu);
KSTATUS st = gHal->InterruptController.RouteGsi(gsi, vector,
target->HardwareId, flags, true);
if (!KSUCCESS(st)) {
KeInterruptDisconnect(handler);
return st;
}
_InterlockedExchange(&route->Enabled, 1);
(void)gHal->InterruptController.MaskGsi(gsi, false);
} else if (newRoute) {
_InterlockedExchange(&route->Enabled, 1);
}
*out = handler;
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeInterruptConnectGsi(uint32_t gsi, uint16_t flags,
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
}
__declspec(dllexport) KSTATUS KeInterruptConnectIsa(uint8_t isaIrq, uint16_t sourceFlags, uint64_t affinity,
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
uint32_t gsi;
uint16_t flags;
if (!gHal || !gHal->InterruptController.ResolveIsa)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = gHal->InterruptController.ResolveIsa(isaIrq, &gsi, &flags);
if (!KSUCCESS(st)) return st;
if (sourceFlags) flags = sourceFlags;
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
}
__declspec(dllexport) KSTATUS KeInterruptReserveMessage(uint64_t affinity,
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
return KiReserveInterrupt(true, UINT32_MAX, 0, affinity, routine, context, out);
}
__declspec(dllexport) KSTATUS KeInterruptComposeMessage(const KE_INTERRUPT* interrupt,
NTL_DDK_MSI_MESSAGE* out) {
if (!interrupt || !out || !gHal) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route ||
!interrupt->Route->Message || interrupt->Route->Draining) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
HAL_MSI_MESSAGE message;
KSTATUS st = cpu ? gHal->InterruptController.ComposeMsi(
cpu->HardwareId, route->Vector, &message) : NTLSTATUS_INVALID_PARAMETER;
KeReleaseSpinlock(&gInterruptLock, cookie);
if (!KSUCCESS(st)) return st;
out->Address = message.Address;
out->Data = message.Data;
out->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeInterruptQuery(const KE_INTERRUPT* interrupt,
KE_INTERRUPT_INFO* out) {
if (!interrupt || !out) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route || interrupt->Route->Draining) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
out->CpuIndex = route->Cpu;
out->HardwareId = cpu ? cpu->HardwareId : UINT32_MAX;
out->Gsi = route->Gsi;
out->Flags = route->Flags;
out->Vector = route->Vector;
out->IsMessage = route->Message ? 1u : 0u;
out->Delivered = route->Delivered;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
// A message device registers its update operation once PCI has enabled the source.
__declspec(dllexport) KSTATUS KeInterruptBindMessageSource(KE_INTERRUPT* interrupt,
KE_INTERRUPT_MESSAGE_PROGRAM program, void* context) {
if (!interrupt || !program) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
(interrupt->Route->Draining || interrupt->Route->Closing) || interrupt->Program) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
interrupt->Program = program;
interrupt->ProgramContext = context;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
// The bus owner enables migration only after the device source is programmed.
__declspec(dllexport) KSTATUS KeInterruptActivateMessage(KE_INTERRUPT* interrupt) {
if (!interrupt) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
(interrupt->Route->Draining || interrupt->Route->Closing) || !interrupt->Program) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
interrupt->Route->MessageActive = true;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
// Moves the complete physical source to one new CPU/vector, including PCI
// message programming. The old vector remains quarantined through APIC drain.
__declspec(dllexport) KSTATUS KeInterruptSetAffinity(KE_INTERRUPT* interrupt,
uint64_t permittedCpuMask) {
if (!interrupt || !permittedCpuMask || !gHal ||
KeGetCurrentIrql() != PASSIVE_LEVEL || KeGetCurrentCpu()->InterruptDepth)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route ||
interrupt->Route->Draining || interrupt->Route->Closing) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
KI_INTERRUPT_ROUTE* route = interrupt->Route;
if (route->Message && !route->MessageActive) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
// One shared line must satisfy every handler's affinity constraint.
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next) {
if (h->Affinity && !(h->Affinity & permittedCpuMask)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
if (route->Message && (!h->Program || h != interrupt)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
}
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
if (h->Affinity) permittedCpuMask &= h->Affinity;
if (!permittedCpuMask) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
if (permittedCpuMask & (1ull << route->Cpu)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
uint32_t newCpu = 0;
uint8_t newVector = 0;
if (!KiPickVector(permittedCpuMask, &newCpu, &newVector)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
const uint32_t oldCpu = route->Cpu;
const uint8_t oldVector = route->Vector;
gCpuVectorReserved[newCpu][newVector] = true;
route->Draining = true;
KeReleaseSpinlock(&gInterruptLock, cookie);
NTL_DDK_MSI_MESSAGE updated = {0};
if (route->Message) {
const KCPU* destination = KeGetCpuByIndex(newCpu);
HAL_MSI_MESSAGE message;
KSTATUS st = gHal->InterruptController.ComposeMsi(destination->HardwareId,
newVector, &message);
if (!KSUCCESS(st)) goto rollback_reservation;
updated.Address = message.Address;
updated.Data = message.Data;
st = interrupt->Program(interrupt->ProgramContext, KeInterruptMessageQuiesce, 0);
if (!KSUCCESS(st)) goto rollback_reservation;
} else {
KSTATUS st = gHal->InterruptController.MaskGsi(route->Gsi, true);
if (!KSUCCESS(st)) goto rollback_reservation;
}
if (list->Count >= MAX_IRQ_HANDLERS) {
DbgPrint("KeIrqRegister: IRQ %u handler table full\n", irq);
_InterlockedExchange(&route->Enabled, 0);
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], 0);
uint32_t spins = 0;
while (route->Active || gCpuVectorReaders[oldCpu][oldVector]) {
if (++spins == 100000000u)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: affinity migration drain timeout\n");
_mm_pause();
}
KSTATUS update = NTLSTATUS_SUCCESS;
if (route->Message)
update = interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageProgram, &updated);
else {
const KCPU* destination = KeGetCpuByIndex(newCpu);
update = gHal->InterruptController.RouteGsi(route->Gsi, newVector,
destination->HardwareId, route->Flags, true);
}
if (!KSUCCESS(update)) {
// Restore the existing route and re-enable its original source.
if (route->Message) {
const KCPU* previous = KeGetCpuByIndex(oldCpu);
HAL_MSI_MESSAGE msg;
if (!KSUCCESS(gHal->InterruptController.ComposeMsi(previous->HardwareId,
oldVector, &msg)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original MSI route\n");
NTL_DDK_MSI_MESSAGE original = {msg.Address, msg.Data, 0};
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageProgram, &original)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore MSI message\n");
} else {
const KCPU* previous = KeGetCpuByIndex(oldCpu);
if (!KSUCCESS(gHal->InterruptController.RouteGsi(route->Gsi, oldVector,
previous->HardwareId, route->Flags, true)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original GSI route\n");
}
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], route);
_InterlockedExchange(&route->Enabled, 1);
if (route->Message) {
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageResume, 0)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI restore resume failed\n");
} else (void)gHal->InterruptController.MaskGsi(route->Gsi, false);
cookie = KeAcquireSpinlock(&gInterruptLock);
route->Draining = false;
gCpuVectorReserved[newCpu][newVector] = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
return update;
}
cookie = KeAcquireSpinlock(&gInterruptLock);
route->Cpu = newCpu;
route->Vector = newVector;
gCpuVectorRetired[oldCpu][oldVector] = true;
gCpuVectorQuietTicks[oldCpu][oldVector] = 0;
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[newCpu][newVector], route);
_InterlockedExchange(&route->Enabled, 1);
KeReleaseSpinlock(&gInterruptLock, cookie);
if (route->Message) {
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageResume, 0)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI migration resume failed\n");
} else if (!KSUCCESS(gHal->InterruptController.MaskGsi(route->Gsi, false)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: GSI migration unmask failed\n");
cookie = KeAcquireSpinlock(&gInterruptLock);
route->Draining = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
rollback_reservation:
cookie = KeAcquireSpinlock(&gInterruptLock);
gCpuVectorReserved[newCpu][newVector] = false;
route->Draining = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
// The scheduler must evacuate every device route before stopping a processor.
KSTATUS KiInterruptPrepareCpuOffline(uint32_t cpuIndex) {
if (cpuIndex >= gCpuCount || cpuIndex >= KE_MAX_CPUS ||
KeGetCurrentIrql() != PASSIVE_LEVEL) return NTLSTATUS_INVALID_PARAMETER;
uint64_t allowed = 0;
for (uint32_t i = 0; i < gCpuCount && i < 64u; i++)
if (i != cpuIndex && KiCpuAvailable(i, 0)) allowed |= 1ull << i;
if (!allowed) return NTLSTATUS_NOT_SUPPORTED;
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
KI_INTERRUPT_ROUTE* route = &gRoutes[i];
if (route->Used && route->Cpu == cpuIndex && route->Handlers) {
KSTATUS st = KeInterruptSetAffinity(route->Handlers, allowed);
if (!KSUCCESS(st)) return st;
}
}
return NTLSTATUS_SUCCESS;
}
// Serializes device-source disable with affinity migration. A caller must
// close the route before masking the PCI source, preventing a racing resume.
__declspec(dllexport) void KeInterruptBeginDisconnect(KE_INTERRUPT* interrupt) {
if (!interrupt || KeGetCurrentIrql() != PASSIVE_LEVEL ||
KeGetCurrentCpu()->InterruptDepth)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: invalid disconnect preparation context\n");
for (uint32_t spins = 0;; spins++) {
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return;
}
KI_INTERRUPT_ROUTE* route = interrupt->Route;
if (!route->Draining && !route->Closing) {
route->Closing = true;
KeReleaseSpinlock(&gInterruptLock, cookie);
return;
}
KeReleaseSpinlock(&gInterruptLock, cookie);
if (spins >= 100000000u)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect serialization timed out\n");
_mm_pause();
}
}
// Disconnect runs in thread context after the device has stopped producing messages.
// The route is unpublished before draining readers, so a concurrent dispatcher
// either holds a counted reference or observes an empty vector.
__declspec(dllexport) void KeInterruptDisconnect(KE_INTERRUPT* interrupt) {
if (!interrupt) return;
if (KeGetCurrentCpu()->InterruptDepth || KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect requires PASSIVE_LEVEL\n");
// PCI callers have already serialized source disable. A direct GSI
// caller prepares the route here before masking it inside this routine.
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return;
}
KI_INTERRUPT_ROUTE* route = interrupt->Route;
if (!route->Closing) {
KeReleaseSpinlock(&gInterruptLock, cookie);
KeInterruptBeginDisconnect(interrupt);
cookie = KeAcquireSpinlock(&gInterruptLock);
route = interrupt->Route;
if (!route) { KeReleaseSpinlock(&gInterruptLock, cookie); return; }
}
if (route->Draining) {
KeReleaseSpinlock(&gInterruptLock, cookie);
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: unexpected concurrent route drain\n");
}
route->Draining = true;
_InterlockedExchange(&route->Enabled, 0);
if (!route->Message)
(void)gHal->InterruptController.MaskGsi(route->Gsi, true);
const uint32_t cpu = route->Cpu;
const uint8_t vector = route->Vector;
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], 0);
KeReleaseSpinlock(&gInterruptLock, cookie);
list->Handlers[list->Count++] = handler;
if (!gIrqSoftwareVector[irq])
gHal->InterruptController.RouteIrq(irq);
}
// Implements the irq Dispatch operation.
void KeIrqDispatch(uint8_t irq) {
if (!gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: dispatch before interrupt-manager initialization, irq=%u\n", irq);
KIRQL targetIrql = KiIrqToIrql(irq);
KIRQL currentIrql = KeGetCurrentIrql();
// Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected
// while IF is clear, but preserving monotonic IRQL keeps nesting safe.
if (targetIrql < currentIrql)
targetIrql = currentIrql;
KIRQL oldIrql = KeRaiseIrql(targetIrql);
if (irq == KE_VECTOR_RESCHEDULE_IPI) {
// Nothing to do here. The sender has already set this CPU's
// ReschedulePending flag. KiExitInterrupt consumes it after EOI and
// after the interrupted frame unwinds to PASSIVE_LEVEL.
} else if (irq < MAX_IRQS) {
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
for (uint8_t i = 0; i < list->Count; i++) {
if (list->Handlers[i])
list->Handlers[i]();
}
// Drain both ISRs already invoking callbacks and readers that fetched the
// route just before the unpublish. Reserved vectors remain unavailable.
uint32_t spins = 0;
while (route->Active || gCpuVectorReaders[cpu][vector]) {
if (++spins == 100000000u)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: route rundown timed out cpu=%u vector=%02x\n",
cpu, vector);
_mm_pause();
}
// Handlers are no longer allowed to context-switch directly. Timer IRQs
// only request rescheduling; KiExitInterrupt performs the switch after this
// function returns, so the EOI is guaranteed to be sent first. Keep the
// software level at DIRQL until after the controller has been acknowledged.
gHal->InterruptController.SendEoi();
cookie = KeAcquireSpinlock(&gInterruptLock);
KE_INTERRUPT** link = &route->Handlers;
while (*link && *link != interrupt) link = &(*link)->Next;
if (*link != interrupt)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: missing handler during disconnect\n");
*link = interrupt->Next;
interrupt->Used = false;
interrupt->Route = 0;
interrupt->Routine = 0;
interrupt->Context = 0;
interrupt->Program = 0;
interrupt->ProgramContext = 0;
interrupt->Affinity = 0;
interrupt->Next = 0;
if (!route->Handlers) {
route->Used = false;
// Retain the vector while the target APIC drains any pending delivery.
// Only its target CPU can inspect its IRR and ISR registers.
gCpuVectorRetired[cpu][vector] = true;
gCpuVectorQuietTicks[cpu][vector] = 0;
} else {
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
_InterlockedExchange(&route->Enabled, 1);
if (!route->Message)
(void)gHal->InterruptController.MaskGsi(route->Gsi, false);
}
route->Draining = false;
route->Closing = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
}
// The local timer gives a retiring vector a grace period on its owning CPU.
// Source drivers must disable message generation before disconnecting. A
// vector is reusable after consecutive timer checks find no APIC delivery.
static void KiReapRetiredVectors(void) {
const uint32_t cpu = KeGetCurrentCpuIndex();
if (cpu >= KE_MAX_CPUS) return;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
if (!gCpuVectorRetired[cpu][v]) continue;
if (gCpuVectorReaders[cpu][v] ||
gHal->InterruptController.VectorPending((uint8_t)v)) {
gCpuVectorQuietTicks[cpu][v] = 0;
} else if (++gCpuVectorQuietTicks[cpu][v] >= 3u) {
gCpuVectorRetired[cpu][v] = false;
gCpuVectorReserved[cpu][v] = false;
gCpuVectorQuietTicks[cpu][v] = 0;
}
}
KeReleaseSpinlock(&gInterruptLock, cookie);
}
void KeIrqRegisterLocalTimerHandler(void (*handler)(void)) {
if (!handler || !gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid local timer registration\n");
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
if (gLocalTimerHandlers[i] == handler) return;
if (gLocalTimerHandlerCount >= 2u)
KePanic(KE_PANIC_CODE_INIT, "IRQ: local timer handler count exceeded\n");
gLocalTimerHandlers[gLocalTimerHandlerCount++] = handler;
}
void KeIrqDispatch(uint8_t vector) {
if (!gHal) KePanic(KE_PANIC_CODE_INIT, "IRQ: uninitialized dispatch\n");
if (vector == KE_VECTOR_SPURIOUS) return;
KIRQL target = (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST)
? KiVectorIrql(vector) : (vector == KE_VECTOR_CPU_TIMER ? (KIRQL)27 : (KIRQL)29);
if (target < KeGetCurrentIrql()) target = KeGetCurrentIrql();
KIRQL oldIrql = KeRaiseIrql(target);
if (vector == KE_VECTOR_CPU_TIMER) {
KiReapRetiredVectors();
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
gLocalTimerHandlers[i]();
} else if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
uint32_t cpu = KeGetCurrentCpuIndex();
if (cpu < KE_MAX_CPUS) {
_InterlockedIncrement(&gCpuVectorReaders[cpu][vector]);
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)
_InterlockedCompareExchangePointer(
(void* volatile*)&gCpuVectors[cpu][vector], 0, 0);
if (route) {
_InterlockedIncrement(&route->Active);
if (route->Enabled) {
route->Delivered++;
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
if (h->Used && h->Routine) h->Routine(h->Context);
}
_InterlockedDecrement(&route->Active);
}
}
}
gHal->InterruptController.SendEoi();
if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
uint32_t cpu = KeGetCurrentCpuIndex();
if (cpu < KE_MAX_CPUS)
_InterlockedDecrement(&gCpuVectorReaders[cpu][vector]);
}
KeLowerIrql(oldIrql);
}
// Implements the enter Interrupt operation.
void KiEnterInterrupt(void) {
KeGetCurrentCpu()->InterruptDepth++;
}
// Implements the exit Interrupt operation.
void KiExitInterrupt(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu->InterruptDepth)
cpu->InterruptDepth--;
// A reschedule is only legal after the interrupt unwinds back to the
// interrupted thread's PASSIVE context. If an interrupt hit DISPATCH/DIRQL
// code, leave the request pending instead of switching out of a spinlock or
// high-IRQL region.
if (cpu->InterruptDepth) cpu->InterruptDepth--;
if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
THREAD* current = cpu->CurrentThread;
if (current && !current->IsIdleThread && current->Owner &&
@@ -140,13 +647,7 @@ void KiExitInterrupt(void) {
KeExitThread(current, current->Owner->ExitCode);
while (true) ;
}
KiDrainDpcQueue();
// KeExitThread owns a terminal stack/rundown interval. A timer or IPI
// may interrupt it, but switching away here would abandon teardown
// before the process ExitEvent is necessarily published. The explicit
// KeYield at the end of KeExitThread performs the one terminal switch.
current = cpu->CurrentThread;
if (KeGetCurrentIrql() == PASSIVE_LEVEL &&
(!current || !KeThreadExitInProgress(current)))
code/kernel/ke_irql.c
+25 -3
View File
@@ -1,8 +1,9 @@
// Kernel IRQL model. This file owns the software interrupt-priority state used by IRQ and scheduler code.
// Kernel IRQL model. This file synchronizes per-CPU IRQL with APIC task priority.
#include <stdbool.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_hal.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_apc.h>
#include <intrin.h>
@@ -26,6 +27,27 @@ __declspec(dllexport) KIRQL KeGetCurrentIrql(void) {
return KeGetCurrentCpu()->CurrentIrql;
}
// IRQL classes 16-25 map to vectors 0x30-0xC0. The TPR masks the current
// class and all lower-priority external interrupts. HIGH_LEVEL masks all.
static uint8_t KiIrqlTaskPriority(KIRQL level) {
if (level == HIGH_LEVEL) return 0xFFu;
if (level >= 28u) return 0xF0u;
if (level >= DIRQL_BASE) return (uint8_t)((level - DIRQL_BASE + 3u) << 4u);
if (level >= DISPATCH_LEVEL) return 0x20u;
if (level >= APC_LEVEL) return 0x10u;
return 0u;
}
// Applies the priority before publishing a raised IRQL and after a lowered IRQL.
static void KiSetIrql(KCPU* cpu, KIRQL level, bool raising) {
const HAL_API* hal = KeGetHal();
if (raising && hal && hal->InterruptController.SetTaskPriority)
hal->InterruptController.SetTaskPriority(KiIrqlTaskPriority(level));
cpu->CurrentIrql = level;
if (!raising && hal && hal->InterruptController.SetTaskPriority)
hal->InterruptController.SetTaskPriority(KiIrqlTaskPriority(level));
}
// Implements the raise Irql operation.
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
KCPU* cpu = KeGetCurrentCpu();
@@ -41,7 +63,7 @@ __declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
KeIrqlName(newIrql), (unsigned)newIrql);
}
cpu->CurrentIrql = newIrql;
KiSetIrql(cpu, newIrql, true);
return oldIrql;
}
@@ -60,7 +82,7 @@ __declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KeIrqlName(oldIrql), (unsigned)oldIrql);
}
cpu->CurrentIrql = oldIrql;
KiSetIrql(cpu, oldIrql, false);
if (oldIrql == PASSIVE_LEVEL && currentIrql != PASSIVE_LEVEL &&
(__readeflags() & 0x200u) != 0 && cpu->InterruptDepth == 0)
code/kernel/ke_power.c
+85 -48
View File
@@ -1,14 +1,13 @@
#include <ntlite/atomic.h>
// Kernel power coordinator and optional firmware-provider binding.
#include <ntlite/syscall.h>
#include <ntlite/ddk/power.h>
#include <kernel/ke_power.h>
#include <kernel/ke_process.h>
#include <kernel/ke_event.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irq.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_settings.h>
#include <kernel/ke_debug.h>
#include <kernel/hal/hal_api.h>
#include <string.h>
static SPINLOCK gPowerLock = SPINLOCK_INIT;
@@ -17,9 +16,12 @@ static EVENT* gPowerComplete;
static PROCESS* gInit;
static uint32_t gAction;
static bool gCommitted;
static const HAL_API* gButtonHal;
static bool gPolicyReady;
static bool gEarlyButtonPressed;
static uint32_t gButtonAction;
static KDPC gPowerButtonDpc;
static NTL_FIRMWARE_POWER_PROVIDER gFirmwareProvider;
static bool gFirmwarePrepared;
static uint32_t gPreparedAction;
KSTATUS KePowerInitialize(void) {
if (gPowerRequest) return NTLSTATUS_ALREADY_EXISTS;
@@ -29,58 +31,93 @@ KSTATUS KePowerInitialize(void) {
return st;
}
// Delivers the physical action after interrupt-exit processing.
static void KiPowerButtonDpc(KDPC* dpc, void* context, void* arg1, void* arg2) {
(void)dpc;
(void)context;
(void)arg1;
(void)arg2;
(void)KePowerRequestHardware(gButtonAction);
// The resident firmware owner may be ACPICA or another platform implementation.
__declspec(dllexport) KSTATUS KeRegisterFirmwarePowerProvider(
const NTL_FIRMWARE_POWER_PROVIDER* provider) {
if (!provider || provider->Size != sizeof(*provider) ||
!provider->Prepare || !provider->Enter)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
KSTATUS status = gFirmwareProvider.Size ? NTLSTATUS_ALREADY_EXISTS : NTLSTATUS_SUCCESS;
if (KSUCCESS(status)) gFirmwareProvider = *provider;
KeReleaseSpinlock(&gPowerLock, flags);
if (KSUCCESS(status)) DbgPrint("power: firmware provider registered\n");
return status;
}
// The ACPI SCI handler only acknowledges the fixed event and queues normal work.
static void KiPowerButtonIsr(void) {
if (!gButtonHal || !gButtonHal->Power.ConsumeFixedButton ||
!gButtonHal->Power.ConsumeFixedButton()) return;
(void)KeInsertQueueDpc(&gPowerButtonDpc, NULL, NULL);
// Events before system-root policy becomes available are retained once.
__declspec(dllexport) KSTATUS KeNotifyPowerButton(void) {
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
if (!gPowerRequest) {
KeReleaseSpinlock(&gPowerLock, flags);
return NTLSTATUS_NOT_SUPPORTED;
}
if (!gPolicyReady) {
gEarlyButtonPressed = true;
KeReleaseSpinlock(&gPowerLock, flags);
return NTLSTATUS_SUCCESS;
}
uint32_t action = gButtonAction;
KeReleaseSpinlock(&gPowerLock, flags);
return action ? KePowerRequestHardware(action) : NTLSTATUS_SUCCESS;
}
// Reads immutable boot-time button policy and attaches it to the fixed ACPI SCI.
void KePowerInitializeButton(const HAL_API* hal) {
// System-root policy is loaded independently of the presence of a firmware driver.
void KePowerInitializePolicy(void) {
char mode[24];
KSTATUS status = KeGetSettingString("power", "button", mode, sizeof(mode));
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_INIT, "power: missing [power] button, status=0x%x\n", status);
if (strcmp(mode, "ignore") == 0) {
DbgPrint("power: ACPI power button ignored by policy\n");
return;
}
if (strcmp(mode, "shutdown") == 0) gButtonAction = SYS_POWER_ACTION_SHUTDOWN;
else if (strcmp(mode, "restart") == 0) gButtonAction = SYS_POWER_ACTION_RESTART;
else KePanic(KE_PANIC_CODE_INIT, "power: invalid [power] button=%s\n", mode);
uint32_t action = 0;
if (strcmp(mode, "shutdown") == 0) action = SYS_POWER_ACTION_SHUTDOWN;
else if (strcmp(mode, "restart") == 0) action = SYS_POWER_ACTION_RESTART;
else if (strcmp(mode, "ignore") != 0)
KePanic(KE_PANIC_CODE_INIT, "power: invalid [power] button=%s\n", mode);
if (!hal || !hal->Power.QueryFixedButtonIrq || !hal->Power.ArmFixedButton ||
!hal->Power.ConsumeFixedButton) {
DbgPrint("power: no ACPI fixed power button HAL support\n");
return;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
gButtonAction = action;
gPolicyReady = true;
bool pending = gEarlyButtonPressed;
bool providerPresent = gFirmwareProvider.Size != 0;
gEarlyButtonPressed = false;
KeReleaseSpinlock(&gPowerLock, flags);
DbgPrint("power: button policy=%s firmware provider=%s\n",
mode, providerPresent ? "present" : "absent");
if (pending && action) (void)KePowerRequestHardware(action);
}
// A provider prepares firmware while threads, device resources and AML can run.
KSTATUS KePowerPrepareFirmware(uint32_t action) {
if (action != SYS_POWER_ACTION_SHUTDOWN && action != SYS_POWER_ACTION_RESTART)
return NTLSTATUS_INVALID_PARAMETER;
NTL_FIRMWARE_POWER_PROVIDER provider;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
provider = gFirmwareProvider;
KeReleaseSpinlock(&gPowerLock, flags);
if (!provider.Size) return NTLSTATUS_NOT_SUPPORTED;
flags = KeAcquireSpinlock(&gPowerLock);
gFirmwarePrepared = false;
gPreparedAction = 0;
KeReleaseSpinlock(&gPowerLock, flags);
KSTATUS status = provider.Prepare(provider.Context, action);
if (KSUCCESS(status)) {
flags = KeAcquireSpinlock(&gPowerLock);
gFirmwarePrepared = true;
gPreparedAction = action;
KeReleaseSpinlock(&gPowerLock, flags);
}
uint8_t irq = 0;
status = hal->Power.QueryFixedButtonIrq(&irq);
if (!KSUCCESS(status)) {
DbgPrint("power: ACPI fixed button unavailable (0x%x); AML button not yet supported\n", status);
return;
}
// The IOAPIC is still masked for this IRQ. Arm the firmware source first,
// then install the kernel handler and finally unmask via KeIrqRegister.
status = hal->Power.ArmFixedButton();
if (!KSUCCESS(status)) {
DbgPrint("power: cannot arm fixed button IRQ %u, status=0x%x\n", irq, status);
return;
}
KeInitializeDpc(&gPowerButtonDpc, KiPowerButtonDpc, NULL);
gButtonHal = hal;
KeIrqRegister(irq, KiPowerButtonIsr);
DbgPrint("power: ACPI fixed button armed on IRQ %u, action=%s\n", irq, mode);
return status;
}
// Any returned transition leaves the HAL as the supported final fallback.
KSTATUS KePowerEnterFirmware(uint32_t action) {
NTL_FIRMWARE_POWER_PROVIDER provider;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
bool prepared = gFirmwarePrepared && gPreparedAction == action;
provider = gFirmwareProvider;
KeReleaseSpinlock(&gPowerLock, flags);
if (!prepared || !provider.Size) return NTLSTATUS_NOT_SUPPORTED;
return provider.Enter(provider.Context, action);
}
void KePowerSetInitProcess(PROCESS* init) {
code/kernel/ke_syscall.asm
+154 -4
View File
@@ -1,4 +1,4 @@
; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64)
; ke_syscall.asm - INT 0x2F, fault, and IRQ kernel entries (MASM64)
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
@@ -64,7 +64,7 @@ KeSyscallEntry PROC
mov rax, KTRAP_FRAME_MAGIC
mov [rbx + TF_Magic], rax
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
mov dword ptr [rbx + TF_Vector], 40h
mov dword ptr [rbx + TF_Vector], 02Fh
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
mov [rbx + TF_Rcx], rcx
@@ -781,7 +781,7 @@ IrqStub MACRO num
PUBLIC KeIrqStub&num
KeIrqStub&num PROC
push rax
mov al, num
mov al, (030h + num)
jmp KeIrqCommon
KeIrqStub&num ENDP
ENDM
@@ -810,6 +810,156 @@ IrqStub 20
IrqStub 21
IrqStub 22
IrqStub 23
IrqStub 24
IrqStub 25
IrqStub 26
IrqStub 27
IrqStub 28
IrqStub 29
IrqStub 30
IrqStub 31
IrqStub 32
IrqStub 33
IrqStub 34
IrqStub 35
IrqStub 36
IrqStub 37
IrqStub 38
IrqStub 39
IrqStub 40
IrqStub 41
IrqStub 42
IrqStub 43
IrqStub 44
IrqStub 45
IrqStub 46
IrqStub 47
IrqStub 48
IrqStub 49
IrqStub 50
IrqStub 51
IrqStub 52
IrqStub 53
IrqStub 54
IrqStub 55
IrqStub 56
IrqStub 57
IrqStub 58
IrqStub 59
IrqStub 60
IrqStub 61
IrqStub 62
IrqStub 63
IrqStub 64
IrqStub 65
IrqStub 66
IrqStub 67
IrqStub 68
IrqStub 69
IrqStub 70
IrqStub 71
IrqStub 72
IrqStub 73
IrqStub 74
IrqStub 75
IrqStub 76
IrqStub 77
IrqStub 78
IrqStub 79
IrqStub 80
IrqStub 81
IrqStub 82
IrqStub 83
IrqStub 84
IrqStub 85
IrqStub 86
IrqStub 87
IrqStub 88
IrqStub 89
IrqStub 90
IrqStub 91
IrqStub 92
IrqStub 93
IrqStub 94
IrqStub 95
IrqStub 96
IrqStub 97
IrqStub 98
IrqStub 99
IrqStub 100
IrqStub 101
IrqStub 102
IrqStub 103
IrqStub 104
IrqStub 105
IrqStub 106
IrqStub 107
IrqStub 108
IrqStub 109
IrqStub 110
IrqStub 111
IrqStub 112
IrqStub 113
IrqStub 114
IrqStub 115
IrqStub 116
IrqStub 117
IrqStub 118
IrqStub 119
IrqStub 120
IrqStub 121
IrqStub 122
IrqStub 123
IrqStub 124
IrqStub 125
IrqStub 126
IrqStub 127
IrqStub 128
IrqStub 129
IrqStub 130
IrqStub 131
IrqStub 132
IrqStub 133
IrqStub 134
IrqStub 135
IrqStub 136
IrqStub 137
IrqStub 138
IrqStub 139
IrqStub 140
IrqStub 141
IrqStub 142
IrqStub 143
IrqStub 144
IrqStub 145
IrqStub 146
IrqStub 147
IrqStub 148
IrqStub 149
IrqStub 150
IrqStub 151
IrqStub 152
IrqStub 153
IrqStub 154
IrqStub 155
IrqStub 156
IrqStub 157
IrqStub 158
IrqStub 159
PUBLIC KeLocalTimerStub
KeLocalTimerStub PROC
push rax
mov al, 0E0h
jmp KeIrqCommon
KeLocalTimerStub ENDP
PUBLIC KeSpuriousStub
KeSpuriousStub PROC
push rax
mov al, 0FFh
jmp KeIrqCommon
KeSpuriousStub ENDP
PUBLIC KeRescheduleIpiStub
KeRescheduleIpiStub PROC
@@ -819,7 +969,7 @@ KeRescheduleIpiStub PROC
KeRescheduleIpiStub ENDP
KeIrqCommon PROC
; al = irq number, original rax saved by the per-IRQ stub.
; al = IDT vector, original rax saved by the vector stub.
push rcx
push rdx
push r8
code/kernel/ke_syscall.c
+1 -1
View File
@@ -1,6 +1,6 @@
// Kernel syscall dispatcher. This file validates user requests and bridges user-mode ABI calls into kernel services.
// KeSyscallEntry (ke_syscall.asm) handles the INT 0x40 ring-3 -> ring-0
// KeSyscallEntry (ke_syscall.asm) handles the INT 0x2F ring-3 -> ring-0
// transition, re-orders the arguments to match MS x64, and calls
// KeSyscallDispatch with the syscall number plus at most four register arguments.
// Compound operations use size-tagged request structures instead of extending
code/kernel/kmain.c
+14 -6
View File
@@ -16,6 +16,7 @@
#include <kernel/ob_object.h>
#include <kernel/ke_process.h>
#include <kernel/ke_power.h>
#include <kernel/ke_firmware.h>
#include <kernel/mm_file_cache.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
@@ -554,13 +555,12 @@ void SystemInitThread(void* arg) {
DbgPrint("SystemInitThread: Thread started. Igniting the system...\n");
KiInitializeDrivers(args->BootInfo, args->Hal);
KSTATUS powerStatus = KePowerInitialize();
if (!KSUCCESS(powerStatus))
KePanic(KE_PANIC_CODE_INIT, "Power coordinator failed (0x%x)\n", powerStatus);
KePowerInitializeButton(args->Hal);
KiInitializeDrivers(args->BootInfo, args->Hal);
KePowerInitializePolicy();
KeTimeReloadSettings();
KiInitializeSwapFromSettings();
@@ -604,12 +604,19 @@ void SystemInitThread(void* arg) {
flush = MmFileCacheFlushAll();
if (!KSUCCESS(flush)) DbgPrint("shutdown: final cache flush status 0x%x\n", flush);
KiFlushRegisteredDisks("final");
// Firmware preparation runs while platform drivers can still execute methods.
KSTATUS firmwarePrepare = KePowerPrepareFirmware(action);
if (!KSUCCESS(firmwarePrepare) && firmwarePrepare != NTLSTATUS_NOT_SUPPORTED)
DbgPrint("shutdown: firmware prepare failed 0x%x; using HAL fallback\n", firmwarePrepare);
KSTATUS prepare = DmPrepareShutdown();
if (!KSUCCESS(prepare)) DbgPrint("shutdown: prepare status 0x%x\n", prepare);
KSTATUS pnpShutdown = IoPnpShutdown(10000u);
if (!KSUCCESS(pnpShutdown))
DbgPrint("shutdown: device stop status 0x%x\n", pnpShutdown);
DmCommitShutdown();
KSTATUS firmwareEnter = KePowerEnterFirmware(action);
if (firmwareEnter != NTLSTATUS_NOT_SUPPORTED)
DbgPrint("shutdown: firmware transition returned 0x%x; using HAL fallback\n", firmwareEnter);
if (action == SYS_POWER_ACTION_RESTART) {
if (!gKmainHal->Power.Reset)
KePanic(KE_PANIC_CODE_INIT, "shutdown: reset HAL callback unavailable\n");
@@ -773,9 +780,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeStopProcessor();
IoInitializeDdk(hal, bootInfo->PhysicalMapBase);
KeFirmwareInitialize(bootInfo);
gKmainHal = hal;
KeInitDebug(hal, KSUCCESS(bootConfigStatus) ? bootConfig.Debug : 0);
KeFirmwareLog();
if (!KSUCCESS(bootConfigStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"boot.ini: malformed or missing mandatory configuration, status=0x%x\n",
@@ -786,7 +795,6 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Bind IRQ dispatch while IF remains clear. Interrupt sources are armed
// later, after their kernel consumers have registered.
KeIrqInitialize(hal);
DbgPrint("CPU: PAT owns WB/WC/UC policy; MTRR default=WB fixed=off variable=off.\n");
KeTimeInitialize(bootInfo, hal);
KeDisplayInitialize(hal);
@@ -805,7 +813,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeInputInitialize(hal);
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.HandleInterrupt);
KeIrqRegisterLocalTimerHandler(hal->Timer.HandleInterrupt);
KeSyscallInit(hal);
@@ -822,7 +830,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Scheduler Prep & Status Update.
KeInitializeScheduler();
KeIrqRegister(hal->Timer.IrqNumber, KeSchedulerTick);
KeIrqRegisterLocalTimerHandler(KeSchedulerTick);
KeConfigureCurrentProcessorTimer();
KSTATUS volumeManagerStatus = IoInitializeVolumeManager();
code/kernel/mm_virt.c
+1
View File
@@ -7,6 +7,7 @@
// FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings.
// FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map.
// FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys.
// FFFF980000000000-FFFF9FFFFFFFFFFF is the generic device-MMIO aperture.
// FFFFA00000000000-FFFFAFFFFFFFFFFF is the kernel shared-view arena.
// FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool.
//