// HAL APIC support. This file owns local APIC, I/O APIC, and interrupt-controller setup. #include #include #include #include #include #include #include // ---- Local APIC ----. static volatile uint32_t* gLapic = (volatile uint32_t*)0; #define LAPIC_REG(off) (*(volatile uint32_t*)((uint8_t*)gLapic + (off))) #define LAPIC_ID 0x020 #define LAPIC_EOI 0x0B0 #define LAPIC_SPURIOUS 0x0F0 #define LAPIC_ICR_LO 0x300 #define LAPIC_ICR_HI 0x310 #define LAPIC_ENABLE 0x100 // bit in spurious vector register #define IA32_APIC_BASE 0x0000001Bu #define IA32_APIC_BASE_ENABLE (1ULL << 11) #define IA32_APIC_BASE_X2APIC (1ULL << 10) #define IA32_X2APIC_ID 0x00000802u #define IA32_X2APIC_EOI 0x0000080Bu #define IA32_X2APIC_SVR 0x0000080Fu #define IA32_X2APIC_ICR 0x00000830u #define CPUID_FEATURE_ECX_X2APIC (1u << 21) // ---- I/O APIC ----. static volatile uint32_t* gIoapic = (volatile uint32_t*)0; static uint32_t gBspLapicId = 0; static uint8_t gX2ApicSupported = 0; static uint8_t gX2ApicEnabled = 0; static uint32_t gIoapicGsiBase = 0; static uint32_t gIoapicRedirectionCount = 0; static uint32_t gIsaGsi[16]; static uint16_t gIsaFlags[16]; static uint16_t gSciIrq = UINT16_MAX; static bool gIsaOverridesInitialized = false; #define IOAPIC_IDX 0x00 #define IOAPIC_DATA 0x10 // Implements the ioapic Read operation. static uint32_t IoapicRead(uint8_t reg) { *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg; return *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA); } // Implements the ioapic Write operation. static void IoapicWrite(uint8_t reg, uint32_t val) { *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg; *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA) = val; } // Implements the apic Detect Mode operation. static void HalApicDetectMode(void) { int regs[4] = {0, 0, 0, 0}; __cpuid(regs, 1); gX2ApicSupported = (regs[2] & CPUID_FEATURE_ECX_X2APIC) ? 1u : 0u; uint64_t apicBase = __readmsr(IA32_APIC_BASE); gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u; } // Stalls for the requested bootstrap interval using the HAL timer fallback chain. static void HalApicStallUs(uint32_t us) { HalTimerStall(us); } // Implements the apic Wait Icr Idle operation. static void HalApicWaitIcrIdle(void) { if (gX2ApicEnabled) return; for (uint32_t i = 0; i < 1000000u; i++) { if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12))) return; _mm_pause(); } } // Implements the apic Send Ipi operation. static void HalApicSendIpi(uint32_t lapicId, uint32_t low) { if (gX2ApicEnabled) { __writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low); return; } HalApicWaitIcrIdle(); LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24; LAPIC_REG(LAPIC_ICR_LO) = low; HalApicWaitIcrIdle(); } // IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high). #define IOREDTBL_LO(n) (0x10 + 2*(n)) #define IOREDTBL_HI(n) (0x11 + 2*(n)) // Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ... #define IRQ_VECTOR_BASE 0x20 // Returns the local interrupt-controller hardware ID of the boot processor. uint32_t HalApicGetBootProcessorId(void) { return gBspLapicId; } // Returns whether the local interrupt controller is using x2APIC mode. uint8_t HalApicUsingX2Apic(void) { return gX2ApicEnabled; } // Returns whether the active local interrupt-controller mode can address one hardware ID. uint8_t HalApicCanAddressId(uint32_t hardwareId) { if (gX2ApicEnabled) return 1u; return (hardwareId < 256u) ? 1u : 0u; } // Initializes the local interrupt controller on the current processor. void HalApicInitializeCurrentProcessor(void) { uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE; if (gX2ApicEnabled) { apicBase |= IA32_APIC_BASE_X2APIC; __writemsr(IA32_APIC_BASE, apicBase); __writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu); return; } __writemsr(IA32_APIC_BASE, apicBase); if (gLapic) LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF; } // Broadcasts an NMI to every processor except the caller. void HalApicBroadcastNmi(void) { if (!gLapic && !gX2ApicEnabled) return; // Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed // vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still // enter the panic path. The NMI vector is already installed in the IDT on // every CPU that has reached kernel-owned state. HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI } // Sends one fixed-vector processor interrupt without exposing APIC mechanics. void HalApicSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector) { if ((!gLapic && !gX2ApicEnabled) || vector < 0x20u || !HalApicCanAddressId(hardwareId)) return; HalApicSendIpi(hardwareId, vector); } // Sends one non-maskable processor interrupt. void HalApicSendProcessorNmi(uint32_t hardwareId) { if ((!gLapic && !gX2ApicEnabled) || !HalApicCanAddressId(hardwareId)) return; HalApicSendIpi(hardwareId, (4u << 8)); } // Starts one secondary processor through the local interrupt-controller startup protocol. KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical) { if ((!gLapic && !gX2ApicEnabled) || !startPhysical || (startPhysical & 0xFFFu) != 0 || startPhysical >= 0x100000u || hardwareId == gBspLapicId) { return NTLSTATUS_INVALID_PARAMETER; } if (!HalApicCanAddressId(hardwareId)) return NTLSTATUS_NOT_SUPPORTED; uint8_t startupVector = (uint8_t)(startPhysical >> 12); // INIT assert/deassert, then two SIPIs. The target begins at startPhysical. HalApicSendIpi(hardwareId, 0x0000C500u); // INIT, level assert, level-triggered HalApicStallUs(10000); HalApicSendIpi(hardwareId, 0x00008500u); // INIT, level deassert HalApicStallUs(200); HalApicSendIpi(hardwareId, 0x00000600u | startupVector); // SIPI HalApicStallUs(200); HalApicSendIpi(hardwareId, 0x00000600u | startupVector); // SIPI retry HalApicStallUs(200); return NTLSTATUS_SUCCESS; } // Composes an MSI message targeting the boot processor without exposing APIC mode. KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut) { if (!messageOut || vector < 0x20u) return NTLSTATUS_INVALID_PARAMETER; if (gBspLapicId >= 256u) return NTLSTATUS_NOT_SUPPORTED; messageOut->Address = 0xFEE00000ull | ((uint64_t)(gBspLapicId & 0xFFu) << 12); messageOut->Data = vector; messageOut->Reserved = 0; return NTLSTATUS_SUCCESS; } // Initializes the identity ISA map before processing MADT source overrides. static void HalApicInitializeIsaMap(void) { if (gIsaOverridesInitialized) return; for (uint32_t irq = 0; irq < 16; ++irq) { gIsaGsi[irq] = irq; gIsaFlags[irq] = 0; } gIsaOverridesInitialized = true; } // Records the global-system-interrupt base of the currently supported IOAPIC. void HalApicSetGsiBase(uint32_t gsiBase) { gIoapicGsiBase = gsiBase; } // Preserves MADT interrupt-source overrides for legacy ISA devices. void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags) { if (irq >= 16) return; HalApicInitializeIsaMap(); gIsaGsi[irq] = gsi; gIsaFlags[irq] = flags; } // Marks the FADT SCI as a low-active, level-triggered interrupt by default. void HalApicSetSciIrq(uint16_t irq) { gSciIrq = irq; } // Resolves one logical IRQ to a physical pin on the selected IOAPIC. static bool HalApicResolveIrq(uint8_t irq, uint32_t* pinOut, uint32_t* flagsOut) { HalApicInitializeIsaMap(); uint32_t gsi = irq < 16 ? gIsaGsi[irq] : irq; if (gsi < gIoapicGsiBase || gsi - gIoapicGsiBase >= gIoapicRedirectionCount) return false; uint32_t flags = 0; if (irq == gSciIrq) flags = (1u << 13) | (1u << 15); if (irq < 16) { uint16_t override = gIsaFlags[irq]; // MADT 00 is bus-conformant; 11 selects active low or level triggered. if ((override & 3u) == 3u) flags |= (1u << 13); else if ((override & 3u) == 1u) flags &= ~(1u << 13); if (((override >> 2) & 3u) == 3u) flags |= (1u << 15); else if (((override >> 2) & 3u) == 1u) flags &= ~(1u << 15); } *pinOut = gsi - gIoapicGsiBase; *flagsOut = flags; return true; } // Reports whether a platform IRQ can be routed by the current IOAPIC. bool HalApicCanRouteIrq(uint8_t irq) { uint32_t pin, flags; return irq < 24 && HalApicResolveIrq(irq, &pin, &flags); } // Implements the apic Route Irq operation. void HalApicRouteIrq(uint8_t irq) { // Full initial setup of the redirection entry. // Sets the vector and clears the mask bit in one write. uint32_t pin, flags; if (!HalApicResolveIrq(irq, &pin, &flags)) return; uint32_t vector = IRQ_VECTOR_BASE + irq; IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24); IoapicWrite(IOREDTBL_LO(pin), vector | flags); } // Implements the apic Mask Irq operation. void HalApicMaskIrq(uint8_t irq) { uint32_t pin, flags; if (!HalApicResolveIrq(irq, &pin, &flags)) return; uint32_t lo = IoapicRead(IOREDTBL_LO(pin)); lo |= (1 << 16); IoapicWrite(IOREDTBL_LO(pin), lo); } // Implements the apic Unmask Irq operation. void HalApicUnmaskIrq(uint8_t irq) { uint32_t pin, flags; if (!HalApicResolveIrq(irq, &pin, &flags)) return; uint32_t lo = IoapicRead(IOREDTBL_LO(pin)); lo = (lo & ~(0xFFu | (1u << 16) | (1u << 13) | (1u << 15))) | (IRQ_VECTOR_BASE + irq) | flags; IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24); IoapicWrite(IOREDTBL_LO(pin), lo); } // Implements the apic Send Eoi operation. void HalApicSendEoi(void) { if (gX2ApicEnabled) { __writemsr(IA32_X2APIC_EOI, 0); return; } LAPIC_REG(LAPIC_EOI) = 0; } // Implements the apic Initialize operation. void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) { HalApicDetectMode(); gLapic = (volatile uint32_t*)(HalPhysicalToVirtual(lapicPhys)); gIoapic = (volatile uint32_t*)(HalPhysicalToVirtual(ioapicPhys)); // The BSP is the CPU executing HAL init. Keep all external/device IRQs // routed to this LAPIC; CPU-local IPIs use the local APIC directly. if (gX2ApicEnabled) gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID); else gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF; HalApicInitializeCurrentProcessor(); // Mask all IOAPIC lines initially. uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF; gIoapicRedirectionCount = maxRedir + 1u; for (uint32_t i = 0; i <= maxRedir; i++) { IoapicWrite(IOREDTBL_LO(i), (1u << 16) | IRQ_VECTOR_BASE); IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24); } }