// HAL APIC support. This file owns local APIC, I/O APIC, and interrupt-controller setup. #include #include #include #include extern HAL_API gApi; // ---- Local APIC ----. static volatile uint32_t* gLapic = (volatile uint32_t *)0; #define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off))) #define LAPIC_ID 0x020 #define LAPIC_EOI 0x0B0 #define LAPIC_SPURIOUS 0x0F0 #define LAPIC_ICR_LO 0x300 #define LAPIC_ICR_HI 0x310 #define LAPIC_ENABLE 0x100 // bit in spurious vector register #define IA32_APIC_BASE 0x0000001Bu #define IA32_APIC_BASE_ENABLE (1ULL << 11) #define IA32_APIC_BASE_X2APIC (1ULL << 10) #define IA32_X2APIC_ID 0x00000802u #define IA32_X2APIC_EOI 0x0000080Bu #define IA32_X2APIC_SVR 0x0000080Fu #define IA32_X2APIC_ICR 0x00000830u #define CPUID_FEATURE_ECX_X2APIC (1u << 21) #define APIC_VECTOR_RESCHEDULE 0xF0u // ---- I/O APIC ----. static volatile uint32_t* gIoapic = (volatile uint32_t *)0; static uint32_t gBspLapicId = 0; static uint8_t gX2ApicSupported = 0; static uint8_t gX2ApicEnabled = 0; #define IOAPIC_IDX 0x00 #define IOAPIC_DATA 0x10 // Implements the ioapic Read operation. static uint32_t IoapicRead(uint8_t reg) { *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg; return *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA); } // Implements the ioapic Write operation. static void IoapicWrite(uint8_t reg, uint32_t val) { *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_IDX) = reg; *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val; } // Implements the apic Detect Mode operation. static void HalApicDetectMode(void) { int regs[4] = {0, 0, 0, 0}; __cpuid(regs, 1); gX2ApicSupported = (regs[2] & CPUID_FEATURE_ECX_X2APIC) ? 1u : 0u; uint64_t apicBase = __readmsr(IA32_APIC_BASE); gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u; } // Implements the apic Stall Fallback operation. static void HalApicStallFallback(uint32_t loops) { for (volatile uint32_t i = 0; i < loops; i++) { _mm_pause(); } } // Implements the apic Stall Us operation. static void HalApicStallUs(uint32_t us) { if (gApi.Timer.Stall) gApi.Timer.Stall(us); else HalApicStallFallback(us * 128u + 1u); } // Implements the apic Wait Icr Idle operation. static void HalApicWaitIcrIdle(void) { if (gX2ApicEnabled) return; for (uint32_t i = 0; i < 1000000u; i++) { if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12))) return; _mm_pause(); } } // Implements the apic Send Ipi operation. static void HalApicSendIpi(uint32_t lapicId, uint32_t low) { if (gX2ApicEnabled) { __writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low); return; } HalApicWaitIcrIdle(); LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24; LAPIC_REG(LAPIC_ICR_LO) = low; HalApicWaitIcrIdle(); } // IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high). #define IOREDTBL_LO(n) (0x10 + 2*(n)) #define IOREDTBL_HI(n) (0x11 + 2*(n)) // Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ... #define IRQ_VECTOR_BASE 0x20 // Returns bsp Lapic Id. __declspec(dllexport) uint32_t HalGetBspLapicId(void) { return gBspLapicId; } // Implements the apic Using X2 Apic operation. __declspec(dllexport) uint8_t HalApicUsingX2Apic(void) { return gX2ApicEnabled; } // Implements the apic Can Address Id operation. __declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId) { if (gX2ApicEnabled) return 1u; return (lapicId < 256u) ? 1u : 0u; } // Implements the apic Enable Current Cpu operation. __declspec(dllexport) void HalApicEnableCurrentCpu(void) { uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE; if (gX2ApicEnabled) { apicBase |= IA32_APIC_BASE_X2APIC; __writemsr(IA32_APIC_BASE, apicBase); __writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu); return; } __writemsr(IA32_APIC_BASE, apicBase); if (gLapic) LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF; } // Implements the apic Send Panic Ipi operation. __declspec(dllexport) void HalApicSendPanicIpi(void) { if (!gLapic && !gX2ApicEnabled) return; // Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed // vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still // enter the panic path. The NMI vector is already installed in the IDT on // every CPU that has reached kernel-owned state. HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI } // Implements the apic Send Reschedule Ipi operation. __declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) { if (!gLapic && !gX2ApicEnabled) return; HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination } // Implements the apic Send Tlb Shootdown Ipi operation. __declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) { if (!gLapic && !gX2ApicEnabled) return; // TLB shootdown must reach CPUs that may be spinning with IF=0 inside // high-IRQL/spinlock regions, so use NMI delivery rather than a fixed // vector. The NMI stub handles active shootdowns and returns; unexpected // NMIs still go through the normal fault/panic path. HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination } // Starts application Processor. __declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) { if ((!gLapic && !gX2ApicEnabled) || !startupVector || lapicId == gBspLapicId) return STATUS_INVALID_PARAMETER; if (!HalApicCanAddressId(lapicId)) return STATUS_NOT_SUPPORTED; // INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12. HalApicSendIpi(lapicId, 0x0000C500u); // INIT, level assert, level-triggered HalApicStallUs(10000); HalApicSendIpi(lapicId, 0x00008500u); // INIT, level deassert HalApicStallUs(200); HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI HalApicStallUs(200); HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI retry HalApicStallUs(200); return STATUS_SUCCESS; } // Implements the apic Route Irq operation. void HalApicRouteIrq(uint8_t irq) { // Full initial setup of the redirection entry. // Sets the vector and clears the mask bit in one write. uint32_t vector = IRQ_VECTOR_BASE + irq; // e.g. IRQ 1 -> vector 0x21 IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24); // external IRQs target the BSP IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed } // Implements the apic Mask Irq operation. void HalApicMaskIrq(uint8_t irq) { uint32_t lo = IoapicRead(IOREDTBL_LO(irq)); lo |= (1 << 16); IoapicWrite(IOREDTBL_LO(irq), lo); } // Implements the apic Unmask Irq operation. void HalApicUnmaskIrq(uint8_t irq) { uint32_t lo = IoapicRead(IOREDTBL_LO(irq)); lo &= ~(1 << 16); // clear mask bit lo = (lo & ~0xFF) | (IRQ_VECTOR_BASE + irq); IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24); IoapicWrite(IOREDTBL_LO(irq), lo); } // Implements the apic Send Eoi operation. void HalApicSendEoi(void) { if (gX2ApicEnabled) { __writemsr(IA32_X2APIC_EOI, 0); return; } LAPIC_REG(LAPIC_EOI) = 0; } // Implements the apic Initialize operation. void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) { HalApicDetectMode(); gLapic = (volatile uint32_t *)(gApi.PhysToVirt(lapicPhys)); gIoapic = (volatile uint32_t *)(gApi.PhysToVirt(ioapicPhys)); // The BSP is the CPU executing HAL init. Keep all external/device IRQs // routed to this LAPIC; CPU-local IPIs use the local APIC directly. if (gX2ApicEnabled) gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID); else gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF; HalApicEnableCurrentCpu(); // Mask all IOAPIC lines initially. uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF; for (uint8_t i = 0; i <= maxRedir; i++) { IoapicWrite(IOREDTBL_LO(i), (1 << 16) | (IRQ_VECTOR_BASE + i)); IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24); } }