nt-lite/code/kernel/hal/hal_apic.c

// HAL APIC support. This file owns local APIC, I/O APIC, and interrupt-controller setup.

#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_timer.h>
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>


// ---- Local APIC ----.
static volatile uint32_t* gLapic = (volatile uint32_t*)0;
#define LAPIC_REG(off)   (*(volatile uint32_t*)((uint8_t*)gLapic + (off)))
#define LAPIC_ID         0x020
#define LAPIC_EOI        0x0B0
#define LAPIC_SPURIOUS   0x0F0
#define LAPIC_ICR_LO     0x300
#define LAPIC_ICR_HI     0x310
#define LAPIC_ENABLE     0x100   // bit in spurious vector register

#define IA32_APIC_BASE          0x0000001Bu
#define IA32_APIC_BASE_ENABLE   (1ULL << 11)
#define IA32_APIC_BASE_X2APIC   (1ULL << 10)
#define IA32_X2APIC_ID          0x00000802u
#define IA32_X2APIC_EOI         0x0000080Bu
#define IA32_X2APIC_SVR         0x0000080Fu
#define IA32_X2APIC_ICR         0x00000830u

#define CPUID_FEATURE_ECX_X2APIC (1u << 21)


// ---- I/O APIC ----.
static volatile uint32_t* gIoapic = (volatile uint32_t*)0;
static uint32_t gBspLapicId = 0;
static uint8_t gX2ApicSupported = 0;
static uint8_t gX2ApicEnabled = 0;
static uint32_t gIoapicGsiBase = 0;
static uint32_t gIoapicRedirectionCount = 0;
static uint32_t gIsaGsi[16];
static uint16_t gIsaFlags[16];
static uint16_t gSciIrq = UINT16_MAX;
static bool gIsaOverridesInitialized = false;
#define IOAPIC_IDX   0x00
#define IOAPIC_DATA  0x10

// Implements the ioapic Read operation.
static uint32_t IoapicRead(uint8_t reg) {
    *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
    return *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA);
}
// Implements the ioapic Write operation.
static void IoapicWrite(uint8_t reg, uint32_t val) {
    *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
    *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA) = val;
}

// Implements the apic Detect Mode operation.
static void HalApicDetectMode(void) {
    int regs[4] = {0, 0, 0, 0};
    __cpuid(regs, 1);
    gX2ApicSupported = (regs[2] & CPUID_FEATURE_ECX_X2APIC) ? 1u : 0u;

    uint64_t apicBase = __readmsr(IA32_APIC_BASE);
    gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u;
}

// Stalls for the requested bootstrap interval using the HAL timer fallback chain.
static void HalApicStallUs(uint32_t us) {
    HalTimerStall(us);
}

// Implements the apic Wait Icr Idle operation.
static void HalApicWaitIcrIdle(void) {
    if (gX2ApicEnabled)
        return;

    for (uint32_t i = 0; i < 1000000u; i++) {
        if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12)))
            return;
        _mm_pause();
    }
}

// Implements the apic Send Ipi operation.
static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
    if (gX2ApicEnabled) {
        __writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low);
        return;
    }

    HalApicWaitIcrIdle();
    LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24;
    LAPIC_REG(LAPIC_ICR_LO) = low;
    HalApicWaitIcrIdle();
}

// IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high).
#define IOREDTBL_LO(n)  (0x10 + 2*(n))
#define IOREDTBL_HI(n)  (0x11 + 2*(n))
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
#define IRQ_VECTOR_BASE  0x20

// Returns the local interrupt-controller hardware ID of the boot processor.
uint32_t HalApicGetBootProcessorId(void) {
    return gBspLapicId;
}

// Returns whether the local interrupt controller is using x2APIC mode.
uint8_t HalApicUsingX2Apic(void) {
    return gX2ApicEnabled;
}

// Returns whether the active local interrupt-controller mode can address one hardware ID.
uint8_t HalApicCanAddressId(uint32_t hardwareId) {
    if (gX2ApicEnabled)
        return 1u;
    return (hardwareId < 256u) ? 1u : 0u;
}

// Initializes the local interrupt controller on the current processor.
void HalApicInitializeCurrentProcessor(void) {
    uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE;

    if (gX2ApicEnabled) {
        apicBase |= IA32_APIC_BASE_X2APIC;
        __writemsr(IA32_APIC_BASE, apicBase);
        __writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu);
        return;
    }

    __writemsr(IA32_APIC_BASE, apicBase);
    if (gLapic)
        LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
}

// Broadcasts an NMI to every processor except the caller.
void HalApicBroadcastNmi(void) {
    if (!gLapic && !gX2ApicEnabled) return;

    // Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed
    // vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still
    // enter the panic path. The NMI vector is already installed in the IDT on
    // every CPU that has reached kernel-owned state.
    HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI
}

// Sends one fixed-vector processor interrupt without exposing APIC mechanics.
void HalApicSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector) {
    if ((!gLapic && !gX2ApicEnabled) || vector < 0x20u || !HalApicCanAddressId(hardwareId))
        return;
    HalApicSendIpi(hardwareId, vector);
}

// Sends one non-maskable processor interrupt.
void HalApicSendProcessorNmi(uint32_t hardwareId) {
    if ((!gLapic && !gX2ApicEnabled) || !HalApicCanAddressId(hardwareId))
        return;
    HalApicSendIpi(hardwareId, (4u << 8));
}

// Starts one secondary processor through the local interrupt-controller startup protocol.
KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical) {
    if ((!gLapic && !gX2ApicEnabled) || !startPhysical ||
        (startPhysical & 0xFFFu) != 0 || startPhysical >= 0x100000u ||
        hardwareId == gBspLapicId) {
        return NTLSTATUS_INVALID_PARAMETER;
    }
    if (!HalApicCanAddressId(hardwareId))
        return NTLSTATUS_NOT_SUPPORTED;

    uint8_t startupVector = (uint8_t)(startPhysical >> 12);
    // INIT assert/deassert, then two SIPIs. The target begins at startPhysical.
    HalApicSendIpi(hardwareId, 0x0000C500u); // INIT, level assert, level-triggered
    HalApicStallUs(10000);
    HalApicSendIpi(hardwareId, 0x00008500u); // INIT, level deassert
    HalApicStallUs(200);
    HalApicSendIpi(hardwareId, 0x00000600u | startupVector); // SIPI
    HalApicStallUs(200);
    HalApicSendIpi(hardwareId, 0x00000600u | startupVector); // SIPI retry
    HalApicStallUs(200);
    return NTLSTATUS_SUCCESS;
}

// Composes an MSI message targeting the boot processor without exposing APIC mode.
KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut) {
    if (!messageOut || vector < 0x20u)
        return NTLSTATUS_INVALID_PARAMETER;
    if (gBspLapicId >= 256u)
        return NTLSTATUS_NOT_SUPPORTED;

    messageOut->Address = 0xFEE00000ull | ((uint64_t)(gBspLapicId & 0xFFu) << 12);
    messageOut->Data = vector;
    messageOut->Reserved = 0;
    return NTLSTATUS_SUCCESS;
}

// Initializes the identity ISA map before processing MADT source overrides.
static void HalApicInitializeIsaMap(void) {
    if (gIsaOverridesInitialized) return;
    for (uint32_t irq = 0; irq < 16; ++irq) {
        gIsaGsi[irq] = irq;
        gIsaFlags[irq] = 0;
    }
    gIsaOverridesInitialized = true;
}

// Records the global-system-interrupt base of the currently supported IOAPIC.
void HalApicSetGsiBase(uint32_t gsiBase) { gIoapicGsiBase = gsiBase; }

// Preserves MADT interrupt-source overrides for legacy ISA devices.
void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags) {
    if (irq >= 16) return;
    HalApicInitializeIsaMap();
    gIsaGsi[irq] = gsi;
    gIsaFlags[irq] = flags;
}

// Marks the FADT SCI as a low-active, level-triggered interrupt by default.
void HalApicSetSciIrq(uint16_t irq) { gSciIrq = irq; }

// Resolves one logical IRQ to a physical pin on the selected IOAPIC.
static bool HalApicResolveIrq(uint8_t irq, uint32_t* pinOut, uint32_t* flagsOut) {
    HalApicInitializeIsaMap();
    uint32_t gsi = irq < 16 ? gIsaGsi[irq] : irq;
    if (gsi < gIoapicGsiBase || gsi - gIoapicGsiBase >= gIoapicRedirectionCount)
        return false;
    uint32_t flags = 0;
    if (irq == gSciIrq) flags = (1u << 13) | (1u << 15);
    if (irq < 16) {
        uint16_t override = gIsaFlags[irq];
        // MADT 00 is bus-conformant; 11 selects active low or level triggered.
        if ((override & 3u) == 3u) flags |= (1u << 13);
        else if ((override & 3u) == 1u) flags &= ~(1u << 13);
        if (((override >> 2) & 3u) == 3u) flags |= (1u << 15);
        else if (((override >> 2) & 3u) == 1u) flags &= ~(1u << 15);
    }
    *pinOut = gsi - gIoapicGsiBase;
    *flagsOut = flags;
    return true;
}

// Reports whether a platform IRQ can be routed by the current IOAPIC.
bool HalApicCanRouteIrq(uint8_t irq) {
    uint32_t pin, flags;
    return irq < 24 && HalApicResolveIrq(irq, &pin, &flags);
}

// Implements the apic Route Irq operation.
void HalApicRouteIrq(uint8_t irq) {
    // Full initial setup of the redirection entry.
    // Sets the vector and clears the mask bit in one write.
    uint32_t pin, flags;
    if (!HalApicResolveIrq(irq, &pin, &flags)) return;
    uint32_t vector = IRQ_VECTOR_BASE + irq;
    IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
    IoapicWrite(IOREDTBL_LO(pin), vector | flags);
}

// Implements the apic Mask Irq operation.
void HalApicMaskIrq(uint8_t irq) {
    uint32_t pin, flags;
    if (!HalApicResolveIrq(irq, &pin, &flags)) return;
    uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
    lo |= (1 << 16);
    IoapicWrite(IOREDTBL_LO(pin), lo);
}

// Implements the apic Unmask Irq operation.
void HalApicUnmaskIrq(uint8_t irq) {
    uint32_t pin, flags;
    if (!HalApicResolveIrq(irq, &pin, &flags)) return;
    uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
    lo = (lo & ~(0xFFu | (1u << 16) | (1u << 13) | (1u << 15))) |
         (IRQ_VECTOR_BASE + irq) | flags;
    IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
    IoapicWrite(IOREDTBL_LO(pin), lo);
}

// Implements the apic Send Eoi operation.
void HalApicSendEoi(void) {
    if (gX2ApicEnabled) {
        __writemsr(IA32_X2APIC_EOI, 0);
        return;
    }
    LAPIC_REG(LAPIC_EOI) = 0;
}

// Implements the apic Initialize operation.
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
    HalApicDetectMode();

    gLapic  = (volatile uint32_t*)(HalPhysicalToVirtual(lapicPhys));
    gIoapic = (volatile uint32_t*)(HalPhysicalToVirtual(ioapicPhys));

    // The BSP is the CPU executing HAL init. Keep all external/device IRQs
    // routed to this LAPIC; CPU-local IPIs use the local APIC directly.
    if (gX2ApicEnabled)
        gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID);
    else
        gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;

    HalApicInitializeCurrentProcessor();

    // Mask all IOAPIC lines initially.
    uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
    gIoapicRedirectionCount = maxRedir + 1u;
    for (uint32_t i = 0; i <= maxRedir; i++) {
        IoapicWrite(IOREDTBL_LO(i), (1u << 16) | IRQ_VECTOR_BASE);
        IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24);
    }
}