// Interrupt manager. Per-CPU vector ownership is independent of physical sources.

#include <kernel/ke_irq.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_vectors.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <intrin.h>

#define KI_INTERRUPT_ROUTES 256u
#define KI_INTERRUPT_HANDLERS 512u

typedef struct KI_INTERRUPT_ROUTE KI_INTERRUPT_ROUTE;
struct KE_INTERRUPT {
    KE_INTERRUPT_ROUTINE Routine;
    void* Context;
    KI_INTERRUPT_ROUTE* Route;
    KE_INTERRUPT_MESSAGE_PROGRAM Program;
    void* ProgramContext;
    uint64_t Affinity;
    struct KE_INTERRUPT* Next;
    bool Used;
};

struct KI_INTERRUPT_ROUTE {
    KE_INTERRUPT* Handlers;
    uint32_t Gsi;
    uint32_t Cpu;
    uint8_t Vector;
    uint16_t Flags;
    bool Used;
    bool Message;
    bool Draining;
    bool Closing;
    bool MessageActive;
    volatile long Active;
    volatile long Enabled;
    volatile uint64_t Delivered;
};

static KI_INTERRUPT_ROUTE gRoutes[KI_INTERRUPT_ROUTES];
static KE_INTERRUPT gHandlers[KI_INTERRUPT_HANDLERS];
static KI_INTERRUPT_ROUTE* volatile gCpuVectors[KE_MAX_CPUS][256];
// A reader is registered before it resolves the route pointer. A retired route
// waits for all readers on its former CPU/vector before releasing handler state.
static volatile long gCpuVectorReaders[KE_MAX_CPUS][256];
// Vector reservations outlive publication while a route is draining.
static bool gCpuVectorReserved[KE_MAX_CPUS][256];
static bool gCpuVectorRetired[KE_MAX_CPUS][256];
static uint8_t gCpuVectorQuietTicks[KE_MAX_CPUS][256];
static SPINLOCK gInterruptLock = SPINLOCK_INIT;
static const HAL_API* gHal;
static void (*gLocalTimerHandlers[2])(void);
static uint8_t gLocalTimerHandlerCount;
static uint32_t gPlacementCursor;

static KIRQL KiVectorIrql(uint8_t vector) {
    uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)((vector >> 4u) - 3u);
    if (level >= HIGH_LEVEL) level = HIGH_LEVEL - 1u;
    return (KIRQL)level;
}

void KeIrqInitialize(const HAL_API* hal) {
    if (!hal || !hal->InterruptController.SendEoi ||
        !hal->InterruptController.RouteGsi || !hal->InterruptController.ResolveIsa ||
        !hal->InterruptController.ComposeMsi)
        KePanic(KE_PANIC_CODE_INIT, "IRQ: incomplete interrupt-controller API\n");
    gHal = hal;
}

static bool KiCpuAvailable(uint32_t index, uint64_t affinity) {
    if (index >= gCpuCount || index >= KE_MAX_CPUS ||
        (affinity && !(affinity & (1ull << index)))) return false;
    KCPU* cpu = KeGetCpuByIndex(index);
    return cpu && (cpu->IsBootProcessor || cpu->State >= KeCpuSchedulerActive) &&
           cpu->HardwareId <= 255u;
}

static bool KiPickVector(uint64_t affinity, uint32_t* cpuOut, uint8_t* vectorOut) {
    uint32_t count = gCpuCount;
    if (!count || count > KE_MAX_CPUS) count = 1;
    // Rotate the first candidate CPU, and allocate the least occupied vector.
    uint32_t start = gPlacementCursor++ % count;
    uint32_t bestLoad = UINT32_MAX;
    bool found = false;
    for (uint32_t i = 0; i < count; i++) {
        uint32_t cpu = (start + i) % count;
        if (!KiCpuAvailable(cpu, affinity)) continue;
        uint32_t occupied = 0;
        for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++)
            if (gCpuVectorReserved[cpu][v]) occupied++;
        if (occupied >= KE_VECTOR_DEVICE_LAST - KE_VECTOR_DEVICE_FIRST + 1u || occupied >= bestLoad)
            continue;
        for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
            if (!gCpuVectorReserved[cpu][v]) {
                bestLoad = occupied;
                *cpuOut = cpu;
                *vectorOut = (uint8_t)v;
                found = true;
                break;
            }
        }
    }
    return found;
}

static KSTATUS KiReserveInterrupt(bool message, uint32_t gsi, uint16_t flags,
    uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context,
    KE_INTERRUPT** out) {
    if (!out || !routine || !gHal) return NTLSTATUS_INVALID_PARAMETER;
    *out = (KE_INTERRUPT*)0;
    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)0;
    if (!message) {
        for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
            if (gRoutes[i].Used && !gRoutes[i].Message && gRoutes[i].Gsi == gsi) {
                if (gRoutes[i].Draining || gRoutes[i].Closing) {
                    KeReleaseSpinlock(&gInterruptLock, cookie);
                    return NTLSTATUS_NOT_SUPPORTED;
                }
                if (gRoutes[i].Flags != flags ||
                    (affinity && !(affinity & (1ull << gRoutes[i].Cpu)))) {
                    KeReleaseSpinlock(&gInterruptLock, cookie);
                    return NTLSTATUS_INVALID_PARAMETER;
                }
                route = &gRoutes[i];
                break;
            }
        }
    }
    bool newRoute = route == 0;
    uint32_t cpu = 0;
    uint8_t vector = 0;
    if (newRoute) {
        if (!KiPickVector(affinity, &cpu, &vector)) {
            KeReleaseSpinlock(&gInterruptLock, cookie);
            return NTLSTATUS_NO_MEMORY;
        }
        for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++)
            if (!gRoutes[i].Used) { route = &gRoutes[i]; break; }
        if (!route) {
            KeReleaseSpinlock(&gInterruptLock, cookie);
            return NTLSTATUS_NO_MEMORY;
        }
    }
    KE_INTERRUPT* handler = 0;
    for (uint32_t i = 0; i < KI_INTERRUPT_HANDLERS; i++)
        if (!gHandlers[i].Used) { handler = &gHandlers[i]; break; }
    if (!handler) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_NO_MEMORY;
    }
    if (newRoute) {
        route->Used = true;
        route->Message = message;
        route->Draining = false;
        route->Closing = false;
        route->MessageActive = false;
        route->Gsi = gsi;
        route->Cpu = cpu;
        route->Vector = vector;
        route->Flags = flags;
        route->Handlers = 0;
        route->Active = 0;
        route->Enabled = 0;
        route->Delivered = 0;
        gCpuVectorReserved[cpu][vector] = true;
        gCpuVectorRetired[cpu][vector] = false;
        gCpuVectorQuietTicks[cpu][vector] = 0;
        _InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
    }
    handler->Used = true;
    handler->Routine = routine;
    handler->Context = context;
    handler->Route = route;
    handler->Program = 0;
    handler->ProgramContext = 0;
    handler->Affinity = affinity;
    handler->Next = route->Handlers;
    route->Handlers = handler;
    KeReleaseSpinlock(&gInterruptLock, cookie);
    if (newRoute && !message) {
        KCPU* target = KeGetCpuByIndex(cpu);
        KSTATUS st = gHal->InterruptController.RouteGsi(gsi, vector,
                          target->HardwareId, flags, true);
        if (!KSUCCESS(st)) {
            KeInterruptDisconnect(handler);
            return st;
        }
        _InterlockedExchange(&route->Enabled, 1);
        (void)gHal->InterruptController.MaskGsi(gsi, false);
    } else if (newRoute) {
        _InterlockedExchange(&route->Enabled, 1);
    }
    *out = handler;
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS KeInterruptConnectGsi(uint32_t gsi, uint16_t flags,
    uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
    return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
}

__declspec(dllexport) KSTATUS KeInterruptConnectIsa(uint8_t isaIrq, uint16_t sourceFlags, uint64_t affinity,
    KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
    uint32_t gsi;
    uint16_t flags;
    if (!gHal || !gHal->InterruptController.ResolveIsa)
        return NTLSTATUS_INVALID_PARAMETER;
    KSTATUS st = gHal->InterruptController.ResolveIsa(isaIrq, &gsi, &flags);
    if (!KSUCCESS(st)) return st;
    if (sourceFlags) flags = sourceFlags;
    return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
}

__declspec(dllexport) KSTATUS KeInterruptReserveMessage(uint64_t affinity,
    KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
    return KiReserveInterrupt(true, UINT32_MAX, 0, affinity, routine, context, out);
}

__declspec(dllexport) KSTATUS KeInterruptComposeMessage(const KE_INTERRUPT* interrupt,
    NTL_DDK_MSI_MESSAGE* out) {
    if (!interrupt || !out || !gHal) return NTLSTATUS_INVALID_PARAMETER;
    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    if (!interrupt->Used || !interrupt->Route ||
        !interrupt->Route->Message || interrupt->Route->Draining) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    const KI_INTERRUPT_ROUTE* route = interrupt->Route;
    const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
    HAL_MSI_MESSAGE message;
    KSTATUS st = cpu ? gHal->InterruptController.ComposeMsi(
        cpu->HardwareId, route->Vector, &message) : NTLSTATUS_INVALID_PARAMETER;
    KeReleaseSpinlock(&gInterruptLock, cookie);
    if (!KSUCCESS(st)) return st;
    out->Address = message.Address;
    out->Data = message.Data;
    out->Reserved = 0;
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS KeInterruptQuery(const KE_INTERRUPT* interrupt,
                                               KE_INTERRUPT_INFO* out) {
    if (!interrupt || !out) return NTLSTATUS_INVALID_PARAMETER;
    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    if (!interrupt->Used || !interrupt->Route || interrupt->Route->Draining) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    const KI_INTERRUPT_ROUTE* route = interrupt->Route;
    const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
    out->CpuIndex = route->Cpu;
    out->HardwareId = cpu ? cpu->HardwareId : UINT32_MAX;
    out->Gsi = route->Gsi;
    out->Flags = route->Flags;
    out->Vector = route->Vector;
    out->IsMessage = route->Message ? 1u : 0u;
    out->Delivered = route->Delivered;
    KeReleaseSpinlock(&gInterruptLock, cookie);
    return NTLSTATUS_SUCCESS;
}

// A message device registers its update operation once PCI has enabled the source.
__declspec(dllexport) KSTATUS KeInterruptBindMessageSource(KE_INTERRUPT* interrupt,
    KE_INTERRUPT_MESSAGE_PROGRAM program, void* context) {
    if (!interrupt || !program) return NTLSTATUS_INVALID_PARAMETER;
    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
        (interrupt->Route->Draining || interrupt->Route->Closing) || interrupt->Program) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    interrupt->Program = program;
    interrupt->ProgramContext = context;
    KeReleaseSpinlock(&gInterruptLock, cookie);
    return NTLSTATUS_SUCCESS;
}

// The bus owner enables migration only after the device source is programmed.
__declspec(dllexport) KSTATUS KeInterruptActivateMessage(KE_INTERRUPT* interrupt) {
    if (!interrupt) return NTLSTATUS_INVALID_PARAMETER;
    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
        (interrupt->Route->Draining || interrupt->Route->Closing) || !interrupt->Program) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    interrupt->Route->MessageActive = true;
    KeReleaseSpinlock(&gInterruptLock, cookie);
    return NTLSTATUS_SUCCESS;
}

// Moves the complete physical source to one new CPU/vector, including PCI
// message programming. The old vector remains quarantined through APIC drain.
__declspec(dllexport) KSTATUS KeInterruptSetAffinity(KE_INTERRUPT* interrupt,
    uint64_t permittedCpuMask) {
    if (!interrupt || !permittedCpuMask || !gHal ||
        KeGetCurrentIrql() != PASSIVE_LEVEL || KeGetCurrentCpu()->InterruptDepth)
        return NTLSTATUS_INVALID_PARAMETER;

    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    if (!interrupt->Used || !interrupt->Route ||
        interrupt->Route->Draining || interrupt->Route->Closing) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    KI_INTERRUPT_ROUTE* route = interrupt->Route;
    if (route->Message && !route->MessageActive) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_NOT_SUPPORTED;
    }
    // One shared line must satisfy every handler's affinity constraint.
    for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next) {
        if (h->Affinity && !(h->Affinity & permittedCpuMask)) {
            KeReleaseSpinlock(&gInterruptLock, cookie);
            return NTLSTATUS_NOT_SUPPORTED;
        }
        if (route->Message && (!h->Program || h != interrupt)) {
            KeReleaseSpinlock(&gInterruptLock, cookie);
            return NTLSTATUS_NOT_SUPPORTED;
        }
    }
    for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
        if (h->Affinity) permittedCpuMask &= h->Affinity;
    if (!permittedCpuMask) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_NOT_SUPPORTED;
    }
    if (permittedCpuMask & (1ull << route->Cpu)) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_SUCCESS;
    }
    uint32_t newCpu = 0;
    uint8_t newVector = 0;
    if (!KiPickVector(permittedCpuMask, &newCpu, &newVector)) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return NTLSTATUS_NO_MEMORY;
    }
    const uint32_t oldCpu = route->Cpu;
    const uint8_t oldVector = route->Vector;
    gCpuVectorReserved[newCpu][newVector] = true;
    route->Draining = true;
    KeReleaseSpinlock(&gInterruptLock, cookie);

    NTL_DDK_MSI_MESSAGE updated = {0};
    if (route->Message) {
        const KCPU* destination = KeGetCpuByIndex(newCpu);
        HAL_MSI_MESSAGE message;
        KSTATUS st = gHal->InterruptController.ComposeMsi(destination->HardwareId,
                                                           newVector, &message);
        if (!KSUCCESS(st)) goto rollback_reservation;
        updated.Address = message.Address;
        updated.Data = message.Data;
        st = interrupt->Program(interrupt->ProgramContext, KeInterruptMessageQuiesce, 0);
        if (!KSUCCESS(st)) goto rollback_reservation;
    } else {
        KSTATUS st = gHal->InterruptController.MaskGsi(route->Gsi, true);
        if (!KSUCCESS(st)) goto rollback_reservation;
    }

    _InterlockedExchange(&route->Enabled, 0);
    _InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], 0);
    uint32_t spins = 0;
    while (route->Active || gCpuVectorReaders[oldCpu][oldVector]) {
        if (++spins == 100000000u)
            KePanic(KE_PANIC_CODE_DRIVER, "IRQ: affinity migration drain timeout\n");
        _mm_pause();
    }

    KSTATUS update = NTLSTATUS_SUCCESS;
    if (route->Message)
        update = interrupt->Program(interrupt->ProgramContext,
                                    KeInterruptMessageProgram, &updated);
    else {
        const KCPU* destination = KeGetCpuByIndex(newCpu);
        update = gHal->InterruptController.RouteGsi(route->Gsi, newVector,
                                                     destination->HardwareId, route->Flags, true);
    }
    if (!KSUCCESS(update)) {
        // Restore the existing route and re-enable its original source.
        if (route->Message) {
            const KCPU* previous = KeGetCpuByIndex(oldCpu);
            HAL_MSI_MESSAGE msg;
            if (!KSUCCESS(gHal->InterruptController.ComposeMsi(previous->HardwareId,
                                                                oldVector, &msg)))
                KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original MSI route\n");
            NTL_DDK_MSI_MESSAGE original = {msg.Address, msg.Data, 0};
            if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
                                              KeInterruptMessageProgram, &original)))
                KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore MSI message\n");
        } else {
            const KCPU* previous = KeGetCpuByIndex(oldCpu);
            if (!KSUCCESS(gHal->InterruptController.RouteGsi(route->Gsi, oldVector,
                                                               previous->HardwareId, route->Flags, true)))
                KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original GSI route\n");
        }
        _InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], route);
        _InterlockedExchange(&route->Enabled, 1);
        if (route->Message) {
            if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
                                              KeInterruptMessageResume, 0)))
                KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI restore resume failed\n");
        } else (void)gHal->InterruptController.MaskGsi(route->Gsi, false);
        cookie = KeAcquireSpinlock(&gInterruptLock);
        route->Draining = false;
        gCpuVectorReserved[newCpu][newVector] = false;
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return update;
    }

    cookie = KeAcquireSpinlock(&gInterruptLock);
    route->Cpu = newCpu;
    route->Vector = newVector;
    gCpuVectorRetired[oldCpu][oldVector] = true;
    gCpuVectorQuietTicks[oldCpu][oldVector] = 0;
    _InterlockedExchangePointer((void* volatile*)&gCpuVectors[newCpu][newVector], route);
    _InterlockedExchange(&route->Enabled, 1);
    KeReleaseSpinlock(&gInterruptLock, cookie);

    if (route->Message) {
        if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
                                          KeInterruptMessageResume, 0)))
            KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI migration resume failed\n");
    } else if (!KSUCCESS(gHal->InterruptController.MaskGsi(route->Gsi, false)))
        KePanic(KE_PANIC_CODE_DRIVER, "IRQ: GSI migration unmask failed\n");
    cookie = KeAcquireSpinlock(&gInterruptLock);
    route->Draining = false;
    KeReleaseSpinlock(&gInterruptLock, cookie);
    return NTLSTATUS_SUCCESS;

rollback_reservation:
    cookie = KeAcquireSpinlock(&gInterruptLock);
    gCpuVectorReserved[newCpu][newVector] = false;
    route->Draining = false;
    KeReleaseSpinlock(&gInterruptLock, cookie);
    return NTLSTATUS_NOT_SUPPORTED;
}

// The scheduler must evacuate every device route before stopping a processor.
KSTATUS KiInterruptPrepareCpuOffline(uint32_t cpuIndex) {
    if (cpuIndex >= gCpuCount || cpuIndex >= KE_MAX_CPUS ||
        KeGetCurrentIrql() != PASSIVE_LEVEL) return NTLSTATUS_INVALID_PARAMETER;
    uint64_t allowed = 0;
    for (uint32_t i = 0; i < gCpuCount && i < 64u; i++)
        if (i != cpuIndex && KiCpuAvailable(i, 0)) allowed |= 1ull << i;
    if (!allowed) return NTLSTATUS_NOT_SUPPORTED;
    for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
        KI_INTERRUPT_ROUTE* route = &gRoutes[i];
        if (route->Used && route->Cpu == cpuIndex && route->Handlers) {
            KSTATUS st = KeInterruptSetAffinity(route->Handlers, allowed);
            if (!KSUCCESS(st)) return st;
        }
    }
    return NTLSTATUS_SUCCESS;
}

// Serializes device-source disable with affinity migration. A caller must
// close the route before masking the PCI source, preventing a racing resume.
__declspec(dllexport) void KeInterruptBeginDisconnect(KE_INTERRUPT* interrupt) {
    if (!interrupt || KeGetCurrentIrql() != PASSIVE_LEVEL ||
        KeGetCurrentCpu()->InterruptDepth)
        KePanic(KE_PANIC_CODE_DRIVER, "IRQ: invalid disconnect preparation context\n");
    for (uint32_t spins = 0;; spins++) {
        uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
        if (!interrupt->Used || !interrupt->Route) {
            KeReleaseSpinlock(&gInterruptLock, cookie);
            return;
        }
        KI_INTERRUPT_ROUTE* route = interrupt->Route;
        if (!route->Draining && !route->Closing) {
            route->Closing = true;
            KeReleaseSpinlock(&gInterruptLock, cookie);
            return;
        }
        KeReleaseSpinlock(&gInterruptLock, cookie);
        if (spins >= 100000000u)
            KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect serialization timed out\n");
        _mm_pause();
    }
}

// Disconnect runs in thread context after the device has stopped producing messages.
// The route is unpublished before draining readers, so a concurrent dispatcher
// either holds a counted reference or observes an empty vector.
__declspec(dllexport) void KeInterruptDisconnect(KE_INTERRUPT* interrupt) {
    if (!interrupt) return;
    if (KeGetCurrentCpu()->InterruptDepth || KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect requires PASSIVE_LEVEL\n");

    // PCI callers have already serialized source disable. A direct GSI
    // caller prepares the route here before masking it inside this routine.
    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    if (!interrupt->Used || !interrupt->Route) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        return;
    }
    KI_INTERRUPT_ROUTE* route = interrupt->Route;
    if (!route->Closing) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        KeInterruptBeginDisconnect(interrupt);
        cookie = KeAcquireSpinlock(&gInterruptLock);
        route = interrupt->Route;
        if (!route) { KeReleaseSpinlock(&gInterruptLock, cookie); return; }
    }
    if (route->Draining) {
        KeReleaseSpinlock(&gInterruptLock, cookie);
        KePanic(KE_PANIC_CODE_DRIVER, "IRQ: unexpected concurrent route drain\n");
    }
    route->Draining = true;
    _InterlockedExchange(&route->Enabled, 0);
    if (!route->Message)
        (void)gHal->InterruptController.MaskGsi(route->Gsi, true);
    const uint32_t cpu = route->Cpu;
    const uint8_t vector = route->Vector;
    _InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], 0);
    KeReleaseSpinlock(&gInterruptLock, cookie);

    // Drain both ISRs already invoking callbacks and readers that fetched the
    // route just before the unpublish. Reserved vectors remain unavailable.
    uint32_t spins = 0;
    while (route->Active || gCpuVectorReaders[cpu][vector]) {
        if (++spins == 100000000u)
            KePanic(KE_PANIC_CODE_DRIVER, "IRQ: route rundown timed out cpu=%u vector=%02x\n",
                    cpu, vector);
        _mm_pause();
    }

    cookie = KeAcquireSpinlock(&gInterruptLock);
    KE_INTERRUPT** link = &route->Handlers;
    while (*link && *link != interrupt) link = &(*link)->Next;
    if (*link != interrupt)
        KePanic(KE_PANIC_CODE_DRIVER, "IRQ: missing handler during disconnect\n");
    *link = interrupt->Next;
    interrupt->Used = false;
    interrupt->Route = 0;
    interrupt->Routine = 0;
    interrupt->Context = 0;
    interrupt->Program = 0;
    interrupt->ProgramContext = 0;
    interrupt->Affinity = 0;
    interrupt->Next = 0;
    if (!route->Handlers) {
        route->Used = false;
        // Retain the vector while the target APIC drains any pending delivery.
        // Only its target CPU can inspect its IRR and ISR registers.
        gCpuVectorRetired[cpu][vector] = true;
        gCpuVectorQuietTicks[cpu][vector] = 0;
    } else {
        _InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
        _InterlockedExchange(&route->Enabled, 1);
        if (!route->Message)
            (void)gHal->InterruptController.MaskGsi(route->Gsi, false);
    }
    route->Draining = false;
    route->Closing = false;
    KeReleaseSpinlock(&gInterruptLock, cookie);
}

// The local timer gives a retiring vector a grace period on its owning CPU.
// Source drivers must disable message generation before disconnecting. A
// vector is reusable after consecutive timer checks find no APIC delivery.
static void KiReapRetiredVectors(void) {
    const uint32_t cpu = KeGetCurrentCpuIndex();
    if (cpu >= KE_MAX_CPUS) return;
    uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
    for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
        if (!gCpuVectorRetired[cpu][v]) continue;
        if (gCpuVectorReaders[cpu][v] ||
            gHal->InterruptController.VectorPending((uint8_t)v)) {
            gCpuVectorQuietTicks[cpu][v] = 0;
        } else if (++gCpuVectorQuietTicks[cpu][v] >= 3u) {
            gCpuVectorRetired[cpu][v] = false;
            gCpuVectorReserved[cpu][v] = false;
            gCpuVectorQuietTicks[cpu][v] = 0;
        }
    }
    KeReleaseSpinlock(&gInterruptLock, cookie);
}

void KeIrqRegisterLocalTimerHandler(void (*handler)(void)) {
    if (!handler || !gHal)
        KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid local timer registration\n");
    for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
        if (gLocalTimerHandlers[i] == handler) return;
    if (gLocalTimerHandlerCount >= 2u)
        KePanic(KE_PANIC_CODE_INIT, "IRQ: local timer handler count exceeded\n");
    gLocalTimerHandlers[gLocalTimerHandlerCount++] = handler;
}

void KeIrqDispatch(uint8_t vector) {
    if (!gHal) KePanic(KE_PANIC_CODE_INIT, "IRQ: uninitialized dispatch\n");
    if (vector == KE_VECTOR_SPURIOUS) return;
    KIRQL target = (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST)
        ? KiVectorIrql(vector) : (vector == KE_VECTOR_CPU_TIMER ? (KIRQL)27 : (KIRQL)29);
    if (target < KeGetCurrentIrql()) target = KeGetCurrentIrql();
    KIRQL oldIrql = KeRaiseIrql(target);
    if (vector == KE_VECTOR_CPU_TIMER) {
        KiReapRetiredVectors();
        for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
            gLocalTimerHandlers[i]();
    } else if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
        uint32_t cpu = KeGetCurrentCpuIndex();
        if (cpu < KE_MAX_CPUS) {
            _InterlockedIncrement(&gCpuVectorReaders[cpu][vector]);
            KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)
                _InterlockedCompareExchangePointer(
                    (void* volatile*)&gCpuVectors[cpu][vector], 0, 0);
            if (route) {
                _InterlockedIncrement(&route->Active);
                if (route->Enabled) {
                    route->Delivered++;
                    for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
                        if (h->Used && h->Routine) h->Routine(h->Context);
                }
                _InterlockedDecrement(&route->Active);
            }
        }
    }
    gHal->InterruptController.SendEoi();
    if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
        uint32_t cpu = KeGetCurrentCpuIndex();
        if (cpu < KE_MAX_CPUS)
            _InterlockedDecrement(&gCpuVectorReaders[cpu][vector]);
    }
    KeLowerIrql(oldIrql);
}

void KiEnterInterrupt(void) {
    KeGetCurrentCpu()->InterruptDepth++;
}

void KiExitInterrupt(void) {
    KCPU* cpu = KeGetCurrentCpu();
    if (cpu->InterruptDepth) cpu->InterruptDepth--;
    if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
        THREAD* current = cpu->CurrentThread;
        if (current && !current->IsIdleThread && current->Owner &&
            !KeThreadExitInProgress(current) &&
            KeIsProcessTerminating(current->Owner) &&
            !KeThreadTerminationDeferred(current)) {
            KeExitThread(current, current->Owner->ExitCode);
            while (true) ;
        }
        KiDrainDpcQueue();
        current = cpu->CurrentThread;
        if (KeGetCurrentIrql() == PASSIVE_LEVEL &&
            (!current || !KeThreadExitInProgress(current)))
            KiScheduleFromInterrupt();
    }
}