657 lines
27 KiB
C
657 lines
27 KiB
C
// Interrupt manager. Per-CPU vector ownership is independent of physical sources.
|
|
|
|
#include <kernel/ke_irq.h>
|
|
#include <kernel/ke_sched.h>
|
|
#include <kernel/ke_cpu.h>
|
|
#include <kernel/ke_vectors.h>
|
|
#include <kernel/ke_irql.h>
|
|
#include <kernel/ke_dpc.h>
|
|
#include <kernel/ke_thread.h>
|
|
#include <kernel/ke_process.h>
|
|
#include <kernel/ke_debug.h>
|
|
#include <kernel/ke_spinlock.h>
|
|
#include <intrin.h>
|
|
|
|
#define KI_INTERRUPT_ROUTES 256u
|
|
#define KI_INTERRUPT_HANDLERS 512u
|
|
|
|
typedef struct KI_INTERRUPT_ROUTE KI_INTERRUPT_ROUTE;
|
|
struct KE_INTERRUPT {
|
|
KE_INTERRUPT_ROUTINE Routine;
|
|
void* Context;
|
|
KI_INTERRUPT_ROUTE* Route;
|
|
KE_INTERRUPT_MESSAGE_PROGRAM Program;
|
|
void* ProgramContext;
|
|
uint64_t Affinity;
|
|
struct KE_INTERRUPT* Next;
|
|
bool Used;
|
|
};
|
|
|
|
struct KI_INTERRUPT_ROUTE {
|
|
KE_INTERRUPT* Handlers;
|
|
uint32_t Gsi;
|
|
uint32_t Cpu;
|
|
uint8_t Vector;
|
|
uint16_t Flags;
|
|
bool Used;
|
|
bool Message;
|
|
bool Draining;
|
|
bool Closing;
|
|
bool MessageActive;
|
|
volatile long Active;
|
|
volatile long Enabled;
|
|
volatile uint64_t Delivered;
|
|
};
|
|
|
|
static KI_INTERRUPT_ROUTE gRoutes[KI_INTERRUPT_ROUTES];
|
|
static KE_INTERRUPT gHandlers[KI_INTERRUPT_HANDLERS];
|
|
static KI_INTERRUPT_ROUTE* volatile gCpuVectors[KE_MAX_CPUS][256];
|
|
// A reader is registered before it resolves the route pointer. A retired route
|
|
// waits for all readers on its former CPU/vector before releasing handler state.
|
|
static volatile long gCpuVectorReaders[KE_MAX_CPUS][256];
|
|
// Vector reservations outlive publication while a route is draining.
|
|
static bool gCpuVectorReserved[KE_MAX_CPUS][256];
|
|
static bool gCpuVectorRetired[KE_MAX_CPUS][256];
|
|
static uint8_t gCpuVectorQuietTicks[KE_MAX_CPUS][256];
|
|
static SPINLOCK gInterruptLock = SPINLOCK_INIT;
|
|
static const HAL_API* gHal;
|
|
static void (*gLocalTimerHandlers[2])(void);
|
|
static uint8_t gLocalTimerHandlerCount;
|
|
static uint32_t gPlacementCursor;
|
|
|
|
static KIRQL KiVectorIrql(uint8_t vector) {
|
|
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)((vector >> 4u) - 3u);
|
|
if (level >= HIGH_LEVEL) level = HIGH_LEVEL - 1u;
|
|
return (KIRQL)level;
|
|
}
|
|
|
|
void KeIrqInitialize(const HAL_API* hal) {
|
|
if (!hal || !hal->InterruptController.SendEoi ||
|
|
!hal->InterruptController.RouteGsi || !hal->InterruptController.ResolveIsa ||
|
|
!hal->InterruptController.ComposeMsi)
|
|
KePanic(KE_PANIC_CODE_INIT, "IRQ: incomplete interrupt-controller API\n");
|
|
gHal = hal;
|
|
}
|
|
|
|
static bool KiCpuAvailable(uint32_t index, uint64_t affinity) {
|
|
if (index >= gCpuCount || index >= KE_MAX_CPUS ||
|
|
(affinity && !(affinity & (1ull << index)))) return false;
|
|
KCPU* cpu = KeGetCpuByIndex(index);
|
|
return cpu && (cpu->IsBootProcessor || cpu->State >= KeCpuSchedulerActive) &&
|
|
cpu->HardwareId <= 255u;
|
|
}
|
|
|
|
static bool KiPickVector(uint64_t affinity, uint32_t* cpuOut, uint8_t* vectorOut) {
|
|
uint32_t count = gCpuCount;
|
|
if (!count || count > KE_MAX_CPUS) count = 1;
|
|
// Rotate the first candidate CPU, and allocate the least occupied vector.
|
|
uint32_t start = gPlacementCursor++ % count;
|
|
uint32_t bestLoad = UINT32_MAX;
|
|
bool found = false;
|
|
for (uint32_t i = 0; i < count; i++) {
|
|
uint32_t cpu = (start + i) % count;
|
|
if (!KiCpuAvailable(cpu, affinity)) continue;
|
|
uint32_t occupied = 0;
|
|
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++)
|
|
if (gCpuVectorReserved[cpu][v]) occupied++;
|
|
if (occupied >= KE_VECTOR_DEVICE_LAST - KE_VECTOR_DEVICE_FIRST + 1u || occupied >= bestLoad)
|
|
continue;
|
|
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
|
|
if (!gCpuVectorReserved[cpu][v]) {
|
|
bestLoad = occupied;
|
|
*cpuOut = cpu;
|
|
*vectorOut = (uint8_t)v;
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return found;
|
|
}
|
|
|
|
static KSTATUS KiReserveInterrupt(bool message, uint32_t gsi, uint16_t flags,
|
|
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context,
|
|
KE_INTERRUPT** out) {
|
|
if (!out || !routine || !gHal) return NTLSTATUS_INVALID_PARAMETER;
|
|
*out = (KE_INTERRUPT*)0;
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)0;
|
|
if (!message) {
|
|
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
|
|
if (gRoutes[i].Used && !gRoutes[i].Message && gRoutes[i].Gsi == gsi) {
|
|
if (gRoutes[i].Draining || gRoutes[i].Closing) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NOT_SUPPORTED;
|
|
}
|
|
if (gRoutes[i].Flags != flags ||
|
|
(affinity && !(affinity & (1ull << gRoutes[i].Cpu)))) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
}
|
|
route = &gRoutes[i];
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
bool newRoute = route == 0;
|
|
uint32_t cpu = 0;
|
|
uint8_t vector = 0;
|
|
if (newRoute) {
|
|
if (!KiPickVector(affinity, &cpu, &vector)) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++)
|
|
if (!gRoutes[i].Used) { route = &gRoutes[i]; break; }
|
|
if (!route) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
}
|
|
KE_INTERRUPT* handler = 0;
|
|
for (uint32_t i = 0; i < KI_INTERRUPT_HANDLERS; i++)
|
|
if (!gHandlers[i].Used) { handler = &gHandlers[i]; break; }
|
|
if (!handler) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
if (newRoute) {
|
|
route->Used = true;
|
|
route->Message = message;
|
|
route->Draining = false;
|
|
route->Closing = false;
|
|
route->MessageActive = false;
|
|
route->Gsi = gsi;
|
|
route->Cpu = cpu;
|
|
route->Vector = vector;
|
|
route->Flags = flags;
|
|
route->Handlers = 0;
|
|
route->Active = 0;
|
|
route->Enabled = 0;
|
|
route->Delivered = 0;
|
|
gCpuVectorReserved[cpu][vector] = true;
|
|
gCpuVectorRetired[cpu][vector] = false;
|
|
gCpuVectorQuietTicks[cpu][vector] = 0;
|
|
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
|
|
}
|
|
handler->Used = true;
|
|
handler->Routine = routine;
|
|
handler->Context = context;
|
|
handler->Route = route;
|
|
handler->Program = 0;
|
|
handler->ProgramContext = 0;
|
|
handler->Affinity = affinity;
|
|
handler->Next = route->Handlers;
|
|
route->Handlers = handler;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
if (newRoute && !message) {
|
|
KCPU* target = KeGetCpuByIndex(cpu);
|
|
KSTATUS st = gHal->InterruptController.RouteGsi(gsi, vector,
|
|
target->HardwareId, flags, true);
|
|
if (!KSUCCESS(st)) {
|
|
KeInterruptDisconnect(handler);
|
|
return st;
|
|
}
|
|
_InterlockedExchange(&route->Enabled, 1);
|
|
(void)gHal->InterruptController.MaskGsi(gsi, false);
|
|
} else if (newRoute) {
|
|
_InterlockedExchange(&route->Enabled, 1);
|
|
}
|
|
*out = handler;
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS KeInterruptConnectGsi(uint32_t gsi, uint16_t flags,
|
|
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
|
|
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS KeInterruptConnectIsa(uint8_t isaIrq, uint16_t sourceFlags, uint64_t affinity,
|
|
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
|
|
uint32_t gsi;
|
|
uint16_t flags;
|
|
if (!gHal || !gHal->InterruptController.ResolveIsa)
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
KSTATUS st = gHal->InterruptController.ResolveIsa(isaIrq, &gsi, &flags);
|
|
if (!KSUCCESS(st)) return st;
|
|
if (sourceFlags) flags = sourceFlags;
|
|
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS KeInterruptReserveMessage(uint64_t affinity,
|
|
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
|
|
return KiReserveInterrupt(true, UINT32_MAX, 0, affinity, routine, context, out);
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS KeInterruptComposeMessage(const KE_INTERRUPT* interrupt,
|
|
NTL_DDK_MSI_MESSAGE* out) {
|
|
if (!interrupt || !out || !gHal) return NTLSTATUS_INVALID_PARAMETER;
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
if (!interrupt->Used || !interrupt->Route ||
|
|
!interrupt->Route->Message || interrupt->Route->Draining) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
}
|
|
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
|
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
|
|
HAL_MSI_MESSAGE message;
|
|
KSTATUS st = cpu ? gHal->InterruptController.ComposeMsi(
|
|
cpu->HardwareId, route->Vector, &message) : NTLSTATUS_INVALID_PARAMETER;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
if (!KSUCCESS(st)) return st;
|
|
out->Address = message.Address;
|
|
out->Data = message.Data;
|
|
out->Reserved = 0;
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS KeInterruptQuery(const KE_INTERRUPT* interrupt,
|
|
KE_INTERRUPT_INFO* out) {
|
|
if (!interrupt || !out) return NTLSTATUS_INVALID_PARAMETER;
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
if (!interrupt->Used || !interrupt->Route || interrupt->Route->Draining) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
}
|
|
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
|
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
|
|
out->CpuIndex = route->Cpu;
|
|
out->HardwareId = cpu ? cpu->HardwareId : UINT32_MAX;
|
|
out->Gsi = route->Gsi;
|
|
out->Flags = route->Flags;
|
|
out->Vector = route->Vector;
|
|
out->IsMessage = route->Message ? 1u : 0u;
|
|
out->Delivered = route->Delivered;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
// A message device registers its update operation once PCI has enabled the source.
|
|
__declspec(dllexport) KSTATUS KeInterruptBindMessageSource(KE_INTERRUPT* interrupt,
|
|
KE_INTERRUPT_MESSAGE_PROGRAM program, void* context) {
|
|
if (!interrupt || !program) return NTLSTATUS_INVALID_PARAMETER;
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
|
|
(interrupt->Route->Draining || interrupt->Route->Closing) || interrupt->Program) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
}
|
|
interrupt->Program = program;
|
|
interrupt->ProgramContext = context;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
// The bus owner enables migration only after the device source is programmed.
|
|
__declspec(dllexport) KSTATUS KeInterruptActivateMessage(KE_INTERRUPT* interrupt) {
|
|
if (!interrupt) return NTLSTATUS_INVALID_PARAMETER;
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
|
|
(interrupt->Route->Draining || interrupt->Route->Closing) || !interrupt->Program) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
}
|
|
interrupt->Route->MessageActive = true;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
// Moves the complete physical source to one new CPU/vector, including PCI
|
|
// message programming. The old vector remains quarantined through APIC drain.
|
|
__declspec(dllexport) KSTATUS KeInterruptSetAffinity(KE_INTERRUPT* interrupt,
|
|
uint64_t permittedCpuMask) {
|
|
if (!interrupt || !permittedCpuMask || !gHal ||
|
|
KeGetCurrentIrql() != PASSIVE_LEVEL || KeGetCurrentCpu()->InterruptDepth)
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
if (!interrupt->Used || !interrupt->Route ||
|
|
interrupt->Route->Draining || interrupt->Route->Closing) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
}
|
|
KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
|
if (route->Message && !route->MessageActive) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NOT_SUPPORTED;
|
|
}
|
|
// One shared line must satisfy every handler's affinity constraint.
|
|
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next) {
|
|
if (h->Affinity && !(h->Affinity & permittedCpuMask)) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NOT_SUPPORTED;
|
|
}
|
|
if (route->Message && (!h->Program || h != interrupt)) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NOT_SUPPORTED;
|
|
}
|
|
}
|
|
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
|
|
if (h->Affinity) permittedCpuMask &= h->Affinity;
|
|
if (!permittedCpuMask) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NOT_SUPPORTED;
|
|
}
|
|
if (permittedCpuMask & (1ull << route->Cpu)) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
uint32_t newCpu = 0;
|
|
uint8_t newVector = 0;
|
|
if (!KiPickVector(permittedCpuMask, &newCpu, &newVector)) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
const uint32_t oldCpu = route->Cpu;
|
|
const uint8_t oldVector = route->Vector;
|
|
gCpuVectorReserved[newCpu][newVector] = true;
|
|
route->Draining = true;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
|
|
NTL_DDK_MSI_MESSAGE updated = {0};
|
|
if (route->Message) {
|
|
const KCPU* destination = KeGetCpuByIndex(newCpu);
|
|
HAL_MSI_MESSAGE message;
|
|
KSTATUS st = gHal->InterruptController.ComposeMsi(destination->HardwareId,
|
|
newVector, &message);
|
|
if (!KSUCCESS(st)) goto rollback_reservation;
|
|
updated.Address = message.Address;
|
|
updated.Data = message.Data;
|
|
st = interrupt->Program(interrupt->ProgramContext, KeInterruptMessageQuiesce, 0);
|
|
if (!KSUCCESS(st)) goto rollback_reservation;
|
|
} else {
|
|
KSTATUS st = gHal->InterruptController.MaskGsi(route->Gsi, true);
|
|
if (!KSUCCESS(st)) goto rollback_reservation;
|
|
}
|
|
|
|
_InterlockedExchange(&route->Enabled, 0);
|
|
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], 0);
|
|
uint32_t spins = 0;
|
|
while (route->Active || gCpuVectorReaders[oldCpu][oldVector]) {
|
|
if (++spins == 100000000u)
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: affinity migration drain timeout\n");
|
|
_mm_pause();
|
|
}
|
|
|
|
KSTATUS update = NTLSTATUS_SUCCESS;
|
|
if (route->Message)
|
|
update = interrupt->Program(interrupt->ProgramContext,
|
|
KeInterruptMessageProgram, &updated);
|
|
else {
|
|
const KCPU* destination = KeGetCpuByIndex(newCpu);
|
|
update = gHal->InterruptController.RouteGsi(route->Gsi, newVector,
|
|
destination->HardwareId, route->Flags, true);
|
|
}
|
|
if (!KSUCCESS(update)) {
|
|
// Restore the existing route and re-enable its original source.
|
|
if (route->Message) {
|
|
const KCPU* previous = KeGetCpuByIndex(oldCpu);
|
|
HAL_MSI_MESSAGE msg;
|
|
if (!KSUCCESS(gHal->InterruptController.ComposeMsi(previous->HardwareId,
|
|
oldVector, &msg)))
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original MSI route\n");
|
|
NTL_DDK_MSI_MESSAGE original = {msg.Address, msg.Data, 0};
|
|
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
|
|
KeInterruptMessageProgram, &original)))
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore MSI message\n");
|
|
} else {
|
|
const KCPU* previous = KeGetCpuByIndex(oldCpu);
|
|
if (!KSUCCESS(gHal->InterruptController.RouteGsi(route->Gsi, oldVector,
|
|
previous->HardwareId, route->Flags, true)))
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original GSI route\n");
|
|
}
|
|
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], route);
|
|
_InterlockedExchange(&route->Enabled, 1);
|
|
if (route->Message) {
|
|
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
|
|
KeInterruptMessageResume, 0)))
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI restore resume failed\n");
|
|
} else (void)gHal->InterruptController.MaskGsi(route->Gsi, false);
|
|
cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
route->Draining = false;
|
|
gCpuVectorReserved[newCpu][newVector] = false;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return update;
|
|
}
|
|
|
|
cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
route->Cpu = newCpu;
|
|
route->Vector = newVector;
|
|
gCpuVectorRetired[oldCpu][oldVector] = true;
|
|
gCpuVectorQuietTicks[oldCpu][oldVector] = 0;
|
|
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[newCpu][newVector], route);
|
|
_InterlockedExchange(&route->Enabled, 1);
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
|
|
if (route->Message) {
|
|
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
|
|
KeInterruptMessageResume, 0)))
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI migration resume failed\n");
|
|
} else if (!KSUCCESS(gHal->InterruptController.MaskGsi(route->Gsi, false)))
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: GSI migration unmask failed\n");
|
|
cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
route->Draining = false;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_SUCCESS;
|
|
|
|
rollback_reservation:
|
|
cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
gCpuVectorReserved[newCpu][newVector] = false;
|
|
route->Draining = false;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return NTLSTATUS_NOT_SUPPORTED;
|
|
}
|
|
|
|
// The scheduler must evacuate every device route before stopping a processor.
|
|
KSTATUS KiInterruptPrepareCpuOffline(uint32_t cpuIndex) {
|
|
if (cpuIndex >= gCpuCount || cpuIndex >= KE_MAX_CPUS ||
|
|
KeGetCurrentIrql() != PASSIVE_LEVEL) return NTLSTATUS_INVALID_PARAMETER;
|
|
uint64_t allowed = 0;
|
|
for (uint32_t i = 0; i < gCpuCount && i < 64u; i++)
|
|
if (i != cpuIndex && KiCpuAvailable(i, 0)) allowed |= 1ull << i;
|
|
if (!allowed) return NTLSTATUS_NOT_SUPPORTED;
|
|
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
|
|
KI_INTERRUPT_ROUTE* route = &gRoutes[i];
|
|
if (route->Used && route->Cpu == cpuIndex && route->Handlers) {
|
|
KSTATUS st = KeInterruptSetAffinity(route->Handlers, allowed);
|
|
if (!KSUCCESS(st)) return st;
|
|
}
|
|
}
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
// Serializes device-source disable with affinity migration. A caller must
|
|
// close the route before masking the PCI source, preventing a racing resume.
|
|
__declspec(dllexport) void KeInterruptBeginDisconnect(KE_INTERRUPT* interrupt) {
|
|
if (!interrupt || KeGetCurrentIrql() != PASSIVE_LEVEL ||
|
|
KeGetCurrentCpu()->InterruptDepth)
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: invalid disconnect preparation context\n");
|
|
for (uint32_t spins = 0;; spins++) {
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
if (!interrupt->Used || !interrupt->Route) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return;
|
|
}
|
|
KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
|
if (!route->Draining && !route->Closing) {
|
|
route->Closing = true;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return;
|
|
}
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
if (spins >= 100000000u)
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect serialization timed out\n");
|
|
_mm_pause();
|
|
}
|
|
}
|
|
|
|
// Disconnect runs in thread context after the device has stopped producing messages.
|
|
// The route is unpublished before draining readers, so a concurrent dispatcher
|
|
// either holds a counted reference or observes an empty vector.
|
|
__declspec(dllexport) void KeInterruptDisconnect(KE_INTERRUPT* interrupt) {
|
|
if (!interrupt) return;
|
|
if (KeGetCurrentCpu()->InterruptDepth || KeGetCurrentIrql() != PASSIVE_LEVEL)
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect requires PASSIVE_LEVEL\n");
|
|
|
|
// PCI callers have already serialized source disable. A direct GSI
|
|
// caller prepares the route here before masking it inside this routine.
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
if (!interrupt->Used || !interrupt->Route) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
return;
|
|
}
|
|
KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
|
if (!route->Closing) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
KeInterruptBeginDisconnect(interrupt);
|
|
cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
route = interrupt->Route;
|
|
if (!route) { KeReleaseSpinlock(&gInterruptLock, cookie); return; }
|
|
}
|
|
if (route->Draining) {
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: unexpected concurrent route drain\n");
|
|
}
|
|
route->Draining = true;
|
|
_InterlockedExchange(&route->Enabled, 0);
|
|
if (!route->Message)
|
|
(void)gHal->InterruptController.MaskGsi(route->Gsi, true);
|
|
const uint32_t cpu = route->Cpu;
|
|
const uint8_t vector = route->Vector;
|
|
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], 0);
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
|
|
// Drain both ISRs already invoking callbacks and readers that fetched the
|
|
// route just before the unpublish. Reserved vectors remain unavailable.
|
|
uint32_t spins = 0;
|
|
while (route->Active || gCpuVectorReaders[cpu][vector]) {
|
|
if (++spins == 100000000u)
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: route rundown timed out cpu=%u vector=%02x\n",
|
|
cpu, vector);
|
|
_mm_pause();
|
|
}
|
|
|
|
cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
KE_INTERRUPT** link = &route->Handlers;
|
|
while (*link && *link != interrupt) link = &(*link)->Next;
|
|
if (*link != interrupt)
|
|
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: missing handler during disconnect\n");
|
|
*link = interrupt->Next;
|
|
interrupt->Used = false;
|
|
interrupt->Route = 0;
|
|
interrupt->Routine = 0;
|
|
interrupt->Context = 0;
|
|
interrupt->Program = 0;
|
|
interrupt->ProgramContext = 0;
|
|
interrupt->Affinity = 0;
|
|
interrupt->Next = 0;
|
|
if (!route->Handlers) {
|
|
route->Used = false;
|
|
// Retain the vector while the target APIC drains any pending delivery.
|
|
// Only its target CPU can inspect its IRR and ISR registers.
|
|
gCpuVectorRetired[cpu][vector] = true;
|
|
gCpuVectorQuietTicks[cpu][vector] = 0;
|
|
} else {
|
|
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
|
|
_InterlockedExchange(&route->Enabled, 1);
|
|
if (!route->Message)
|
|
(void)gHal->InterruptController.MaskGsi(route->Gsi, false);
|
|
}
|
|
route->Draining = false;
|
|
route->Closing = false;
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
}
|
|
|
|
// The local timer gives a retiring vector a grace period on its owning CPU.
|
|
// Source drivers must disable message generation before disconnecting. A
|
|
// vector is reusable after consecutive timer checks find no APIC delivery.
|
|
static void KiReapRetiredVectors(void) {
|
|
const uint32_t cpu = KeGetCurrentCpuIndex();
|
|
if (cpu >= KE_MAX_CPUS) return;
|
|
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
|
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
|
|
if (!gCpuVectorRetired[cpu][v]) continue;
|
|
if (gCpuVectorReaders[cpu][v] ||
|
|
gHal->InterruptController.VectorPending((uint8_t)v)) {
|
|
gCpuVectorQuietTicks[cpu][v] = 0;
|
|
} else if (++gCpuVectorQuietTicks[cpu][v] >= 3u) {
|
|
gCpuVectorRetired[cpu][v] = false;
|
|
gCpuVectorReserved[cpu][v] = false;
|
|
gCpuVectorQuietTicks[cpu][v] = 0;
|
|
}
|
|
}
|
|
KeReleaseSpinlock(&gInterruptLock, cookie);
|
|
}
|
|
|
|
void KeIrqRegisterLocalTimerHandler(void (*handler)(void)) {
|
|
if (!handler || !gHal)
|
|
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid local timer registration\n");
|
|
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
|
|
if (gLocalTimerHandlers[i] == handler) return;
|
|
if (gLocalTimerHandlerCount >= 2u)
|
|
KePanic(KE_PANIC_CODE_INIT, "IRQ: local timer handler count exceeded\n");
|
|
gLocalTimerHandlers[gLocalTimerHandlerCount++] = handler;
|
|
}
|
|
|
|
void KeIrqDispatch(uint8_t vector) {
|
|
if (!gHal) KePanic(KE_PANIC_CODE_INIT, "IRQ: uninitialized dispatch\n");
|
|
if (vector == KE_VECTOR_SPURIOUS) return;
|
|
KIRQL target = (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST)
|
|
? KiVectorIrql(vector) : (vector == KE_VECTOR_CPU_TIMER ? (KIRQL)27 : (KIRQL)29);
|
|
if (target < KeGetCurrentIrql()) target = KeGetCurrentIrql();
|
|
KIRQL oldIrql = KeRaiseIrql(target);
|
|
if (vector == KE_VECTOR_CPU_TIMER) {
|
|
KiReapRetiredVectors();
|
|
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
|
|
gLocalTimerHandlers[i]();
|
|
} else if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
|
|
uint32_t cpu = KeGetCurrentCpuIndex();
|
|
if (cpu < KE_MAX_CPUS) {
|
|
_InterlockedIncrement(&gCpuVectorReaders[cpu][vector]);
|
|
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)
|
|
_InterlockedCompareExchangePointer(
|
|
(void* volatile*)&gCpuVectors[cpu][vector], 0, 0);
|
|
if (route) {
|
|
_InterlockedIncrement(&route->Active);
|
|
if (route->Enabled) {
|
|
route->Delivered++;
|
|
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
|
|
if (h->Used && h->Routine) h->Routine(h->Context);
|
|
}
|
|
_InterlockedDecrement(&route->Active);
|
|
}
|
|
}
|
|
}
|
|
gHal->InterruptController.SendEoi();
|
|
if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
|
|
uint32_t cpu = KeGetCurrentCpuIndex();
|
|
if (cpu < KE_MAX_CPUS)
|
|
_InterlockedDecrement(&gCpuVectorReaders[cpu][vector]);
|
|
}
|
|
KeLowerIrql(oldIrql);
|
|
}
|
|
|
|
void KiEnterInterrupt(void) {
|
|
KeGetCurrentCpu()->InterruptDepth++;
|
|
}
|
|
|
|
void KiExitInterrupt(void) {
|
|
KCPU* cpu = KeGetCurrentCpu();
|
|
if (cpu->InterruptDepth) cpu->InterruptDepth--;
|
|
if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
|
|
THREAD* current = cpu->CurrentThread;
|
|
if (current && !current->IsIdleThread && current->Owner &&
|
|
!KeThreadExitInProgress(current) &&
|
|
KeIsProcessTerminating(current->Owner) &&
|
|
!KeThreadTerminationDeferred(current)) {
|
|
KeExitThread(current, current->Owner->ExitCode);
|
|
while (true) ;
|
|
}
|
|
KiDrainDpcQueue();
|
|
current = cpu->CurrentThread;
|
|
if (KeGetCurrentIrql() == PASSIVE_LEVEL &&
|
|
(!current || !KeThreadExitInProgress(current)))
|
|
KiScheduleFromInterrupt();
|
|
}
|
|
}
|