replace nob with ms' build, move disk creation to build.bat, major comment cleanup, remove old and stupid tests, overhaul interrupt management with per-CPU vector allocation, affinity migration and safe teardown, add proper IOAPIC routing and PCI MSI/MSI-X/INTx support, migrate drivers to the new interrupt API, integrate ACPICA with a proper OS services layer, move ACPI firmware events, power button handling and S5 shutdown into acpi.sys, add parallel driver startup and atomic publication, replace PnP arrays with dynamic lists, redesign service startup again to be asynchronous, add delayed startup mode for services, allow omitting keys in services.ini, fix acpi ec deferred work handling, fix kernel threads having garbage exit code.
This commit is contained in:
maksw committed
1 parent
d36f19509c
commit
36eef8f58d
696 files changed
+147281
-7530
No files matched your search
+599
-98
@@ -1,136 +1,643 @@
|
||||
// Interrupt manager. This file owns IRQ routing, handler dispatch, and post-interrupt scheduling hooks.
|
||||
// Interrupt manager. Per-CPU vector ownership is independent of physical sources.
|
||||
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_cpu.h>
|
||||
#include <kernel/ke_idt.h>
|
||||
#include <kernel/ke_vectors.h>
|
||||
#include <kernel/ke_irql.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_process.h>
|
||||
#include <kernel/hal/hal_api.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <intrin.h>
|
||||
|
||||
#define MAX_IRQS 24
|
||||
#define MAX_IRQ_HANDLERS 8
|
||||
#define MSI_IRQ_FIRST 16u
|
||||
#define MSI_IRQ_LAST 23u
|
||||
#define KI_INTERRUPT_ROUTES 256u
|
||||
#define KI_INTERRUPT_HANDLERS 512u
|
||||
|
||||
typedef struct IRQ_HANDLER_LIST {
|
||||
void (*Handlers[MAX_IRQ_HANDLERS])(void);
|
||||
uint8_t Count;
|
||||
} IRQ_HANDLER_LIST;
|
||||
typedef struct KI_INTERRUPT_ROUTE KI_INTERRUPT_ROUTE;
|
||||
struct KE_INTERRUPT {
|
||||
KE_INTERRUPT_ROUTINE Routine;
|
||||
void* Context;
|
||||
KI_INTERRUPT_ROUTE* Route;
|
||||
KE_INTERRUPT_MESSAGE_PROGRAM Program;
|
||||
void* ProgramContext;
|
||||
uint64_t Affinity;
|
||||
struct KE_INTERRUPT* Next;
|
||||
bool Used;
|
||||
};
|
||||
|
||||
static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
|
||||
static bool gIrqSoftwareVector[MAX_IRQS];
|
||||
struct KI_INTERRUPT_ROUTE {
|
||||
KE_INTERRUPT* Handlers;
|
||||
uint32_t Gsi;
|
||||
uint32_t Cpu;
|
||||
uint8_t Vector;
|
||||
uint16_t Flags;
|
||||
bool Used;
|
||||
bool Message;
|
||||
bool Draining;
|
||||
bool Closing;
|
||||
bool MessageActive;
|
||||
volatile long Active;
|
||||
volatile long Enabled;
|
||||
volatile uint64_t Delivered;
|
||||
};
|
||||
|
||||
static KI_INTERRUPT_ROUTE gRoutes[KI_INTERRUPT_ROUTES];
|
||||
static KE_INTERRUPT gHandlers[KI_INTERRUPT_HANDLERS];
|
||||
static KI_INTERRUPT_ROUTE* volatile gCpuVectors[KE_MAX_CPUS][256];
|
||||
// A reader is registered before it resolves the route pointer. A retired route
|
||||
// waits for all readers on its former CPU/vector before releasing handler state.
|
||||
static volatile long gCpuVectorReaders[KE_MAX_CPUS][256];
|
||||
// Vector reservations outlive publication while a route is draining.
|
||||
static bool gCpuVectorReserved[KE_MAX_CPUS][256];
|
||||
static bool gCpuVectorRetired[KE_MAX_CPUS][256];
|
||||
static uint8_t gCpuVectorQuietTicks[KE_MAX_CPUS][256];
|
||||
static SPINLOCK gInterruptLock = SPINLOCK_INIT;
|
||||
static const HAL_API* gHal;
|
||||
static void (*gLocalTimerHandlers[2])(void);
|
||||
static uint8_t gLocalTimerHandlerCount;
|
||||
static uint32_t gPlacementCursor;
|
||||
|
||||
// Implements the irq To Irql operation.
|
||||
static KIRQL KiIrqToIrql(uint8_t irq) {
|
||||
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)irq;
|
||||
if (level >= HIGH_LEVEL)
|
||||
level = (uint32_t)(HIGH_LEVEL - 1);
|
||||
static KIRQL KiVectorIrql(uint8_t vector) {
|
||||
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)((vector >> 4u) - 3u);
|
||||
if (level >= HIGH_LEVEL) level = HIGH_LEVEL - 1u;
|
||||
return (KIRQL)level;
|
||||
}
|
||||
|
||||
// Implements the irq Initialize operation.
|
||||
void KeIrqInitialize(const HAL_API* hal) {
|
||||
if (!hal || !hal->InterruptController.RouteIrq || !hal->InterruptController.SendEoi)
|
||||
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid interrupt-controller API during initialization\n");
|
||||
if (!hal || !hal->InterruptController.SendEoi ||
|
||||
!hal->InterruptController.RouteGsi || !hal->InterruptController.ResolveIsa ||
|
||||
!hal->InterruptController.ComposeMsi)
|
||||
KePanic(KE_PANIC_CODE_INIT, "IRQ: incomplete interrupt-controller API\n");
|
||||
gHal = hal;
|
||||
}
|
||||
|
||||
// Implements the irq Allocate Msi Vector operation.
|
||||
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
|
||||
if (!outIrq) return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
for (uint8_t irq = MSI_IRQ_FIRST; irq <= MSI_IRQ_LAST; irq++) {
|
||||
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
|
||||
if (!gIrqSoftwareVector[irq] && list->Count == 0) {
|
||||
gIrqSoftwareVector[irq] = true;
|
||||
*outIrq = irq;
|
||||
DbgPrint("KeIrq: allocated MSI vector slot IRQ %u vector 0x%02x\n",
|
||||
irq, 0x20u + irq);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
static bool KiCpuAvailable(uint32_t index, uint64_t affinity) {
|
||||
if (index >= gCpuCount || index >= KE_MAX_CPUS ||
|
||||
(affinity && !(affinity & (1ull << index)))) return false;
|
||||
KCPU* cpu = KeGetCpuByIndex(index);
|
||||
return cpu && (cpu->IsBootProcessor || cpu->State >= KeCpuSchedulerActive) &&
|
||||
cpu->HardwareId <= 255u;
|
||||
}
|
||||
|
||||
// Implements the irq Register operation.
|
||||
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
|
||||
if (irq >= MAX_IRQS || !handler) return;
|
||||
if (!gHal)
|
||||
KePanic(KE_PANIC_CODE_INIT, "IRQ: registration before interrupt-manager initialization\n");
|
||||
static bool KiPickVector(uint64_t affinity, uint32_t* cpuOut, uint8_t* vectorOut) {
|
||||
uint32_t count = gCpuCount;
|
||||
if (!count || count > KE_MAX_CPUS) count = 1;
|
||||
// Rotate the first candidate CPU, and allocate the least occupied vector.
|
||||
uint32_t start = gPlacementCursor++ % count;
|
||||
uint32_t bestLoad = UINT32_MAX;
|
||||
bool found = false;
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
uint32_t cpu = (start + i) % count;
|
||||
if (!KiCpuAvailable(cpu, affinity)) continue;
|
||||
uint32_t occupied = 0;
|
||||
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++)
|
||||
if (gCpuVectorReserved[cpu][v]) occupied++;
|
||||
if (occupied >= KE_VECTOR_DEVICE_LAST - KE_VECTOR_DEVICE_FIRST + 1u || occupied >= bestLoad)
|
||||
continue;
|
||||
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
|
||||
if (!gCpuVectorReserved[cpu][v]) {
|
||||
bestLoad = occupied;
|
||||
*cpuOut = cpu;
|
||||
*vectorOut = (uint8_t)v;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
|
||||
for (uint8_t i = 0; i < list->Count; i++) {
|
||||
if (list->Handlers[i] == handler)
|
||||
return;
|
||||
static KSTATUS KiReserveInterrupt(bool message, uint32_t gsi, uint16_t flags,
|
||||
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context,
|
||||
KE_INTERRUPT** out) {
|
||||
if (!out || !routine || !gHal) return NTLSTATUS_INVALID_PARAMETER;
|
||||
*out = (KE_INTERRUPT*)0;
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)0;
|
||||
if (!message) {
|
||||
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
|
||||
if (gRoutes[i].Used && !gRoutes[i].Message && gRoutes[i].Gsi == gsi) {
|
||||
if (gRoutes[i].Draining || gRoutes[i].Closing) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
}
|
||||
if (gRoutes[i].Flags != flags ||
|
||||
(affinity && !(affinity & (1ull << gRoutes[i].Cpu)))) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
route = &gRoutes[i];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
bool newRoute = route == 0;
|
||||
uint32_t cpu = 0;
|
||||
uint8_t vector = 0;
|
||||
if (newRoute) {
|
||||
if (!KiPickVector(affinity, &cpu, &vector)) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
}
|
||||
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++)
|
||||
if (!gRoutes[i].Used) { route = &gRoutes[i]; break; }
|
||||
if (!route) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
}
|
||||
}
|
||||
KE_INTERRUPT* handler = 0;
|
||||
for (uint32_t i = 0; i < KI_INTERRUPT_HANDLERS; i++)
|
||||
if (!gHandlers[i].Used) { handler = &gHandlers[i]; break; }
|
||||
if (!handler) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
}
|
||||
if (newRoute) {
|
||||
route->Used = true;
|
||||
route->Message = message;
|
||||
route->Draining = false;
|
||||
route->Closing = false;
|
||||
route->MessageActive = false;
|
||||
route->Gsi = gsi;
|
||||
route->Cpu = cpu;
|
||||
route->Vector = vector;
|
||||
route->Flags = flags;
|
||||
route->Handlers = 0;
|
||||
route->Active = 0;
|
||||
route->Enabled = 0;
|
||||
route->Delivered = 0;
|
||||
gCpuVectorReserved[cpu][vector] = true;
|
||||
gCpuVectorRetired[cpu][vector] = false;
|
||||
gCpuVectorQuietTicks[cpu][vector] = 0;
|
||||
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
|
||||
}
|
||||
handler->Used = true;
|
||||
handler->Routine = routine;
|
||||
handler->Context = context;
|
||||
handler->Route = route;
|
||||
handler->Program = 0;
|
||||
handler->ProgramContext = 0;
|
||||
handler->Affinity = affinity;
|
||||
handler->Next = route->Handlers;
|
||||
route->Handlers = handler;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
if (newRoute && !message) {
|
||||
KCPU* target = KeGetCpuByIndex(cpu);
|
||||
KSTATUS st = gHal->InterruptController.RouteGsi(gsi, vector,
|
||||
target->HardwareId, flags, true);
|
||||
if (!KSUCCESS(st)) {
|
||||
KeInterruptDisconnect(handler);
|
||||
return st;
|
||||
}
|
||||
_InterlockedExchange(&route->Enabled, 1);
|
||||
(void)gHal->InterruptController.MaskGsi(gsi, false);
|
||||
} else if (newRoute) {
|
||||
_InterlockedExchange(&route->Enabled, 1);
|
||||
}
|
||||
*out = handler;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInterruptConnectGsi(uint32_t gsi, uint16_t flags,
|
||||
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
|
||||
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInterruptConnectIsa(uint8_t isaIrq, uint16_t sourceFlags, uint64_t affinity,
|
||||
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
|
||||
uint32_t gsi;
|
||||
uint16_t flags;
|
||||
if (!gHal || !gHal->InterruptController.ResolveIsa)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
KSTATUS st = gHal->InterruptController.ResolveIsa(isaIrq, &gsi, &flags);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
if (sourceFlags) flags = sourceFlags;
|
||||
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInterruptReserveMessage(uint64_t affinity,
|
||||
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
|
||||
return KiReserveInterrupt(true, UINT32_MAX, 0, affinity, routine, context, out);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInterruptComposeMessage(const KE_INTERRUPT* interrupt,
|
||||
NTL_DDK_MSI_MESSAGE* out) {
|
||||
if (!interrupt || !out || !gHal) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
if (!interrupt->Used || !interrupt->Route ||
|
||||
!interrupt->Route->Message || interrupt->Route->Draining) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
||||
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
|
||||
HAL_MSI_MESSAGE message;
|
||||
KSTATUS st = cpu ? gHal->InterruptController.ComposeMsi(
|
||||
cpu->HardwareId, route->Vector, &message) : NTLSTATUS_INVALID_PARAMETER;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
out->Address = message.Address;
|
||||
out->Data = message.Data;
|
||||
out->Reserved = 0;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS KeInterruptQuery(const KE_INTERRUPT* interrupt,
|
||||
KE_INTERRUPT_INFO* out) {
|
||||
if (!interrupt || !out) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
if (!interrupt->Used || !interrupt->Route || interrupt->Route->Draining) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
||||
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
|
||||
out->CpuIndex = route->Cpu;
|
||||
out->HardwareId = cpu ? cpu->HardwareId : UINT32_MAX;
|
||||
out->Gsi = route->Gsi;
|
||||
out->Flags = route->Flags;
|
||||
out->Vector = route->Vector;
|
||||
out->IsMessage = route->Message ? 1u : 0u;
|
||||
out->Delivered = route->Delivered;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// A message device registers its update operation once PCI has enabled the source.
|
||||
__declspec(dllexport) KSTATUS KeInterruptBindMessageSource(KE_INTERRUPT* interrupt,
|
||||
KE_INTERRUPT_MESSAGE_PROGRAM program, void* context) {
|
||||
if (!interrupt || !program) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
|
||||
(interrupt->Route->Draining || interrupt->Route->Closing) || interrupt->Program) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
interrupt->Program = program;
|
||||
interrupt->ProgramContext = context;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// The bus owner enables migration only after the device source is programmed.
|
||||
__declspec(dllexport) KSTATUS KeInterruptActivateMessage(KE_INTERRUPT* interrupt) {
|
||||
if (!interrupt) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
|
||||
(interrupt->Route->Draining || interrupt->Route->Closing) || !interrupt->Program) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
interrupt->Route->MessageActive = true;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Moves the complete physical source to one new CPU/vector, including PCI
|
||||
// message programming. The old vector remains quarantined through APIC drain.
|
||||
__declspec(dllexport) KSTATUS KeInterruptSetAffinity(KE_INTERRUPT* interrupt,
|
||||
uint64_t permittedCpuMask) {
|
||||
if (!interrupt || !permittedCpuMask || !gHal ||
|
||||
KeGetCurrentIrql() != PASSIVE_LEVEL || KeGetCurrentCpu()->InterruptDepth)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
if (!interrupt->Used || !interrupt->Route ||
|
||||
interrupt->Route->Draining || interrupt->Route->Closing) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
||||
if (route->Message && !route->MessageActive) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
}
|
||||
// One shared line must satisfy every handler's affinity constraint.
|
||||
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next) {
|
||||
if (h->Affinity && !(h->Affinity & permittedCpuMask)) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
}
|
||||
if (route->Message && (!h->Program || h != interrupt)) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
}
|
||||
}
|
||||
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
|
||||
if (h->Affinity) permittedCpuMask &= h->Affinity;
|
||||
if (!permittedCpuMask) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
}
|
||||
if (permittedCpuMask & (1ull << route->Cpu)) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
uint32_t newCpu = 0;
|
||||
uint8_t newVector = 0;
|
||||
if (!KiPickVector(permittedCpuMask, &newCpu, &newVector)) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
}
|
||||
const uint32_t oldCpu = route->Cpu;
|
||||
const uint8_t oldVector = route->Vector;
|
||||
gCpuVectorReserved[newCpu][newVector] = true;
|
||||
route->Draining = true;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
|
||||
NTL_DDK_MSI_MESSAGE updated = {0};
|
||||
if (route->Message) {
|
||||
const KCPU* destination = KeGetCpuByIndex(newCpu);
|
||||
HAL_MSI_MESSAGE message;
|
||||
KSTATUS st = gHal->InterruptController.ComposeMsi(destination->HardwareId,
|
||||
newVector, &message);
|
||||
if (!KSUCCESS(st)) goto rollback_reservation;
|
||||
updated.Address = message.Address;
|
||||
updated.Data = message.Data;
|
||||
st = interrupt->Program(interrupt->ProgramContext, KeInterruptMessageQuiesce, 0);
|
||||
if (!KSUCCESS(st)) goto rollback_reservation;
|
||||
} else {
|
||||
KSTATUS st = gHal->InterruptController.MaskGsi(route->Gsi, true);
|
||||
if (!KSUCCESS(st)) goto rollback_reservation;
|
||||
}
|
||||
|
||||
if (list->Count >= MAX_IRQ_HANDLERS) {
|
||||
DbgPrint("KeIrqRegister: IRQ %u handler table full\n", irq);
|
||||
_InterlockedExchange(&route->Enabled, 0);
|
||||
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], 0);
|
||||
uint32_t spins = 0;
|
||||
while (route->Active || gCpuVectorReaders[oldCpu][oldVector]) {
|
||||
if (++spins == 100000000u)
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: affinity migration drain timeout\n");
|
||||
_mm_pause();
|
||||
}
|
||||
|
||||
KSTATUS update = NTLSTATUS_SUCCESS;
|
||||
if (route->Message)
|
||||
update = interrupt->Program(interrupt->ProgramContext,
|
||||
KeInterruptMessageProgram, &updated);
|
||||
else {
|
||||
const KCPU* destination = KeGetCpuByIndex(newCpu);
|
||||
update = gHal->InterruptController.RouteGsi(route->Gsi, newVector,
|
||||
destination->HardwareId, route->Flags, true);
|
||||
}
|
||||
if (!KSUCCESS(update)) {
|
||||
// Restore the existing route and re-enable its original source.
|
||||
if (route->Message) {
|
||||
const KCPU* previous = KeGetCpuByIndex(oldCpu);
|
||||
HAL_MSI_MESSAGE msg;
|
||||
if (!KSUCCESS(gHal->InterruptController.ComposeMsi(previous->HardwareId,
|
||||
oldVector, &msg)))
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original MSI route\n");
|
||||
NTL_DDK_MSI_MESSAGE original = {msg.Address, msg.Data, 0};
|
||||
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
|
||||
KeInterruptMessageProgram, &original)))
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore MSI message\n");
|
||||
} else {
|
||||
const KCPU* previous = KeGetCpuByIndex(oldCpu);
|
||||
if (!KSUCCESS(gHal->InterruptController.RouteGsi(route->Gsi, oldVector,
|
||||
previous->HardwareId, route->Flags, true)))
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original GSI route\n");
|
||||
}
|
||||
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], route);
|
||||
_InterlockedExchange(&route->Enabled, 1);
|
||||
if (route->Message) {
|
||||
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
|
||||
KeInterruptMessageResume, 0)))
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI restore resume failed\n");
|
||||
} else (void)gHal->InterruptController.MaskGsi(route->Gsi, false);
|
||||
cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
route->Draining = false;
|
||||
gCpuVectorReserved[newCpu][newVector] = false;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return update;
|
||||
}
|
||||
|
||||
cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
route->Cpu = newCpu;
|
||||
route->Vector = newVector;
|
||||
gCpuVectorRetired[oldCpu][oldVector] = true;
|
||||
gCpuVectorQuietTicks[oldCpu][oldVector] = 0;
|
||||
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[newCpu][newVector], route);
|
||||
_InterlockedExchange(&route->Enabled, 1);
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
|
||||
if (route->Message) {
|
||||
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
|
||||
KeInterruptMessageResume, 0)))
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI migration resume failed\n");
|
||||
} else if (!KSUCCESS(gHal->InterruptController.MaskGsi(route->Gsi, false)))
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: GSI migration unmask failed\n");
|
||||
cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
route->Draining = false;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
|
||||
rollback_reservation:
|
||||
cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
gCpuVectorReserved[newCpu][newVector] = false;
|
||||
route->Draining = false;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
// The scheduler must evacuate every device route before stopping a processor.
|
||||
KSTATUS KiInterruptPrepareCpuOffline(uint32_t cpuIndex) {
|
||||
if (cpuIndex >= gCpuCount || cpuIndex >= KE_MAX_CPUS ||
|
||||
KeGetCurrentIrql() != PASSIVE_LEVEL) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uint64_t allowed = 0;
|
||||
for (uint32_t i = 0; i < gCpuCount && i < 64u; i++)
|
||||
if (i != cpuIndex && KiCpuAvailable(i, 0)) allowed |= 1ull << i;
|
||||
if (!allowed) return NTLSTATUS_NOT_SUPPORTED;
|
||||
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
|
||||
KI_INTERRUPT_ROUTE* route = &gRoutes[i];
|
||||
if (route->Used && route->Cpu == cpuIndex && route->Handlers) {
|
||||
KSTATUS st = KeInterruptSetAffinity(route->Handlers, allowed);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
}
|
||||
}
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Serializes device-source disable with affinity migration. A caller must
|
||||
// close the route before masking the PCI source, preventing a racing resume.
|
||||
__declspec(dllexport) void KeInterruptBeginDisconnect(KE_INTERRUPT* interrupt) {
|
||||
if (!interrupt || KeGetCurrentIrql() != PASSIVE_LEVEL ||
|
||||
KeGetCurrentCpu()->InterruptDepth)
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: invalid disconnect preparation context\n");
|
||||
for (uint32_t spins = 0;; spins++) {
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
if (!interrupt->Used || !interrupt->Route) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return;
|
||||
}
|
||||
KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
||||
if (!route->Draining && !route->Closing) {
|
||||
route->Closing = true;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return;
|
||||
}
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
if (spins >= 100000000u)
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect serialization timed out\n");
|
||||
_mm_pause();
|
||||
}
|
||||
}
|
||||
|
||||
// Disconnect runs in thread context after the device has stopped producing messages.
|
||||
// The route is unpublished before draining readers, so a concurrent dispatcher
|
||||
// either holds a counted reference or observes an empty vector.
|
||||
__declspec(dllexport) void KeInterruptDisconnect(KE_INTERRUPT* interrupt) {
|
||||
if (!interrupt) return;
|
||||
if (KeGetCurrentCpu()->InterruptDepth || KeGetCurrentIrql() != PASSIVE_LEVEL)
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect requires PASSIVE_LEVEL\n");
|
||||
|
||||
// PCI callers have already serialized source disable. A direct GSI
|
||||
// caller prepares the route here before masking it inside this routine.
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
if (!interrupt->Used || !interrupt->Route) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
return;
|
||||
}
|
||||
KI_INTERRUPT_ROUTE* route = interrupt->Route;
|
||||
if (!route->Closing) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
KeInterruptBeginDisconnect(interrupt);
|
||||
cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
route = interrupt->Route;
|
||||
if (!route) { KeReleaseSpinlock(&gInterruptLock, cookie); return; }
|
||||
}
|
||||
if (route->Draining) {
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: unexpected concurrent route drain\n");
|
||||
}
|
||||
route->Draining = true;
|
||||
_InterlockedExchange(&route->Enabled, 0);
|
||||
if (!route->Message)
|
||||
(void)gHal->InterruptController.MaskGsi(route->Gsi, true);
|
||||
const uint32_t cpu = route->Cpu;
|
||||
const uint8_t vector = route->Vector;
|
||||
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], 0);
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
|
||||
list->Handlers[list->Count++] = handler;
|
||||
if (!gIrqSoftwareVector[irq])
|
||||
gHal->InterruptController.RouteIrq(irq);
|
||||
}
|
||||
|
||||
// Implements the irq Dispatch operation.
|
||||
void KeIrqDispatch(uint8_t irq) {
|
||||
if (!gHal)
|
||||
KePanic(KE_PANIC_CODE_INIT, "IRQ: dispatch before interrupt-manager initialization, irq=%u\n", irq);
|
||||
|
||||
KIRQL targetIrql = KiIrqToIrql(irq);
|
||||
KIRQL currentIrql = KeGetCurrentIrql();
|
||||
|
||||
// Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected
|
||||
// while IF is clear, but preserving monotonic IRQL keeps nesting safe.
|
||||
if (targetIrql < currentIrql)
|
||||
targetIrql = currentIrql;
|
||||
|
||||
KIRQL oldIrql = KeRaiseIrql(targetIrql);
|
||||
|
||||
if (irq == KE_VECTOR_RESCHEDULE_IPI) {
|
||||
// Nothing to do here. The sender has already set this CPU's
|
||||
// ReschedulePending flag. KiExitInterrupt consumes it after EOI and
|
||||
// after the interrupted frame unwinds to PASSIVE_LEVEL.
|
||||
} else if (irq < MAX_IRQS) {
|
||||
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
|
||||
for (uint8_t i = 0; i < list->Count; i++) {
|
||||
if (list->Handlers[i])
|
||||
list->Handlers[i]();
|
||||
}
|
||||
// Drain both ISRs already invoking callbacks and readers that fetched the
|
||||
// route just before the unpublish. Reserved vectors remain unavailable.
|
||||
uint32_t spins = 0;
|
||||
while (route->Active || gCpuVectorReaders[cpu][vector]) {
|
||||
if (++spins == 100000000u)
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: route rundown timed out cpu=%u vector=%02x\n",
|
||||
cpu, vector);
|
||||
_mm_pause();
|
||||
}
|
||||
|
||||
// Handlers are no longer allowed to context-switch directly. Timer IRQs
|
||||
// only request rescheduling; KiExitInterrupt performs the switch after this
|
||||
// function returns, so the EOI is guaranteed to be sent first. Keep the
|
||||
// software level at DIRQL until after the controller has been acknowledged.
|
||||
gHal->InterruptController.SendEoi();
|
||||
cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
KE_INTERRUPT** link = &route->Handlers;
|
||||
while (*link && *link != interrupt) link = &(*link)->Next;
|
||||
if (*link != interrupt)
|
||||
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: missing handler during disconnect\n");
|
||||
*link = interrupt->Next;
|
||||
interrupt->Used = false;
|
||||
interrupt->Route = 0;
|
||||
interrupt->Routine = 0;
|
||||
interrupt->Context = 0;
|
||||
interrupt->Program = 0;
|
||||
interrupt->ProgramContext = 0;
|
||||
interrupt->Affinity = 0;
|
||||
interrupt->Next = 0;
|
||||
if (!route->Handlers) {
|
||||
route->Used = false;
|
||||
// Retain the vector while the target APIC drains any pending delivery.
|
||||
// Only its target CPU can inspect its IRR and ISR registers.
|
||||
gCpuVectorRetired[cpu][vector] = true;
|
||||
gCpuVectorQuietTicks[cpu][vector] = 0;
|
||||
} else {
|
||||
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
|
||||
_InterlockedExchange(&route->Enabled, 1);
|
||||
if (!route->Message)
|
||||
(void)gHal->InterruptController.MaskGsi(route->Gsi, false);
|
||||
}
|
||||
route->Draining = false;
|
||||
route->Closing = false;
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
}
|
||||
|
||||
// The local timer gives a retiring vector a grace period on its owning CPU.
|
||||
// Source drivers must disable message generation before disconnecting. A
|
||||
// vector is reusable after consecutive timer checks find no APIC delivery.
|
||||
static void KiReapRetiredVectors(void) {
|
||||
const uint32_t cpu = KeGetCurrentCpuIndex();
|
||||
if (cpu >= KE_MAX_CPUS) return;
|
||||
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
|
||||
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
|
||||
if (!gCpuVectorRetired[cpu][v]) continue;
|
||||
if (gCpuVectorReaders[cpu][v] ||
|
||||
gHal->InterruptController.VectorPending((uint8_t)v)) {
|
||||
gCpuVectorQuietTicks[cpu][v] = 0;
|
||||
} else if (++gCpuVectorQuietTicks[cpu][v] >= 3u) {
|
||||
gCpuVectorRetired[cpu][v] = false;
|
||||
gCpuVectorReserved[cpu][v] = false;
|
||||
gCpuVectorQuietTicks[cpu][v] = 0;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gInterruptLock, cookie);
|
||||
}
|
||||
|
||||
void KeIrqRegisterLocalTimerHandler(void (*handler)(void)) {
|
||||
if (!handler || !gHal)
|
||||
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid local timer registration\n");
|
||||
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
|
||||
if (gLocalTimerHandlers[i] == handler) return;
|
||||
if (gLocalTimerHandlerCount >= 2u)
|
||||
KePanic(KE_PANIC_CODE_INIT, "IRQ: local timer handler count exceeded\n");
|
||||
gLocalTimerHandlers[gLocalTimerHandlerCount++] = handler;
|
||||
}
|
||||
|
||||
void KeIrqDispatch(uint8_t vector) {
|
||||
if (!gHal) KePanic(KE_PANIC_CODE_INIT, "IRQ: uninitialized dispatch\n");
|
||||
if (vector == KE_VECTOR_SPURIOUS) return;
|
||||
KIRQL target = (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST)
|
||||
? KiVectorIrql(vector) : (vector == KE_VECTOR_CPU_TIMER ? (KIRQL)27 : (KIRQL)29);
|
||||
if (target < KeGetCurrentIrql()) target = KeGetCurrentIrql();
|
||||
KIRQL oldIrql = KeRaiseIrql(target);
|
||||
if (vector == KE_VECTOR_CPU_TIMER) {
|
||||
KiReapRetiredVectors();
|
||||
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
|
||||
gLocalTimerHandlers[i]();
|
||||
} else if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
|
||||
uint32_t cpu = KeGetCurrentCpuIndex();
|
||||
if (cpu < KE_MAX_CPUS) {
|
||||
_InterlockedIncrement(&gCpuVectorReaders[cpu][vector]);
|
||||
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)
|
||||
_InterlockedCompareExchangePointer(
|
||||
(void* volatile*)&gCpuVectors[cpu][vector], 0, 0);
|
||||
if (route) {
|
||||
_InterlockedIncrement(&route->Active);
|
||||
if (route->Enabled) {
|
||||
route->Delivered++;
|
||||
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
|
||||
if (h->Used && h->Routine) h->Routine(h->Context);
|
||||
}
|
||||
_InterlockedDecrement(&route->Active);
|
||||
}
|
||||
}
|
||||
}
|
||||
gHal->InterruptController.SendEoi();
|
||||
if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
|
||||
uint32_t cpu = KeGetCurrentCpuIndex();
|
||||
if (cpu < KE_MAX_CPUS)
|
||||
_InterlockedDecrement(&gCpuVectorReaders[cpu][vector]);
|
||||
}
|
||||
KeLowerIrql(oldIrql);
|
||||
}
|
||||
|
||||
// Implements the enter Interrupt operation.
|
||||
void KiEnterInterrupt(void) {
|
||||
KeGetCurrentCpu()->InterruptDepth++;
|
||||
}
|
||||
|
||||
// Implements the exit Interrupt operation.
|
||||
void KiExitInterrupt(void) {
|
||||
KCPU* cpu = KeGetCurrentCpu();
|
||||
if (cpu->InterruptDepth)
|
||||
cpu->InterruptDepth--;
|
||||
|
||||
// A reschedule is only legal after the interrupt unwinds back to the
|
||||
// interrupted thread's PASSIVE context. If an interrupt hit DISPATCH/DIRQL
|
||||
// code, leave the request pending instead of switching out of a spinlock or
|
||||
// high-IRQL region.
|
||||
if (cpu->InterruptDepth) cpu->InterruptDepth--;
|
||||
if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
|
||||
THREAD* current = cpu->CurrentThread;
|
||||
if (current && !current->IsIdleThread && current->Owner &&
|
||||
@@ -140,13 +647,7 @@ void KiExitInterrupt(void) {
|
||||
KeExitThread(current, current->Owner->ExitCode);
|
||||
while (true) ;
|
||||
}
|
||||
|
||||
KiDrainDpcQueue();
|
||||
|
||||
// KeExitThread owns a terminal stack/rundown interval. A timer or IPI
|
||||
// may interrupt it, but switching away here would abandon teardown
|
||||
// before the process ExitEvent is necessarily published. The explicit
|
||||
// KeYield at the end of KeExitThread performs the one terminal switch.
|
||||
current = cpu->CurrentThread;
|
||||
if (KeGetCurrentIrql() == PASSIVE_LEVEL &&
|
||||
(!current || !KeThreadExitInProgress(current)))
|
||||
|
||||
Reference in new issue
Block a user