116 lines
4.1 KiB
C
116 lines
4.1 KiB
C
// Kernel IRQL model. This file synchronizes per-CPU IRQL with APIC task priority.
|
|
|
|
#include <stdbool.h>
|
|
#include <kernel/ke_irql.h>
|
|
#include <kernel/ke_cpu.h>
|
|
#include <kernel/ke_hal.h>
|
|
#include <kernel/ke_debug.h>
|
|
#include <kernel/ke_apc.h>
|
|
#include <intrin.h>
|
|
|
|
// Implements the irql Name operation.
|
|
__declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
|
|
switch (irql) {
|
|
case PASSIVE_LEVEL: return "PASSIVE_LEVEL";
|
|
case APC_LEVEL: return "APC_LEVEL";
|
|
case DISPATCH_LEVEL: return "DISPATCH_LEVEL";
|
|
case HIGH_LEVEL: return "HIGH_LEVEL";
|
|
default:
|
|
if (irql >= DIRQL_BASE && irql < HIGH_LEVEL)
|
|
return "DIRQL";
|
|
return "INVALID_IRQL";
|
|
}
|
|
}
|
|
|
|
// Returns current Irql.
|
|
__declspec(dllexport) KIRQL KeGetCurrentIrql(void) {
|
|
return KeGetCurrentCpu()->CurrentIrql;
|
|
}
|
|
|
|
// IRQL classes 16-25 map to vectors 0x30-0xC0. The TPR masks the current
|
|
// class and all lower-priority external interrupts. HIGH_LEVEL masks all.
|
|
static uint8_t KiIrqlTaskPriority(KIRQL level) {
|
|
if (level == HIGH_LEVEL) return 0xFFu;
|
|
if (level >= 28u) return 0xF0u;
|
|
if (level >= DIRQL_BASE) return (uint8_t)((level - DIRQL_BASE + 3u) << 4u);
|
|
if (level >= DISPATCH_LEVEL) return 0x20u;
|
|
if (level >= APC_LEVEL) return 0x10u;
|
|
return 0u;
|
|
}
|
|
|
|
// Applies the priority before publishing a raised IRQL and after a lowered IRQL.
|
|
static void KiSetIrql(KCPU* cpu, KIRQL level, bool raising) {
|
|
const HAL_API* hal = KeGetHal();
|
|
if (raising && hal && hal->InterruptController.SetTaskPriority)
|
|
hal->InterruptController.SetTaskPriority(KiIrqlTaskPriority(level));
|
|
cpu->CurrentIrql = level;
|
|
if (!raising && hal && hal->InterruptController.SetTaskPriority)
|
|
hal->InterruptController.SetTaskPriority(KiIrqlTaskPriority(level));
|
|
}
|
|
|
|
// Implements the raise Irql operation.
|
|
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
|
|
KCPU* cpu = KeGetCurrentCpu();
|
|
KIRQL oldIrql = cpu->CurrentIrql;
|
|
|
|
if (newIrql > HIGH_LEVEL) {
|
|
KePanic(KE_PANIC_CODE_IRQL, "KeRaiseIrql: invalid target IRQL %u\n", (unsigned)newIrql);
|
|
}
|
|
|
|
if (newIrql < oldIrql) {
|
|
KePanic(KE_PANIC_CODE_IRQL, "KeRaiseIrql: attempted raise from %s(%u) down to %s(%u)\n",
|
|
KeIrqlName(oldIrql), (unsigned)oldIrql,
|
|
KeIrqlName(newIrql), (unsigned)newIrql);
|
|
}
|
|
|
|
KiSetIrql(cpu, newIrql, true);
|
|
return oldIrql;
|
|
}
|
|
|
|
// Implements the lower Irql operation.
|
|
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
|
|
KCPU* cpu = KeGetCurrentCpu();
|
|
KIRQL currentIrql = cpu->CurrentIrql;
|
|
|
|
if (oldIrql > HIGH_LEVEL) {
|
|
KePanic(KE_PANIC_CODE_IRQL, "KeLowerIrql: invalid target IRQL %u\n", (unsigned)oldIrql);
|
|
}
|
|
|
|
if (oldIrql > currentIrql) {
|
|
KePanic(KE_PANIC_CODE_IRQL, "KeLowerIrql: attempted lower from %s(%u) up to %s(%u)\n",
|
|
KeIrqlName(currentIrql), (unsigned)currentIrql,
|
|
KeIrqlName(oldIrql), (unsigned)oldIrql);
|
|
}
|
|
|
|
KiSetIrql(cpu, oldIrql, false);
|
|
|
|
if (oldIrql == PASSIVE_LEVEL && currentIrql != PASSIVE_LEVEL &&
|
|
(__readeflags() & 0x200u) != 0 && cpu->InterruptDepth == 0)
|
|
KiDeliverKernelApcs();
|
|
}
|
|
|
|
// Implements the assert Irql operation.
|
|
__declspec(dllexport) void KeAssertIrql(KIRQL expected) {
|
|
KIRQL currentIrql = KeGetCurrentIrql();
|
|
if (currentIrql != expected) {
|
|
KePanic(KE_PANIC_CODE_IRQL, "IRQL assertion failed: expected %s(%u), current %s(%u)\n",
|
|
KeIrqlName(expected), (unsigned)expected,
|
|
KeIrqlName(currentIrql), (unsigned)currentIrql);
|
|
}
|
|
}
|
|
|
|
// Implements the assert Irql At Or Below operation.
|
|
__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
|
|
KIRQL currentIrql = KeGetCurrentIrql();
|
|
if (currentIrql > maxIrql) {
|
|
KePanic(KE_PANIC_CODE_IRQL, "IRQL assertion failed: expected <= %s(%u), current %s(%u)\n",
|
|
KeIrqlName(maxIrql), (unsigned)maxIrql,
|
|
KeIrqlName(currentIrql), (unsigned)currentIrql);
|
|
}
|
|
}
|
|
|
|
// Checks whether wait.
|
|
__declspec(dllexport) bool KeCanWait(void) {
|
|
return KeGetCurrentIrql() == PASSIVE_LEVEL;
|
|
}
|