maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

replace nob with ms' build, move disk creation to build.bat, major comment cleanup, remove old and stupid tests, overhaul interrupt management with per-CPU vector allocation, affinity migration and safe teardown, add proper IOAPIC routing and PCI MSI/MSI-X/INTx support, migrate drivers to the new interrupt API, integrate ACPICA with a proper OS services layer, move ACPI firmware events, power button handling and S5 shutdown into acpi.sys, add parallel driver startup and atomic publication, replace PnP arrays with dynamic lists, redesign service startup again to be asynchronous, add delayed startup mode for services, allow omitting keys in services.ini, fix acpi ec deferred work handling, fix kernel threads having garbage exit code.

Browse Source
This commit is contained in:
maksw committed 2026-10-28 15:18:43 +01:00
1 parent d36f19509c
commit 36eef8f58d
696 files changed
+147281 -7530
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/core/makefile
+1
View File
@@ -0,0 +1 @@
!include $(BASEDIR)\makefile.def
code/kernel/core/sources
+21
View File
@@ -0,0 +1,21 @@
TARGETNAME=kernel
TARGETPATH=obj
TARGETTYPE=PROGRAM
IMAGE_KIND=KERNEL
FLAVOR=KERNEL
EXTRA_CFLAGS=/DNTL_KERNEL_BUILD /DKE_DISPLAY_BUILD
TARGETLIBS="$(BASEDIR)\bin\hal.lib" "$(BASEDIR)\bin\printf.lib" "$(BASEDIR)\bin\libc.lib"
EXTRA_LFLAGS=/ENTRY:KernelMain /MAP:"$(BASEDIR)\bin\kernel.map" /FIXED /DYNAMICBASE:NO /HIGHENTROPYVA:NO /RELEASE /BASE:0xFFFF800000000000
INCLUDES=$(BASEDIR)\code\kernel\include
SOURCES=@/code/kernel/dm_config.c @/code/kernel/driver_module.c @/code/kernel/io_async.c @/code/kernel/io_ddk.c @/code/kernel/io_device.c @/code/kernel/io_dma.c \
@/code/kernel/io_file.c @/code/kernel/io_partition.c @/code/kernel/io_pipe.c @/code/kernel/io_pnp.c @/code/kernel/io_volume.c @/code/kernel/ke_apc.c \
@/code/kernel/ke_audio.c @/code/kernel/ke_boot_config.c @/code/kernel/ke_cpu.c @/code/kernel/ke_debug.c @/code/kernel/ke_display.c @/code/kernel/ke_dpc.c \
@/code/kernel/ke_event.c @/code/kernel/ke_exception.c @/code/kernel/ke_firmware.c @/code/kernel/ke_hal.c @/code/kernel/ke_handle.c @/code/kernel/ke_id.c \
@/code/kernel/ke_idt.c @/code/kernel/ke_input.c @/code/kernel/ke_irp.c @/code/kernel/ke_irq.c @/code/kernel/ke_irql.c @/code/kernel/ke_list.c \
@/code/kernel/ke_mutex.c @/code/kernel/ke_port.c @/code/kernel/ke_power.c @/code/kernel/ke_process.c @/code/kernel/ke_queue.c @/code/kernel/ke_sched.c \
@/code/kernel/ke_selftest.c @/code/kernel/ke_semaphore.c @/code/kernel/ke_settings.c @/code/kernel/ke_smp.c @/code/kernel/ke_surface.c @/code/kernel/ke_syscall.c \
@/code/kernel/ke_system_root.c @/code/kernel/ke_system_service.c @/code/kernel/ke_thread.c @/code/kernel/ke_time.c @/code/kernel/ke_timer.c @/code/kernel/ke_trap.c \
@/code/kernel/ke_work.c @/code/kernel/kmain.c @/code/kernel/ldr_image.c @/code/kernel/ldr_section.c @/code/kernel/mm_file_cache.c @/code/kernel/mm_phys.c \
@/code/kernel/mm_pool.c @/code/kernel/mm_pte.c @/code/kernel/mm_swap.c @/code/kernel/mm_virt.c @/code/kernel/net_device.c @/code/kernel/net_socket.c \
@/code/kernel/net_stack.c @/code/kernel/ob_object.c @/code/kernel/se_security.c @/code/kernel/ke_gdt_reload.asm @/code/kernel/ke_switch.asm @/code/kernel/ke_syscall.asm \
@/code/common/boot_volume_selector.c
code/kernel/dirs
+1
View File
@@ -0,0 +1 @@
DIRS=hal core
code/kernel/dm_config.c
+3 -2
View File
@@ -354,7 +354,7 @@ bool DmClassSettled(const DM_CONFIG* config, const char* classId) {
const DM_DRIVER_CONFIG* driver = &config->Drivers[i];
if (!strcmp(driver->Class, classId) &&
(config->ActivePhase == DmStartRuntime || driver->Phase == DmStartBoot) &&
driver->State == DmRunWaiting) return false;
(driver->State == DmRunWaiting || driver->State == DmRunStarting)) return false;
}
return true;
}
@@ -377,7 +377,8 @@ static bool DmEdgesReady(const DM_CONFIG* config, const DM_EDGES* edges, bool re
int index = DmFindDriver(config, edge->Id);
if (index < 0) return false;
DM_RUN_STATE state = config->Drivers[index].State;
if (required ? state != DmRunStarted : state == DmRunWaiting) return false;
if (required ? state != DmRunStarted :
(state == DmRunWaiting || state == DmRunStarting)) return false;
}
}
return true;
code/kernel/driver_module.c
+76 -19
View File
@@ -10,11 +10,14 @@
#include <kernel/mm_virt.h>
#include <kernel/io_pnp.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#define DM_NAME_MAX 64u
typedef enum DM_MODULE_STATE {
DmModuleStarting,
DmModulePrepared,
DmModuleCommitting,
DmModuleStarted,
DmModuleFailed,
DmModuleStopping,
@@ -26,11 +29,14 @@ struct NTL_MODULE {
char Name[DM_NAME_MAX];
uintptr_t ImageBase;
DM_MODULE_STATE State;
NTL_MODULE_COMMIT_START CommitStart;
NTL_MODULE_ABORT_START AbortStart;
NTL_MODULE_PREPARE_SHUTDOWN PrepareShutdown;
NTL_MODULE_COMMIT_SHUTDOWN CommitShutdown;
};
static NTL_MODULE* gDriverModules;
static SPINLOCK gDriverModuleLock = SPINLOCK_INIT;
static bool DmImageContains(uintptr_t base, size_t imageSize, uintptr_t ptr, size_t bytes) {
return ptr >= base && ptr - base <= imageSize && bytes <= imageSize - (ptr - base);
@@ -94,7 +100,9 @@ static bool DmValidIoDriver(uintptr_t imageBase, size_t imageSize, const DRIVER_
return true;
}
KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
KSTATUS DmPrepareLoadedDriver(const char* name, uintptr_t imageBase, bool* canReleaseImage) {
if (!canReleaseImage) return NTLSTATUS_INVALID_PARAMETER;
*canReleaseImage = true;
if (!name || !*name || !imageBase)
return NTLSTATUS_INVALID_PARAMETER;
@@ -104,18 +112,13 @@ KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
if (!length || length == DM_NAME_MAX)
return NTLSTATUS_INVALID_PARAMETER;
for (NTL_MODULE* it = gDriverModules; it; it = it->Next) {
if (strcmp(it->Name, name) == 0 || it->ImageBase == imageBase)
return NTLSTATUS_ALREADY_EXISTS;
}
size_t imageSize = LdrGetImageSize(imageBase);
if (!imageSize)
return NTLSTATUS_INVALID_IMAGE_FORMAT;
const NTL_MODULE_EXPORT* manifest =
(const NTL_MODULE_EXPORT*)LdrGetImageExport(imageBase, NTL_DDK_MODULE_EXPORT_NAME);
NTL_MODULE_INITIALIZE initialize = (NTL_MODULE_INITIALIZE)0;
NTL_MODULE_PREPARE_START prepareStart = (NTL_MODULE_PREPARE_START)0;
if (!manifest) return NTLSTATUS_INVALID_IMAGE_FORMAT;
{
@@ -124,15 +127,19 @@ KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
manifest->Version != NTL_DDK_MODULE_ABI_VERSION ||
manifest->Size != sizeof(*manifest) || manifest->Flags != 0u ||
manifest->Reserved != 0u || manifest->Reserved2 != 0u ||
(manifest->Initialize && !DmImageContains(imageBase, imageSize,
(uintptr_t)manifest->Initialize, 1u)) ||
(manifest->PrepareStart && !DmImageContains(imageBase, imageSize,
(uintptr_t)manifest->PrepareStart, 1u)) ||
(manifest->PrepareShutdown && !DmImageContains(imageBase, imageSize,
(uintptr_t)manifest->PrepareShutdown, 1u)) ||
(manifest->CommitShutdown && !DmImageContains(imageBase, imageSize,
(uintptr_t)manifest->CommitShutdown, 1u)))
(uintptr_t)manifest->CommitShutdown, 1u)) ||
(manifest->CommitStart && !DmImageContains(imageBase, imageSize,
(uintptr_t)manifest->CommitStart, 1u)) ||
(manifest->AbortStart && !DmImageContains(imageBase, imageSize,
(uintptr_t)manifest->AbortStart, 1u)))
return NTLSTATUS_INVALID_IMAGE_FORMAT;
if ((manifest->DeviceCount && !manifest->Devices) ||
manifest->DeviceCount > 64u ||
manifest->DeviceCount > imageSize / sizeof(NTL_DEVICE_DESC) ||
(manifest->DeviceCount &&
!DmImageContains(imageBase, imageSize, (uintptr_t)manifest->Devices,
(size_t)manifest->DeviceCount * sizeof(NTL_DEVICE_DESC))))
@@ -157,7 +164,7 @@ KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
}
}
if ((manifest->IoDriverCount && !manifest->IoDrivers) ||
manifest->IoDriverCount > 64u ||
manifest->IoDriverCount > imageSize / sizeof(*manifest->IoDrivers) ||
(manifest->IoDriverCount &&
!DmImageContains(imageBase, imageSize, (uintptr_t)manifest->IoDrivers,
(size_t)manifest->IoDriverCount * sizeof(*manifest->IoDrivers))))
@@ -167,7 +174,7 @@ KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
(uintptr_t)manifest->IoDrivers[i], sizeof(DRIVER_OBJECT)))
return NTLSTATUS_INVALID_IMAGE_FORMAT;
}
initialize = manifest->Initialize;
prepareStart = manifest->PrepareStart;
}
NTL_MODULE* module = (NTL_MODULE*)MmAllocatePool(sizeof(*module));
@@ -178,12 +185,30 @@ KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
memcpy(module->Name, name, length);
module->ImageBase = imageBase;
module->State = DmModuleStarting;
module->CommitStart = manifest->CommitStart;
module->AbortStart = manifest->AbortStart;
module->PrepareShutdown = manifest->PrepareShutdown;
module->CommitShutdown = manifest->CommitShutdown;
module->Next = gDriverModules;
gDriverModules = module;
KSTATUS status = NTLSTATUS_SUCCESS;
uintptr_t flags = KeAcquireSpinlock(&gDriverModuleLock);
for (NTL_MODULE* it = gDriverModules; it; it = it->Next) {
if (strcmp(it->Name, name) == 0 || it->ImageBase == imageBase) {
status = NTLSTATUS_ALREADY_EXISTS;
break;
}
}
if (KSUCCESS(status)) {
module->Next = gDriverModules;
gDriverModules = module;
}
KeReleaseSpinlock(&gDriverModuleLock, flags);
if (!KSUCCESS(status)) {
MmFreePool(module);
return status;
}
KSTATUS status = initialize ? initialize(module) : NTLSTATUS_SUCCESS;
if (prepareStart && !module->AbortStart) *canReleaseImage = false;
status = prepareStart ? prepareStart(module) : NTLSTATUS_SUCCESS;
if (!KSUCCESS(status) && !KFAILED(status))
status = NTLSTATUS_INVALID_PARAMETER;
@@ -196,10 +221,42 @@ KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
}
}
if (KSUCCESS(status))
status = IoPnpPublishModule(module, manifest->Devices, manifest->DeviceCount,
status = IoPnpPrepareModule(module, manifest->Devices, manifest->DeviceCount,
manifest->IoDrivers, manifest->IoDriverCount);
module->State = KSUCCESS(status) ? DmModuleStarted : DmModuleFailed;
return status;
if (!KSUCCESS(status)) {
if (module->AbortStart) module->AbortStart(module);
IoPnpAbortModule(module);
uintptr_t flags = KeAcquireSpinlock(&gDriverModuleLock);
NTL_MODULE** link = &gDriverModules;
while (*link && *link != module) link = &(*link)->Next;
if (*link) *link = module->Next;
KeReleaseSpinlock(&gDriverModuleLock, flags);
MmFreePool(module);
return status;
}
module->State = DmModulePrepared;
return NTLSTATUS_SUCCESS;
}
// Commit activates the module, exports, and PnP registrations.
void DmCommitLoadedDriver(uintptr_t imageBase, LDR_KERNEL_MODULE* image) {
NTL_MODULE* module = (NTL_MODULE*)0;
uintptr_t flags = KeAcquireSpinlock(&gDriverModuleLock);
for (NTL_MODULE* it = gDriverModules; it; it = it->Next) {
if (it->ImageBase == imageBase && it->State == DmModulePrepared) {
module = it;
module->State = DmModuleCommitting;
break;
}
}
KeReleaseSpinlock(&gDriverModuleLock, flags);
if (!module) return;
if (module->CommitStart) module->CommitStart(module);
LdrPublishModule(image);
IoPnpCommitModule(module);
flags = KeAcquireSpinlock(&gDriverModuleLock);
module->State = DmModuleStarted;
KeReleaseSpinlock(&gDriverModuleLock, flags);
}
KSTATUS DmPrepareShutdown(void) {
code/kernel/hal/hal_apic.c
+161 -109
View File
@@ -4,6 +4,8 @@
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_timer.h>
#include <kernel/hal/hal_processor.h>
#include <kernel/ke_vectors.h>
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>
@@ -14,6 +16,9 @@ static volatile uint32_t* gLapic = (volatile uint32_t*)0;
#define LAPIC_REG(off) (*(volatile uint32_t*)((uint8_t*)gLapic + (off)))
#define LAPIC_ID 0x020
#define LAPIC_EOI 0x0B0
#define LAPIC_TPR 0x080
#define LAPIC_ISR_BASE 0x100
#define LAPIC_IRR_BASE 0x200
#define LAPIC_SPURIOUS 0x0F0
#define LAPIC_ICR_LO 0x300
#define LAPIC_ICR_HI 0x310
@@ -24,38 +29,70 @@ static volatile uint32_t* gLapic = (volatile uint32_t*)0;
#define IA32_APIC_BASE_X2APIC (1ULL << 10)
#define IA32_X2APIC_ID 0x00000802u
#define IA32_X2APIC_EOI 0x0000080Bu
#define IA32_X2APIC_TPR 0x00000808u
#define IA32_X2APIC_ISR_BASE 0x00000810u
#define IA32_X2APIC_IRR_BASE 0x00000820u
#define IA32_X2APIC_SVR 0x0000080Fu
#define IA32_X2APIC_ICR 0x00000830u
#define CPUID_FEATURE_ECX_X2APIC (1u << 21)
// ---- I/O APIC ----.
static volatile uint32_t* gIoapic = (volatile uint32_t*)0;
static uint32_t gBspLapicId = 0;
static uint8_t gX2ApicSupported = 0;
static uint8_t gX2ApicEnabled = 0;
static uint32_t gIoapicGsiBase = 0;
static uint32_t gIoapicRedirectionCount = 0;
// ---- I/O APIC interrupt domains ----.
#define HAL_IOAPIC_MAX 8u
#define IOAPIC_IDX 0x00u
#define IOAPIC_DATA 0x10u
typedef struct HAL_IOAPIC {
volatile uint32_t* Registers;
uintptr_t Physical;
uint32_t GsiBase;
uint32_t Pins;
volatile long RegisterLock;
} HAL_IOAPIC;
static HAL_IOAPIC gIoapics[HAL_IOAPIC_MAX];
static uint32_t gIoapicCount;
static uint32_t gBspLapicId;
static uint8_t gX2ApicSupported;
static uint8_t gX2ApicEnabled;
static uint32_t gIsaGsi[16];
static uint16_t gIsaFlags[16];
static uint16_t gSciIrq = UINT16_MAX;
static bool gIsaOverridesInitialized = false;
#define IOAPIC_IDX 0x00
#define IOAPIC_DATA 0x10
static bool gIsaOverridesInitialized;
// Implements the ioapic Read operation.
static uint32_t IoapicRead(uint8_t reg) {
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
return *(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA);
}
// Implements the ioapic Write operation.
static void IoapicWrite(uint8_t reg, uint32_t val) {
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_IDX) = reg;
*(volatile uint32_t*)((uint8_t*)gIoapic + IOAPIC_DATA) = val;
// I/O APIC register selectors are shared by every CPU accessing that APIC.
static uint32_t IoapicRead(HAL_IOAPIC* apic, uint8_t reg) {
uintptr_t interruptFlags = HalDisableInterrupts();
while (_InterlockedCompareExchange(&apic->RegisterLock, 1, 0))
_mm_pause();
apic->Registers[IOAPIC_IDX / 4u] = reg;
uint32_t value = apic->Registers[IOAPIC_DATA / 4u];
_InterlockedExchange(&apic->RegisterLock, 0);
HalRestoreInterrupts(interruptFlags);
return value;
}
static void IoapicWrite(HAL_IOAPIC* apic, uint8_t reg, uint32_t value) {
uintptr_t interruptFlags = HalDisableInterrupts();
while (_InterlockedCompareExchange(&apic->RegisterLock, 1, 0))
_mm_pause();
apic->Registers[IOAPIC_IDX / 4u] = reg;
apic->Registers[IOAPIC_DATA / 4u] = value;
_InterlockedExchange(&apic->RegisterLock, 0);
HalRestoreInterrupts(interruptFlags);
}
static HAL_IOAPIC* HalApicFindGsi(uint32_t gsi, uint32_t* pinOut) {
for (uint32_t i = 0; i < gIoapicCount; i++) {
HAL_IOAPIC* apic = &gIoapics[i];
if (gsi >= apic->GsiBase && gsi - apic->GsiBase < apic->Pins) {
*pinOut = gsi - apic->GsiBase;
return apic;
}
}
return (HAL_IOAPIC*)0;
}
// Implements the apic Detect Mode operation.
static void HalApicDetectMode(void) {
int regs[4] = {0, 0, 0, 0};
__cpuid(regs, 1);
@@ -70,7 +107,6 @@ static void HalApicStallUs(uint32_t us) {
HalTimerStall(us);
}
// Implements the apic Wait Icr Idle operation.
static void HalApicWaitIcrIdle(void) {
if (gX2ApicEnabled)
return;
@@ -82,7 +118,6 @@ static void HalApicWaitIcrIdle(void) {
}
}
// Implements the apic Send Ipi operation.
static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low);
@@ -98,8 +133,6 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
// IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high).
#define IOREDTBL_LO(n) (0x10 + 2*(n))
#define IOREDTBL_HI(n) (0x11 + 2*(n))
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
#define IRQ_VECTOR_BASE 0x20
// Returns the local interrupt-controller hardware ID of the boot processor.
uint32_t HalApicGetBootProcessorId(void) {
@@ -126,12 +159,46 @@ void HalApicInitializeCurrentProcessor(void) {
apicBase |= IA32_APIC_BASE_X2APIC;
__writemsr(IA32_APIC_BASE, apicBase);
__writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu);
HalApicSetTaskPriority(0);
return;
}
__writemsr(IA32_APIC_BASE, apicBase);
if (gLapic)
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
HalApicSetTaskPriority(0);
}
// Controls the current CPU's interrupt priority without changing its interrupt-enable flag.
void HalApicSetTaskPriority(uint8_t priority) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_TPR, priority);
} else if (gLapic) {
LAPIC_REG(LAPIC_TPR) = priority;
}
}
uint8_t HalApicQueryTaskPriority(void) {
if (gX2ApicEnabled)
return (uint8_t)__readmsr(IA32_X2APIC_TPR);
return gLapic ? (uint8_t)LAPIC_REG(LAPIC_TPR) : 0u;
}
// Reads the current processor's local delivery state after a source is masked.
bool HalApicVectorPending(uint8_t vector) {
const uint32_t index = (uint32_t)vector >> 5u;
const uint32_t mask = 1u << (vector & 31u);
uint32_t irr, isr;
if (gX2ApicEnabled) {
irr = (uint32_t)__readmsr(IA32_X2APIC_IRR_BASE + index);
isr = (uint32_t)__readmsr(IA32_X2APIC_ISR_BASE + index);
} else if (gLapic) {
irr = LAPIC_REG(LAPIC_IRR_BASE + 0x10u * index);
isr = LAPIC_REG(LAPIC_ISR_BASE + 0x10u * index);
} else {
return true;
}
return ((irr | isr) & mask) != 0u;
}
// Broadcasts an NMI to every processor except the caller.
@@ -182,102 +249,85 @@ KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical) {
return NTLSTATUS_SUCCESS;
}
// Composes an MSI message targeting the boot processor without exposing APIC mode.
KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut) {
if (!messageOut || vector < 0x20u)
// MSI destination encoding without interrupt remapping supports an 8-bit APIC ID.
KSTATUS HalApicComposeMsi(uint32_t hardwareId, uint8_t vector, HAL_MSI_MESSAGE* out) {
if (!out || vector < KE_VECTOR_DEVICE_FIRST || vector > KE_VECTOR_DEVICE_LAST)
return NTLSTATUS_INVALID_PARAMETER;
if (gBspLapicId >= 256u)
if (hardwareId > 255u)
return NTLSTATUS_NOT_SUPPORTED;
messageOut->Address = 0xFEE00000ull | ((uint64_t)(gBspLapicId & 0xFFu) << 12);
messageOut->Data = vector;
messageOut->Reserved = 0;
out->Address = 0xFEE00000ull | ((uint64_t)hardwareId << 12);
out->Data = vector;
out->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
// Initializes the identity ISA map before processing MADT source overrides.
static void HalApicInitializeIsaMap(void) {
if (gIsaOverridesInitialized) return;
for (uint32_t irq = 0; irq < 16; ++irq) {
for (uint32_t irq = 0; irq < 16; irq++) {
gIsaGsi[irq] = irq;
gIsaFlags[irq] = 0;
}
gIsaOverridesInitialized = true;
}
// Records the global-system-interrupt base of the currently supported IOAPIC.
void HalApicSetGsiBase(uint32_t gsiBase) { gIoapicGsiBase = gsiBase; }
void HalApicAddIoapic(uintptr_t physical, uint32_t gsiBase) {
if (!physical || gIoapicCount >= HAL_IOAPIC_MAX)
return;
for (uint32_t i = 0; i < gIoapicCount; i++)
if (gIoapics[i].Physical == physical)
return;
HAL_IOAPIC* apic = &gIoapics[gIoapicCount++];
apic->Physical = physical;
apic->GsiBase = gsiBase;
}
// Preserves MADT interrupt-source overrides for legacy ISA devices.
void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags) {
if (irq >= 16) return;
if (irq >= 16u) return;
HalApicInitializeIsaMap();
gIsaGsi[irq] = gsi;
gIsaFlags[irq] = flags;
}
// Marks the FADT SCI as a low-active, level-triggered interrupt by default.
void HalApicSetSciIrq(uint16_t irq) { gSciIrq = irq; }
// Resolves one logical IRQ to a physical pin on the selected IOAPIC.
static bool HalApicResolveIrq(uint8_t irq, uint32_t* pinOut, uint32_t* flagsOut) {
KSTATUS HalApicResolveIsa(uint8_t irq, uint32_t* gsiOut, uint16_t* flagsOut) {
if (irq >= 16u || !gsiOut || !flagsOut)
return NTLSTATUS_INVALID_PARAMETER;
HalApicInitializeIsaMap();
uint32_t gsi = irq < 16 ? gIsaGsi[irq] : irq;
if (gsi < gIoapicGsiBase || gsi - gIoapicGsiBase >= gIoapicRedirectionCount)
return false;
uint32_t flags = 0;
if (irq == gSciIrq) flags = (1u << 13) | (1u << 15);
if (irq < 16) {
uint16_t override = gIsaFlags[irq];
// MADT 00 is bus-conformant; 11 selects active low or level triggered.
if ((override & 3u) == 3u) flags |= (1u << 13);
else if ((override & 3u) == 1u) flags &= ~(1u << 13);
if (((override >> 2) & 3u) == 3u) flags |= (1u << 15);
else if (((override >> 2) & 3u) == 1u) flags &= ~(1u << 15);
}
*pinOut = gsi - gIoapicGsiBase;
*flagsOut = flags;
return true;
uint32_t pin;
if (!HalApicFindGsi(gIsaGsi[irq], &pin))
return NTLSTATUS_NOT_SUPPORTED;
*gsiOut = gIsaGsi[irq];
*flagsOut = gIsaFlags[irq];
return NTLSTATUS_SUCCESS;
}
// Reports whether a platform IRQ can be routed by the current IOAPIC.
bool HalApicCanRouteIrq(uint8_t irq) {
uint32_t pin, flags;
return irq < 24 && HalApicResolveIrq(irq, &pin, &flags);
// ACPI polarity bits 1:0 and trigger bits 3:2 encode bus-defined defaults.
KSTATUS HalApicRouteGsi(uint32_t gsi, uint8_t vector, uint32_t hardwareId,
uint16_t flags, bool masked) {
if (vector < KE_VECTOR_DEVICE_FIRST || vector > KE_VECTOR_DEVICE_LAST || hardwareId > 255u)
return NTLSTATUS_INVALID_PARAMETER;
uint32_t pin;
HAL_IOAPIC* apic = HalApicFindGsi(gsi, &pin);
if (!apic) return NTLSTATUS_NOT_SUPPORTED;
uint32_t mode = 0;
if ((flags & 3u) == 3u) mode |= (1u << 13);
if (((flags >> 2) & 3u) == 3u) mode |= (1u << 15);
IoapicWrite(apic, IOREDTBL_LO(pin), (1u << 16) | vector | mode);
IoapicWrite(apic, IOREDTBL_HI(pin), (hardwareId & 255u) << 24);
if (!masked) IoapicWrite(apic, IOREDTBL_LO(pin), vector | mode);
return NTLSTATUS_SUCCESS;
}
// Implements the apic Route Irq operation.
void HalApicRouteIrq(uint8_t irq) {
// Full initial setup of the redirection entry.
// Sets the vector and clears the mask bit in one write.
uint32_t pin, flags;
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
uint32_t vector = IRQ_VECTOR_BASE + irq;
IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
IoapicWrite(IOREDTBL_LO(pin), vector | flags);
KSTATUS HalApicMaskGsi(uint32_t gsi, bool masked) {
uint32_t pin;
HAL_IOAPIC* apic = HalApicFindGsi(gsi, &pin);
if (!apic) return NTLSTATUS_NOT_SUPPORTED;
uint32_t lo = IoapicRead(apic, IOREDTBL_LO(pin));
if (masked) lo |= 1u << 16;
else lo &= ~(1u << 16);
IoapicWrite(apic, IOREDTBL_LO(pin), lo);
return NTLSTATUS_SUCCESS;
}
// Implements the apic Mask Irq operation.
void HalApicMaskIrq(uint8_t irq) {
uint32_t pin, flags;
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
lo |= (1 << 16);
IoapicWrite(IOREDTBL_LO(pin), lo);
}
// Implements the apic Unmask Irq operation.
void HalApicUnmaskIrq(uint8_t irq) {
uint32_t pin, flags;
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
lo = (lo & ~(0xFFu | (1u << 16) | (1u << 13) | (1u << 15))) |
(IRQ_VECTOR_BASE + irq) | flags;
IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
IoapicWrite(IOREDTBL_LO(pin), lo);
}
// Implements the apic Send Eoi operation.
void HalApicSendEoi(void) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_EOI, 0);
@@ -286,27 +336,29 @@ void HalApicSendEoi(void) {
LAPIC_REG(LAPIC_EOI) = 0;
}
// Implements the apic Initialize operation.
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
void HalApicInitialize(uintptr_t lapicPhys) {
// The PIC remains masked: legacy input arrives through programmed I/O APICs.
__outbyte(0x21, 0xFF);
__outbyte(0xA1, 0xFF);
HalApicDetectMode();
gLapic = (volatile uint32_t*)(HalPhysicalToVirtual(lapicPhys));
gIoapic = (volatile uint32_t*)(HalPhysicalToVirtual(ioapicPhys));
// The BSP is the CPU executing HAL init. Keep all external/device IRQs
// routed to this LAPIC; CPU-local IPIs use the local APIC directly.
gLapic = (volatile uint32_t*)HalPhysicalToVirtual(lapicPhys);
if (gX2ApicEnabled)
gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID);
else
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFFu;
HalApicInitializeCurrentProcessor();
if (gIoapicCount == 0)
HalApicAddIoapic(0xFEC00000u, 0);
// Mask all IOAPIC lines initially.
uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
gIoapicRedirectionCount = maxRedir + 1u;
for (uint32_t i = 0; i <= maxRedir; i++) {
IoapicWrite(IOREDTBL_LO(i), (1u << 16) | IRQ_VECTOR_BASE);
IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24);
for (uint32_t index = 0; index < gIoapicCount; index++) {
HAL_IOAPIC* apic = &gIoapics[index];
apic->Registers = (volatile uint32_t*)HalPhysicalToVirtual(apic->Physical);
uint32_t version = IoapicRead(apic, 1u);
apic->Pins = ((version >> 16) & 0xFFu) + 1u;
for (uint32_t pin = 0; pin < apic->Pins; pin++) {
IoapicWrite(apic, IOREDTBL_LO(pin), (1u << 16) | KE_VECTOR_DEVICE_FIRST);
IoapicWrite(apic, IOREDTBL_HI(pin), (gBspLapicId & 255u) << 24);
}
}
}
code/kernel/hal/hal_main.c
+12 -114
View File
@@ -39,10 +39,13 @@ static HAL_API gApi = {
.Start = HalApicStartProcessor,
},
.InterruptController = {
.RouteIrq = HalApicRouteIrq,
.MaskIrq = HalApicMaskIrq,
.UnmaskIrq = HalApicUnmaskIrq,
.RouteGsi = HalApicRouteGsi,
.MaskGsi = HalApicMaskGsi,
.ResolveIsa = HalApicResolveIsa,
.SendEoi = HalApicSendEoi,
.SetTaskPriority = HalApicSetTaskPriority,
.QueryTaskPriority = HalApicQueryTaskPriority,
.VectorPending = HalApicVectorPending,
.InitializeCurrentProcessor = HalApicInitializeCurrentProcessor,
.SendProcessorInterrupt = HalApicSendProcessorInterrupt,
.SendProcessorNmi = HalApicSendProcessorNmi,
@@ -122,12 +125,7 @@ typedef struct ACPI_FIXED_REGISTER {
static ACPI_FIXED_REGISTER gPm1aCnt;
static ACPI_FIXED_REGISTER gPm1bCnt;
static ACPI_FIXED_REGISTER gPm1aStatus;
static ACPI_FIXED_REGISTER gPm1bStatus;
static ACPI_FIXED_REGISTER gPm1aEnable;
static ACPI_FIXED_REGISTER gPm1bEnable;
static uint16_t gSciIrq = 0;
static bool gFixedPowerButtonArmed = false;
static ACPI_FIXED_REGISTER gSleepControl;
static uint32_t gLegacyPm1aPort = 0;
static uint32_t gLegacyPm1bPort = 0;
@@ -179,13 +177,6 @@ void* HalPhysicalToVirtual(uint64_t phys) {
// systems may put the real DSDT and PM1 control registers in the extended
// X_* fields, so do not rely only on the legacy 32-bit fields.
#define FADT_SCI_INT_OFFSET 46u
#define FADT_PM1A_EVT_BLK_OFFSET 56u
#define FADT_PM1B_EVT_BLK_OFFSET 60u
#define FADT_PM1_EVT_LEN_OFFSET 88u
#define FADT_X_PM1A_EVT_BLK_OFFSET 148u
#define FADT_X_PM1B_EVT_BLK_OFFSET 160u
#define FADT_FLAG_POWER_BUTTON (1u << 4)
#define ACPI_PM1_POWER_BUTTON_BIT (1u << 8)
#define FADT_DSDT_OFFSET 40u
#define FADT_SMI_CMD_OFFSET 48u
#define FADT_ACPI_ENABLE_OFFSET 52u
@@ -217,7 +208,6 @@ typedef struct {
static ACPI_FIXED_REGISTER gResetRegister;
static uint8_t gResetValue;
// Implements the acpi Read U32 operation.
static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) {
uint32_t value = 0;
if (off + sizeof(value) <= tableLen)
@@ -225,7 +215,6 @@ static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off)
return value;
}
// Implements the acpi Read U64 operation.
static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off) {
uint64_t value = 0;
if (off + sizeof(value) <= tableLen)
@@ -310,77 +299,6 @@ static void AcpiWriteFixedRegister(const ACPI_FIXED_REGISTER* reg, uint32_t valu
else *(volatile uint32_t*)address = value;
}
// Selects a PM1 event block as two adjacent equal-width status and enable registers.
static void AcpiConfigurePm1EventBlock(const void* fadt, uint32_t len,
uint32_t legacyOffset, uint32_t gasOffset,
uint8_t halfLength, ACPI_FIXED_REGISTER* status,
ACPI_FIXED_REGISTER* enable) {
ACPI_FIXED_REGISTER block = {0};
uint32_t legacy = AcpiReadU32(fadt, len, legacyOffset);
if (legacy && legacy <= 0xFFFFu) {
block.AddressSpaceId = 1;
block.Address = legacy;
}
ACPI_FIXED_REGISTER extended = {0};
if (AcpiReadGasRegister(fadt, len, gasOffset, &extended)) block = extended;
if (!block.Address || (block.AddressSpaceId == 1 &&
block.Address + halfLength * 2u - 1u > 0xFFFFu)) return;
block.WidthBytes = halfLength;
*status = block;
block.Address += halfLength;
*enable = block;
}
// Reports the fixed-feature power-button interrupt supported by this HAL.
static KSTATUS AcpiQueryFixedButtonIrq(uint8_t* irqOut) {
if (!irqOut) return NTLSTATUS_INVALID_PARAMETER;
// A FADT control-method power button requires AML evaluation, not PM1 polling.
if (gHwReducedAcpi || (gFadtFlags & FADT_FLAG_POWER_BUTTON) ||
!gSciIrq || gSciIrq >= 16u || !gPm1aStatus.Address || !gPm1aEnable.Address ||
!HalApicCanRouteIrq((uint8_t)gSciIrq))
return NTLSTATUS_NOT_SUPPORTED;
*irqOut = (uint8_t)gSciIrq;
return NTLSTATUS_SUCCESS;
}
// Discards any stale press then enables the PM1 fixed power-button SCI.
static KSTATUS AcpiArmFixedButton(void) {
uint8_t irq;
KSTATUS status = AcpiQueryFixedButtonIrq(&irq);
if (!KSUCCESS(status)) return status;
// No SCI_EN means firmware still owns ACPI events. Never steal ownership
// through SMI_CMD just to install a power-button handler during boot.
if (!(AcpiReadFixedRegister(&gPm1aCnt) & 1u))
return NTLSTATUS_NOT_SUPPORTED;
AcpiWriteFixedRegister(&gPm1aStatus, ACPI_PM1_POWER_BUTTON_BIT);
if (gPm1bStatus.Address)
AcpiWriteFixedRegister(&gPm1bStatus, ACPI_PM1_POWER_BUTTON_BIT);
uint32_t enabled = AcpiReadFixedRegister(&gPm1aEnable);
AcpiWriteFixedRegister(&gPm1aEnable, enabled | ACPI_PM1_POWER_BUTTON_BIT);
if (gPm1bEnable.Address) {
enabled = AcpiReadFixedRegister(&gPm1bEnable);
AcpiWriteFixedRegister(&gPm1bEnable, enabled | ACPI_PM1_POWER_BUTTON_BIT);
}
gFixedPowerButtonArmed = true;
return NTLSTATUS_SUCCESS;
}
// Acknowledges the fixed-button source only; the SCI may carry unrelated events.
static bool AcpiConsumeFixedButton(void) {
if (!gFixedPowerButtonArmed) return false;
bool pressed = false;
if (AcpiReadFixedRegister(&gPm1aStatus) & ACPI_PM1_POWER_BUTTON_BIT) {
AcpiWriteFixedRegister(&gPm1aStatus, ACPI_PM1_POWER_BUTTON_BIT);
pressed = true;
}
if (gPm1bStatus.Address &&
(AcpiReadFixedRegister(&gPm1bStatus) & ACPI_PM1_POWER_BUTTON_BIT)) {
AcpiWriteFixedRegister(&gPm1bStatus, ACPI_PM1_POWER_BUTTON_BIT);
pressed = true;
}
return pressed;
}
// Detects whether PM1 control is using the standard Intel PCH PMBASE layout.
static void AcpiDetectIntelPchPmBase(void) {
gIntelPchPmBase = 0;
@@ -486,7 +404,6 @@ static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
return p;
}
// Implements the aml Read Integer operation.
static bool AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
const uint8_t* p = *pp;
if (p >= end) return false;
@@ -580,10 +497,6 @@ static void ParseFadt(const ACPISDTHeader* header) {
memset(&gPm1aCnt, 0, sizeof(gPm1aCnt));
memset(&gPm1bCnt, 0, sizeof(gPm1bCnt));
memset(&gPm1aStatus, 0, sizeof(gPm1aStatus));
memset(&gPm1bStatus, 0, sizeof(gPm1bStatus));
memset(&gPm1aEnable, 0, sizeof(gPm1aEnable));
memset(&gPm1bEnable, 0, sizeof(gPm1bEnable));
gSciIrq = 0;
if (len >= FADT_SCI_INT_OFFSET + sizeof(gSciIrq))
memcpy(&gSciIrq, (const uint8_t*)fadt + FADT_SCI_INT_OFFSET, sizeof(gSciIrq));
@@ -628,16 +541,6 @@ static void ParseFadt(const ACPISDTHeader* header) {
if (gHwReducedAcpi)
(void)AcpiReadByteGasRegister(fadt, len, FADT_SLEEP_CONTROL_OFFSET, &gSleepControl);
uint8_t eventLength = FADT_PM1_EVT_LEN_OFFSET < len ?
*((const uint8_t*)fadt + FADT_PM1_EVT_LEN_OFFSET) : 0u;
if (!gHwReducedAcpi && (eventLength == 4u || eventLength == 8u)) {
uint8_t half = eventLength / 2u;
AcpiConfigurePm1EventBlock(fadt, len, FADT_PM1A_EVT_BLK_OFFSET,
FADT_X_PM1A_EVT_BLK_OFFSET, half, &gPm1aStatus, &gPm1aEnable);
AcpiConfigurePm1EventBlock(fadt, len, FADT_PM1B_EVT_BLK_OFFSET,
FADT_X_PM1B_EVT_BLK_OFFSET, half, &gPm1bStatus, &gPm1bEnable);
}
gSmiCommandPort = AcpiReadU32(fadt, len, FADT_SMI_CMD_OFFSET);
gAcpiEnableValue = FADT_ACPI_ENABLE_OFFSET < len
? *((const uint8_t*)fadt + FADT_ACPI_ENABLE_OFFSET)
@@ -652,10 +555,12 @@ static void ParseFadt(const ACPISDTHeader* header) {
centuryRegister = *((const uint8_t*)fadt + FADT_CENTURY_OFFSET);
HalRtcSetCenturyRegister(centuryRegister);
HalDbgPrint("HAL: FADT flags=0x%08x hw_reduced=%u PM1a_CNT=%u:0x%016llx legacy=0x%04x "
HalDbgPrint("HAL: FADT flags=0x%08x sci_irq=%u power_button=%s hw_reduced=%u PM1a_CNT=%u:0x%016llx legacy=0x%04x "
"PM1b_CNT=%u:0x%016llx legacy=0x%04x SLEEP_CTL=%u:0x%016llx "
"SMI=0x%08x DSDT=0x%016llx RTC_CENTURY=0x%02x\n",
gFadtFlags, gHwReducedAcpi ? 1u : 0u,
gFadtFlags, (unsigned)gSciIrq,
(gFadtFlags & (1u << 4)) ? "control-method" : "fixed",
gHwReducedAcpi ? 1u : 0u,
gPm1aCnt.AddressSpaceId, gPm1aCnt.Address, gLegacyPm1aPort,
gPm1bCnt.AddressSpaceId, gPm1bCnt.Address, gLegacyPm1bPort,
gSleepControl.AddressSpaceId, gSleepControl.Address,
@@ -674,7 +579,6 @@ static void ParseFadt(const ACPISDTHeader* header) {
}
}
// Implements the add Cpu Record operation.
static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
if (gCpuTopology.Count >= HAL_MAX_PROCESSORS)
return;
@@ -711,7 +615,7 @@ static void ParseMadt(MADT* madt) {
gIoapicPhys = *(uint32_t*)(p + 4);
uint32_t base;
memcpy(&base, p + 8, sizeof(base));
HalApicSetGsiBase(base);
HalApicAddIoapic(*(uint32_t*)(p + 4), base);
} else if (type == 2 && len >= 10 && p[2] == 0) { // ISA source override
uint32_t gsi;
uint16_t flags;
@@ -732,7 +636,6 @@ static void ParseMadt(MADT* madt) {
}
}
// Implements the walk Acpi operation.
static void WalkAcpi(uintptr_t rsdpPhys) {
if (!rsdpPhys) return;
@@ -995,8 +898,7 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
HalDbgPrint("HAL: LAPIC=%016llx IOAPIC=%016llx HPET=%016llx\n",
gLapicPhys, gIoapicPhys, gHpetPhys);
HalApicSetSciIrq(gSciIrq);
HalApicInitialize(gLapicPhys, gIoapicPhys);
HalApicInitialize(gLapicPhys);
HalDbgPrint("HAL: local APIC mode=%s BSP_ID=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
HalApicGetBootProcessorId());
@@ -1008,9 +910,6 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
HalPciInitialize(&gApi.Pci);
gApi.Power.Shutdown = HalShutdown;
gApi.Power.Reset = HalReset;
gApi.Power.QueryFixedButtonIrq = AcpiQueryFixedButtonIrq;
gApi.Power.ArmFixedButton = AcpiArmFixedButton;
gApi.Power.ConsumeFixedButton = AcpiConsumeFixedButton;
AcpiDetectIntelPchPmBase();
gInitialised = 1;
@@ -1026,7 +925,6 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
gApi.Timer.Stall(1); // Micro-stalls
}
HalDbgPrint("HAL: initialisation complete.\n");
return NTLSTATUS_SUCCESS;
}
code/kernel/hal/hal_pci.c
-1
View File
@@ -41,7 +41,6 @@ static void PciWrite32(uint16_t segment,
__outdword(PCI_CONFIG_DATA, value);
}
// Implements the pci Initialize operation.
void HalPciInitialize(HAL_PCI* out) {
out->Read32 = PciRead32;
out->Write32 = PciWrite32;
code/kernel/hal/hal_rtc.c
-10
View File
@@ -23,23 +23,19 @@
static uint8_t gCenturyRegister;
// Implements the rtc Read operation.
static uint8_t RtcRead(uint8_t reg) {
__outbyte(CMOS_INDEX_PORT, (uint8_t)(reg | 0x80u)); // keep NMI masked while selecting.
return __inbyte(CMOS_DATA_PORT);
}
// Implements the rtc Update In Progress operation.
static bool RtcUpdateInProgress(void) {
return (RtcRead(RTC_REG_STATUS_A) & 0x80u) != 0;
}
// Implements the rtc Bcd To Binary operation.
static uint8_t RtcBcdToBinary(uint8_t value) {
return (uint8_t)(((value >> 4) * 10u) + (value & 0x0Fu));
}
// Implements the rtc Read Stable operation.
static bool RtcReadStable(HAL_RTC_TIME* out) {
if (!out) return false;
@@ -116,12 +112,10 @@ static bool RtcReadStable(HAL_RTC_TIME* out) {
return true;
}
// Implements the rtc Leap Year operation.
static bool RtcLeapYear(uint16_t year) {
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
}
// Implements the rtc Days In Month operation.
static uint8_t RtcDaysInMonth(uint16_t year, uint8_t month) {
static const uint8_t days[12] = { 31,28,31,30,31,30,31,31,30,31,30,31 };
if (month < 1u || month > 12u) return 0;
@@ -129,7 +123,6 @@ static uint8_t RtcDaysInMonth(uint16_t year, uint8_t month) {
return days[month - 1u];
}
// Implements the rtc Valid operation.
static bool RtcValid(const HAL_RTC_TIME* t) {
if (!t) return false;
if (t->Year < 1970u || t->Year > 9999u) return false;
@@ -140,7 +133,6 @@ static bool RtcValid(const HAL_RTC_TIME* t) {
return true;
}
// Implements the rtc Query Time operation.
static KSTATUS HalRtcQueryTime(HAL_RTC_TIME* outTime) {
if (!outTime) return NTLSTATUS_INVALID_PARAMETER;
@@ -153,12 +145,10 @@ static KSTATUS HalRtcQueryTime(HAL_RTC_TIME* outTime) {
return NTLSTATUS_SUCCESS;
}
// Implements the rtc Set Century Register operation.
void HalRtcSetCenturyRegister(uint8_t centuryRegister) {
gCenturyRegister = centuryRegister;
}
// Implements the rtc Initialize operation.
void HalRtcInitialize(HAL_RTC* out) {
if (!out) return;
out->QueryTime = HalRtcQueryTime;
code/kernel/hal/hal_screen.c
-8
View File
@@ -39,7 +39,6 @@ static inline void PutPixelRaw(int32_t x, int32_t y, uint32_t color) {
gFb[y * gStride + x] = color;
}
// Implements the screen Flush Rows operation.
static void ScreenFlushRows(int startY, int rows) {
if (!gBackbuffer || rows <= 0)
return;
@@ -73,12 +72,10 @@ static void ScreenClear(uint32_t color) {
gCursorX = 0; gCursorY = 0;
}
// Implements the screen Put Pixel operation.
static void ScreenPutPixel(int32_t x, int32_t y, uint32_t color) {
PutPixelRaw(x, y, color);
}
// Implements the screen Blit operation.
static void ScreenBlit(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels) {
// Calculate clipping bounds.
int startX = (x < 0) ? 0 : x;
@@ -111,10 +108,8 @@ static void ScreenBlit(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_
}
}
// Implements the screen Get Size operation.
static void ScreenGetSize(int* w, int* h) { *w = gWidth; *h = gHeight; }
// Implements the screen Get Framebuffer operation.
static KSTATUS ScreenGetFramebuffer(HAL_FRAMEBUFFER_INFO* outInfo) {
if (!outInfo || !gFb || gWidth <= 0 || gHeight <= 0 || gStride <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -127,7 +122,6 @@ static KSTATUS ScreenGetFramebuffer(HAL_FRAMEBUFFER_INFO* outInfo) {
return NTLSTATUS_SUCCESS;
}
// Implements the screen Scroll operation.
static void ScreenScroll(int32_t lines) {
if (lines <= 0)
return;
@@ -153,7 +147,6 @@ static void ScreenScroll(int32_t lines) {
ScreenFlushRows(0, gHeight);
}
// Implements the screen Put Char operation.
static void ScreenPutChar(char c, int32_t x, int32_t y, uint32_t fg, uint32_t bg) {
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16);
for (int32_t row = 0; row < 16; row++) {
@@ -165,7 +158,6 @@ static void ScreenPutChar(char c, int32_t x, int32_t y, uint32_t fg, uint32_t bg
}
}
// Implements the screen Print Char operation.
static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
int cols = gWidth / FONT_W;
int rows = gHeight / FONT_H;
code/kernel/hal/hal_timer.c
+5 -11
View File
@@ -6,13 +6,14 @@
#include <kernel/hal/hal_timer.h>
#include <kernel/hal/hal_internal.h>
#include <kernel/hal/hal_debug.h>
#include <kernel/ke_vectors.h>
#define LAPIC_TIMER_LVT 0x320
#define LAPIC_TIMER_INIT 0x380
#define LAPIC_TIMER_CUR 0x390
#define LAPIC_TIMER_DIV 0x3E0
#define TIMER_VECTOR 0x20 // vector 0x20 = IRQ 0 slot in IDT
#define TIMER_VECTOR KE_VECTOR_CPU_TIMER
#define TIMER_PERIODIC (1 << 17)
#define TIMER_MASKED (1 << 16)
#define TIMER_DIV16 0x3
@@ -30,9 +31,7 @@ static volatile uint64_t* gHpet = NULL;
static volatile uint64_t gTicks = 0;
// Implements the outb operation.
static inline void outb(uint16_t port, uint8_t val) { __outbyte(port, val); }
// Implements the inb operation.
static inline uint8_t inb(uint16_t port) { return __inbyte(port); }
// Records one local timer interrupt for HAL timekeeping fallback.
@@ -87,7 +86,6 @@ static uint64_t QueryTickFrequency(void) {
return 1000000ull;
}
// Implements the stall operation.
void HalTimerStall(uint32_t us) {
// Priority 1: Use HPET (Highest Precision).
if (gHpet && gHpetTicksPerUs > 0) {
@@ -187,7 +185,6 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
return elapsed / 10; // APIC ticks per ms
}
// Implements the timer Configure Current Cpu operation.
void HalTimerConfigureCurrentProcessor(void) {
if (!gLapic || !gApicTicksPerQuantum) return;
LAPIC_REG(LAPIC_TIMER_DIV) = TIMER_DIV16;
@@ -195,14 +192,12 @@ void HalTimerConfigureCurrentProcessor(void) {
LAPIC_REG(LAPIC_TIMER_INIT) = gApicTicksPerQuantum;
}
// Implements the timer Initialize operation.
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* out) {
gLapic = (volatile uint32_t*)HalPhysicalToVirtual(lapicPhys);
HalDbgPrint("HAL: lapicPhys=0x%016llX, lapicVirt=0x%016llX, hpetPhys=0x%016llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
uint32_t ticksPerMs = 0;
if (hpetPhys) {
HalDbgPrint("HAL: Attempting to calibrate APIC timer via HPET...\n");
ticksPerMs = CalibrateApicTimerViaHpet(hpetPhys);
}
if (!ticksPerMs) {
@@ -210,7 +205,7 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* o
ticksPerMs = CalibrateApicTimerViaPit();
}
if (!ticksPerMs) {
HalDbgPrint("HAL: APIC calibration failed. How?\n");
HalDbgPrint("hal: APIC timer calibration failed\n");
return NTLSTATUS_DEVICE_ERROR;
}
@@ -226,14 +221,13 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* o
HalDbgPrint("HAL: APIC timer: %u ticks/ms\n", ticksPerMs);
// Leave the BSP timer masked until the kernel registers its scheduler
// callback and IRQ route. APs arm their local timer after per-CPU setup.
// Leave the BSP timer masked until the kernel installs its timer handlers.
// APs arm their local timer after per-CPU setup.
out->QueryTicks = HalTimerQueryTicks;
out->QueryTickFrequency = QueryTickFrequency;
out->Stall = HalTimerStall;
out->HandleInterrupt = TimerInterrupt;
out->ConfigureCurrentProcessor = HalTimerConfigureCurrentProcessor;
out->IrqNumber = 0;
return NTLSTATUS_SUCCESS;
}
code/kernel/hal/makefile
+1
View File
@@ -0,0 +1 @@
!include $(BASEDIR)\makefile.def
code/kernel/hal/sources
+11
View File
@@ -0,0 +1,11 @@
TARGETNAME=hal
TARGETPATH=obj
TARGETTYPE=DYNLINK
IMAGE_KIND=DLL
FLAVOR=KERNEL
EXTRA_CFLAGS=/DNTL_HAL_BUILD
TARGETLIBS="$(BASEDIR)\bin\printf.lib" "$(BASEDIR)\bin\libc.lib"
EXTRA_LFLAGS=/NOENTRY /FIXED /DYNAMICBASE:NO /HIGHENTROPYVA:NO /RELEASE /BASE:0xFFFF800010000000 /IGNORE:4216
INCLUDES=$(BASEDIR)\code\kernel\include
SOURCES=@/code/kernel/hal/hal_apic.c @/code/kernel/hal/hal_disk.c @/code/kernel/hal/hal_keyboard.c @/code/kernel/hal/hal_main.c @/code/kernel/hal/hal_mouse.c @/code/kernel/hal/hal_output.c \
@/code/kernel/hal/hal_pci.c @/code/kernel/hal/hal_processor.c @/code/kernel/hal/hal_rtc.c @/code/kernel/hal/hal_screen.c @/code/kernel/hal/hal_serial.c @/code/kernel/hal/hal_timer.c
code/kernel/include/kernel/dm_config.h
+1
View File
@@ -36,6 +36,7 @@ typedef struct DM_EDGES {
typedef enum DM_RUN_STATE {
DmRunWaiting,
DmRunStarting,
DmRunStarted,
DmRunFailed,
DmRunBlocked,
code/kernel/include/kernel/driver_module.h
+4 -1
View File
@@ -1,9 +1,12 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/status.h>
KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase);
KSTATUS DmPrepareLoadedDriver(const char* name, uintptr_t imageBase, bool* canReleaseImage);
struct LDR_KERNEL_MODULE;
void DmCommitLoadedDriver(uintptr_t imageBase, struct LDR_KERNEL_MODULE* image);
KSTATUS DmPrepareShutdown(void);
void DmCommitShutdown(void);
code/kernel/include/kernel/hal/hal_api.h
+14 -12
View File
@@ -132,7 +132,6 @@ typedef struct HAL_TIMER {
void (*Stall)(uint32_t us);
void (*HandleInterrupt)(void);
void (*ConfigureCurrentProcessor)(void);
uint8_t IrqNumber;
} HAL_TIMER;
#define HAL_RTC_TIME_FLAG_VALID 0x0001u
@@ -173,24 +172,27 @@ typedef struct HAL_PCI {
} HAL_PCI;
typedef struct HAL_INTERRUPT_CONTROLLER {
void (*RouteIrq)(uint8_t irq);
void (*MaskIrq)(uint8_t irq);
void (*UnmaskIrq)(uint8_t irq);
KSTATUS (*RouteGsi)(uint32_t gsi, uint8_t vector, uint32_t hardwareId,
uint16_t flags, bool masked);
KSTATUS (*MaskGsi)(uint32_t gsi, bool masked);
KSTATUS (*ResolveIsa)(uint8_t isaIrq, uint32_t* gsiOut, uint16_t* flagsOut);
void (*SendEoi)(void);
// The APIC task priority must follow the kernel's per-CPU IRQL.
void (*SetTaskPriority)(uint8_t priority);
uint8_t (*QueryTaskPriority)(void);
// Queries the current CPU's pending and in-service APIC bits for a vector.
bool (*VectorPending)(uint8_t vector);
void (*InitializeCurrentProcessor)(void);
void (*SendProcessorInterrupt)(uint32_t hardwareId, uint8_t vector);
void (*SendProcessorNmi)(uint32_t hardwareId);
void (*BroadcastNmi)(void);
KSTATUS (*ComposeMsi)(uint8_t vector, HAL_MSI_MESSAGE* messageOut);
KSTATUS (*ComposeMsi)(uint32_t hardwareId, uint8_t vector,
HAL_MSI_MESSAGE* messageOut);
} HAL_INTERRUPT_CONTROLLER;
typedef struct HAL_POWER {
// ACPI fixed-feature button events. Control-method devices require an AML owner.
KSTATUS (*QueryFixedButtonIrq)(uint8_t* irqOut);
KSTATUS (*ArmFixedButton)(void);
bool (*ConsumeFixedButton)(void);
// ACPI S5 soft-off. Disables interrupts, writes SLP_EN to PM1a/b_CNT,
// and halts if firmware doesn't respond. Does not return.
// Basic platform shutdown and reset, available without a firmware driver.
// The HAL halts if its final hardware transition returns.
void (*Shutdown)(void);
void (*Reset)(void);
} HAL_POWER;
@@ -231,7 +233,7 @@ typedef struct HAL_INIT_PARAMS {
} HAL_INIT_PARAMS;
// Increment whenever the kernel/HAL table ABI changes.
#define HAL_API_VERSION 4u
#define HAL_API_VERSION 8u
// Returns the HAL table. Processor interrupt-state operations are valid before HalInitialize().
HAL_ENTRY const HAL_API* HalGetApi(void);
code/kernel/include/kernel/hal/hal_apic.h
+10 -8
View File
@@ -6,15 +6,17 @@
#include <ntlite/status.h>
#include <kernel/hal/hal_api.h>
void HalApicRouteIrq(uint8_t irq);
void HalApicSetGsiBase(uint32_t gsiBase);
KSTATUS HalApicRouteGsi(uint32_t gsi, uint8_t vector, uint32_t hardwareId,
uint16_t flags, bool masked);
KSTATUS HalApicMaskGsi(uint32_t gsi, bool masked);
KSTATUS HalApicResolveIsa(uint8_t irq, uint32_t* gsiOut, uint16_t* flagsOut);
void HalApicAddIoapic(uintptr_t physical, uint32_t gsiBase);
void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags);
void HalApicSetSciIrq(uint16_t irq);
bool HalApicCanRouteIrq(uint8_t irq);
void HalApicMaskIrq(uint8_t irq);
void HalApicUnmaskIrq(uint8_t irq);
void HalApicSendEoi(void);
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys);
void HalApicSetTaskPriority(uint8_t priority);
uint8_t HalApicQueryTaskPriority(void);
bool HalApicVectorPending(uint8_t vector);
void HalApicInitialize(uintptr_t lapicPhys);
uint32_t HalApicGetBootProcessorId(void);
uint8_t HalApicUsingX2Apic(void);
uint8_t HalApicCanAddressId(uint32_t hardwareId);
@@ -23,4 +25,4 @@ void HalApicBroadcastNmi(void);
void HalApicSendProcessorInterrupt(uint32_t hardwareId, uint8_t vector);
void HalApicSendProcessorNmi(uint32_t hardwareId);
KSTATUS HalApicStartProcessor(uint32_t hardwareId, uintptr_t startPhysical);
KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut);
KSTATUS HalApicComposeMsi(uint32_t hardwareId, uint8_t vector, HAL_MSI_MESSAGE* messageOut);
code/kernel/include/kernel/io_pnp.h
+3 -1
View File
@@ -9,9 +9,11 @@ struct NTL_MODULE;
struct DRIVER_OBJECT;
KSTATUS IoPnpInitialize(void);
KSTATUS IoPnpPublishModule(struct NTL_MODULE* module,
KSTATUS IoPnpPrepareModule(struct NTL_MODULE* module,
const NTL_DEVICE_DESC* descriptors, uint32_t count,
struct DRIVER_OBJECT* const* ioDrivers, uint32_t ioCount);
void IoPnpCommitModule(struct NTL_MODULE* module);
void IoPnpAbortModule(struct NTL_MODULE* module);
KSTATUS IoPnpPhysicalDeviceArrived(struct DEVICE_OBJECT* pdo);
void IoPnpPhysicalDevicePublished(struct DEVICE_OBJECT* pdo);
void IoPnpPhysicalDeviceRemoved(struct DEVICE_OBJECT* pdo);
code/kernel/include/kernel/ke_boot_config.h
-2
View File
@@ -6,14 +6,12 @@
#include <stdint.h>
#include <ntlite/status.h>
#define KE_BOOT_COMMAND_LINE_MAX 256u
#define KE_BOOT_SYSTEM_VOLUME_MAX 128u
typedef struct KE_BOOT_CONFIG {
uint32_t Debug;
// Opaque system-volume selector interpreted by storage policy, never as a legacy drive slot.
char SystemVolume[KE_BOOT_SYSTEM_VOLUME_MAX];
char CommandLine[KE_BOOT_COMMAND_LINE_MAX];
} KE_BOOT_CONFIG;
// Parses the kernel-owned boot.ini settings from the raw bootloader string.
code/kernel/include/kernel/ke_firmware.h
+8
View File
@@ -0,0 +1,8 @@
// Firmware roots retained from the verified bootloader handoff.
#pragma once
#include <ntlite/boot_info.h>
void KeFirmwareInitialize(const BOOT_INFO* bootInfo);
void KeFirmwareLog(void);
code/kernel/include/kernel/ke_idt.h
+1 -3
View File
@@ -51,6 +51,4 @@ extern void KeFaultPF(void); // #PF page fault
extern void KeFaultUD(void); // #UD invalid opcode
extern void KeNmiIpiStub(void); // returning NMI SMP-control stub
// CPU-local IPI vector. This uses the common interrupt entry machinery but is
// not an external/device IRQ and is never routed through the external interrupt controller.
#define KE_VECTOR_RESCHEDULE_IPI 0xF0u
#include <kernel/ke_vectors.h>
code/kernel/include/kernel/ke_irq.h
+7 -12
View File
@@ -1,18 +1,13 @@
// Kernel ke irq declarations shared by implementation files and kernel clients.
// Kernel interrupt-manager declarations.
#pragma once
#include <stdint.h>
#include <ntlite/ddk/interrupt.h>
#include <kernel/hal/hal_api.h>
#include <ntlite/status.h>
void KeIrqInitialize(const HAL_API* hal);
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq);
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void));
void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm
// Low-level interrupt entry/exit hooks. These are called by the common IRQ
// assembly stub around KeIrqDispatch. The exit hook is where deferred
// interrupt-time scheduling now happens, so IRQ handlers never switch away
// before KeIrqDispatch has returned and sent EOI.
void KeIrqRegisterLocalTimerHandler(void (*handler)(void));
void KeIrqDispatch(uint8_t vector);
void KiEnterInterrupt(void);
void KiExitInterrupt(void);
// CPU offlining requires the interrupt manager to evacuate every route first.
KSTATUS KiInterruptPrepareCpuOffline(uint32_t cpuIndex);
code/kernel/include/kernel/ke_irql.h
+1 -3
View File
@@ -2,9 +2,7 @@
#pragma once
// ke_irql.h - software IRQL skeleton.
// The level constants are DDK-visible because current synchronization and IRP
// layouts expose KIRQL. Kernel routines remain declared here.
// IRQL levels are DDK-visible; the kernel synchronizes them with APIC TPR.
#include <stdbool.h>
#include <ntlite/ddk/irql.h>
code/kernel/include/kernel/ke_power.h
+3 -2
View File
@@ -5,9 +5,10 @@
struct PROCESS;
struct EVENT;
struct HAL_API;
KSTATUS KePowerInitialize(void);
void KePowerInitializeButton(const struct HAL_API* hal);
void KePowerInitializePolicy(void);
KSTATUS KePowerPrepareFirmware(uint32_t action);
KSTATUS KePowerEnterFirmware(uint32_t action);
void KePowerSetInitProcess(struct PROCESS* init);
KSTATUS KePowerRequest(uint32_t action);
KSTATUS KePowerRequestHardware(uint32_t action);
code/kernel/include/kernel/ke_vectors.h
+12
View File
@@ -0,0 +1,12 @@
// x86-64 interrupt-vector assignments shared by the kernel and HAL.
#pragma once
// CPU exception gates occupy vectors 0x00 through 0x1f.
// Software gates occupy 0x20 through 0x2f; only the syscall gate is active.
#define KE_VECTOR_SYSCALL 0x2Fu
#define KE_VECTOR_DEVICE_FIRST 0x30u
#define KE_VECTOR_DEVICE_LAST 0xCFu
#define KE_VECTOR_CPU_TIMER 0xE0u
#define KE_VECTOR_RESCHEDULE_IPI 0xF0u
#define KE_VECTOR_SPURIOUS 0xFFu
code/kernel/include/kernel/ldr_image.h
+5 -2
View File
@@ -42,6 +42,9 @@ KSTATUS LdrProtectImageForProcess(struct PROCESS* process, uintptr_t imageBase,
// Called by KernelMain to register the fixed kernel/HAL bootstrap images so LdrGetExport can find them.
struct BOOT_INFO; // forward declaration
KSTATUS LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
KSTATUS LdrRegisterModule(const char* name, uintptr_t base);
KSTATUS LdrSetModuleExportsAvailable(const char* name, bool available);
typedef struct LDR_KERNEL_MODULE LDR_KERNEL_MODULE;
KSTATUS LdrRegisterModule(const char* name, uintptr_t base, LDR_KERNEL_MODULE** out);
void LdrPublishModule(LDR_KERNEL_MODULE* module);
void LdrDiscardModule(LDR_KERNEL_MODULE* module);
void LdrReleaseKernelImage(uintptr_t imageBase);
const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut, size_t* sizeOut);
code/kernel/include/kernel/syscall_nr.h
+1 -1
View File
@@ -2,7 +2,7 @@
#pragma once
// INT 0x40 calling convention, MS x64.
// INT 0x2F calling convention, MS x64.
// RAX selects the syscall number. RCX, RDX, R8, and R9 carry args 0-3.
// Syscalls use at most four register arguments; larger operations use request structures.
// RAX carries the return value.
code/kernel/io_ddk.c
+119 -12
View File
@@ -4,6 +4,9 @@
#include <kernel/io_dma.h>
#include <kernel/ke_audio.h>
#include <kernel/ke_irq.h>
#include <kernel/ke_vectors.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/mm_pte.h>
static const HAL_API* gHal;
@@ -14,6 +17,10 @@ void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase) {
gPhysicalWindowBase = physicalWindowBase;
}
__declspec(dllexport) uintptr_t KeCurrentThreadIdentity(void) {
return (uintptr_t)KeGetCurrentThread();
}
__declspec(dllexport) uint64_t KeQueryTicks(void) {
return gHal && gHal->Timer.QueryTicks ? gHal->Timer.QueryTicks() : 0;
}
@@ -30,18 +37,6 @@ __declspec(dllexport) void KePciWriteConfig32(uint16_t segment, uint8_t bus, uin
if (gHal && gHal->Pci.Write32) gHal->Pci.Write32(segment, bus, device, function, offset, value);
}
__declspec(dllexport) KSTATUS KeIrqComposeMsiMessage(uint8_t irq, NTL_DDK_MSI_MESSAGE* out) {
if (!gHal || !out || irq >= 24u || !gHal->InterruptController.ComposeMsi)
return NTLSTATUS_INVALID_PARAMETER;
HAL_MSI_MESSAGE message;
KSTATUS status = gHal->InterruptController.ComposeMsi((uint8_t)(0x20u + irq), &message);
if (!KSUCCESS(status)) return status;
out->Address = message.Address;
out->Data = message.Data;
out->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) void* MmDdkPhysToVirt(uint64_t physicalAddress) {
if (!gPhysicalWindowBase || physicalAddress > UINTPTR_MAX - gPhysicalWindowBase)
return (void*)0;
@@ -53,6 +48,118 @@ __declspec(dllexport) KSTATUS MmDdkMapPage(uintptr_t virtualAddress, uint64_t ph
mode == NtlDdkMapReplaceExisting ? MmPteMapReplaceExisting : MmPteMapFailIfPresent);
}
// Each slot is a recyclable 2 MiB VA reservation; no pool allocation is needed
// when reserving a device mapping. This aperture stays separate from PCI BARs.
#define DDK_IO_VA_BASE 0xFFFF980000000000ULL
#define DDK_IO_SLOT_SIZE (2ULL * 1024ULL * 1024ULL)
#define DDK_IO_SLOT_COUNT 256u
typedef struct DDK_IO_MAPPING {
uint8_t State;
uint32_t Pages;
uintptr_t Offset;
size_t Length;
} DDK_IO_MAPPING;
static SPINLOCK gDdkIoMapLock = SPINLOCK_INIT;
static DDK_IO_MAPPING gDdkIoMaps[DDK_IO_SLOT_COUNT];
// The caller owns the mapping until an exact-length unmap completes.
__declspec(dllexport) KSTATUS MmDdkMapIoMemory(uint64_t physicalAddress, size_t length, void** outVirtual) {
if (!outVirtual) return NTLSTATUS_INVALID_PARAMETER;
*outVirtual = NULL;
if (!KeCanWait()) return NTLSTATUS_NOT_SUPPORTED;
if (!length || physicalAddress > UINT64_MAX - length) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t offset = (uintptr_t)(physicalAddress & (PAGE_SIZE - 1u));
if (length > DDK_IO_SLOT_SIZE - offset) return NTLSTATUS_INVALID_PARAMETER;
uint32_t pages = (uint32_t)((offset + length + PAGE_SIZE - 1u) / PAGE_SIZE);
if (pages == 0 || pages > DDK_IO_SLOT_SIZE / PAGE_SIZE) return NTLSTATUS_INVALID_PARAMETER;
// Reject a UC alias of RAM already covered by the firmware's WB/WC
// direct map. Real device registers have no direct mapping or an UC one.
for (uint32_t i = 0; i < pages; i++) {
uint64_t frame = (physicalAddress - offset) + (uint64_t)i * PAGE_SIZE;
if (gPhysicalWindowBase && frame <= UINTPTR_MAX - gPhysicalWindowBase) {
PHYSICAL_ADDRESS existing = 0;
uint64_t existingFlags = 0;
if (KSUCCESS(MmPteQueryPage(gPhysicalWindowBase + (uintptr_t)frame,
&existing, &existingFlags))) {
if ((existing & PAGE_MASK) != frame ||
(existingFlags & PAGE_CACHE_MASK) != PAGE_CACHE_UC)
return NTLSTATUS_NOT_SUPPORTED;
}
}
}
uintptr_t flags = KeAcquireSpinlock(&gDdkIoMapLock);
uint32_t slot;
for (slot = 0; slot < DDK_IO_SLOT_COUNT; slot++) {
if (!gDdkIoMaps[slot].State) {
gDdkIoMaps[slot].State = 1;
gDdkIoMaps[slot].Length = length;
gDdkIoMaps[slot].Offset = offset;
gDdkIoMaps[slot].Pages = pages;
break;
}
}
KeReleaseSpinlock(&gDdkIoMapLock, flags);
if (slot == DDK_IO_SLOT_COUNT) return NTLSTATUS_NO_MEMORY;
uintptr_t base = (uintptr_t)(DDK_IO_VA_BASE + (uint64_t)slot * DDK_IO_SLOT_SIZE);
uint64_t physBase = physicalAddress - offset;
uint32_t mapped = 0;
KSTATUS status = NTLSTATUS_SUCCESS;
for (; mapped < pages; mapped++) {
status = MmPteMapPage(base + (uintptr_t)mapped * PAGE_SIZE,
physBase + (uint64_t)mapped * PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC | PAGE_NX,
MmPteMapFailIfPresent);
if (!KSUCCESS(status)) break;
}
if (mapped != pages) {
while (mapped) (void)MmPteUnmapPage(base + (uintptr_t)(--mapped) * PAGE_SIZE);
flags = KeAcquireSpinlock(&gDdkIoMapLock);
gDdkIoMaps[slot].State = 0;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
return status;
}
flags = KeAcquireSpinlock(&gDdkIoMapLock);
gDdkIoMaps[slot].State = 2;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
*outVirtual = (void*)(base + offset);
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmDdkUnmapIoMemory(void* virtualAddress, size_t length) {
if (!KeCanWait()) return NTLSTATUS_NOT_SUPPORTED;
if (!virtualAddress || !length) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t address = (uintptr_t)virtualAddress;
if (address < DDK_IO_VA_BASE ||
(uint64_t)(address - DDK_IO_VA_BASE) >= DDK_IO_SLOT_SIZE * DDK_IO_SLOT_COUNT)
return NTLSTATUS_INVALID_PARAMETER;
uint32_t slot = (uint32_t)((address - DDK_IO_VA_BASE) / DDK_IO_SLOT_SIZE);
uintptr_t base = (uintptr_t)(DDK_IO_VA_BASE + (uint64_t)slot * DDK_IO_SLOT_SIZE);
uintptr_t flags = KeAcquireSpinlock(&gDdkIoMapLock);
DDK_IO_MAPPING* map = &gDdkIoMaps[slot];
if (map->State != 2 || map->Length != length || address != base + map->Offset) {
KeReleaseSpinlock(&gDdkIoMapLock, flags);
return NTLSTATUS_INVALID_PARAMETER;
}
map->State = 3;
uint32_t pages = map->Pages;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
KSTATUS result = NTLSTATUS_SUCCESS;
for (uint32_t i = 0; i < pages; i++) {
KSTATUS status = MmPteUnmapPage(base + (uintptr_t)i * PAGE_SIZE);
if (!KSUCCESS(status)) result = status;
}
flags = KeAcquireSpinlock(&gDdkIoMapLock);
// Preserve the slot after a failed unmap to prevent stale PTE reuse.
if (KSUCCESS(result)) map->State = 0;
KeReleaseSpinlock(&gDdkIoMapLock, flags);
return result;
}
__declspec(dllexport) KSTATUS IoDmaGetAdapter(struct DEVICE_OBJECT* device, const IO_DMA_REQUIREMENTS* requirements, IO_DMA_ADAPTER** outAdapter) {
return IoGetDmaDdk()->GetAdapter(device, requirements, outAdapter);
}
code/kernel/io_device.c
+11 -16
View File
@@ -159,12 +159,10 @@ __declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
return NTLSTATUS_SUCCESS;
}
// Implements the reference Device Object operation.
__declspec(dllexport) void IoReferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObReferenceObject(&device->Header);
}
// Implements the dereference Device Object operation.
__declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
if (device) ObDereferenceObject(&device->Header);
}
@@ -228,7 +226,6 @@ __declspec(dllexport) DEVICE_OBJECT* IoAttachDeviceToDeviceStack(DEVICE_OBJECT*
return top;
}
// Implements the detach Device operation.
__declspec(dllexport) void IoDetachDevice(DEVICE_OBJECT* source) {
if (!source || !source->LowerDevice) return;
@@ -258,7 +255,6 @@ __declspec(dllexport) uint8_t IoGetDeviceStackSize(DEVICE_OBJECT* device) {
return count ? count : 1;
}
// Implements the forward Irp To Lower operation.
__declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !irp || !device->LowerDevice)
return NTLSTATUS_NOT_SUPPORTED;
@@ -373,7 +369,6 @@ static void IoDiskStatsNoteWrite(uint8_t driveIndex, uint32_t count) {
KeReleaseSpinlock(&state->Lock, flags);
}
// Implements the notify Disk Change operation.
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
IO_DISK_CHANGE_CALLBACK_ENTRY* entry = &gDiskChangeCallbacks[i];
@@ -429,7 +424,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
return NTLSTATUS_SUCCESS;
}
// Implements the notify Disk Surprise Removed operation.
// Invalidates mounted partitions when a disk disappears unexpectedly.
__declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return NTLSTATUS_INVALID_PARAMETER;
@@ -442,29 +437,32 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
return NTLSTATUS_SUCCESS;
}
// Implements the unregister Disk Device operation.
// Releases volume consumers before the backing device completes orderly teardown.
__declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return NTLSTATUS_INVALID_PARAMETER;
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return NTLSTATUS_NOT_FOUND;
if (dev->PnpState != DevicePnpSurpriseRemoved) {
bool surpriseRemoved = dev->PnpState == DevicePnpSurpriseRemoved;
if (!surpriseRemoved) {
IoNotifyDiskChange(driveIndex, dev, IoDeviceQueryRemove);
KSTATUS queryStatus = IoSendPnpIrpSync(dev, IRP_MN_QUERY_REMOVE_DEVICE);
if (!KSUCCESS(queryStatus) && queryStatus != NTLSTATUS_NOT_SUPPORTED)
if (!KSUCCESS(queryStatus) && queryStatus != NTLSTATUS_NOT_SUPPORTED) {
(void)IoSendPnpIrpSync(dev, IRP_MN_CANCEL_REMOVE_DEVICE);
return queryStatus;
}
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
// Notify volume owners while the backing disk can still service teardown I/O.
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
}
dev->PnpState = DevicePnpRemovePending;
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
if (surpriseRemoved)
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return NTLSTATUS_SUCCESS;
}
@@ -526,19 +524,16 @@ __declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
return gDiskDevices[driveIndex];
}
// Implements the reference Disk Device operation.
__declspec(dllexport) DEVICE_OBJECT* IoReferenceDiskDevice(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
if (dev) IoReferenceDeviceObject(dev);
return dev;
}
// Implements the disk Device Present operation.
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
}
// Implements the disk Sector Count operation.
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
return dev ? dev->u.Disk.SectorCount : 0;
code/kernel/io_file.c
-4
View File
@@ -254,7 +254,6 @@ static KSTATUS KiIoCancelAsyncFileRequest(IO_ASYNC_REQUEST_OBJECT* async, void*
return NTLSTATUS_OBJECT_DEAD;
}
// Implements the io Async File Dequeue operation.
static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
LIST_ENTRY* node = ListPopHead(&gAsyncFileQueue);
@@ -275,7 +274,6 @@ static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
return req;
}
// Implements the io Async File Worker operation.
static void KiIoAsyncFileWorker(void* arg) {
(void)arg;
@@ -317,7 +315,6 @@ static void KiIoAsyncFileWorker(void* arg) {
}
}
// Implements the io Ensure Async File Io Started operation.
static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileInitLock);
if (gAsyncFileInitState == 2) {
@@ -517,7 +514,6 @@ __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const vo
lockedBuffer, lockedIosb, requestOut);
}
// Implements the async File Request Object operation.
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? &req->Async.Header : (OBJECT*)0;
}
code/kernel/io_pnp.c
+128 -114
View File
@@ -12,8 +12,6 @@
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#define PNP_MAX_DESCRIPTORS 64u
#define PNP_MAX_DEVICES 128u
#define PNP_WORKERS 2u
typedef enum PNP_BIND_STATE {
@@ -29,12 +27,15 @@ typedef enum PNP_BIND_STATE {
} PNP_BIND_STATE;
typedef struct PNP_DRIVER {
struct PNP_DRIVER* Next;
const NTL_DEVICE_DESC* Descriptor;
DRIVER_OBJECT* IoDriver;
struct NTL_MODULE* Module;
bool Committed;
} PNP_DRIVER;
typedef struct PNP_NODE {
struct PNP_NODE* Next;
DEVICE_OBJECT* Pdo;
DEVICE_OBJECT* ParentPdo;
NTL_DEVICE* Binding;
@@ -63,10 +64,11 @@ struct NTL_DEVICE {
};
static SPINLOCK gPnpLock = SPINLOCK_INIT;
static PNP_DRIVER gPnpDrivers[PNP_MAX_DESCRIPTORS];
static uint32_t gPnpDriverCount;
static PNP_NODE* gPnpNodes[PNP_MAX_DEVICES];
static uint32_t gPnpNodeCount;
static PNP_DRIVER* gPnpDrivers;
static PNP_DRIVER* gPnpDriversTail;
static PNP_NODE* gPnpNodes;
static PNP_NODE* gPnpNodesTail;
static size_t gPnpNodeCount;
static EVENT* gPnpReady;
static EVENT* gPnpChanged;
static bool gPnpStarted;
@@ -113,18 +115,19 @@ static bool IoPnpMatchesIoDriver(const DRIVER_OBJECT* driver, const DEVICE_OBJEC
}
static const PNP_DRIVER* IoPnpFindDriver(const DEVICE_OBJECT* pdo) {
for (uint32_t i = 0; i < gPnpDriverCount; ++i) {
if ((gPnpDrivers[i].Descriptor && IoPnpMatches(gPnpDrivers[i].Descriptor, pdo)) ||
(gPnpDrivers[i].IoDriver && IoPnpMatchesIoDriver(gPnpDrivers[i].IoDriver, pdo)))
return &gPnpDrivers[i];
for (PNP_DRIVER* driver = gPnpDrivers; driver; driver = driver->Next) {
if (!driver->Committed) continue;
if ((driver->Descriptor && IoPnpMatches(driver->Descriptor, pdo)) ||
(driver->IoDriver && IoPnpMatchesIoDriver(driver->IoDriver, pdo)))
return driver;
}
return (const PNP_DRIVER*)0;
}
static PNP_NODE* IoPnpFindNode(const DEVICE_OBJECT* pdo) {
for (uint32_t i = 0; i < gPnpNodeCount; ++i)
if (gPnpNodes[i]->Pdo == pdo)
return gPnpNodes[i];
for (PNP_NODE* node = gPnpNodes; node; node = node->Next)
if (node->Pdo == pdo)
return node;
return (PNP_NODE*)0;
}
@@ -139,9 +142,9 @@ static DEVICE_OBJECT* IoPnpParentPdo(const DEVICE_OBJECT* pdo) {
}
static bool IoPnpHasChildren(const DEVICE_OBJECT* parent) {
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
const DEVICE_OBJECT* ancestor = gPnpNodes[i]->ParentPdo;
for (uint32_t depth = 0; ancestor && depth < PNP_MAX_DEVICES; ++depth) {
for (PNP_NODE* candidate = gPnpNodes; candidate; candidate = candidate->Next) {
const DEVICE_OBJECT* ancestor = candidate->ParentPdo;
for (size_t depth = 0; ancestor && depth < gPnpNodeCount; ++depth) {
if (ancestor == parent) return true;
const PNP_NODE* node = IoPnpFindNode(ancestor);
ancestor = node ? node->ParentPdo : (const DEVICE_OBJECT*)0;
@@ -180,12 +183,15 @@ static void IoPnpDispose(PNP_NODE* node) {
}
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
if (gPnpNodes[i] == node) {
gPnpNodes[i] = gPnpNodes[--gPnpNodeCount];
gPnpNodes[gPnpNodeCount] = (PNP_NODE*)0;
break;
PNP_NODE** link = &gPnpNodes;
while (*link && *link != node) link = &(*link)->Next;
if (*link) {
*link = node->Next;
if (gPnpNodesTail == node) {
gPnpNodesTail = (PNP_NODE*)0;
for (PNP_NODE* it = gPnpNodes; it; it = it->Next) gPnpNodesTail = it;
}
--gPnpNodeCount;
}
KeReleaseSpinlock(&gPnpLock, flags);
IoDereferenceDeviceObject(node->Pdo);
@@ -247,8 +253,7 @@ static void IoPnpWorker(void* ignored) {
bool cleanup = false;
bool cancel = false;
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
PNP_NODE* candidate = gPnpNodes[i];
for (PNP_NODE* candidate = gPnpNodes; candidate; candidate = candidate->Next) {
if (candidate->State == PnpPending && candidate->Removed &&
!candidate->CancelIssued && (candidate->Driver->Descriptor && candidate->Driver->Descriptor->CancelStart)) {
candidate->CancelIssued = true;
@@ -338,82 +343,105 @@ KSTATUS IoPnpInitialize(void) {
return NTLSTATUS_SUCCESS;
}
KSTATUS IoPnpPublishModule(struct NTL_MODULE* module,
// Reserve declarative and DRIVER_OBJECT names in one namespace.
KSTATUS IoPnpPrepareModule(struct NTL_MODULE* module,
const NTL_DEVICE_DESC* descriptors, uint32_t count,
DRIVER_OBJECT* const* ioDrivers, uint32_t ioCount) {
if (!gPnpStarted || !module || (count && !descriptors) || (ioCount && !ioDrivers))
return NTLSTATUS_INVALID_PARAMETER;
if (count > UINT32_MAX - ioCount) return NTLSTATUS_INVALID_PARAMETER;
PNP_DRIVER* first = (PNP_DRIVER*)0;
PNP_DRIVER* last = (PNP_DRIVER*)0;
for (uint32_t i = 0; i < count + ioCount; ++i) {
PNP_DRIVER* driver = (PNP_DRIVER*)MmAllocatePool(sizeof(*driver));
if (!driver) {
while (first) {
PNP_DRIVER* next = first->Next;
MmFreePool(first);
first = next;
}
return NTLSTATUS_NO_MEMORY;
}
memset(driver, 0, sizeof(*driver));
driver->Module = module;
if (i < count) driver->Descriptor = &descriptors[i];
else driver->IoDriver = ioDrivers[i - count];
if (last) last->Next = driver;
else first = driver;
last = driver;
}
KSTATUS status = NTLSTATUS_SUCCESS;
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
if (gPnpShutdown) {
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_DEVICE_REMOVED;
}
if (count > PNP_MAX_DESCRIPTORS - gPnpDriverCount ||
ioCount > PNP_MAX_DESCRIPTORS - gPnpDriverCount - count) {
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_NO_MEMORY;
}
for (uint32_t i = 0; i < count; ++i) {
for (uint32_t j = 0; j < gPnpDriverCount; ++j) {
SYS_STRING_VIEW existing = gPnpDrivers[j].IoDriver ?
gPnpDrivers[j].IoDriver->Name :
SysSvFromParts(gPnpDrivers[j].Descriptor->Name,
strlen(gPnpDrivers[j].Descriptor->Name));
SYS_STRING_VIEW incoming = SysSvFromParts(descriptors[i].Name,
strlen(descriptors[i].Name));
if (SysSvEqual(existing, incoming)) {
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_ALREADY_EXISTS;
if (gPnpShutdown) status = NTLSTATUS_DEVICE_REMOVED;
for (PNP_DRIVER* proposed = first; KSUCCESS(status) && proposed;
proposed = proposed->Next) {
SYS_STRING_VIEW name = proposed->IoDriver ? proposed->IoDriver->Name :
SysSvFromCstr(proposed->Descriptor->Name);
for (PNP_DRIVER* existing = gPnpDrivers; existing; existing = existing->Next) {
SYS_STRING_VIEW oldName = existing->IoDriver ? existing->IoDriver->Name :
SysSvFromCstr(existing->Descriptor->Name);
if (SysSvEqual(name, oldName)) {
status = NTLSTATUS_ALREADY_EXISTS;
break;
}
}
for (uint32_t j = i + 1; j < count; ++j) {
if (!strcmp(descriptors[i].Name, descriptors[j].Name)) {
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_ALREADY_EXISTS;
}
for (PNP_DRIVER* other = proposed->Next; KSUCCESS(status) && other;
other = other->Next) {
SYS_STRING_VIEW otherName = other->IoDriver ? other->IoDriver->Name :
SysSvFromCstr(other->Descriptor->Name);
if (SysSvEqual(name, otherName)) status = NTLSTATUS_ALREADY_EXISTS;
}
}
for (uint32_t i = 0; i < ioCount; ++i) {
DRIVER_OBJECT* driver = ioDrivers[i];
for (uint32_t j = 0; j < gPnpDriverCount; ++j) {
SYS_STRING_VIEW existing = gPnpDrivers[j].IoDriver ?
gPnpDrivers[j].IoDriver->Name :
SysSvFromParts(gPnpDrivers[j].Descriptor->Name,
strlen(gPnpDrivers[j].Descriptor->Name));
if (SysSvEqual(driver->Name, existing)) {
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_ALREADY_EXISTS;
}
}
for (uint32_t j = i + 1; j < ioCount; ++j) {
if (SysSvEqual(driver->Name, ioDrivers[j]->Name)) {
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_ALREADY_EXISTS;
}
}
for (uint32_t j = 0; j < count; ++j) {
if (SysSvEqual(driver->Name, SysSvFromParts(descriptors[j].Name,
strlen(descriptors[j].Name)))) {
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_ALREADY_EXISTS;
}
}
}
for (uint32_t i = 0; i < count; ++i) {
gPnpDrivers[gPnpDriverCount].Descriptor = &descriptors[i];
gPnpDrivers[gPnpDriverCount].IoDriver = (DRIVER_OBJECT*)0;
gPnpDrivers[gPnpDriverCount].Module = module;
++gPnpDriverCount;
}
for (uint32_t i = 0; i < ioCount; ++i) {
gPnpDrivers[gPnpDriverCount].Descriptor = (const NTL_DEVICE_DESC*)0;
gPnpDrivers[gPnpDriverCount].IoDriver = ioDrivers[i];
gPnpDrivers[gPnpDriverCount].Module = module;
++gPnpDriverCount;
if (KSUCCESS(status) && first) {
if (gPnpDriversTail) gPnpDriversTail->Next = first;
else gPnpDrivers = first;
gPnpDriversTail = last;
}
KeReleaseSpinlock(&gPnpLock, flags);
if (!KSUCCESS(status)) {
while (first) {
PNP_DRIVER* next = first->Next;
MmFreePool(first);
first = next;
}
}
return status;
}
// Reserved registrations become eligible for binding without allocation.
void IoPnpCommitModule(struct NTL_MODULE* module) {
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
for (PNP_DRIVER* driver = gPnpDrivers; driver; driver = driver->Next)
if (driver->Module == module) driver->Committed = true;
KeReleaseSpinlock(&gPnpLock, flags);
IoPnpWake();
return NTLSTATUS_SUCCESS;
}
// Abort releases reservations before any binding can reference them.
void IoPnpAbortModule(struct NTL_MODULE* module) {
PNP_DRIVER* removed = (PNP_DRIVER*)0;
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
PNP_DRIVER** link = &gPnpDrivers;
while (*link) {
PNP_DRIVER* driver = *link;
if (driver->Module != module || driver->Committed) {
link = &driver->Next;
continue;
}
*link = driver->Next;
driver->Next = removed;
removed = driver;
}
gPnpDriversTail = (PNP_DRIVER*)0;
for (PNP_DRIVER* it = gPnpDrivers; it; it = it->Next) gPnpDriversTail = it;
KeReleaseSpinlock(&gPnpLock, flags);
while (removed) {
PNP_DRIVER* next = removed->Next;
MmFreePool(removed);
removed = next;
}
}
KSTATUS IoPnpPhysicalDeviceArrived(DEVICE_OBJECT* pdo) {
@@ -432,21 +460,16 @@ KSTATUS IoPnpPhysicalDeviceArrived(DEVICE_OBJECT* pdo) {
MmFreePool(node);
return NTLSTATUS_DEVICE_REMOVED;
}
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
if (gPnpNodes[i]->Pdo == pdo) {
KeReleaseSpinlock(&gPnpLock, flags);
IoDereferenceDeviceObject(pdo);
MmFreePool(node);
return NTLSTATUS_ALREADY_EXISTS;
}
}
if (gPnpNodeCount == PNP_MAX_DEVICES) {
if (IoPnpFindNode(pdo)) {
KeReleaseSpinlock(&gPnpLock, flags);
IoDereferenceDeviceObject(pdo);
MmFreePool(node);
return NTLSTATUS_NO_MEMORY;
return NTLSTATUS_ALREADY_EXISTS;
}
gPnpNodes[gPnpNodeCount++] = node;
if (gPnpNodesTail) gPnpNodesTail->Next = node;
else gPnpNodes = node;
gPnpNodesTail = node;
++gPnpNodeCount;
KeReleaseSpinlock(&gPnpLock, flags);
return NTLSTATUS_SUCCESS;
}
@@ -454,12 +477,8 @@ KSTATUS IoPnpPhysicalDeviceArrived(DEVICE_OBJECT* pdo) {
void IoPnpPhysicalDevicePublished(DEVICE_OBJECT* pdo) {
if (!pdo || !gPnpStarted) return;
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
if (gPnpNodes[i]->Pdo == pdo) {
gPnpNodes[i]->Published = true;
break;
}
}
PNP_NODE* node = IoPnpFindNode(pdo);
if (node) node->Published = true;
KeReleaseSpinlock(&gPnpLock, flags);
IoPnpWake();
}
@@ -467,12 +486,8 @@ void IoPnpPhysicalDevicePublished(DEVICE_OBJECT* pdo) {
void IoPnpPhysicalDeviceRemoved(DEVICE_OBJECT* pdo) {
if (!pdo || !gPnpStarted) return;
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
if (gPnpNodes[i]->Pdo == pdo) {
gPnpNodes[i]->Removed = true;
break;
}
}
PNP_NODE* node = IoPnpFindNode(pdo);
if (node) node->Removed = true;
KeReleaseSpinlock(&gPnpLock, flags);
IoPnpWake();
}
@@ -483,8 +498,7 @@ KSTATUS IoPnpWaitForIdle(uint64_t timeoutMs) {
for (;;) {
bool idle = true;
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
PNP_NODE* node = gPnpNodes[i];
for (PNP_NODE* node = gPnpNodes; node; node = node->Next) {
if (node->State == PnpReserved || node->State == PnpStarting ||
node->State == PnpPending || node->State == PnpCancelling ||
node->State == PnpCleaning ||
@@ -512,8 +526,8 @@ KSTATUS IoPnpShutdown(uint64_t timeoutMs) {
if (!gPnpStarted) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
gPnpShutdown = true;
for (uint32_t i = 0; i < gPnpNodeCount; ++i)
gPnpNodes[i]->Removed = true;
for (PNP_NODE* node = gPnpNodes; node; node = node->Next)
node->Removed = true;
KeReleaseSpinlock(&gPnpLock, flags);
IoPnpWake();
return IoPnpWaitForIdle(timeoutMs);
code/kernel/io_volume.c
+3 -1
View File
@@ -379,7 +379,9 @@ __declspec(dllexport) KSTATUS IoRemoveVolume(IO_VOLUME* volume, IO_VOLUME_REMOVA
KiIoVolumeAddStat(&gVolumeStats.MountedVolumes, -1);
KiIoVolumeAddStat(&gVolumeStats.RemovedVolumes, 1);
KiIoNotifyVolumeChange();
DbgPrint("io: volume %u removed reason=%u\n", volume->InstanceId, (unsigned)reason);
const char* reasonName = reason == IoVolumeUnmounted ? "unmounted" :
reason == IoVolumeBackingRemoved ? "backing-removed" : "provider-dead";
DbgPrint("io: volume %u removed reason=%s\n", volume->InstanceId, reasonName);
if (dropRegistryReference)
ObDereferenceObject(&volume->Header);
return NTLSTATUS_SUCCESS;
code/kernel/ke_apc.c
-8
View File
@@ -144,7 +144,6 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
return true;
}
// Implements the enter Critical Region operation.
__declspec(dllexport) void KeEnterCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
@@ -154,7 +153,6 @@ __declspec(dllexport) void KeEnterCriticalRegion(void) {
KeReleaseSpinlock(&thread->ApcLock, flags);
}
// Implements the leave Critical Region operation.
__declspec(dllexport) void KeLeaveCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
@@ -169,14 +167,12 @@ __declspec(dllexport) void KeLeaveCriticalRegion(void) {
KiDeliverKernelApcs();
}
// Implements the are Kernel Apcs Disabled operation.
__declspec(dllexport) bool KeAreKernelApcsDisabled(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return true;
return thread->KernelApcDisable != 0;
}
// Implements the pop Kernel Apc operation.
static KAPC* KiPopKernelApc(THREAD* thread) {
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
LIST_ENTRY* node = ListPopHead(&thread->ApcQueue);
@@ -189,7 +185,6 @@ static KAPC* KiPopKernelApc(THREAD* thread) {
return CONTAINING_RECORD(node, KAPC, Link);
}
// Implements the deliver Kernel Apcs operation.
void KiDeliverKernelApcs(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread)
@@ -322,7 +317,6 @@ static KUSER_APC* KiPopUserApc(THREAD* thread, bool alertableDelivery) {
return (KUSER_APC*)0;
}
// Implements the check For User Apc Delivery operation.
void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
if (!frame || !KeTrapFrameFromUser(frame))
return;
@@ -394,7 +388,6 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
MmFreePool(apc);
}
// Implements the rundown Apc State operation.
void KiRundownApcState(THREAD* thread) {
if (!thread) return;
@@ -435,7 +428,6 @@ void KiRundownApcState(THREAD* thread) {
thread->UserApcSavedRax = 0;
}
// Implements the user Apc Return operation.
__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* userFrame) {
THREAD* thread = KeGetCurrentThread();
if (!thread)
code/kernel/ke_audio.c
-1
View File
@@ -1447,7 +1447,6 @@ static KSTATUS KiAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* reg
return NTLSTATUS_SUCCESS;
}
// Implements the provider-facing endpoint registration operation.
static KSTATUS KiAudioDdkRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration,
IO_AUDIO_ENDPOINT** outEndpoint) {
return KiAudioRegisterEndpoint(registration, outEndpoint);
code/kernel/ke_boot_config.c
-5
View File
@@ -146,11 +146,6 @@ KSTATUS KeParseBootConfiguration(const char* text, size_t length, KE_BOOT_CONFIG
return status;
if (config.Debug & ~KE_DEBUG_VALID_MASK)
return NTLSTATUS_INVALID_PARAMETER;
} else if (strcmp(section, "init") == 0 && KiBootConfigSpanEquals(keyBegin, keyEnd, "commandline")) {
KSTATUS status = KiBootConfigCopyString(valueBegin, valueEnd,
config.CommandLine, sizeof(config.CommandLine));
if (!KSUCCESS(status))
return status;
}
}
code/kernel/ke_debug.c
+3 -81
View File
@@ -30,7 +30,6 @@ static const HAL_API* gDebugHal;
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_FATAL_OUTPUT_LINE_SIZE 2048u
#define KE_STACK_DUMP_SLOT_COUNT 12u
#define KE_RBP_CALL_STACK_DEPTH 16u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
@@ -458,7 +457,6 @@ static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
return "unknown";
}
// Implements the resolve Module operation.
static const char* KiResolveModule(PROCESS* proc, uintptr_t addr) {
return KiResolveModuleEx(proc, addr, (uintptr_t*)0, (size_t*)0);
}
@@ -639,28 +637,6 @@ static void KiDumpModules(PROCESS* proc) {
}
#if 0
// Returns whether a range is inside the current kernel stack.
static bool KiKernelStackContains(const THREAD* thread, uintptr_t addr, size_t size) {
if (!thread || !thread->KernelStackBase || !thread->KernelStackTop || !addr || size == 0)
return false;
if (size - 1u > (size_t)(UINTPTR_MAX - addr))
return false;
uintptr_t base = (uintptr_t)thread->KernelStackBase;
uintptr_t top = thread->KernelStackTop;
uintptr_t end = addr + size;
return addr >= base && end <= top;
}
// Returns whether the fault frame is using the current kernel stack.
static bool KiFaultFrameOnKernelStack(const THREAD* thread, const KFAULT_FRAME* f) {
if (!f || !thread)
return false;
return KiKernelStackContains(thread, (uintptr_t)f->Rsp, sizeof(uint64_t));
}
#endif
// Dumps stack words near RSP.
static void KiDumpStack(PROCESS* proc, uintptr_t rsp) {
KiFaultPrintf("Stack @ %016llx:\n", (unsigned long long)rsp);
@@ -684,58 +660,8 @@ static void KiDumpStack(PROCESS* proc, uintptr_t rsp) {
}
}
#if 0
// Dumps a best-effort frame-pointer call stack.
static void KiDumpRbpCallStack(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
uintptr_t rbp = f ? (uintptr_t)f->Rbp : 0;
KiFaultPrintf("RBP call stack @ %016llx:\n", (unsigned long long)rbp);
if (!rbp) {
KiFaultPrintf(" <no RBP captured>\n");
return;
}
int kernelStack = KiFaultFrameOnKernelStack(thread, f);
for (uint32_t depth = 0; depth < KE_RBP_CALL_STACK_DEPTH; depth++) {
if (rbp & (sizeof(uint64_t) - 1u)) {
KiFaultPrintf(" stop: unaligned RBP %016llx\n",
(unsigned long long)rbp);
return;
}
if (kernelStack && !KiKernelStackContains(thread, rbp, sizeof(uint64_t) * 2u)) {
KiFaultPrintf(" stop: RBP outside kernel stack %016llx\n",
(unsigned long long)rbp);
return;
}
uint64_t next = 0;
uint64_t ret = 0;
if (!KiUserReadU64(rbp, &next) || !KiUserReadU64(rbp + sizeof(uint64_t), &ret)) {
KiFaultPrintf(" stop: frame unreadable at %016llx\n",
(unsigned long long)rbp);
return;
}
KiFaultPrintf(" #%02u rbp=%016llx ret=",
(unsigned)depth,
(unsigned long long)rbp);
KiPrintAddressValue(proc, (uintptr_t)ret);
KiFaultPrintf(" next=%016llx\n", (unsigned long long)next);
if (!next)
return;
if ((uintptr_t)next <= rbp) {
KiFaultPrintf(" stop: non-forward RBP chain %016llx -> %016llx\n",
(unsigned long long)rbp,
(unsigned long long)next);
return;
}
rbp = (uintptr_t)next;
}
}
#endif
// Dumps a fault frame.
static void KiDumpFaultFrame(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
if (!f) return;
KiFaultPrintf("Regs:\n");
@@ -771,9 +697,6 @@ static void KiDumpFaultFrame(PROCESS* proc, THREAD* thread, const KFAULT_FRAME*
}
KiDumpStack(proc, (uintptr_t)f->Rsp);
#if 0
KiDumpRbpCallStack(proc, thread, f);
#endif
}
static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
@@ -804,7 +727,6 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
(unsigned long long)errCode);
}
// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
int exitCode = (int)exitStatus;
if (proc) (void)KeTerminateProcess(proc, exitCode);
@@ -849,7 +771,7 @@ static _Noreturn void KiPanicKernelFault(PROCESS* proc, THREAD* thread, const KF
thread,
proc ? proc->ProcessId : 0,
proc ? proc->Name : "<none>");
KiDumpFaultFrame(proc, thread, frame);
KiDumpFaultFrame(proc, frame);
if (vector == 14) {
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
if (frame->Rip) MmDebugDescribeAddress((uintptr_t)frame->Rip);
@@ -894,7 +816,7 @@ static _Noreturn void KiTerminateUserFault(PROCESS* proc, THREAD* thread, const
(uintptr_t)frame->Rip,
(uintptr_t)frame->ErrorCode,
(uintptr_t)frame->Cr2);
KiDumpFaultFrame(proc, thread, frame);
KiDumpFaultFrame(proc, frame);
KiDumpModules(proc);
KiFinishFaultedProcess(proc, thread, KeExceptionProcessExitStatus((uint32_t)frame->Vector));
code/kernel/ke_display.c
-36
View File
@@ -837,14 +837,12 @@ static KSTATUS KiDisplayFinishProviderDraw(DISPLAY_DEVICE* device,
return KSUCCESS(drawStatus) ? presentStatus : drawStatus;
}
// Implements the display Dirty All operation.
static void KiDisplayDirtyAll(void) {
gDisplayStats.DirtyAllCount++;
gDirtyAll = true;
gDirtyRectCount = 0;
}
// Implements the display Add Dirty operation.
static void KiDisplayAddDirty(int32_t x, int32_t y, int32_t w, int32_t h) {
if (!KiDisplayHasDevice() || gDirtyAll || w <= 0 || h <= 0)
return;
@@ -872,7 +870,6 @@ static void KiDisplayAddDirty(int32_t x, int32_t y, int32_t w, int32_t h) {
gDirtyRects[gDirtyRectCount++] = rect;
}
// Implements the display Flush Dirty operation.
static KSTATUS KiDisplayFlushDirty(void) {
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Present)
return NTLSTATUS_SUCCESS;
@@ -906,7 +903,6 @@ static KSTATUS KiDisplayFlushDirty(void) {
return s;
}
// Implements the display Hide Cursor For Draw operation.
static void KiDisplayHideCursorForDraw(void) {
if (!gCursorDrawn || !KiDisplayUsesSoftwareCursor())
return;
@@ -915,7 +911,6 @@ static void KiDisplayHideCursorForDraw(void) {
gCursorDrawn = false;
}
// Implements the display Restore Cursor After Draw operation.
static void KiDisplayRestoreCursorAfterDraw(void) {
if (!gCursorVisible || gCursorDrawn || !KiDisplayHasCursorOps())
return;
@@ -927,13 +922,11 @@ static void KiDisplayRestoreCursorAfterDraw(void) {
gCursorDrawn = true;
}
// Implements the display Begin Draw operation.
static void KiDisplayBeginDraw(void) {
if (gDrawDepth++ == 0)
KiDisplayHideCursorForDraw();
}
// Implements the display End Draw operation.
static KSTATUS KiDisplayEndDraw(void) {
KSTATUS s = NTLSTATUS_SUCCESS;
if (gDrawDepth > 0)
@@ -946,7 +939,6 @@ static KSTATUS KiDisplayEndDraw(void) {
return s;
}
// Implements the display Draw Blit No Cursor operation.
static KSTATUS KiDisplayDrawBlitNoCursor(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels) {
if (!pixels || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -962,7 +954,6 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int32_t x, int32_t y, int32_t w, int32_
return s;
}
// Implements the display Fill Rect No Cursor operation.
static KSTATUS KiDisplayFillRectNoCursor(int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) {
if (w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -978,7 +969,6 @@ static KSTATUS KiDisplayFillRectNoCursor(int32_t x, int32_t y, int32_t w, int32_
return s;
}
// Implements the display Copy Rect No Cursor operation.
static KSTATUS KiDisplayCopyRectNoCursor(int32_t dstX, int32_t dstY, int32_t srcX, int32_t srcY, int32_t w, int32_t h) {
if (w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -994,12 +984,10 @@ static KSTATUS KiDisplayCopyRectNoCursor(int32_t dstX, int32_t dstY, int32_t src
return s;
}
// Implements the display Console Index operation.
static size_t KiDisplayConsoleIndex(int32_t col, int32_t row) {
return (size_t)row * DISPLAY_CONSOLE_MAX_COLS + (size_t)col;
}
// Implements the display Console Color operation.
static uint32_t KiDisplayConsoleColor(uint8_t index) {
static const uint32_t colors[16] = {
0x00000000u, 0x000000AAu, 0x0000AA00u, 0x0000AAAAu,
@@ -1010,17 +998,14 @@ static uint32_t KiDisplayConsoleColor(uint8_t index) {
return colors[index & 0x0Fu];
}
// Implements the display Console Fg operation.
static uint32_t KiDisplayConsoleFg(uint16_t attr) {
return KiDisplayConsoleColor((uint8_t)(attr & 0x0Fu));
}
// Implements the display Console Bg operation.
static uint32_t KiDisplayConsoleBg(uint16_t attr) {
return KiDisplayConsoleColor((uint8_t)((attr >> 4) & 0x0Fu));
}
// Implements the display Get Console Grid Locked operation.
static bool KiDisplayGetConsoleGridLocked(int* outCols, int* outRows) {
DISPLAY_MODE mode;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->GetMode)
@@ -1040,7 +1025,6 @@ static bool KiDisplayGetConsoleGridLocked(int* outCols, int* outRows) {
return true;
}
// Implements the display Console Reset Cells Locked operation.
static void KiDisplayConsoleResetCellsLocked(void) {
for (size_t i = 0; i < DISPLAY_CONSOLE_MAX_CELLS; i++) {
gConsoleCells[i] = ' ';
@@ -1049,7 +1033,6 @@ static void KiDisplayConsoleResetCellsLocked(void) {
gConsoleAttribute = DISPLAY_CONSOLE_DEFAULT_ATTR;
}
// Implements the display Console Scroll Cells Locked operation.
static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines) {
if (cols <= 0 || rows <= 0 || cellLines <= 0)
return;
@@ -1083,7 +1066,6 @@ static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines)
}
}
// Implements the display Draw Char Pixels Locked operation.
static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int32_t x, int32_t y, uint32_t fg, uint32_t bg) {
uint32_t glyphPixels[FONT_W * FONT_H];
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u);
@@ -1097,7 +1079,6 @@ static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int32_t x, int32_t y, uint3
return KiDisplayDrawBlitNoCursor(x, y, FONT_W, FONT_H, glyphPixels);
}
// Implements the display Draw Console Cell Locked operation.
static KSTATUS KiDisplayDrawConsoleCellLocked(int32_t col, int32_t row, char c, uint16_t attr, bool updateCell) {
if (updateCell) {
size_t i = KiDisplayConsoleIndex(col, row);
@@ -1555,7 +1536,6 @@ DISPLAY_API KSTATUS KeDisplayUnregisterDevice(DISPLAY_DEVICE* device) {
return NTLSTATUS_SUCCESS;
}
// Implements the display Seed New Primary operation.
static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE* newDevice) {
if (!oldDevice || !newDevice || oldDevice == newDevice)
return NTLSTATUS_SUCCESS;
@@ -1598,7 +1578,6 @@ DISPLAY_API void KeDisplaySuspendConsole(bool suspend) {
NtlAtomicExchange32(&gDisplayConsoleSuspended, suspend ? 1 : 0);
}
// Implements the display Output Suspended operation.
static bool KiDisplayOutputSuspended(void) {
return NtlAtomicLoad32(&gDisplayConsoleSuspended) != 0;
}
@@ -2501,12 +2480,10 @@ DISPLAY_API KSTATUS KeDisplayShowCursor(bool show) {
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Context operation.
static BOOTFB_CONTEXT* BootFbContext(DISPLAY_DEVICE* device) {
return device ? (BOOTFB_CONTEXT*)device->DriverContext : NULL;
}
// Implements the boot Fb Get Mode operation.
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !outMode)
@@ -2622,7 +2599,6 @@ static void BootFbCopyToShadowAndScanout(uint32_t* shadow,
}
}
// Implements the boot framebuffer pitched blit operation.
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels, uint32_t pitchBytes) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
@@ -2702,7 +2678,6 @@ static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Fill Rect operation.
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
@@ -2728,7 +2703,6 @@ static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int3
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Copy Rect operation.
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int32_t dstX, int32_t dstY, int32_t srcX, int32_t srcY, int32_t w, int32_t h) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
@@ -2767,7 +2741,6 @@ static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int32_t dstX, int32_t dstY
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Read Rect operation.
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t* outPixels, uint32_t outStride) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
@@ -2792,7 +2765,6 @@ static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int3
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Present operation.
static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer)
@@ -2827,14 +2799,12 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects,
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Sync operation.
static KSTATUS BootFbSync(DISPLAY_DEVICE* device) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (fb) fb->Stats.SyncCount++;
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Set Cursor Image operation.
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
(void)device;
(void)width;
@@ -2845,7 +2815,6 @@ static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int32_t width, int32
return NTLSTATUS_NOT_SUPPORTED;
}
// Implements the boot Fb Set Cursor Position operation.
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int32_t x, int32_t y) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (fb) fb->Stats.CursorMoveCount++;
@@ -2854,7 +2823,6 @@ static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int32_t x, int32_
return NTLSTATUS_SUCCESS;
}
// Implements the display Blend Pixel operation.
static inline uint32_t KiDisplayBlendPixel(uint32_t dst, uint32_t src) {
uint8_t a = (uint8_t)(src >> 24);
if (a == 0) return dst;
@@ -2873,7 +2841,6 @@ static inline uint32_t KiDisplayBlendPixel(uint32_t dst, uint32_t src) {
return ((uint32_t)rr << 16) | ((uint32_t)gg << 8) | bb;
}
// Implements the boot Fb Restore Cursor operation.
static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
if (!fb || !fb->Framebuffer || !fb->Backbuffer || gCursorDrawX < 0)
return;
@@ -2893,7 +2860,6 @@ static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
gCursorUpdatePending = false;
}
// Implements the boot Fb Draw Cursor operation.
static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int32_t x, int32_t y) {
if (!fb || !fb->Framebuffer || !fb->Backbuffer)
return;
@@ -2918,7 +2884,6 @@ static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int32_t x, int32_t y) {
}
}
// Implements the boot Fb Show Cursor operation.
static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !fb->Backbuffer)
@@ -2935,7 +2900,6 @@ static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show) {
return NTLSTATUS_SUCCESS;
}
// Implements the boot Fb Query Stats operation.
static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !outStats)
code/kernel/ke_dpc.c
-2
View File
@@ -24,7 +24,6 @@ void KeInitializeDpcSystem(void) {
gDpcInitialized = 1;
}
// Implements the ensure Dpc Initialized operation.
static void KiEnsureDpcInitialized(void) {
if (gDpcInitialized)
return;
@@ -69,7 +68,6 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
return true;
}
// Implements the drain Dpc Queue operation.
void KiDrainDpcQueue(void) {
if (!gDpcInitialized)
return;
code/kernel/ke_event.c
-1
View File
@@ -33,7 +33,6 @@ struct KE_EVENT_MULTI_WAIT_CONTEXT {
uint32_t SelectedIndex;
};
// Implements the event Destroy operation.
static void KiEventDestroy(OBJECT* obj) { (void)obj; }
// Completes a multi-wait block while the source event lock is held.
code/kernel/ke_firmware.c
+106
View File
@@ -0,0 +1,106 @@
// Firmware root discovery for optional platform drivers.
#include <kernel/ke_firmware.h>
#include <ntlite/ddk/platform.h>
#include <ntlite/memory_map.h>
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <kernel/ke_debug.h>
#include <string.h>
#define KI_RSDP_V1_SIZE 20u
#define KI_RSDP_V2_SIZE 36u
#define KI_RSDP_MAX_SIZE 4096u
static uint64_t gAcpiRsdpPhysical;
static const char* gAcpiRootDiagnostic = "absent";
static uint8_t gAcpiRsdpRevision;
// Firmware roots must remain in reserved memory after ExitBootServices.
static bool KiFirmwareReservedSpan(const MEMORY_MAP* map, uint64_t start, uint32_t length) {
if (!map || map->Magic != MEMORY_MAP_MAGIC ||
map->RegionCount > MEMORY_MAX_REGIONS || !length || start > UINT64_MAX - length)
return false;
uint64_t end = start + length;
for (uint32_t i = 0; i < map->RegionCount; i++) {
const MEMORY_REGION* region = &map->Regions[i];
if (region->Type != MemoryReserved ||
region->Base > UINT64_MAX - region->Length)
continue;
if (start >= region->Base && end <= region->Base + region->Length)
return true;
}
return false;
}
static bool KiFirmwareChecksumValid(const uint8_t* bytes, uint32_t length) {
uint8_t sum = 0;
for (uint32_t i = 0; i < length; i++) sum = (uint8_t)(sum + bytes[i]);
return sum == 0;
}
void KeFirmwareInitialize(const BOOT_INFO* bootInfo) {
gAcpiRsdpPhysical = 0;
gAcpiRsdpRevision = 0;
gAcpiRootDiagnostic = "absent";
if (!bootInfo || !bootInfo->RsdpPhysical) return;
if (!bootInfo->PhysicalMapBase ||
!KiFirmwareReservedSpan(&bootInfo->MemoryMap,
bootInfo->RsdpPhysical, KI_RSDP_V1_SIZE) ||
bootInfo->RsdpPhysical > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
gAcpiRootDiagnostic = "invalid physical range";
return;
}
const uint8_t* rsdp = (const uint8_t*)(bootInfo->PhysicalMapBase + bootInfo->RsdpPhysical);
if (memcmp(rsdp, "RSD PTR ", 8) != 0) {
gAcpiRootDiagnostic = "invalid signature";
return;
}
if (!KiFirmwareChecksumValid(rsdp, KI_RSDP_V1_SIZE)) {
gAcpiRootDiagnostic = "invalid legacy checksum";
return;
}
if (rsdp[15] >= 2u) {
if (!KiFirmwareReservedSpan(&bootInfo->MemoryMap,
bootInfo->RsdpPhysical, KI_RSDP_V2_SIZE)) {
gAcpiRootDiagnostic = "truncated extended root";
return;
}
uint32_t length;
memcpy(&length, rsdp + 20, sizeof(length));
if (length < KI_RSDP_V2_SIZE || length > KI_RSDP_MAX_SIZE ||
!KiFirmwareReservedSpan(&bootInfo->MemoryMap,
bootInfo->RsdpPhysical, length)) {
gAcpiRootDiagnostic = "invalid extended length";
return;
}
if (!KiFirmwareChecksumValid(rsdp, length)) {
gAcpiRootDiagnostic = "invalid extended checksum";
return;
}
}
gAcpiRsdpPhysical = (uint64_t)bootInfo->RsdpPhysical;
gAcpiRsdpRevision = rsdp[15];
gAcpiRootDiagnostic = "valid";
}
// A driver chooses its own ACPI interpreter and maps tables through the DDK.
__declspec(dllexport) KSTATUS KeFirmwareQueryAcpiRsdp(uint64_t* physicalAddressOut) {
if (!physicalAddressOut) return NTLSTATUS_INVALID_PARAMETER;
*physicalAddressOut = 0;
if (!gAcpiRsdpPhysical) return NTLSTATUS_NOT_SUPPORTED;
*physicalAddressOut = gAcpiRsdpPhysical;
return NTLSTATUS_SUCCESS;
}
// Reports the firmware handoff state through the normal kernel diagnostic stream.
void KeFirmwareLog(void) {
DbgPrint("firmware: ACPI RSDP %s phys=0x%016llX revision=%u\n",
gAcpiRootDiagnostic,
(unsigned long long)gAcpiRsdpPhysical,
(unsigned)gAcpiRsdpRevision);
}
code/kernel/ke_hal.c
+4 -2
View File
@@ -30,8 +30,10 @@ static bool KiHalRuntimeTableValid(const HAL_API* hal) {
hal->Timer.QueryTicks && hal->Timer.QueryTickFrequency && hal->Timer.Stall &&
hal->Timer.HandleInterrupt && hal->Timer.ConfigureCurrentProcessor &&
hal->Rtc.QueryTime && hal->Pci.Read32 && hal->Pci.Write32 &&
hal->InterruptController.RouteIrq && hal->InterruptController.MaskIrq &&
hal->InterruptController.UnmaskIrq && hal->InterruptController.SendEoi &&
hal->InterruptController.RouteGsi && hal->InterruptController.MaskGsi &&
hal->InterruptController.ResolveIsa && hal->InterruptController.SendEoi &&
hal->InterruptController.SetTaskPriority && hal->InterruptController.QueryTaskPriority &&
hal->InterruptController.VectorPending &&
hal->InterruptController.InitializeCurrentProcessor &&
hal->InterruptController.SendProcessorInterrupt &&
hal->InterruptController.SendProcessorNmi && hal->InterruptController.BroadcastNmi &&
code/kernel/ke_idt.c
+309 -14
View File
@@ -12,6 +12,7 @@
#include <string.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_vectors.h>
// External assembly procedures.
extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
@@ -41,7 +42,145 @@ extern void KeIrqStub20(void);
extern void KeIrqStub21(void);
extern void KeIrqStub22(void);
extern void KeIrqStub23(void);
extern void KeIrqStub24(void);
extern void KeIrqStub25(void);
extern void KeIrqStub26(void);
extern void KeIrqStub27(void);
extern void KeIrqStub28(void);
extern void KeIrqStub29(void);
extern void KeIrqStub30(void);
extern void KeIrqStub31(void);
extern void KeIrqStub32(void);
extern void KeIrqStub33(void);
extern void KeIrqStub34(void);
extern void KeIrqStub35(void);
extern void KeIrqStub36(void);
extern void KeIrqStub37(void);
extern void KeIrqStub38(void);
extern void KeIrqStub39(void);
extern void KeIrqStub40(void);
extern void KeIrqStub41(void);
extern void KeIrqStub42(void);
extern void KeIrqStub43(void);
extern void KeIrqStub44(void);
extern void KeIrqStub45(void);
extern void KeIrqStub46(void);
extern void KeIrqStub47(void);
extern void KeIrqStub48(void);
extern void KeIrqStub49(void);
extern void KeIrqStub50(void);
extern void KeIrqStub51(void);
extern void KeIrqStub52(void);
extern void KeIrqStub53(void);
extern void KeIrqStub54(void);
extern void KeIrqStub55(void);
extern void KeIrqStub56(void);
extern void KeIrqStub57(void);
extern void KeIrqStub58(void);
extern void KeIrqStub59(void);
extern void KeIrqStub60(void);
extern void KeIrqStub61(void);
extern void KeIrqStub62(void);
extern void KeIrqStub63(void);
extern void KeIrqStub64(void);
extern void KeIrqStub65(void);
extern void KeIrqStub66(void);
extern void KeIrqStub67(void);
extern void KeIrqStub68(void);
extern void KeIrqStub69(void);
extern void KeIrqStub70(void);
extern void KeIrqStub71(void);
extern void KeIrqStub72(void);
extern void KeIrqStub73(void);
extern void KeIrqStub74(void);
extern void KeIrqStub75(void);
extern void KeIrqStub76(void);
extern void KeIrqStub77(void);
extern void KeIrqStub78(void);
extern void KeIrqStub79(void);
extern void KeIrqStub80(void);
extern void KeIrqStub81(void);
extern void KeIrqStub82(void);
extern void KeIrqStub83(void);
extern void KeIrqStub84(void);
extern void KeIrqStub85(void);
extern void KeIrqStub86(void);
extern void KeIrqStub87(void);
extern void KeIrqStub88(void);
extern void KeIrqStub89(void);
extern void KeIrqStub90(void);
extern void KeIrqStub91(void);
extern void KeIrqStub92(void);
extern void KeIrqStub93(void);
extern void KeIrqStub94(void);
extern void KeIrqStub95(void);
extern void KeIrqStub96(void);
extern void KeIrqStub97(void);
extern void KeIrqStub98(void);
extern void KeIrqStub99(void);
extern void KeIrqStub100(void);
extern void KeIrqStub101(void);
extern void KeIrqStub102(void);
extern void KeIrqStub103(void);
extern void KeIrqStub104(void);
extern void KeIrqStub105(void);
extern void KeIrqStub106(void);
extern void KeIrqStub107(void);
extern void KeIrqStub108(void);
extern void KeIrqStub109(void);
extern void KeIrqStub110(void);
extern void KeIrqStub111(void);
extern void KeIrqStub112(void);
extern void KeIrqStub113(void);
extern void KeIrqStub114(void);
extern void KeIrqStub115(void);
extern void KeIrqStub116(void);
extern void KeIrqStub117(void);
extern void KeIrqStub118(void);
extern void KeIrqStub119(void);
extern void KeIrqStub120(void);
extern void KeIrqStub121(void);
extern void KeIrqStub122(void);
extern void KeIrqStub123(void);
extern void KeIrqStub124(void);
extern void KeIrqStub125(void);
extern void KeIrqStub126(void);
extern void KeIrqStub127(void);
extern void KeIrqStub128(void);
extern void KeIrqStub129(void);
extern void KeIrqStub130(void);
extern void KeIrqStub131(void);
extern void KeIrqStub132(void);
extern void KeIrqStub133(void);
extern void KeIrqStub134(void);
extern void KeIrqStub135(void);
extern void KeIrqStub136(void);
extern void KeIrqStub137(void);
extern void KeIrqStub138(void);
extern void KeIrqStub139(void);
extern void KeIrqStub140(void);
extern void KeIrqStub141(void);
extern void KeIrqStub142(void);
extern void KeIrqStub143(void);
extern void KeIrqStub144(void);
extern void KeIrqStub145(void);
extern void KeIrqStub146(void);
extern void KeIrqStub147(void);
extern void KeIrqStub148(void);
extern void KeIrqStub149(void);
extern void KeIrqStub150(void);
extern void KeIrqStub151(void);
extern void KeIrqStub152(void);
extern void KeIrqStub153(void);
extern void KeIrqStub154(void);
extern void KeIrqStub155(void);
extern void KeIrqStub156(void);
extern void KeIrqStub157(void);
extern void KeIrqStub158(void);
extern void KeIrqStub159(void);
extern void KeRescheduleIpiStub(void);
extern void KeLocalTimerStub(void);
extern void KeSpuriousStub(void);
extern void KeFaultDB(void);
extern void KeFaultNMI(void);
@@ -74,7 +213,7 @@ extern void KeFaultVC(void);
extern void KeFaultSX(void);
extern void KeFaultRSV31(void);
#define NUM_IRQS 24
#define NUM_DEVICE_VECTORS (KE_VECTOR_DEVICE_LAST - KE_VECTOR_DEVICE_FIRST + 1u)
// GDT.
static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
@@ -88,7 +227,6 @@ static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit,
((uint64_t)(base >> 24 & 0xFF)<< 56);
}
// Implements the gdt Set Tss operation.
static void KiGdtSetTss(uint64_t* gdt, int idx, uintptr_t base, uint32_t limit) {
// TSS descriptor is 16 bytes (two GDT slots).
gdt[idx] = 0;
@@ -219,23 +357,180 @@ void KeInitializeIdt(void) {
// User-mode INT3 must enter #BP, not fail the gate-DPL check and become #GP.
KiIdtSetGate(3, (uintptr_t)KeFaultBP, KI_IDT_TYPE_USER_INTERRUPT_GATE, 0);
KiIdtSetGate(0x40, (uintptr_t)KeSyscallEntry, KI_IDT_TYPE_USER_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_SYSCALL, (uintptr_t)KeSyscallEntry, KI_IDT_TYPE_USER_INTERRUPT_GATE, 0);
static void* irqStubs[NUM_IRQS] = {
(void*)KeIrqStub0, (void*)KeIrqStub1, (void*)KeIrqStub2,
(void*)KeIrqStub3, (void*)KeIrqStub4, (void*)KeIrqStub5,
(void*)KeIrqStub6, (void*)KeIrqStub7, (void*)KeIrqStub8,
(void*)KeIrqStub9, (void*)KeIrqStub10, (void*)KeIrqStub11,
(void*)KeIrqStub12, (void*)KeIrqStub13, (void*)KeIrqStub14,
(void*)KeIrqStub15, (void*)KeIrqStub16, (void*)KeIrqStub17,
(void*)KeIrqStub18, (void*)KeIrqStub19, (void*)KeIrqStub20,
(void*)KeIrqStub21, (void*)KeIrqStub22, (void*)KeIrqStub23,
static void* irqStubs[NUM_DEVICE_VECTORS] = {
(void*)KeIrqStub0,
(void*)KeIrqStub1,
(void*)KeIrqStub2,
(void*)KeIrqStub3,
(void*)KeIrqStub4,
(void*)KeIrqStub5,
(void*)KeIrqStub6,
(void*)KeIrqStub7,
(void*)KeIrqStub8,
(void*)KeIrqStub9,
(void*)KeIrqStub10,
(void*)KeIrqStub11,
(void*)KeIrqStub12,
(void*)KeIrqStub13,
(void*)KeIrqStub14,
(void*)KeIrqStub15,
(void*)KeIrqStub16,
(void*)KeIrqStub17,
(void*)KeIrqStub18,
(void*)KeIrqStub19,
(void*)KeIrqStub20,
(void*)KeIrqStub21,
(void*)KeIrqStub22,
(void*)KeIrqStub23,
(void*)KeIrqStub24,
(void*)KeIrqStub25,
(void*)KeIrqStub26,
(void*)KeIrqStub27,
(void*)KeIrqStub28,
(void*)KeIrqStub29,
(void*)KeIrqStub30,
(void*)KeIrqStub31,
(void*)KeIrqStub32,
(void*)KeIrqStub33,
(void*)KeIrqStub34,
(void*)KeIrqStub35,
(void*)KeIrqStub36,
(void*)KeIrqStub37,
(void*)KeIrqStub38,
(void*)KeIrqStub39,
(void*)KeIrqStub40,
(void*)KeIrqStub41,
(void*)KeIrqStub42,
(void*)KeIrqStub43,
(void*)KeIrqStub44,
(void*)KeIrqStub45,
(void*)KeIrqStub46,
(void*)KeIrqStub47,
(void*)KeIrqStub48,
(void*)KeIrqStub49,
(void*)KeIrqStub50,
(void*)KeIrqStub51,
(void*)KeIrqStub52,
(void*)KeIrqStub53,
(void*)KeIrqStub54,
(void*)KeIrqStub55,
(void*)KeIrqStub56,
(void*)KeIrqStub57,
(void*)KeIrqStub58,
(void*)KeIrqStub59,
(void*)KeIrqStub60,
(void*)KeIrqStub61,
(void*)KeIrqStub62,
(void*)KeIrqStub63,
(void*)KeIrqStub64,
(void*)KeIrqStub65,
(void*)KeIrqStub66,
(void*)KeIrqStub67,
(void*)KeIrqStub68,
(void*)KeIrqStub69,
(void*)KeIrqStub70,
(void*)KeIrqStub71,
(void*)KeIrqStub72,
(void*)KeIrqStub73,
(void*)KeIrqStub74,
(void*)KeIrqStub75,
(void*)KeIrqStub76,
(void*)KeIrqStub77,
(void*)KeIrqStub78,
(void*)KeIrqStub79,
(void*)KeIrqStub80,
(void*)KeIrqStub81,
(void*)KeIrqStub82,
(void*)KeIrqStub83,
(void*)KeIrqStub84,
(void*)KeIrqStub85,
(void*)KeIrqStub86,
(void*)KeIrqStub87,
(void*)KeIrqStub88,
(void*)KeIrqStub89,
(void*)KeIrqStub90,
(void*)KeIrqStub91,
(void*)KeIrqStub92,
(void*)KeIrqStub93,
(void*)KeIrqStub94,
(void*)KeIrqStub95,
(void*)KeIrqStub96,
(void*)KeIrqStub97,
(void*)KeIrqStub98,
(void*)KeIrqStub99,
(void*)KeIrqStub100,
(void*)KeIrqStub101,
(void*)KeIrqStub102,
(void*)KeIrqStub103,
(void*)KeIrqStub104,
(void*)KeIrqStub105,
(void*)KeIrqStub106,
(void*)KeIrqStub107,
(void*)KeIrqStub108,
(void*)KeIrqStub109,
(void*)KeIrqStub110,
(void*)KeIrqStub111,
(void*)KeIrqStub112,
(void*)KeIrqStub113,
(void*)KeIrqStub114,
(void*)KeIrqStub115,
(void*)KeIrqStub116,
(void*)KeIrqStub117,
(void*)KeIrqStub118,
(void*)KeIrqStub119,
(void*)KeIrqStub120,
(void*)KeIrqStub121,
(void*)KeIrqStub122,
(void*)KeIrqStub123,
(void*)KeIrqStub124,
(void*)KeIrqStub125,
(void*)KeIrqStub126,
(void*)KeIrqStub127,
(void*)KeIrqStub128,
(void*)KeIrqStub129,
(void*)KeIrqStub130,
(void*)KeIrqStub131,
(void*)KeIrqStub132,
(void*)KeIrqStub133,
(void*)KeIrqStub134,
(void*)KeIrqStub135,
(void*)KeIrqStub136,
(void*)KeIrqStub137,
(void*)KeIrqStub138,
(void*)KeIrqStub139,
(void*)KeIrqStub140,
(void*)KeIrqStub141,
(void*)KeIrqStub142,
(void*)KeIrqStub143,
(void*)KeIrqStub144,
(void*)KeIrqStub145,
(void*)KeIrqStub146,
(void*)KeIrqStub147,
(void*)KeIrqStub148,
(void*)KeIrqStub149,
(void*)KeIrqStub150,
(void*)KeIrqStub151,
(void*)KeIrqStub152,
(void*)KeIrqStub153,
(void*)KeIrqStub154,
(void*)KeIrqStub155,
(void*)KeIrqStub156,
(void*)KeIrqStub157,
(void*)KeIrqStub158,
(void*)KeIrqStub159,
};
for (int i = 0; i < NUM_IRQS; i++)
KiIdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], KI_IDT_TYPE_INTERRUPT_GATE, 0);
for (uint32_t i = 0; i < NUM_DEVICE_VECTORS; i++)
KiIdtSetGate(KE_VECTOR_DEVICE_FIRST + i, (uintptr_t)irqStubs[i],
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_CPU_TIMER, (uintptr_t)KeLocalTimerStub,
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_RESCHEDULE_IPI, (uintptr_t)KeRescheduleIpiStub,
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KiIdtSetGate(KE_VECTOR_SPURIOUS, (uintptr_t)KeSpuriousStub,
KI_IDT_TYPE_INTERRUPT_GATE, 0);
KeLoadIdtForCurrentCpu();
}
code/kernel/ke_input.c
-9
View File
@@ -41,7 +41,6 @@ static void KiInputQueueManagerEvent(KE_INPUT_CONTEXT* context, const KE_INPUT_M
KiInputRefreshManagerEvent(context);
}
// Implements the input Initialize operation.
void KeInputInitialize(const HAL_API* hal) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
ObInitializeObjectHeader(&gInputManager.Header, ObjectTypeInputManager);
@@ -114,7 +113,6 @@ KSTATUS KeInputManagerReadEvent(KE_INPUT_MANAGER* manager, uint64_t timeoutMs, S
}
}
// Implements the input Register Device operation.
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) {
(void)flags;
uint32_t id = 0;
@@ -125,7 +123,6 @@ __declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t f
return NTLSTATUS_SUCCESS;
}
// Implements the input Submit Key operation.
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
@@ -139,7 +136,6 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
KiInputQueueManagerEvent(context, &managerEvent);
}
// Implements the input Commit Mouse Position operation.
static void KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int32_t nx, int32_t ny) {
int32_t bw = context->MouseBoundsWidth;
int32_t bh = context->MouseBoundsHeight;
@@ -155,7 +151,6 @@ static void KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int32_t nx, in
context->MouseY = ny;
}
// Implements the input Submit Mouse operation.
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
@@ -174,7 +169,6 @@ __declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
KiInputCommitMousePosition(context, nx, ny);
}
// Implements the input Submit Mouse Absolute operation.
__declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
@@ -196,7 +190,6 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
KiInputCommitMousePosition(context, event->x, event->y);
}
// Implements the input Submit Mouse Absolute Scaled operation.
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int32_t wheel, uint8_t buttons) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
int32_t bw = context->MouseBoundsWidth;
@@ -212,7 +205,6 @@ __declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint3
KeInputSubmitMouseAbsolute(&ev);
}
// Implements the input Mouse Set Position operation.
void KeInputMouseSetPosition(int32_t x, int32_t y) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
int32_t bw = context->MouseBoundsWidth;
@@ -227,7 +219,6 @@ void KeInputMouseSetPosition(int32_t x, int32_t y) {
context->MouseY = y;
}
// Implements the input Mouse Set Bounds operation.
void KeInputMouseSetBounds(int32_t w, int32_t h) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (w < 1) w = 1;
code/kernel/ke_irp.c
-5
View File
@@ -21,7 +21,6 @@ __declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
LIST_INIT(&q->Head);
}
// Implements the irp Queue Enqueue operation.
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
if (irp->Flags & IRP_FLAG_PRIORITY)
@@ -32,7 +31,6 @@ __declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
KeSignalSemaphore((SEMAPHORE*)q->Semaphore);
}
// Implements the irp Queue Dequeue operation.
__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
LIST_ENTRY* node = ListPopHead(&q->Head);
@@ -62,7 +60,6 @@ __declspec(dllexport) void IrpQueueCancelAllWithCallback(IRP_QUEUE* q,
}
}
// Implements the irp Queue Cancel All operation.
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
IrpQueueCancelAllWithCallback(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
}
@@ -88,7 +85,6 @@ __declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
}
}
// Implements the acquire Remove Lock operation.
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return NTLSTATUS_INVALID_PARAMETER;
@@ -179,7 +175,6 @@ __declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) {
return &irp->Stack[irp->CurrentLocation + 1u];
}
// Implements the skip Current Irp Stack Location operation.
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) {
if (!irp) return;
if (irp->CurrentLocation + 1u < irp->StackCount)
code/kernel/ke_irq.c
+599 -98
View File
@@ -1,136 +1,643 @@
// Interrupt manager. This file owns IRQ routing, handler dispatch, and post-interrupt scheduling hooks.
// Interrupt manager. Per-CPU vector ownership is independent of physical sources.
#include <kernel/ke_irq.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_vectors.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <intrin.h>
#define MAX_IRQS 24
#define MAX_IRQ_HANDLERS 8
#define MSI_IRQ_FIRST 16u
#define MSI_IRQ_LAST 23u
#define KI_INTERRUPT_ROUTES 256u
#define KI_INTERRUPT_HANDLERS 512u
typedef struct IRQ_HANDLER_LIST {
void (*Handlers[MAX_IRQ_HANDLERS])(void);
uint8_t Count;
} IRQ_HANDLER_LIST;
typedef struct KI_INTERRUPT_ROUTE KI_INTERRUPT_ROUTE;
struct KE_INTERRUPT {
KE_INTERRUPT_ROUTINE Routine;
void* Context;
KI_INTERRUPT_ROUTE* Route;
KE_INTERRUPT_MESSAGE_PROGRAM Program;
void* ProgramContext;
uint64_t Affinity;
struct KE_INTERRUPT* Next;
bool Used;
};
static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
static bool gIrqSoftwareVector[MAX_IRQS];
struct KI_INTERRUPT_ROUTE {
KE_INTERRUPT* Handlers;
uint32_t Gsi;
uint32_t Cpu;
uint8_t Vector;
uint16_t Flags;
bool Used;
bool Message;
bool Draining;
bool Closing;
bool MessageActive;
volatile long Active;
volatile long Enabled;
volatile uint64_t Delivered;
};
static KI_INTERRUPT_ROUTE gRoutes[KI_INTERRUPT_ROUTES];
static KE_INTERRUPT gHandlers[KI_INTERRUPT_HANDLERS];
static KI_INTERRUPT_ROUTE* volatile gCpuVectors[KE_MAX_CPUS][256];
// A reader is registered before it resolves the route pointer. A retired route
// waits for all readers on its former CPU/vector before releasing handler state.
static volatile long gCpuVectorReaders[KE_MAX_CPUS][256];
// Vector reservations outlive publication while a route is draining.
static bool gCpuVectorReserved[KE_MAX_CPUS][256];
static bool gCpuVectorRetired[KE_MAX_CPUS][256];
static uint8_t gCpuVectorQuietTicks[KE_MAX_CPUS][256];
static SPINLOCK gInterruptLock = SPINLOCK_INIT;
static const HAL_API* gHal;
static void (*gLocalTimerHandlers[2])(void);
static uint8_t gLocalTimerHandlerCount;
static uint32_t gPlacementCursor;
// Implements the irq To Irql operation.
static KIRQL KiIrqToIrql(uint8_t irq) {
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)irq;
if (level >= HIGH_LEVEL)
level = (uint32_t)(HIGH_LEVEL - 1);
static KIRQL KiVectorIrql(uint8_t vector) {
uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)((vector >> 4u) - 3u);
if (level >= HIGH_LEVEL) level = HIGH_LEVEL - 1u;
return (KIRQL)level;
}
// Implements the irq Initialize operation.
void KeIrqInitialize(const HAL_API* hal) {
if (!hal || !hal->InterruptController.RouteIrq || !hal->InterruptController.SendEoi)
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid interrupt-controller API during initialization\n");
if (!hal || !hal->InterruptController.SendEoi ||
!hal->InterruptController.RouteGsi || !hal->InterruptController.ResolveIsa ||
!hal->InterruptController.ComposeMsi)
KePanic(KE_PANIC_CODE_INIT, "IRQ: incomplete interrupt-controller API\n");
gHal = hal;
}
// Implements the irq Allocate Msi Vector operation.
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
if (!outIrq) return NTLSTATUS_INVALID_PARAMETER;
for (uint8_t irq = MSI_IRQ_FIRST; irq <= MSI_IRQ_LAST; irq++) {
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
if (!gIrqSoftwareVector[irq] && list->Count == 0) {
gIrqSoftwareVector[irq] = true;
*outIrq = irq;
DbgPrint("KeIrq: allocated MSI vector slot IRQ %u vector 0x%02x\n",
irq, 0x20u + irq);
return NTLSTATUS_SUCCESS;
}
}
return NTLSTATUS_NO_MEMORY;
static bool KiCpuAvailable(uint32_t index, uint64_t affinity) {
if (index >= gCpuCount || index >= KE_MAX_CPUS ||
(affinity && !(affinity & (1ull << index)))) return false;
KCPU* cpu = KeGetCpuByIndex(index);
return cpu && (cpu->IsBootProcessor || cpu->State >= KeCpuSchedulerActive) &&
cpu->HardwareId <= 255u;
}
// Implements the irq Register operation.
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
if (!gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: registration before interrupt-manager initialization\n");
static bool KiPickVector(uint64_t affinity, uint32_t* cpuOut, uint8_t* vectorOut) {
uint32_t count = gCpuCount;
if (!count || count > KE_MAX_CPUS) count = 1;
// Rotate the first candidate CPU, and allocate the least occupied vector.
uint32_t start = gPlacementCursor++ % count;
uint32_t bestLoad = UINT32_MAX;
bool found = false;
for (uint32_t i = 0; i < count; i++) {
uint32_t cpu = (start + i) % count;
if (!KiCpuAvailable(cpu, affinity)) continue;
uint32_t occupied = 0;
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++)
if (gCpuVectorReserved[cpu][v]) occupied++;
if (occupied >= KE_VECTOR_DEVICE_LAST - KE_VECTOR_DEVICE_FIRST + 1u || occupied >= bestLoad)
continue;
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
if (!gCpuVectorReserved[cpu][v]) {
bestLoad = occupied;
*cpuOut = cpu;
*vectorOut = (uint8_t)v;
found = true;
break;
}
}
}
return found;
}
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
for (uint8_t i = 0; i < list->Count; i++) {
if (list->Handlers[i] == handler)
return;
static KSTATUS KiReserveInterrupt(bool message, uint32_t gsi, uint16_t flags,
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context,
KE_INTERRUPT** out) {
if (!out || !routine || !gHal) return NTLSTATUS_INVALID_PARAMETER;
*out = (KE_INTERRUPT*)0;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)0;
if (!message) {
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
if (gRoutes[i].Used && !gRoutes[i].Message && gRoutes[i].Gsi == gsi) {
if (gRoutes[i].Draining || gRoutes[i].Closing) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
if (gRoutes[i].Flags != flags ||
(affinity && !(affinity & (1ull << gRoutes[i].Cpu)))) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
route = &gRoutes[i];
break;
}
}
}
bool newRoute = route == 0;
uint32_t cpu = 0;
uint8_t vector = 0;
if (newRoute) {
if (!KiPickVector(affinity, &cpu, &vector)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++)
if (!gRoutes[i].Used) { route = &gRoutes[i]; break; }
if (!route) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
}
KE_INTERRUPT* handler = 0;
for (uint32_t i = 0; i < KI_INTERRUPT_HANDLERS; i++)
if (!gHandlers[i].Used) { handler = &gHandlers[i]; break; }
if (!handler) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
if (newRoute) {
route->Used = true;
route->Message = message;
route->Draining = false;
route->Closing = false;
route->MessageActive = false;
route->Gsi = gsi;
route->Cpu = cpu;
route->Vector = vector;
route->Flags = flags;
route->Handlers = 0;
route->Active = 0;
route->Enabled = 0;
route->Delivered = 0;
gCpuVectorReserved[cpu][vector] = true;
gCpuVectorRetired[cpu][vector] = false;
gCpuVectorQuietTicks[cpu][vector] = 0;
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
}
handler->Used = true;
handler->Routine = routine;
handler->Context = context;
handler->Route = route;
handler->Program = 0;
handler->ProgramContext = 0;
handler->Affinity = affinity;
handler->Next = route->Handlers;
route->Handlers = handler;
KeReleaseSpinlock(&gInterruptLock, cookie);
if (newRoute && !message) {
KCPU* target = KeGetCpuByIndex(cpu);
KSTATUS st = gHal->InterruptController.RouteGsi(gsi, vector,
target->HardwareId, flags, true);
if (!KSUCCESS(st)) {
KeInterruptDisconnect(handler);
return st;
}
_InterlockedExchange(&route->Enabled, 1);
(void)gHal->InterruptController.MaskGsi(gsi, false);
} else if (newRoute) {
_InterlockedExchange(&route->Enabled, 1);
}
*out = handler;
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeInterruptConnectGsi(uint32_t gsi, uint16_t flags,
uint64_t affinity, KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
}
__declspec(dllexport) KSTATUS KeInterruptConnectIsa(uint8_t isaIrq, uint16_t sourceFlags, uint64_t affinity,
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
uint32_t gsi;
uint16_t flags;
if (!gHal || !gHal->InterruptController.ResolveIsa)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS st = gHal->InterruptController.ResolveIsa(isaIrq, &gsi, &flags);
if (!KSUCCESS(st)) return st;
if (sourceFlags) flags = sourceFlags;
return KiReserveInterrupt(false, gsi, flags, affinity, routine, context, out);
}
__declspec(dllexport) KSTATUS KeInterruptReserveMessage(uint64_t affinity,
KE_INTERRUPT_ROUTINE routine, void* context, KE_INTERRUPT** out) {
return KiReserveInterrupt(true, UINT32_MAX, 0, affinity, routine, context, out);
}
__declspec(dllexport) KSTATUS KeInterruptComposeMessage(const KE_INTERRUPT* interrupt,
NTL_DDK_MSI_MESSAGE* out) {
if (!interrupt || !out || !gHal) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route ||
!interrupt->Route->Message || interrupt->Route->Draining) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
HAL_MSI_MESSAGE message;
KSTATUS st = cpu ? gHal->InterruptController.ComposeMsi(
cpu->HardwareId, route->Vector, &message) : NTLSTATUS_INVALID_PARAMETER;
KeReleaseSpinlock(&gInterruptLock, cookie);
if (!KSUCCESS(st)) return st;
out->Address = message.Address;
out->Data = message.Data;
out->Reserved = 0;
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeInterruptQuery(const KE_INTERRUPT* interrupt,
KE_INTERRUPT_INFO* out) {
if (!interrupt || !out) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route || interrupt->Route->Draining) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
const KI_INTERRUPT_ROUTE* route = interrupt->Route;
const KCPU* cpu = KeGetCpuByIndex(route->Cpu);
out->CpuIndex = route->Cpu;
out->HardwareId = cpu ? cpu->HardwareId : UINT32_MAX;
out->Gsi = route->Gsi;
out->Flags = route->Flags;
out->Vector = route->Vector;
out->IsMessage = route->Message ? 1u : 0u;
out->Delivered = route->Delivered;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
// A message device registers its update operation once PCI has enabled the source.
__declspec(dllexport) KSTATUS KeInterruptBindMessageSource(KE_INTERRUPT* interrupt,
KE_INTERRUPT_MESSAGE_PROGRAM program, void* context) {
if (!interrupt || !program) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
(interrupt->Route->Draining || interrupt->Route->Closing) || interrupt->Program) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
interrupt->Program = program;
interrupt->ProgramContext = context;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
// The bus owner enables migration only after the device source is programmed.
__declspec(dllexport) KSTATUS KeInterruptActivateMessage(KE_INTERRUPT* interrupt) {
if (!interrupt) return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route || !interrupt->Route->Message ||
(interrupt->Route->Draining || interrupt->Route->Closing) || !interrupt->Program) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
interrupt->Route->MessageActive = true;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
// Moves the complete physical source to one new CPU/vector, including PCI
// message programming. The old vector remains quarantined through APIC drain.
__declspec(dllexport) KSTATUS KeInterruptSetAffinity(KE_INTERRUPT* interrupt,
uint64_t permittedCpuMask) {
if (!interrupt || !permittedCpuMask || !gHal ||
KeGetCurrentIrql() != PASSIVE_LEVEL || KeGetCurrentCpu()->InterruptDepth)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route ||
interrupt->Route->Draining || interrupt->Route->Closing) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_INVALID_PARAMETER;
}
KI_INTERRUPT_ROUTE* route = interrupt->Route;
if (route->Message && !route->MessageActive) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
// One shared line must satisfy every handler's affinity constraint.
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next) {
if (h->Affinity && !(h->Affinity & permittedCpuMask)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
if (route->Message && (!h->Program || h != interrupt)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
}
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
if (h->Affinity) permittedCpuMask &= h->Affinity;
if (!permittedCpuMask) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
if (permittedCpuMask & (1ull << route->Cpu)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
}
uint32_t newCpu = 0;
uint8_t newVector = 0;
if (!KiPickVector(permittedCpuMask, &newCpu, &newVector)) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NO_MEMORY;
}
const uint32_t oldCpu = route->Cpu;
const uint8_t oldVector = route->Vector;
gCpuVectorReserved[newCpu][newVector] = true;
route->Draining = true;
KeReleaseSpinlock(&gInterruptLock, cookie);
NTL_DDK_MSI_MESSAGE updated = {0};
if (route->Message) {
const KCPU* destination = KeGetCpuByIndex(newCpu);
HAL_MSI_MESSAGE message;
KSTATUS st = gHal->InterruptController.ComposeMsi(destination->HardwareId,
newVector, &message);
if (!KSUCCESS(st)) goto rollback_reservation;
updated.Address = message.Address;
updated.Data = message.Data;
st = interrupt->Program(interrupt->ProgramContext, KeInterruptMessageQuiesce, 0);
if (!KSUCCESS(st)) goto rollback_reservation;
} else {
KSTATUS st = gHal->InterruptController.MaskGsi(route->Gsi, true);
if (!KSUCCESS(st)) goto rollback_reservation;
}
if (list->Count >= MAX_IRQ_HANDLERS) {
DbgPrint("KeIrqRegister: IRQ %u handler table full\n", irq);
_InterlockedExchange(&route->Enabled, 0);
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], 0);
uint32_t spins = 0;
while (route->Active || gCpuVectorReaders[oldCpu][oldVector]) {
if (++spins == 100000000u)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: affinity migration drain timeout\n");
_mm_pause();
}
KSTATUS update = NTLSTATUS_SUCCESS;
if (route->Message)
update = interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageProgram, &updated);
else {
const KCPU* destination = KeGetCpuByIndex(newCpu);
update = gHal->InterruptController.RouteGsi(route->Gsi, newVector,
destination->HardwareId, route->Flags, true);
}
if (!KSUCCESS(update)) {
// Restore the existing route and re-enable its original source.
if (route->Message) {
const KCPU* previous = KeGetCpuByIndex(oldCpu);
HAL_MSI_MESSAGE msg;
if (!KSUCCESS(gHal->InterruptController.ComposeMsi(previous->HardwareId,
oldVector, &msg)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original MSI route\n");
NTL_DDK_MSI_MESSAGE original = {msg.Address, msg.Data, 0};
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageProgram, &original)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore MSI message\n");
} else {
const KCPU* previous = KeGetCpuByIndex(oldCpu);
if (!KSUCCESS(gHal->InterruptController.RouteGsi(route->Gsi, oldVector,
previous->HardwareId, route->Flags, true)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: cannot restore original GSI route\n");
}
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[oldCpu][oldVector], route);
_InterlockedExchange(&route->Enabled, 1);
if (route->Message) {
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageResume, 0)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI restore resume failed\n");
} else (void)gHal->InterruptController.MaskGsi(route->Gsi, false);
cookie = KeAcquireSpinlock(&gInterruptLock);
route->Draining = false;
gCpuVectorReserved[newCpu][newVector] = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
return update;
}
cookie = KeAcquireSpinlock(&gInterruptLock);
route->Cpu = newCpu;
route->Vector = newVector;
gCpuVectorRetired[oldCpu][oldVector] = true;
gCpuVectorQuietTicks[oldCpu][oldVector] = 0;
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[newCpu][newVector], route);
_InterlockedExchange(&route->Enabled, 1);
KeReleaseSpinlock(&gInterruptLock, cookie);
if (route->Message) {
if (!KSUCCESS(interrupt->Program(interrupt->ProgramContext,
KeInterruptMessageResume, 0)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: MSI migration resume failed\n");
} else if (!KSUCCESS(gHal->InterruptController.MaskGsi(route->Gsi, false)))
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: GSI migration unmask failed\n");
cookie = KeAcquireSpinlock(&gInterruptLock);
route->Draining = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_SUCCESS;
rollback_reservation:
cookie = KeAcquireSpinlock(&gInterruptLock);
gCpuVectorReserved[newCpu][newVector] = false;
route->Draining = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
return NTLSTATUS_NOT_SUPPORTED;
}
// The scheduler must evacuate every device route before stopping a processor.
KSTATUS KiInterruptPrepareCpuOffline(uint32_t cpuIndex) {
if (cpuIndex >= gCpuCount || cpuIndex >= KE_MAX_CPUS ||
KeGetCurrentIrql() != PASSIVE_LEVEL) return NTLSTATUS_INVALID_PARAMETER;
uint64_t allowed = 0;
for (uint32_t i = 0; i < gCpuCount && i < 64u; i++)
if (i != cpuIndex && KiCpuAvailable(i, 0)) allowed |= 1ull << i;
if (!allowed) return NTLSTATUS_NOT_SUPPORTED;
for (uint32_t i = 0; i < KI_INTERRUPT_ROUTES; i++) {
KI_INTERRUPT_ROUTE* route = &gRoutes[i];
if (route->Used && route->Cpu == cpuIndex && route->Handlers) {
KSTATUS st = KeInterruptSetAffinity(route->Handlers, allowed);
if (!KSUCCESS(st)) return st;
}
}
return NTLSTATUS_SUCCESS;
}
// Serializes device-source disable with affinity migration. A caller must
// close the route before masking the PCI source, preventing a racing resume.
__declspec(dllexport) void KeInterruptBeginDisconnect(KE_INTERRUPT* interrupt) {
if (!interrupt || KeGetCurrentIrql() != PASSIVE_LEVEL ||
KeGetCurrentCpu()->InterruptDepth)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: invalid disconnect preparation context\n");
for (uint32_t spins = 0;; spins++) {
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return;
}
KI_INTERRUPT_ROUTE* route = interrupt->Route;
if (!route->Draining && !route->Closing) {
route->Closing = true;
KeReleaseSpinlock(&gInterruptLock, cookie);
return;
}
KeReleaseSpinlock(&gInterruptLock, cookie);
if (spins >= 100000000u)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect serialization timed out\n");
_mm_pause();
}
}
// Disconnect runs in thread context after the device has stopped producing messages.
// The route is unpublished before draining readers, so a concurrent dispatcher
// either holds a counted reference or observes an empty vector.
__declspec(dllexport) void KeInterruptDisconnect(KE_INTERRUPT* interrupt) {
if (!interrupt) return;
if (KeGetCurrentCpu()->InterruptDepth || KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: disconnect requires PASSIVE_LEVEL\n");
// PCI callers have already serialized source disable. A direct GSI
// caller prepares the route here before masking it inside this routine.
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
if (!interrupt->Used || !interrupt->Route) {
KeReleaseSpinlock(&gInterruptLock, cookie);
return;
}
KI_INTERRUPT_ROUTE* route = interrupt->Route;
if (!route->Closing) {
KeReleaseSpinlock(&gInterruptLock, cookie);
KeInterruptBeginDisconnect(interrupt);
cookie = KeAcquireSpinlock(&gInterruptLock);
route = interrupt->Route;
if (!route) { KeReleaseSpinlock(&gInterruptLock, cookie); return; }
}
if (route->Draining) {
KeReleaseSpinlock(&gInterruptLock, cookie);
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: unexpected concurrent route drain\n");
}
route->Draining = true;
_InterlockedExchange(&route->Enabled, 0);
if (!route->Message)
(void)gHal->InterruptController.MaskGsi(route->Gsi, true);
const uint32_t cpu = route->Cpu;
const uint8_t vector = route->Vector;
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], 0);
KeReleaseSpinlock(&gInterruptLock, cookie);
list->Handlers[list->Count++] = handler;
if (!gIrqSoftwareVector[irq])
gHal->InterruptController.RouteIrq(irq);
}
// Implements the irq Dispatch operation.
void KeIrqDispatch(uint8_t irq) {
if (!gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: dispatch before interrupt-manager initialization, irq=%u\n", irq);
KIRQL targetIrql = KiIrqToIrql(irq);
KIRQL currentIrql = KeGetCurrentIrql();
// Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected
// while IF is clear, but preserving monotonic IRQL keeps nesting safe.
if (targetIrql < currentIrql)
targetIrql = currentIrql;
KIRQL oldIrql = KeRaiseIrql(targetIrql);
if (irq == KE_VECTOR_RESCHEDULE_IPI) {
// Nothing to do here. The sender has already set this CPU's
// ReschedulePending flag. KiExitInterrupt consumes it after EOI and
// after the interrupted frame unwinds to PASSIVE_LEVEL.
} else if (irq < MAX_IRQS) {
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
for (uint8_t i = 0; i < list->Count; i++) {
if (list->Handlers[i])
list->Handlers[i]();
}
// Drain both ISRs already invoking callbacks and readers that fetched the
// route just before the unpublish. Reserved vectors remain unavailable.
uint32_t spins = 0;
while (route->Active || gCpuVectorReaders[cpu][vector]) {
if (++spins == 100000000u)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: route rundown timed out cpu=%u vector=%02x\n",
cpu, vector);
_mm_pause();
}
// Handlers are no longer allowed to context-switch directly. Timer IRQs
// only request rescheduling; KiExitInterrupt performs the switch after this
// function returns, so the EOI is guaranteed to be sent first. Keep the
// software level at DIRQL until after the controller has been acknowledged.
gHal->InterruptController.SendEoi();
cookie = KeAcquireSpinlock(&gInterruptLock);
KE_INTERRUPT** link = &route->Handlers;
while (*link && *link != interrupt) link = &(*link)->Next;
if (*link != interrupt)
KePanic(KE_PANIC_CODE_DRIVER, "IRQ: missing handler during disconnect\n");
*link = interrupt->Next;
interrupt->Used = false;
interrupt->Route = 0;
interrupt->Routine = 0;
interrupt->Context = 0;
interrupt->Program = 0;
interrupt->ProgramContext = 0;
interrupt->Affinity = 0;
interrupt->Next = 0;
if (!route->Handlers) {
route->Used = false;
// Retain the vector while the target APIC drains any pending delivery.
// Only its target CPU can inspect its IRR and ISR registers.
gCpuVectorRetired[cpu][vector] = true;
gCpuVectorQuietTicks[cpu][vector] = 0;
} else {
_InterlockedExchangePointer((void* volatile*)&gCpuVectors[cpu][vector], route);
_InterlockedExchange(&route->Enabled, 1);
if (!route->Message)
(void)gHal->InterruptController.MaskGsi(route->Gsi, false);
}
route->Draining = false;
route->Closing = false;
KeReleaseSpinlock(&gInterruptLock, cookie);
}
// The local timer gives a retiring vector a grace period on its owning CPU.
// Source drivers must disable message generation before disconnecting. A
// vector is reusable after consecutive timer checks find no APIC delivery.
static void KiReapRetiredVectors(void) {
const uint32_t cpu = KeGetCurrentCpuIndex();
if (cpu >= KE_MAX_CPUS) return;
uintptr_t cookie = KeAcquireSpinlock(&gInterruptLock);
for (uint32_t v = KE_VECTOR_DEVICE_FIRST; v <= KE_VECTOR_DEVICE_LAST; v++) {
if (!gCpuVectorRetired[cpu][v]) continue;
if (gCpuVectorReaders[cpu][v] ||
gHal->InterruptController.VectorPending((uint8_t)v)) {
gCpuVectorQuietTicks[cpu][v] = 0;
} else if (++gCpuVectorQuietTicks[cpu][v] >= 3u) {
gCpuVectorRetired[cpu][v] = false;
gCpuVectorReserved[cpu][v] = false;
gCpuVectorQuietTicks[cpu][v] = 0;
}
}
KeReleaseSpinlock(&gInterruptLock, cookie);
}
void KeIrqRegisterLocalTimerHandler(void (*handler)(void)) {
if (!handler || !gHal)
KePanic(KE_PANIC_CODE_INIT, "IRQ: invalid local timer registration\n");
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
if (gLocalTimerHandlers[i] == handler) return;
if (gLocalTimerHandlerCount >= 2u)
KePanic(KE_PANIC_CODE_INIT, "IRQ: local timer handler count exceeded\n");
gLocalTimerHandlers[gLocalTimerHandlerCount++] = handler;
}
void KeIrqDispatch(uint8_t vector) {
if (!gHal) KePanic(KE_PANIC_CODE_INIT, "IRQ: uninitialized dispatch\n");
if (vector == KE_VECTOR_SPURIOUS) return;
KIRQL target = (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST)
? KiVectorIrql(vector) : (vector == KE_VECTOR_CPU_TIMER ? (KIRQL)27 : (KIRQL)29);
if (target < KeGetCurrentIrql()) target = KeGetCurrentIrql();
KIRQL oldIrql = KeRaiseIrql(target);
if (vector == KE_VECTOR_CPU_TIMER) {
KiReapRetiredVectors();
for (uint8_t i = 0; i < gLocalTimerHandlerCount; i++)
gLocalTimerHandlers[i]();
} else if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
uint32_t cpu = KeGetCurrentCpuIndex();
if (cpu < KE_MAX_CPUS) {
_InterlockedIncrement(&gCpuVectorReaders[cpu][vector]);
KI_INTERRUPT_ROUTE* route = (KI_INTERRUPT_ROUTE*)
_InterlockedCompareExchangePointer(
(void* volatile*)&gCpuVectors[cpu][vector], 0, 0);
if (route) {
_InterlockedIncrement(&route->Active);
if (route->Enabled) {
route->Delivered++;
for (KE_INTERRUPT* h = route->Handlers; h; h = h->Next)
if (h->Used && h->Routine) h->Routine(h->Context);
}
_InterlockedDecrement(&route->Active);
}
}
}
gHal->InterruptController.SendEoi();
if (vector >= KE_VECTOR_DEVICE_FIRST && vector <= KE_VECTOR_DEVICE_LAST) {
uint32_t cpu = KeGetCurrentCpuIndex();
if (cpu < KE_MAX_CPUS)
_InterlockedDecrement(&gCpuVectorReaders[cpu][vector]);
}
KeLowerIrql(oldIrql);
}
// Implements the enter Interrupt operation.
void KiEnterInterrupt(void) {
KeGetCurrentCpu()->InterruptDepth++;
}
// Implements the exit Interrupt operation.
void KiExitInterrupt(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu->InterruptDepth)
cpu->InterruptDepth--;
// A reschedule is only legal after the interrupt unwinds back to the
// interrupted thread's PASSIVE context. If an interrupt hit DISPATCH/DIRQL
// code, leave the request pending instead of switching out of a spinlock or
// high-IRQL region.
if (cpu->InterruptDepth) cpu->InterruptDepth--;
if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
THREAD* current = cpu->CurrentThread;
if (current && !current->IsIdleThread && current->Owner &&
@@ -140,13 +647,7 @@ void KiExitInterrupt(void) {
KeExitThread(current, current->Owner->ExitCode);
while (true) ;
}
KiDrainDpcQueue();
// KeExitThread owns a terminal stack/rundown interval. A timer or IPI
// may interrupt it, but switching away here would abandon teardown
// before the process ExitEvent is necessarily published. The explicit
// KeYield at the end of KeExitThread performs the one terminal switch.
current = cpu->CurrentThread;
if (KeGetCurrentIrql() == PASSIVE_LEVEL &&
(!current || !KeThreadExitInProgress(current)))
code/kernel/ke_irql.c
+25 -8
View File
@@ -1,13 +1,13 @@
// Kernel IRQL model. This file owns the software interrupt-priority state used by IRQ and scheduler code.
// Kernel IRQL model. This file synchronizes per-CPU IRQL with APIC task priority.
#include <stdbool.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_hal.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_apc.h>
#include <intrin.h>
// Implements the irql Name operation.
__declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
switch (irql) {
case PASSIVE_LEVEL: return "PASSIVE_LEVEL";
@@ -26,7 +26,27 @@ __declspec(dllexport) KIRQL KeGetCurrentIrql(void) {
return KeGetCurrentCpu()->CurrentIrql;
}
// Implements the raise Irql operation.
// IRQL classes 16-25 map to vectors 0x30-0xC0. The TPR masks the current
// class and all lower-priority external interrupts. HIGH_LEVEL masks all.
static uint8_t KiIrqlTaskPriority(KIRQL level) {
if (level == HIGH_LEVEL) return 0xFFu;
if (level >= 28u) return 0xF0u;
if (level >= DIRQL_BASE) return (uint8_t)((level - DIRQL_BASE + 3u) << 4u);
if (level >= DISPATCH_LEVEL) return 0x20u;
if (level >= APC_LEVEL) return 0x10u;
return 0u;
}
// Applies the priority before publishing a raised IRQL and after a lowered IRQL.
static void KiSetIrql(KCPU* cpu, KIRQL level, bool raising) {
const HAL_API* hal = KeGetHal();
if (raising && hal && hal->InterruptController.SetTaskPriority)
hal->InterruptController.SetTaskPriority(KiIrqlTaskPriority(level));
cpu->CurrentIrql = level;
if (!raising && hal && hal->InterruptController.SetTaskPriority)
hal->InterruptController.SetTaskPriority(KiIrqlTaskPriority(level));
}
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
KCPU* cpu = KeGetCurrentCpu();
KIRQL oldIrql = cpu->CurrentIrql;
@@ -41,11 +61,10 @@ __declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
KeIrqlName(newIrql), (unsigned)newIrql);
}
cpu->CurrentIrql = newIrql;
KiSetIrql(cpu, newIrql, true);
return oldIrql;
}
// Implements the lower Irql operation.
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KCPU* cpu = KeGetCurrentCpu();
KIRQL currentIrql = cpu->CurrentIrql;
@@ -60,14 +79,13 @@ __declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KeIrqlName(oldIrql), (unsigned)oldIrql);
}
cpu->CurrentIrql = oldIrql;
KiSetIrql(cpu, oldIrql, false);
if (oldIrql == PASSIVE_LEVEL && currentIrql != PASSIVE_LEVEL &&
(__readeflags() & 0x200u) != 0 && cpu->InterruptDepth == 0)
KiDeliverKernelApcs();
}
// Implements the assert Irql operation.
__declspec(dllexport) void KeAssertIrql(KIRQL expected) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql != expected) {
@@ -77,7 +95,6 @@ __declspec(dllexport) void KeAssertIrql(KIRQL expected) {
}
}
// Implements the assert Irql At Or Below operation.
__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql > maxIrql) {
code/kernel/ke_power.c
+98 -52
View File
@@ -1,14 +1,14 @@
#include <ntlite/atomic.h>
// Kernel power coordinator and optional firmware-provider binding.
#include <ntlite/syscall.h>
#include <ntlite/ddk/power.h>
#include <kernel/ke_power.h>
#include <kernel/ke_process.h>
#include <kernel/ke_event.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irq.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_settings.h>
#include <kernel/ke_debug.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ob_object.h>
#include <string.h>
static SPINLOCK gPowerLock = SPINLOCK_INIT;
@@ -17,70 +17,116 @@ static EVENT* gPowerComplete;
static PROCESS* gInit;
static uint32_t gAction;
static bool gCommitted;
static const HAL_API* gButtonHal;
static bool gPolicyReady;
static bool gEarlyButtonPressed;
static uint32_t gButtonAction;
static KDPC gPowerButtonDpc;
static NTL_FIRMWARE_POWER_PROVIDER gFirmwareProvider;
static bool gFirmwarePrepared;
static uint32_t gPreparedAction;
KSTATUS KePowerInitialize(void) {
if (gPowerRequest) return NTLSTATUS_ALREADY_EXISTS;
KSTATUS st = KeEventCreate(EventManualReset, false, &gPowerRequest);
if (!KSUCCESS(st)) return st;
st = KeEventCreate(EventManualReset, false, &gPowerComplete);
return st;
EVENT* request = NULL;
EVENT* complete = NULL;
KSTATUS status = KeEventCreate(EventManualReset, false, &request);
if (!KSUCCESS(status)) return status;
status = KeEventCreate(EventManualReset, false, &complete);
if (!KSUCCESS(status)) {
ObDereferenceObject((OBJECT*)request);
return status;
}
gPowerRequest = request;
gPowerComplete = complete;
return NTLSTATUS_SUCCESS;
}
// Delivers the physical action after interrupt-exit processing.
static void KiPowerButtonDpc(KDPC* dpc, void* context, void* arg1, void* arg2) {
(void)dpc;
(void)context;
(void)arg1;
(void)arg2;
(void)KePowerRequestHardware(gButtonAction);
// The resident firmware owner may be ACPICA or another platform implementation.
__declspec(dllexport) KSTATUS KeRegisterFirmwarePowerProvider(
const NTL_FIRMWARE_POWER_PROVIDER* provider) {
if (!provider || provider->Size != sizeof(*provider) ||
!provider->Prepare || !provider->Enter)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
KSTATUS status = gFirmwareProvider.Size ? NTLSTATUS_ALREADY_EXISTS : NTLSTATUS_SUCCESS;
if (KSUCCESS(status)) gFirmwareProvider = *provider;
KeReleaseSpinlock(&gPowerLock, flags);
if (KSUCCESS(status)) DbgPrint("power: firmware provider registered\n");
return status;
}
// The ACPI SCI handler only acknowledges the fixed event and queues normal work.
static void KiPowerButtonIsr(void) {
if (!gButtonHal || !gButtonHal->Power.ConsumeFixedButton ||
!gButtonHal->Power.ConsumeFixedButton()) return;
(void)KeInsertQueueDpc(&gPowerButtonDpc, NULL, NULL);
// Events before system-root policy becomes available are retained once.
__declspec(dllexport) KSTATUS KeNotifyPowerButton(void) {
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
if (!gPowerRequest) {
KeReleaseSpinlock(&gPowerLock, flags);
return NTLSTATUS_NOT_SUPPORTED;
}
if (!gPolicyReady) {
gEarlyButtonPressed = true;
KeReleaseSpinlock(&gPowerLock, flags);
return NTLSTATUS_SUCCESS;
}
uint32_t action = gButtonAction;
KeReleaseSpinlock(&gPowerLock, flags);
return action ? KePowerRequestHardware(action) : NTLSTATUS_SUCCESS;
}
// Reads immutable boot-time button policy and attaches it to the fixed ACPI SCI.
void KePowerInitializeButton(const HAL_API* hal) {
// System-root policy is loaded independently of the presence of a firmware driver.
void KePowerInitializePolicy(void) {
char mode[24];
KSTATUS status = KeGetSettingString("power", "button", mode, sizeof(mode));
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_INIT, "power: missing [power] button, status=0x%x\n", status);
if (strcmp(mode, "ignore") == 0) {
DbgPrint("power: ACPI power button ignored by policy\n");
return;
}
if (strcmp(mode, "shutdown") == 0) gButtonAction = SYS_POWER_ACTION_SHUTDOWN;
else if (strcmp(mode, "restart") == 0) gButtonAction = SYS_POWER_ACTION_RESTART;
else KePanic(KE_PANIC_CODE_INIT, "power: invalid [power] button=%s\n", mode);
uint32_t action = 0;
if (strcmp(mode, "shutdown") == 0) action = SYS_POWER_ACTION_SHUTDOWN;
else if (strcmp(mode, "restart") == 0) action = SYS_POWER_ACTION_RESTART;
else if (strcmp(mode, "ignore") != 0)
KePanic(KE_PANIC_CODE_INIT, "power: invalid [power] button=%s\n", mode);
if (!hal || !hal->Power.QueryFixedButtonIrq || !hal->Power.ArmFixedButton ||
!hal->Power.ConsumeFixedButton) {
DbgPrint("power: no ACPI fixed power button HAL support\n");
return;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
gButtonAction = action;
gPolicyReady = true;
bool pending = gEarlyButtonPressed;
bool providerPresent = gFirmwareProvider.Size != 0;
gEarlyButtonPressed = false;
KeReleaseSpinlock(&gPowerLock, flags);
DbgPrint("power: button policy=%s firmware provider=%s\n",
mode, providerPresent ? "present" : "absent");
if (pending && action) (void)KePowerRequestHardware(action);
}
// A provider prepares firmware while threads, device resources and AML can run.
KSTATUS KePowerPrepareFirmware(uint32_t action) {
if (action != SYS_POWER_ACTION_SHUTDOWN && action != SYS_POWER_ACTION_RESTART)
return NTLSTATUS_INVALID_PARAMETER;
NTL_FIRMWARE_POWER_PROVIDER provider;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
provider = gFirmwareProvider;
KeReleaseSpinlock(&gPowerLock, flags);
if (!provider.Size) return NTLSTATUS_NOT_SUPPORTED;
flags = KeAcquireSpinlock(&gPowerLock);
gFirmwarePrepared = false;
gPreparedAction = 0;
KeReleaseSpinlock(&gPowerLock, flags);
KSTATUS status = provider.Prepare(provider.Context, action);
if (KSUCCESS(status)) {
flags = KeAcquireSpinlock(&gPowerLock);
gFirmwarePrepared = true;
gPreparedAction = action;
KeReleaseSpinlock(&gPowerLock, flags);
}
uint8_t irq = 0;
status = hal->Power.QueryFixedButtonIrq(&irq);
if (!KSUCCESS(status)) {
DbgPrint("power: ACPI fixed button unavailable (0x%x); AML button not yet supported\n", status);
return;
}
// The IOAPIC is still masked for this IRQ. Arm the firmware source first,
// then install the kernel handler and finally unmask via KeIrqRegister.
status = hal->Power.ArmFixedButton();
if (!KSUCCESS(status)) {
DbgPrint("power: cannot arm fixed button IRQ %u, status=0x%x\n", irq, status);
return;
}
KeInitializeDpc(&gPowerButtonDpc, KiPowerButtonDpc, NULL);
gButtonHal = hal;
KeIrqRegister(irq, KiPowerButtonIsr);
DbgPrint("power: ACPI fixed button armed on IRQ %u, action=%s\n", irq, mode);
return status;
}
// Any returned transition leaves the HAL as the supported final fallback.
KSTATUS KePowerEnterFirmware(uint32_t action) {
NTL_FIRMWARE_POWER_PROVIDER provider;
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
bool prepared = gFirmwarePrepared && gPreparedAction == action;
provider = gFirmwareProvider;
KeReleaseSpinlock(&gPowerLock, flags);
if (!prepared || !provider.Size) return NTLSTATUS_NOT_SUPPORTED;
return provider.Enter(provider.Context, action);
}
void KePowerSetInitProcess(PROCESS* init) {
code/kernel/ke_sched.c
-30
View File
@@ -129,7 +129,6 @@ bool KiSchedulerRefreshMutexDonation(MUTEX* mutex) {
return changed;
}
// Implements the default Thread Affinity Mask operation.
static uint64_t KiDefaultThreadAffinityMask(void) {
// Normal kernel, user, and driver-created threads are boot-processor-affine by
// default. Code that is explicitly audited for AP execution can widen its
@@ -137,14 +136,12 @@ static uint64_t KiDefaultThreadAffinityMask(void) {
return 1ULL;
}
// Implements the thread Is Killed Locked operation.
static bool KiThreadIsKilledLocked(const THREAD* thread) {
return thread && !thread->IsIdleThread && thread->Owner &&
KeIsProcessTerminating(thread->Owner) &&
!KeThreadTerminationDeferred(thread);
}
// Implements the thread Can Run On Cpu Locked operation.
static bool KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
if (!thread || !cpu) return false;
if (thread->IsIdleThread) return thread == cpu->IdleThread;
@@ -152,7 +149,6 @@ static bool KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
}
// Implements the cpu Is Idle Locked operation.
static bool KiCpuIsIdleLocked(const KCPU* cpu) {
if (!cpu || cpu->State != KeCpuOnline)
return false;
@@ -160,7 +156,6 @@ static bool KiCpuIsIdleLocked(const KCPU* cpu) {
return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
}
// Implements the any Idle Primary For Thread Locked operation.
static bool KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
if (!thread || thread->IsIdleThread)
return false;
@@ -193,7 +188,6 @@ static bool KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
return KiAnyIdlePrimaryForThreadLocked(thread);
}
// Implements the request Reschedule Cpu Locked operation.
static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || target->State != KeCpuOnline)
return;
@@ -206,7 +200,6 @@ static void KiRequestRescheduleCpuLocked(KCPU* target) {
}
}
// Implements the poke Runnable Cpus Locked operation.
static void KiPokeRunnableCpusLocked(THREAD* thread) {
if (!thread || thread->IsIdleThread)
return;
@@ -256,7 +249,6 @@ static void KiPokeRunnableCpusLocked(THREAD* thread) {
}
}
// Implements the base Quantum For Priority operation.
static uint32_t KiBaseQuantumForPriority(uint32_t priority) {
switch ((THREAD_PRIORITY)priority) {
case PriorityHigh: return SCHED_HIGH_QUANTUM_TICKS;
@@ -266,7 +258,6 @@ static uint32_t KiBaseQuantumForPriority(uint32_t priority) {
}
}
// Implements the compute Quantum Locked operation.
static uint32_t KiComputeQuantumLocked(THREAD* thread) {
if (!thread || thread == KeGetCurrentCpu()->IdleThread)
return 1;
@@ -288,7 +279,6 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) {
return quantum ? quantum : 1;
}
// Implements the refill Quantum Locked operation.
static void KiRefillQuantumLocked(THREAD* thread) {
if (!thread) return;
thread->QuantumTicks = KiComputeQuantumLocked(thread);
@@ -308,7 +298,6 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the arm Post Switch Unlock operation.
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
// The scheduler lock is intentionally held across KeSwitchContext(). The
// unlock cookie belongs to the thread continuation that will resume after
@@ -319,12 +308,10 @@ static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
thread->SwitchLockCookie = cookie;
}
// Implements the post Switch Unlock Cookie operation.
static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
return thread ? thread->SwitchLockCookie : 0x202ULL;
}
// Implements the scheduler Bug operation.
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"scheduler invariant failed: %s: thread=0x%016llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
@@ -338,7 +325,6 @@ static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
why ? why : "?");
}
// Implements the assert Ready Queue Candidate Locked operation.
static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null thread");
@@ -352,7 +338,6 @@ static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread)
KiSchedulerBug(where, thread, "thread still reserved/running on a CPU");
}
// Implements the assert Picked Thread Locked operation.
static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) {
if (!thread)
KiSchedulerBug(where, thread, "null picked thread");
@@ -694,7 +679,6 @@ KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority) {
static bool KiAnyRunnableForCpuLocked(KCPU* cpu);
// Implements the any Runnable For Current Cpu operation.
static int KiAnyRunnableForCurrentCpu(void) {
KCPU* cpu = KeGetCurrentCpu();
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -703,7 +687,6 @@ static int KiAnyRunnableForCurrentCpu(void) {
return any;
}
// Implements the idle Loop operation.
static void KiIdleLoop(void) {
while (true) {
KCPU* cpu = KeGetCurrentCpu();
@@ -900,7 +883,6 @@ uint64_t KeGetDefaultThreadAffinityMask(void) {
return KiDefaultThreadAffinityMask();
}
// Implements the ready Thread operation.
void KeReadyThread(THREAD* thread) {
if (!thread) return;
@@ -968,7 +950,6 @@ KSTATUS KeCancelSuspendedThread(THREAD* thread) {
return NTLSTATUS_SUCCESS;
}
// Implements the unready Thread operation.
void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -979,7 +960,6 @@ void KeUnreadyThread(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the any Runnable For Cpu Locked operation.
static bool KiAnyRunnableForCpuLocked(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -1028,7 +1008,6 @@ static THREAD* KiPickFromQueueLocked(KCPU* cpu, LIST_ENTRY* head) {
return (THREAD*)0;
}
// Implements the pick Next For Cpu operation.
static THREAD* KiPickNextForCpu(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -1049,7 +1028,6 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
return cpu->IdleThread;
}
// Implements the request Reschedule operation.
void KiRequestReschedule(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu)
@@ -1062,7 +1040,6 @@ bool KiIsReschedulePending(void) {
return cpu && NtlAtomicLoad32(&cpu->ReschedulePending) != 0;
}
// Implements the schedule operation.
static void KiSchedule(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -1143,13 +1120,11 @@ static void KiSchedule(void) {
KeSchedulerPostSwitch();
}
// Implements the schedule From Interrupt operation.
void KiScheduleFromInterrupt(void) {
if (!KiIsReschedulePending()) return;
KiSchedule();
}
// Implements the scheduler Mark Thread Waiting operation.
void KiSchedulerMarkThreadWaiting(THREAD* thread) {
if (!thread) return;
@@ -1196,7 +1171,6 @@ static void KiCancelThreadTimeoutLocked(THREAD* thread) {
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
}
// Implements the scheduler Note Voluntary Block operation.
void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -1263,7 +1237,6 @@ void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the scheduler Wake Thread operation.
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiWakeThreadLocked(thread, status);
@@ -1430,7 +1403,6 @@ static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) {
return requestReschedule;
}
// Implements the scheduler Tick operation.
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
THREAD* cur = cpu->CurrentThread;
@@ -1513,7 +1485,6 @@ KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus)
return NTLSTATUS_SUCCESS;
}
// Implements the delay Thread operation.
__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
if (milliseconds == 0) {
KeYield();
@@ -1554,7 +1525,6 @@ __declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
return KeCommitBlockThread(self, 0);
}
// Implements the yield operation.
__declspec(dllexport) void KeYield(void) {
KeAssertIrql(PASSIVE_LEVEL);
KiSchedule();
code/kernel/ke_semaphore.c
-2
View File
@@ -8,10 +8,8 @@
#include <kernel/ke_irql.h>
#include <kernel/ob_object.h>
// Implements the semaphore Destroy operation.
static void KiSemaphoreDestroy(OBJECT* obj) { (void)obj; }
// Implements the semaphore Create operation.
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE** out) {
if (!out || initial < 0 || max < 1 || initial > max)
return NTLSTATUS_INVALID_PARAMETER;
code/kernel/ke_smp.c
-13
View File
@@ -108,23 +108,15 @@ static void KiSmpPoolCheckpoint(const char* phase, uint32_t cpuIndex) {
(void)MmValidateKernelPool(phase ? phase : "smp");
}
// Implements the emit8 operation.
static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
// Implements the emit16 operation.
static void KiEmit16(uint8_t** p, uint16_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
// Implements the emit32 operation.
static void KiEmit32(uint8_t** p, uint32_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
// Implements the emit64 operation.
static void KiEmit64(uint8_t** p, uint64_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); }
// Implements the patch16 operation.
static void KiPatch16(uint8_t* base, size_t off, uint16_t v) { memcpy(base + off, &v, sizeof(v)); }
// Implements the patch32 operation.
static void KiPatch32(uint8_t* base, size_t off, uint32_t v) { memcpy(base + off, &v, sizeof(v)); }
// Implements the patch64 operation.
static void KiPatch64(uint8_t* base, size_t off, uint64_t v) { memcpy(base + off, &v, sizeof(v)); }
// Implements the gdt Descriptor operation.
static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) {
return ((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
@@ -394,7 +386,6 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_PROCESSOR_INFO* info
cpu->ReschedulePending = 0;
}
// Implements the ceil Log2 U32 operation.
static uint32_t KiCeilLog2U32(uint32_t value) {
if (value <= 1u)
return 0u;
@@ -407,7 +398,6 @@ static uint32_t KiCeilLog2U32(uint32_t value) {
return shift;
}
// Implements the mask Width operation.
static uint32_t KiMaskWidth(uint32_t shift) {
if (shift >= 32u)
return 0xFFFFFFFFu;
@@ -451,7 +441,6 @@ static void KiReadCpuidTopologyLeaf(uint32_t leaf, KI_CPUID_TOPOLOGY_PARAMS* out
}
}
// Implements the detect Cpuid Topology Params operation.
static KI_CPUID_TOPOLOGY_PARAMS KiDetectCpuidTopologyParams(void) {
KI_CPUID_TOPOLOGY_PARAMS topo;
memset(&topo, 0, sizeof(topo));
@@ -640,7 +629,6 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
return NTLSTATUS_SUCCESS;
}
// Implements the ap Stage Name operation.
static const char* KiApStageName(uint32_t stage) {
switch (stage) {
case AP_STAGE_NONE: return "none";
@@ -801,7 +789,6 @@ KSTATUS KeStartApplicationProcessors(void) {
return NTLSTATUS_SUCCESS;
}
// Implements the admit Application Processors To Scheduler operation.
KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
if (gCpuCount <= 1) {
DbgPrint("SMP: no AP scheduler admission needed\n");
code/kernel/ke_syscall.asm
+154 -4
View File
@@ -1,4 +1,4 @@
; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64)
; ke_syscall.asm - INT 0x2F, fault, and IRQ kernel entries (MASM64)
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
@@ -64,7 +64,7 @@ KeSyscallEntry PROC
mov rax, KTRAP_FRAME_MAGIC
mov [rbx + TF_Magic], rax
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
mov dword ptr [rbx + TF_Vector], 40h
mov dword ptr [rbx + TF_Vector], 02Fh
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
mov [rbx + TF_Rcx], rcx
@@ -781,7 +781,7 @@ IrqStub MACRO num
PUBLIC KeIrqStub&num
KeIrqStub&num PROC
push rax
mov al, num
mov al, (030h + num)
jmp KeIrqCommon
KeIrqStub&num ENDP
ENDM
@@ -810,6 +810,156 @@ IrqStub 20
IrqStub 21
IrqStub 22
IrqStub 23
IrqStub 24
IrqStub 25
IrqStub 26
IrqStub 27
IrqStub 28
IrqStub 29
IrqStub 30
IrqStub 31
IrqStub 32
IrqStub 33
IrqStub 34
IrqStub 35
IrqStub 36
IrqStub 37
IrqStub 38
IrqStub 39
IrqStub 40
IrqStub 41
IrqStub 42
IrqStub 43
IrqStub 44
IrqStub 45
IrqStub 46
IrqStub 47
IrqStub 48
IrqStub 49
IrqStub 50
IrqStub 51
IrqStub 52
IrqStub 53
IrqStub 54
IrqStub 55
IrqStub 56
IrqStub 57
IrqStub 58
IrqStub 59
IrqStub 60
IrqStub 61
IrqStub 62
IrqStub 63
IrqStub 64
IrqStub 65
IrqStub 66
IrqStub 67
IrqStub 68
IrqStub 69
IrqStub 70
IrqStub 71
IrqStub 72
IrqStub 73
IrqStub 74
IrqStub 75
IrqStub 76
IrqStub 77
IrqStub 78
IrqStub 79
IrqStub 80
IrqStub 81
IrqStub 82
IrqStub 83
IrqStub 84
IrqStub 85
IrqStub 86
IrqStub 87
IrqStub 88
IrqStub 89
IrqStub 90
IrqStub 91
IrqStub 92
IrqStub 93
IrqStub 94
IrqStub 95
IrqStub 96
IrqStub 97
IrqStub 98
IrqStub 99
IrqStub 100
IrqStub 101
IrqStub 102
IrqStub 103
IrqStub 104
IrqStub 105
IrqStub 106
IrqStub 107
IrqStub 108
IrqStub 109
IrqStub 110
IrqStub 111
IrqStub 112
IrqStub 113
IrqStub 114
IrqStub 115
IrqStub 116
IrqStub 117
IrqStub 118
IrqStub 119
IrqStub 120
IrqStub 121
IrqStub 122
IrqStub 123
IrqStub 124
IrqStub 125
IrqStub 126
IrqStub 127
IrqStub 128
IrqStub 129
IrqStub 130
IrqStub 131
IrqStub 132
IrqStub 133
IrqStub 134
IrqStub 135
IrqStub 136
IrqStub 137
IrqStub 138
IrqStub 139
IrqStub 140
IrqStub 141
IrqStub 142
IrqStub 143
IrqStub 144
IrqStub 145
IrqStub 146
IrqStub 147
IrqStub 148
IrqStub 149
IrqStub 150
IrqStub 151
IrqStub 152
IrqStub 153
IrqStub 154
IrqStub 155
IrqStub 156
IrqStub 157
IrqStub 158
IrqStub 159
PUBLIC KeLocalTimerStub
KeLocalTimerStub PROC
push rax
mov al, 0E0h
jmp KeIrqCommon
KeLocalTimerStub ENDP
PUBLIC KeSpuriousStub
KeSpuriousStub PROC
push rax
mov al, 0FFh
jmp KeIrqCommon
KeSpuriousStub ENDP
PUBLIC KeRescheduleIpiStub
KeRescheduleIpiStub PROC
@@ -819,7 +969,7 @@ KeRescheduleIpiStub PROC
KeRescheduleIpiStub ENDP
KeIrqCommon PROC
; al = irq number, original rax saved by the per-IRQ stub.
; al = IDT vector, original rax saved by the vector stub.
push rcx
push rdx
push r8
code/kernel/ke_syscall.c
+1 -7
View File
@@ -1,6 +1,6 @@
// Kernel syscall dispatcher. This file validates user requests and bridges user-mode ABI calls into kernel services.
// KeSyscallEntry (ke_syscall.asm) handles the INT 0x40 ring-3 -> ring-0
// KeSyscallEntry (ke_syscall.asm) handles the INT 0x2F ring-3 -> ring-0
// transition, re-orders the arguments to match MS x64, and calls
// KeSyscallDispatch with the syscall number plus at most four register arguments.
// Compound operations use size-tagged request structures instead of extending
@@ -135,7 +135,6 @@ static void KiFileMappingDestroy(OBJECT* obj) {
}
// Implements the syscall Init operation.
void KeSyscallInit(const HAL_API* hal) {
gHal = hal;
}
@@ -301,7 +300,6 @@ static KSTATUS KiCopyInt32ToUser(int32_t* valueOut, int32_t value) {
return CopyToUser(valueOut, &value, sizeof(value));
}
// Implements the mul Size operation.
static bool KiMulSize(size_t a, size_t b, size_t* out) {
if (!out) return false;
if (a && b > ((size_t)-1) / a) return false;
@@ -1044,7 +1042,6 @@ static KSTATUS KiVmProtectToPageFlags(uint32_t protect, uint64_t* flagsOut) {
}
}
// Implements the page Flags To Vm Protect operation.
static uint32_t KiPageFlagsToVmProtect(uint64_t flags) {
if (!(flags & PAGE_PRESENT)) return SYS_VM_NOACCESS;
bool writable = (flags & PAGE_WRITE) != 0;
@@ -2070,7 +2067,6 @@ static void SyscallUserAsyncKernelRoutine(KAPC* apc,
KeGetCurrentIrql());
}
// Implements the user Async Apc Context Free operation.
static void SyscallUserAsyncApcContextFree(SYS_USER_ASYNC_APC_CONTEXT* ctx) {
if (!ctx) return;
if (ctx->TargetThread) {
@@ -2080,7 +2076,6 @@ static void SyscallUserAsyncApcContextFree(SYS_USER_ASYNC_APC_CONTEXT* ctx) {
MmFreePool(ctx);
}
// Implements the user Async Apc Rundown operation.
static void SyscallUserAsyncApcRundown(KAPC* apc) {
if (!apc) return;
SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc);
@@ -2110,7 +2105,6 @@ static void SyscallUserAsyncNormalRoutine(void* normalContext,
SyscallUserAsyncApcContextFree(ctx);
}
// Implements the async File Common operation.
static uint64_t SyscallAsyncFileCommon(const SYS_ASYNC_FILE_REQUEST* request, bool write) {
if (!request) return NTLSTATUS_INVALID_PARAMETER;
code/kernel/ke_thread.c
+9 -10
View File
@@ -136,7 +136,6 @@ void KeFinalizeThreadExit(THREAD* thread) {
}
}
// Implements the thread Destroy operation.
static void KiThreadDestroy(OBJECT* obj) {
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
@@ -450,7 +449,6 @@ bool KeThreadExitInProgress(const THREAD* thread) {
return thread && NtlAtomicLoad32(&thread->ExitInProgress) != 0;
}
// Implements the exit Thread operation.
__declspec(dllexport) void KeExitThread(THREAD* thread, int32_t code) {
(void)code;
if (!thread) return;
@@ -601,6 +599,15 @@ static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
ctx->Rip = (uint64_t)KeKernelThreadEntry;
thread->SavedContext = ctx;
// Publish thread completion before the new kernel thread can run.
EVENT* exitEvent = (EVENT*)0;
KSTATUS eventStatus = KeEventCreate(EventManualReset, false, &exitEvent);
if (!KSUCCESS(eventStatus)) {
ObDereferenceObject(&thread->Header);
return eventStatus;
}
thread->ExitEvent = exitEvent;
// Register with the owning process.
ObReferenceObject(&owner->Header);
thread->Owner = owner;
@@ -716,7 +723,6 @@ void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
routine(thread, context, status);
}
// Implements the commit Block Thread operation.
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
KeAssertIrql(PASSIVE_LEVEL);
if (!thread) return NTLSTATUS_INVALID_PARAMETER;
@@ -766,13 +772,11 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
return thread->WaitStatus;
}
// Implements the block Thread operation.
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
KePrepareBlockThread(thread, waitObject, alertable);
return KeCommitBlockThread(thread, alertable);
}
// Implements the unblock Thread With Status operation.
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
KiSchedulerWakeThread(thread, status);
@@ -864,12 +868,10 @@ bool KeAbortThreadWait(THREAD* thread, KSTATUS status) {
return true;
}
// Implements the unblock Thread operation.
void KeUnblockThread(THREAD* thread) {
KeUnblockThreadWithStatus(thread, NTLSTATUS_SUCCESS);
}
// Implements the alert Thread With Status operation.
static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return NTLSTATUS_INVALID_PARAMETER;
@@ -881,17 +883,14 @@ static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
: NTLSTATUS_PENDING;
}
// Implements the alert Thread operation.
KSTATUS KeAlertThread(THREAD* thread) {
return KiAlertThreadWithStatus(thread, NTLSTATUS_ALERTED);
}
// Implements the wake Thread For Kernel Apc operation.
void KiWakeThreadForKernelApc(THREAD* thread) {
(void)KiAlertThreadWithStatus(thread, NTLSTATUS_KERNEL_APC);
}
// Implements the wake Thread For User Apc operation.
void KiWakeThreadForUserApc(THREAD* thread) {
(void)KiAlertThreadWithStatus(thread, NTLSTATUS_USER_APC);
}
code/kernel/ke_time.c
-15
View File
@@ -56,7 +56,6 @@ static bool KiIsLeapYear(uint32_t year) {
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
}
// Implements the days In Month operation.
static uint32_t KiDaysInMonth(uint32_t year, uint32_t month) {
static const uint8_t days[12] = { 31,28,31,30,31,30,31,31,30,31,30,31 };
if (month == 2u && KiIsLeapYear(year)) return 29u;
@@ -76,7 +75,6 @@ static bool KiValidCalendar(uint32_t year, uint32_t month, uint32_t day,
return true;
}
// Implements the days Before Year operation.
static uint64_t KiDaysBeforeYear(uint32_t year) {
uint64_t days = 0;
for (uint32_t y = 1970u; y < year; y++)
@@ -93,7 +91,6 @@ static uint64_t KiCalendarToEpochSeconds(uint32_t year, uint32_t month, uint32_t
return (((days * 24u) + hour) * 60u + minute) * 60u + second;
}
// Implements the epoch Seconds To Calendar operation.
static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out) {
uint64_t days = epochSeconds / 86400u;
uint32_t sod = (uint32_t)(epochSeconds % 86400u);
@@ -124,20 +121,17 @@ static void KiEpochSecondsToCalendar(uint64_t epochSeconds, SYS_SYSTEM_TIME* out
out->DayOfWeek = (uint8_t)((totalDays + 4u) % 7u); // 1970-01-01 was Thursday.
}
// Implements the day Of Week operation.
static uint32_t KiDayOfWeek(uint32_t year, uint32_t month, uint32_t day) {
uint64_t epoch = KiCalendarToEpochSeconds(year, month, day, 0, 0, 0);
return (uint32_t)(((epoch / 86400u) + 4u) % 7u);
}
// Implements the last Sunday Of Month operation.
static uint32_t KiLastSundayOfMonth(uint32_t year, uint32_t month) {
uint32_t last = KiDaysInMonth(year, month);
uint32_t dow = KiDayOfWeek(year, month, last); // Sunday == 0.
return last - dow;
}
// Implements the central Europe Dst Active Utc operation.
static bool KiCentralEuropeDstActiveUtc(uint64_t utcEpochSeconds) {
SYS_SYSTEM_TIME utc;
memset(&utc, 0, sizeof(utc));
@@ -150,7 +144,6 @@ static bool KiCentralEuropeDstActiveUtc(uint64_t utcEpochSeconds) {
return utcEpochSeconds >= startUtc && utcEpochSeconds < endUtc;
}
// Implements the offset For Utc Epoch operation.
static int16_t KiOffsetForUtcEpoch(uint64_t utcEpochSeconds, bool* daylight) {
int32_t offset = gTimeZonePolicy.StandardOffsetMinutes;
bool dst = false;
@@ -166,7 +159,6 @@ static int16_t KiOffsetForUtcEpoch(uint64_t utcEpochSeconds, bool* daylight) {
return (int16_t)offset;
}
// Implements the local Epoch To Utc operation.
static uint64_t KiLocalEpochToUtc(uint64_t localEpochSeconds, int16_t offsetMinutes) {
uint64_t offsetSeconds = (uint64_t)((offsetMinutes < 0) ? -offsetMinutes : offsetMinutes) * 60u;
if (offsetMinutes >= 0)
@@ -174,7 +166,6 @@ static uint64_t KiLocalEpochToUtc(uint64_t localEpochSeconds, int16_t offsetMinu
return localEpochSeconds + offsetSeconds;
}
// Implements the rebase Utc From Source operation.
static void KiRebaseUtcFromSource(void) {
if (gSourceTimeZoneKnown) {
gBaseUtcEpochSeconds = KiLocalEpochToUtc(gSourceRawEpochSeconds, gSourceTimeZoneMinutes);
@@ -194,13 +185,11 @@ static void KiRebaseUtcFromSource(void) {
gBaseTickMicroseconds = gSourceTickMicroseconds;
}
// Implements the ascii Lower operation.
static int KiAsciiLower(int c) {
if (c >= 'A' && c <= 'Z') return c + ('a' - 'A');
return c;
}
// Implements the string Equals No Case operation.
static bool KiStringEqualsNoCase(const char* a, const char* b) {
if (!a || !b) return false;
while (*a && *b) {
@@ -237,7 +226,6 @@ static bool KiParseInt32(const char* s, int32_t* out) {
return true;
}
// Implements the starts With No Case operation.
static bool KiStartsWithNoCase(const char* s, const char* prefix) {
if (!s || !prefix) return false;
while (*prefix) {
@@ -307,7 +295,6 @@ static bool KiParseUtcOffset(const char* s, int16_t* out) {
return true;
}
// Implements the apply Default Time Zone Policy operation.
static void KiApplyDefaultTimeZonePolicy(void) {
gTimeZonePolicy.StandardOffsetMinutes = KE_CE_STANDARD_OFFSET;
gTimeZonePolicy.DstRule = KeDstRuleCentralEurope;
@@ -315,7 +302,6 @@ static void KiApplyDefaultTimeZonePolicy(void) {
gHardwareClockMode = KeHardwareClockUtc;
}
// Implements the time Initialize operation.
void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal) {
gTimeHal = hal;
KiApplyDefaultTimeZonePolicy();
@@ -369,7 +355,6 @@ void KeTimeInitialize(const BOOT_INFO* bootInfo, const HAL_API* hal) {
}
}
// Implements the time Reload Settings operation.
KSTATUS KeTimeReloadSettings(void) {
KE_TIME_ZONE_POLICY policy = gTimeZonePolicy;
char value[64];
code/kernel/ke_timer.c
-1
View File
@@ -52,7 +52,6 @@ static uint8_t KiTimerRemoveLocked(KTIMER* timer) {
return 1;
}
// Implements the timer Destroy operation.
static void KiTimerDestroy(OBJECT* obj) {
KTIMER* timer = (KTIMER*)obj;
if (!timer)
code/kernel/ke_trap.c
-3
View File
@@ -45,7 +45,6 @@ STATIC_ASSERT_OFFSET(KTRAP_FRAME, SsSlot, 216);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousFrame, 224);
_Static_assert(sizeof(KTRAP_FRAME) == 240, "KTRAP_FRAME size must match ke_syscall.asm");
// Implements the trap Frame From User operation.
bool KeTrapFrameFromUser(const KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) return false;
return (frame->Cs & 3u) == 3u;
@@ -58,7 +57,6 @@ KTRAP_FRAME* KeGetCurrentTrapFrame(void) {
return t->TrapFrame;
}
// Implements the enter Trap Frame operation.
void KiEnterTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
@@ -71,7 +69,6 @@ void KiEnterTrapFrame(KTRAP_FRAME* frame) {
t->TrapFrame = frame;
}
// Implements the exit Trap Frame operation.
void KiExitTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
code/kernel/kmain.c
+219 -73
View File
@@ -16,6 +16,7 @@
#include <kernel/ob_object.h>
#include <kernel/ke_process.h>
#include <kernel/ke_power.h>
#include <kernel/ke_firmware.h>
#include <kernel/mm_file_cache.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
@@ -43,6 +44,8 @@
#include <kernel/ke_audio.h>
#include <kernel/ke_settings.h>
#include <kernel/ke_work.h>
#include <kernel/ke_event.h>
#include <ntlite/atomic.h>
#include <kernel/ke_system_root.h>
#include <kernel/se_security.h>
#include <kernel/ke_system_service.h>
@@ -134,7 +137,6 @@ static void KiInitializeSwapFromSettings(void) {
// HAL pointer cached at boot for subsystems that still need platform services.
static const HAL_API* gKmainHal;
// Implements the page In Bootloader Region operation.
static bool KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) {
const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i];
@@ -178,6 +180,24 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
KiReclaimBootBlob(bootInfo, bootInfo->DriversIni.PhysicalBase, bootInfo->DriversIni.PageCount);
}
// Runs the shared module prepare/publication sequence for both image sources.
static KSTATUS KiStartLoadedDriver(const DM_DRIVER_CONFIG* driver, uintptr_t base) {
KSTATUS status = LdrResolveImports(base);
LDR_KERNEL_MODULE* reservation = (LDR_KERNEL_MODULE*)0;
bool canReleaseImage = true;
if (KSUCCESS(status))
status = LdrRegisterModule(driver->Image, base, &reservation);
if (KSUCCESS(status)) status = DmPrepareLoadedDriver(driver->Id, base, &canReleaseImage);
if (KSUCCESS(status)) {
DmCommitLoadedDriver(base, reservation);
return NTLSTATUS_SUCCESS;
}
LdrDiscardModule(reservation);
if (canReleaseImage) LdrReleaseKernelImage(base);
else DbgPrint("driver: failed image retained for callback lifetime %s\n", driver->Image);
return status;
}
// Loads a runtime image from the selected system root, keeping PE identity separate
// from the configured module id used for lifecycle and dependency accounting.
static KSTATUS KiLoadRuntimeDriver(const DM_DRIVER_CONFIG* driver) {
@@ -210,14 +230,7 @@ static KSTATUS KiLoadRuntimeDriver(const DM_DRIVER_CONFIG* driver) {
status = LdrLoadKernelImage(memory, (size_t)size, &base);
MmFreePool(memory);
if (!KSUCCESS(status)) return status;
status = LdrResolveImports(base);
if (!KSUCCESS(status)) return status;
status = LdrRegisterModule(driver->Image, base);
if (!KSUCCESS(status)) return status;
status = DmStartLoadedDriver(driver->Id, base);
if (KSUCCESS(status))
status = LdrSetModuleExportsAvailable(driver->Image, true);
return status;
return KiStartLoadedDriver(driver, base);
}
// Starts a preloaded boot driver by its configuration identity, not array position.
@@ -237,14 +250,7 @@ static KSTATUS KiLoadBootDriver(const BOOT_INFO* bootInfo,
KSTATUS status = LdrLoadKernelImage(
(const void*)(bootInfo->PhysicalMapBase + found->PhysicalBase), found->Length, &base);
if (!KSUCCESS(status)) return status;
status = LdrResolveImports(base);
if (!KSUCCESS(status)) return status;
status = LdrRegisterModule(driver->Image, base);
if (!KSUCCESS(status)) return status;
status = DmStartLoadedDriver(driver->Id, base);
if (KSUCCESS(status))
status = LdrSetModuleExportsAvailable(driver->Image, true);
return status;
return KiStartLoadedDriver(driver, base);
}
// Validates that the firmware preloaded exactly the configured boot modules.
@@ -266,54 +272,195 @@ static bool KiCheckBootDriverHandoff(const BOOT_INFO* bootInfo, const DM_CONFIG*
return expected == bootInfo->BootDrivers.Count;
}
// Starts the lexicographically first eligible driver, independent of INI order.
// Failed providers block requires= dependents, while after= waits for settlement.
// Bounded workers execute independent driver starts while the coordinator owns graph state.
#define KI_DRIVER_START_WORKERS 4u
typedef enum KI_DRIVER_WORK_STATE {
KiDriverWorkIdle,
KiDriverWorkQueued,
KiDriverWorkRunning,
KiDriverWorkFinished,
KiDriverWorkExit,
KiDriverWorkExited
} KI_DRIVER_WORK_STATE;
typedef struct KI_DRIVER_START_WORKER {
EVENT* Wake;
EVENT* Completion;
THREAD* Thread;
const DM_CONFIG* Config;
const BOOT_INFO* BootInfo;
volatile int32_t State;
uint32_t Index;
KSTATUS Status;
} KI_DRIVER_START_WORKER;
static void KiDriverStartWorker(void* argument) {
KI_DRIVER_START_WORKER* worker = (KI_DRIVER_START_WORKER*)argument;
for (;;) {
EVENT* wake = worker->Wake;
if (!wake)
KePanic(KE_PANIC_CODE_INIT,
"Driver worker: null wake worker=0x%016llx index=%u state=%d\n",
(uint64_t)(uintptr_t)worker, worker->Index,
NtlAtomicLoad32(&worker->State));
KSTATUS status = KeWaitForEvent(wake, false);
if (!KSUCCESS(status)) {
THREAD* self = KeGetCurrentThread();
KePanic(KE_PANIC_CODE_INIT,
"Driver worker wait status=0x%x worker=0x%016llx index=%u state=%d\n"
" wake=0x%016llx currentWake=0x%016llx thread=0x%016llx\n"
" waitStatus=0x%x threadState=%u waitObject=0x%016llx\n",
status, (uint64_t)(uintptr_t)worker, worker->Index,
NtlAtomicLoad32(&worker->State),
(uint64_t)(uintptr_t)wake,
(uint64_t)(uintptr_t)worker->Wake,
(uint64_t)(uintptr_t)self,
self ? self->WaitStatus : 0u,
self ? (unsigned)self->State : 0u,
self ? (uint64_t)(uintptr_t)self->WaitObject : 0ull);
}
// A wake alone does not claim work; stale signals may arrive after completion.
int32_t prior = NtlAtomicCompareExchange32(&worker->State,
KiDriverWorkRunning, KiDriverWorkQueued);
if (prior == KiDriverWorkExit) {
NtlAtomicExchange32(&worker->State, KiDriverWorkExited);
return;
}
if (prior != KiDriverWorkQueued) continue;
uint32_t index = worker->Index;
if (index >= worker->Config->DriverCount)
KePanic(KE_PANIC_CODE_INIT, "Driver worker: invalid assignment %u\n", index);
const DM_DRIVER_CONFIG* driver = &worker->Config->Drivers[index];
DbgPrint("driver: start id=%s image=%s\n", driver->Id, driver->Image);
worker->Status = driver->Phase == DmStartBoot ?
KiLoadBootDriver(worker->BootInfo, driver) : KiLoadRuntimeDriver(driver);
NtlAtomicExchange32(&worker->State, KiDriverWorkFinished);
KeSignalEvent(worker->Completion);
}
}
static int KiSelectEligibleDriver(const DM_CONFIG* config, DM_START_PHASE phase) {
int selected = -1;
for (uint32_t i = 0; i < config->DriverCount; ++i) {
const DM_DRIVER_CONFIG* d = &config->Drivers[i];
if (d->Phase != phase || !DmDriverCanStart(config, i)) continue;
if (selected < 0 || strcmp(d->Id, config->Drivers[selected].Id) < 0)
selected = (int)i;
}
return selected;
}
static void KiRunDriverPhase(DM_CONFIG* config, const BOOT_INFO* bootInfo,
DM_START_PHASE phase) {
config->ActivePhase = phase;
EVENT* completion = (EVENT*)0;
KSTATUS status = KeEventCreate(EventAutoReset, false, &completion);
if (!KSUCCESS(status)) KePanic(KE_PANIC_CODE_INIT, "Driver scheduler: completion event failed\n");
// Worker threads retain these records until their exit has been acknowledged.
KI_DRIVER_START_WORKER* workers = (KI_DRIVER_START_WORKER*)
MmAllocatePoolTag(sizeof(KI_DRIVER_START_WORKER) * KI_DRIVER_START_WORKERS,
MM_POOL_TAG('d', 'r', 'w', 'k'));
if (!workers)
KePanic(KE_PANIC_CODE_INIT, "Driver scheduler: worker allocation failed\n");
memset(workers, 0, sizeof(KI_DRIVER_START_WORKER) * KI_DRIVER_START_WORKERS);
for (uint32_t i = 0; i < KI_DRIVER_START_WORKERS; ++i) {
KI_DRIVER_START_WORKER* worker = &workers[i];
worker->Index = UINT32_MAX;
worker->Config = config;
worker->BootInfo = bootInfo;
worker->Completion = completion;
status = KeEventCreate(EventAutoReset, false, &worker->Wake);
if (!KSUCCESS(status)) KePanic(KE_PANIC_CODE_INIT, "Driver scheduler: work event failed\n");
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "driver.start");
status = KeCreateKernelThread((uintptr_t)KiDriverStartWorker,
(uintptr_t)worker, &parameters, &worker->Thread);
if (!KSUCCESS(status)) KePanic(KE_PANIC_CODE_INIT, "Driver scheduler: worker failed\n");
}
uint32_t active = 0;
for (;;) {
int selected = -1;
for (uint32_t i = 0; i < config->DriverCount; ++i) {
const DM_DRIVER_CONFIG* d = &config->Drivers[i];
if (d->Phase != phase || !DmDriverCanStart(config, i)) continue;
if (selected < 0 || strcmp(d->Id, config->Drivers[selected].Id) < 0)
selected = (int)i;
}
if (selected >= 0) {
DM_DRIVER_CONFIG* d = &config->Drivers[selected];
DbgPrint("Driver manager: starting %s (%s)\n", d->Id, d->Image);
KSTATUS status = phase == DmStartBoot
? KiLoadBootDriver(bootInfo, d) : KiLoadRuntimeDriver(d);
d->State = KSUCCESS(status) ? DmRunStarted : DmRunFailed;
if (!KSUCCESS(status)) {
DbgPrint("Driver manager: failed %s, status=0x%x\n", d->Id, status);
if (d->Critical)
KePanic(KE_PANIC_CODE_INIT,
"Critical driver %s failed (0x%x)\n", d->Id, status);
for (uint32_t i = 0; i < KI_DRIVER_START_WORKERS; ++i) {
KI_DRIVER_START_WORKER* worker = &workers[i];
if (NtlAtomicCompareExchange32(&worker->State,
KiDriverWorkIdle, KiDriverWorkFinished) !=
KiDriverWorkFinished) continue;
uint32_t index = worker->Index;
if (index >= config->DriverCount)
KePanic(KE_PANIC_CODE_INIT, "Driver scheduler: invalid completion %u\n", index);
DM_DRIVER_CONFIG* driver = &config->Drivers[index];
driver->State = KSUCCESS(worker->Status) ? DmRunStarted : DmRunFailed;
if (!KSUCCESS(worker->Status)) {
DbgPrint("driver: failed id=%s status=0x%x\n", driver->Id, worker->Status);
if (driver->Critical)
KePanic(KE_PANIC_CODE_INIT, "Critical driver %s failed (0x%x)\n",
driver->Id, worker->Status);
}
worker->Index = UINT32_MAX;
--active;
}
for (uint32_t i = 0; i < KI_DRIVER_START_WORKERS; ++i) {
KI_DRIVER_START_WORKER* worker = &workers[i];
if (NtlAtomicLoad32(&worker->State) != KiDriverWorkIdle) continue;
int selected = KiSelectEligibleDriver(config, phase);
if (selected < 0) break;
DM_DRIVER_CONFIG* driver = &config->Drivers[selected];
driver->State = DmRunStarting;
worker->Index = (uint32_t)selected;
NtlAtomicExchange32(&worker->State, KiDriverWorkQueued);
++active;
KeSignalEvent(worker->Wake);
}
if (active) {
status = KeWaitForEvent(completion, false);
if (!KSUCCESS(status)) KePanic(KE_PANIC_CODE_INIT, "Driver scheduler: wait failed\n");
continue;
}
bool changed = false;
uint32_t waiting = 0;
for (uint32_t i = 0; i < config->DriverCount; ++i) {
DM_DRIVER_CONFIG* d = &config->Drivers[i];
if (d->Phase != phase || d->State != DmRunWaiting) continue;
DM_DRIVER_CONFIG* driver = &config->Drivers[i];
if (driver->Phase != phase || driver->State != DmRunWaiting) continue;
++waiting;
if (DmDriverIsBlocked(config, i)) {
d->State = DmRunBlocked;
DbgPrint("Driver manager: blocked %s by unsatisfied dependency\n", d->Id);
if (d->Critical)
KePanic(KE_PANIC_CODE_INIT, "Critical driver %s blocked\n", d->Id);
changed = true;
}
if (!DmDriverIsBlocked(config, i)) continue;
driver->State = DmRunBlocked;
DbgPrint("driver: blocked id=%s dependency failed\n", driver->Id);
if (driver->Critical)
KePanic(KE_PANIC_CODE_INIT, "Critical driver %s blocked\n", driver->Id);
changed = true;
}
if (changed) continue;
if (waiting)
KePanic(KE_PANIC_CODE_INIT,
"Driver manager: no eligible drivers in phase %u (waiting=%u)\n",
"Driver manager: deadlocked phase=%u waiting=%u\n",
(unsigned)phase, waiting);
return;
break;
}
for (uint32_t i = 0; i < KI_DRIVER_START_WORKERS; ++i) {
if (NtlAtomicCompareExchange32(&workers[i].State,
KiDriverWorkExit, KiDriverWorkIdle) != KiDriverWorkIdle)
KePanic(KE_PANIC_CODE_INIT, "Driver scheduler: worker still active at shutdown\n");
KeSignalEvent(workers[i].Wake);
}
for (uint32_t i = 0; i < KI_DRIVER_START_WORKERS; ++i) {
status = KeWaitForEvent(workers[i].Thread->ExitEvent, false);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_INIT,
"Driver scheduler: worker %u exit wait failed (0x%x)\n", i, status);
if (NtlAtomicLoad32(&workers[i].State) != KiDriverWorkExited)
KePanic(KE_PANIC_CODE_INIT,
"Driver scheduler: worker %u exited without acknowledgment (state=%d)\n",
i, NtlAtomicLoad32(&workers[i].State));
ObDereferenceObject(&workers[i].Thread->Header);
ObDereferenceObject(&workers[i].Wake->Header);
}
ObDereferenceObject(&completion->Header);
MmFreePool(workers);
}
static DM_CONFIG* gDriverConfig;
@@ -359,7 +506,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
// Kernel-launched user images are rooted at the selected system volume.
s = KeOpenSystemRootFile(filePath, &f);
if (s != NTLSTATUS_SUCCESS) {
DbgPrint("Failed to load system file: \"%s\" (status %d)!\n", filePath, s);
DbgPrint("process: image load failed path=%s status=0x%x\n", filePath, s);
return 0;
}
@@ -376,13 +523,13 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
IoCloseFile(f);
return 0;
}
DbgPrint("Created process (PID %u) with command line: %s\n", proc->ProcessId, (args && args[0]) ? args : filePath);
DbgPrint("process: created pid=%u image=%s\n", proc->ProcessId, filePath);
s = KeSetupUserProcessEnvironment(proc,
SysSvFromCstr(filePath),
SysSvFromCstr(args));
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s);
DbgPrint("process: environment failed status=0x%x\n", s);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
@@ -461,7 +608,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
}
}
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to create bootstrap startup capabilities. Status: %d\n", s);
DbgPrint("process: bootstrap capabilities failed status=0x%x\n", s);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
@@ -476,7 +623,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
};
s = KeSetProcessStartupCapabilities(proc, startupCapabilities, 6);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to install startup capabilities. Status: %d\n", s);
DbgPrint("process: capability install failed status=0x%x\n", s);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
@@ -487,7 +634,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
s = LdrLoadImageFileForProcess(proc, f, &imageBase);
IoCloseFile(f);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: LdrLoadImageFileForProcess failed with status: %d\n", s);
DbgPrint("process: image map failed status=0x%x\n", s);
ObDereferenceObject(&proc->Header);
return 0;
}
@@ -501,7 +648,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
s = LdrProtectImageForProcess(proc, imageBase, true);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to protect EXE image for %s (status %d)\n", filePath, s);
DbgPrint("process: image protect failed path=%s status=0x%x\n", filePath, s);
ObDereferenceObject(&proc->Header);
return 0;
}
@@ -520,14 +667,12 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, &threadParameters, &thread);
if (!KSUCCESS(s)) {
KePowerSetInitProcess((PROCESS*)0);
DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
DbgPrint("process: thread creation failed status=0x%x\n", s);
proc->ImageBase = 0;
proc->ImageSize = 0;
ObDereferenceObject(&proc->Header);
return 0;
}
uint32_t tid = thread->ThreadId;
DbgPrint("KiLaunchUserProcess: Thread created and queued! TID=%u\n", tid);
ObDereferenceObject(&thread->Header);
// Caller owns the reference returned by KeCreateProcess.
@@ -548,19 +693,15 @@ static void KiFlushRegisteredDisks(const char* phase) {
DbgPrint("shutdown: %s disk enumeration failed 0x%x\n", phase, st);
}
// Implements the system Init Thread operation.
void SystemInitThread(void* arg) {
SYSTEM_INIT_ARGS* args = (SYSTEM_INIT_ARGS*)arg;
DbgPrint("SystemInitThread: Thread started. Igniting the system...\n");
KiInitializeDrivers(args->BootInfo, args->Hal);
KSTATUS powerStatus = KePowerInitialize();
if (!KSUCCESS(powerStatus))
KePanic(KE_PANIC_CODE_INIT, "Power coordinator failed (0x%x)\n", powerStatus);
KePowerInitializeButton(args->Hal);
KiInitializeDrivers(args->BootInfo, args->Hal);
KePowerInitializePolicy();
KeTimeReloadSettings();
KiInitializeSwapFromSettings();
@@ -571,7 +712,7 @@ void SystemInitThread(void* arg) {
KiRunKernelSelfTests();
// Launch the bootstrap service manager. Its allocated PID is not a system contract.
PROCESS* initProcess = KiLaunchUserProcess("system/init.exe", args->BootConfig.CommandLine);
PROCESS* initProcess = KiLaunchUserProcess("system/init.exe", "");
if (!initProcess) {
DbgPrint("SystemInitThread: Failed to start system/init.exe.\n");
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n");
@@ -588,7 +729,7 @@ void SystemInitThread(void* arg) {
if (exitCode != 0) {
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init.exe exited with code %d.\n", exitCode);
}
DbgPrint("SystemInitThread: power commit accepted; init exited with code %d.\n", exitCode);
DbgPrint("shutdown: init exit=%d\n", exitCode);
uint32_t action = KePowerCommittedAction();
if (!action) KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: missing power action\n");
KSTATUS flush = MmFileCacheFlushAll();
@@ -604,12 +745,19 @@ void SystemInitThread(void* arg) {
flush = MmFileCacheFlushAll();
if (!KSUCCESS(flush)) DbgPrint("shutdown: final cache flush status 0x%x\n", flush);
KiFlushRegisteredDisks("final");
// Firmware preparation runs while platform drivers can still execute methods.
KSTATUS firmwarePrepare = KePowerPrepareFirmware(action);
if (!KSUCCESS(firmwarePrepare) && firmwarePrepare != NTLSTATUS_NOT_SUPPORTED)
DbgPrint("shutdown: firmware prepare failed 0x%x; using HAL fallback\n", firmwarePrepare);
KSTATUS prepare = DmPrepareShutdown();
if (!KSUCCESS(prepare)) DbgPrint("shutdown: prepare status 0x%x\n", prepare);
KSTATUS pnpShutdown = IoPnpShutdown(10000u);
if (!KSUCCESS(pnpShutdown))
DbgPrint("shutdown: device stop status 0x%x\n", pnpShutdown);
DmCommitShutdown();
KSTATUS firmwareEnter = KePowerEnterFirmware(action);
if (firmwareEnter != NTLSTATUS_NOT_SUPPORTED)
DbgPrint("shutdown: firmware transition returned 0x%x; using HAL fallback\n", firmwareEnter);
if (action == SYS_POWER_ACTION_RESTART) {
if (!gKmainHal->Power.Reset)
KePanic(KE_PANIC_CODE_INIT, "shutdown: reset HAL callback unavailable\n");
@@ -741,9 +889,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeInitializeGdt();
KeInitializeIdt();
// Register boot modules (Allows LdrGetExport to work for drivers and the HAL)
// Note: Kept to a single call here to avoid the duplicate in your original code.
// Register boot exports before resolving driver imports.
KSTATUS moduleRegistryStatus = LdrRegisterModulesFromBootInfo(bootInfo);
if (!KSUCCESS(moduleRegistryStatus))
KeStopProcessor();
@@ -773,9 +919,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeStopProcessor();
IoInitializeDdk(hal, bootInfo->PhysicalMapBase);
KeFirmwareInitialize(bootInfo);
gKmainHal = hal;
KeInitDebug(hal, KSUCCESS(bootConfigStatus) ? bootConfig.Debug : 0);
KeFirmwareLog();
if (!KSUCCESS(bootConfigStatus)) {
KePanic(KE_PANIC_CODE_INIT,
"boot.ini: malformed or missing mandatory configuration, status=0x%x\n",
@@ -786,7 +934,6 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Bind IRQ dispatch while IF remains clear. Interrupt sources are armed
// later, after their kernel consumers have registered.
KeIrqInitialize(hal);
DbgPrint("CPU: PAT owns WB/WC/UC policy; MTRR default=WB fixed=off variable=off.\n");
KeTimeInitialize(bootInfo, hal);
KeDisplayInitialize(hal);
@@ -805,7 +952,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeInputInitialize(hal);
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.HandleInterrupt);
KeIrqRegisterLocalTimerHandler(hal->Timer.HandleInterrupt);
KeSyscallInit(hal);
@@ -822,7 +969,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Scheduler Prep & Status Update.
KeInitializeScheduler();
KeIrqRegister(hal->Timer.IrqNumber, KeSchedulerTick);
KeIrqRegisterLocalTimerHandler(KeSchedulerTick);
KeConfigureCurrentProcessorTimer();
KSTATUS volumeManagerStatus = IoInitializeVolumeManager();
@@ -873,7 +1020,6 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st);
}
DbgPrint("KernelMain: Handing over to scheduler...\n");
// The bootstrap CPU becomes the scheduler idle context once interrupts are enabled.
while (true)
code/kernel/ldr_image.c
+82 -58
View File
@@ -9,6 +9,7 @@
#include <kernel/io_file.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <ntlite/syscall.h>
#ifndef LDR_DEBUG_VERBOSE
@@ -89,12 +90,10 @@ typedef struct {
uint32_t Characteristics;
} SECTION_HEADER;
// Implements the align Pe Size operation.
static size_t KiAlignPeSize(size_t value) {
return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
}
// Implements the compute Required Image Size operation.
static KSTATUS KiComputeRequiredImageSize(const PE_HEADER* pe, size_t fileSize, size_t* outSize) {
if (!pe || !outSize) return NTLSTATUS_INVALID_PARAMETER;
@@ -166,15 +165,16 @@ typedef struct {
// Global kernel-module registry. Entries are allocated as modules arrive so boot-driver count is not an ABI limit.
#define LDR_KERNEL_MODULE_NAME_MAX 64u
typedef struct MODULE_ENTRY {
typedef struct LDR_KERNEL_MODULE {
char Name[LDR_KERNEL_MODULE_NAME_MAX];
uintptr_t Base;
size_t Size;
bool ExportsAvailable;
struct MODULE_ENTRY* Next;
} MODULE_ENTRY;
struct LDR_KERNEL_MODULE* Next;
} LDR_KERNEL_MODULE;
static MODULE_ENTRY* gModules;
static LDR_KERNEL_MODULE* gModules;
static SPINLOCK gModuleLock = SPINLOCK_INIT;
// Ensures a user image page is privately writable before the kernel patches it.
static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t address, size_t size) {
@@ -182,66 +182,82 @@ static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t add
return MmVadMakeImageRangePrivate(process, (void*)address, size);
}
// Allocates and inserts one global kernel-module registry entry.
// Reserves an exact module identity before exposing its exports.
static KSTATUS KiRegisterGlobalModule(const char* name, uintptr_t base, size_t size,
bool exportsAvailable) {
bool exportsAvailable, LDR_KERNEL_MODULE** out) {
if (!name || !*name || !base || !size)
return NTLSTATUS_INVALID_PARAMETER;
if (out) *out = (LDR_KERNEL_MODULE*)0;
size_t nameLength = 0;
while (nameLength < LDR_KERNEL_MODULE_NAME_MAX && name[nameLength]) nameLength++;
if (nameLength == 0 || nameLength >= LDR_KERNEL_MODULE_NAME_MAX ||
while (nameLength < LDR_KERNEL_MODULE_NAME_MAX && name[nameLength]) ++nameLength;
if (!nameLength || nameLength >= LDR_KERNEL_MODULE_NAME_MAX ||
!SysSvIsUtf8Text(SysSvFromParts(name, nameLength)))
return NTLSTATUS_INVALID_PARAMETER;
MODULE_ENTRY* entry = (MODULE_ENTRY*)MmAllocatePool(sizeof(MODULE_ENTRY));
if (!entry)
return NTLSTATUS_NO_MEMORY;
LDR_KERNEL_MODULE* entry = (LDR_KERNEL_MODULE*)MmAllocatePool(sizeof(*entry));
if (!entry) return NTLSTATUS_NO_MEMORY;
memset(entry, 0, sizeof(*entry));
memcpy(entry->Name, name, nameLength);
entry->Name[nameLength] = '\0';
entry->Base = base;
entry->Size = size;
entry->ExportsAvailable = exportsAvailable;
entry->Next = gModules;
gModules = entry;
return NTLSTATUS_SUCCESS;
}
// Registers the two fixed bootstrap images described explicitly by BOOT_INFO.
KSTATUS LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
if (!bootInfo)
return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = KiRegisterGlobalModule("kernel.exe",
bootInfo->KernelImage.VirtualBase,
bootInfo->KernelImage.ImageSize, true);
if (!KSUCCESS(status))
return status;
return KiRegisterGlobalModule("hal.dll",
bootInfo->HalImage.VirtualBase,
bootInfo->HalImage.ImageSize, true);
}
// Registers one loaded kernel module without imposing a fixed registry capacity.
KSTATUS LdrRegisterModule(const char* name, uintptr_t base) {
size_t size = base ? LdrGetImageSize(base) : 0;
return KiRegisterGlobalModule(name, base, size, false);
}
// Makes an initialized module's exports visible to future import resolution.
// Failed images remain mapped for diagnostics but must not provide exports.
KSTATUS LdrSetModuleExportsAvailable(const char* name, bool available) {
if (!name) return NTLSTATUS_INVALID_PARAMETER;
for (MODULE_ENTRY* entry = gModules; entry; entry = entry->Next) {
if (_stricmp(entry->Name, name) == 0) {
entry->ExportsAvailable = available;
return NTLSTATUS_SUCCESS;
KSTATUS status = NTLSTATUS_SUCCESS;
uintptr_t flags = KeAcquireSpinlock(&gModuleLock);
for (LDR_KERNEL_MODULE* it = gModules; it; it = it->Next) {
if (_stricmp(it->Name, name) == 0 || it->Base == base) {
status = NTLSTATUS_ALREADY_EXISTS;
break;
}
}
return NTLSTATUS_NOT_FOUND;
if (KSUCCESS(status)) {
entry->Next = gModules;
gModules = entry;
if (out) *out = entry;
}
KeReleaseSpinlock(&gModuleLock, flags);
if (!KSUCCESS(status)) MmFreePool(entry);
return status;
}
KSTATUS LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
if (!bootInfo) return NTLSTATUS_INVALID_PARAMETER;
KSTATUS status = KiRegisterGlobalModule("kernel.exe",
bootInfo->KernelImage.VirtualBase,
bootInfo->KernelImage.ImageSize, true, (LDR_KERNEL_MODULE**)0);
if (!KSUCCESS(status)) return status;
return KiRegisterGlobalModule("hal.dll", bootInfo->HalImage.VirtualBase,
bootInfo->HalImage.ImageSize, true, (LDR_KERNEL_MODULE**)0);
}
KSTATUS LdrRegisterModule(const char* name, uintptr_t base, LDR_KERNEL_MODULE** out) {
size_t size = base ? LdrGetImageSize(base) : 0;
if (!out) return NTLSTATUS_INVALID_PARAMETER;
return KiRegisterGlobalModule(name, base, size, false, out);
}
// Only the owner of the reservation can publish or discard it.
void LdrPublishModule(LDR_KERNEL_MODULE* module) {
uintptr_t flags = KeAcquireSpinlock(&gModuleLock);
module->ExportsAvailable = true;
KeReleaseSpinlock(&gModuleLock, flags);
}
void LdrDiscardModule(LDR_KERNEL_MODULE* module) {
if (!module) return;
uintptr_t flags = KeAcquireSpinlock(&gModuleLock);
LDR_KERNEL_MODULE** link = &gModules;
while (*link && *link != module) link = &(*link)->Next;
if (*link && !module->ExportsAvailable) *link = module->Next;
else module = (LDR_KERNEL_MODULE*)0;
KeReleaseSpinlock(&gModuleLock, flags);
if (module) MmFreePool(module);
}
void LdrReleaseKernelImage(uintptr_t imageBase) {
if (imageBase) MmFreePool((void*)imageBase);
}
// Finds the loaded kernel module that owns an address.
@@ -249,7 +265,9 @@ const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut,
if (baseOut) *baseOut = 0;
if (sizeOut) *sizeOut = 0;
for (MODULE_ENTRY* module = gModules; module; module = module->Next) {
uintptr_t flags = KeAcquireSpinlock(&gModuleLock);
for (LDR_KERNEL_MODULE* module = gModules; module; module = module->Next) {
if (!module->ExportsAvailable) continue;
uintptr_t base = module->Base;
size_t size = module->Size;
if (!base || !size || base > UINTPTR_MAX - size)
@@ -257,20 +275,28 @@ const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut,
if (address >= base && address < base + size) {
if (baseOut) *baseOut = base;
if (sizeOut) *sizeOut = size;
return module->Name;
const char* name = module->Name;
KeReleaseSpinlock(&gModuleLock, flags);
return name;
}
}
KeReleaseSpinlock(&gModuleLock, flags);
return "unknown";
}
// Returns the loaded global kernel module base for one exact module name.
static uintptr_t KiFindGlobalModuleBase(const char* name) {
if (!name) return 0;
for (MODULE_ENTRY* module = gModules; module; module = module->Next) {
if (module->ExportsAvailable && _stricmp(module->Name, name) == 0)
return module->Base;
uintptr_t base = 0;
uintptr_t flags = KeAcquireSpinlock(&gModuleLock);
for (LDR_KERNEL_MODULE* module = gModules; module; module = module->Next) {
if (module->ExportsAvailable && _stricmp(module->Name, name) == 0) {
base = module->Base;
break;
}
}
return 0;
KeReleaseSpinlock(&gModuleLock, flags);
return base;
}
// Returns export At Base.
@@ -313,7 +339,6 @@ void* LdrGetExport(const char* moduleName, const char* functionName) {
return KiGetExportAtBase(base, functionName);
}
// Implements the ldr Get Image Size operation.
size_t LdrGetImageSize(uintptr_t base) {
if (!base) return 0;
uint32_t peOffset = *(uint32_t*)(base + 0x3C);
@@ -322,7 +347,6 @@ size_t LdrGetImageSize(uintptr_t base) {
return (size_t)pe->OptionalHeader.SizeOfImage;
}
// Implements the ldr Get Mapped Image Size operation.
size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
if (!base) return 0;
if (process) {
code/kernel/mm_pool.c
-5
View File
@@ -67,19 +67,15 @@ _Static_assert((sizeof(BLOCK_FTR) % POOL_ALIGNMENT) == 0, "pool footer must pres
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
// Implements the hdr To Ftr operation.
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR*)((char*)h + HDR_SIZE + h->Size);
}
// Implements the ftr To Hdr operation.
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR*)((char*)f - HDR_SIZE - f->Size);
}
// Implements the next Block operation.
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR*)((char*)h + HDR_SIZE + h->Size + FTR_SIZE);
}
// Implements the prev Ftr operation.
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR*)((char*)h - FTR_SIZE);
}
@@ -385,7 +381,6 @@ static void KiFreeListRemoveLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller) {
blk->PrevFree = blk->NextFree = (BLOCK_HDR*)0;
}
// Implements the heap Try Decommit Tail operation.
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
if (!h || !blk || blk->Free == BLOCK_STATE_USED) return;
code/kernel/mm_virt.c
+1 -6
View File
@@ -7,6 +7,7 @@
// FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings.
// FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map.
// FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys.
// FFFF980000000000-FFFF9FFFFFFFFFFF is the generic device-MMIO aperture.
// FFFFA00000000000-FFFFAFFFFFFFFFFF is the kernel shared-view arena.
// FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool.
//
@@ -43,7 +44,6 @@ static KSTATUS KiCopyToUserPaged(void* userDst, const void* kernelSrc, size_t si
static PHYSICAL_ADDRESS gVadZeroPageFrame;
// User/kernel copy boundary helpers.
// Implements the user Range Valid operation.
static bool KiUserRangeValid(uintptr_t addr, size_t size) {
if (size == 0) return true;
if (addr < USER_BASE) return false;
@@ -2584,7 +2584,6 @@ size_t MmVadGetSizeByBase(const void* base) {
return 0;
}
// Implements the VAD Type Name operation.
static const char* KiVadTypeName(MM_VAD_TYPE type) {
switch (type) {
case MmVadGeneric: return "generic";
@@ -2600,7 +2599,6 @@ static const char* KiVadTypeName(MM_VAD_TYPE type) {
}
}
// Implements the debug Dump VADs operation.
void MmVadDebugDumpProcess(PROCESS* process) {
if (!process) {
DbgPrint("MM: VADs: <no process>\n");
@@ -2629,7 +2627,6 @@ void MmVadDebugDumpProcess(PROCESS* process) {
if (!count) DbgPrint(" <none>\n");
}
// Implements the debug Describe Address operation.
void MmDebugDescribeAddress(uintptr_t addr) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
@@ -2705,7 +2702,6 @@ __declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* pt
(void)MmVadDestroy(process, ptr);
}
// Implements the reallocate User Memory For Process operation.
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) {
if (!process) return (void*)0;
if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize);
@@ -2737,7 +2733,6 @@ __declspec(dllexport) void MmFreeUserMemory(void* ptr) {
MmFreeUserMemoryForProcess(process, ptr);
}
// Implements the reallocate User Memory operation.
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
PROCESS* process = KeGetCurrentProcess();
return MmReallocateUserMemoryForProcess(process, ptr, newSize);
code/kernel/net_device.c
-7
View File
@@ -15,7 +15,6 @@ static NET_DEVICE* gNetDevices[NET_MAX_DEVICES];
static uint32_t gNetDeviceCount = 0;
static SPINLOCK gNetLock = SPINLOCK_INIT;
// Implements the net Make Default Name operation.
static void NetMakeDefaultName(NET_DEVICE* device, uint32_t index) {
if (!device || device->Name[0])
return;
@@ -27,7 +26,6 @@ static void NetMakeDefaultName(NET_DEVICE* device, uint32_t index) {
device->Name[4] = '\0';
}
// Implements the net Is Device Registered Locked operation.
static bool NetIsDeviceRegisteredLocked(const NET_DEVICE* device) {
for (uint32_t i = 0; i < gNetDeviceCount; ++i) {
if (gNetDevices[i] == device)
@@ -36,12 +34,10 @@ static bool NetIsDeviceRegisteredLocked(const NET_DEVICE* device) {
return false;
}
// Implements the net Device Has Ethernet Address operation.
static bool NetDeviceHasEthernetAddress(const NET_DEVICE* device) {
return device && device->HardwareAddressLength == NET_ETHERNET_ADDRESS_LENGTH;
}
// Implements the net Log Hardware Address operation.
static void NetLogHardwareAddress(const NET_DEVICE* device) {
if (NetDeviceHasEthernetAddress(device)) {
DbgPrint("%02x:%02x:%02x:%02x:%02x:%02x",
@@ -53,7 +49,6 @@ static void NetLogHardwareAddress(const NET_DEVICE* device) {
}
}
// Implements the net Register Device operation.
__declspec(dllexport) KSTATUS NetRegisterDevice(NET_DEVICE* device) {
if (!device || !device->SendFrame)
return NTLSTATUS_INVALID_PARAMETER;
@@ -96,7 +91,6 @@ __declspec(dllexport) KSTATUS NetRegisterDevice(NET_DEVICE* device) {
return NTLSTATUS_SUCCESS;
}
// Implements the net Get Device Count operation.
__declspec(dllexport) uint32_t NetGetDeviceCount(void) {
uintptr_t flags = KeAcquireSpinlock(&gNetLock);
uint32_t count = gNetDeviceCount;
@@ -104,7 +98,6 @@ __declspec(dllexport) uint32_t NetGetDeviceCount(void) {
return count;
}
// Implements the net Get Device operation.
__declspec(dllexport) NET_DEVICE* NetGetDevice(uint32_t index) {
uintptr_t flags = KeAcquireSpinlock(&gNetLock);
NET_DEVICE* device = (index < gNetDeviceCount) ? gNetDevices[index] : (NET_DEVICE*)0;
code/kernel/net_stack.c
-3
View File
@@ -11,7 +11,6 @@
static const NET_STACK_PROVIDER* gNetStackProvider = 0;
static SPINLOCK gNetStackLock = SPINLOCK_INIT;
// Implements the net Validate Stack Provider operation.
static KSTATUS NetValidateStackProvider(const NET_STACK_PROVIDER* p) {
if (!p || p->Version != NET_STACK_PROVIDER_VERSION || !p->Name)
return NTLSTATUS_INVALID_PARAMETER;
@@ -26,7 +25,6 @@ static KSTATUS NetValidateStackProvider(const NET_STACK_PROVIDER* p) {
return NTLSTATUS_SUCCESS;
}
// Implements the net Register Stack Provider operation.
__declspec(dllexport) KSTATUS NetRegisterStackProvider(const NET_STACK_PROVIDER* provider) {
KSTATUS st = NetValidateStackProvider(provider);
if (!KSUCCESS(st))
@@ -59,7 +57,6 @@ __declspec(dllexport) KSTATUS NetRegisterStackProvider(const NET_STACK_PROVIDER*
return NTLSTATUS_SUCCESS;
}
// Implements the net Get Stack Provider operation.
__declspec(dllexport) const NET_STACK_PROVIDER* NetGetStackProvider(void) {
uintptr_t flags = KeAcquireSpinlock(&gNetStackLock);
const NET_STACK_PROVIDER* p = gNetStackProvider;