maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move keyboard and mouse out of HAL into a standalone ps2.sys driver, implement usb stack with xhci controller discovery, hid keyboard/mouse, and bulk-only mass storage, add pci msi support with capability chain walking and software irq vector allocation (slots 16-23), rewrite user virtual memory as a USER_VA_REGION descriptor arena with per-process ownership tracking and guard page support, replacing the old bump allocator and top-down image region, add MmCreateUserRegion/MmDestroyAllUserRegions so process teardown reclaims all user va in one pass, allocate a pool-backed shadow framebuffer in hal_screen to fix cursor tearing and scroll corruption, reserve ap trampoline page below 1mb in uefi bootloader before boot services exit, rewrite memset/memcpy/memmove/memcmp with __stosq/__movsq intrinsics, promote driver flag literals to macros and add multi-tu STANDALONE_DRIVERS table to build.c, split rtl8139/rtl816x into header+source, rename KbReadScancode to KbReadKey, improve page fault output with MmDebugDescribeAddress and user region dump, fix %p to %016llX throughout.

Browse Source
This commit is contained in:
maksw committed 2026-06-02 00:01:20 +02:00
1 parent b970d7f59c
commit b838f8d83d
130 files changed
+8137 -2932
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/hal/hal_apic.c
+76 -22
View File
@@ -16,11 +16,23 @@ static volatile uint32_t* gLapic = (volatile uint32_t *)0;
#define LAPIC_ICR_HI 0x310
#define LAPIC_ENABLE 0x100 // bit in spurious vector register
#define IA32_APIC_BASE 0x0000001Bu
#define IA32_APIC_BASE_ENABLE (1ULL << 11)
#define IA32_APIC_BASE_X2APIC (1ULL << 10)
#define IA32_X2APIC_ID 0x00000802u
#define IA32_X2APIC_EOI 0x0000080Bu
#define IA32_X2APIC_SVR 0x0000080Fu
#define IA32_X2APIC_ICR 0x00000830u
#define CPUID_FEATURE_ECX_X2APIC (1u << 21)
#define APIC_VECTOR_RESCHEDULE 0xF0u
// ---- I/O APIC ----
static volatile uint32_t* gIoapic = (volatile uint32_t *)0;
static uint32_t gBspLapicId = 0;
static uint8_t gX2ApicSupported = 0;
static uint8_t gX2ApicEnabled = 0;
#define IOAPIC_IDX 0x00
#define IOAPIC_DATA 0x10
@@ -33,6 +45,14 @@ static void IoapicWrite(uint8_t reg, uint32_t val) {
*(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val;
}
static void HalApicDetectMode(void) {
int regs[4] = {0, 0, 0, 0};
__cpuid(regs, 1);
gX2ApicSupported = (regs[2] & CPUID_FEATURE_ECX_X2APIC) ? 1u : 0u;
uint64_t apicBase = __readmsr(IA32_APIC_BASE);
gX2ApicEnabled = (gX2ApicSupported && (apicBase & IA32_APIC_BASE_X2APIC)) ? 1u : 0u;
}
static void HalApicStallFallback(uint32_t loops) {
for (volatile uint32_t i = 0; i < loops; i++) {
@@ -46,6 +66,9 @@ static void HalApicStallUs(uint32_t us) {
}
static void HalApicWaitIcrIdle(void) {
if (gX2ApicEnabled)
return;
for (uint32_t i = 0; i < 1000000u; i++) {
if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12)))
return;
@@ -54,6 +77,11 @@ static void HalApicWaitIcrIdle(void) {
}
static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_ICR, ((uint64_t)lapicId << 32) | low);
return;
}
HalApicWaitIcrIdle();
LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24;
LAPIC_REG(LAPIC_ICR_LO) = low;
@@ -63,52 +91,70 @@ static void HalApicSendIpi(uint32_t lapicId, uint32_t low) {
// IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high).
#define IOREDTBL_LO(n) (0x10 + 2*(n))
#define IOREDTBL_HI(n) (0x11 + 2*(n))
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> 0x21, ...
// Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> vector 0x21, ...
#define IRQ_VECTOR_BASE 0x20
__declspec(dllexport) uint32_t HalGetBspLapicId(void) {
return gBspLapicId;
}
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
if (!gLapic) return;
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void) {
return gX2ApicEnabled;
}
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId) {
if (gX2ApicEnabled)
return 1u;
return (lapicId < 256u) ? 1u : 0u;
}
__declspec(dllexport) void HalApicEnableCurrentCpu(void) {
uint64_t apicBase = __readmsr(IA32_APIC_BASE) | IA32_APIC_BASE_ENABLE;
if (gX2ApicEnabled) {
apicBase |= IA32_APIC_BASE_X2APIC;
__writemsr(IA32_APIC_BASE, apicBase);
__writemsr(IA32_X2APIC_SVR, LAPIC_ENABLE | 0xFFu);
return;
}
__writemsr(IA32_APIC_BASE, apicBase);
if (gLapic)
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
}
__declspec(dllexport) void HalApicSendPanicIpi(void) {
if (!gLapic) return;
if (!gLapic && !gX2ApicEnabled) return;
// Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed
// Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed
// vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still
// enter the panic path. The NMI vector is already installed in the IDT on
// enter the panic path. The NMI vector is already installed in the IDT on
// every CPU that has reached kernel-owned state.
HalApicWaitIcrIdle();
LAPIC_REG(LAPIC_ICR_HI) = 0;
LAPIC_REG(LAPIC_ICR_LO) = (3u << 18) | (4u << 8); // all-excluding-self, NMI
HalApicWaitIcrIdle();
HalApicSendIpi(0, (3u << 18) | (4u << 8)); // all-excluding-self, NMI
}
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) {
if (!gLapic) return;
if (!gLapic && !gX2ApicEnabled) return;
HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination
}
__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) {
if (!gLapic) return;
if (!gLapic && !gX2ApicEnabled) return;
// TLB shootdown must reach CPUs that may be spinning with IF=0 inside
// high-IRQL/spinlock regions, so use NMI delivery rather than a fixed
// vector. The NMI stub handles active shootdowns and returns; unexpected
// vector. The NMI stub handles active shootdowns and returns; unexpected
// NMIs still go through the normal fault/panic path.
HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination
}
__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) {
if (!gLapic || !startupVector || lapicId == gBspLapicId)
if ((!gLapic && !gX2ApicEnabled) || !startupVector || lapicId == gBspLapicId)
return STATUS_INVALID_PARAMETER;
if (!HalApicCanAddressId(lapicId))
return STATUS_NOT_SUPPORTED;
// INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12.
// INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12.
HalApicSendIpi(lapicId, 0x0000C500u); // INIT, level assert, level-triggered
HalApicStallUs(10000);
HalApicSendIpi(lapicId, 0x00008500u); // INIT, level deassert
@@ -124,7 +170,7 @@ void HalApicRouteIrq(uint8_t irq) {
// Full initial setup of the redirection entry.
// Sets the vector and clears the mask bit in one write.
uint32_t vector = IRQ_VECTOR_BASE + irq; // e.g. IRQ 1 -> vector 0x21
IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24); // external IRQs target the BSP
IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24); // external IRQs target the BSP
IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed
}
@@ -138,29 +184,37 @@ void HalApicUnmaskIrq(uint8_t irq) {
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
lo &= ~(1 << 16); // clear mask bit
lo = (lo & ~0xFF) | (IRQ_VECTOR_BASE + irq);
IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24);
IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24);
IoapicWrite(IOREDTBL_LO(irq), lo);
}
void HalApicSendEoi(void) {
if (gX2ApicEnabled) {
__writemsr(IA32_X2APIC_EOI, 0);
return;
}
LAPIC_REG(LAPIC_EOI) = 0;
}
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
HalApicDetectMode();
gLapic = (volatile uint32_t *)(gApi.PhysToVirt(lapicPhys));
gIoapic = (volatile uint32_t *)(gApi.PhysToVirt(ioapicPhys));
// The BSP is the CPU executing HAL init. Keep all external/device IRQs
// routed to this LAPIC; CPU-local IPIs use the local APIC directly.
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
if (gX2ApicEnabled)
gBspLapicId = (uint32_t)__readmsr(IA32_X2APIC_ID);
else
gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF;
// Enable Local APIC, set spurious vector to 0xFF.
LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF;
HalApicEnableCurrentCpu();
// Mask all IOAPIC lines initially.
uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
for (uint8_t i = 0; i <= maxRedir; i++) {
IoapicWrite(IOREDTBL_LO(i), (1 << 16) | (IRQ_VECTOR_BASE + i));
IoapicWrite(IOREDTBL_HI(i), gBspLapicId << 24);
IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24);
}
}
code/kernel/hal/hal_keyboard.c
+1 -204
View File
@@ -1,204 +1 @@
// hal_keyboard.c - PS/2 keyboard, interrupt-driven, with live state tracking
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
// -- Port addresses ------------------------------------------------------------
#define PS2_DATA 0x60u
#define PS2_STAT 0x64u
#define PS2_CMD 0x64u
// -- Ring buffer (for ASCII/typing) -------------------------------------------
#define KBD_BUF_SIZE 256u
#define KBD_BUF_MASK (KBD_BUF_SIZE - 1u)
static uint8_t gBuf[KBD_BUF_SIZE];
static volatile int gHead = 0;
static volatile int gTail = 0;
// -- Deferred callback queue ---------------------------------------------------
// The hard IRQ path updates the key state and stores completed make codes.
// Optional client callbacks are delivered later from the kernel-queued DPC,
// never directly from DIRQL/ISR context.
#define KBD_DPC_BUF_SIZE 128u
#define KBD_DPC_BUF_MASK (KBD_DPC_BUF_SIZE - 1u)
static uint8_t gDpcBuf[KBD_DPC_BUF_SIZE];
static volatile int gDpcHead = 0;
static volatile int gDpcTail = 0;
// -- Live Key State (for GetAsyncKeyState) ------------------------------------
// These arrays track whether a key is currently held down.
static volatile uint8_t gKeyState[128]; // Standard scan codes
static volatile uint8_t gKeyStateExt[128]; // Extended (E0) scan codes
// -- Scan-code set 1 constants -------------------------------------------------
#define SC1_BREAK_BIT 0x80u
#define SC1_MAKE_MASK 0x7Fu
#define SC1_EXT_PREFIX 0xE0u
// -- Modifier flags ------------------------------------------------------------
#define MOD_LSHIFT (1u << 0)
#define MOD_RSHIFT (1u << 1)
#define MOD_LCTRL (1u << 2)
#define MOD_RCTRL (1u << 3)
#define MOD_LALT (1u << 4)
#define MOD_RALT (1u << 5)
#define MOD_CAPS (1u << 6)
#define MOD_NUMLOCK (1u << 7)
#define MOD_SHIFT (MOD_LSHIFT | MOD_RSHIFT)
#define MOD_CTRL (MOD_LCTRL | MOD_RCTRL)
#define MOD_ALT (MOD_LALT | MOD_RALT)
static volatile uint32_t gModifiers = 0;
static volatile bool gNextExtended = false;
static void (* volatile gCallback)(uint8_t) = NULL;
// -- ASCII Translation Tables --------------------------------------------------
static const char gUnshifted[128] = {
0, 0, '1','2','3','4','5','6','7','8','9','0','-','=','\b','\t',
'q','w','e','r','t','y','u','i','o','p','[',']','\n', 0, 'a','s',
'd','f','g','h','j','k','l',';','\'','`', 0, '\\','z','x','c','v',
'b','n','m',',','.','/', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.'
};
static const char gShifted[128] = {
0, 0, '!','@','#','$','%','^','&','*','(',')','_','+','\b','\t',
'Q','W','E','R','T','Y','U','I','O','P','{','}','\n', 0, 'A','S',
'D','F','G','H','J','K','L',':','"','~', 0, '|','Z','X','C','V',
'B','N','M','<','>','?', 0, '*', 0, ' ', 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, '7','8','9','-', '4','5','6','+', '1','2','3','0','.'
};
// -- Deferred callback delivery ------------------------------------------------
static void KbdDpc(void) {
for (;;) {
if (gDpcHead == gDpcTail)
break;
uint8_t make = gDpcBuf[gDpcHead];
gDpcHead = (gDpcHead + 1) & (int)KBD_DPC_BUF_MASK;
void (*cb)(uint8_t) = (void (*)(uint8_t))gCallback;
if (cb)
cb(make);
}
}
// -- ISR: The Engine -----------------------------------------------------------
void KbdIsr(void) {
uint8_t sc = __inbyte(PS2_DATA);
if (sc == SC1_EXT_PREFIX) {
gNextExtended = true;
return;
}
bool extended = gNextExtended;
bool is_break = (sc & SC1_BREAK_BIT) != 0;
uint8_t make = sc & SC1_MAKE_MASK;
gNextExtended = false;
// Update the live state arrays
if (extended) {
gKeyStateExt[make] = is_break ? 0 : 1;
} else {
gKeyState[make] = is_break ? 0 : 1;
}
// Update modifiers
if (!extended) {
switch (make) {
case 0x2A: gModifiers = is_break ? (gModifiers & ~MOD_LSHIFT) : (gModifiers | MOD_LSHIFT); break;
case 0x36: gModifiers = is_break ? (gModifiers & ~MOD_RSHIFT) : (gModifiers | MOD_RSHIFT); break;
case 0x1D: gModifiers = is_break ? (gModifiers & ~MOD_LCTRL) : (gModifiers | MOD_LCTRL); break;
case 0x38: gModifiers = is_break ? (gModifiers & ~MOD_LALT) : (gModifiers | MOD_LALT); break;
case 0x3A: if (!is_break) gModifiers ^= MOD_CAPS; break;
case 0x45: if (!is_break) gModifiers ^= MOD_NUMLOCK; break;
}
} else {
switch (make) {
case 0x1D: gModifiers = is_break ? (gModifiers & ~MOD_RCTRL) : (gModifiers | MOD_RCTRL); break;
case 0x38: gModifiers = is_break ? (gModifiers & ~MOD_RALT) : (gModifiers | MOD_RALT); break;
}
}
// Buffer management (only for 'make' codes)
if (!is_break) {
int next = (gTail + 1) & (int)KBD_BUF_MASK;
if (next != gHead) {
gBuf[gTail] = make;
gTail = next;
}
// Queue optional callback delivery to the DPC path. Keep a separate
// callback queue so the public ReadScancode ring remains independent.
if (gCallback) {
int dpcNext = (gDpcTail + 1) & (int)KBD_DPC_BUF_MASK;
if (dpcNext != gDpcHead) {
gDpcBuf[gDpcTail] = make;
gDpcTail = dpcNext;
}
}
}
}
// -- Public API ----------------------------------------------------------------
// Use this inside your GetAsyncKeyState implementation!
bool KbdIsKeyDown(uint8_t scancode, bool extended) {
if (extended) return gKeyStateExt[scancode & 0x7F] != 0;
return gKeyState[scancode & 0x7F] != 0;
}
static uint8_t KbdPoll(void) {
return (gHead == gTail) ? 0 : gBuf[gHead];
}
static uint8_t KbdReadScancode(void) {
while (gHead == gTail) { _mm_pause(); }
uint8_t sc = gBuf[gHead];
gHead = (gHead + 1) & (int)KBD_BUF_MASK;
return sc;
}
static char KbdReadChar(void) {
uint8_t make = KbdReadScancode();
if (make >= 128) return 0;
uint32_t mods = gModifiers;
bool shifted = (mods & MOD_SHIFT) != 0;
if (mods & MOD_CAPS) {
char base = gUnshifted[make];
if (base >= 'a' && base <= 'z') shifted = !shifted;
}
return shifted ? gShifted[make] : gUnshifted[make];
}
static uint32_t KbdGetModifiers(void) {
return gModifiers;
}
static void KbdSetCallback(void (*cb)(uint8_t scancode)) {
gCallback = cb;
}
void HalKeyboardInitialize(HAL_KEYBOARD *out) {
while (__inbyte(PS2_STAT) & 2);
__outbyte(PS2_CMD, 0xA8);
while (__inbyte(PS2_STAT) & 1) { __inbyte(PS2_DATA); }
out->Poll = KbdPoll;
out->ReadScancode = KbdReadScancode;
out->ReadChar = KbdReadChar;
out->GetModifiers = KbdGetModifiers;
out->SetCallback = KbdSetCallback;
out->IsKeyDown = KbdIsKeyDown;
out->Isr = KbdIsr;
out->Dpc = KbdDpc;
out->IrqNumber = 1;
}
// hal_keyboard.c - input moved out of HAL into kernel input core and ps2.sys.
code/kernel/hal/hal_main.c
+8 -10
View File
@@ -5,12 +5,11 @@
#include <stdint.h>
#include <stdarg.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <common/printf.h>
// Declared in other hal_*.c files:
void HalScreenInitialize(uintptr_t fb, int w, int h, int stride, HAL_SCREEN* out);
void HalKeyboardInitialize(HAL_KEYBOARD* out);
void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH);
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out);
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out);
void HalPciInitialize(HAL_PCI* out);
extern void HalSetDebugEnabled(int v);
@@ -242,7 +241,7 @@ static void ParseMadt(MADT* madt) {
uint32_t lapicId = *(uint32_t *)(p + 4);
uint32_t flags = *(uint32_t *)(p + 8);
uint32_t uid = *(uint32_t *)(p + 12);
if ((flags & 1u) && lapicId < 256u)
if (flags & 1u)
HalAddCpuRecord(uid, lapicId, flags);
}
@@ -395,7 +394,7 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
debugEnabled = initParams->Debug;
// Screen first so DbgPrint works immediately.
HalScreenInitialize(initParams->FramebufferBase + initParams->PhysicalWindowBase,
HalScreenInitialize(initParams->AllocatePool, initParams->FramebufferBase + initParams->PhysicalWindowBase,
(int)initParams->FramebufferWidth,
(int)initParams->FramebufferHeight,
(int)initParams->FramebufferStride,
@@ -412,10 +411,9 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
gLapicPhys, gIoapicPhys, gHpetPhys);
HalApicInitialize(gLapicPhys, gIoapicPhys);
HalKeyboardInitialize(&gApi.Keyboard);
HalMouseInitialize(&gApi.Mouse,
(int)initParams->FramebufferWidth,
(int)initParams->FramebufferHeight);
HalDbgPrint("HAL: local APIC mode=%s BSP_ID=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
HalGetBspLapicId());
KSTATUS ts = HalTimerInitialize(gLapicPhys, gHpetPhys, &gApi.Timer);
if (!KSUCCESS(ts)) HalDbgPrint("HAL: HPET unavailable, using fallback\n");
code/kernel/hal/hal_mouse.c
+1 -305
View File
@@ -1,305 +1 @@
// hal_mouse.c - PS/2 auxiliary mouse driver
//
// Sits on IRQ 12, sharing the 8042 controller with the keyboard (IRQ 1).
// hal_keyboard.c has already issued command 0xA8 (enable aux port) by the
// time HalMouseInitialize is called, so we only need to:
// 1. Enable aux interrupts in the 8042 command byte (bit 1, clears bit 5).
// 2. Send the standard mouse init sequence (reset → defaults → enable).
// 3. Wire up the ISR and expose the HAL_MOUSE vtable.
//
// Packet format - standard 3-byte PS/2 mouse (no scroll wheel):
// Byte 0: [YOV][XOV][YS][XS][1][M][R][L]
// Byte 1: delta-X unsigned magnitude (sign in byte-0 bit 4)
// Byte 2: delta-Y unsigned magnitude (sign in byte-0 bit 5); PS/2 Y+ = up
//
// The ISR accumulates the 3-byte packet, converts it to a screen-space delta
// (Y-axis flipped), and queues the completed packet to the HAL's kernel-queued
// bottom half. Cursor drawing, public event enqueue, and optional callbacks
// run from that bottom half instead of hard IRQ context.
//
// Threading model
// ---------------
// Public events are produced by the mouse DPC and consumed by callers.
// Raw completed packets are produced by the ISR and consumed by the same DPC.
// On SMP, add acquire/release barriers around the ring head/tail pairs.
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_debug.h>
// -- PS/2 port addresses -------------------------------------------------------
#define PS2_DATA 0x60u
#define PS2_STAT 0x64u
#define PS2_CMD 0x64u
// -- Ring buffer ---------------------------------------------------------------
#define MOUSE_BUF_SIZE 64u
#define MOUSE_BUF_MASK (MOUSE_BUF_SIZE - 1u)
typedef struct {
int dx, dy;
uint8_t buttons; // bit0=left, bit1=right, bit2=middle
} MOUSE_EVENT;
static MOUSE_EVENT gBuf[MOUSE_BUF_SIZE];
static volatile int gHead = 0; // public consumer read pointer
static volatile int gTail = 0; // public producer write pointer (updated by DPC)
// Completed hardware packets waiting for the DPC bottom half.
static MOUSE_EVENT gDpcBuf[MOUSE_BUF_SIZE];
static volatile int gDpcHead = 0;
static volatile int gDpcTail = 0;
// -- Absolute cursor position & screen bounds ----------------------------------
static volatile int gMouseX = 0;
static volatile int gMouseY = 0;
static volatile int gBoundsW = 1920;
static volatile int gBoundsH = 1080;
// -- 3-byte packet assembly ----------------------------------------------------
static volatile uint8_t gPacket[3];
static volatile int gPacketIdx = 0; // 0, 1, or 2
// -- Optional callback (called from mouse DPC) ---------------------------------
static void (* volatile gCallback)(int dx, int dy, uint8_t buttons) = NULL;
// -- Cursor helpers provided by hal_screen.c -----------------------------------
// Declared here; defined in hal_screen.c.
extern void HalScreenDrawCursor(int x, int y);
extern void HalScreenEraseCursor(void);
// -- 8042 low-level helpers ----------------------------------------------------
// Wait until the 8042's input buffer is empty (bit 1 of status = 0) so we
// can safely write to PS2_DATA or PS2_CMD.
static void Ps2WaitInput(void) {
int timeout = 100000;
while ((__inbyte(PS2_STAT) & 0x02u) && --timeout) { /* spin */ }
}
// Wait until the 8042's output buffer has data (bit 0 of status = 1).
static void Ps2WaitOutput(void) {
int timeout = 5000000;
while (!(__inbyte(PS2_STAT) & 0x01u) && --timeout) { /* spin */ }
}
// Reads from PS2_DATA until the output buffer is empty
static void Ps2FlushOutput(void) {
int timeout = 100000;
while ((__inbyte(PS2_STAT) & 0x01u) && --timeout) {
__inbyte(PS2_DATA);
}
}
// Route the next write to the auxiliary (mouse) device rather than the keyboard.
static void MouseSendByte(uint8_t byte) {
Ps2WaitInput();
__outbyte(PS2_CMD, 0xD4u); // "write to mouse" prefix
Ps2WaitInput();
__outbyte(PS2_DATA, byte);
}
// Read one byte from the PS/2 output buffer (shared by keyboard and mouse).
static uint8_t MouseRecvByte(void) {
Ps2WaitOutput();
return __inbyte(PS2_DATA);
}
// -- Deferred mouse bottom half -------------------------------------------------
static void MouseDpc(void) {
for (;;) {
if (gDpcHead == gDpcTail)
break;
MOUSE_EVENT ev = gDpcBuf[gDpcHead];
gDpcHead = (gDpcHead + 1) & (int)MOUSE_BUF_MASK;
// Update absolute position and redraw the cursor outside DIRQL.
HalScreenEraseCursor();
int nx = gMouseX + ev.dx;
int ny = gMouseY + ev.dy;
if (nx < 0) nx = 0;
if (nx >= gBoundsW) nx = gBoundsW - 1;
if (ny < 0) ny = 0;
if (ny >= gBoundsH) ny = gBoundsH - 1;
gMouseX = nx;
gMouseY = ny;
HalScreenDrawCursor(nx, ny);
// Publish event to the normal consumer ring.
int next = (gTail + 1) & (int)MOUSE_BUF_MASK;
if (next != gHead) {
gBuf[gTail] = ev;
gTail = next;
}
void (*cb)(int, int, uint8_t) = (void (*)(int, int, uint8_t))gCallback;
if (cb)
cb(ev.dx, ev.dy, ev.buttons);
}
}
// -- Interrupt service routine --------------------------------------------------
// Called with interrupts disabled by KeIrqDispatch (IRQ 12).
// The dispatcher sends the APIC EOI after this function returns.
static void MouseIsr(void) {
uint8_t data = __inbyte(PS2_DATA);
// Synchronisation guard: the first byte of every packet has bit 3 set.
// If we lose sync, wait for the next valid first byte before assembling.
if (gPacketIdx == 0) {
if (!(data & 0x08u)) return; // not a valid first byte - discard
}
gPacket[gPacketIdx++] = data;
if (gPacketIdx < 3) return; // packet not yet complete
gPacketIdx = 0;
// -- Decode flags ----------------------------------------------------------
uint8_t flags = gPacket[0];
// Overflow bits: if either is set the movement data is meaningless.
if (flags & 0xC0u) return;
uint8_t buttons = flags & 0x07u;
// Convert unsigned magnitudes + sign bits to signed 9-bit values.
int dx = (int)gPacket[1] - ((flags & 0x10u) ? 256 : 0);
int dy = (int)gPacket[2] - ((flags & 0x20u) ? 256 : 0);
dy = -dy; // PS/2 Y+ = up; screen Y+ = down, so invert.
// -- Queue completed packet to the DPC bottom half -------------------------
int next = (gDpcTail + 1) & (int)MOUSE_BUF_MASK;
if (next != gDpcHead) {
gDpcBuf[gDpcTail].dx = dx;
gDpcBuf[gDpcTail].dy = dy;
gDpcBuf[gDpcTail].buttons = buttons;
gDpcTail = next;
}
}
// -- Consumer API --------------------------------------------------------------
static bool MousePoll(void) {
return gHead != gTail;
}
static bool MouseRead(int* dx, int* dy, uint8_t* buttons) {
if (gHead == gTail) return false;
*dx = gBuf[gHead].dx;
*dy = gBuf[gHead].dy;
*buttons = gBuf[gHead].buttons;
gHead = (gHead + 1) & (int)MOUSE_BUF_MASK;
return true;
}
static void MouseGetPosition(int* x, int* y) {
*x = gMouseX;
*y = gMouseY;
}
static void MouseSetPosition(int x, int y) {
gMouseX = x;
gMouseY = y;
}
static void MouseSetBounds(int w, int h) {
gBoundsW = w;
gBoundsH = h;
}
static void MouseSetCallback(void (*cb)(int dx, int dy, uint8_t buttons)) {
gCallback = cb;
}
// -- Initializer ---------------------------------------------------------------
// Called from HalInitialize in hal_main.c after the screen is ready.
// hal_keyboard.c has already sent 0xA8 (enable aux port) to port 0x64.
void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH) {
gBoundsW = screenW;
gBoundsH = screenH;
gMouseX = screenW / 2;
gMouseY = screenH / 2;
// -----------------------------------------------------------------
// Step 0: Deafen the controller and flush
// -----------------------------------------------------------------
Ps2WaitInput();
__outbyte(PS2_CMD, 0xA7u); // Disable Aux port (ignores mouse)
Ps2WaitInput();
__outbyte(PS2_CMD, 0xADu); // Disable Main port (ignores keyboard)
// Now flush the buffer. Because the ports are disabled, we guarantee
// no new data can sneak in behind our backs.
Ps2FlushOutput();
// -----------------------------------------------------------------
// Step 1: Safely configure the 8042 Command Byte
// -----------------------------------------------------------------
Ps2WaitInput();
__outbyte(PS2_CMD, 0x20u); // read current command byte
Ps2WaitOutput();
uint8_t cmd = __inbyte(PS2_DATA);
cmd |= 0x02u; // enable aux interrupt
cmd &= ~0x20u; // clear "disable aux clock"
Ps2WaitInput();
__outbyte(PS2_CMD, 0x60u); // write command byte
Ps2WaitInput();
__outbyte(PS2_DATA, cmd);
// Re-enable both ports now that config is safely written
Ps2WaitInput();
__outbyte(PS2_CMD, 0xA8u); // Enable Aux port
Ps2WaitInput();
__outbyte(PS2_CMD, 0xAEu); // Enable Main port
// -----------------------------------------------------------------
// Step 2: Reset mouse (0xFF).
// -----------------------------------------------------------------
MouseSendByte(0xFFu); // reset
// The BIOS or an earlier phase might have left a movement packet
// partially in flight right as we re-enabled the port.
// We loop and discard anything that isn't the 0xFA ACK.
uint8_t ack;
int retries = 10;
do {
ack = MouseRecvByte();
} while (ack != 0xFAu && --retries > 0);
// The output timeout covers the physical hardware self-test latency.
MouseRecvByte(); // 0xAA - BAT passed
MouseRecvByte(); // 0x00 - device ID
// -----------------------------------------------------------------
// Step 3: Set default parameters (sampling rate 100, 4 counts/mm).
// Mouse replies: ACK.
// -----------------------------------------------------------------
MouseSendByte(0xF6u); // set defaults
MouseRecvByte(); // ACK
// -----------------------------------------------------------------
// Step 4: Enable data reporting.
// Mouse will now send 3-byte packets on every movement/button event.
// -----------------------------------------------------------------
MouseSendByte(0xF4u); // enable data reporting
MouseRecvByte(); // ACK
// Draw the initial cursor at the centre of the screen.
HalScreenDrawCursor(gMouseX, gMouseY);
// Populate vtable.
out->Poll = MousePoll;
out->Read = MouseRead;
out->SetCallback = MouseSetCallback;
out->GetPosition = MouseGetPosition;
out->SetPosition = MouseSetPosition;
out->SetBounds = MouseSetBounds;
out->Isr = MouseIsr;
out->Dpc = MouseDpc;
out->IrqNumber = 12;
}
// hal_mouse.c - input moved out of HAL into kernel input core and ps2.sys.
code/kernel/hal/hal_pci.c
+108
View File
@@ -1,10 +1,27 @@
// hal_pci.c - PCI configuration space via Mechanism 1 port I/O
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_debug.h>
#define PCI_CONFIG_ADDR 0xCF8
#define PCI_CONFIG_DATA 0xCFC
#define PCI_REG_COMMAND_STATUS 0x04u
#define PCI_REG_CAP_PTR 0x34u
#define PCI_STATUS_CAP_LIST (1u << 20)
#define PCI_COMMAND_INTX_DISABLE (1u << 10)
#define PCI_CAP_ID_MSI 0x05u
#define PCI_CAP_ID_MSIX 0x11u
#define PCI_MSI_CTRL_ENABLE (1u << 0)
#define PCI_MSI_CTRL_MME_MASK (7u << 4)
#define PCI_MSI_CTRL_64BIT (1u << 7)
#define PCI_MSI_CTRL_PVM (1u << 8)
#define PCI_MSI_ADDRESS_BASE 0xFEE00000u
#define PCI_IRQ_VECTOR_BASE 0x20u
static uint32_t MakeAddr(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) {
return 0x80000000UL
| ((uint32_t)bus << 16)
@@ -24,6 +41,25 @@ static void PciWrite32(uint8_t bus, uint8_t dev, uint8_t fn,
__outdword(PCI_CONFIG_DATA, val);
}
static uint8_t PciRead8(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) {
uint32_t v = PciRead32(bus, dev, fn, reg);
return (uint8_t)(v >> ((reg & 3u) * 8u));
}
static uint16_t PciRead16(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg) {
uint32_t v = PciRead32(bus, dev, fn, reg);
return (uint16_t)(v >> ((reg & 2u) * 8u));
}
static void PciWrite16(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t reg, uint16_t value) {
uint8_t aligned = reg & ~3u;
uint32_t shift = (reg & 2u) * 8u;
uint32_t old = PciRead32(bus, dev, fn, aligned);
old &= ~(0xFFFFu << shift);
old |= ((uint32_t)value << shift);
PciWrite32(bus, dev, fn, aligned, old);
}
static bool PciFindDevice(uint16_t vendor, uint16_t device,
uint8_t* outBus, uint8_t* outDev, uint8_t* outFn) {
for (int bus = 0; bus < 256; bus++)
@@ -61,10 +97,82 @@ static uint8_t PciGetIrqLine(uint8_t bus, uint8_t dev, uint8_t fn) {
return (uint8_t)(val & 0xFF); // 0xFF = not connected
}
static uint8_t PciFindCapability(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t capId) {
uint32_t status = PciRead32(bus, dev, fn, PCI_REG_COMMAND_STATUS);
if ((status & PCI_STATUS_CAP_LIST) == 0)
return 0;
uint8_t cap = PciRead8(bus, dev, fn, PCI_REG_CAP_PTR) & 0xFCu;
for (uint32_t guard = 0; cap && guard < 48u; guard++) {
uint32_t hdr = PciRead32(bus, dev, fn, cap);
uint8_t id = (uint8_t)(hdr & 0xFFu);
uint8_t next = (uint8_t)((hdr >> 8) & 0xFCu);
if (id == capId)
return cap;
cap = next;
}
return 0;
}
static KSTATUS PciEnableMsi(uint8_t bus, uint8_t dev, uint8_t fn, uint8_t irqSlot) {
if (irqSlot >= 24u)
return STATUS_INVALID_PARAMETER;
uint8_t cap = PciFindCapability(bus, dev, fn, PCI_CAP_ID_MSI);
if (!cap) {
HalDbgPrint("PCI %u:%u.%u: MSI capability not present\n", bus, dev, fn);
return STATUS_NOT_SUPPORTED;
}
uint32_t bspLapic = HalGetBspLapicId();
if (!HalApicUsingX2Apic() && bspLapic >= 256u)
return STATUS_NOT_SUPPORTED;
if (HalApicUsingX2Apic() && bspLapic >= 256u) {
// Extended x2APIC MSI destinations need interrupt-remapping awareness.
// Keep the first MSI implementation xAPIC-compatible and explicit.
HalDbgPrint("PCI %u:%u.%u: MSI x2APIC destination %u not supported yet\n",
bus, dev, fn, bspLapic);
return STATUS_NOT_SUPPORTED;
}
uint16_t control = PciRead16(bus, dev, fn, (uint8_t)(cap + 2u));
bool is64 = (control & PCI_MSI_CTRL_64BIT) != 0;
bool pvm = (control & PCI_MSI_CTRL_PVM) != 0;
uint8_t vector = (uint8_t)(PCI_IRQ_VECTOR_BASE + irqSlot);
uint32_t address = PCI_MSI_ADDRESS_BASE | ((bspLapic & 0xFFu) << 12);
uint16_t data = vector;
PciWrite32(bus, dev, fn, (uint8_t)(cap + 4u), address);
if (is64) {
PciWrite32(bus, dev, fn, (uint8_t)(cap + 8u), 0);
PciWrite16(bus, dev, fn, (uint8_t)(cap + 12u), data);
if (pvm)
PciWrite32(bus, dev, fn, (uint8_t)(cap + 16u), 0);
} else {
PciWrite16(bus, dev, fn, (uint8_t)(cap + 8u), data);
if (pvm)
PciWrite32(bus, dev, fn, (uint8_t)(cap + 12u), 0);
}
control &= (uint16_t)~PCI_MSI_CTRL_MME_MASK; // enable exactly one message
control |= PCI_MSI_CTRL_ENABLE;
PciWrite16(bus, dev, fn, (uint8_t)(cap + 2u), control);
uint32_t cmd = PciRead32(bus, dev, fn, PCI_REG_COMMAND_STATUS);
cmd |= PCI_COMMAND_INTX_DISABLE;
PciWrite32(bus, dev, fn, PCI_REG_COMMAND_STATUS, cmd);
HalDbgPrint("PCI %u:%u.%u: MSI enabled cap=0x%02x vector=0x%02x irqslot=%u dest=%u %s\n",
bus, dev, fn, cap, vector, irqSlot, bspLapic, is64 ? "64-bit" : "32-bit");
return STATUS_SUCCESS;
}
void HalPciInitialize(HAL_PCI* out) {
out->Read32 = PciRead32;
out->Write32 = PciWrite32;
out->FindDevice = PciFindDevice;
out->FindClass = PciFindClass;
out->GetIrqLine = PciGetIrqLine;
out->EnableMsi = PciEnableMsi;
}
code/kernel/hal/hal_screen.c
+71 -31
View File
@@ -7,8 +7,10 @@
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/mm_virt.h>
static uint32_t* gFb = (uint32_t *)0;
static uint32_t* gBackbuffer = (uint32_t *)0;
static int gWidth = 0;
static int gHeight = 0;
static int gStride = 0; // pixels per row
@@ -20,14 +22,39 @@ static int gCursorX = 0, gCursorY = 0;
// ---------------------------------------------------------------------------
// 8x16 VGA font - printable ASCII only (glyph 0 = space = 0x20)
// ---------------------------------------------------------------------------
#include <kernel/hal/font8x16.h> // see font8x16.h below
#include <kernel/hal/font8x16.h> // see font8x16.h
// ---------------------------------------------------------------------------
// Internals
// ---------------------------------------------------------------------------
static inline uint32_t* ScreenBackbufferPixel(int x, int y) {
return gBackbuffer ? (gBackbuffer + y * gStride + x) : (gFb + y * gStride + x);
}
static inline void PutPixelRaw(int x, int y, uint32_t color) {
if ((unsigned)x < (unsigned)gWidth && (unsigned)y < (unsigned)gHeight)
gFb[y * gStride + x] = color;
if ((unsigned)x >= (unsigned)gWidth || (unsigned)y >= (unsigned)gHeight)
return;
ScreenBackbufferPixel(x, y)[0] = color;
gFb[y * gStride + x] = color;
}
static void ScreenFlushRows(int startY, int rows) {
if (!gBackbuffer || rows <= 0)
return;
if (startY < 0) {
rows += startY;
startY = 0;
}
if (startY >= gHeight || rows <= 0)
return;
if (startY + rows > gHeight)
rows = gHeight - startY;
size_t rowBytes = (size_t)gStride * sizeof(uint32_t);
memcpy(gFb + startY * gStride, gBackbuffer + startY * gStride,
(size_t)rows * rowBytes);
}
// ---------------------------------------------------------------------------
@@ -36,7 +63,14 @@ static inline void PutPixelRaw(int x, int y, uint32_t color) {
static void ScreenClear(uint32_t color) {
for (int y = 0; y < gHeight; y++)
for (int x = 0; x < gWidth; x++)
gFb[y * gStride + x] = color;
ScreenBackbufferPixel(x, y)[0] = color;
ScreenFlushRows(0, gHeight);
if (!gBackbuffer) {
// ScreenFlushRows is a no-op without a backbuffer.
for (int y = 0; y < gHeight; y++)
for (int x = 0; x < gWidth; x++)
gFb[y * gStride + x] = color;
}
gCursorX = 0; gCursorY = 0;
}
@@ -66,25 +100,13 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
// Calculate bytes to copy per row
size_t bytesPerRow = (size_t)drawW * sizeof(uint32_t);
// If the width we are drawing exactly matches the physical screen stride,
// AND it matches the original source image width, memory is perfectly contiguous.
if (drawW == gStride && drawW == w) {
uint32_t* dst = gFb + (startY * gStride) + startX;
const uint32_t* src = pixels + (srcOffsetY * w) + srcOffsetX;
memcpy(dst, src, bytesPerRow * drawH);
return;
}
// Copy row-by-row
for (int row = 0; row < drawH; row++) {
// Destination in the actual framebuffer (accounting for stride)
uint32_t* dst = gFb + ((startY + row) * gStride) + startX;
// Source from the provided pixel buffer
uint32_t* shadow = ScreenBackbufferPixel(startX, startY + row);
uint32_t* fb = gFb + ((startY + row) * gStride) + startX;
const uint32_t* src = pixels + ((srcOffsetY + row) * w) + srcOffsetX;
memcpy(dst, src, bytesPerRow);
memcpy(shadow, src, bytesPerRow);
memcpy(fb, src, bytesPerRow);
}
}
@@ -92,12 +114,17 @@ static void ScreenGetSize(int* w, int* h) { *w = gWidth; *h = gHeight; }
static void ScreenScroll(int lines) {
if (lines <= 0 || lines >= gHeight) { ScreenClear(0); return; }
int rowBytes = gStride * sizeof(uint32_t);
memmove(gFb, gFb + lines * gStride,
size_t rowBytes = (size_t)gStride * sizeof(uint32_t);
uint32_t* shadow = gBackbuffer ? gBackbuffer : gFb;
memmove(shadow, shadow + lines * gStride,
(size_t)(gHeight - lines) * rowBytes);
// Clear the revealed rows at the bottom.
memset(gFb + (gHeight - lines) * gStride, 0,
memset(shadow + (gHeight - lines) * gStride, 0,
(size_t)lines * rowBytes);
ScreenFlushRows(0, gHeight);
}
static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
@@ -195,7 +222,7 @@ static void CursorSaveBg(int x, int y) {
int fx = x + col;
uint32_t px = 0;
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight)
px = gFb[fy * gStride + fx];
px = ScreenBackbufferPixel(fx, fy)[0];
gCursorSavedBg[row * CURSOR_W + col] = px;
}
}
@@ -207,8 +234,12 @@ static void CursorRestoreBg(int x, int y) {
int fy = y + row;
for (int col = 0; col < CURSOR_W; col++) {
int fx = x + col;
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight)
gFb[fy * gStride + fx] = gCursorSavedBg[row * CURSOR_W + col];
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) {
uint32_t px = gBackbuffer
? ScreenBackbufferPixel(fx, fy)[0]
: gCursorSavedBg[row * CURSOR_W + col];
gFb[fy * gStride + fx] = px;
}
}
}
}
@@ -248,8 +279,8 @@ void HalScreenDrawCursor(int x, int y) {
if ((src >> 24) == 0) continue; // fully transparent, skip
int fx = x + col;
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) {
uint32_t* dst = &gFb[fy * gStride + fx];
*dst = BlendPixel(*dst, src);
uint32_t dst = ScreenBackbufferPixel(fx, fy)[0];
gFb[fy * gStride + fx] = BlendPixel(dst, src);
}
}
}
@@ -266,12 +297,21 @@ void HalScreenEraseCursor(void) {
// ---------------------------------------------------------------------------
// Populate the HAL_SCREEN vtable
// ---------------------------------------------------------------------------
void HalScreenInitialize(uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out) {
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out) {
gFb = (uint32_t *)fb;
gWidth = w;
gHeight = h;
gStride = stride;
size_t backbufferPixels = (size_t)gStride * (size_t)gHeight;
size_t backbufferBytes = backbufferPixels * sizeof(uint32_t);
if (gStride > 0 && gHeight > 0 &&
backbufferPixels / (size_t)gStride == (size_t)gHeight &&
backbufferBytes / sizeof(uint32_t) == backbufferPixels) {
gBackbuffer = (uint32_t*)AllocatePool(backbufferBytes);
}
out->Clear = ScreenClear;
out->PutPixel = ScreenPutPixel;
out->Blit = ScreenBlit;
code/kernel/hal/hal_timer.c
+2 -2
View File
@@ -51,7 +51,7 @@ static uint64_t QueryTicks(void) {
}
// Fallback: combine the coarse periodic tick with the current LAPIC
// countdown. Read the coarse tick twice so an interrupt between gTicks and
// countdown. Read the coarse tick twice so an interrupt between gTicks and
// LAPIC_TIMER_CUR cannot make time jump backwards by almost one quantum.
uint64_t baseTicks1;
uint64_t baseTicks2;
@@ -177,7 +177,7 @@ static uint32_t CalibrateApicTimerViaHpet(uintptr_t hpetPhys) {
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *out) {
gLapic = (volatile uint32_t *)gApi.PhysToVirt(lapicPhys);
HalDbgPrint("HAL: lapicPhys=0x%llX, lapicVirt=0x%llX, hpetPhys=0x%llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
HalDbgPrint("HAL: lapicPhys=0x%016llX, lapicVirt=0x%016llX, hpetPhys=0x%016llX\n", lapicPhys, (uintptr_t)gLapic, hpetPhys);
uint32_t ticksPerMs = 0;
if (hpetPhys) {
code/kernel/include/kernel/hal/hal_apic.h
+2
View File
@@ -28,6 +28,8 @@ typedef struct HAL_CPU_TOPOLOGY {
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out);
__declspec(dllexport) uint32_t HalGetBspLapicId(void);
__declspec(dllexport) uint8_t HalApicUsingX2Apic(void);
__declspec(dllexport) uint8_t HalApicCanAddressId(uint32_t lapicId);
__declspec(dllexport) void HalApicEnableCurrentCpu(void);
__declspec(dllexport) void HalApicSendPanicIpi(void);
__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId);
code/kernel/include/kernel/io_file.h
+2 -2
View File
@@ -14,9 +14,9 @@
typedef SYS_DIR_ENTRY KDIR_ENTRY;
#define KDIR_NAME_MAX SYS_DIR_NAME_MAX
// Native asynchronous filesystem completion. Filesystems that provide
// Native asynchronous filesystem completion. Filesystems that provide
// ReadAsync/WriteAsync accept the operation, return STATUS_PENDING, and later
// call Complete(CompleteContext, status, information). Filesystems without
// call Complete(CompleteContext, status, information). Filesystems without
// these callbacks fall back to the generic VFS async worker.
typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
KSTATUS status,
code/kernel/include/kernel/ke_apc.h
+1 -1
View File
@@ -84,7 +84,7 @@ __declspec(dllexport) void KeEnterCriticalRegion(void);
__declspec(dllexport) void KeLeaveCriticalRegion(void);
__declspec(dllexport) bool KeAreKernelApcsDisabled(void);
// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL,
// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL,
// after context switches, and after waits complete.
void KiDeliverKernelApcs(void);
code/kernel/include/kernel/ke_debug.h
+2 -2
View File
@@ -26,8 +26,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void);
// Fault dispatch - called from the IDT fault stubs (ke_syscall.asm).
//
// The old scalar entry only reported RIP/error/CR2, which is not enough for
// debugging null indirect calls. The frame entry snapshots user GPRs plus the
// CPU exception frame, including user RSP. For a CALL through a null function
// debugging null indirect calls. The frame entry snapshots user GPRs plus the
// CPU exception frame, including user RSP. For a CALL through a null function
// pointer, RIP is 0 and [RSP] is usually the return address after the bad call;
// that is the useful caller site.
// ---------------------------------------------------------------------------
code/kernel/include/kernel/ke_dpc.h
+4 -4
View File
@@ -1,9 +1,9 @@
#pragma once
// ke_dpc.h - deferred procedure calls
//
// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC
// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC
// after acknowledging/capturing device state; the DPC then runs later at
// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not
// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not
// wait, block, or call PASSIVE_LEVEL-only code.
#include <stdint.h>
@@ -34,11 +34,11 @@ __declspec(dllexport) void KeInitializeDpc(KDPC* dpc,
KDEFERRED_ROUTINE routine,
void* context);
// Queue a DPC if it is not already queued. Returns false if the DPC was
// Queue a DPC if it is not already queued. Returns false if the DPC was
// already pending or the arguments are invalid.
__declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
void* systemArgument1,
void* systemArgument2);
// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit.
// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit.
void KiDrainDpcQueue(void);
code/kernel/include/kernel/ke_input.h
+24
View File
@@ -0,0 +1,24 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/input.h>
#include <ntlite/ddi/hal_api.h>
#include <ntlite/status.h>
void KeInputInitialize(const HAL_API* hal);
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId);
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event);
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event);
INPUT_KEY KeInputKbPoll(void);
bool KeInputIsKeyDown(INPUT_KEY key);
INPUT_KEY KeInputReadKey(void);
char KeInputReadChar(void);
uint32_t KeInputGetModifiers(void);
bool KeInputMousePoll(void);
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons);
void KeInputMouseGetPosition(int* x, int* y);
void KeInputMouseSetPosition(int x, int y);
void KeInputMouseSetBounds(int w, int h);
code/kernel/include/kernel/ke_irp.h
+4 -4
View File
@@ -55,7 +55,7 @@ typedef struct IRP {
uint32_t Flags; // IRP_FLAG_*
uint64_t Lba;
uint32_t SectorCount;
void *Buffer; // kernel VA; valid until completion
void* Buffer; // kernel VA; valid until completion
// ---- Completion (caller fills before submit; driver writes Status) ----
@@ -65,17 +65,17 @@ typedef struct IRP {
// Forward-declared pointer: callers that create events include ke_event.h.
struct EVENT *CompletionEvent;
// Optional kernel APC. If set, IoCompleteRequest queues it to its
// Optional kernel APC. If set, IoCompleteRequest queues it to its
// target thread with arg1=irp and arg2=(void*)(uintptr_t)status.
struct KAPC *CompletionApc;
IRP_COMPLETION_FN CompletionFn;
void *CompletionCtx;
void* CompletionCtx;
// ---- Driver-internal --------------------------------------------------
uint8_t HwSlot;
void *DriverCtx;
void* DriverCtx;
} IRP;
// -------------------------------------------------------------------------
code/kernel/include/kernel/ke_irq.h
+4 -2
View File
@@ -1,13 +1,15 @@
#pragma once
#include <stdint.h>
#include <ntlite/ddi/hal_api.h>
#include <ntlite/status.h>
void KeIrqInitialize(const HAL_API *hal);
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq);
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void));
void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm
// Low-level interrupt entry/exit hooks. These are called by the common IRQ
// assembly stub around KeIrqDispatch. The exit hook is where deferred
// Low-level interrupt entry/exit hooks. These are called by the common IRQ
// assembly stub around KeIrqDispatch. The exit hook is where deferred
// interrupt-time scheduling now happens, so IRQ handlers never switch away
// before KeIrqDispatch has returned and sent EOI.
void KiEnterInterrupt(void);
code/kernel/include/kernel/ke_irql.h
+4 -4
View File
@@ -1,9 +1,9 @@
#pragma once
// ke_irql.h - software IRQL skeleton
//
// This is intentionally small for the first IRQL/APC bring-up step. It does
// This is intentionally small for the first IRQL/APC bring-up step. It does
// not yet program interrupt hardware or replace the spinlock interrupt-disable
// path. It gives the kernel one authoritative execution-level state plus
// path. It gives the kernel one authoritative execution-level state plus
// assertions so blocking code can reject illegal callers early.
#include <stdint.h>
@@ -18,11 +18,11 @@ typedef uint8_t KIRQL;
__declspec(dllexport) KIRQL KeGetCurrentIrql(void);
// Raise to newIrql and return the previous level. Raising to a lower level is
// Raise to newIrql and return the previous level. Raising to a lower level is
// a kernel bug and will panic in checked/debug-style builds.
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql);
// Restore a previously returned level. Lowering to a higher level is a kernel
// Restore a previously returned level. Lowering to a higher level is a kernel
// bug and will panic.
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql);
code/kernel/include/kernel/ke_process.h
+6
View File
@@ -75,6 +75,12 @@ typedef struct PROCESS {
uintptr_t UserCwdBase;
size_t UserCwdSize;
/* Per-process user virtual regions. The actual VA arena is global because
NT-Lite currently uses one shared address space; this list tracks
ownership so leaked user mappings can be reclaimed on process exit. */
LIST_ENTRY UserRegions;
SPINLOCK UserRegionLock;
/* Per-process loaded modules (user processes only). */
PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
uint32_t ModuleCount;
code/kernel/include/kernel/ke_sched.h
+4 -4
View File
@@ -37,15 +37,15 @@ KSTATUS KiCreateIdleThreadForCpu(struct KCPU* cpu);
void KeReadyThread(THREAD* thread);
void KeUnreadyThread(THREAD* thread);
// Called from the timer ISR every SCHED_TICK_MS. This performs timer
// Called from the timer ISR every SCHED_TICK_MS. This performs timer
// accounting and requests a reschedule only when the current thread has
// exhausted its scheduler budget, or when idle work becomes available. The
// exhausted its scheduler budget, or when idle work becomes available. The
// actual context switch happens from
// KiExitInterrupt after the IRQ handler has returned and EOI has been sent.
void KeSchedulerTick(void);
// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule
// is cheap and can be called from interrupt context. KiScheduleFromInterrupt
// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule
// is cheap and can be called from interrupt context. KiScheduleFromInterrupt
// consumes the pending request and performs the context switch, if needed.
void KiRequestReschedule(void);
int KiIsReschedulePending(void);
code/kernel/include/kernel/ke_smp.h
+3
View File
@@ -2,6 +2,9 @@
#include <stdint.h>
#include <ntlite/status.h>
// Provide an optional bootloader-reserved low-memory trampoline page.
void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys);
// Start application processors with hardware INIT/SIPI and park them in
// kernel-owned state. APs are not runnable until scheduler admission succeeds.
KSTATUS KeStartApplicationProcessors(void);
code/kernel/include/kernel/ke_spinlock.h
+1
View File
@@ -11,6 +11,7 @@
// Never hold a spinlock across a wait/blocking operation.
#include <stdint.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <kernel/ke_irql.h>
typedef struct SPINLOCK {
code/kernel/include/kernel/ke_thread.h
+6 -6
View File
@@ -71,7 +71,7 @@ typedef struct THREAD {
uintptr_t UserArg;
// Saved scheduler-lock cookie for this thread's suspended switch
// continuation. This is deliberately thread-owned, not CPU-owned:
// continuation. This is deliberately thread-owned, not CPU-owned:
// after KeSwitchContext() resumes a saved KiSchedule() continuation, the
// resumed thread must restore the IF/IRQL state captured when that same
// thread switched out.
@@ -86,7 +86,7 @@ typedef struct THREAD {
void* FpuContext;
uint8_t FpuStorage[528];
// Scheduler accounting. QuantaConsumed is kept in the assembly-stable
// Scheduler accounting. QuantaConsumed is kept in the assembly-stable
// portion of THREAD for layout compatibility; the dynamic scheduler uses
// these fields for per-dispatch budget and CPU-burst tracking.
uint32_t QuantumTicks;
@@ -105,14 +105,14 @@ typedef struct THREAD {
uint64_t AffinityMask;
// Set by wait primitives before blocking and completed by the object
// that wakes the thread. Keep this field after the assembly-visible
// that wakes the thread. Keep this field after the assembly-visible
// context-switch fields: ke_switch.asm hard-codes offsets through
// FpuContext.
KSTATUS WaitStatus;
LIST_ENTRY* WaitListHead;
uint8_t AlertableWait;
// Kernel APC state. APCs are queued to a target thread and delivered
// Kernel APC state. APCs are queued to a target thread and delivered
// when that thread reaches PASSIVE_LEVEL with APC delivery enabled.
LIST_ENTRY ApcQueue;
SPINLOCK ApcLock;
@@ -124,7 +124,7 @@ typedef struct THREAD {
// The pointed-to frame lives on this thread's kernel stack.
KTRAP_FRAME* TrapFrame;
// User APC state. User APCs are queued by kernel completions, but are
// User APC state. User APCs are queued by kernel completions, but are
// delivered only on an alertable return to user mode through the registered
// user-mode dispatcher stub.
LIST_ENTRY UserApcQueue;
@@ -162,7 +162,7 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code);
//
// The split form prevents the classic lost wakeup where an ISR/DPC signals
// the object after the waiter is queued but before the waiter marks itself
// waiting. Alertable waits are explicit at every call site.
// waiting. Alertable waits are explicit at every call site.
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead);
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
code/kernel/include/kernel/ke_userenv.h
+1 -1
View File
@@ -32,7 +32,7 @@ typedef struct SYS_TEB {
uint32_t ThreadId;
uint32_t Magic;
// Native per-thread extension pointer. The kernel leaves this NULL;
// Native per-thread extension pointer. The kernel leaves this NULL;
// user-mode personalities/libraries may hang their own thread-local
// state here without baking subsystem semantics into SYS_TEB.
void* SubsystemData;
code/kernel/include/kernel/ldr_image.h
+5 -1
View File
@@ -6,12 +6,16 @@
#include <stdbool.h>
#include <ntlite/status.h>
struct PROCESS;
// Map a PE32+ image from 'buffer' into the kernel virtual address space.
// On success, *baseOut receives the virtual base address.
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user);
KSTATUS LdrLoadImageForProcess(struct PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut);
// Return the SizeOfImage recorded in the PE header for an already-mapped image.
size_t LdrGetImageSize(uintptr_t base);
size_t LdrGetMappedImageSize(struct PROCESS* process, uintptr_t base);
// Look up a named export in a loaded module.
// 'moduleName' must match a name in BOOT_INFO.Modules (e.g., "hal.dll").
@@ -24,11 +28,11 @@ uintptr_t LdrGetEntryPoint(uintptr_t base);
KSTATUS LdrResolveImports(uintptr_t imageBase);
// Resolve imports using a process-local module registry.
struct PROCESS;
KSTATUS LdrResolveImportsForProcess(struct PROCESS* process, uintptr_t imageBase);
// Apply final page permissions for a loaded image after relocations/imports.
KSTATUS LdrProtectImage(uintptr_t imageBase, bool user);
KSTATUS LdrProtectImageForProcess(struct PROCESS* process, uintptr_t imageBase, bool user);
// Load a DLL for a specific process, resolve it against that process's
// loaded modules, and register it in the process-local registry.
code/kernel/include/kernel/mm_phys.h
+3 -1
View File
@@ -17,8 +17,10 @@ bool MmInitializePhysical(const MEMORY_MAP* map,
// Allocate / free a single 4 KB frame.
// MmAllocateFrame returns 0 on failure.
PHYSICAL_ADDRESS MmAllocateFrame(void);
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void);
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive);
PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive);
void MmFreeFrame(PHYSICAL_ADDRESS frame);
// Allocate 'count' physically contiguous 4 KB frames.
code/kernel/include/kernel/mm_virt.h
+47 -7
View File
@@ -6,6 +6,25 @@
#include <kernel/mm_phys.h>
#include <ntlite/status.h>
struct PROCESS;
typedef struct PROCESS PROCESS;
typedef enum USER_REGION_TYPE {
UserRegionGeneric = 0,
UserRegionHeap,
UserRegionImage,
UserRegionStack,
UserRegionTeb,
UserRegionPeb,
UserRegionEnvironment,
UserRegionMappedFile,
UserRegionShared,
} USER_REGION_TYPE;
#define USER_REGION_GUARD_LOW 0x00000001u
#define USER_REGION_GUARD_HIGH 0x00000002u
#define USER_REGION_ZERO 0x00000004u
// ---------------------------------------------------------------------------
// Page flags (x86-64 PTE bits)
// ---------------------------------------------------------------------------
@@ -32,10 +51,10 @@
// Called by KernelMain after MmInitializePhysical.
// bootPml4Phys: the PML4 set up by the bootloader.
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
void MmActivate(void);
void MmActivate(void);
// ---------------------------------------------------------------------------
// Page mapping
@@ -89,16 +108,37 @@ __declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// ---------------------------------------------------------------------------
// User heap - shared flat space, ring 3 accessible.
// Allocations are writable user pages with NX set by default; PE image
// sections opt back into execution through the loader.
// User virtual regions.
// The kernel has one shared user VA arena, but every user mapping is owned by
// a PROCESS so process teardown can reclaim leaked memory. Guard pages are
// reserved VA with no present PTE.
// ---------------------------------------------------------------------------
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t regionFlags,
USER_REGION_TYPE type, void** outBase);
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base);
void MmDestroyAllUserRegions(PROCESS* process);
size_t MmGetUserRegionSize(PROCESS* process, const void* base);
size_t MmGetUserRegionSizeByBase(const void* base);
// Debug-only ownership/address diagnostics used by fault and launch paths.
void MmDebugDescribeAddress(uintptr_t addr);
void MmDebugDumpUserRegions(PROCESS* process);
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size);
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr);
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize);
// Convenience wrappers for syscalls/current-process allocations.
__declspec(dllexport) void* MmAllocateUserMemory(size_t size);
__declspec(dllexport) void MmFreeUserMemory(void* ptr);
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize);
// Page-aligned user image mapping. PE section protection is page based, so
// executable images must not be allocated from the normal sub-page heap.
// Page-aligned user image mapping. PE section protection is page based.
__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size);
__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size);
__declspec(dllexport) void* MmAllocateUserImage(size_t size);
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
code/kernel/include/kernel/syscall_nr.h
+1 -1
View File
@@ -73,7 +73,7 @@
#define SYS_KB_POLL 64
#define SYS_KB_IS_KEY_DOWN 65
#define SYS_KB_READ_CHAR 66
#define SYS_KB_READ_SCANCODE 67
#define SYS_KB_READ_KEY 67
#define SYS_KB_GET_MODIFIERS 68
#define SYS_MOUSE_POLL 69
#define SYS_MOUSE_READ 70
code/kernel/io_device.c
+162
View File
@@ -3,10 +3,137 @@
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_irp.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#define MAX_PUBLISHED_DEVICES 64
#define MAX_DRIVER_MATCHERS 32
typedef struct IO_DRIVER_MATCHER_ENTRY {
bool Used;
const char* DriverName;
IO_BUS_TYPE BusType;
IO_DRIVER_MATCH_CALLBACK Match;
IO_DRIVER_BIND_CALLBACK Bind;
void* Context;
} IO_DRIVER_MATCHER_ENTRY;
static IO_PUBLISHED_DEVICE gPublishedDevices[MAX_PUBLISHED_DEVICES];
static uint32_t gPublishedDeviceCount;
static IO_DRIVER_MATCHER_ENTRY gDriverMatchers[MAX_DRIVER_MATCHERS];
static uint32_t gDriverMatcherCount;
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
switch (busType) {
case IoBusPci: return "pci";
case IoBusUsb: return "usb";
default: return "unknown";
}
}
static bool IoTryBindDevice(IO_PUBLISHED_DEVICE* device,
IO_DRIVER_MATCHER_ENTRY* matcher) {
if (!device || !matcher || !matcher->Used || device->Bound)
return false;
if (matcher->BusType != IoBusUnknown && matcher->BusType != device->BusType)
return false;
if (!matcher->Match || !matcher->Bind)
return false;
if (!matcher->Match(device, matcher->Context))
return false;
KSTATUS st = matcher->Bind(device, matcher->Context);
if (KSUCCESS(st)) {
device->Bound = true;
device->BoundDriverName = matcher->DriverName;
DbgPrint("io: %s device %s bound by %s\n",
IoBusTypeName(device->BusType),
device->Name ? device->Name : "<unnamed>",
matcher->DriverName ? matcher->DriverName : "<unnamed>");
return true;
}
DbgPrint("io: %s matcher %s accepted %s but bind failed st=0x%x\n",
IoBusTypeName(device->BusType),
matcher->DriverName ? matcher->DriverName : "<unnamed>",
device->Name ? device->Name : "<unnamed>",
st);
return false;
}
__declspec(dllexport) KSTATUS IoRegisterDriverMatcher(const char* driverName,
IO_BUS_TYPE busType,
IO_DRIVER_MATCH_CALLBACK match,
IO_DRIVER_BIND_CALLBACK bind,
void* context) {
if (!driverName || !match || !bind)
return STATUS_INVALID_PARAMETER;
if (gDriverMatcherCount >= MAX_DRIVER_MATCHERS)
return STATUS_NO_MEMORY;
IO_DRIVER_MATCHER_ENTRY* entry = &gDriverMatchers[gDriverMatcherCount++];
memset(entry, 0, sizeof(*entry));
entry->Used = true;
entry->DriverName = driverName;
entry->BusType = busType;
entry->Match = match;
entry->Bind = bind;
entry->Context = context;
DbgPrint("io: registered %s matcher %s\n",
IoBusTypeName(busType), driverName);
for (uint32_t i = 0; i < gPublishedDeviceCount; i++) {
IoTryBindDevice(&gPublishedDevices[i], entry);
}
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoPublishDevice(const IO_PUBLISHED_DEVICE* device) {
if (!device || device->BusType == IoBusUnknown)
return STATUS_INVALID_PARAMETER;
if (gPublishedDeviceCount >= MAX_PUBLISHED_DEVICES)
return STATUS_NO_MEMORY;
IO_PUBLISHED_DEVICE* stored = &gPublishedDevices[gPublishedDeviceCount++];
memcpy(stored, device, sizeof(*stored));
stored->Bound = false;
stored->BoundDriverName = (const char*)0;
for (uint32_t i = 0; i < gDriverMatcherCount; i++) {
if (IoTryBindDevice(stored, &gDriverMatchers[i]))
break;
}
return STATUS_SUCCESS;
}
__declspec(dllexport) uint32_t IoPublishedDeviceCount(void) {
return gPublishedDeviceCount;
}
#define MAX_DISK_DEVICES 8
#define MAX_DISK_ARRIVAL_CALLBACKS 16
typedef struct IO_DISK_CALLBACK_ENTRY {
bool Used;
IO_DISK_ARRIVAL_CALLBACK Callback;
void* Context;
} IO_DISK_CALLBACK_ENTRY;
static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES];
static IO_DISK_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_ARRIVAL_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
static void IoNotifyDiskArrived(uint8_t driveIndex, DEVICE_OBJECT* device) {
for (uint32_t i = 0; i < gDiskArrivalCallbackCount; i++) {
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[i];
if (entry->Used && entry->Callback)
entry->Callback(driveIndex, device, entry->Context);
}
}
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
@@ -43,6 +170,41 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
device->Flags |= DEVICE_OBJECT_DISK;
device->DiskIndex = driveIndex;
gDiskDevices[driveIndex] = device;
IoNotifyDiskArrived(driveIndex, device);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoRegisterDiskArrivalCallback(IO_DISK_ARRIVAL_CALLBACK callback,
void* context) {
if (!callback)
return STATUS_INVALID_PARAMETER;
if (gDiskArrivalCallbackCount >= MAX_DISK_ARRIVAL_CALLBACKS)
return STATUS_NO_MEMORY;
IO_DISK_CALLBACK_ENTRY* entry = &gDiskArrivalCallbacks[gDiskArrivalCallbackCount++];
memset(entry, 0, sizeof(*entry));
entry->Used = true;
entry->Callback = callback;
entry->Context = context;
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
if (gDiskDevices[i])
callback(i, gDiskDevices[i], context);
}
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoEnumerateDiskDevices(IO_DISK_ARRIVAL_CALLBACK callback,
void* context) {
if (!callback)
return STATUS_INVALID_PARAMETER;
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
if (gDiskDevices[i])
callback(i, gDiskDevices[i], context);
}
return STATUS_SUCCESS;
}
code/kernel/io_file.c
+2 -2
View File
@@ -150,7 +150,7 @@ static void KiIoAsyncFileWorker(void* arg) {
}
if (req->CancelRequested) {
// The operation may already have reached the filesystem. The
// The operation may already have reached the filesystem. The
// completion contract still reports cancellation so waiters/APCs do
// not mistake a cancelled request for useful data.
s = STATUS_CANCELLED;
@@ -628,7 +628,7 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
}
// ---------------------------------------------------------------------------
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
code/kernel/ke_apc.c
+3 -3
View File
@@ -74,13 +74,13 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
KeReleaseSpinlock(&thread->ApcLock, flags);
// If the target opted into an alertable wait, make the APC itself the
// wait completion. The target thread will deliver the APC before
// wait completion. The target thread will deliver the APC before
// KeWaitForEvent(..., true) returns STATUS_KERNEL_APC.
if (wakeAlertable)
KiWakeThreadForKernelApc(thread);
// If the caller queued an APC to itself at PASSIVE_LEVEL, try to deliver
// immediately. Cross-thread APCs run the next time the target thread
// immediately. Cross-thread APCs run the next time the target thread
// returns to PASSIVE_LEVEL.
if (thread == KeGetCurrentThread())
KiDeliverKernelApcs();
@@ -165,7 +165,7 @@ void KiDeliverKernelApcs(void) {
&arg2);
// After the kernel routine has had a chance to adjust the callout,
// stop touching APC storage before invoking the normal routine. The
// stop touching APC storage before invoking the normal routine. The
// normal routine is allowed to free the APC's containing allocation.
apc->SystemArgument1 = (void*)0;
apc->SystemArgument2 = (void*)0;
code/kernel/ke_cpu.c
+1 -1
View File
@@ -26,7 +26,7 @@ void KeLoadKernelGsBase(KCPU* cpu) {
}
void KeSetUserGsBase(uintptr_t userGsBase) {
// While running in the kernel, GS_BASE is the current KCPU. The user TEB
// While running in the kernel, GS_BASE is the current KCPU. The user TEB
// base is parked in IA32_KERNEL_GS_BASE so swapgs can exchange it with the
// KCPU pointer on return to ring 3.
__writemsr(IA32_KERNEL_GS_BASE, (uint64_t)userGsBase);
code/kernel/ke_debug.c
+30 -11
View File
@@ -5,6 +5,7 @@
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <kernel/ke_debug.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
@@ -31,8 +32,8 @@ static int KiPanicIsActive(void) {
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
}
// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
// unexpected NMI is not handled here and will fall through to the normal fault
// path in the assembly stub.
__declspec(dllexport) int KeHandleNmiIpi(void) {
@@ -72,13 +73,13 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
// Silence this CPU first.
_disable();
// Only one CPU owns the visible panic report. CPUs that fault while the
// Only one CPU owns the visible panic report. CPUs that fault while the
// system is already panicking immediately join the stopped set.
if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) != 0) {
KiPanicSecondaryHalt();
}
// Stop every other CPU as early as possible. This uses an NMI IPI so it
// Stop every other CPU as early as possible. This uses an NMI IPI so it
// still reaches CPUs with IF cleared inside spinlock/high-IRQL regions.
HalApicSendPanicIpi();
@@ -98,7 +99,7 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
(unsigned)cpu->CpuIndex,
(unsigned)cpu->LapicId);
if (thread)
printf(" thread=%p tid=%llu",
printf(" thread=0x%016llX tid=%llu",
thread,
(unsigned long long)thread->ThreadId);
printf("\n");
@@ -192,7 +193,7 @@ static int KiFaultHasErrorCode(int vector) {
// Walk the process module list to find which image owns an address.
// Returns the module name string, or "unknown" if the address is outside all
// known image ranges. Base/size are optional and are useful for offset dumps.
// known image ranges. Base/size are optional and are useful for offset dumps.
static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
uintptr_t* baseOut, size_t* sizeOut) {
if (baseOut) *baseOut = 0;
@@ -230,7 +231,7 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
const char* mod = KiResolveModuleEx(proc, addr, &base, &size);
if (base) {
printf("%s%016llx <%s+0x%llx>\n",
printf("%s%016llx <%s+0x%016llX>\n",
label,
(unsigned long long)addr,
mod,
@@ -374,7 +375,7 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
size_t size = 0;
const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size);
if (base) {
printf(" [%02d] %016llx : %016llx <%s+0x%llx>\n",
printf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n",
i,
(unsigned long long)slot,
(unsigned long long)value,
@@ -486,7 +487,7 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
}
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
// the interrupted context on a secondary CPU once another CPU owns panic.
if (KiPanicIsActive())
KiPanicSecondaryHalt();
@@ -503,10 +504,28 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
int vector = (int)frame->Vector;
const char* name = KiFaultVectorName(vector);
DbgPrint("FAULT: kernel exception vector=%d name=%s rip=%016llx rsp=%016llx err=%04llx cr2=%016llx thread=%p pid=%u proc=%s\n",
vector,
name,
(unsigned long long)frame->Rip,
(unsigned long long)frame->Rsp,
(unsigned long long)frame->ErrorCode,
(unsigned long long)frame->Cr2,
thread,
proc ? proc->ProcessId : 0,
proc ? proc->Name : "<none>");
KiDumpFaultFrame(proc, frame);
if (vector == 14) {
KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
MmDebugDumpUserRegions(proc);
}
if (vector == 14) {
KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | RSP: %016llx | Code: %04llx | CR2: %016llx\n",
name,
(unsigned long long)frame->Rip,
(unsigned long long)frame->Rsp,
(unsigned long long)frame->ErrorCode,
(unsigned long long)frame->Cr2);
} else if (KiFaultHasErrorCode(vector)) {
@@ -547,7 +566,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
//
// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and
// call here. We check the CS RPL to decide: kernel fault → panic and halt,
// call here. We check the CS RPL to decide: kernel fault → panic and halt,
// user fault → KeBugCheckProcess (kill process, resume scheduler).
// ---------------------------------------------------------------------------
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
code/kernel/ke_dpc.c
+1 -1
View File
@@ -46,7 +46,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
KiEnsureDpcInitialized();
// Coalesce repeated inserts of the same KDPC. The first insert owns the
// Coalesce repeated inserts of the same KDPC. The first insert owns the
// argument pair until the DPC is popped by the dispatcher.
if (_InterlockedCompareExchange(&dpc->Inserted, 1, 0) != 0)
return false;
code/kernel/ke_event.c
+1 -1
View File
@@ -38,7 +38,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
} else {
if (!LIST_IS_EMPTY(&event->WaitList)) {
// Auto-reset handoff: transfer the signal directly to exactly one
// waiter. Do not leave Signaled=1, because a later waiter can steal
// waiter. Do not leave Signaled=1, because a later waiter can steal
// the disk/APC completion before the selected thread resumes.
event->Signaled = 0;
LIST_ENTRY* node = ListPopHead(&event->WaitList);
code/kernel/ke_idt.c
+1 -1
View File
@@ -140,7 +140,7 @@ void KeLoadGdtForCurrentCpu(KCPU* cpu) {
KeFlushGdt(&cpu->Gdtr, 0x08, 0x10, 0x28);
// Reloading GS as a segment register during GDT reload can disturb the
// hidden GS base. Reinstall the kernel per-CPU base immediately after
// hidden GS base. Reinstall the kernel per-CPU base immediately after
// the descriptor reload so KeGetCurrentCpu() remains GS-backed.
KeLoadKernelGsBase(cpu);
}
code/kernel/ke_input.c
+352
View File
@@ -0,0 +1,352 @@
// ke_input.c - kernel-owned canonical input core
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <kernel/ke_input.h>
#include <kernel/ke_debug.h>
#define KEY_QUEUE_SIZE 256u
#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u)
#define MOUSE_QUEUE_SIZE 128u
#define MOUSE_QUEUE_MASK (MOUSE_QUEUE_SIZE - 1u)
#define MAX_INPUT_KEYS 256u
#define MAX_INPUT_DEVICES 16u
#define KEY_REPEAT_DELAY_US 500000ull
#define KEY_REPEAT_PERIOD_US 33000ull
typedef struct KEY_CHAR_PAIR {
INPUT_KEY key;
char normal;
char shifted;
} KEY_CHAR_PAIR;
static const HAL_API* gInputHal;
static volatile uint8_t gKeyDown[MAX_INPUT_KEYS];
static volatile uint32_t gModifiers;
typedef struct INPUT_KEY_QUEUE_ENTRY {
INPUT_KEY key;
char ch;
} INPUT_KEY_QUEUE_ENTRY;
static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE];
static volatile int gKeyHead;
static volatile int gKeyTail;
static volatile INPUT_KEY gRepeatKey;
static volatile uint64_t gRepeatNextTick;
static INPUT_MOUSE_EVENT gMouseQueue[MOUSE_QUEUE_SIZE];
static volatile int gMouseHead;
static volatile int gMouseTail;
static volatile int gMouseX;
static volatile int gMouseY;
static volatile int gMouseBoundsW = 1920;
static volatile int gMouseBoundsH = 1080;
static volatile uint8_t gMouseButtons;
static volatile uint32_t gNextInputDeviceId = 1;
static char KiInputKeyToChar(INPUT_KEY key);
static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' },
{ INPUT_KEY_3, '3', '#' }, { INPUT_KEY_4, '4', '$' },
{ INPUT_KEY_5, '5', '%' }, { INPUT_KEY_6, '6', '^' },
{ INPUT_KEY_7, '7', '&' }, { INPUT_KEY_8, '8', '*' },
{ INPUT_KEY_9, '9', '(' }, { INPUT_KEY_0, '0', ')' },
{ INPUT_KEY_MINUS, '-', '_' }, { INPUT_KEY_EQUAL, '=', '+' },
{ INPUT_KEY_LEFT_BRACKET, '[', '{' }, { INPUT_KEY_RIGHT_BRACKET, ']', '}' },
{ INPUT_KEY_BACKSLASH, '\\', '|' }, { INPUT_KEY_SEMICOLON, ';', ':' },
{ INPUT_KEY_APOSTROPHE, '\'', '"' }, { INPUT_KEY_GRAVE, '`', '~' },
{ INPUT_KEY_COMMA, ',', '<' }, { INPUT_KEY_PERIOD, '.', '>' },
{ INPUT_KEY_SLASH, '/', '?' },
{ INPUT_KEY_SPACE, ' ', ' ' }, { INPUT_KEY_TAB, '\t', '\t' },
{ INPUT_KEY_ENTER, '\n', '\n' }, { INPUT_KEY_BACKSPACE, '\b', '\b' },
{ INPUT_KEY_KP_DIVIDE, '/', '/' }, { INPUT_KEY_KP_MULTIPLY, '*', '*' },
{ INPUT_KEY_KP_MINUS, '-', '-' }, { INPUT_KEY_KP_PLUS, '+', '+' },
{ INPUT_KEY_KP_ENTER, '\n', '\n' }, { INPUT_KEY_KP_1, '1', '1' },
{ INPUT_KEY_KP_2, '2', '2' }, { INPUT_KEY_KP_3, '3', '3' },
{ INPUT_KEY_KP_4, '4', '4' }, { INPUT_KEY_KP_5, '5', '5' },
{ INPUT_KEY_KP_6, '6', '6' }, { INPUT_KEY_KP_7, '7', '7' },
{ INPUT_KEY_KP_8, '8', '8' }, { INPUT_KEY_KP_9, '9', '9' },
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
static void KiInputQueueKey(INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
if (next == gKeyHead) return;
gKeyQueue[gKeyTail].key = key;
gKeyQueue[gKeyTail].ch = ch;
gKeyTail = next;
}
static bool KiInputQueueHasSpace(void) {
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
return next != gKeyHead;
}
static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return false;
switch (key) {
case INPUT_KEY_LEFT_SHIFT:
case INPUT_KEY_RIGHT_SHIFT:
case INPUT_KEY_LEFT_CTRL:
case INPUT_KEY_RIGHT_CTRL:
case INPUT_KEY_LEFT_ALT:
case INPUT_KEY_RIGHT_ALT:
case INPUT_KEY_LEFT_GUI:
case INPUT_KEY_RIGHT_GUI:
case INPUT_KEY_CAPS_LOCK:
case INPUT_KEY_NUM_LOCK:
return false;
default:
return true;
}
}
static uint64_t KiInputNowTicks(void) {
if (gInputHal && gInputHal->Timer.QueryTicks)
return gInputHal->Timer.QueryTicks();
return 0;
}
static void KiInputStartRepeat(INPUT_KEY key, uint64_t now) {
if (!KiInputKeyIsRepeatable(key)) return;
gRepeatKey = key;
gRepeatNextTick = now + KEY_REPEAT_DELAY_US;
}
static void KiInputStopRepeat(INPUT_KEY key) {
if (gRepeatKey == key) {
gRepeatKey = INPUT_KEY_NONE;
gRepeatNextTick = 0;
}
}
static void KiInputServiceRepeat(void) {
INPUT_KEY key = gRepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key]) {
gRepeatKey = INPUT_KEY_NONE;
gRepeatNextTick = 0;
return;
}
uint64_t now = KiInputNowTicks();
if (now == 0 || now < gRepeatNextTick)
return;
if (!KiInputQueueHasSpace()) {
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
return;
}
KiInputQueueKey(key, KiInputKeyToChar(key));
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
}
static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
uint32_t bit = 0;
switch (key) {
case INPUT_KEY_LEFT_SHIFT: bit = INPUT_MOD_LSHIFT; break;
case INPUT_KEY_RIGHT_SHIFT: bit = INPUT_MOD_RSHIFT; break;
case INPUT_KEY_LEFT_CTRL: bit = INPUT_MOD_LCTRL; break;
case INPUT_KEY_RIGHT_CTRL: bit = INPUT_MOD_RCTRL; break;
case INPUT_KEY_LEFT_ALT: bit = INPUT_MOD_LALT; break;
case INPUT_KEY_RIGHT_ALT: bit = INPUT_MOD_RALT; break;
case INPUT_KEY_LEFT_GUI: bit = INPUT_MOD_LGUI; break;
case INPUT_KEY_RIGHT_GUI: bit = INPUT_MOD_RGUI; break;
default: break;
}
if (bit) {
if (pressed) gModifiers |= bit;
else gModifiers &= ~bit;
}
if (pressed && key == INPUT_KEY_CAPS_LOCK) gModifiers ^= INPUT_MOD_CAPS_LOCK;
if (pressed && key == INPUT_KEY_NUM_LOCK) gModifiers ^= INPUT_MOD_NUM_LOCK;
}
static char KiInputKeyToChar(INPUT_KEY key) {
if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) {
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
bool caps = (gModifiers & INPUT_MOD_CAPS_LOCK) != 0;
char c = (char)('a' + (key - INPUT_KEY_A));
if (shifted ^ caps) c = (char)(c - 'a' + 'A');
return c;
}
bool shifted = (gModifiers & INPUT_MOD_SHIFT) != 0;
for (size_t i = 0; i < sizeof(gCharMap) / sizeof(gCharMap[0]); i++) {
if (gCharMap[i].key == key)
return shifted ? gCharMap[i].shifted : gCharMap[i].normal;
}
return 0;
}
void KeInputInitialize(const HAL_API* hal) {
gInputHal = hal;
memset((void*)gKeyDown, 0, sizeof(gKeyDown));
gModifiers = 0;
gKeyHead = gKeyTail = 0;
gRepeatKey = INPUT_KEY_NONE;
gRepeatNextTick = 0;
gMouseHead = gMouseTail = 0;
gMouseButtons = 0;
int w = 1920, h = 1080;
if (hal && hal->Screen.GetSize) hal->Screen.GetSize(&w, &h);
KeInputMouseSetBounds(w, h);
KeInputMouseSetPosition(w / 2, h / 2);
if (hal && hal->Screen.DrawCursor)
hal->Screen.DrawCursor(gMouseX, gMouseY);
DbgPrint("Input: kernel input core initialized (%dx%d)\n", w, h);
}
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) {
(void)flags;
uint32_t id = gNextInputDeviceId++;
if (id >= MAX_INPUT_DEVICES) return STATUS_NO_MEMORY;
if (outDeviceId) *outDeviceId = id;
DbgPrint("Input: registered device %u (%s)\n", id, name ? name : "unnamed");
return STATUS_SUCCESS;
}
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
if (!event) return;
INPUT_KEY key = event->Key;
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return;
bool wasDown = gKeyDown[key] != 0;
bool pressed = event->Pressed;
if (pressed) {
// The input core owns typematic repeat. Treat repeated hardware make
// events as idempotent so PS/2 typematic and USB HID share one policy.
if (wasDown)
return;
gKeyDown[key] = 1u;
KiInputSetModifier(key, true);
KiInputQueueKey(key, KiInputKeyToChar(key));
KiInputStartRepeat(key, KiInputNowTicks());
} else {
if (!wasDown)
return;
gKeyDown[key] = 0u;
KiInputSetModifier(key, false);
KiInputStopRepeat(key);
}
}
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
if (!event) return;
int nx = gMouseX + event->dx;
int ny = gMouseY + event->dy;
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
if (nx < 0) nx = 0;
if (ny < 0) ny = 0;
if (nx >= bw) nx = bw - 1;
if (ny >= bh) ny = bh - 1;
if (gInputHal && gInputHal->Screen.EraseCursor)
gInputHal->Screen.EraseCursor();
gMouseX = nx;
gMouseY = ny;
gMouseButtons = event->buttons;
if (gInputHal && gInputHal->Screen.DrawCursor)
gInputHal->Screen.DrawCursor(nx, ny);
int next = (gMouseTail + 1) & (int)MOUSE_QUEUE_MASK;
if (next != gMouseHead) {
gMouseQueue[gMouseTail] = *event;
gMouseTail = next;
}
}
INPUT_KEY KeInputKbPoll(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
}
bool KeInputIsKeyDown(INPUT_KEY key) {
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
return gKeyDown[key] != 0;
}
INPUT_KEY KeInputReadKey(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return INPUT_KEY_NONE;
INPUT_KEY key = gKeyQueue[gKeyHead].key;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
return key;
}
char KeInputReadChar(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return 0;
char ch = gKeyQueue[gKeyHead].ch;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
return ch;
}
uint32_t KeInputGetModifiers(void) {
return gModifiers;
}
bool KeInputMousePoll(void) {
return gMouseHead != gMouseTail;
}
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
if (!dx || !dy || !buttons) return false;
if (gMouseHead == gMouseTail) return false;
INPUT_MOUSE_EVENT ev = gMouseQueue[gMouseHead];
gMouseHead = (gMouseHead + 1) & (int)MOUSE_QUEUE_MASK;
*dx = ev.dx;
*dy = ev.dy;
*buttons = ev.buttons;
return true;
}
void KeInputMouseGetPosition(int* x, int* y) {
if (x) *x = gMouseX;
if (y) *y = gMouseY;
}
void KeInputMouseSetPosition(int x, int y) {
int bw = gMouseBoundsW;
int bh = gMouseBoundsH;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
if (x < 0) x = 0;
if (y < 0) y = 0;
if (x >= bw) x = bw - 1;
if (y >= bh) y = bh - 1;
gMouseX = x;
gMouseY = y;
}
void KeInputMouseSetBounds(int w, int h) {
if (w < 1) w = 1;
if (h < 1) h = 1;
gMouseBoundsW = w;
gMouseBoundsH = h;
KeInputMouseSetPosition(gMouseX, gMouseY);
}
code/kernel/ke_irq.c
+24 -3
View File
@@ -10,6 +10,8 @@
#define MAX_IRQS 24
#define MAX_IRQ_HANDLERS 8
#define MSI_IRQ_FIRST 16u
#define MSI_IRQ_LAST 23u
typedef struct IRQ_HANDLER_LIST {
void (*Handlers[MAX_IRQ_HANDLERS])(void);
@@ -17,6 +19,7 @@ typedef struct IRQ_HANDLER_LIST {
} IRQ_HANDLER_LIST;
static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS];
static bool gIrqSoftwareVector[MAX_IRQS];
static const HAL_API *gHal;
static KIRQL KiIrqToIrql(uint8_t irq) {
@@ -30,6 +33,23 @@ void KeIrqInitialize(const HAL_API *hal) {
gHal = hal;
}
__declspec(dllexport) KSTATUS KeIrqAllocateMsiVector(uint8_t* outIrq) {
if (!outIrq) return STATUS_INVALID_PARAMETER;
for (uint8_t irq = MSI_IRQ_FIRST; irq <= MSI_IRQ_LAST; irq++) {
IRQ_HANDLER_LIST* list = &gIrqHandlers[irq];
if (!gIrqSoftwareVector[irq] && list->Count == 0) {
gIrqSoftwareVector[irq] = true;
*outIrq = irq;
DbgPrint("KeIrq: allocated MSI vector slot IRQ %u vector 0x%02x\n",
irq, 0x20u + irq);
return STATUS_SUCCESS;
}
}
return STATUS_NO_MEMORY;
}
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
@@ -45,7 +65,8 @@ __declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
}
list->Handlers[list->Count++] = handler;
gHal->Intc.RouteIrq(irq);
if (!gIrqSoftwareVector[irq])
gHal->Intc.RouteIrq(irq);
}
void KeIrqDispatch(uint8_t irq) {
@@ -71,9 +92,9 @@ void KeIrqDispatch(uint8_t irq) {
}
}
// Handlers are no longer allowed to context-switch directly. Timer IRQs
// Handlers are no longer allowed to context-switch directly. Timer IRQs
// only request rescheduling; KiExitInterrupt performs the switch after this
// function returns, so the EOI is guaranteed to be sent first. Keep the
// function returns, so the EOI is guaranteed to be sent first. Keep the
// software level at DIRQL until after the controller has been acknowledged.
gHal->Intc.SendEoi();
code/kernel/ke_mutex.c
+1 -1
View File
@@ -61,7 +61,7 @@ void KeReleaseMutex(MUTEX* mutex) {
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
// Direct handoff: the selected waiter owns the mutex when its wait
// returns. Do not drop Owner to NULL, or another thread can steal it.
// returns. Do not drop Owner to NULL, or another thread can steal it.
mutex->Owner = waiter;
KeReleaseSpinlock(&mutex->Lock, flags);
KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS);
code/kernel/ke_process.c
+33 -52
View File
@@ -96,45 +96,21 @@ static void KiProcessDestroy(OBJECT* obj) {
}
}
// Release the main executable image.
if (proc->ImageBase && proc->ImageSize) {
MmFreeUserImage((void *)proc->ImageBase, proc->ImageSize);
proc->ImageBase = 0;
proc->ImageSize = 0;
}
// Release per-process modules.
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
if (proc->Modules[i].Base && proc->Modules[i].Size) {
MmFreeUserImage((void *)proc->Modules[i].Base, proc->Modules[i].Size);
}
proc->Modules[i].Base = 0;
proc->Modules[i].Size = 0;
proc->Modules[i].Name[0] = '\0';
}
// Release every user VA region owned by this process. This includes the
// main image, DLLs, TEBs/stacks that survived thread teardown, process
// environment strings, and leaked user allocations.
MmDestroyAllUserRegions(proc);
proc->ImageBase = 0;
proc->ImageSize = 0;
proc->ModuleCount = 0;
// Release the user-mode process environment.
if (proc->UserPebBase && proc->UserPebSize) {
MmFreeUserMemory((void *)proc->UserPebBase);
proc->UserPebBase = 0;
proc->UserPebSize = 0;
}
if (proc->UserImagePathBase && proc->UserImagePathSize) {
MmFreeUserMemory((void *)proc->UserImagePathBase);
proc->UserImagePathBase = 0;
proc->UserImagePathSize = 0;
}
if (proc->UserCmdlineBase && proc->UserCmdlineSize) {
MmFreeUserMemory((void *)proc->UserCmdlineBase);
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
}
if (proc->UserCwdBase && proc->UserCwdSize) {
MmFreeUserMemory((void *)proc->UserCwdBase);
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
}
proc->UserPebBase = 0;
proc->UserPebSize = 0;
proc->UserImagePathBase = 0;
proc->UserImagePathSize = 0;
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
// Release the exit event.
if (proc->ExitEvent) {
@@ -189,6 +165,8 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->CurrentDirectoryLength = 0;
proc->CurrentDrive = 0;
proc->CwdLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&proc->UserRegions);
proc->UserRegionLock = (SPINLOCK)SPINLOCK_INIT;
// Slot 0 is the invalid handle - leave it NULL.
memset(proc->Handles, 0, sizeof(proc->Handles));
@@ -213,7 +191,8 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
// ---------------------------------------------------------------------------
// KeSetupUserProcessEnvironment
// ---------------------------------------------------------------------------
static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src,
static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
SYS_STRING_VIEW src,
uintptr_t* baseOut,
size_t* sizeOut) {
if (!baseOut || !sizeOut) return STATUS_INVALID_PARAMETER;
@@ -225,7 +204,7 @@ static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src,
if (src.Length == (size_t)-1) return STATUS_BUFFER_TOO_SMALL;
size_t len = src.Length + 1;
char* dst = (char*)MmAllocateUserMemory(len);
char* dst = (char*)MmAllocateUserMemoryForProcess(proc, len);
if (!dst) return STATUS_NO_MEMORY;
if (src.Length) memcpy(dst, src.Data, src.Length);
@@ -236,7 +215,8 @@ static KSTATUS KiCopyEnvironmentStringView(SYS_STRING_VIEW src,
return STATUS_SUCCESS;
}
static KSTATUS KiCopyEnvironmentStringViewFixed(SYS_STRING_VIEW src,
static KSTATUS KiCopyEnvironmentStringViewFixed(PROCESS* proc,
SYS_STRING_VIEW src,
size_t capacity,
uintptr_t* baseOut,
size_t* sizeOut) {
@@ -248,7 +228,7 @@ static KSTATUS KiCopyEnvironmentStringViewFixed(SYS_STRING_VIEW src,
if (SysSvContainsByte(src, '\0')) return STATUS_INVALID_PARAMETER;
if (src.Length + 1 > capacity) return STATUS_BUFFER_TOO_SMALL;
char* dst = (char*)MmAllocateUserMemory(capacity);
char* dst = (char*)MmAllocateUserMemoryForProcess(proc, capacity);
if (!dst) return STATUS_NO_MEMORY;
memset(dst, 0, capacity);
@@ -264,22 +244,22 @@ static void KiFreePartialProcessEnvironment(PROCESS* proc) {
if (!proc) return;
if (proc->UserPebBase && proc->UserPebSize) {
MmFreeUserMemory((void*)proc->UserPebBase);
MmFreeUserMemoryForProcess(proc, (void*)proc->UserPebBase);
proc->UserPebBase = 0;
proc->UserPebSize = 0;
}
if (proc->UserImagePathBase && proc->UserImagePathSize) {
MmFreeUserMemory((void*)proc->UserImagePathBase);
MmFreeUserMemoryForProcess(proc, (void*)proc->UserImagePathBase);
proc->UserImagePathBase = 0;
proc->UserImagePathSize = 0;
}
if (proc->UserCmdlineBase && proc->UserCmdlineSize) {
MmFreeUserMemory((void*)proc->UserCmdlineBase);
MmFreeUserMemoryForProcess(proc, (void*)proc->UserCmdlineBase);
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
}
if (proc->UserCwdBase && proc->UserCwdSize) {
MmFreeUserMemory((void*)proc->UserCwdBase);
MmFreeUserMemoryForProcess(proc, (void*)proc->UserCwdBase);
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
}
@@ -337,19 +317,20 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
KiFreePartialProcessEnvironment(proc);
s = KiCopyEnvironmentStringView(imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
s = KiCopyEnvironmentStringView(proc, imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
if (!KSUCCESS(s)) goto fail;
s = KiCopyEnvironmentStringView(finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize);
s = KiCopyEnvironmentStringView(proc, finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize);
if (!KSUCCESS(s)) goto fail;
s = KiCopyEnvironmentStringViewFixed(SysSvFromParts(finalCwd, finalCwdLength),
s = KiCopyEnvironmentStringViewFixed(proc,
SysSvFromParts(finalCwd, finalCwdLength),
SYS_CWD_MAX,
&proc->UserCwdBase,
&proc->UserCwdSize);
if (!KSUCCESS(s)) goto fail;
SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemory(sizeof(SYS_PEB));
SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemoryForProcess(proc, sizeof(SYS_PEB));
if (!peb) { s = STATUS_NO_MEMORY; goto fail; }
memset(peb, 0, sizeof(SYS_PEB));
@@ -389,7 +370,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
// The calling thread exits immediately after this in SysExitProcess. Do
// The calling thread exits immediately after this in SysExitProcess. Do
// not drop the process object here: each thread owns one process reference,
// and KiThreadDestroy releases that reference after unlinking the thread.
KiCompleteProcessExitIfLastThread(process);
@@ -402,7 +383,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
_InterlockedExchange(&process->Terminating, 1);
// Wake alertable waiters so they can notice Terminating at the syscall/
// wait boundary. This is intentionally conservative: fully asynchronous
// wait boundary. This is intentionally conservative: fully asynchronous
// forced thread teardown needs object-specific wait-list locking and is a
// larger process-lifecycle project.
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
code/kernel/ke_sched.c
+11 -10
View File
@@ -2,6 +2,7 @@
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
#pragma intrinsic(_enable)
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_process.h>
@@ -84,7 +85,7 @@ static uint32_t KiComputeQuantumLocked(THREAD* thread) {
uint32_t quantum = KiBaseQuantumForPriority(thread->Priority);
// Low-priority CPU-bound threads get longer batches once they have proven
// they mostly burn CPU. Higher-priority work still preempts them as soon
// they mostly burn CPU. Higher-priority work still preempts them as soon
// as it becomes ready, so this reduces switch overhead without hurting
// interactive wake latency.
if (thread->Priority == PriorityLow && thread->CpuBurstTicks >=
@@ -113,7 +114,7 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
// The scheduler lock is intentionally held across KeSwitchContext(). The
// The scheduler lock is intentionally held across KeSwitchContext(). The
// unlock cookie belongs to the thread continuation that will resume after
// the switch, not to the physical CPU. Another dispatch may resume this
// thread from an unrelated interrupt, so using a KCPU slot here would
@@ -129,7 +130,7 @@ static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"scheduler invariant failed: %s: thread=0x%llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
"scheduler invariant failed: %s: thread=0x%016llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
where,
(uint64_t)thread,
thread ? thread->ThreadId : 0,
@@ -201,7 +202,7 @@ static void KiIdleLoop(void) {
continue;
}
// HLT only wakes for maskable IPIs when IF=1. A CPU that reaches
// HLT only wakes for maskable IPIs when IF=1. A CPU that reaches
// the idle thread through an unusual switch path must not sleep with
// interrupts disabled, otherwise reschedule/TLB shootdown IPIs can
// be delayed forever.
@@ -282,7 +283,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
KeInitThreadFpuContext(idle);
// The hand-built CPU idle thread is not created through KeCreateThread(),
// so register it manually with the idle process. This keeps PID 0 visible
// so register it manually with the idle process. This keeps PID 0 visible
// as the real idle process while kernel worker threads live under PID 1.
ObReferenceObject(&gIdleProcess->Header);
uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock);
@@ -418,8 +419,8 @@ static void KiSchedule(void) {
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiArmPostSwitchUnlock(cur, flags);
// The currently running thread is not present in a run queue. If it is
// still runnable and nobody else is ready, keep executing it. Otherwise a
// The currently running thread is not present in a run queue. If it is
// still runnable and nobody else is ready, keep executing it. Otherwise a
// lone CPU-bound thread gets alternated with the idle thread on every timer
// tick, effectively turning a periodic preemption check into a 50% throttle.
if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableForCpuLocked(cpu)) {
@@ -508,7 +509,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
// Complete the wait while holding the scheduler lock. On SMP a wake can
// Complete the wait while holding the scheduler lock. On SMP a wake can
// race the prepare->commit window on another CPU: the target thread is
// already on the object's wait list and has State=ThreadWaiting, but it is
// still executing with RunningCpu set until it actually calls KeYield().
@@ -529,7 +530,7 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
thread->QuantumRemaining = 0;
if (thread->RunningCpu != KE_INVALID_CPU) {
// Prepare->commit race: the waiter is still running somewhere. Its
// Prepare->commit race: the waiter is still running somewhere. Its
// KeCommitBlockThread() will see State != ThreadWaiting and return the
// completed wait status without ever entering the ready queue.
thread->State = ThreadRunning;
@@ -598,7 +599,7 @@ void KeSchedulerTick(void) {
}
// Periodic aging pass to prevent indefinite starvation of PriorityLow and
// PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
// PriorityNormal threads. Ready threads promoted by aging get a fresh CPU
// burst history; they earned the boost by waiting, not by burning CPU.
gAgingCounter++;
if (gAgingCounter >= SCHED_AGING_INTERVAL_TICKS) {
code/kernel/ke_selftest.c
+16 -14
View File
@@ -42,7 +42,7 @@ static void KiApcSelfTestKernelRoutine(KAPC* apc,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("APC selftest: kernel routine apc=0x%llX normal=0x%llX ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n",
DbgPrint("APC selftest: kernel routine apc=0x%016llX normal=0x%016llX ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)apc,
(uint64_t)*normalRoutine,
(uint64_t)*normalContext,
@@ -56,7 +56,7 @@ static void KiApcSelfTestNormalRoutine(void* normalContext,
void* systemArgument2) {
volatile int* counter = (volatile int*)normalContext;
DbgPrint("APC selftest: normal routine ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n",
DbgPrint("APC selftest: normal routine ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)normalContext,
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
@@ -73,7 +73,7 @@ static void KiApcSelfTestCrossThreadRoutine(void* normalContext,
void* systemArgument2) {
KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext;
DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n",
DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
@@ -124,7 +124,7 @@ static void KiApcAlertableSelfTestNormalRoutine(void* normalContext,
KI_APC_ALERTABLE_SELFTEST_STATE* st =
(KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext;
DbgPrint("APC selftest: alertable APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n",
DbgPrint("APC selftest: alertable APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
@@ -366,7 +366,7 @@ static void KiIrpApcSelfTestKernelRoutine(KAPC* apc,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("IRP/APC selftest: kernel routine apc=0x%llX irp=0x%llX status=%u irql=%u\n",
DbgPrint("IRP/APC selftest: kernel routine apc=0x%016llX irp=0x%016llX status=%u irql=%u\n",
(uint64_t)apc,
(uint64_t)*systemArgument1,
(uint32_t)(uintptr_t)*systemArgument2,
@@ -383,7 +383,7 @@ static void KiIrpApcSelfTestNormalRoutine(void* normalContext,
IRP* irp = (IRP*)systemArgument1;
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
DbgPrint("IRP/APC selftest: APC delivered irp=0x%llX status=%u info=%llu irql=%u\n",
DbgPrint("IRP/APC selftest: APC delivered irp=0x%016llX status=%u info=%llu irql=%u\n",
(uint64_t)irp,
status,
(uint64_t)(irp ? irp->Information : 0),
@@ -553,7 +553,7 @@ static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%llX iosb=0x%llX status=%u irql=%u\n",
DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%016llX iosb=0x%016llX status=%u irql=%u\n",
(uint64_t)apc,
(uint64_t)*systemArgument1,
(uint32_t)(uintptr_t)*systemArgument2,
@@ -570,7 +570,7 @@ static void KiFileAsyncSelfTestNormalRoutine(void* normalContext,
(KI_FILE_ASYNC_SELFTEST_STATE*)normalContext;
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%llX status=%u iosb=%u/%llu irql=%u\n",
DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%016llX status=%u iosb=%u/%llu irql=%u\n",
(uint64_t)systemArgument1,
status,
st->Iosb.Status,
@@ -759,10 +759,10 @@ void KiRunCoreHardeningSelfTests(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu != &gCpus[0])
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%llX cpu0=0x%llX\n",
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%016llX cpu0=0x%016llX\n",
(uint64_t)cpu, (uint64_t)&gCpus[0]);
if (cpu->Self != cpu)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%llX cpu=0x%llX\n",
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%016llX cpu=0x%016llX\n",
(uint64_t)cpu->Self, (uint64_t)cpu);
if (KeGetCurrentCpuIndex() != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n");
@@ -776,15 +776,17 @@ void KiRunCoreHardeningSelfTests(void) {
if (KeGetCurrentTss() != &cpu->Tss)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n");
if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64))
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%llX iomap=%u\n",
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%016llX iomap=%u\n",
cpu->Tss.RSP0, cpu->Tss.IoMapBase);
if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%llX gdt=0x%llX\n",
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%016llX gdt=0x%016llX\n",
cpu->Gdtr.Base, (uint64_t)cpu->Gdt);
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* ap = &gCpus[i];
if (!ap->Present || !ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
if (!ap->Present)
continue;
if (!ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n",
i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp);
@@ -795,7 +797,7 @@ void KiRunCoreHardeningSelfTests(void) {
if (!ap->KernelStack || !ap->KernelStackTop)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%llX iomap=%u\n",
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%016llX iomap=%u\n",
i, ap->Tss.RSP0, ap->Tss.IoMapBase);
if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i);
code/kernel/ke_semaphore.c
+1 -1
View File
@@ -57,7 +57,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
if (!sem) return;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (!LIST_IS_EMPTY(&sem->WaitList)) {
// Transfer this semaphore release directly to the selected waiter. The
// Transfer this semaphore release directly to the selected waiter. The
// count stays unchanged; the waiter returns success without looping and
// consuming a count later.
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
code/kernel/ke_smp.c
+135 -20
View File
@@ -8,6 +8,8 @@
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <ntlite/status.h>
#include <kernel/ke_smp.h>
#include <kernel/ke_cpu.h>
@@ -22,6 +24,8 @@
#include <kernel/hal/hal_apic.h>
#define AP_TRAMPOLINE_MAX_PHYS 0x000A0000ULL
#define AP_TRAMPOLINE_PREFERRED_MIN 0x00070000ULL
#define AP_TRAMPOLINE_FALLBACK_MIN 0x00010000ULL
#define AP_TRAMPOLINE_BOOT_OFF 0x0800u
#define AP_STACK_PAGES 8u
#define AP_START_TIMEOUT_SPINS 20000000u
@@ -74,6 +78,7 @@ _Static_assert(offsetof(AP_BOOT_BLOCK, State) == 48, "AP_BOOT_BLOCK.State offset
_Static_assert(offsetof(AP_BOOT_BLOCK, Stage) == 52, "AP_BOOT_BLOCK.Stage offset");
static PHYSICAL_ADDRESS gApTrampolinePhys;
static PHYSICAL_ADDRESS gReservedApTrampolinePhys;
static AP_BOOT_BLOCK* gApBootBlock;
static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; }
@@ -94,13 +99,55 @@ static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, u
((uint64_t)((base >> 24) & 0xFF) << 56);
}
void KeSetApplicationProcessorTrampoline(uint64_t trampolinePhys) {
if (trampolinePhys >= PAGE_SIZE &&
trampolinePhys < AP_TRAMPOLINE_MAX_PHYS &&
!(trampolinePhys & (PAGE_SIZE - 1))) {
gReservedApTrampolinePhys = (PHYSICAL_ADDRESS)trampolinePhys;
} else {
gReservedApTrampolinePhys = 0;
}
}
static KSTATUS KiBuildApTrampoline(void) {
if (gApTrampolinePhys)
return STATUS_SUCCESS;
PHYSICAL_ADDRESS frame = MmAllocateFrameBelow(AP_TRAMPOLINE_MAX_PHYS);
if (!frame || frame < PAGE_SIZE || (frame & (PAGE_SIZE - 1)))
PHYSICAL_ADDRESS frame = gReservedApTrampolinePhys;
if (frame) {
if (frame < AP_TRAMPOLINE_FALLBACK_MIN ||
frame >= AP_TRAMPOLINE_MAX_PHYS ||
(frame & (PAGE_SIZE - 1))) {
DbgPrint("SMP: rejecting bootloader AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
frame = 0;
} else {
DbgPrint("SMP: using bootloader-reserved AP trampoline frame 0x%016llX\n",
(uint64_t)frame);
}
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_PREFERRED_MIN,
AP_TRAMPOLINE_MAX_PHYS);
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_FALLBACK_MIN,
AP_TRAMPOLINE_PREFERRED_MIN);
}
if (!frame) {
DbgPrint("SMP: no free PMM frame available below 0x%016llX for AP trampoline\n",
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
return STATUS_NO_MEMORY;
}
DbgPrint("SMP: AP trampoline frame 0x%016llX selected\n", (uint64_t)frame);
uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame);
memset(page, 0, PAGE_SIZE);
@@ -150,7 +197,7 @@ static KSTATUS KiBuildApTrampoline(void) {
while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90);
uint32_t long64Off = (uint32_t)(p - page);
// --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). ---
// --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). ---
KiEmit8(&p, 0x48); KiEmit8(&p, 0xBB); size_t bootBlockPatch = (size_t)(p - page); KiEmit64(&p, 0); // mov rbx, bootblock
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Stage)); KiEmit32(&p, AP_STAGE_LONG64);
KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, State)); KiEmit32(&p, AP_STATE_LONG_MODE);
@@ -201,7 +248,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
}
// APs start from the SIPI reset state, so do not assume the firmware/BSP
// CR4 feature bits are present. The scheduler and context-switch paths
// CR4 feature bits are present. The scheduler and context-switch paths
// use FXSAVE/FXRSTOR on every CPU; without OSFXSR, those instructions
// raise #UD as soon as an AP enters its idle thread.
uintptr_t cr4 = __readcr4();
@@ -282,12 +329,20 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
}
static KSTATUS KiAllocateApStack(KCPU* cpu) {
PHYSICAL_ADDRESS stackPhys = MmAllocateContiguous(AP_STACK_PAGES);
if (!stackPhys) return STATUS_NO_MEMORY;
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
// available, which is fragile on real machines and unrelated to AP startup.
size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE;
void* stack = MmAllocatePool(stackBytes);
if (!stack) {
DbgPrint("SMP: AP%u stack allocation failed bytes=%llu\n",
cpu ? cpu->CpuIndex : 0xFFFFFFFFu, (uint64_t)stackBytes);
return STATUS_NO_MEMORY;
}
cpu->KernelStack = (void*)(PHYS_MAP_BASE + stackPhys);
cpu->KernelStackTop = (uintptr_t)cpu->KernelStack + AP_STACK_PAGES * PAGE_SIZE;
memset(cpu->KernelStack, 0, AP_STACK_PAGES * PAGE_SIZE);
cpu->KernelStack = stack;
cpu->KernelStackTop = (uintptr_t)stack + stackBytes;
memset(cpu->KernelStack, 0, stackBytes);
memset(&cpu->Tss, 0, sizeof(cpu->Tss));
cpu->Tss.IoMapBase = sizeof(TSS64);
cpu->Tss.RSP0 = cpu->KernelStackTop;
@@ -295,6 +350,20 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
return STATUS_SUCCESS;
}
static const char* KiApStageName(uint32_t stage) {
switch (stage) {
case AP_STAGE_NONE: return "none";
case AP_STAGE_LONG64: return "long64";
case AP_STAGE_ENTRY: return "entry";
case AP_STAGE_GS: return "gs";
case AP_STAGE_GDT: return "gdt";
case AP_STAGE_IDT: return "idt";
case AP_STAGE_PARK: return "park";
default: return "unknown";
}
}
static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) {
memset(gApBootBlock, 0, sizeof(*gApBootBlock));
gApBootBlock->Magic = AP_BOOT_MAGIC;
@@ -326,30 +395,53 @@ KSTATUS KeStartApplicationProcessors(void) {
bsp->SchedulerCommand = 0;
bsp->SchedulerActive = 1;
DbgPrint("SMP: APIC mode=%s BSP LAPIC=%u MADT CPUs=%u\n",
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
bspLapic, topo.CpuCount);
uint32_t count = 1;
for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) {
HAL_CPU_INFO* info = &topo.Cpus[i];
DbgPrint("SMP: topology[%u] uid=%u lapic=%u flags=0x%x enabled=%u bsp=%u addressable=%u\n",
i, info->ProcessorUid, info->LapicId, info->Flags,
info->Enabled, info->IsBsp, HalApicCanAddressId(info->LapicId));
if (!info->Enabled) continue;
if (info->LapicId == bspLapic) {
bsp->AcpiProcessorId = info->ProcessorUid;
continue;
}
if (!HalApicCanAddressId(info->LapicId)) {
DbgPrint("SMP: skipping AP uid=%u lapic=%u; current APIC mode cannot address it\n",
info->ProcessorUid, info->LapicId);
continue;
}
KiPrepareCpuRecord(count, info);
count++;
}
gCpuCount = count;
DbgPrint("SMP: discovered %u CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic);
DbgPrint("SMP: discovered %u addressable CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic);
if (gCpuCount <= 1) {
DbgPrint("SMP: no application processors present\n");
return STATUS_SUCCESS;
}
if ((__readcr3() >> 32) != 0)
return STATUS_NOT_SUPPORTED;
if ((__readcr3() >> 32) != 0) {
DbgPrint("SMP: disabled; bootstrap CR3 is above 4GiB (cr3=0x%016llX), AP trampoline cannot load it yet\n",
(uint64_t)__readcr3());
gCpuCount = 1;
return STATUS_SUCCESS;
}
DbgPrint("SMP: bootloader AP trampoline phys=0x%016llX\n",
(uint64_t)gReservedApTrampolinePhys);
st = KiBuildApTrampoline();
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
DbgPrint("SMP: disabled; AP trampoline allocation/build failed status=0x%x\n", st);
gCpuCount = 1;
return STATUS_SUCCESS;
}
uint8_t sipiVector = (uint8_t)(gApTrampolinePhys >> 12);
uint32_t claimed = 0;
@@ -359,13 +451,29 @@ KSTATUS KeStartApplicationProcessors(void) {
if (!cpu->Present) continue;
st = KiAllocateApStack(cpu);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
continue;
}
KiFillBootBlockForCpu(cpu, i);
DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector);
st = HalStartApplicationProcessor(cpu->LapicId, sipiVector);
if (!KSUCCESS(st)) return st;
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u LAPIC=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
i, cpu->LapicId, st);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
continue;
}
uint64_t lastHeartbeat = gApBootBlock->Heartbeat;
int parked = 0;
@@ -382,10 +490,14 @@ KSTATUS KeStartApplicationProcessors(void) {
}
if (!parked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x heartbeat=%llu last=%llu\n",
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
gApBootBlock->Heartbeat, lastHeartbeat);
DbgPrint("SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage,
KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
cpu->Present = 0;
cpu->Started = 0;
cpu->Parked = 0;
cpu->Online = 0;
continue;
}
cpu->Started = 1;
@@ -395,6 +507,9 @@ KSTATUS KeStartApplicationProcessors(void) {
DbgPrint("SMP: AP%u reached kernel park\n", i);
}
if (claimed == 0 && gCpuCount > 1)
DbgPrint("SMP: no APs reached kernel-owned state; continuing BSP-only\n");
DbgPrint("SMP: kernel claimed %u APs\n", claimed);
return STATUS_SUCCESS;
}
@@ -482,7 +597,7 @@ struct SMP_AP_WORK_TEST {
static void KiSmpApControlledWorker(void* context) {
SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context;
if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%llX\n", (uint64_t)arg);
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%016llX\n", (uint64_t)arg);
}
_InterlockedExchange(&arg->Ready, 1);
code/kernel/ke_switch.asm
+4 -4
View File
@@ -50,7 +50,7 @@ KeSwitchContext PROC
; rsp = to->SavedContext
mov rsp, [rdx + THREAD_SavedContext]
; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the
; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the
; incoming thread's user TEB in IA32_KERNEL_GS_BASE; user-mode entry/exit
; swaps it with the kernel GS base using swapgs.
mov rax, [rdx + THREAD_Teb] ; Fetch the incoming user GS/TEB pointer
@@ -80,9 +80,9 @@ KeSwitchContext ENDP
; void KeStartInitialThread(THREAD* to)
; RCX = to
;
; Used by AP scheduler admission. The AP is already in kernel long mode on a
; Used by AP scheduler admission. The AP is already in kernel long mode on a
; AP bootstrap stack; this helper installs the idle thread's stack/TSS
; state and returns directly into that thread's saved context. There is no
; state and returns directly into that thread's saved context. There is no
; outgoing THREAD and no scheduler lock to release.
; -------------------------------------------------------------------------
KeStartInitialThread PROC
@@ -94,7 +94,7 @@ KeStartInitialThread PROC
call KeSetCurrentTssRsp0
add rsp, 40
; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel
; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel
; threads have a NULL TEB, which is fine.
mov rax, [rbx + THREAD_Teb]
mov ecx, 0C0000102h
code/kernel/ke_syscall.asm
+15 -10
View File
@@ -1,7 +1,7 @@
; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64)
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
; slots used by user APC return redirection.
EXTERN KeSyscallDispatch:PROC
@@ -147,8 +147,8 @@ KeSyscallEntry ENDP
; Fault stubs - snapshot a full user fault frame before entering C.
;
; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless
; by itself. The captured user RSP lets the C crash dumper read [RSP], which is
; usually the return address immediately after the bad CALL. The GPR snapshot
; by itself. The captured user RSP lets the C crash dumper read [RSP], which is
; usually the return address immediately after the bad CALL. The GPR snapshot
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
@@ -239,7 +239,8 @@ KeFaultDE PROC
jmp de_call
de_kernel_frame:
mov qword ptr [rax + KF_Rsp], 0
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
de_call:
@@ -304,7 +305,8 @@ KeFaultGP PROC
jmp gp_call
gp_kernel_frame:
mov qword ptr [rax + KF_Rsp], 0
lea r10, [rdx+32] ; same-privilege fault: original RSP is above ERR/RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
gp_call:
@@ -370,7 +372,8 @@ KeFaultPF PROC
jmp pf_call
pf_kernel_frame:
mov qword ptr [rax + KF_Rsp], 0
lea r10, [rdx+32] ; same-privilege #PF frame has ERR/RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
pf_call:
@@ -438,7 +441,8 @@ StubName PROC
jmp noerr_call
noerr_kernel_frame:
mov qword ptr [rax + KF_Rsp], 0
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
noerr_call:
@@ -502,7 +506,8 @@ StubName PROC
jmp err_call
err_kernel_frame:
mov qword ptr [rax + KF_Rsp], 0
lea r10, [rdx+32] ; same-privilege error-code frame has ERR/RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
err_call:
@@ -516,8 +521,8 @@ ENDM
EXTERN KeHandleNmiIpi:PROC
PUBLIC KeNmiIpiStub
KeNmiIpiStub PROC
; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
; the NMI was a panic/TLB-control event and is fully handled. If it
; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
; the NMI was a panic/TLB-control event and is fully handled. If it
; returns 0, restore the interrupted context and fall through to the normal
; #NMI fault stub so unexpected NMIs still panic with a useful frame.
push rax
code/kernel/ke_syscall.c
+61 -51
View File
@@ -42,6 +42,16 @@
#include <kernel/ke_apc.h>
#include <kernel/ke_queue.h>
#include <kernel/ke_debug.h>
#ifndef PROCESS_LAUNCH_DEBUG
#define PROCESS_LAUNCH_DEBUG 0
#endif
#if PROCESS_LAUNCH_DEBUG
#define ProcDbg(...) DbgPrint(__VA_ARGS__)
#else
#define ProcDbg(...) ((void)0)
#endif
#include <kernel/mm_virt.h>
#include <kernel/ldr_image.h>
#include <kernel/syscall_nr.h>
@@ -49,6 +59,7 @@
#include <kernel/io_file.h>
#include <kernel/net_socket.h>
#include <kernel/ke_trap.h>
#include <kernel/ke_input.h>
#include <ntlite/syscall.h>
// ---------------------------------------------------------------------------
@@ -121,7 +132,7 @@ static KSTATUS KiAllocHandleForCurrentWithRights(OBJECT* obj, uint32_t rights, u
KSTATUS s = KeAllocateHandleWithRights(proc, obj, rights, handleOut);
// Most syscall create/open paths receive a fresh creator reference from the
// subsystem. The handle table takes its own reference; this helper always
// subsystem. The handle table takes its own reference; this helper always
// drops the creator reference so the object is owned only by the returned
// handle (and any subsystem-internal refs).
ObDereferenceObject(obj);
@@ -752,7 +763,7 @@ static void SyscallUserAsyncKernelRoutine(KAPC* apc,
(void)systemArgument2;
SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc);
DbgPrint("USER/APC async: kernel routine ctx=0x%llX iosb=0x%llX status=%u info=%llu irql=%u\n",
DbgPrint("USER/APC async: kernel routine ctx=0x%016llX iosb=0x%016llX status=%u info=%llu irql=%u\n",
(uint64_t)ctx,
(uint64_t)(uintptr_t)apc->SystemArgument1,
(uint32_t)(uintptr_t)apc->SystemArgument2,
@@ -1150,81 +1161,72 @@ static uint64_t SyscallScreenPutChar(uint64_t a0, uint64_t a1, uint64_t a2,
}
// ---------------------------------------------------------------------------
// SYS_KB_POLL (40) -> 1 or 0
// SYS_KB_POLL (64) -> next INPUT_KEY or INPUT_KEY_NONE
// ---------------------------------------------------------------------------
static uint64_t SyscallKbPoll(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!gHal) return 0;
return gHal->Keyboard.Poll();
return (uint64_t)KeInputKbPoll();
}
// ---------------------------------------------------------------------------
// SYS_KB_IS_KEY_DOWN (41) a0=scancode a1=extended -> 1 or 0
// SYS_KB_IS_KEY_DOWN (65) a0=INPUT_KEY -> 1 or 0
// ---------------------------------------------------------------------------
static uint64_t SyscallKbIsKeyDown(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
// Check gHal and ensure the HAL struct actually has the new function pointer
if (!gHal || !gHal->Keyboard.IsKeyDown) return 0;
return gHal->Keyboard.IsKeyDown((uint8_t)a0, (bool)a1) ? 1u : 0u;
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KeInputIsKeyDown((INPUT_KEY)a0) ? 1u : 0u;
}
// ---------------------------------------------------------------------------
// SYS_KB_READ_CHAR (42) -> char (blocks, yields while waiting)
// SYS_KB_READ_CHAR (66) -> char (blocks, yields while waiting)
// ---------------------------------------------------------------------------
static uint64_t SyscallKbReadChar(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!gHal) return 0;
while (!gHal->Keyboard.Poll()) KeYield();
return (uint64_t)(uint8_t)gHal->Keyboard.ReadChar();
while (!KeInputKbPoll()) KeYield();
return (uint64_t)(uint8_t)KeInputReadChar();
}
// ---------------------------------------------------------------------------
// SYS_KB_READ_SCANCODE (43) -> uint8_t (blocks, yields while waiting)
// SYS_KB_READ_KEY (67) -> INPUT_KEY (blocks, yields while waiting)
// ---------------------------------------------------------------------------
static uint64_t SyscallKbReadScancode(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
static uint64_t SyscallKbReadKey(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!gHal) return 0;
while (!gHal->Keyboard.Poll()) KeYield();
return (uint64_t)gHal->Keyboard.ReadScancode();
while (!KeInputKbPoll()) KeYield();
return (uint64_t)KeInputReadKey();
}
// ---------------------------------------------------------------------------
// SYS_KB_GET_MODIFIERS (44) -> uint32_t
// SYS_KB_GET_MODIFIERS (68) -> uint32_t INPUT_MOD_* flags
// ---------------------------------------------------------------------------
static uint64_t SyscallKbGetModifiers(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!gHal) return 0;
return (uint64_t)gHal->Keyboard.GetModifiers();
return (uint64_t)KeInputGetModifiers();
}
// ---------------------------------------------------------------------------
// SYS_MOUSE_POLL (45) -> 1 or 0
// SYS_MOUSE_POLL (69) -> 1 or 0
// ---------------------------------------------------------------------------
static uint64_t SyscallMousePoll(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a0; (void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!gHal) return 0;
return gHal->Mouse.Poll() ? 1u : 0u;
return KeInputMousePoll() ? 1u : 0u;
}
// ---------------------------------------------------------------------------
// SYS_MOUSE_READ (46) a0=*dx a1=*dy a2=*buttons -> 1 or 0
// SYS_MOUSE_READ (70) a0=*dx a1=*dy a2=*buttons -> 1 or 0
// ---------------------------------------------------------------------------
static uint64_t SyscallMouseRead(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a3; (void)a4; (void)a5;
if (!gHal || !a0 || !a1 || !a2) return 0;
if (!a0 || !a1 || !a2) return 0;
int dx = 0, dy = 0;
uint8_t buttons = 0;
if (!gHal->Mouse.Read(&dx, &dy, &buttons)) return 0;
if (!KeInputMouseRead(&dx, &dy, &buttons)) return 0;
if (!KSUCCESS(CopyToUser((void*)(uintptr_t)a0, &dx, sizeof(dx)))) return 0;
if (!KSUCCESS(CopyToUser((void*)(uintptr_t)a1, &dy, sizeof(dy)))) return 0;
@@ -1233,15 +1235,15 @@ static uint64_t SyscallMouseRead(uint64_t a0, uint64_t a1, uint64_t a2,
}
// ---------------------------------------------------------------------------
// SYS_MOUSE_GET_POS (47) a0=*x a1=*y
// SYS_MOUSE_GET_POS (71) a0=*x a1=*y
// ---------------------------------------------------------------------------
static uint64_t SyscallMouseGetPos(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
if (!gHal || !a0 || !a1) return STATUS_INVALID_PARAMETER;
if (!a0 || !a1) return STATUS_INVALID_PARAMETER;
int x = 0, y = 0;
gHal->Mouse.GetPosition(&x, &y);
KeInputMouseGetPosition(&x, &y);
KSTATUS s = KiCopyIntToUser(a0, x);
if (!KSUCCESS(s)) return s;
return KiCopyIntToUser(a1, y);
@@ -1334,22 +1336,25 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
// Read the executable from disk.
FILE_OBJECT* f;
ProcDbg("CreateProcess: path='%v' cmd='%v' callerPid=%u\n", path, cmdlineView, KeGetCurrentProcess() ? KeGetCurrentProcess()->ProcessId : 0);
s = IoOpenFile(path, &f);
if (!KSUCCESS(s)) return s;
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: IoOpenFile failed st=%d\n", s); return s; }
uint64_t size = IoGetFileSize(f);
ProcDbg("CreateProcess: fileSize=%llu\n", size);
void* mem = MmAllocatePool(size);
if (!mem) { IoCloseFile(f); return STATUS_NO_MEMORY; }
if (!mem) { IoCloseFile(f); ProcDbg("CreateProcess: raw image pool alloc failed size=%llu\n", size); return STATUS_NO_MEMORY; }
size_t got;
s = IoReadFile(f, mem, size, &got);
IoCloseFile(f);
if (!KSUCCESS(s)) { MmFreePool(mem); return s; }
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: IoReadFile failed st=%d got=%llu\n", s, (unsigned long long)got); MmFreePool(mem); return s; }
// Create the process object.
PROCESS* proc;
s = KeCreateProcess(pathBuf, &proc);
if (!KSUCCESS(s)) { MmFreePool(mem); return s; }
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: KeCreateProcess failed st=%d\n", s); MmFreePool(mem); return s; }
ProcDbg("CreateProcess: created pid=%u name=%s\n", proc->ProcessId, proc->Name);
// Inherit CWD from parent
PROCESS* parent = KeGetCurrentProcess();
@@ -1368,34 +1373,37 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
path,
cmdlineView,
SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength));
if (!KSUCCESS(s)) { MmFreePool(mem); ObDereferenceObject(&proc->Header); return s; }
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: env setup failed st=%d\n", s); MmFreePool(mem); ObDereferenceObject(&proc->Header); return s; }
// Map the PE image into memory.
uintptr_t imageBase = 0;
s = LdrLoadImage(mem, size, &imageBase, true);
s = LdrLoadImageForProcess(proc, mem, size, &imageBase);
MmFreePool(mem);
if (!KSUCCESS(s)) { ObDereferenceObject(&proc->Header); return s; }
if (!KSUCCESS(s)) { ProcDbg("CreateProcess: LdrLoadImage failed st=%d\n", s); ObDereferenceObject(&proc->Header); return s; }
ProcDbg("CreateProcess: image mapped pid=%u base=%016llx size=%llx\n", proc->ProcessId, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(proc, imageBase));
s = LdrResolveImportsForProcess(proc, imageBase);
if (!KSUCCESS(s)) {
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
ProcDbg("CreateProcess: resolve imports failed st=%d pid=%u image=%016llx\n", s, proc->ProcessId, (unsigned long long)imageBase);
MmDebugDumpUserRegions(proc);
ObDereferenceObject(&proc->Header);
return s;
}
ProcDbg("CreateProcess: imports resolved pid=%u modules=%u\n", proc->ProcessId, proc->ModuleCount);
s = LdrProtectImage(imageBase, true);
s = LdrProtectImageForProcess(proc, imageBase, true);
if (!KSUCCESS(s)) {
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
ProcDbg("CreateProcess: protect image failed st=%d pid=%u\n", s, proc->ProcessId);
MmDebugDumpUserRegions(proc);
ObDereferenceObject(&proc->Header);
return s;
}
proc->ImageBase = imageBase;
proc->ImageSize = LdrGetImageSize(imageBase);
proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
PROCESS* caller = KeGetCurrentProcess();
if (!caller) {
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
ObDereferenceObject(&proc->Header);
return STATUS_FAILURE;
}
@@ -1405,7 +1413,6 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
if (!KSUCCESS(s)) {
proc->ImageBase = 0;
proc->ImageSize = 0;
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
ObDereferenceObject(&proc->Header);
return s;
}
@@ -1414,10 +1421,13 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread);
if (!KSUCCESS(s)) {
ProcDbg("CreateProcess: KeCreateThread failed st=%d pid=%u entry=%016llx\n", s, proc->ProcessId, (unsigned long long)entryPoint);
MmDebugDumpUserRegions(proc);
KeCloseHandle(caller, handle);
ObDereferenceObject(&proc->Header);
return s;
}
ProcDbg("CreateProcess: thread created pid=%u tid=%llu entry=%016llx handle=%u\n", proc->ProcessId, (unsigned long long)thread->ThreadId, (unsigned long long)entryPoint, handle);
s = KiCopyHandleToUser((uint64_t)(uintptr_t)req.HandleOut, handle);
if (!KSUCCESS(s)) {
@@ -1427,7 +1437,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
return s;
}
// Drop the creator's local reference. The returned handle and live thread
// Drop the creator's local reference. The returned handle and live thread
// references now own the process lifetime.
ObDereferenceObject(&proc->Header);
return STATUS_SUCCESS;
@@ -1666,7 +1676,7 @@ static uint64_t SyscallGetProcessPid(uint64_t a0, uint64_t a1, uint64_t a2,
// Named Queue Registry
//
// A flat, fixed-size table of { name -> QUEUE* } pairs protected by a
// spinlock. The registry holds one reference to each published queue for
// spinlock. The registry holds one reference to each published queue for
// the lifetime of the entry, so the queue object cannot be destroyed while
// it is registered even if every process closes its own handle to it.
//
@@ -1747,7 +1757,7 @@ static uint64_t SyscallPublishQueue(uint64_t a0, uint64_t a1, uint64_t a2,
// SYS_OPEN_NAMED_QUEUE (61) a0=*name a1=*handleOut
//
// Looks up 'name' in the registry and allocates a handle to the queue in the
// calling process's handle table. Returns STATUS_NOT_FOUND if no match.
// calling process's handle table. Returns STATUS_NOT_FOUND if no match.
// Callers that race against the publisher should retry on NOT_FOUND.
// ---------------------------------------------------------------------------
static uint64_t SyscallOpenNamedQueue(uint64_t a0, uint64_t a1, uint64_t a2,
@@ -2629,7 +2639,7 @@ static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
/* 64 SYS_KB_POLL */ SyscallKbPoll,
/* 65 SYS_KB_IS_KEY_DOWN */ SyscallKbIsKeyDown,
/* 66 SYS_KB_READ_CHAR */ SyscallKbReadChar,
/* 67 SYS_KB_READ_SCANCODE */ SyscallKbReadScancode,
/* 67 SYS_KB_READ_KEY */ SyscallKbReadKey,
/* 68 SYS_KB_GET_MODIFIERS */ SyscallKbGetModifiers,
/* 69 SYS_MOUSE_POLL */ SyscallMousePoll,
/* 70 SYS_MOUSE_READ */ SyscallMouseRead,
code/kernel/ke_thread.c
+31 -22
View File
@@ -16,7 +16,7 @@
#define FPU_FCW_DEFAULT 0x037Fu
#define FPU_MXCSR_DEFAULT 0x1F80u
// These offsets are consumed directly by ke_switch.asm. Keep the asserts
// These offsets are consumed directly by ke_switch.asm. Keep the asserts
// here so THREAD layout edits fail at build time instead of corrupting
// context switches at runtime.
#define STATIC_ASSERT_OFFSET(type, field, expected) \
@@ -41,6 +41,20 @@ static void KiThreadDestroy(OBJECT* obj) {
thread->ExitEvent = (EVENT*)0;
}
PROCESS* ownerForUserRegions = thread->Owner;
if (ownerForUserRegions) {
if (thread->Teb && thread->TebSize) {
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
}
if (thread->UserStackBase) {
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase);
thread->UserStackBase = (void *)0;
}
}
if (thread->Owner) {
PROCESS* owner = thread->Owner;
uint32_t remaining = 0;
@@ -63,17 +77,6 @@ static void KiThreadDestroy(OBJECT* obj) {
MmFreePool(thread->KernelStackBase);
thread->KernelStackBase = (void *)0;
}
if (thread->Teb && thread->TebSize) {
MmFreeUserMemory(thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
}
if (thread->UserStackBase) {
MmFreeUserMemory(thread->UserStackBase);
thread->UserStackBase = (void *)0;
}
}
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
@@ -119,8 +122,14 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
thread->TssRsp0 = thread->KernelStackTop;
void* ustack = MmAllocateUserMemory(USTACK_SIZE);
if (!ustack) {
KSTATUS s;
void* ustack = (void*)0;
s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
UserRegionStack,
&ustack);
if (!KSUCCESS(s) || !ustack) {
MmFreePool(kstack);
thread->KernelStackBase = (void*)0;
ObDereferenceObject(&thread->Header);
@@ -128,14 +137,14 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
}
thread->UserStackBase = ustack;
// Ring-3 starts at the image entry via IRET, not CALL, so there is no
// pushed return address. The x64 ABI expects RSP % 16 == 8 at function
// pushed return address. The x64 ABI expects RSP % 16 == 8 at function
// entry; bias the initial stack as if a call had happened so compiler
// generated aligned SSE spills (movaps) are safe.
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8;
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemory(sizeof(SYS_TEB));
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
if (!teb) {
MmFreeUserMemory(ustack);
MmFreeUserMemoryForProcess(owner, ustack);
thread->UserStackBase = (void*)0;
MmFreePool(kstack);
thread->KernelStackBase = (void*)0;
@@ -177,12 +186,12 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
// Allocate the exit event before the thread becomes visible. Manual-reset
// so multiple joiners (if any) all wake up when the thread exits.
EVENT* exitEvent;
KSTATUS s = KeEventCreate(EventManualReset, 0, &exitEvent);
s = KeEventCreate(EventManualReset, 0, &exitEvent);
if (!KSUCCESS(s)) {
if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb);
if (thread->Teb && thread->TebSize) MmFreeUserMemoryForProcess(owner, thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
MmFreeUserMemory(ustack);
MmFreeUserMemoryForProcess(owner, ustack);
thread->UserStackBase = (void*)0;
MmFreePool(kstack);
thread->KernelStackBase = (void*)0;
@@ -238,7 +247,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority,
uint64_t affinityMask,
THREAD** out) {
// Kernel worker threads are owned by the system process. The scheduler's
// Kernel worker threads are owned by the system process. The scheduler's
// hand-built CPU idle thread stays attached to PID 0 (idle), while driver,
// async I/O, and init worker threads are grouped under PID 1 (system).
PROCESS* owner = KeGetSystemProcess();
@@ -331,7 +340,7 @@ __declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uin
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
// Called with the synchronization object's spinlock held, so the caller is
// normally at DISPATCH_LEVEL here. This function only marks intent to
// normally at DISPATCH_LEVEL here. This function only marks intent to
// block; the actual scheduling point is KeCommitBlockThread at PASSIVE.
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
code/kernel/ke_trap.c
+6 -6
View File
@@ -56,7 +56,7 @@ KTRAP_FRAME* KeGetCurrentTrapFrame(void) {
void KiEnterTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame);
KePanic(KE_PANIC_CODE_TRAP, "KiEnterTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
THREAD* t = KeGetCurrentThread();
if (!t) return;
@@ -68,13 +68,13 @@ void KiEnterTrapFrame(KTRAP_FRAME* frame) {
void KiExitTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame);
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: invalid trap frame 0x%016llX\n", (uint64_t)frame);
// Give alertable user-mode returns a chance to redirect through the
// registered user APC dispatcher before committing IRET slots.
KiCheckForUserApcDelivery(frame);
// Commit editable return state. Kernel-mode interrupts have no RSP/SS
// Commit editable return state. Kernel-mode interrupts have no RSP/SS
// hardware slots, so those pointers are NULL in that case.
if (frame->RipSlot) *frame->RipSlot = frame->Rip;
if (frame->CsSlot) *frame->CsSlot = frame->Cs;
@@ -85,7 +85,7 @@ void KiExitTrapFrame(KTRAP_FRAME* frame) {
THREAD* t = KeGetCurrentThread();
if (t) {
if (t->TrapFrame != frame)
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: trap-frame stack mismatch top=0x%llX frame=0x%llX\n",
KePanic(KE_PANIC_CODE_TRAP, "KiExitTrapFrame: trap-frame stack mismatch top=0x%016llX frame=0x%016llX\n",
(uint64_t)t->TrapFrame, (uint64_t)frame);
t->TrapFrame = frame->PreviousFrame;
}
@@ -98,11 +98,11 @@ void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
if (!f)
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: no current trap frame for nr=%llu\n", nr);
if (f->Magic != KTRAP_FRAME_MAGIC)
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: bad magic 0x%llX\n", f->Magic);
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: bad magic 0x%016llX\n", f->Magic);
if (f->Kind != KTRAP_KIND_SYSCALL)
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: kind=%u, expected syscall\n", f->Kind);
if (!KeTrapFrameFromUser(f) || f->PreviousMode != KTRAP_MODE_USER)
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: syscall did not come from user cs=0x%llX mode=%u\n",
KePanic(KE_PANIC_CODE_TRAP, "syscall trap validation: syscall did not come from user cs=0x%016llX mode=%u\n",
f->Cs, f->PreviousMode);
if (f->Rax != nr || f->Rcx != a0 || f->Rdx != a1 || f->R8 != a2 ||
f->R9 != a3 || f->R10 != a4 || f->R11 != a5) {
code/kernel/kmain.c
+10 -73
View File
@@ -2,6 +2,8 @@
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <ntlite/boot_info.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_virt.h>
@@ -22,6 +24,7 @@
#include <kernel/io_file.h>
#include <kernel/ke_selftest.h>
#include <kernel/ke_smp.h>
#include <kernel/ke_input.h>
#include <common/ini_parser.h>
typedef struct {
@@ -31,52 +34,9 @@ typedef struct {
typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *);
// HAL pointer cached at boot so kernel threads can reach it without
// threading a parameter through every call chain.
// HAL pointer cached at boot for subsystems that still need platform services.
static const HAL_API* gKmainHal;
// Kernel-owned DPC wrappers for HAL input devices. The HAL keeps the tiny
// hardware-specific ISR fast and exposes a bottom-half entry point; the kernel
// owns the actual DPC objects and scheduling policy.
static KDPC gKeyboardDpc;
static KDPC gMouseDpc;
static void KeyboardDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
if (gKmainHal && gKmainHal->Keyboard.Dpc)
gKmainHal->Keyboard.Dpc();
}
static void KiMouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
if (gKmainHal && gKmainHal->Mouse.Dpc)
gKmainHal->Mouse.Dpc();
}
static void KeyboardIrqWrapper(void) {
if (gKmainHal && gKmainHal->Keyboard.Isr)
gKmainHal->Keyboard.Isr();
if (gKmainHal && gKmainHal->Keyboard.Dpc)
KeInsertQueueDpc(&gKeyboardDpc, (void*)0, (void*)0);
}
static void KiMouseIrqWrapper(void) {
if (gKmainHal && gKmainHal->Mouse.Isr)
gKmainHal->Mouse.Isr();
if (gKmainHal && gKmainHal->Mouse.Dpc)
KeInsertQueueDpc(&gMouseDpc, (void*)0, (void*)0);
}
static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) {
const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i];
@@ -288,17 +248,14 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
// Read the executable file into a raw buffer.
s = IoOpenFile(SysSvFromCstr(filePath), &f);
DbgPrint("KiLaunchUserProcess: IoOpenFile: %d\n", s);
if (s != STATUS_SUCCESS) {
DbgPrint("Failed to load file: \"%s\"!\n", filePath);
return 0;
}
uint64_t size = IoGetFileSize(f);
DbgPrint("KiLaunchUserProcess: IoGetFileSize: %llu\n", size);
void* mem = MmAllocatePool(size);
DbgPrint("KiLaunchUserProcess: MmAllocatePool: 0x%llX\n", mem);
if (!mem) {
IoCloseFile(f);
return 0;
@@ -306,7 +263,6 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
size_t got = 0;
s = IoReadFile(f, mem, size, &got);
DbgPrint("KiLaunchUserProcess: IoReadFile: %d\n", s);
IoCloseFile(f);
if (!KSUCCESS(s)) {
MmFreePool(mem);
@@ -338,7 +294,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
// Parse the raw PE file and map it into executable memory.
uintptr_t imageBase = 0;
s = LdrLoadImage(mem, size, &imageBase, true);
s = LdrLoadImageForProcess(proc, mem, size, &imageBase);
MmFreePool(mem);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s);
@@ -349,16 +305,14 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
s = LdrResolveImportsForProcess(proc, imageBase);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s);
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
ObDereferenceObject(&proc->Header);
return 0;
}
proc->ImageBase = imageBase;
proc->ImageSize = LdrGetImageSize(imageBase);
proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
DbgPrint("KiLaunchUserProcess: PE Image mapped! Base 0x%llx, Entry 0x%llX\n", imageBase, entryPoint);
// Create the main thread last. KeCreateThread queues it before returning.
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread);
@@ -366,7 +320,6 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
proc->ImageBase = 0;
proc->ImageSize = 0;
MmFreeUserImage((void*)imageBase, LdrGetImageSize(imageBase));
ObDereferenceObject(&proc->Header);
return 0;
}
@@ -388,19 +341,6 @@ void SystemInitThread(void* arg) {
KiResolveSystemImports(args->bootInfo);
KiReclaimBootloaderMemory(args->bootInfo);
args->hal->Screen.Clear(0x000000);
// Print memory status now that the system is fully alive
DbgPrint(
"Memory Status:\n"
" -> Free (PMM): %.2f MB\n"
" -> Kernel Pool: %.2f MB\n"
" -> User Heap: %.2f MB\n",
(float)MmFreeFrameCount() * PAGE_SIZE / (1024.0f * 1024.0f),
(float)MmGetKernelPoolUsage() / (1024.0f * 1024.0f),
(float)MmGetUserHeapUsage() / (1024.0f * 1024.0f)
);
if (args->bootInfo->Debug & 2)
KiRunKernelSelfTests(args->hal);
@@ -464,7 +404,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
.FramebufferHeight = bootInfo->FramebufferHeight,
.FramebufferStride = bootInfo->FramebufferStride,
.RsdpPhysical = bootInfo->RsdpPhysical,
.Debug = bootInfo->Debug
.Debug = bootInfo->Debug,
.AllocatePool = MmAllocatePool
};
KSTATUS s = HalInitialize(&halParams);
@@ -482,6 +423,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// the kernel owns KCPU records, per-CPU stacks, GDT/TSS/IDT state, and the
// AP park loop. External device IRQs remain routed to the BSP during this
// step; APs only become runnable after scheduler admission below.
KeSetApplicationProcessorTrampoline((uint64_t)bootInfo->ApTrampolinePhys);
KSTATUS smpStatus = KeStartApplicationProcessors();
if (!KSUCCESS(smpStatus)) {
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to start application processors, status=0x%x\n", smpStatus);
@@ -490,14 +432,9 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeIrqInitialize(hal);
KeInitializeDpcSystem();
KeInitializeDpc(&gKeyboardDpc, KeyboardDpcRoutine, (void*)0);
KeInitializeDpc(&gMouseDpc, KiMouseDpcRoutine, (void*)0);
KeInputInitialize(hal);
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr);
KeIrqRegister(hal->Keyboard.IrqNumber,
hal->Keyboard.Dpc ? KeyboardIrqWrapper : hal->Keyboard.Isr);
KeIrqRegister(hal->Mouse.IrqNumber,
hal->Mouse.Dpc ? KiMouseIrqWrapper : hal->Mouse.Isr);
HalRegisterScheduler(KeSchedulerTick);
code/kernel/ldr_image.c
+157 -21
View File
@@ -8,6 +8,16 @@
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#ifndef LDR_DEBUG_VERBOSE
#define LDR_DEBUG_VERBOSE 0
#endif
#if LDR_DEBUG_VERBOSE
#define LdrDbg(...) DbgPrint(__VA_ARGS__)
#else
#define LdrDbg(...) ((void)0)
#endif
int _stricmp(const char* a, const char* b);
// ---------------------------------------------------------------------------
@@ -78,6 +88,42 @@ typedef struct {
uint32_t Characteristics;
} SECTION_HEADER;
static size_t KiAlignPeSize(size_t value) {
return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
}
static KSTATUS KiComputeRequiredImageSize(const PE_HEADER* pe, size_t fileSize, size_t* outSize) {
if (!pe || !outSize) return STATUS_INVALID_PARAMETER;
const OPT_HEADER64* opt = &pe->OptionalHeader;
size_t required = KiAlignPeSize(opt->SizeOfImage);
size_t headers = KiAlignPeSize(opt->SizeOfHeaders);
if (headers > required) required = headers;
const SECTION_HEADER* sec = (const SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
for (int i = 0; i < pe->NumberOfSections; i++) {
size_t va = (size_t)sec[i].VirtualAddress;
size_t virt = (size_t)sec[i].VirtualSize;
size_t raw = (size_t)sec[i].SizeOfRawData;
size_t span = virt ? virt : raw;
if (raw > span) span = raw;
if (!span) continue;
if (va > (size_t)-1 - span) return STATUS_NO_MEMORY;
size_t end = KiAlignPeSize(va + span);
if (end > required) required = end;
if (raw) {
size_t rawBase = (size_t)sec[i].PointerToRawData;
if (rawBase > fileSize || raw > fileSize - rawBase) return STATUS_FAILURE;
}
}
if (!required) return STATUS_FAILURE;
*outSize = required;
return STATUS_SUCCESS;
}
typedef struct {
uint32_t Characteristics;
uint32_t TimeDateStamp;
@@ -246,33 +292,77 @@ void* LdrGetExportForProcess(PROCESS* process, const char* moduleName, const cha
}
size_t LdrGetImageSize(uintptr_t base) {
if (!base) return 0;
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
if (pe->Signature != PE_SIGNATURE) return 0;
return (size_t)pe->OptionalHeader.SizeOfImage;
}
static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
if (!imageBase) return;
size_t imageSize = LdrGetImageSize(imageBase);
if (user) MmFreeUserImage((void*)imageBase, imageSize);
else MmFreePool((void*)imageBase);
size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
if (!base) return 0;
if (process) {
size_t regionSize = MmGetUserRegionSize(process, (const void*)base);
if (regionSize) return regionSize;
}
size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base);
if (globalRegionSize) return globalRegionSize;
return LdrGetImageSize(base);
}
KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) {
static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return;
size_t imageSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase);
if (user) MmFreeUserImageForProcess(process, (void*)imageBase, imageSize);
else MmFreePool((void*)imageBase);
}
static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user);
}
KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return STATUS_INVALID_PARAMETER;
size_t mappedSize = user ? LdrGetMappedImageSize(process, imageBase) : LdrGetImageSize(imageBase);
if (!mappedSize) return STATUS_FAILURE;
uint32_t peOffset = *(uint32_t *)(imageBase + 0x3C);
if ((size_t)peOffset > mappedSize || mappedSize - (size_t)peOffset < sizeof(uint32_t) + 20)
return STATUS_FAILURE;
PE_HEADER* pe = (PE_HEADER *)(imageBase + peOffset);
if (pe->Signature != PE_SIGNATURE) return STATUS_FAILURE;
SECTION_HEADER* sec = (SECTION_HEADER *)((uint8_t *)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader;
size_t sectionTableSize = (size_t)pe->NumberOfSections * sizeof(SECTION_HEADER);
if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset)
return STATUS_FAILURE;
SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset);
for (int i = 0; i < pe->NumberOfSections; i++) {
size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData;
if (!protectSize) continue;
uintptr_t secBase = imageBase + (uintptr_t)sec[i].VirtualAddress;
uintptr_t secEnd = secBase + protectSize;
uintptr_t mapEnd = imageBase + mappedSize;
if (secBase < imageBase || secBase >= mapEnd) {
LdrDbg("LDR: protect skipping out-of-range section %d base=%016llx size=%llx image=%016llx mapped=%llx\n",
i, (unsigned long long)secBase, (unsigned long long)protectSize,
(unsigned long long)imageBase, (unsigned long long)mappedSize);
continue;
}
if (secEnd < secBase) return STATUS_NO_MEMORY;
if (secEnd > mapEnd) {
LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n",
i, (unsigned long long)secEnd, (unsigned long long)mapEnd);
protectSize = (size_t)(mapEnd - secBase);
}
uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE)
prot |= PAGE_WRITE;
@@ -280,12 +370,27 @@ KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) {
(sec[i].Characteristics & IMAGE_SCN_CNT_CODE)))
prot |= PAGE_NX;
KSTATUS s = MmSetPageProtect(imageBase + sec[i].VirtualAddress, protectSize, prot);
if (!KSUCCESS(s)) return s;
KSTATUS s;
if (user) {
s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot);
} else {
s = MmSetPageProtect(secBase, protectSize, prot);
}
if (!KSUCCESS(s)) {
LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
i, (unsigned long long)secBase, (unsigned long long)protectSize,
(unsigned long long)prot, s,
(unsigned long long)imageBase, (unsigned long long)mappedSize);
return s;
}
}
return STATUS_SUCCESS;
}
KSTATUS LdrProtectImage(uintptr_t imageBase, bool user) {
return LdrProtectImageForProcess(user ? (PROCESS*)0 : (PROCESS*)0, imageBase, user);
}
// ---------------------------------------------------------------------------
// Entry point lookup
// ---------------------------------------------------------------------------
@@ -368,6 +473,7 @@ static KSTATUS LdrResolveImportsCommon(
//Compatibility remap.
dllName = KiCanonicalModuleName(dllName);
LdrDbg("LDR: resolve import dll=%s image=%016llx pid=%u\n", dllName, (unsigned long long)imageBase, process ? process->ProcessId : 0);
//
// Ensure dependency is loaded into the process
@@ -469,7 +575,7 @@ static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uint
strncpy(mod->Name, name, 31);
mod->Name[31] = '\0';
mod->Base = base;
mod->Size = LdrGetImageSize(base);
mod->Size = LdrGetMappedImageSize(process, base);
return STATUS_SUCCESS;
}
@@ -500,6 +606,7 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
}
uint64_t fileSize = IoGetFileSize(f);
LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize);
void* mem = MmAllocatePool((size_t)fileSize);
if (!mem) {
IoCloseFile(f);
@@ -515,39 +622,45 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
}
uintptr_t imageBase = 0;
s = LdrLoadImage(mem, (size_t)fileSize, &imageBase, true);
s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase);
MmFreePool(mem);
if (!KSUCCESS(s)) {
DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s);
return s;
}
LdrDbg("LDR: module mapped pid=%u file=%s base=%016llx size=%llx\n",
process->ProcessId, filename, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(process, imageBase));
s = LdrResolveImportsForProcess(process, imageBase);
if (!KSUCCESS(s)) {
KiFreeLoadedImage(imageBase, true);
KiFreeLoadedImageForProcess(process, imageBase, true);
return s;
}
s = LdrProtectImage(imageBase, true);
s = LdrProtectImageForProcess(process, imageBase, true);
if (!KSUCCESS(s)) {
KiFreeLoadedImage(imageBase, true);
KiFreeLoadedImageForProcess(process, imageBase, true);
return s;
}
s = LdrRegisterProcessModule(process, filename, imageBase);
if (!KSUCCESS(s)) {
KiFreeLoadedImage(imageBase, true);
KiFreeLoadedImageForProcess(process, imageBase, true);
return s;
}
LdrDbg("LDR: module registered pid=%u name=%s base=%016llx modules=%u\n",
process->ProcessId, filename, (unsigned long long)imageBase, process->ModuleCount);
if (baseOut) *baseOut = imageBase;
return STATUS_SUCCESS;
}
// LdrLoadImage - load a PE from a raw buffer into virtual space.
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER;
if (user && !process) return STATUS_INVALID_PARAMETER;
if (size < 0x40) return STATUS_FAILURE;
const uint8_t* bytes = (const uint8_t*)buffer;
@@ -579,10 +692,23 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u
PE_HEADER* pe = (PE_HEADER *)(bytes + peOffset);
OPT_HEADER64* opt = &pe->OptionalHeader;
size_t imageSize = opt->SizeOfImage;
size_t imageSize = 0;
KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize);
if (!KSUCCESS(sizeStatus)) return sizeStatus;
void* mem = user ? MmAllocateUserImage(imageSize) : MmAllocatePool(imageSize);
if (!mem) return STATUS_NO_MEMORY;
LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n",
opt->SizeOfImage,
(unsigned long long)imageSize,
(unsigned long long)opt->ImageBase,
pe->NumberOfSections,
user ? 1 : 0,
process ? process->ProcessId : 0);
void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize);
if (!mem) {
DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n",
(unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0);
return STATUS_NO_MEMORY;
}
memset(mem, 0, imageSize);
memcpy(mem, buffer, opt->SizeOfHeaders);
@@ -591,7 +717,9 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u
void* dst = (uint8_t *)mem + sec[i].VirtualAddress;
const void* src = (const uint8_t *)buffer + sec[i].PointerToRawData;
if (sec[i].SizeOfRawData) {
memcpy(dst, src, sec[i].SizeOfRawData < sec[i].VirtualSize ? sec[i].SizeOfRawData : sec[i].VirtualSize);
size_t copySize = sec[i].SizeOfRawData;
if (sec[i].VirtualSize && sec[i].VirtualSize < copySize) copySize = sec[i].VirtualSize;
memcpy(dst, src, copySize);
}
uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0);
@@ -608,3 +736,11 @@ KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool u
*baseOut = (uintptr_t)mem;
return STATUS_SUCCESS;
}
KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) {
return KiLdrLoadImageCommon(process, buffer, size, baseOut, true);
}
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user);
}
code/kernel/mm_phys.c
+29 -8
View File
@@ -101,27 +101,48 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
// ---------------------------------------------------------------------------
// Public API
// ---------------------------------------------------------------------------
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) {
if (maxExclusive <= PAGE_SIZE) return (PHYSICAL_ADDRESS)0;
static PHYSICAL_ADDRESS KiPopFrameAtIndexLocked(size_t idx) {
PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx];
size_t tail = gFrameStack.Top - idx - 1;
for (size_t j = 0; j < tail; j++)
gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j];
gFrameStack.Top--;
return frame;
}
PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive) {
minInclusive = (minInclusive + PAGE_SIZE - 1) & PAGE_MASK;
maxExclusive &= PAGE_MASK;
if (maxExclusive <= minInclusive)
return (PHYSICAL_ADDRESS)0;
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
// Search high-to-low within the requested window. For low-memory users
// such as the AP trampoline this naturally prefers the traditional
// 0x70000-0x9f000 area and avoids consuming lower real-mode space first.
for (size_t i = gFrameStack.Top; i > 0; i--) {
size_t idx = i - 1;
PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx];
if (frame && frame < maxExclusive && !(frame & (PAGE_SIZE - 1))) {
size_t tail = gFrameStack.Top - idx - 1;
for (size_t j = 0; j < tail; j++)
gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j];
gFrameStack.Top--;
if (frame >= minInclusive && frame < maxExclusive &&
!(frame & (PAGE_SIZE - 1))) {
frame = KiPopFrameAtIndexLocked(idx);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return frame;
}
}
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return (PHYSICAL_ADDRESS)0;
}
PHYSICAL_ADDRESS MmAllocateFrame(void) {
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) {
return MmAllocateFrameInRange(PAGE_SIZE, maxExclusive);
}
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void) {
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
if (gFrameStack.Top == 0) {
KeReleaseSpinlock(&gFrameStack.Lock, flags);
code/kernel/mm_virt.c
+628 -94
View File
@@ -16,6 +16,7 @@
#include <string.h>
#include <intrin.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_process.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
@@ -139,7 +140,7 @@ void MmFlushTlbForKernelMapping(uintptr_t virt) {
if (acks < targets) {
KePanic(KE_PANIC_CODE_GENERIC,
"TLB shootdown timeout: va=0x%llX targets=%ld acks=%ld selfCpu=%u\n",
"TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n",
(uint64_t)virt, targets, acks, selfIndex);
}
}
@@ -506,6 +507,35 @@ typedef struct HEAP {
SPINLOCK Lock;
} HEAP;
static int KiHeapEnsureMapped(HEAP* h, uintptr_t start, size_t size) {
if (!h || !size) return 1;
if (start < h->Start) return 0;
if (start > (uintptr_t)-1 - (size - 1)) return 0;
uintptr_t begin = start & PAGE_MASK;
uintptr_t end = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
if (end < begin || end > h->Limit) return 0;
for (uintptr_t va = begin; va < end; va += PAGE_SIZE) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
KSTATUS qs = MmQueryPage(va, &phys, &flags);
if (KSUCCESS(qs) && (flags & PAGE_PRESENT))
continue;
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) return 0;
KSTATUS ms = MmMapPage(va, frame, h->MapFlags);
if (!KSUCCESS(ms)) {
MmFreeFrame(frame);
return 0;
}
memset((void*)va, 0, PAGE_SIZE);
DbgPrint("MM: repaired missing heap page va=%016llx\n", (unsigned long long)va);
}
return 1;
}
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int KiHeapGrow(HEAP* h, size_t need) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
@@ -521,6 +551,9 @@ static int KiHeapGrow(HEAP* h, size_t need) {
memset((void *)va, 0, PAGE_SIZE);
}
if (!KiHeapEnsureMapped(h, h->Brk, grow))
return 0;
// Make the new pages a single free block.
BLOCK_HDR* blk = (BLOCK_HDR *)h->Brk;
blk->Magic = BLOCK_MAGIC;
@@ -547,34 +580,18 @@ static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
}
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
(void)h;
if (!blk || blk->Free == 0) return;
while (h->Brk > h->Start) {
uintptr_t blockEnd = (uintptr_t)blk + HDR_SIZE + blk->Size + FTR_SIZE;
if (blockEnd != h->Brk) break;
// Keep at least one page for the block header/footer and a small payload.
if (blk->Size < PAGE_SIZE + sizeof(void*)) break;
uintptr_t pageStart = h->Brk - PAGE_SIZE;
if (pageStart <= (uintptr_t)blk) break; // would unmap the page containing the header
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(pageStart);
if (!phys) break;
if (!KSUCCESS(MmUnmapPage(pageStart))) {
break;
}
MmFreeFrame(phys);
h->Brk -= PAGE_SIZE;
blk->Size -= PAGE_SIZE;
}
if (blk->Free) {
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
// Do not decommit kernel-pool tail pages for now. The pool allocator keeps
// free-list nodes and boundary tags inside the managed arena; shrinking the
// mapped tail while large launch-failure cleanup is walking/freeing many
// objects can leave stale metadata pointers into unmapped pages. Physical
// pages are still reclaimed for user regions by MmDestroyUserRegion(); pool
// pages remain committed until we have a dedicated, audited heap shrinker.
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
static void* KiHeapAlloc(HEAP* h, size_t size) {
@@ -583,6 +600,11 @@ static void* KiHeapAlloc(HEAP* h, size_t size) {
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
if (h->Brk > h->Start && !KiHeapEnsureMapped(h, h->Start, h->Brk - h->Start)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void*)0;
}
// Best-fit search.
BLOCK_HDR* best = (BLOCK_HDR *)0;
for (BLOCK_HDR* cur = h->FreeList; cur; cur = cur->NextFree) {
@@ -735,12 +757,29 @@ static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
static HEAP gPoolHeap;
// ---------------------------------------------------------------------------
// User heap
// User virtual region arena
// ---------------------------------------------------------------------------
static HEAP gUserHeap;
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
static SPINLOCK gUserImageLock = SPINLOCK_INIT;
static uintptr_t gUserImageNext = USER_LIMIT;
typedef struct USER_VA_REGION {
LIST_ENTRY ProcessLink;
LIST_ENTRY GlobalLink;
PROCESS* Owner;
uintptr_t ReservationBase;
size_t ReservationSize;
uintptr_t Base;
size_t Size;
size_t CommittedSize;
uint64_t PageFlags;
USER_REGION_TYPE Type;
uint32_t Flags;
} USER_VA_REGION;
static LIST_ENTRY gUserRegionList;
static SPINLOCK gUserRegionLock = SPINLOCK_INIT;
static int gUserRegionListReady = 0;
#define USER_REGION_ARENA_BASE 0x0000100000000000ULL
// ---------------------------------------------------------------------------
// Module init
@@ -763,20 +802,11 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
KiHeapGrow(&gPoolHeap, poolSize);
}
// User heap.
gUserHeap.FreeList = (BLOCK_HDR *)0;
gUserHeap.Start = USER_BASE;
gUserHeap.Brk = USER_BASE;
gUserHeap.Limit = USER_LIMIT;
gUserHeap.MapFlags = PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX;
gUserHeap.Lock = (SPINLOCK)SPINLOCK_INIT;
// Executable image mappings grow downward from the top of user space.
// The regular user heap grows upward from USER_BASE. Keeping them in
// separate page-aligned regions prevents W^X section protection from
// changing heap metadata or adjacent allocations.
gUserImageLock = (SPINLOCK)SPINLOCK_INIT;
gUserImageNext = USER_LIMIT;
// User VA arena. One global lower-half virtual-address bucket, with
// per-process ownership tracked by USER_VA_REGION descriptors.
LIST_INIT(&gUserRegionList);
gUserRegionLock = (SPINLOCK)SPINLOCK_INIT;
gUserRegionListReady = 1;
}
void MmActivate(void) {
@@ -791,69 +821,86 @@ __declspec(dllexport) void MmFreePool(void* ptr) { KiHeap
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); }
// ---------------------------------------------------------------------------
// Public user heap API
// User virtual region API
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapAlloc(&gUserHeap, size);
static inline uintptr_t KiAlignUpPtr(uintptr_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
if (align & (align - 1)) return 0;
return (v + (uintptr_t)align - 1) & ~((uintptr_t)align - 1);
}
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
KeAssertIrql(PASSIVE_LEVEL);
KiHeapFree(&gUserHeap, ptr);
static inline size_t KiAlignUpSize(size_t v, size_t align) {
if (align < PAGE_SIZE) align = PAGE_SIZE;
if (align & (align - 1)) return 0;
return (v + align - 1) & ~(align - 1);
}
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapRealloc(&gUserHeap, ptr, newSize);
static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) {
uintptr_t aEnd = aBase + aSize;
uintptr_t bEnd = bBase + bSize;
return aBase < bEnd && bBase < aEnd;
}
static KSTATUS KiFindUserVaRange(size_t reservationSize, size_t alignment, uintptr_t* baseOut) {
if (!baseOut || !reservationSize) return STATUS_INVALID_PARAMETER;
if (!gUserRegionListReady) return STATUS_FAILURE;
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
if (!size) return (void*)0;
uintptr_t candidate = KiAlignUpPtr(USER_REGION_ARENA_BASE, alignment);
if (!candidate) return STATUS_INVALID_PARAMETER;
size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (pagesSize < size) return (void*)0;
for (;;) {
if (candidate < USER_REGION_ARENA_BASE || reservationSize > (size_t)(USER_LIMIT - candidate))
return STATUS_NO_MEMORY;
uintptr_t flags = KeAcquireSpinlock(&gUserImageLock);
uintptr_t next = candidate;
int collided = 0;
if (pagesSize > gUserImageNext || gUserImageNext - pagesSize < gUserHeap.Brk) {
KeReleaseSpinlock(&gUserImageLock, flags);
return (void*)0;
}
uintptr_t base = gUserImageNext - pagesSize;
gUserImageNext = base;
KeReleaseSpinlock(&gUserImageLock, flags);
for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) {
MmFreeUserImage((void*)base, va - base);
return (void*)0;
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
if (KiRangesOverlap(candidate, reservationSize, r->ReservationBase, r->ReservationSize)) {
uintptr_t end = r->ReservationBase + r->ReservationSize;
next = KiAlignUpPtr(end, alignment);
if (!next || next <= candidate) return STATUS_NO_MEMORY;
collided = 1;
break;
}
}
KSTATUS s = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
if (!KSUCCESS(s)) {
MmFreeFrame(frame);
MmFreeUserImage((void*)base, va - base);
return (void*)0;
}
}
memset((void*)base, 0, pagesSize);
return (void*)base;
if (!collided) {
*baseOut = candidate;
return STATUS_SUCCESS;
}
candidate = next;
}
}
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
if (!ptr || !size) return;
static USER_VA_REGION* KiFindUserRegionLocked(PROCESS* process, const void* base) {
if (!process || !base) return (USER_VA_REGION*)0;
uintptr_t b = (uintptr_t)base;
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
if (r->Base == b) return r;
}
return (USER_VA_REGION*)0;
}
uintptr_t base = (uintptr_t)ptr & PAGE_MASK;
size_t pagesSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (pagesSize < size) return;
static USER_VA_REGION* KiFindUserRegionContainingLocked(PROCESS* process, uintptr_t base, size_t size) {
if (!process || !base || !size) return (USER_VA_REGION*)0;
if (base > (uintptr_t)-1 - (size - 1)) return (USER_VA_REGION*)0;
uintptr_t end = base + size;
for (uintptr_t va = base; va < base + pagesSize; va += PAGE_SIZE) {
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
uintptr_t regionEnd = r->Base + r->CommittedSize;
if (base >= r->Base && end <= regionEnd) return r;
}
return (USER_VA_REGION*)0;
}
static void KiUnmapUserRegionPayload(USER_VA_REGION* r) {
if (!r) return;
uintptr_t end = r->Base + r->CommittedSize;
for (uintptr_t va = r->Base; va < end; va += PAGE_SIZE) {
PHYSICAL_ADDRESS phys = MmGetPhysicalAddress(va);
if (!phys) continue;
if (KSUCCESS(MmUnmapPage(va))) {
@@ -862,12 +909,499 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
}
}
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t regionFlags,
USER_REGION_TYPE type, void** outBase) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !outBase || !size) return STATUS_INVALID_PARAMETER;
*outBase = (void*)0;
if (!(pageFlags & PAGE_USER)) pageFlags |= PAGE_USER;
if (!(pageFlags & PAGE_PRESENT)) pageFlags |= PAGE_PRESENT;
size_t payloadSize = (size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (payloadSize < size) return STATUS_NO_MEMORY;
size_t guardLow = (regionFlags & USER_REGION_GUARD_LOW) ? PAGE_SIZE : 0;
size_t guardHigh = (regionFlags & USER_REGION_GUARD_HIGH) ? PAGE_SIZE : 0;
size_t reservationSize = guardLow + payloadSize + guardHigh;
if (reservationSize < payloadSize) return STATUS_NO_MEMORY;
if (alignment < PAGE_SIZE) alignment = PAGE_SIZE;
if (alignment & (alignment - 1)) return STATUS_INVALID_PARAMETER;
USER_VA_REGION* r = (USER_VA_REGION*)MmAllocatePool(sizeof(USER_VA_REGION));
if (!r) return STATUS_NO_MEMORY;
memset(r, 0, sizeof(*r));
LIST_INIT(&r->ProcessLink);
LIST_INIT(&r->GlobalLink);
uintptr_t lockFlags = KeAcquireSpinlock(&gUserRegionLock);
uintptr_t reservationBase = 0;
KSTATUS s = KiFindUserVaRange(reservationSize, alignment, &reservationBase);
if (KSUCCESS(s)) {
r->Owner = process;
r->ReservationBase = reservationBase;
r->ReservationSize = reservationSize;
r->Base = reservationBase + guardLow;
r->Size = size;
r->CommittedSize = payloadSize;
r->PageFlags = pageFlags;
r->Type = type;
r->Flags = regionFlags;
ListInsertTail(&gUserRegionList, &r->GlobalLink);
}
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
if (!KSUCCESS(s)) {
MmFreePool(r);
return s;
}
uintptr_t va = r->Base;
for (; va < r->Base + payloadSize; va += PAGE_SIZE) {
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) {
s = STATUS_NO_MEMORY;
break;
}
s = MmMapPage(va, frame, pageFlags);
if (!KSUCCESS(s)) {
MmFreeFrame(frame);
break;
}
}
if (!KSUCCESS(s)) {
r->CommittedSize = (va > r->Base) ? (size_t)(va - r->Base) : 0;
KiUnmapUserRegionPayload(r);
lockFlags = KeAcquireSpinlock(&gUserRegionLock);
ListRemove(&r->GlobalLink);
KeReleaseSpinlock(&gUserRegionLock, lockFlags);
MmFreePool(r);
return s;
}
if (regionFlags & USER_REGION_ZERO) {
memset((void*)r->Base, 0, payloadSize);
}
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
ListInsertTail(&process->UserRegions, &r->ProcessLink);
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
*outBase = (void*)r->Base;
return STATUS_SUCCESS;
}
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
uintptr_t start = (uintptr_t)base;
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
uintptr_t alignedStart = start & PAGE_MASK;
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
uintptr_t regionBase = r ? r->Base : 0;
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
uint64_t defaultFlags = r ? r->PageFlags : 0;
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
if (!r) {
DbgPrint("MM: ensure user mapping failed: pid=%u range=%016llx-%016llx not inside owned committed region\n",
process->ProcessId,
(unsigned long long)alignedStart,
(unsigned long long)alignedEnd);
return STATUS_NOT_FOUND;
}
uint64_t flags = pageFlags ? pageFlags : defaultFlags;
flags |= PAGE_PRESENT | PAGE_USER;
uint32_t committed = 0;
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
if (va < regionBase || va >= regionEnd) return STATUS_ACCESS_DENIED;
PHYSICAL_ADDRESS phys = 0;
uint64_t existingFlags = 0;
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
if (KSUCCESS(qs) && (existingFlags & PAGE_PRESENT))
continue;
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) return STATUS_NO_MEMORY;
KSTATUS ms = MmMapPage(va, frame, flags);
if (!KSUCCESS(ms)) {
MmFreeFrame(frame);
return ms;
}
memset((void*)va, 0, PAGE_SIZE);
committed++;
}
return STATUS_SUCCESS;
}
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base || !size) return STATUS_INVALID_PARAMETER;
uintptr_t start = (uintptr_t)base;
if (start > (uintptr_t)-1 - (size - 1)) return STATUS_NO_MEMORY;
uintptr_t alignedStart = start & PAGE_MASK;
uintptr_t alignedEnd = (start + size + PAGE_SIZE - 1) & PAGE_MASK;
if (alignedEnd <= alignedStart) return STATUS_NO_MEMORY;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionContainingLocked(process, alignedStart, alignedEnd - alignedStart);
uintptr_t regionBase = r ? r->Base : 0;
uintptr_t regionEnd = r ? (r->Base + r->CommittedSize) : 0;
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
if (!r) {
DbgPrint("MM: protect user region failed: pid=%u range=%016llx-%016llx not owned/committed\n",
process->ProcessId,
(unsigned long long)alignedStart,
(unsigned long long)alignedEnd);
return STATUS_NOT_FOUND;
}
uint64_t finalFlags = pageFlags | PAGE_PRESENT | PAGE_USER;
for (uintptr_t va = alignedStart; va < alignedEnd; va += PAGE_SIZE) {
if (va < regionBase || va >= regionEnd) {
DbgPrint("MM: protect user region hit guard/outside page pid=%u va=%016llx region=%016llx-%016llx\n",
process->ProcessId,
(unsigned long long)va,
(unsigned long long)regionBase,
(unsigned long long)regionEnd);
return STATUS_ACCESS_DENIED;
}
PHYSICAL_ADDRESS phys = 0;
uint64_t existingFlags = 0;
KSTATUS qs = MmQueryPage(va, &phys, &existingFlags);
if (!KSUCCESS(qs) || !(existingFlags & PAGE_PRESENT)) {
PHYSICAL_ADDRESS frame = MmAllocateFrame();
if (!frame) return STATUS_NO_MEMORY;
KSTATUS ms = MmMapPage(va, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
if (!KSUCCESS(ms)) {
MmFreeFrame(frame);
return ms;
}
memset((void*)va, 0, PAGE_SIZE);
DbgPrint("MM: committed missing user page during protect pid=%u va=%016llx\n",
process->ProcessId, (unsigned long long)va);
}
KSTATUS ps = MmSetPageProtect(va, PAGE_SIZE, finalFlags);
if (!KSUCCESS(ps)) {
PHYSICAL_ADDRESS dbgPhys = 0;
uint64_t dbgFlags = 0;
KSTATUS dbg = MmQueryPage(va, &dbgPhys, &dbgFlags);
DbgPrint("MM: protect user page failed pid=%u va=%016llx st=%d query=%d phys=%016llx flags=%016llx final=%016llx\n",
process->ProcessId,
(unsigned long long)va,
ps, dbg,
(unsigned long long)dbgPhys,
(unsigned long long)dbgFlags,
(unsigned long long)finalFlags);
return ps;
}
}
return STATUS_SUCCESS;
}
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process || !base) return STATUS_INVALID_PARAMETER;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
if (!r) {
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
return STATUS_NOT_FOUND;
}
ListRemove(&r->ProcessLink);
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
ListRemove(&r->GlobalLink);
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
KiUnmapUserRegionPayload(r);
MmFreePool(r);
return STATUS_SUCCESS;
}
void MmDestroyAllUserRegions(PROCESS* process) {
KeAssertIrql(PASSIVE_LEVEL);
if (!process) return;
for (;;) {
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
LIST_ENTRY* node = ListPopHead(&process->UserRegions);
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
if (!node) break;
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
uintptr_t globalFlags = KeAcquireSpinlock(&gUserRegionLock);
ListRemove(&r->GlobalLink);
KeReleaseSpinlock(&gUserRegionLock, globalFlags);
KiUnmapUserRegionPayload(r);
MmFreePool(r);
}
}
size_t MmGetUserRegionSize(PROCESS* process, const void* base) {
if (!process || !base) return 0;
uintptr_t procFlags = KeAcquireSpinlock(&process->UserRegionLock);
USER_VA_REGION* r = KiFindUserRegionLocked(process, base);
size_t size = r ? r->Size : 0;
KeReleaseSpinlock(&process->UserRegionLock, procFlags);
return size;
}
size_t MmGetUserRegionSizeByBase(const void* base) {
if (!base) return 0;
uintptr_t b = (uintptr_t)base;
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
if (r->Base == b) {
size_t size = r->Size;
KeReleaseSpinlock(&gUserRegionLock, flags);
return size;
}
}
KeReleaseSpinlock(&gUserRegionLock, flags);
return 0;
}
static const char* KiUserRegionTypeName(USER_REGION_TYPE type) {
switch (type) {
case UserRegionGeneric: return "generic";
case UserRegionHeap: return "heap";
case UserRegionImage: return "image";
case UserRegionStack: return "stack";
case UserRegionTeb: return "teb";
case UserRegionPeb: return "peb";
case UserRegionEnvironment: return "env";
case UserRegionMappedFile: return "mapped";
case UserRegionShared: return "shared";
default: return "?";
}
}
void MmDebugDumpUserRegions(PROCESS* process) {
if (!process) {
DbgPrint("MM: user regions: <no process>\n");
return;
}
DbgPrint("MM: user regions for pid=%u name=%s:\n", process->ProcessId, process->Name);
uintptr_t flags = KeAcquireSpinlock(&process->UserRegionLock);
uint32_t count = 0;
for (LIST_ENTRY* node = process->UserRegions.Fwd; node != &process->UserRegions; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, ProcessLink);
DbgPrint(" #%u %-7s res=%016llx-%016llx base=%016llx size=%llx committed=%llx flags=%x pte=%llx\n",
count,
KiUserRegionTypeName(r->Type),
(unsigned long long)r->ReservationBase,
(unsigned long long)(r->ReservationBase + r->ReservationSize),
(unsigned long long)r->Base,
(unsigned long long)r->Size,
(unsigned long long)r->CommittedSize,
r->Flags,
(unsigned long long)r->PageFlags);
count++;
}
KeReleaseSpinlock(&process->UserRegionLock, flags);
if (!count) DbgPrint(" <none>\n");
}
static void KiDebugDescribePoolAddress(uintptr_t addr) {
DbgPrint("MM: kernel pool start=%016llx brk=%016llx limit=%016llx usage=%llu KiB\n",
(unsigned long long)gPoolHeap.Start,
(unsigned long long)gPoolHeap.Brk,
(unsigned long long)gPoolHeap.Limit,
(unsigned long long)((gPoolHeap.Brk - gPoolHeap.Start) / 1024));
if (addr < gPoolHeap.Start || addr >= gPoolHeap.Limit) return;
if (addr >= gPoolHeap.Brk) {
DbgPrint("MM: address %016llx is inside pool VA range but above committed brk by +0x%llx\n",
(unsigned long long)addr,
(unsigned long long)(addr - gPoolHeap.Brk));
return;
}
uintptr_t flags = KeAcquireSpinlock(&gPoolHeap.Lock);
for (BLOCK_HDR* cur = (BLOCK_HDR*)gPoolHeap.Start;
(uintptr_t)cur < gPoolHeap.Brk;
cur = KiNextBlock(cur)) {
if (cur->Magic != BLOCK_MAGIC) {
DbgPrint("MM: pool walk corrupt at hdr=%p magic=%08x\n", cur, cur->Magic);
break;
}
uintptr_t hdr = (uintptr_t)cur;
uintptr_t data = hdr + HDR_SIZE;
uintptr_t end = data + cur->Size;
uintptr_t ftr = end;
if (addr >= hdr && addr < ftr + FTR_SIZE) {
DbgPrint("MM: address %016llx in pool block hdr=%016llx data=%016llx-%016llx size=%llx free=%u\n",
(unsigned long long)addr,
(unsigned long long)hdr,
(unsigned long long)data,
(unsigned long long)end,
(unsigned long long)cur->Size,
cur->Free);
break;
}
if (KiNextBlock(cur) <= cur) {
DbgPrint("MM: pool walk stopped: non-advancing block hdr=%p size=%llx\n", cur, (unsigned long long)cur->Size);
break;
}
}
KeReleaseSpinlock(&gPoolHeap.Lock, flags);
}
void MmDebugDescribeAddress(uintptr_t addr) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
KSTATUS qs = MmQueryPage(addr, &phys, &flags);
DbgPrint("MM: describe va=%016llx query=%d phys=%016llx flags=%016llx\n",
(unsigned long long)addr,
qs,
(unsigned long long)phys,
(unsigned long long)flags);
if (addr >= POOL_BASE && addr < POOL_LIMIT) {
KiDebugDescribePoolAddress(addr);
return;
}
if (addr >= USER_REGION_ARENA_BASE && addr < USER_LIMIT) {
uintptr_t gl = KeAcquireSpinlock(&gUserRegionLock);
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
if (addr >= r->ReservationBase && addr < r->ReservationBase + r->ReservationSize) {
DbgPrint("MM: address belongs to pid=%u name=%s region=%s res=%016llx-%016llx base=%016llx size=%llx flags=%x\n",
r->Owner ? r->Owner->ProcessId : 0,
r->Owner ? r->Owner->Name : "?",
KiUserRegionTypeName(r->Type),
(unsigned long long)r->ReservationBase,
(unsigned long long)(r->ReservationBase + r->ReservationSize),
(unsigned long long)r->Base,
(unsigned long long)r->Size,
r->Flags);
KeReleaseSpinlock(&gUserRegionLock, gl);
return;
}
}
KeReleaseSpinlock(&gUserRegionLock, gl);
DbgPrint("MM: address is in user arena but not reserved by any USER_VA_REGION\n");
}
}
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size) {
void* base = (void*)0;
if (!process || !size) return (void*)0;
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
USER_REGION_ZERO,
UserRegionHeap,
&base);
return KSUCCESS(s) ? base : (void*)0;
}
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr) {
if (!process || !ptr) return;
(void)MmDestroyUserRegion(process, ptr);
}
__declspec(dllexport) void* MmReallocateUserMemoryForProcess(PROCESS* process, void* ptr, size_t newSize) {
if (!process) return (void*)0;
if (!ptr) return MmAllocateUserMemoryForProcess(process, newSize);
if (!newSize) {
MmFreeUserMemoryForProcess(process, ptr);
return (void*)0;
}
size_t oldSize = MmGetUserRegionSize(process, ptr);
if (!oldSize) return (void*)0;
if (newSize <= oldSize) return ptr;
void* newPtr = MmAllocateUserMemoryForProcess(process, newSize);
if (!newPtr) return (void*)0;
memcpy(newPtr, ptr, oldSize);
MmFreeUserMemoryForProcess(process, ptr);
return newPtr;
}
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
PROCESS* process = KeGetCurrentProcess();
return MmAllocateUserMemoryForProcess(process, size);
}
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
PROCESS* process = KeGetCurrentProcess();
MmFreeUserMemoryForProcess(process, ptr);
}
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
PROCESS* process = KeGetCurrentProcess();
return MmReallocateUserMemoryForProcess(process, ptr, newSize);
}
__declspec(dllexport) void* MmAllocateUserImageForProcess(PROCESS* process, size_t size) {
void* base = (void*)0;
if (!process || !size) return (void*)0;
KSTATUS s = MmCreateUserRegion(process, size, PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
UserRegionImage,
&base);
return KSUCCESS(s) ? base : (void*)0;
}
__declspec(dllexport) void MmFreeUserImageForProcess(PROCESS* process, void* ptr, size_t size) {
(void)size;
if (!process || !ptr) return;
(void)MmDestroyUserRegion(process, ptr);
}
__declspec(dllexport) void* MmAllocateUserImage(size_t size) {
PROCESS* process = KeGetCurrentProcess();
return MmAllocateUserImageForProcess(process, size);
}
__declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size) {
PROCESS* process = KeGetCurrentProcess();
MmFreeUserImageForProcess(process, ptr, size);
}
size_t MmGetKernelPoolUsage(void) {
// Returns bytes committed to the kernel pool
return (size_t)(gPoolHeap.Brk - gPoolHeap.Start);
}
size_t MmGetUserHeapUsage(void) {
// Returns bytes committed to the user heap
return (size_t)(gUserHeap.Brk - gUserHeap.Start);
size_t total = 0;
if (!gUserRegionListReady) return 0;
uintptr_t flags = KeAcquireSpinlock(&gUserRegionLock);
for (LIST_ENTRY* node = gUserRegionList.Fwd; node != &gUserRegionList; node = node->Fwd) {
USER_VA_REGION* r = CONTAINING_RECORD(node, USER_VA_REGION, GlobalLink);
total += r->CommittedSize;
}
KeReleaseSpinlock(&gUserRegionLock, flags);
return total;
}