maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add ntfs.sys standalone driver target, change default usb vhd layout formatting to ntfs, implement Fat32CreateDir for directory creation support in the fat32 file system driver, add a dedicated HAL serial output endpoint for COM1 writes in hal_serial.c and subscribe serial to HAL text streams, introduce waitable kernel timers with KeTimerCreate, KeTimerSet, and KeTimerCancel alongside spinlock-protected global timer list initialization and scheduler tick tracking, implement SyscallWriteFileAsync system call for asynchronous non-blocking file writes, refactor NetSocketConnect and NetSocketSendTo to use generic SYS_SOCKET_ADDRESS_K structures instead of being hardcoded to IPv4, update gethostbyname and DNS name resolution to support IPv6 parsing via ResolveNameFamily, update tcpserve.exe with -4 and -6 command-line flags to accept and display IPv6 addresses, append size and version headers to all kernel telemetry and statistics structures including SYS_PMM_STATS, SYS_CPU_STATS, and SYS_SCHEDULER_STATS, update winmm timeGetTime to look up high-resolution hardware frequencies via SysGetTickFrequency, integrate 14 native smoke test applications.

Browse Source
This commit is contained in:
maksw committed 2026-06-26 15:18:48 +02:00
1 parent 2c4a68a75f
commit ed3d065abd
156 files changed
+10509 -2643
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/hal/hal_main.c
+39 -24
View File
@@ -2,11 +2,11 @@
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <kernel/hal/hal_output.h>
#include <string.h>
#include <stdint.h>
#include <stdarg.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <common/printf.h>
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
@@ -92,26 +92,41 @@ static uint8_t gSlpTypB = 0;
static int gS5Found = 0;
int debugEnabled = 0;
int _fltused; // MSVC floating-point helper anchor
// Implements the hal putchar operation.
static void hal_putchar(char c) {
gApi.Screen.PrintChar(c, 0xCCCCCC, 0x012456);
}
// Implements the dbg Print operation.
void HalDbgPrint(const char* fmt, ...) {
if (!debugEnabled) return;
int _fltused; // MSVC floating-point helper anchor.
// Writes a formatted early HAL diagnostic through the output router.
void HalDbgPrint(const char* fmt, ...) {
if (!debugEnabled || !fmt || !gApi.Output.Write)
return;
char line[1024];
va_list args;
va_start(args, fmt);
vprintf(fmt, args);
int count = vsnprintf_(line, sizeof(line), fmt, args);
va_end(args);
size_t length = 0;
if (count > 0) {
length = (size_t)count;
if (length >= sizeof(line))
length = sizeof(line) - 1u;
}
HAL_OUTPUT_RECORD record;
record.Stream = HAL_OUTPUT_STREAM_DEBUG;
record.Flags = 0;
record.Foreground = 0x00CCCCCCu;
record.Background = 0x00012456u;
record.Buffer = line;
record.Length = length;
gApi.Output.Write(&record);
}
static void *HalPhysToVirt(uint64_t phys) {
return (void *)(phys + gPhysBase);
}
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
// systems may put the real DSDT and PM1 control registers in the extended
// X_* fields, so do not rely only on the legacy 32-bit fields.
#define FADT_DSDT_OFFSET 40u
@@ -155,7 +170,7 @@ static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
// 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(),
// 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(),
// so ignore MMIO GAS entries until the HAL grows MMIO register access here.
if (gas.AddressSpaceId != 0x01 || gas.Address == 0 || gas.Address > 0xFFFF)
return 0;
@@ -257,7 +272,7 @@ static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out)
return 1;
}
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and
// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants.
static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
@@ -343,7 +358,6 @@ static void ParseFadt(const ACPISDTHeader* header) {
}
}
// Implements the add Cpu Record operation.
static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) {
if (gCpuTopology.CpuCount >= HAL_MAX_CPUS)
@@ -470,7 +484,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
}
// Parse HPET.
else if (memcmp(header->Signature, "HPET", 4) == 0) {
// HPET base address block is located at offset 44
// HPET base address block is located at offset 44
// (which is 8 bytes past the 36-byte ACPISDTHeader).
gHpetPhys = *(uint64_t*)((uint8_t*)header + 44) & ~0xFFFULL;
}
@@ -479,13 +493,13 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
// HalShutdown - ACPI S5 soft power-off. Does not return.
static void HalShutdown(void) {
// Silence! Stop the world!
_disable();
// Refuse further maskable interrupts before entering firmware power state.
(void)HalDisableInterrupts();
if (!gS5Found || !gPm1aCntBlk) {
// Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets
// Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets
// put into S3/S4 sleep instead of S5 soft-off.
while (true) __halt();
HalStopProcessor();
}
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10).
@@ -495,8 +509,7 @@ static void HalShutdown(void) {
__outword(gPm1aCntBlk, valA);
if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB);
// Die!
while (true) __halt();
HalStopProcessor();
}
__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) {
@@ -542,14 +555,16 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
gApi.PhysToVirt = HalPhysToVirt;
debugEnabled = initParams->Debug;
// Screen first so DbgPrint works immediately.
HalOutputInitialize(&gApi.Output);
HalSerialInitialize(&gApi.Output);
// Screen follows serial so diagnostics still work without a framebuffer.
HalScreenInitialize(initParams->AllocatePool, initParams->FramebufferBase + initParams->PhysicalWindowBase,
(int)initParams->FramebufferWidth,
(int)initParams->FramebufferHeight,
(int)initParams->FramebufferStride,
&gApi.Screen);
printf_set_putchar(hal_putchar);
HalScreenRegisterOutputEndpoint(&gApi.Output);
HalDbgPrint("HAL: screen %dx%d\n",
initParams->FramebufferWidth, initParams->FramebufferHeight);
code/kernel/hal/hal_output.c
+142
View File
@@ -0,0 +1,142 @@
// HAL output router. This file fans typed text streams out to registered physical and runtime endpoints.
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_output.h>
#define HAL_OUTPUT_ENDPOINT_CAPACITY 8u
typedef struct HAL_OUTPUT_SLOT {
HAL_OUTPUT_ENDPOINT Endpoint;
volatile long Published;
} HAL_OUTPUT_SLOT;
static HAL_OUTPUT_SLOT gOutputSlots[HAL_OUTPUT_ENDPOINT_CAPACITY];
static volatile long gOutputRegistrationLock;
static volatile long long gOutputRecordCount;
static volatile long long gOutputByteCount;
static volatile long long gOutputEndpointWriteCount;
static volatile long long gOutputEmergencyRecordCount;
static volatile long gOutputEndpointCount;
// Acquires the endpoint registration lock.
static void HalOutputAcquireRegistrationLock(void) {
while (_InterlockedCompareExchange(&gOutputRegistrationLock, 1, 0) != 0)
_mm_pause();
}
// Releases the endpoint registration lock.
static void HalOutputReleaseRegistrationLock(void) {
_InterlockedExchange(&gOutputRegistrationLock, 0);
}
// Registers one output endpoint in the fixed endpoint table.
static KSTATUS HalOutputRegisterEndpoint(const HAL_OUTPUT_ENDPOINT* endpoint, uint32_t* endpointIdOut) {
if (!endpoint || !endpointIdOut || !endpoint->Name || !endpoint->Write ||
endpoint->StreamMask == 0 || (endpoint->StreamMask & ~HAL_OUTPUT_STREAM_ALL) != 0)
return STATUS_INVALID_PARAMETER;
KSTATUS status = STATUS_NO_MEMORY;
HalOutputAcquireRegistrationLock();
for (uint32_t i = 0; i < HAL_OUTPUT_ENDPOINT_CAPACITY; i++) {
HAL_OUTPUT_SLOT* slot = &gOutputSlots[i];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) != 0)
continue;
slot->Endpoint = *endpoint;
_ReadWriteBarrier();
_InterlockedExchange(&slot->Published, 1);
_InterlockedIncrement(&gOutputEndpointCount);
*endpointIdOut = i;
status = STATUS_SUCCESS;
break;
}
HalOutputReleaseRegistrationLock();
return status;
}
// Replaces the stream subscription mask for a registered endpoint.
static KSTATUS HalOutputSetEndpointStreams(uint32_t endpointId, uint32_t streamMask) {
if (endpointId >= HAL_OUTPUT_ENDPOINT_CAPACITY ||
(streamMask & ~HAL_OUTPUT_STREAM_ALL) != 0)
return STATUS_INVALID_PARAMETER;
HAL_OUTPUT_SLOT* slot = &gOutputSlots[endpointId];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) == 0)
return STATUS_NOT_FOUND;
_InterlockedExchange((volatile long*)&slot->Endpoint.StreamMask, (long)streamMask);
return STATUS_SUCCESS;
}
// Fans one text record out to every subscribed endpoint.
static void HalOutputWrite(const HAL_OUTPUT_RECORD* record) {
if (!record || !record->Buffer || record->Length == 0 ||
(record->Stream & HAL_OUTPUT_STREAM_ALL) == 0 ||
(record->Stream & (record->Stream - 1u)) != 0)
return;
_InterlockedIncrement64(&gOutputRecordCount);
_InterlockedExchangeAdd64(&gOutputByteCount, (long long)record->Length);
if (record->Flags & HAL_OUTPUT_WRITE_EMERGENCY)
_InterlockedIncrement64(&gOutputEmergencyRecordCount);
for (uint32_t i = 0; i < HAL_OUTPUT_ENDPOINT_CAPACITY; i++) {
HAL_OUTPUT_SLOT* slot = &gOutputSlots[i];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) == 0)
continue;
uint32_t streams = (uint32_t)_InterlockedCompareExchange(
(volatile long*)&slot->Endpoint.StreamMask, 0, 0);
if ((streams & record->Stream) == 0)
continue;
if ((record->Flags & HAL_OUTPUT_WRITE_EMERGENCY) &&
!(slot->Endpoint.Capabilities & HAL_OUTPUT_ENDPOINT_PANIC_SAFE))
continue;
HAL_OUTPUT_ENDPOINT_WRITE write = slot->Endpoint.Write;
void* context = slot->Endpoint.Context;
if (!write)
continue;
write(context, record);
_InterlockedIncrement64(&gOutputEndpointWriteCount);
}
}
// Returns a snapshot of output router counters.
static void HalOutputQueryStats(HAL_OUTPUT_STATS* statsOut) {
if (!statsOut)
return;
memset(statsOut, 0, sizeof(*statsOut));
statsOut->RecordCount = (uint64_t)_InterlockedCompareExchange64(&gOutputRecordCount, 0, 0);
statsOut->ByteCount = (uint64_t)_InterlockedCompareExchange64(&gOutputByteCount, 0, 0);
statsOut->EndpointWriteCount = (uint64_t)_InterlockedCompareExchange64(&gOutputEndpointWriteCount, 0, 0);
statsOut->EmergencyRecordCount = (uint64_t)_InterlockedCompareExchange64(&gOutputEmergencyRecordCount, 0, 0);
statsOut->EndpointCount = (uint32_t)_InterlockedCompareExchange(&gOutputEndpointCount, 0, 0);
}
// Initializes the fixed HAL output endpoint table.
void HalOutputInitialize(HAL_OUTPUT_ROUTER* output) {
if (!output)
return;
memset(gOutputSlots, 0, sizeof(gOutputSlots));
gOutputRegistrationLock = 0;
gOutputRecordCount = 0;
gOutputByteCount = 0;
gOutputEndpointWriteCount = 0;
gOutputEmergencyRecordCount = 0;
gOutputEndpointCount = 0;
memset(output, 0, sizeof(*output));
output->RegisterEndpoint = HalOutputRegisterEndpoint;
output->SetEndpointStreams = HalOutputSetEndpointStreams;
output->Write = HalOutputWrite;
output->QueryStats = HalOutputQueryStats;
output->SerialEndpointId = HAL_OUTPUT_ENDPOINT_INVALID;
output->BootFramebufferEndpointId = HAL_OUTPUT_ENDPOINT_INVALID;
}
code/kernel/hal/hal_processor.c
+42
View File
@@ -0,0 +1,42 @@
// HAL processor control exports. This file owns interrupt masking and processor halt instructions.
#include <stdint.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
// Masks interrupts and returns the previous RFLAGS value.
__declspec(dllexport) uintptr_t HalDisableInterrupts(void) {
uintptr_t flags = __readeflags();
_disable();
return flags;
}
// Restores a previously saved RFLAGS value.
__declspec(dllexport) void HalRestoreInterrupts(uintptr_t flags) {
__writeeflags(flags);
}
// Enables maskable interrupts.
__declspec(dllexport) void HalEnableInterrupts(void) {
_enable();
}
// Enables interrupts and halts until an interrupt arrives.
__declspec(dllexport) void HalWaitForInterrupt(void) {
_enable();
__halt();
}
// Halts once without changing interrupt state.
__declspec(dllexport) void HalHalt(void) {
__halt();
}
// Masks interrupts and permanently stops the current processor.
__declspec(dllexport) _Noreturn void HalStopProcessor(void) {
_disable();
for (;;)
__halt();
}
code/kernel/hal/hal_screen.c
+30 -8
View File
@@ -5,9 +5,9 @@
// each byte is the leftmost pixel.
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_output.h>
static uint32_t* gFb = (uint32_t *)0;
static uint32_t* gBackbuffer = (uint32_t *)0;
@@ -142,7 +142,7 @@ static void ScreenScroll(int lines) {
memmove(shadow, shadow + lines * gStride,
(size_t)(gHeight - lines) * rowBytes);
// Fill the revealed rows with the console background. Do not memset here:
// Fill the revealed rows with the console background. Do not memset here:
// the background color is not zero.
uint32_t* fill = shadow + (gHeight - lines) * gStride;
for (int y = 0; y < lines; y++) {
@@ -170,11 +170,6 @@ static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
int cols = gWidth / FONT_W;
int rows = gHeight / FONT_H;
// Mirror to serial.
while ((__inbyte(0x3FD) & 0x20) == 0);
if (c == '\n') __outbyte(0x3F8, '\r');
__outbyte(0x3F8, c);
if (c == '\n') {
gCursorX = 0;
gCursorY++;
@@ -198,7 +193,7 @@ static void ScreenPrintChar(char c, uint32_t fg, uint32_t bg) {
}
// Mouse cursor – 12×19 monochrome arrow sprite
// 0 = transparent 1 = black outline 2 = white fill
// 0 = transparent 1 = black outline 2 = white fill
// The hotspot is at (0,0) (top-left pixel = tip of the arrow).
#define CURSOR_W 16
#define CURSOR_H 25
@@ -318,6 +313,33 @@ void HalScreenEraseCursor(void) {
gCursorDrawY = -1;
}
// Writes one HAL output record through the boot framebuffer text console.
static void HalScreenWriteOutputRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer)
return;
for (size_t i = 0; i < record->Length; i++)
ScreenPrintChar(record->Buffer[i], record->Foreground, record->Background);
}
// Registers the boot framebuffer text endpoint.
void HalScreenRegisterOutputEndpoint(HAL_OUTPUT_ROUTER* output) {
if (!output || !output->RegisterEndpoint || !gFb || gWidth <= 0 || gHeight <= 0)
return;
HAL_OUTPUT_ENDPOINT endpoint;
endpoint.Name = "boot-framebuffer";
endpoint.StreamMask = HAL_OUTPUT_STREAM_ALL;
endpoint.Capabilities = HAL_OUTPUT_ENDPOINT_PANIC_SAFE;
endpoint.Write = HalScreenWriteOutputRecord;
endpoint.Context = 0;
uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
if (KSUCCESS(output->RegisterEndpoint(&endpoint, &endpointId)))
output->BootFramebufferEndpointId = endpointId;
}
// Populate the HAL_SCREEN vtable.
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
HAL_SCREEN* out) {
code/kernel/hal/hal_serial.c
+78
View File
@@ -0,0 +1,78 @@
// HAL serial output endpoint. This file owns polling COM1 writes and subscribes serial to HAL text streams.
#include <stdint.h>
#include <intrin.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_output.h>
#define HAL_COM1_DATA_PORT 0x03F8u
#define HAL_COM1_STATUS_PORT 0x03FDu
#define HAL_COM1_TX_READY 0x20u
static volatile long gSerialWriteLock;
// Waits until COM1 can accept one byte.
static void HalSerialWaitForTransmit(void) {
while ((__inbyte(HAL_COM1_STATUS_PORT) & HAL_COM1_TX_READY) == 0)
_mm_pause();
}
// Writes one byte to COM1.
static void HalSerialWriteByte(uint8_t value) {
HalSerialWaitForTransmit();
__outbyte(HAL_COM1_DATA_PORT, value);
}
// Acquires the normal serial writer lock.
static void HalSerialAcquireWriteLock(void) {
while (_InterlockedCompareExchange(&gSerialWriteLock, 1, 0) != 0)
_mm_pause();
}
// Releases the normal serial writer lock.
static void HalSerialReleaseWriteLock(void) {
_InterlockedExchange(&gSerialWriteLock, 0);
}
// Writes one output record through polling COM1.
static void HalSerialWriteRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer)
return;
bool emergency = (record->Flags & HAL_OUTPUT_WRITE_EMERGENCY) != 0;
uintptr_t interruptFlags = 0;
if (!emergency) {
interruptFlags = HalDisableInterrupts();
HalSerialAcquireWriteLock();
}
for (size_t i = 0; i < record->Length; i++) {
char c = record->Buffer[i];
if (c == '\n')
HalSerialWriteByte('\r');
HalSerialWriteByte((uint8_t)c);
}
if (!emergency) {
HalSerialReleaseWriteLock();
HalRestoreInterrupts(interruptFlags);
}
}
// Registers the polling serial endpoint.
void HalSerialInitialize(HAL_OUTPUT_ROUTER* output) {
if (!output || !output->RegisterEndpoint)
return;
HAL_OUTPUT_ENDPOINT endpoint;
endpoint.Name = "serial0";
endpoint.StreamMask = HAL_OUTPUT_STREAM_ALL;
endpoint.Capabilities = HAL_OUTPUT_ENDPOINT_PANIC_SAFE;
endpoint.Write = HalSerialWriteRecord;
endpoint.Context = 0;
uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
if (KSUCCESS(output->RegisterEndpoint(&endpoint, &endpointId)))
output->SerialEndpointId = endpointId;
}
code/kernel/hal/hal_timer.c
+10 -4
View File
@@ -81,6 +81,11 @@ static uint64_t QueryTicks(void) {
return (baseTicks1 * TIMER_PERIOD_MS * 1000ull) + subQuantumUs;
}
// Returns the frequency used by QueryTicks.
static uint64_t QueryTickFrequency(void) {
return 1000000ull;
}
// Implements the stall operation.
static void Stall(uint32_t us) {
// Priority 1: Use HPET (Highest Precision).
@@ -224,9 +229,10 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER *o
// after enabling their local APIC.
HalTimerConfigureCurrentCpu();
out->QueryTicks = QueryTicks;
out->Stall = Stall;
out->Isr = TimerIsr;
out->IrqNumber = 0;
out->QueryTicks = QueryTicks;
out->QueryTickFrequency = QueryTickFrequency;
out->Stall = Stall;
out->Isr = TimerIsr;
out->IrqNumber = 0;
return STATUS_SUCCESS;
}
code/kernel/include/kernel/hal/hal_apic.h
-1
View File
@@ -10,7 +10,6 @@ void HalApicUnmaskIrq(uint8_t irq);
void HalApicSendEoi(void);
void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys);
#define HAL_MAX_CPUS 32u
typedef struct HAL_CPU_INFO {
code/kernel/include/kernel/hal/hal_output.h
+14
View File
@@ -0,0 +1,14 @@
// HAL output router internals. This file connects built-in endpoint drivers to the public HAL output table.
#pragma once
#include <ntlite/ddi/hal_api.h>
// Initializes the fixed HAL output endpoint table.
void HalOutputInitialize(HAL_OUTPUT_ROUTER* output);
// Registers the polling serial endpoint.
void HalSerialInitialize(HAL_OUTPUT_ROUTER* output);
// Registers the boot framebuffer text endpoint.
void HalScreenRegisterOutputEndpoint(HAL_OUTPUT_ROUTER* output);
code/kernel/include/kernel/io_file.h
+16 -11
View File
@@ -12,6 +12,7 @@
#include <kernel/ke_spinlock.h>
#include <ntlite/string_view.h>
#include <ntlite/dirent.h>
#include <ntlite/file_stat.h>
// Kernel/filesystem code uses the public directory-entry ABI directly.
typedef SYS_DIR_ENTRY KDIR_ENTRY;
@@ -26,6 +27,7 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
size_t information);
// Filesystem-level cache and traversal statistics.
// Drivers map private metadata work onto these generic counters.
typedef struct IO_FILESYSTEM_STATS {
uint64_t PathLookups;
uint64_t PathCacheHits;
@@ -34,14 +36,8 @@ typedef struct IO_FILESYSTEM_STATS {
uint64_t PathCacheInvalidations;
uint64_t PathCacheGeneration;
uint64_t DirectorySearches;
uint64_t DirectoryClustersRead;
uint64_t DirectoryBlocksRead;
uint64_t DirectoryEntriesScanned;
uint64_t FatChainCacheLookups;
uint64_t FatChainCacheHits;
uint64_t FatChainCacheMisses;
uint64_t FatChainCacheStores;
uint64_t FatChainCacheInvalidations;
uint64_t FatChainWalkSteps;
uint64_t DataReadRuns;
uint64_t DataReadSectors;
uint64_t DataReadRunMerges;
@@ -107,6 +103,12 @@ typedef struct IO_FILESYSTEM {
// Return the file size in bytes.
uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Query metadata for an open file or path.
KSTATUS (*QueryInfo)(struct IO_FILESYSTEM* fs, void* fileCtx,
SYS_FILE_STAT* out);
KSTATUS (*Stat)(struct IO_FILESYSTEM* fs, SYS_STRING_VIEW path,
SYS_FILE_STAT* out);
// Return a stable file identity for caches that outlive one open handle.
KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx,
IO_FILE_IDENTITY* out);
@@ -120,8 +122,6 @@ typedef struct IO_FILESYSTEM {
// Optional driver-private data.
void* DriverCtx;
// Directory enumeration (optional - may be NULL if not supported).
// Open a directory iterator for 'dirPath' (relative or drive-prefixed).
// Returns an opaque per-iteration driver cookie in *dirCtx.
KSTATUS (*OpenDir)(struct IO_FILESYSTEM* fs,
@@ -137,7 +137,6 @@ typedef struct IO_FILESYSTEM {
void (*CloseDir)(struct IO_FILESYSTEM* fs, void* dirCtx);
} IO_FILESYSTEM;
// IO_STATUS_BLOCK - completion status visible to async callers.
typedef struct IO_STATUS_BLOCK {
volatile KSTATUS Status;
@@ -152,6 +151,7 @@ typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
// FILE_OBJECT - the kernel object returned to callers.
#define IO_FILE_FLAG_NO_CACHE 0x00000001u
#define IO_FILE_FLAG_SKIP_ZERO_EXTEND 0x00000002u
#define IO_FILE_FLAG_APPEND 0x00000004u
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
@@ -227,7 +227,12 @@ __declspec(dllexport) KSTATUS IoRemoveDir (SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize);
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) KSTATUS IoGetFileStat(FILE_OBJECT* file, SYS_FILE_STAT* out);
__declspec(dllexport) KSTATUS IoStatFile(SYS_STRING_VIEW path, SYS_FILE_STAT* out);
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out);
__declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, struct PROCESS* owner, uint32_t mode, uint64_t offset, uint64_t length, uint64_t timeoutMilliseconds);
__declspec(dllexport) KSTATUS IoUnlockFile(FILE_OBJECT* file, struct PROCESS* owner, uint64_t offset, uint64_t length);
__declspec(dllexport) void IoReleaseFileLocksForProcess(struct PROCESS* owner);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// Directory enumeration API.
@@ -237,4 +242,4 @@ __declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir);
// Current Working Directory.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size);
__declspec(dllexport) KSTATUS IoRealPath(SYS_STRING_VIEW path, char* buffer, size_t bufferSize, size_t* requiredSizeOut);
code/kernel/include/kernel/ke_apc.h
+1 -4
View File
@@ -35,10 +35,7 @@ typedef struct KUSER_APC KUSER_APC;
typedef struct SYS_USER_APC_FRAME_K {
uint64_t Magic;
uint64_t OriginalRip;
uint64_t OriginalRsp;
uint64_t OriginalRflags;
uint64_t OriginalRax;
uint64_t ReturnToken;
uint64_t Routine;
uint64_t NormalContext;
uint64_t SystemArgument1;
code/kernel/include/kernel/ke_debug.h
+2 -2
View File
@@ -7,6 +7,7 @@
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <ntlite/ddi/hal_api.h>
#define KE_DEBUG_CONSOLE 0x00000001u
#define KE_DEBUG_SELFTEST 0x00000002u
@@ -22,8 +23,7 @@
#define KE_DEBUG_VM_ASSERTS 0x00000400u
#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t));
void KeInitDebug(const HAL_API* hal, uint32_t flags);
uint32_t KeGetDebugFlags(void);
__declspec(dllexport) void DbgPrint(const char* fmt, ...);
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len);
code/kernel/include/kernel/ke_display.h
-2
View File
@@ -16,7 +16,6 @@
#endif
#include <ntlite/ddi/display.h>
typedef struct KE_CONSOLE_INFO {
uint32_t Columns;
uint32_t Rows;
@@ -55,7 +54,6 @@ DISPLAY_API KSTATUS KeDisplayGetConsoleInfo(KE_CONSOLE_INFO* outInfo);
DISPLAY_API KSTATUS KeDisplaySetConsoleCursor(int col, int row);
DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int col, int row, size_t length, size_t* writtenOut);
DISPLAY_API KSTATUS KeDisplayFillConsoleAttrs(uint16_t attr, int col, int row, size_t length, size_t* writtenOut);
void KeDisplayDebugPrintChar(char c, uint32_t fg, uint32_t bg);
DISPLAY_API KSTATUS KeDisplayPresent(const DISPLAY_RECT* rects, size_t rectCount);
DISPLAY_API KSTATUS KeDisplaySync(void);
DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo);
code/kernel/include/kernel/ke_input.h
+4 -3
View File
@@ -23,7 +23,8 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons);
INPUT_KEY KeInputKbPoll(void);
KSTATUS KeInputWaitKey(uint8_t alertable);
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs);
void KeInputTimerTick(void);
bool KeInputIsKeyDown(INPUT_KEY key);
bool KeInputPeekEvent(INPUT_KEY_RECORD* record);
bool KeInputReadEvent(INPUT_KEY_RECORD* record);
@@ -33,8 +34,8 @@ INPUT_KEY KeInputReadKey(void);
char KeInputReadChar(void);
uint32_t KeInputGetModifiers(void);
bool KeInputMousePoll(void);
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons);
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event);
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event);
void KeInputMouseGetPosition(int* x, int* y);
void KeInputMouseSetPosition(int x, int y);
void KeInputMouseSetBounds(int w, int h);
code/kernel/include/kernel/ke_irp.h
+2 -2
View File
@@ -1,7 +1,7 @@
// Kernel ke irp declarations shared by implementation files and kernel clients.
#pragma once
// ke_irp.h - I/O Request Packet and remove-lock primitives
// ke_irp.h - I/O Request Packet and remove-lock primitives
// IRPs are stack-based. A caller builds an IRP with one or more
// IO_STACK_LOCATION entries, submits it to a DEVICE_OBJECT, and the I/O
@@ -142,7 +142,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp);
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp);
// IRP_QUEUE (per-device, driver-internal).
// IRP_QUEUE (per-device, driver-internal).
typedef struct IRP_QUEUE {
SPINLOCK Lock;
code/kernel/include/kernel/ke_process.h
+12 -4
View File
@@ -17,6 +17,10 @@
#define PROCESS_MODULE_MAX 16
#define PROCESS_CWD_MAX 512
#define KE_PROCESS_FLAG_SYSTEM 0x00000001u
#define KE_PROCESS_FLAG_IDLE 0x00000002u
#define KE_PROCESS_FLAG_TERMINATION_PROTECTED 0x00000004u
#define HANDLE_RIGHT_READ 0x00000001u
#define HANDLE_RIGHT_WRITE 0x00000002u
#define HANDLE_RIGHT_WAIT 0x00000004u
@@ -24,6 +28,7 @@
#define HANDLE_RIGHT_DUPLICATE 0x00000010u
#define HANDLE_RIGHT_QUERY 0x00000020u
#define HANDLE_RIGHT_TERMINATE 0x00000040u
#define HANDLE_RIGHT_LOCK 0x00000080u
#define HANDLE_RIGHT_ALL 0xFFFFFFFFu
typedef struct HANDLE_ENTRY {
@@ -42,6 +47,7 @@ typedef struct PROCESS {
OBJECT Header; // must be first
uint32_t ProcessId;
uint32_t Flags;
char Name[PROCESS_NAME_MAX]; // UTF-8
char CurrentDirectory[PROCESS_CWD_MAX];
@@ -59,7 +65,7 @@ typedef struct PROCESS {
HANDLE_ENTRY Handles[HANDLE_TABLE_SIZE];
SPINLOCK HandleLock;
int ExitCode;
int32_t ExitCode;
volatile long Terminating;
volatile long Exited;
volatile long ResourcesReleased;
@@ -101,11 +107,12 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* process,
SYS_STRING_VIEW imagePath,
SYS_STRING_VIEW commandLine,
SYS_STRING_VIEW currentDirectory);
void KeExitProcess(PROCESS* process, int exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode);
KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode);
void KeCompleteProcessExitIfLastThread(PROCESS* process);
int KeIsProcessTerminating(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut);
int KeIsProcessTerminationProtected(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int32_t* exitCodeOut);
// Handle table operations.
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut);
@@ -114,6 +121,7 @@ void KeCloseHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle);
OBJECT* KeLookupHandleWithRights(PROCESS* process, uint32_t handle, uint32_t requiredRights);
uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle);
KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle, PROCESS* targetProcess, uint32_t desiredRights, uint32_t duplicateFlags, uint32_t* targetHandleOut);
// Return the global process list head (kernel-internal).
LIST_ENTRY* KeGetProcessList(void);
code/kernel/include/kernel/ke_queue.h
+15 -11
View File
@@ -1,34 +1,38 @@
// Kernel ke queue declarations shared by implementation files and kernel clients.
#pragma once
// ke_queue.h - fixed-depth message queue
// Processes share an address space, so a pointer in Param[] is valid
// on both sides. The queue is only a synchronisation and ordering tool.
// Fixed-depth message queue.
// Processes share an address space, so a pointer in Param[] is valid on both sides.
// The queue orders fixed control records and does not copy pointed-to payloads.
#include <stdint.h>
#include <stdbool.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_list.h>
#include <kernel/ke_event.h>
#include <ntlite/status.h>
#define QUEUE_MESSAGE_SIZE 64 // bytes, fixed
#define QUEUE_MESSAGE_SIZE 56 // Bytes, fixed ABI copy size.
#define QUEUE_DEPTH 256
typedef struct QUEUE_MESSAGE {
uint32_t Type;
uint32_t Flags;
uint64_t Param[6]; // 48 bytes of payload
} QUEUE_MESSAGE; // total: 64 bytes
uint64_t Param[6]; // 48 bytes of payload.
} QUEUE_MESSAGE;
typedef char QUEUE_MESSAGE_SIZE_CHECK[
(sizeof(QUEUE_MESSAGE) == QUEUE_MESSAGE_SIZE) ? 1 : -1
];
typedef struct QUEUE {
OBJECT Header;
SPINLOCK Lock;
QUEUE_MESSAGE Ring[QUEUE_DEPTH];
uint32_t Head, Tail, Count;
LIST_ENTRY SendWaiters;
LIST_ENTRY RecvWaiters;
EVENT* NotEmptyEvent;
EVENT* NotFullEvent;
} QUEUE;
KSTATUS KeQueueCreate(QUEUE **out);
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking);
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking);
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds);
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds);
code/kernel/include/kernel/ke_smp.h
-1
View File
@@ -15,7 +15,6 @@ KSTATUS KeStartApplicationProcessors(void);
// park loop into their idle threads.
KSTATUS KeAdmitApplicationProcessorsToScheduler(void);
// Decode CPUID topology and build per-KCPU package/core/SMT sibling masks.
// KeStartApplicationProcessors calls this after MADT enumeration; it is also
// safe to call again after failed AP startup disables a CPU record.
code/kernel/include/kernel/ke_spinlock.h
+5 -7
View File
@@ -13,8 +13,8 @@
// Never hold a spinlock across a wait/blocking operation.
#include <stdint.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#include <kernel/ke_irql.h>
#include <ntlite/ddi/hal_api.h>
typedef struct SPINLOCK {
volatile long Locked;
@@ -44,8 +44,7 @@ static __forceinline KIRQL KiSpinlockCookieIrql(uintptr_t cookie) {
// Save RFLAGS, mask interrupts, raise to DISPATCH_LEVEL if needed, acquire.
// If already running at DIRQL, the higher IRQL is preserved.
static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) {
uintptr_t flags = __readeflags();
_disable();
uintptr_t flags = HalDisableInterrupts();
KIRQL oldIrql = KeGetCurrentIrql();
if (oldIrql < DISPATCH_LEVEL)
@@ -64,13 +63,12 @@ static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t cookie) {
_InterlockedExchange(&sl->Locked, 0);
KeLowerIrql(oldIrql);
__writeeflags(flags);
HalRestoreInterrupts(flags);
}
// Implements the try Acquire Spinlock operation.
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie) {
uintptr_t flags = __readeflags();
_disable();
uintptr_t flags = HalDisableInterrupts();
KIRQL oldIrql = KeGetCurrentIrql();
if (oldIrql < DISPATCH_LEVEL)
@@ -82,7 +80,7 @@ static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie
}
KeLowerIrql(oldIrql);
__writeeflags(flags);
HalRestoreInterrupts(flags);
return 0;
}
code/kernel/include/kernel/ke_thread.h
+8 -2
View File
@@ -147,11 +147,17 @@ struct THREAD {
uintptr_t UserApcDispatcher;
uint8_t UserApcDeliverable;
uint8_t UserApcDeliveryActive;
uintptr_t UserApcFrameUserAddress;
uint64_t UserApcReturnToken;
uint64_t UserApcSavedRip;
uint64_t UserApcSavedRsp;
uint64_t UserApcSavedRflags;
uint64_t UserApcSavedRax;
};
// Creates and queues a user thread. On success, *out receives a referenced
// Creates and queues a user thread. On success, *out receives a referenced
// THREAD pointer owned by the caller; release it with ObDereferenceObject()
// after copying any needed IDs/state. The live scheduler/thread lifetime is
// after copying any needed IDs/state. The live scheduler/thread lifetime is
// separate and remains valid until the thread exits and is reaped.
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
code/kernel/include/kernel/ke_timer.h
+35
View File
@@ -0,0 +1,35 @@
// Kernel timer object declarations shared by implementation files and kernel clients.
#pragma once
// ke_timer.h - waitable one-shot and periodic timers.
#include <stdint.h>
#include <kernel/ke_event.h>
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ob_object.h>
#include <ntlite/status.h>
typedef struct KTIMER {
OBJECT Header;
EVENT* Event;
LIST_ENTRY Link;
uint64_t DueTick;
uint64_t PeriodTicks;
uint8_t Active;
uint8_t Linked;
} KTIMER;
// Creates an unset waitable timer object.
KSTATUS KeTimerCreate(KTIMER** out);
// Arms or rearms a timer with a relative due time and optional period.
KSTATUS KeTimerSet(KTIMER* timer, uint64_t dueTimeMilliseconds, uint64_t periodMilliseconds);
// Cancels a timer and clears any retained signal.
KSTATUS KeTimerCancel(KTIMER* timer);
// Returns the event that backs timer waits.
EVENT* KeTimerWaitEvent(KTIMER* timer);
// Services expired timers from the scheduler tick path.
void KeTimerTick(void);
code/kernel/include/kernel/ke_work.h
+2 -2
View File
@@ -5,9 +5,9 @@
#include <kernel/ke_list.h>
#include <ntlite/status.h>
// Work items are caller-owned embedded objects. They may be queued from
// Work items are caller-owned embedded objects. They may be queued from
// interrupt/DPC/spinlock context and run later on a persistent system worker
// thread at PASSIVE_LEVEL. Queueing the same item while it is already queued
// thread at PASSIVE_LEVEL. Queueing the same item while it is already queued
// is coalesced.
typedef void (*KWORK_ROUTINE)(void* context);
code/kernel/include/kernel/ldr_image.h
+2 -1
View File
@@ -43,5 +43,6 @@ KSTATUS LdrLoadModuleForProcess(struct PROCESS* process, const char* filename, u
// Called by KernelMain to register all modules from BOOT_INFO so that
// LdrGetExport can find them.
struct BOOT_INFO; // forward declaration
void LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
void LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
void LdrRegisterModule(const char* name, uintptr_t base);
const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut, size_t* sizeOut);
code/kernel/include/kernel/mm_pte.h
+6 -7
View File
@@ -19,18 +19,17 @@
// The physical direct map is a kernel-only alias used to reach page-table frames and early physical memory.
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
// Page-table telemetry separates cumulative operations from the current number
// of 4 KiB leaf mappings owned by this layer. Huge boot mappings may exist but
// are not counted as leaves here.
typedef enum MM_PTE_QUERY_PURPOSE {
MmPteQueryGeneric = 0,
MmPteQueryUserProbe,
MmPteQueryHeap,
MmPteQueryVad,
MmPteQueryDebug,
MmPteQuerySelfTest,
MmPteQueryPurposeCount,
MmPteQueryUserProbe = 1,
MmPteQueryHeap = 2,
MmPteQueryVad = 3,
MmPteQueryDebug = 4,
MmPteQuerySelfTest = 5,
MmPteQueryPurposeCount = 6,
} MM_PTE_QUERY_PURPOSE;
typedef struct MM_PTE_STATS {
code/kernel/include/kernel/net_socket.h
+6 -6
View File
@@ -26,16 +26,16 @@ typedef struct SOCKET_OBJECT {
} SOCKET_OBJECT;
KSTATUS NetSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, SOCKET_OBJECT** out);
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
KSTATUS NetSocketListen(SOCKET_OBJECT* socket, uint32_t backlog);
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut);
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize);
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_K* remoteOut);
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize);
KSTATUS NetSocketSend(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags, size_t* sentOut);
KSTATUS NetSocketReceive(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags, size_t* receivedOut);
KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags, const SYS_SOCKET_ADDRESS_IPV4_K* remote, size_t remoteSize, size_t* sentOut);
KSTATUS NetSocketReceiveFrom(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut, size_t* receivedOut);
KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags, const SYS_SOCKET_ADDRESS_K* remote, size_t remoteSize, size_t* sentOut);
KSTATUS NetSocketReceiveFrom(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags, SYS_SOCKET_ADDRESS_K* remoteOut, size_t* receivedOut);
KSTATUS NetSocketShutdown(SOCKET_OBJECT* socket, uint32_t how);
KSTATUS NetSocketSetOption(SOCKET_OBJECT* socket, uint32_t option, const void* value, size_t valueSize);
KSTATUS NetSocketGetOption(SOCKET_OBJECT* socket, uint32_t option, void* value, size_t valueSize, size_t* actualSizeOut);
KSTATUS NetSocketPoll(SOCKET_OBJECT* socket, uint32_t events, uint32_t* returnedEventsOut);
KSTATUS NetResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut);
KSTATUS NetResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut);
code/kernel/include/kernel/ob_object.h
+1
View File
@@ -27,6 +27,7 @@ typedef enum OBJECT_TYPE {
ObjectTypeSurface = 16,
ObjectTypeInputQueue = 17,
ObjectTypeImageSection = 18,
ObjectTypeTimer = 19,
} OBJECT_TYPE;
typedef struct OBJECT {
code/kernel/include/kernel/syscall_nr.h
+138 -117
View File
@@ -1,120 +1,141 @@
// Kernel syscall nr declarations shared by implementation files and kernel clients.
// Kernel syscall number declarations shared by implementation files and kernel clients.
#pragma once
// syscall_nr.h
// INT 0x40 calling convention, MS x64:
// RAX = syscall number
// RCX, RDX, R8, R9 = args 0-3
// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6)
// RAX = return value.
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_WAIT_SINGLE 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_RETIRED_WAIT_THREAD 9
#define SYS_GET_CURRENT_TID 10
#define SYS_GET_CURRENT_PID 11
#define SYS_SET_THREAD_PRIORITY 12
#define SYS_YIELD 13
#define SYS_SLEEP 14
#define SYS_SET_USER_APC_DISPATCHER 15
#define SYS_APC_RETURN 16
#define SYS_ALLOC_VM 17
#define SYS_FREE_VM 18
#define SYS_PROTECT_VM 19
#define SYS_QUERY_VM 20
#define SYS_CLOSE_HANDLE 21
#define SYS_CREATE_MUTEX 22
#define SYS_RETIRED_ACQUIRE_MUTEX 23
#define SYS_TRY_ACQUIRE_MUTEX 24
#define SYS_RELEASE_MUTEX 25
#define SYS_CREATE_SEM 26
#define SYS_RETIRED_WAIT_SEM 27
#define SYS_TRY_WAIT_SEM 28
#define SYS_SIGNAL_SEM 29
#define SYS_CREATE_EVENT 30
#define SYS_SIGNAL_EVENT 31
#define SYS_CLEAR_EVENT 32
#define SYS_RETIRED_WAIT_EVENT 33
#define SYS_CREATE_QUEUE 34
#define SYS_SEND_QUEUE 35
#define SYS_RECV_QUEUE 36
#define SYS_PUBLISH_QUEUE 37
#define SYS_OPEN_NAMED_QUEUE 38
#define SYS_DUPLICATE_QUEUE 39
#define SYS_GET_NAMED_QUEUE_OWNER_PID 40
#define SYS_OPEN_FILE 41
#define SYS_CREATE_FILE 42
#define SYS_READ_FILE 43
#define SYS_WRITE_FILE 44
#define SYS_READ_FILE_ASYNC 45
#define SYS_WRITE_FILE_ASYNC 46
#define SYS_CANCEL_IO 47
#define SYS_SEEK_FILE 48
#define SYS_GET_FILE_POS 49
#define SYS_GET_FILE_SIZE 50
#define SYS_TRUNCATE_FILE 51
#define SYS_DELETE_FILE 52
#define SYS_RENAME_FILE 53
#define SYS_OPEN_DIR 54
#define SYS_READ_DIR 55
#define SYS_CREATE_DIR 56
#define SYS_REMOVE_DIR 57
#define SYS_SET_CWD 58
#define SYS_GET_CWD 59
#define SYS_WRITE_CONSOLE 60
#define SYS_GET_CONSOLE_INFO 61
#define SYS_SET_CONSOLE_CURSOR 62
#define SYS_FILL_CONSOLE_CHARACTER 63
#define SYS_FILL_CONSOLE_ATTRIBUTE 64
#define SYS_PUT_SCREEN_CHAR 65
#define SYS_CLEAR_SCREEN 66
#define SYS_GET_SCREEN_SIZE 67
#define SYS_QUERY_DISPLAY_INFO 68
#define SYS_POLL_KEYBOARD 69
#define SYS_IS_KEY_DOWN 70
#define SYS_READ_KEYBOARD_CHAR 71
#define SYS_READ_KEY 72
#define SYS_GET_KEY_MODIFIERS 73
#define SYS_POLL_MOUSE 74
#define SYS_READ_MOUSE 75
#define SYS_GET_MOUSE_POS 76
#define SYS_GET_TICK 77
#define SYS_PRINT_DEBUG 78
#define SYS_QUERY_TRAP_FRAME 79
#define SYS_CREATE_SOCKET 80
#define SYS_BIND_SOCKET 81
#define SYS_LISTEN_SOCKET 82
#define SYS_ACCEPT_SOCKET 83
#define SYS_CONNECT_SOCKET 84
#define SYS_SEND_SOCKET 85
#define SYS_RECV_SOCKET 86
#define SYS_SHUTDOWN_SOCKET 87
#define SYS_SEND_TO_SOCKET 88
#define SYS_RECV_FROM_SOCKET 89
#define SYS_RESOLVE_HOST 90
#define SYS_SET_SOCKET_OPTION 91
#define SYS_GET_SOCKET_OPTION 92
#define SYS_POLL 93
#define SYS_GET_MEMORY_STATUS 94
#define SYS_QUERY_KERNEL_STATS 95
#define SYS_GET_SYSTEM_TIME 96
#define SYS_CREATE_FILE_MAPPING 97
#define SYS_MAP_FILE 98
#define SYS_CREATE_SURFACE 99
#define SYS_MAP_SURFACE 100
#define SYS_QUERY_SURFACE 101
#define SYS_PRESENT_SURFACE 102
#define SYS_GET_THREAD_LIST 103
#define SYS_READ_INPUT_EVENT 104
#define SYS_PEEK_INPUT_EVENT 105
#define SYS_WAIT_MANY 106
#define SYS_OPEN_INPUT_QUEUE 107
#define SYS_FLUSH_FILE 108
#define SYS_MAX 109
// INT 0x40 calling convention, MS x64.
// RAX selects the syscall number. RCX, RDX, R8, and R9 carry args 0-3.
// R10 and R11 carry args 4-5 and are loaded by six-argument stubs.
// RAX carries the return value.
// Process, threads, waits, and APC.
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_OPEN_PROCESS 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_GET_THREAD_LIST 9
#define SYS_GET_CURRENT_TID 10
#define SYS_GET_CURRENT_PID 11
#define SYS_SET_THREAD_PRIORITY 12
#define SYS_YIELD 13
#define SYS_SLEEP 14
#define SYS_WAIT_SINGLE 15
#define SYS_WAIT_MANY 16
#define SYS_SET_USER_APC_DISPATCHER 17
#define SYS_APC_RETURN 18
// Virtual memory, mappings, and handles.
#define SYS_ALLOC_VM 19
#define SYS_FREE_VM 20
#define SYS_PROTECT_VM 21
#define SYS_QUERY_VM 22
#define SYS_CLOSE_HANDLE 23
#define SYS_DUPLICATE_HANDLE 24
#define SYS_CREATE_FILE_MAPPING 25
#define SYS_MAP_FILE 26
// Synchronization objects.
#define SYS_CREATE_MUTEX 27
#define SYS_TRY_ACQUIRE_MUTEX 28
#define SYS_RELEASE_MUTEX 29
#define SYS_CREATE_SEM 30
#define SYS_TRY_WAIT_SEM 31
#define SYS_SIGNAL_SEM 32
#define SYS_CREATE_EVENT 33
#define SYS_SIGNAL_EVENT 34
#define SYS_CLEAR_EVENT 35
// Queues and named queues.
#define SYS_CREATE_QUEUE 36
#define SYS_SEND_QUEUE 37
#define SYS_RECV_QUEUE 38
#define SYS_PUBLISH_QUEUE 39
#define SYS_OPEN_NAMED_QUEUE 40
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 41
// Files, directories, and current directory.
#define SYS_OPEN_FILE 42
#define SYS_READ_FILE 43
#define SYS_WRITE_FILE 44
#define SYS_READ_FILE_ASYNC 45
#define SYS_WRITE_FILE_ASYNC 46
#define SYS_CANCEL_IO 47
#define SYS_SEEK_FILE 48
#define SYS_GET_FILE_POS 49
#define SYS_GET_FILE_SIZE 50
#define SYS_GET_FILE_STAT 51
#define SYS_STAT_FILE 52
#define SYS_TRUNCATE_FILE 53
#define SYS_FLUSH_FILE 54
#define SYS_DELETE_FILE 55
#define SYS_RENAME_FILE 56
#define SYS_LOCK_FILE 57
#define SYS_UNLOCK_FILE 58
#define SYS_OPEN_DIR 59
#define SYS_READ_DIR 60
#define SYS_CREATE_DIR 61
#define SYS_REMOVE_DIR 62
#define SYS_SET_CWD 63
#define SYS_REAL_PATH 64
// Console, display, and surfaces.
#define SYS_WRITE_CONSOLE 65
#define SYS_GET_CONSOLE_INFO 66
#define SYS_SET_CONSOLE_CURSOR 67
#define SYS_FILL_CONSOLE_CHARACTER 68
#define SYS_FILL_CONSOLE_ATTRIBUTE 69
#define SYS_PUT_SCREEN_CHAR 70
#define SYS_CLEAR_SCREEN 71
#define SYS_GET_SCREEN_SIZE 72
#define SYS_QUERY_DISPLAY_INFO 73
#define SYS_CREATE_SURFACE 74
#define SYS_MAP_SURFACE 75
#define SYS_QUERY_SURFACE 76
#define SYS_PRESENT_SURFACE 77
// Input.
#define SYS_POLL_KEYBOARD 78
#define SYS_IS_KEY_DOWN 79
#define SYS_READ_KEYBOARD_CHAR 80
#define SYS_READ_KEY 81
#define SYS_GET_KEY_MODIFIERS 82
#define SYS_OPEN_INPUT_QUEUE 83
#define SYS_READ_INPUT_EVENT 84
#define SYS_PEEK_INPUT_EVENT 85
#define SYS_PEEK_MOUSE 86
#define SYS_READ_MOUSE 87
#define SYS_GET_MOUSE_POS 88
// Time, debug, and system information.
#define SYS_CREATE_TIMER 89
#define SYS_SET_TIMER 90
#define SYS_CANCEL_TIMER 91
#define SYS_GET_TICK 92
#define SYS_GET_TICK_FREQUENCY 93
#define SYS_GET_SYSTEM_TIME 94
#define SYS_PRINT_DEBUG 95
#define SYS_QUERY_TRAP_FRAME 96
#define SYS_GET_MEMORY_STATUS 97
#define SYS_QUERY_KERNEL_STATS 98
// Native sockets and polling.
#define SYS_CREATE_SOCKET 99
#define SYS_BIND_SOCKET 100
#define SYS_LISTEN_SOCKET 101
#define SYS_ACCEPT_SOCKET 102
#define SYS_CONNECT_SOCKET 103
#define SYS_SEND_SOCKET 104
#define SYS_RECV_SOCKET 105
#define SYS_SHUTDOWN_SOCKET 106
#define SYS_SEND_TO_SOCKET 107
#define SYS_RECV_FROM_SOCKET 108
#define SYS_RESOLVE_HOST 109
#define SYS_SET_SOCKET_OPTION 110
#define SYS_GET_SOCKET_OPTION 111
#define SYS_POLL 112
#define SYS_MAX 113
code/kernel/io_device.c
-1
View File
@@ -253,7 +253,6 @@ __declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
SYS_STRING_VIEW name) {
if (!device || !SysSvIsValid(name))
code/kernel/io_file.c
+384 -25
View File
@@ -67,6 +67,167 @@ static SPINLOCK gAsyncFileQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gAsyncFileQueue;
static SEMAPHORE* gAsyncFileSemaphore;
typedef struct IO_FILE_LOCK_ENTRY {
LIST_ENTRY Link;
IO_FILE_IDENTITY Identity;
PROCESS* Owner;
uint64_t Offset;
uint64_t Length;
uint32_t Mode;
} IO_FILE_LOCK_ENTRY;
typedef struct IO_FILE_LOCK_WAITER {
LIST_ENTRY Link;
IO_FILE_IDENTITY Identity;
PROCESS* Owner;
uint64_t Offset;
uint64_t Length;
uint32_t Mode;
EVENT* Event;
uint8_t Armed;
} IO_FILE_LOCK_WAITER;
static SPINLOCK gFileLockLock = SPINLOCK_INIT;
static LIST_ENTRY gFileLocks;
static LIST_ENTRY gFileLockWaiters;
static uint8_t gFileLocksInitialized;
// Initializes the global advisory file-lock lists on first use.
static void KiIoEnsureFileLockState(void) {
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
if (!gFileLocksInitialized) {
LIST_INIT(&gFileLocks);
LIST_INIT(&gFileLockWaiters);
gFileLocksInitialized = 1;
}
KeReleaseSpinlock(&gFileLockLock, flags);
}
// Compares the stable filesystem identity fields used for lock ownership.
static bool KiIoSameFileIdentity(const IO_FILE_IDENTITY* a, const IO_FILE_IDENTITY* b) {
if (!a || !b) return false;
return a->VolumeId == b->VolumeId &&
a->FileId == b->FileId &&
a->Generation == b->Generation;
}
// Returns the exclusive end offset for a public lock range.
static uint64_t KiIoFileLockEnd(uint64_t offset, uint64_t length) {
if (length == SYS_FILE_LOCK_TO_EOF)
return UINT64_MAX;
uint64_t end = offset + length;
if (end < offset)
return UINT64_MAX;
return end;
}
// Tests whether two byte ranges overlap.
static bool KiIoFileLockRangesOverlap(uint64_t aOffset, uint64_t aLength,
uint64_t bOffset, uint64_t bLength) {
uint64_t aEnd = KiIoFileLockEnd(aOffset, aLength);
uint64_t bEnd = KiIoFileLockEnd(bOffset, bLength);
return aOffset < bEnd && bOffset < aEnd;
}
// Tests whether one existing lock conflicts with a requested lock.
static bool KiIoFileLockConflicts(const IO_FILE_LOCK_ENTRY* lock,
const IO_FILE_IDENTITY* identity,
PROCESS* owner,
uint32_t mode,
uint64_t offset,
uint64_t length) {
if (!lock || !identity || !owner)
return false;
if (lock->Owner == owner)
return false;
if (!KiIoSameFileIdentity(&lock->Identity, identity))
return false;
if (!KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length))
return false;
if (lock->Mode == SYS_FILE_LOCK_EXCLUSIVE || mode == SYS_FILE_LOCK_EXCLUSIVE)
return true;
return false;
}
// Tests whether the requested lock is currently blocked by another process.
static bool KiIoFileLockHasConflictLocked(const IO_FILE_IDENTITY* identity,
PROCESS* owner,
uint32_t mode,
uint64_t offset,
uint64_t length) {
LIST_ENTRY* head = &gFileLocks;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (KiIoFileLockConflicts(lock, identity, owner, mode, offset, length))
return true;
}
return false;
}
// Returns the scheduler tick value used by finite file-lock waits.
static uint64_t KiIoFileLockReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Converts a millisecond timeout to at least one scheduler tick.
static uint64_t KiIoFileLockTimeoutTicks(uint64_t timeoutMilliseconds) {
uint64_t ticks = timeoutMilliseconds / SCHED_TICK_MS;
if ((timeoutMilliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Computes an absolute scheduler-tick deadline for a finite timeout.
static uint64_t KiIoFileLockTimeoutDeadline(uint64_t timeoutMilliseconds) {
uint64_t now = KiIoFileLockReadSchedulerTick();
uint64_t ticks = KiIoFileLockTimeoutTicks(timeoutMilliseconds);
uint64_t deadline = now + ticks;
return deadline < now ? UINT64_MAX : deadline;
}
// Computes the remaining wait time for a finite timeout deadline.
static uint64_t KiIoFileLockRemainingMilliseconds(uint64_t deadlineTick) {
uint64_t now = KiIoFileLockReadSchedulerTick();
if (deadlineTick <= now)
return 0;
uint64_t ticks = deadlineTick - now;
if (ticks > UINT64_MAX / SCHED_TICK_MS)
return UINT64_MAX;
return ticks * SCHED_TICK_MS;
}
// Waits for a file-lock conflict to change.
static KSTATUS KiIoWaitForFileLockChange(EVENT* event, uint64_t timeoutMilliseconds, uint64_t deadlineTick) {
if (!event)
return STATUS_INVALID_PARAMETER;
uint64_t waitMilliseconds = timeoutMilliseconds;
if (timeoutMilliseconds != SYS_WAIT_TIMEOUT_INFINITE) {
waitMilliseconds = KiIoFileLockRemainingMilliseconds(deadlineTick);
if (waitMilliseconds == 0)
return STATUS_TIMEOUT;
}
EVENT* events[1] = { event };
return KeWaitForMultipleEvents(events, 1, 0, waitMilliseconds, (uint32_t*)0);
}
// Wakes waiters whose requested file identity may have become available.
static void KiIoSignalFileLockWaitersLocked(const IO_FILE_IDENTITY* identity) {
if (!identity || !gFileLocksInitialized)
return;
LIST_ENTRY* head = &gFileLockWaiters;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
IO_FILE_LOCK_WAITER* waiter = CONTAINING_RECORD(node, IO_FILE_LOCK_WAITER, Link);
if (waiter->Armed && waiter->Event && KiIoSameFileIdentity(&waiter->Identity, identity))
KeSignalEvent(waiter->Event);
}
}
// Implements the io Async File Request Destroy operation.
static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
@@ -416,7 +577,6 @@ __declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJ
return req ? &req->Header : (OBJECT*)0;
}
// Returns the wait event owned by an async file request.
__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? req->WaitEvent : (EVENT*)0;
@@ -488,25 +648,14 @@ static size_t KiFormatCwd(uint8_t drive, SYS_STRING_VIEW rel, char* buf, size_t
buf[0] = (char)('0' + drive);
buf[1] = ':';
buf[2] = '/';
if (rel.Length) memcpy(buf + 3, rel.Data, rel.Length);
for (size_t i = 0; i < rel.Length; i++) {
char c = rel.Data[i];
buf[3 + i] = (c == '\\') ? '/' : c;
}
buf[3 + rel.Length] = '\0';
return needed;
}
// Returns cwd.
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
if (!buf || size == 0) return STATUS_INVALID_PARAMETER;
PROCESS* proc = KeGetCurrentProcess();
if (!proc) return STATUS_ACCESS_DENIED;
uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
size_t needed = KiFormatCwd(proc->CurrentDrive, SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength), buf, size);
KeReleaseSpinlock(&proc->CwdLock, flags);
return needed && needed <= size ? STATUS_SUCCESS : STATUS_BUFFER_TOO_SMALL;
}
// Implements the sv Has Drive Root operation.
static int KiSvHasDriveRoot(SYS_STRING_VIEW path) {
return path.Length >= 3 &&
@@ -615,6 +764,29 @@ static KSTATUS KiResolvePathV(SYS_STRING_VIEW input, KRESOLVED_PATH* out) {
return STATUS_SUCCESS;
}
// Builds a canonical native path using the kernel-owned process CWD.
__declspec(dllexport) KSTATUS IoRealPath(SYS_STRING_VIEW path, char* buffer, size_t bufferSize, size_t* requiredSizeOut) {
if (requiredSizeOut) *requiredSizeOut = 0;
if (path.Length && !path.Data) return STATUS_INVALID_PARAMETER;
if (path.Length == 0) return STATUS_INVALID_PARAMETER;
if (!buffer && bufferSize != 0) return STATUS_INVALID_PARAMETER;
KRESOLVED_PATH resolved;
KSTATUS s = KiResolvePathV(path, &resolved);
if (!KSUCCESS(s)) return s;
char formatted[SYS_CWD_MAX];
size_t required = KiFormatCwd(resolved.Drive, resolved.Path, formatted, sizeof(formatted));
if (!required || required > sizeof(formatted)) return STATUS_BUFFER_TOO_SMALL;
if (requiredSizeOut) *requiredSizeOut = required;
if (!buffer) return STATUS_SUCCESS;
if (bufferSize < required) return STATUS_BUFFER_TOO_SMALL;
memcpy(buffer, formatted, required);
return STATUS_SUCCESS;
}
// Updates cwd.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path) {
if (path.Length && !path.Data) return STATUS_INVALID_PARAMETER;
@@ -722,14 +894,8 @@ __declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
if (one.PathCacheGeneration > out->PathCacheGeneration)
out->PathCacheGeneration = one.PathCacheGeneration;
out->DirectorySearches += one.DirectorySearches;
out->DirectoryClustersRead += one.DirectoryClustersRead;
out->DirectoryBlocksRead += one.DirectoryBlocksRead;
out->DirectoryEntriesScanned += one.DirectoryEntriesScanned;
out->FatChainCacheLookups += one.FatChainCacheLookups;
out->FatChainCacheHits += one.FatChainCacheHits;
out->FatChainCacheMisses += one.FatChainCacheMisses;
out->FatChainCacheStores += one.FatChainCacheStores;
out->FatChainCacheInvalidations += one.FatChainCacheInvalidations;
out->FatChainWalkSteps += one.FatChainWalkSteps;
out->DataReadRuns += one.DataReadRuns;
out->DataReadSectors += one.DataReadSectors;
out->DataReadRunMerges += one.DataReadRunMerges;
@@ -785,7 +951,6 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
return STATUS_SUCCESS;
}
static uint64_t KiAlignUpPage64(uint64_t value) {
return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
}
@@ -976,7 +1141,7 @@ __declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t offset = file->Offset;
uint64_t offset = (file->Flags & IO_FILE_FLAG_APPEND) ? file->Size : file->Offset;
KeReleaseSpinlock(&file->Lock, flags);
KSTATUS s = IoWriteFileAt(file, offset, buf, size, bytesWritten);
@@ -1130,6 +1295,45 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
return size;
}
// Queries metadata for an open file object.
__declspec(dllexport) KSTATUS IoGetFileStat(FILE_OBJECT* file, SYS_FILE_STAT* out) {
if (!file || !out) return STATUS_INVALID_PARAMETER;
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
out->Type = SYS_FILE_TYPE_REGULAR;
out->LinkCount = 1;
out->FileSize = IoGetFileSize(file);
if (!file->Fs || !file->Fs->QueryInfo)
return STATUS_SUCCESS;
KSTATUS s = file->Fs->QueryInfo(file->Fs, file->FileCtx, out);
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
file->Size = out->FileSize;
KeReleaseSpinlock(&file->Lock, flags);
}
return s;
}
// Queries metadata for a filesystem path.
__declspec(dllexport) KSTATUS IoStatFile(SYS_STRING_VIEW path, SYS_FILE_STAT* out) {
if ((!path.Data && path.Length) || path.Length == 0 || !out) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
KRESOLVED_PATH resolved;
KSTATUS s = KiResolvePathV(path, &resolved);
if (!KSUCCESS(s)) return s;
IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
if (!fs) return STATUS_NOT_FOUND;
if (!fs->Stat) return STATUS_NOT_SUPPORTED;
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
return fs->Stat(fs, resolved.Path, out);
}
// Queries the filesystem identity for a file object.
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) {
if (!file || !out)
@@ -1140,6 +1344,161 @@ __declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDE
return file->Fs->QueryIdentity(file->Fs, file->FileCtx, out);
}
// Acquires an advisory process-owned byte-range lock for one file identity.
__declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint32_t mode,
uint64_t offset, uint64_t length,
uint64_t timeoutMilliseconds) {
if (!file || !owner) return STATUS_INVALID_PARAMETER;
if (mode != SYS_FILE_LOCK_SHARED && mode != SYS_FILE_LOCK_EXCLUSIVE) return STATUS_INVALID_PARAMETER;
if (length == 0) return STATUS_INVALID_PARAMETER;
if (timeoutMilliseconds != 0 && !KeCanWait()) return STATUS_INVALID_IRQL;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(file, &identity);
if (!KSUCCESS(s)) return s;
KiIoEnsureFileLockState();
uint64_t deadlineTick = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != SYS_WAIT_TIMEOUT_INFINITE)
deadlineTick = KiIoFileLockTimeoutDeadline(timeoutMilliseconds);
EVENT* waitEvent = (EVENT*)0;
IO_FILE_LOCK_WAITER waiter;
memset(&waiter, 0, sizeof(waiter));
for (;;) {
IO_FILE_LOCK_ENTRY* entry = (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k'));
if (!entry) {
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return STATUS_NO_MEMORY;
}
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
if (!KiIoFileLockHasConflictLocked(&identity, owner, mode, offset, length)) {
entry->Identity = identity;
entry->Owner = owner;
entry->Offset = offset;
entry->Length = length;
entry->Mode = mode;
LIST_INIT(&entry->Link);
ListInsertTail(&gFileLocks, &entry->Link);
KeReleaseSpinlock(&gFileLockLock, flags);
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&gFileLockLock, flags);
MmFreePool(entry);
if (timeoutMilliseconds == 0) {
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return STATUS_WOULD_BLOCK;
}
if (!waitEvent) {
s = KeEventCreate(EventAutoReset, 0, &waitEvent);
if (!KSUCCESS(s)) return s;
waiter.Identity = identity;
waiter.Owner = owner;
waiter.Offset = offset;
waiter.Length = length;
waiter.Mode = mode;
waiter.Event = waitEvent;
LIST_INIT(&waiter.Link);
} else {
KeClearEvent(waitEvent);
}
flags = KeAcquireSpinlock(&gFileLockLock);
if (!KiIoFileLockHasConflictLocked(&identity, owner, mode, offset, length)) {
KeReleaseSpinlock(&gFileLockLock, flags);
continue;
}
waiter.Armed = 1;
ListInsertTail(&gFileLockWaiters, &waiter.Link);
KeReleaseSpinlock(&gFileLockLock, flags);
s = KiIoWaitForFileLockChange(waitEvent, timeoutMilliseconds, deadlineTick);
flags = KeAcquireSpinlock(&gFileLockLock);
if (waiter.Armed) {
ListRemove(&waiter.Link);
waiter.Armed = 0;
}
KeReleaseSpinlock(&gFileLockLock, flags);
if (!KSUCCESS(s)) {
ObDereferenceObject(&waitEvent->Header);
return s;
}
}
}
// Releases this process's overlapping advisory locks for one file identity.
__declspec(dllexport) KSTATUS IoUnlockFile(FILE_OBJECT* file, PROCESS* owner, uint64_t offset, uint64_t length) {
if (!file || !owner || length == 0) return STATUS_INVALID_PARAMETER;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(file, &identity);
if (!KSUCCESS(s)) return s;
KiIoEnsureFileLockState();
bool released = false;
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
LIST_ENTRY* head = &gFileLocks;
LIST_ENTRY* node = head->Fwd;
while (node != head) {
LIST_ENTRY* next = node->Fwd;
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (lock->Owner == owner && KiIoSameFileIdentity(&lock->Identity, &identity) &&
KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length)) {
ListRemove(&lock->Link);
MmFreePool(lock);
released = true;
}
node = next;
}
if (released)
KiIoSignalFileLockWaitersLocked(&identity);
KeReleaseSpinlock(&gFileLockLock, flags);
return released ? STATUS_SUCCESS : STATUS_NOT_FOUND;
}
// Releases every advisory file lock owned by one exiting process.
__declspec(dllexport) void IoReleaseFileLocksForProcess(PROCESS* owner) {
if (!owner) return;
KiIoEnsureFileLockState();
bool released = false;
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
LIST_ENTRY* head = &gFileLocks;
LIST_ENTRY* node = head->Fwd;
while (node != head) {
LIST_ENTRY* next = node->Fwd;
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (lock->Owner == owner) {
ListRemove(&lock->Link);
MmFreePool(lock);
released = true;
}
node = next;
}
if (released) {
LIST_ENTRY* waiterHead = &gFileLockWaiters;
for (LIST_ENTRY* waiterNode = waiterHead->Fwd; waiterNode != waiterHead; waiterNode = waiterNode->Fwd) {
IO_FILE_LOCK_WAITER* waiter = CONTAINING_RECORD(waiterNode, IO_FILE_LOCK_WAITER, Link);
if (waiter->Armed && waiter->Event)
KeSignalEvent(waiter->Event);
}
}
KeReleaseSpinlock(&gFileLockLock, flags);
}
// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
if (file) ObDereferenceObject(&file->Header);
code/kernel/ke_apc.c
+128 -17
View File
@@ -6,10 +6,55 @@
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_trap.h>
#include <kernel/ke_debug.h>
#include <kernel/mm_virt.h>
#include <string.h>
#include <intrin.h>
#define KI_USER_CS 0x1BULL
#define KI_USER_SS 0x23ULL
#define KI_RFLAGS_FIXED 0x2ULL
#define KI_RFLAGS_IOPL 0x3000ULL
#define KI_RFLAGS_NT 0x4000ULL
#define KI_RFLAGS_VM 0x20000ULL
static volatile long long gUserApcReturnTokenSeed;
// Returns whether an address is in the canonical user half accepted by IRET.
static bool KiUserAddressValid(uint64_t address) {
return address >= USER_BASE && address < USER_LIMIT;
}
// Returns whether a saved user RFLAGS value can be restored safely.
static bool KiUserRflagsValid(uint64_t rflags) {
if ((rflags & KI_RFLAGS_FIXED) == 0)
return false;
if ((rflags & (KI_RFLAGS_IOPL | KI_RFLAGS_NT | KI_RFLAGS_VM)) != 0)
return false;
return true;
}
// Validates the kernel-owned state that will be restored after a user APC.
static KSTATUS KiValidateUserApcResumeState(uint64_t rip, uint64_t rsp, uint64_t rflags,
uint64_t cs, uint64_t ss) {
if (!KiUserAddressValid(rip) || !KiUserAddressValid(rsp))
return STATUS_ACCESS_DENIED;
if (cs != KI_USER_CS || ss != KI_USER_SS)
return STATUS_ACCESS_DENIED;
if (!KiUserRflagsValid(rflags))
return STATUS_ACCESS_DENIED;
return STATUS_SUCCESS;
}
// Allocates a nonzero token for one active user APC return frame.
static uint64_t KiNextUserApcReturnToken(THREAD* thread, uintptr_t frameAddress) {
uint64_t seed = (uint64_t)_InterlockedIncrement64(&gUserApcReturnTokenSeed);
uint64_t token = seed ^ (uint64_t)(uintptr_t)thread ^ (uint64_t)frameAddress ^ SYS_USER_APC_FRAME_MAGIC;
if (token == 0)
token = SYS_USER_APC_FRAME_MAGIC;
return token;
}
// Initializes apc State.
void KiInitializeApcState(THREAD* thread) {
if (!thread) return;
@@ -25,6 +70,12 @@ void KiInitializeApcState(THREAD* thread) {
thread->UserApcDispatcher = 0;
thread->UserApcDeliverable = 0;
thread->UserApcDeliveryActive = 0;
thread->UserApcFrameUserAddress = 0;
thread->UserApcReturnToken = 0;
thread->UserApcSavedRip = 0;
thread->UserApcSavedRsp = 0;
thread->UserApcSavedRflags = 0;
thread->UserApcSavedRax = 0;
}
__declspec(dllexport) void KeInitializeApc(KAPC* apc,
@@ -184,12 +235,16 @@ void KiDeliverKernelApcs(void) {
thread->ApcDeliveryActive = 0;
}
// Updates user Apc Dispatcher.
__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return STATUS_FAILURE;
if (!dispatcher) return STATUS_INVALID_PARAMETER;
if (!KiUserAddressValid((uint64_t)(uintptr_t)dispatcher)) return STATUS_ACCESS_DENIED;
KSTATUS s = MmProbeUserRead(dispatcher, 1);
if (!KSUCCESS(s)) return s;
thread->UserApcDispatcher = (uintptr_t)dispatcher;
return STATUS_SUCCESS;
}
@@ -201,6 +256,8 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
void* systemArgument2) {
if (!thread || !routine)
return STATUS_INVALID_PARAMETER;
if (!KiUserAddressValid((uint64_t)(uintptr_t)routine))
return STATUS_ACCESS_DENIED;
if (thread->State == ThreadDead)
return STATUS_OBJECT_DEAD;
@@ -262,24 +319,33 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
if (!apc)
return;
KSTATUS s = KiValidateUserApcResumeState(frame->Rip, frame->Rsp, frame->Rflags,
frame->Cs, frame->Ss);
if (!KSUCCESS(s)) {
DbgPrint("USER/APC: rejected saved return state rip=0x%016llX rsp=0x%016llX rflags=0x%016llX cs=0x%016llX ss=0x%016llX status=%u\n",
frame->Rip, frame->Rsp, frame->Rflags, frame->Cs, frame->Ss, s);
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
ListInsertTail(&thread->UserApcQueue, &apc->Link);
thread->UserApcPending = 1;
KeReleaseSpinlock(&thread->UserApcLock, flags);
return;
}
uintptr_t oldRsp = (uintptr_t)frame->Rsp;
uintptr_t newRsp = oldRsp - sizeof(SYS_USER_APC_FRAME_K);
newRsp &= ~(uintptr_t)0xFULL;
newRsp -= 8; // make injected dispatcher entry look like a normal call target
newRsp -= 8; // Make injected dispatcher entry look like a normal call target.
SYS_USER_APC_FRAME_K userFrame;
memset(&userFrame, 0, sizeof(userFrame));
userFrame.Magic = SYS_USER_APC_FRAME_MAGIC;
userFrame.OriginalRip = frame->Rip;
userFrame.OriginalRsp = frame->Rsp;
userFrame.OriginalRflags = frame->Rflags;
userFrame.OriginalRax = STATUS_USER_APC;
userFrame.ReturnToken = KiNextUserApcReturnToken(thread, newRsp);
userFrame.Routine = (uint64_t)(uintptr_t)apc->Routine;
userFrame.NormalContext = (uint64_t)(uintptr_t)apc->NormalContext;
userFrame.SystemArgument1 = (uint64_t)(uintptr_t)apc->SystemArgument1;
userFrame.SystemArgument2 = (uint64_t)(uintptr_t)apc->SystemArgument2;
KSTATUS s = CopyToUser((void*)newRsp, &userFrame, sizeof(userFrame));
s = CopyToUser((void*)newRsp, &userFrame, sizeof(userFrame));
if (!KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
ListInsertTail(&thread->UserApcQueue, &apc->Link);
@@ -288,6 +354,13 @@ void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
return;
}
thread->UserApcFrameUserAddress = newRsp;
thread->UserApcReturnToken = userFrame.ReturnToken;
thread->UserApcSavedRip = frame->Rip;
thread->UserApcSavedRsp = frame->Rsp;
thread->UserApcSavedRflags = frame->Rflags;
thread->UserApcSavedRax = STATUS_USER_APC;
frame->Rip = thread->UserApcDispatcher;
frame->Rsp = newRsp;
frame->Rax = STATUS_USER_APC;
@@ -331,32 +404,70 @@ void KiRundownApcState(THREAD* thread) {
thread->UserApcDispatcher = 0;
thread->UserApcDeliverable = 0;
thread->UserApcDeliveryActive = 0;
thread->UserApcFrameUserAddress = 0;
thread->UserApcReturnToken = 0;
thread->UserApcSavedRip = 0;
thread->UserApcSavedRsp = 0;
thread->UserApcSavedRflags = 0;
thread->UserApcSavedRax = 0;
}
// Implements the user Apc Return operation.
__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* userFrame) {
THREAD* thread = KeGetCurrentThread();
if (!thread)
return STATUS_FAILURE;
if (!thread->UserApcDeliveryActive || !userFrame)
return STATUS_INVALID_PARAMETER;
SYS_USER_APC_FRAME_K frameCopy;
KSTATUS s = CopyFromUser(&frameCopy, userFrame, sizeof(frameCopy));
if (!KSUCCESS(s))
return s;
if (frameCopy.Magic != SYS_USER_APC_FRAME_MAGIC)
if (frameCopy.Magic != SYS_USER_APC_FRAME_MAGIC ||
frameCopy.ReturnToken != thread->UserApcReturnToken ||
(uintptr_t)userFrame != thread->UserApcFrameUserAddress) {
DbgPrint("USER/APC: rejected return frame user=0x%016llX active=0x%016llX token=0x%016llX expected=0x%016llX\n",
(uint64_t)(uintptr_t)userFrame,
(uint64_t)thread->UserApcFrameUserAddress,
frameCopy.ReturnToken,
thread->UserApcReturnToken);
return STATUS_INVALID_PARAMETER;
THREAD* thread = KeGetCurrentThread();
if (!thread)
return STATUS_FAILURE;
}
KTRAP_FRAME* frame = KeGetCurrentTrapFrame();
if (!frame || !KeTrapFrameFromUser(frame) || frame->Kind != KTRAP_KIND_SYSCALL)
return STATUS_FAILURE;
frame->Rip = frameCopy.OriginalRip;
frame->Rsp = frameCopy.OriginalRsp;
frame->Rflags = frameCopy.OriginalRflags;
frame->Rax = frameCopy.OriginalRax;
s = KiValidateUserApcResumeState(thread->UserApcSavedRip,
thread->UserApcSavedRsp,
thread->UserApcSavedRflags,
frame->Cs,
frame->Ss);
if (!KSUCCESS(s)) {
DbgPrint("USER/APC: rejected kernel-owned resume state rip=0x%016llX rsp=0x%016llX rflags=0x%016llX status=%u\n",
thread->UserApcSavedRip,
thread->UserApcSavedRsp,
thread->UserApcSavedRflags,
s);
return s;
}
uint64_t savedRax = thread->UserApcSavedRax;
frame->Rip = thread->UserApcSavedRip;
frame->Rsp = thread->UserApcSavedRsp;
frame->Rflags = thread->UserApcSavedRflags;
frame->Rax = savedRax;
thread->UserApcDeliveryActive = 0;
thread->UserApcDeliverable = 0;
return frameCopy.OriginalRax;
thread->UserApcFrameUserAddress = 0;
thread->UserApcReturnToken = 0;
thread->UserApcSavedRip = 0;
thread->UserApcSavedRsp = 0;
thread->UserApcSavedRflags = 0;
thread->UserApcSavedRax = 0;
return savedRax;
}
code/kernel/ke_debug.c
+229 -349
View File
@@ -7,46 +7,35 @@
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <kernel/ke_debug.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_work.h>
#include <kernel/mm_virt.h>
#include <kernel/ldr_image.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor
int _fltused; // MSVC floating-point helper anchor.
int gDebugEnabled = 0;
static uint32_t gDebugFlags = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
static uint32_t gDebugFlags;
static const HAL_API* gDebugHal;
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_DEBUG_SCREEN_DRAIN_SIZE 512u
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_FATAL_OUTPUT_LINE_SIZE 2048u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static char* gDebugScreenBuffer;
static size_t gDebugScreenSize;
static size_t gDebugScreenRead;
static size_t gDebugScreenWrite;
static size_t gDebugScreenUsed;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile long gDebugLogFlushing;
static volatile long gDebugConsoleOwnsSerial;
static volatile long gDebugConsoleWorkReady;
static KWORK_ITEM gDebugConsoleWorkItem;
// Implements the debug Log Append Locked operation.
// Appends bytes to the in-memory debug log while its lock is held.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
return;
@@ -57,7 +46,8 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
}
size_t first = gDebugLogSize - gDebugLogWrite;
if (first > len) first = len;
if (first > len)
first = len;
memcpy(gDebugLogBuffer + gDebugLogWrite, text, first);
if (len > first)
memcpy(gDebugLogBuffer, text + first, len - first);
@@ -69,137 +59,52 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
gDebugLogUsed += len;
}
// Implements the debug Screen Append Locked operation.
static void KiDebugScreenAppendLocked(const char* text, size_t len) {
if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0)
// Writes one record through the HAL output router.
static void KiOutputWrite(uint32_t stream, uint32_t flags,
uint32_t foreground, uint32_t background,
const char* text, size_t length) {
if (!gDebugHal || !gDebugHal->Output.Write || !text || length == 0)
return;
if (len >= gDebugScreenSize) {
text += len - (gDebugScreenSize - 1u);
len = gDebugScreenSize - 1u;
}
while (gDebugScreenUsed + len > gDebugScreenSize) {
gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize;
gDebugScreenUsed--;
}
size_t first = gDebugScreenSize - gDebugScreenWrite;
if (first > len) first = len;
memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first);
if (len > first)
memcpy(gDebugScreenBuffer, text + first, len - first);
gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize;
gDebugScreenUsed += len;
HAL_OUTPUT_RECORD record;
record.Stream = stream;
record.Flags = flags;
record.Foreground = foreground;
record.Background = background;
record.Buffer = text;
record.Length = length;
gDebugHal->Output.Write(&record);
}
// Implements the debug Screen Drain Locked operation.
static size_t KiDebugScreenDrainLocked(char* out, size_t max) {
if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0)
return 0;
size_t len = gDebugScreenUsed;
if (len > max) len = max;
size_t first = gDebugScreenSize - gDebugScreenRead;
if (first > len) first = len;
memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first);
if (len > first)
memcpy(out + first, gDebugScreenBuffer, len - first);
gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize;
gDebugScreenUsed -= len;
return len;
}
// Ensures the asynchronous screen ring exists before console output starts using it.
static bool KiDebugEnsureScreenBuffer(void) {
if (gDebugScreenBuffer && gDebugScreenSize != 0)
return true;
char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r'));
if (!buffer)
return false;
memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE);
bool installed = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (!gDebugScreenBuffer) {
gDebugScreenBuffer = buffer;
gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE;
gDebugScreenRead = 0;
gDebugScreenWrite = 0;
gDebugScreenUsed = 0;
installed = true;
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (!installed)
MmFreePool(buffer);
return true;
}
// Implements the debug Serial Write Char operation.
static void KiDebugSerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n') {
__outbyte(0x3F8, '\r');
while ((__inbyte(0x3FD) & 0x20) == 0) { }
}
__outbyte(0x3F8, (uint8_t)c);
}
// Implements the debug Serial Write Locked operation.
static void KiDebugSerialWriteLocked(const char* text, size_t len) {
if (!text)
// Formats one bounded output record and routes it through the HAL.
static void KiOutputVPrintf(uint32_t stream, uint32_t flags,
uint32_t foreground, uint32_t background,
const char* format, va_list args) {
if (!format)
return;
for (size_t i = 0; i < len; i++)
KiDebugSerialWriteChar(text[i]);
}
// Implements the debug Console Write operation.
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
return;
for (size_t i = 0; i < len; i++)
PrintChar(text[i], 0xCCCCCC, 0x012456);
}
// Implements the debug Console Work operation.
static void KiDebugConsoleWork(void* context) {
(void)context;
char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE];
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk));
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (len == 0)
break;
KiDebugConsoleWrite(chunk, len);
char line[KE_FATAL_OUTPUT_LINE_SIZE];
int count = vsnprintf_(line, sizeof(line), format, args);
size_t length = 0;
if (count > 0) {
length = (size_t)count;
if (length >= sizeof(line))
length = sizeof(line) - 1u;
}
KiOutputWrite(stream, flags, foreground, background, line, length);
}
// Queues the debug Console Work operation if the passive work queue is ready.
static void KiDebugQueueConsoleWork(void) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted())
(void)KeQueueWorkItem(&gDebugConsoleWorkItem);
}
static volatile long gPanicInProgress;
static volatile uintptr_t gPanicOwnerToken;
static volatile long gPanicStoppedCpuCount;
static volatile long gPanicInProgress = 0;
static volatile uintptr_t gPanicOwnerToken = 0;
static volatile long gPanicStoppedCpuCount = 0;
// Implements the panic Secondary Halt operation.
// Permanently stops a secondary processor that entered an active panic.
static _Noreturn void KiPanicSecondaryHalt(void) {
_disable();
_InterlockedIncrement(&gPanicStoppedCpuCount);
for (;;) __halt();
HalStopProcessor();
}
// Implements the panic Current Token operation.
// Returns a stable token for the current execution context.
static uintptr_t KiPanicCurrentToken(void) {
THREAD* thread = KeGetCurrentThread();
if (thread)
@@ -210,20 +115,20 @@ static uintptr_t KiPanicCurrentToken(void) {
return 1u;
}
// Implements the panic Is Active operation.
// Returns whether a fatal report is active.
static int KiPanicIsActive(void) {
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
}
// Implements the panic Current Cpu Owns Report operation.
// Returns whether the current processor owns the active fatal report.
static int KiPanicCurrentCpuOwnsReport(void) {
uintptr_t token = KiPanicCurrentToken();
return token != 0 && gPanicOwnerToken == token;
}
// Implements the begin Fatal Report operation.
// Claims fatal-report ownership and masks interrupts on this processor.
static void KiBeginFatalReport(void) {
_disable();
(void)HalDisableInterrupts();
uintptr_t token = KiPanicCurrentToken();
if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) == 0) {
gPanicOwnerToken = token;
@@ -233,10 +138,7 @@ static void KiBeginFatalReport(void) {
KiPanicSecondaryHalt();
}
// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
// unexpected NMI is not handled here and will fall through to the normal fault
// path in the assembly stub.
// Handles NMI-based panic stops and TLB shootdowns.
__declspec(dllexport) int KeHandleNmiIpi(void) {
if (KiPanicIsActive())
KiPanicSecondaryHalt();
@@ -247,23 +149,11 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
return 0;
}
// Implements the kernel Put Char operation.
void KiKernelPutChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xCCCCCC, 0x012456);
}
// Initializes debug.
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
// Initializes debug logging and initial output subscriptions.
void KeInitDebug(const HAL_API* hal, uint32_t flags) {
gDebugHal = hal;
gDebugFlags = flags;
gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0))
KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0);
if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
@@ -275,81 +165,56 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
}
}
if (flags & KE_DEBUG_CONSOLE)
(void)KiDebugEnsureScreenBuffer();
if (hal && hal->Output.SetEndpointStreams &&
hal->Output.BootFramebufferEndpointId != HAL_OUTPUT_ENDPOINT_INVALID) {
uint32_t streams = HAL_OUTPUT_STREAM_CONSOLE |
HAL_OUTPUT_STREAM_CRASH |
HAL_OUTPUT_STREAM_PANIC;
if (flags & KE_DEBUG_CONSOLE)
streams |= HAL_OUTPUT_STREAM_DEBUG;
(void)hal->Output.SetEndpointStreams(hal->Output.BootFramebufferEndpointId, streams);
}
}
// Returns debug Flags.
// Returns the active debug flags.
uint32_t KeGetDebugFlags(void) {
return gDebugFlags;
}
// Updates debug Console Writer.
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
if (!PrintPtr)
return;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
// After the display manager owns debug drawing, DbgPrint and user console
// writes mirror to COM1 themselves. The display writer is screen-only so
// asynchronous screen flushing does not duplicate serial bytes.
(void)KiDebugEnsureScreenBuffer();
_InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1);
KiDebugQueueConsoleWork();
}
// Implements the dbg Print operation.
// Writes one formatted kernel diagnostic.
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled || !fmt) return;
if (!gDebugEnabled || !fmt)
return;
char line[KE_DEBUG_LOG_LINE_SIZE];
va_list va;
va_start(va, fmt);
int n = vsnprintf_(line, sizeof(line), fmt, va);
va_end(va);
va_list args;
va_start(args, fmt);
int count = vsnprintf_(line, sizeof(line), fmt, args);
va_end(args);
size_t len = 0;
if (n > 0) {
len = (size_t)n;
if (len >= sizeof(line))
len = sizeof(line) - 1u;
size_t length = 0;
if (count > 0) {
length = (size_t)count;
if (length >= sizeof(line))
length = sizeof(line) - 1u;
}
if (KiPanicIsActive()) {
if (!KiPanicCurrentCpuOwnsReport())
KiPanicSecondaryHalt();
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
KiDebugConsoleWrite(line, len);
}
return;
bool panic = KiPanicIsActive();
if (panic && !KiPanicCurrentCpuOwnsReport())
KiPanicSecondaryHalt();
if (!panic) {
uintptr_t lockFlags = KeAcquireSpinlock(&gDebugPrintLock);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, length);
KeReleaseSpinlock(&gDebugPrintLock, lockFlags);
}
bool queueConsole = false;
bool writeConsoleSync = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (KeIsWorkQueueStarted() && gDebugScreenBuffer) {
KiDebugScreenAppendLocked(line, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(line, len);
KiOutputWrite(panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_DEBUG,
panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0,
panic ? 0x00FF2222u : 0x00CCCCCCu,
panic ? 0x00000000u : 0x00012456u,
line, length);
}
// Flushes debug Log To File.
@@ -402,7 +267,6 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// Attempts to persist the debug log during a panic when the filesystem is usable.
static void KiTryFlushPanicDebugLog(void) {
if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
@@ -419,76 +283,69 @@ static void KiTryFlushPanicDebugLog(void) {
DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
}
// KePanic.
// Writes one formatted panic record through panic-safe endpoints.
static void KiPanicPrintf(const char* format, ...) {
va_list args;
va_start(args, format);
KiOutputVPrintf(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
0x00FF2222u, 0x00000000u, format, args);
va_end(args);
}
// Stops the system after reporting a fatal kernel error.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
KiBeginFatalReport();
// Stop every other CPU for the final panic banner. Kernel fault dispatch
// claims panic ownership before its detailed dump, so this does not steal
// ownership from the CPU that found the fatal error.
// Stop every other CPU before writing the final report. Panic-safe endpoints
// do not depend on scheduler locks held by processors stopped by this IPI.
HalApicSendPanicIpi();
if (PrintChar) {
// Print a bright-red banner so the framebuffer is unmistakable.
static const char banner[] = "\n*** KERNEL PANIC ***\n";
const char* p;
KCPU* cpu = KeGetCurrentCpu();
THREAD* thread = KeGetCurrentThread();
static const char banner[] = "\n*** KERNEL PANIC ***\n";
KiOutputWrite(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
0x00FF2222u, 0x00000000u, banner, sizeof(banner) - 1u);
KiPanicPrintf("Panic code: %d\n", code);
for (p = banner; *p; p++)
PrintChar(*p, 0xFF2222, 0x000000);
printf("Panic code: %d\n", code);
if (cpu) {
printf("Panic CPU: %u LAPIC=%u",
(unsigned)cpu->CpuIndex,
(unsigned)cpu->LapicId);
if (thread)
printf(" thread=0x%016llX tid=%llu",
thread,
(unsigned long long)thread->ThreadId);
printf("\n");
KCPU* cpu = KeGetCurrentCpu();
THREAD* thread = KeGetCurrentThread();
if (cpu) {
KiPanicPrintf("Panic CPU: %u LAPIC=%u",
(unsigned)cpu->CpuIndex,
(unsigned)cpu->LapicId);
if (thread) {
KiPanicPrintf(" thread=0x%016llX tid=%llu",
thread,
(unsigned long long)thread->ThreadId);
}
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
KiPanicPrintf("\n");
}
va_list args;
va_start(args, fmt);
KiOutputVPrintf(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
0x00FF2222u, 0x00000000u, fmt, args);
va_end(args);
KiTryFlushPanicDebugLog();
for (;;) __halt();
HalStopProcessor();
}
// KePrintConsole - write user-visible console text using the default console palette.
// User text mirrors to COM1 immediately, then queues expensive screen drawing
// through the passive debug-console worker when that path is available.
// Writes user-visible console text through subscribed output endpoints.
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
if (!buf || len == 0) return;
KiOutputWrite(HAL_OUTPUT_STREAM_CONSOLE, 0,
0x00CCCCCCu, 0x00012456u, buf, len);
}
bool queueConsole = false;
bool writeConsoleSync = false;
bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer();
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(buf, len);
if (PrintChar) {
if (!KiPanicIsActive() && canQueueScreen) {
KiDebugScreenAppendLocked(buf, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(buf, len);
// Writes one formatted process-fault or kernel-fault diagnostic.
static void KiFaultPrintf(const char* format, ...) {
bool panic = KiPanicIsActive();
va_list args;
va_start(args, format);
KiOutputVPrintf(panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_CRASH,
panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0,
panic ? 0x00FF2222u : 0x00FFAA00u,
0x00000000u,
format, args);
va_end(args);
}
// KeBugCheckProcess - user-mode fault handler
@@ -585,6 +442,15 @@ static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
}
}
uintptr_t kernelBase = 0;
size_t kernelSize = 0;
const char* kernelModule = LdrFindKernelModuleByAddress(addr, &kernelBase, &kernelSize);
if (kernelBase) {
if (baseOut) *baseOut = kernelBase;
if (sizeOut) *sizeOut = kernelSize;
return kernelModule;
}
return "unknown";
}
@@ -600,71 +466,96 @@ static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
const char* mod = KiResolveModuleEx(proc, addr, &base, &size);
if (base) {
printf("%s%016llx <%s+0x%016llX>\n",
KiFaultPrintf("%s%016llx <%s+0x%016llX>\n",
label,
(unsigned long long)addr,
mod,
(unsigned long long)(addr - base));
} else {
printf("%s%016llx <unknown>\n",
KiFaultPrintf("%s%016llx <unknown>\n",
label,
(unsigned long long)addr);
}
}
// Implements the user Read U64 operation.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
if (!out) return 0;
if (!addr) return 0;
if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(uint64_t))))
return 0;
*out = *(const uint64_t*)addr;
// Checks whether a kernel virtual range is mapped before diagnostic reads.
static int KiKernelRangeReadable(uintptr_t addr, size_t size) {
if (!addr || size == 0) return 0;
if (size - 1u > (size_t)(UINTPTR_MAX - addr)) return 0;
uintptr_t start = addr & PAGE_MASK;
uintptr_t last = (addr + size - 1u) & PAGE_MASK;
for (uintptr_t page = start;; page += PAGE_SIZE) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
if (!KSUCCESS(MmPteQueryPageFor(MmPteQueryDebug, page, &phys, &flags)))
return 0;
if ((flags & PAGE_PRESENT) == 0)
return 0;
if (page == last)
break;
if (page > UINTPTR_MAX - PAGE_SIZE)
return 0;
}
return 1;
}
// Implements the user Read U8 operation.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
if (!out) return 0;
if (!addr) return 0;
if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(uint8_t))))
// Reads diagnostic memory from either user space or a mapped kernel range.
static int KiReadDiagnosticMemory(uintptr_t addr, void* out, size_t size) {
if (!out || !addr || size == 0) return 0;
if (addr >= KERNEL_BASE) {
if (!KiKernelRangeReadable(addr, size))
return 0;
memcpy(out, (const void*)addr, size);
return 1;
}
if (!KSUCCESS(MmProbeUserRead((const void*)addr, size)))
return 0;
*out = *(const uint8_t*)addr;
memcpy(out, (const void*)addr, size);
return 1;
}
// Reads one diagnostic uint64_t.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
// Reads one diagnostic byte.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
// Dumps bytes.
static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
if (!center) {
printf("%s: <null>\n", label);
KiFaultPrintf("%s: <null>\n", label);
return;
}
printf("%s around ", label);
KiFaultPrintf("%s around ", label);
KiPrintAddress(proc, "", center);
uintptr_t start = center >= 16 ? center - 16 : 0;
for (int row = 0; row < 2; row++) {
uintptr_t addr = start + (uintptr_t)row * 16u;
printf(" %016llx:", (unsigned long long)addr);
KiFaultPrintf(" %016llx:", (unsigned long long)addr);
for (int i = 0; i < 16; i++) {
uint8_t b = 0;
if (KiUserReadU8(addr + (uintptr_t)i, &b))
printf(" %02x", b);
KiFaultPrintf(" %02x", b);
else
printf(" ??");
KiFaultPrintf(" ??");
}
printf("\n");
KiFaultPrintf("\n");
}
}
// Implements the user Read I32 operation.
// Reads one diagnostic int32_t.
static int KiUserReadI32(uintptr_t addr, int32_t* out) {
if (!out) return 0;
if (!addr) return 0;
if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(int32_t))))
return 0;
*out = *(const int32_t*)addr;
return 1;
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
// Dumps indirect Call Slot.
@@ -693,21 +584,21 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
KiUserReadI32(ret - 4, &disp)) {
insn = ret - 7;
} else {
printf("No RIP-relative indirect call pattern found before return address.\n");
KiFaultPrintf("No RIP-relative indirect call pattern found before return address.\n");
return;
}
uintptr_t slot = ret + (intptr_t)disp;
uint64_t target = 0;
printf("Decoded null indirect call:\n");
KiFaultPrintf("Decoded null indirect call:\n");
KiPrintAddress(proc, " insn=", insn);
KiPrintAddress(proc, " slot=", slot);
if (KiUserReadU64(slot, &target)) {
KiPrintAddress(proc, " *slot=", (uintptr_t)target);
} else {
printf(" *slot=<unreadable>\n");
KiFaultPrintf(" *slot=<unreadable>\n");
}
KiDumpBytes(proc, "Bytes near indirect slot", slot);
@@ -717,9 +608,9 @@ static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
static void KiDumpModules(PROCESS* proc) {
if (!proc) return;
printf("Images:\n");
KiFaultPrintf("Images:\n");
if (proc->ImageBase && proc->ImageSize) {
printf(" %-31s %016llx-%016llx size=%llx\n",
KiFaultPrintf(" %-31s %016llx-%016llx size=%llx\n",
proc->Name,
(unsigned long long)proc->ImageBase,
(unsigned long long)(proc->ImageBase + proc->ImageSize),
@@ -729,7 +620,7 @@ static void KiDumpModules(PROCESS* proc) {
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
uintptr_t base = proc->Modules[i].Base;
size_t size = proc->Modules[i].Size;
printf(" %-31s %016llx-%016llx size=%llx\n",
KiFaultPrintf(" %-31s %016llx-%016llx size=%llx\n",
proc->Modules[i].Name,
(unsigned long long)base,
(unsigned long long)(base + size),
@@ -739,9 +630,9 @@ static void KiDumpModules(PROCESS* proc) {
// Dumps user Stack.
static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
printf("User stack @ %016llx:\n", (unsigned long long)rsp);
KiFaultPrintf("User stack @ %016llx:\n", (unsigned long long)rsp);
if (!rsp) {
printf(" <no user RSP captured>\n");
KiFaultPrintf(" <no user RSP captured>\n");
return;
}
@@ -753,20 +644,20 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
size_t size = 0;
const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size);
if (base) {
printf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n",
KiFaultPrintf(" [%02d] %016llx : %016llx <%s+0x%016llX>\n",
i,
(unsigned long long)slot,
(unsigned long long)value,
mod,
(unsigned long long)((uintptr_t)value - base));
} else {
printf(" [%02d] %016llx : %016llx\n",
KiFaultPrintf(" [%02d] %016llx : %016llx\n",
i,
(unsigned long long)slot,
(unsigned long long)value);
}
} else {
printf(" [%02d] %016llx : <unreadable>\n",
KiFaultPrintf(" [%02d] %016llx : <unreadable>\n",
i,
(unsigned long long)slot);
}
@@ -777,20 +668,20 @@ static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
if (!f) return;
printf("Regs:\n");
printf(" RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n",
KiFaultPrintf("Regs:\n");
KiFaultPrintf(" RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n",
(unsigned long long)f->Rax, (unsigned long long)f->Rbx,
(unsigned long long)f->Rcx, (unsigned long long)f->Rdx);
printf(" RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n",
KiFaultPrintf(" RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n",
(unsigned long long)f->Rsi, (unsigned long long)f->Rdi,
(unsigned long long)f->Rbp, (unsigned long long)f->Rsp);
printf(" R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n",
KiFaultPrintf(" R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n",
(unsigned long long)f->R8, (unsigned long long)f->R9,
(unsigned long long)f->R10, (unsigned long long)f->R11);
printf(" R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n",
KiFaultPrintf(" R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n",
(unsigned long long)f->R12, (unsigned long long)f->R13,
(unsigned long long)f->R14, (unsigned long long)f->R15);
printf(" CS=%04llx SS=%04llx RFLAGS=%016llx\n",
KiFaultPrintf(" CS=%04llx SS=%04llx RFLAGS=%016llx\n",
(unsigned long long)f->Cs,
(unsigned long long)f->Ss,
(unsigned long long)f->Rflags);
@@ -798,12 +689,12 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
if (f->Rip == 0) {
uint64_t ret = 0;
if (KiUserReadU64((uintptr_t)f->Rsp, &ret)) {
printf("Likely null indirect call. Return address from [RSP]:\n");
KiFaultPrintf("Likely null indirect call. Return address from [RSP]:\n");
KiPrintAddress(proc, " ret=", (uintptr_t)ret);
KiDumpBytes(proc, "Bytes near return address", (uintptr_t)ret);
KiDumpIndirectCallSlot(proc, (uintptr_t)ret);
} else {
printf("Likely null indirect call, but [RSP] is unreadable.\n");
KiFaultPrintf("Likely null indirect call, but [RSP] is unreadable.\n");
}
} else {
KiDumpBytes(proc, "Bytes near RIP", (uintptr_t)f->Rip);
@@ -812,14 +703,6 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
KiDumpUserStack(proc, (uintptr_t)f->Rsp);
}
// Prints crash Char.
static void KiPrintCrashChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xFFAA00, 0x000000);
}
static KSTATUS KiFaultExitStatus(int vector) {
switch (vector) {
case 0: return STATUS_PROCESS_DIVIDE_BY_ZERO;
@@ -840,13 +723,11 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
const char* modName = proc ? KiResolveModule(proc, rip) : "unknown";
static const char banner[] = "\n*** APPLICATION CRASH ***\n";
for (const char* p = banner; *p; p++)
KiPrintCrashChar(*p);
printf_set_putchar(KiPrintCrashChar);
KiOutputWrite(HAL_OUTPUT_STREAM_CRASH, 0,
0x00FFAA00u, 0x00000000u, banner, sizeof(banner) - 1u);
if (vector == 14)
printf("Process: %s (PID %u) | Fault: %s\n"
KiFaultPrintf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
procName, pid, faultName,
modName,
@@ -854,7 +735,7 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
(unsigned long long)errCode,
(unsigned long long)cr2);
else
printf("Process: %s (PID %u) | Fault: %s\n"
KiFaultPrintf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx\n",
procName, pid, faultName,
modName,
@@ -864,21 +745,19 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
printf_set_putchar(KiKernelPutChar);
int exitCode = (int)exitStatus;
if (proc) (void)KeTerminateProcess(proc, exitCode);
if (thread) KeExitThread(thread, exitCode);
// KeExitThread normally switches away and never resumes this context. If
// KeExitThread normally switches away and never resumes this context. If
// a fault path re-enters here during scheduler bring-up or with no runnable
// peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
// dead thread to the scheduler with interrupts enabled so waiters such as
// the foreground shell can observe the process exit event.
while (true) {
_enable();
HalEnableInterrupts();
KeYield();
__halt();
HalWaitForInterrupt();
}
}
@@ -932,6 +811,7 @@ __declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
KiDumpFaultFrame(proc, frame);
if (vector == 14) {
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
if (frame->Rip) MmDebugDescribeAddress((uintptr_t)frame->Rip);
if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
MmVadDebugDumpProcess(proc);
}
@@ -994,5 +874,5 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
frame.Rip = rip;
frame.Cs = cs;
(void)KeFaultDispatchFrame(&frame);
while (1) __halt();
HalStopProcessor();
}
code/kernel/ke_display.c
+144 -36
View File
@@ -30,6 +30,14 @@
#define DISPLAY_CONSOLE_MAX_CELLS (DISPLAY_CONSOLE_MAX_COLS * DISPLAY_CONSOLE_MAX_ROWS)
#define DISPLAY_CONSOLE_DEFAULT_ATTR 0x07u
#define DISPLAY_CURSOR_POS_INVALID (-2147483647 - 1)
#define DISPLAY_OUTPUT_BUFFER_CAPACITY 4096u
#define DISPLAY_OUTPUT_DRAIN_COUNT 128u
typedef struct DISPLAY_OUTPUT_GLYPH {
char Character;
uint32_t Foreground;
uint32_t Background;
} DISPLAY_OUTPUT_GLYPH;
typedef struct BOOTFB_CONTEXT {
const HAL_API* Hal;
@@ -78,7 +86,15 @@ static uint16_t gConsoleAttribute;
static char gConsoleCells[DISPLAY_CONSOLE_MAX_CELLS];
static uint8_t gConsoleAttrs[DISPLAY_CONSOLE_MAX_CELLS];
static volatile long gDisplayConsoleSuspended;
static SPINLOCK gDisplayOutputLock = SPINLOCK_INIT;
static DISPLAY_OUTPUT_GLYPH* gDisplayOutputBuffer;
static size_t gDisplayOutputRead;
static size_t gDisplayOutputWrite;
static size_t gDisplayOutputUsed;
static KWORK_ITEM gDisplayOutputWorkItem;
static volatile long gDisplayOutputWorkInitialized;
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg);
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels);
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color);
@@ -105,7 +121,6 @@ static const DISPLAY_DEVICE_OPS gBootFbOps = {
.QueryStats = BootFbQueryStats,
};
static bool KiDisplayOutputSuspended(void);
static void KiDisplayCursorWorker(void* context);
static void KiDisplayDirtyAll(void);
@@ -161,14 +176,13 @@ static void KiDisplayQueueCursorWork(void) {
return;
}
// During single-threaded early boot there is no worker thread yet. The
// During single-threaded early boot there is no worker thread yet. The
// boot framebuffer console still needs synchronous cursor setup; once the
// scheduler work queue exists all cursor commits are asynchronous.
if (!KiDisplayProviderCallbacksRequirePassive())
KiDisplayCursorWorker(0);
}
// Checks whether a primary display provider is available.
static bool KiDisplayHasDevice(void) {
return gPrimaryDisplay && gPrimaryDisplay->Ops;
@@ -195,7 +209,6 @@ static bool KiDisplayHasCursorOps(void) {
gPrimaryDisplay->Ops->ShowCursor;
}
static void KiDisplayCursorWorker(void* context) {
(void)context;
@@ -263,9 +276,9 @@ static void KiDisplayCursorWorker(void* context) {
newCommittedY = DISPLAY_CURSOR_POS_INVALID;
}
} else if (hardwareCursor) {
// Hardware cursors are independent of framebuffer drawing. Commit
// Hardware cursors are independent of framebuffer drawing. Commit
// position only when the logical cursor request changed or the
// provider has not yet been synchronized. This prevents stale or
// provider has not yet been synchronized. This prevents stale or
// duplicate work items from hammering device cursor registers.
bool needPosition = !drawn || x != committedX || y != committedY;
if (needPosition && ops->SetCursorPosition) {
@@ -308,7 +321,7 @@ static void KiDisplayCursorWorker(void* context) {
gCursorCommittedX = newCommittedX;
gCursorCommittedY = newCommittedY;
}
// Do not spin forever on provider failure. A later explicit cursor
// Do not spin forever on provider failure. A later explicit cursor
// request, show/hide request, or display-generation change will retry.
gCursorAppliedGeneration = requestGeneration;
if (gCursorRequestGeneration != requestGeneration)
@@ -349,8 +362,7 @@ static bool KiDisplayClipRect(const DISPLAY_MODE* mode, DISPLAY_RECT* rect) {
return true;
}
// Snapshots the current primary under the manager spinlock. The caller must
// Snapshots the current primary under the manager spinlock. The caller must
// hold the provider mutex before invoking callbacks through the returned ops;
// KeDisplaySetPrimary takes the same mutex, so the provider object cannot be
// switched while the callback sequence is in progress.
@@ -966,12 +978,109 @@ static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t
KiDisplayConsoleBg(attr));
}
// Implements the display Serial Write Char operation.
static void KiDisplaySerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n')
__outbyte(0x3F8, '\r');
__outbyte(0x3F8, (uint8_t)c);
// Appends one glyph to the runtime display output ring while its lock is held.
static void KiDisplayOutputAppendLocked(char character, uint32_t foreground, uint32_t background) {
if (!gDisplayOutputBuffer)
return;
if (gDisplayOutputUsed == DISPLAY_OUTPUT_BUFFER_CAPACITY) {
gDisplayOutputRead = (gDisplayOutputRead + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
gDisplayOutputUsed--;
}
DISPLAY_OUTPUT_GLYPH* glyph = &gDisplayOutputBuffer[gDisplayOutputWrite];
glyph->Character = character;
glyph->Foreground = foreground;
glyph->Background = background;
gDisplayOutputWrite = (gDisplayOutputWrite + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
gDisplayOutputUsed++;
}
// Drains buffered output through the active display provider.
static void KiDisplayOutputWorker(void* context) {
(void)context;
DISPLAY_OUTPUT_GLYPH glyphs[DISPLAY_OUTPUT_DRAIN_COUNT];
for (;;) {
size_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
while (count < DISPLAY_OUTPUT_DRAIN_COUNT && gDisplayOutputUsed != 0) {
glyphs[count++] = gDisplayOutputBuffer[gDisplayOutputRead];
gDisplayOutputRead = (gDisplayOutputRead + 1u) % DISPLAY_OUTPUT_BUFFER_CAPACITY;
gDisplayOutputUsed--;
}
KeReleaseSpinlock(&gDisplayOutputLock, flags);
if (count == 0)
break;
for (size_t i = 0; i < count; i++) {
(void)KiDisplayPrintCharInternal(glyphs[i].Character,
glyphs[i].Foreground,
glyphs[i].Background);
}
}
}
// Queues the runtime display output worker when passive work is available.
static void KiDisplayQueueOutputWorker(void) {
if (_InterlockedCompareExchange(&gDisplayOutputWorkInitialized, 1, 1) &&
KeIsWorkQueueStarted()) {
(void)KeQueueWorkItem(&gDisplayOutputWorkItem);
}
}
// Receives one HAL output record for the runtime display endpoint.
static void KiDisplayWriteOutputRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer || record->Length == 0)
return;
if (KiDisplayOutputSuspended())
return;
if (KeIsWorkQueueStarted() && gDisplayOutputBuffer) {
uintptr_t flags = KeAcquireSpinlock(&gDisplayOutputLock);
for (size_t i = 0; i < record->Length; i++) {
KiDisplayOutputAppendLocked(record->Buffer[i],
record->Foreground,
record->Background);
}
KeReleaseSpinlock(&gDisplayOutputLock, flags);
KiDisplayQueueOutputWorker();
return;
}
for (size_t i = 0; i < record->Length; i++) {
(void)KiDisplayPrintCharInternal(record->Buffer[i],
record->Foreground,
record->Background);
}
}
// Registers the runtime display as a HAL output endpoint.
static void KiDisplayRegisterOutputEndpoint(void) {
if (!gDisplayHal || !gDisplayHal->Output.RegisterEndpoint)
return;
HAL_OUTPUT_ENDPOINT endpoint;
endpoint.Name = "runtime-display";
endpoint.StreamMask = HAL_OUTPUT_STREAM_CONSOLE | HAL_OUTPUT_STREAM_CRASH;
if (KeGetDebugFlags() & KE_DEBUG_CONSOLE)
endpoint.StreamMask |= HAL_OUTPUT_STREAM_DEBUG;
endpoint.Capabilities = 0;
endpoint.Write = KiDisplayWriteOutputRecord;
endpoint.Context = 0;
uint32_t endpointId = HAL_OUTPUT_ENDPOINT_INVALID;
if (!KSUCCESS(gDisplayHal->Output.RegisterEndpoint(&endpoint, &endpointId)))
return;
(void)endpointId;
if (gDisplayHal->Output.SetEndpointStreams &&
gDisplayHal->Output.BootFramebufferEndpointId != HAL_OUTPUT_ENDPOINT_INVALID) {
(void)gDisplayHal->Output.SetEndpointStreams(
gDisplayHal->Output.BootFramebufferEndpointId,
HAL_OUTPUT_STREAM_PANIC);
}
}
// Implements the display Initialize operation.
@@ -1000,6 +1109,16 @@ void KeDisplayInitialize(const HAL_API* hal) {
(void)KeMutexCreate(&gDisplayProviderMutex);
KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
_InterlockedExchange(&gDisplayCursorWorkInitialized, 1);
KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0);
_InterlockedExchange(&gDisplayOutputWorkInitialized, 1);
gDisplayOutputRead = 0;
gDisplayOutputWrite = 0;
gDisplayOutputUsed = 0;
if (!gDisplayOutputBuffer) {
gDisplayOutputBuffer = (DISPLAY_OUTPUT_GLYPH*)MmAllocatePoolTag(
DISPLAY_OUTPUT_BUFFER_CAPACITY * sizeof(DISPLAY_OUTPUT_GLYPH),
MM_POOL_TAG('d', 'o', 'u', 't'));
}
gConsoleX = 0;
gConsoleY = 0;
KiDisplayConsoleResetCellsLocked();
@@ -1047,15 +1166,16 @@ void KeDisplayInitialize(const HAL_API* hal) {
mode.Width, mode.Height, mode.Stride);
}
// KeDisplayInitialize runs while the debug console still points at the
// HAL early console, so the registration messages above are written
// directly through HAL_SCREEN. Resync the bootfb backbuffer once before
// kmain switches DbgPrint to KeDisplayDebugPrintChar.
// The registration messages above were routed through the HAL boot
// framebuffer endpoint. Resync the display-manager backbuffer before
// the runtime display endpoint becomes the normal screen subscriber.
if (gBootFb.Backbuffer) {
size_t pixels2 = (size_t)gBootFb.Stride * (size_t)gBootFb.Height;
memcpy(gBootFb.Backbuffer, gBootFb.Framebuffer, pixels2 * sizeof(uint32_t));
}
}
KiDisplayRegisterOutputEndpoint();
}
DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
@@ -1120,7 +1240,6 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE
return s;
}
DISPLAY_API void KeDisplaySuspendConsole(bool suspend) {
_InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, suspend ? 1 : 0);
}
@@ -1171,7 +1290,7 @@ DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) {
(void)oldDevice->Ops->ShowCursor(oldDevice, false);
// Clean architecture policy: handoff does not read back/copy the old boot
// framebuffer. The new primary is cleared to the kernel console blue and
// framebuffer. The new primary is cleared to the kernel console blue and
// user/session code redraws state above the provider boundary.
if (device->Ops->GetMode && device->Ops->FillRect) {
DISPLAY_MODE mode;
@@ -1523,10 +1642,7 @@ DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t
return s;
}
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg, bool mirrorSerial) {
if (mirrorSerial)
KiDisplaySerialWriteChar(c);
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg) {
if (KiDisplayOutputSuspended()) {
if (!KeCanWait())
return KiDisplayFallbackPrintChar(c, fg, bg);
@@ -1534,11 +1650,8 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg, bool
}
// Once the scheduler/work-queue era begins, display provider callbacks are
// PASSIVE_LEVEL-only. DbgPrint queues screen output through the work queue;
// panic and explicit console paths may still use the serial mirroring entry.
// A panic has interrupts disabled and cannot take the provider mutex, so it
// falls back to the boot framebuffer writer instead of disappearing from
// the screen while still printing to COM1.
// PASSIVE_LEVEL-only. The HAL output endpoint buffers normal text so this
// fallback is reserved for explicit display calls made at elevated IRQL.
if (KiDisplayProviderCallbacksRequirePassive() && !KeCanWait())
return KiDisplayFallbackPrintChar(c, fg, bg);
@@ -1642,12 +1755,7 @@ static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg, bool
}
DISPLAY_API KSTATUS KeDisplayPrintChar(char c, uint32_t fg, uint32_t bg) {
return KiDisplayPrintCharInternal(c, fg, bg, true);
}
// Implements the display Debug Print Char operation.
void KeDisplayDebugPrintChar(char c, uint32_t fg, uint32_t bg) {
(void)KiDisplayPrintCharInternal(c, fg, bg, false);
return KiDisplayPrintCharInternal(c, fg, bg);
}
DISPLAY_API KSTATUS KeDisplayGetConsoleInfo(KE_CONSOLE_INFO* outInfo) {
code/kernel/ke_gdt_reload.asm
+2 -2
View File
@@ -5,8 +5,8 @@
KeFlushGdt PROC
; RCX = pGdtr (pointer to limit/base struct)
; RDX = codeSel
; R8 = dataSel
; R9 = tssSel
; R8 = dataSel
; R9 = tssSel
; 1. Load the GDT
lgdt fword ptr [rcx]
code/kernel/ke_idt.c
+1 -1
View File
@@ -5,7 +5,7 @@
// 0x08 ring-0 code (64-bit)
// 0x10 ring-0 data
// 0x18 ring-3 code (64-bit, DPL=3)
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x28 TSS low
// 0x30 TSS high.
#include <stdint.h>
code/kernel/ke_input.c
+55 -33
View File
@@ -9,6 +9,7 @@
#include <kernel/ke_debug.h>
#include <kernel/ke_display.h>
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
#include <kernel/ob_object.h>
#define KEY_QUEUE_SIZE 256u
@@ -42,6 +43,7 @@ static volatile int gKeyHead;
static volatile int gKeyTail;
static volatile INPUT_KEY gRepeatKey;
static volatile uint64_t gRepeatNextTick;
static KDPC gRepeatDpc;
static INPUT_MOUSE_EVENT gMouseQueue[MOUSE_QUEUE_SIZE];
static volatile int gMouseHead;
@@ -54,9 +56,9 @@ static volatile uint8_t gMouseButtons;
static volatile uint32_t gNextInputDeviceId = 1;
static EVENT* gKeyboardEvent;
static EVENT* gInputStateEvent;
static char KiInputKeyToChar(INPUT_KEY key);
static void KiInputServiceRepeat(void);
static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' },
@@ -82,7 +84,14 @@ static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Queues a keyboard event and wakes one blocked reader when the event was accepted.
// Updates the waitable keyboard event to match queued keyboard input.
static void KiInputRefreshKeyboardEvent(void) {
if (!gKeyboardEvent) return;
if (gKeyHead != gKeyTail) KeSignalEvent(gKeyboardEvent);
else KeClearEvent(gKeyboardEvent);
}
// Queues a keyboard event and wakes waiters when the event was accepted.
static void KiInputQueueKey(INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
@@ -91,9 +100,7 @@ static void KiInputQueueKey(INPUT_KEY key, char ch) {
gKeyQueue[gKeyTail].ch = ch;
gKeyQueue[gKeyTail].modifiers = gModifiers;
gKeyTail = next;
if (gKeyboardEvent)
KeSignalEvent(gKeyboardEvent);
KiInputRefreshKeyboardEvent();
}
// Implements the input Queue Has Space operation.
@@ -167,6 +174,16 @@ static void KiInputServiceRepeat(void) {
gRepeatNextTick = now + KEY_REPEAT_PERIOD_US;
}
// Implements the input Repeat Dpc Routine operation.
static void KiInputRepeatDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
KiInputServiceRepeat();
}
// Implements the input Set Modifier operation.
static void KiInputSetModifier(INPUT_KEY key, bool pressed) {
uint32_t bit = 0;
@@ -219,17 +236,14 @@ void KeInputInitialize(const HAL_API* hal) {
gRepeatNextTick = 0;
gMouseHead = gMouseTail = 0;
gMouseButtons = 0;
KeInitializeDpc(&gRepeatDpc, KiInputRepeatDpcRoutine, (void*)0);
if (!gKeyboardEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gKeyboardEvent);
KSTATUS s = KeEventCreate(EventManualReset, 0, &gKeyboardEvent);
if (!KSUCCESS(s))
DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
}
if (!gInputStateEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gInputStateEvent);
if (!KSUCCESS(s))
DbgPrint("Input: state wait event create failed st=0x%x\n", s);
}
KiInputRefreshKeyboardEvent();
int w = 1920, h = 1080;
DISPLAY_MODE mode;
@@ -272,7 +286,6 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 1u;
KiInputSetModifier(key, true);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputQueueKey(key, KiInputKeyToChar(key));
KiInputStartRepeat(key, KiInputNowTicks());
} else {
@@ -281,7 +294,6 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 0u;
KiInputSetModifier(key, false);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputStopRepeat(key);
}
}
@@ -363,16 +375,17 @@ INPUT_KEY KeInputKbPoll(void) {
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
}
// Waits until at least one keyboard event is available.
KSTATUS KeInputWaitKey(uint8_t alertable) {
// Waits until at least one keyboard event is available or the timeout expires.
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs) {
if (KeInputKbPoll() != INPUT_KEY_NONE)
return STATUS_SUCCESS;
if (!gKeyboardEvent)
return STATUS_FAILURE;
for (;;) {
KSTATUS s = KeWaitForEvent(gKeyboardEvent, alertable);
EVENT* event = gKeyboardEvent;
uint32_t selected = 0;
KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected);
if (!KSUCCESS(s))
return s;
if (KeInputKbPoll() != INPUT_KEY_NONE)
@@ -380,6 +393,17 @@ KSTATUS KeInputWaitKey(uint8_t alertable) {
}
}
// Services typematic repeat from the scheduler timer bottom half.
void KeInputTimerTick(void) {
INPUT_KEY key = gRepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !gKeyDown[key])
return;
uint64_t now = KiInputNowTicks();
if (now != 0 && now >= gRepeatNextTick)
KeInsertQueueDpc(&gRepeatDpc, (void*)0, (void*)0);
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
@@ -395,7 +419,6 @@ bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead];
record->Key = entry.key;
record->Char = entry.ch;
record->Reserved = 0;
record->Modifiers = entry.modifiers;
return true;
}
@@ -404,6 +427,7 @@ bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
if (!KeInputPeekEvent(record)) return false;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
KiInputRefreshKeyboardEvent();
return true;
}
@@ -424,21 +448,19 @@ uint32_t KeInputGetModifiers(void) {
return gModifiers;
}
// Implements the input Mouse Poll operation.
bool KeInputMousePoll(void) {
return gMouseHead != gMouseTail;
// Implements the input Mouse Peek operation.
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event) {
if (!event) return false;
if (gMouseHead == gMouseTail) return false;
*event = gMouseQueue[gMouseHead];
return true;
}
// Implements the input Mouse Read operation.
bool KeInputMouseRead(int* dx, int* dy, uint8_t* buttons) {
if (!dx || !dy || !buttons) return false;
if (gMouseHead == gMouseTail) return false;
INPUT_MOUSE_EVENT ev = gMouseQueue[gMouseHead];
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event) {
if (!KeInputMousePeek(event)) return false;
gMouseHead = (gMouseHead + 1) & (int)MOUSE_QUEUE_MASK;
*dx = ev.dx;
*dy = ev.dy;
*buttons = ev.buttons;
return true;
}
@@ -485,18 +507,18 @@ static void KiInputQueueDestroy(OBJECT* object) {
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
if (!out) return STATUS_INVALID_PARAMETER;
*out = (INPUT_QUEUE*)0;
if (!gInputStateEvent) return STATUS_FAILURE;
if (!gKeyboardEvent) return STATUS_FAILURE;
INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
if (!queue) return STATUS_NO_MEMORY;
queue->Header.Destroy = KiInputQueueDestroy;
ObReferenceObject(&gInputStateEvent->Header);
queue->WaitEvent = gInputStateEvent;
ObReferenceObject(&gKeyboardEvent->Header);
queue->WaitEvent = gKeyboardEvent;
*out = queue;
return STATUS_SUCCESS;
}
// Returns the event that wakes when keyboard state changes.
// Returns the event that wakes when keyboard input can be read.
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
return queue ? queue->WaitEvent : (EVENT*)0;
}
code/kernel/ke_process.c
+73 -11
View File
@@ -66,6 +66,7 @@ KSTATUS KeCreateSystemProcess(PROCESS** out) {
return s;
}
proc->Flags |= KE_PROCESS_FLAG_SYSTEM | KE_PROCESS_FLAG_TERMINATION_PROTECTED;
gSystemProcess = proc;
*out = proc;
return STATUS_SUCCESS;
@@ -100,8 +101,6 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
KiProcessInit();
PROCESS* current = KeGetCurrentProcess();
PROCESS* system = KeGetSystemProcess();
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
size_t count = 0;
LIST_ENTRY* head = &gProcessList;
@@ -120,7 +119,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
uintptr_t handleFlags = KeAcquireSpinlock(&p->HandleLock);
uint32_t threadCount = p->ThreadCount;
uint32_t handleCount = KiCountProcessHandlesLocked(p);
int exitCode = p->ExitCode;
int32_t exitCode = p->ExitCode;
long terminating = p->Terminating;
long exited = p->Exited;
KeReleaseSpinlock(&p->HandleLock, handleFlags);
@@ -139,7 +138,10 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
info->State = SYS_PROCESS_STATE_RUNNING;
}
if (p == system) info->Flags |= SYS_PROCESS_FLAG_SYSTEM;
if (p->Flags & KE_PROCESS_FLAG_SYSTEM) info->Flags |= SYS_PROCESS_FLAG_SYSTEM;
if (p->Flags & KE_PROCESS_FLAG_IDLE) info->Flags |= SYS_PROCESS_FLAG_IDLE;
if (p->Flags & KE_PROCESS_FLAG_TERMINATION_PROTECTED)
info->Flags |= SYS_PROCESS_FLAG_TERMINATION_PROTECTED;
if (p == current) info->Flags |= SYS_PROCESS_FLAG_CURRENT;
MmVadQueryProcessMemory(p, &info->VadCount,
@@ -190,6 +192,7 @@ static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
case ObjectTypeUsbPipe: return SYS_THREAD_WAIT_USB_PIPE;
case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING;
case ObjectTypeSurface: return SYS_THREAD_WAIT_SURFACE;
case ObjectTypeTimer: return SYS_THREAD_WAIT_TIMER;
default: return SYS_THREAD_WAIT_UNKNOWN;
}
}
@@ -276,6 +279,7 @@ static void KiProcessRundownResources(PROCESS* proc) {
if (_InterlockedExchange(&proc->ResourcesReleased, 1) != 0)
return;
IoReleaseFileLocksForProcess(proc);
KiProcessReleaseAllHandles(proc);
// Release every user VA region owned by this process. This includes the
@@ -334,6 +338,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->Header.Destroy = KiProcessDestroy;
proc->ProcessId = _InterlockedExchangeAdd((volatile long *)&gNextProcessId, 1);
proc->Flags = 0;
strncpy(proc->Name, name, PROCESS_NAME_MAX - 1);
proc->Name[PROCESS_NAME_MAX - 1] = '\0';
@@ -381,7 +386,6 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
return STATUS_SUCCESS;
}
// KeSetupUserProcessEnvironment.
static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
SYS_STRING_VIEW src,
@@ -558,8 +562,10 @@ void KeCompleteProcessExitIfLastThread(PROCESS* process) {
}
// Implements the exit Process operation.
void KeExitProcess(PROCESS* process, int exitCode) {
if (!process) return;
KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode) {
if (!process) return STATUS_INVALID_PARAMETER;
if (KeIsProcessTerminationProtected(process)) return STATUS_ACCESS_DENIED;
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
@@ -567,11 +573,13 @@ void KeExitProcess(PROCESS* process, int exitCode) {
// not drop the process object here: each thread owns one process reference,
// and KiThreadDestroy releases that reference after unlinking the thread.
KeCompleteProcessExitIfLastThread(process);
return STATUS_SUCCESS;
}
// Implements the terminate Process operation.
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
if (!process) return STATUS_INVALID_PARAMETER;
if (KeIsProcessTerminationProtected(process)) return STATUS_ACCESS_DENIED;
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
@@ -588,7 +596,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
continue;
// A terminating process must not be held open by worker threads that
// are blocked in non-alertable waits. User-mode DLLs such as Bink can
// are blocked in non-alertable waits. User-mode DLLs such as Bink can
// leave IO/submit workers parked on semaphores; after the foreground
// thread crashes, those waiters would otherwise keep ThreadCount > 0
// and prevent the process ExitEvent from waking the shell.
@@ -616,8 +624,13 @@ int KeIsProcessTerminating(PROCESS* process) {
return process && process->Terminating != 0;
}
// KeGetProcessExitCode.
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut) {
// Returns whether process termination is forbidden.
int KeIsProcessTerminationProtected(PROCESS* process) {
return process && (process->Flags & KE_PROCESS_FLAG_TERMINATION_PROTECTED) != 0;
}
// Copies the recorded process exit code.
KSTATUS KeGetProcessExitCode(PROCESS* process, int32_t* exitCodeOut) {
if (!process || !exitCodeOut) return STATUS_INVALID_PARAMETER;
*exitCodeOut = process->ExitCode;
return STATUS_SUCCESS;
@@ -699,6 +712,55 @@ uint32_t KeGetHandleRights(PROCESS* process, uint32_t handle) {
return rights;
}
// Duplicates a handle from one process table into another.
KSTATUS KeDuplicateHandleToProcess(PROCESS* sourceProcess, uint32_t sourceHandle, PROCESS* targetProcess, uint32_t desiredRights, uint32_t duplicateFlags, uint32_t* targetHandleOut) {
if (!sourceProcess || !targetProcess || !targetHandleOut) return STATUS_INVALID_PARAMETER;
if (sourceHandle == 0 || sourceHandle >= HANDLE_TABLE_SIZE) return STATUS_INVALID_HANDLE;
const uint32_t validDuplicateFlags = SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS;
const uint32_t publicHandleRights = SYS_HANDLE_RIGHT_ALL;
if ((duplicateFlags & ~validDuplicateFlags) != 0) return STATUS_INVALID_PARAMETER;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) != 0 && desiredRights != 0)
return STATUS_INVALID_PARAMETER;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) == 0) {
if (desiredRights == 0) return STATUS_INVALID_PARAMETER;
if ((desiredRights & ~publicHandleRights) != 0) return STATUS_INVALID_PARAMETER;
}
OBJECT* obj = NULL;
uint32_t sourceRights = 0;
uintptr_t savedFlags = KeAcquireSpinlock(&sourceProcess->HandleLock);
HANDLE_ENTRY* entry = &sourceProcess->Handles[sourceHandle];
if (!entry->Object) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return STATUS_INVALID_HANDLE;
}
if ((entry->Rights & HANDLE_RIGHT_DUPLICATE) == 0) {
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
return STATUS_ACCESS_DENIED;
}
obj = entry->Object;
sourceRights = entry->Rights;
ObReferenceObject(obj);
KeReleaseSpinlock(&sourceProcess->HandleLock, savedFlags);
uint32_t targetRights = sourceRights;
if ((duplicateFlags & SYS_DUPLICATE_HANDLE_FLAG_SAME_RIGHTS) == 0) {
if ((desiredRights & ~sourceRights) != 0) {
ObDereferenceObject(obj);
return STATUS_ACCESS_DENIED;
}
targetRights = desiredRights;
}
KSTATUS s = KeAllocateHandleWithRights(targetProcess, obj, targetRights, targetHandleOut);
ObDereferenceObject(obj);
return s;
}
// Returns process By Id.
PROCESS* KeGetProcessById(uint32_t pid) {
KiProcessInit();
code/kernel/ke_queue.c
+145 -53
View File
@@ -1,97 +1,189 @@
// Kernel message queue primitive. This file owns bounded queue storage and blocking send/receive behavior.
// Kernel message queue primitive. This file owns bounded queue storage and queue timeout behavior.
#include <stdint.h>
#include <string.h>
#include <kernel/ke_queue.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ob_object.h>
// Implements the queue Destroy operation.
static void KiQueueDestroy(OBJECT* obj) { (void)obj; }
#define QUEUE_TIMEOUT_INFINITE UINT64_MAX
// Implements the queue Create operation.
// Destroys a queue and releases its private state events.
static void KiQueueDestroy(OBJECT* obj) {
QUEUE* queue = (QUEUE*)obj;
if (!queue)
return;
if (queue->NotEmptyEvent) {
ObDereferenceObject(&queue->NotEmptyEvent->Header);
queue->NotEmptyEvent = (EVENT*)0;
}
if (queue->NotFullEvent) {
ObDereferenceObject(&queue->NotFullEvent->Header);
queue->NotFullEvent = (EVENT*)0;
}
}
// Updates waitable state after one message has been inserted.
static void KiQueueNoteInsertedLocked(QUEUE* queue, uint32_t previousCount) {
if (!queue || !queue->NotEmptyEvent || !queue->NotFullEvent)
return;
if (previousCount == 0)
KeSignalEvent(queue->NotEmptyEvent);
if (queue->Count == QUEUE_DEPTH)
KeClearEvent(queue->NotFullEvent);
}
// Updates waitable state after one message has been removed.
static void KiQueueNoteRemovedLocked(QUEUE* queue, uint32_t previousCount) {
if (!queue || !queue->NotEmptyEvent || !queue->NotFullEvent)
return;
if (previousCount == QUEUE_DEPTH)
KeSignalEvent(queue->NotFullEvent);
if (queue->Count == 0)
KeClearEvent(queue->NotEmptyEvent);
}
// Returns the scheduler tick value used by queue finite-timeout accounting.
static uint64_t KiQueueReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Converts a millisecond timeout to at least one scheduler tick.
static uint64_t KiQueueTimeoutTicks(uint64_t timeoutMilliseconds) {
uint64_t ticks = timeoutMilliseconds / SCHED_TICK_MS;
if ((timeoutMilliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Computes an absolute scheduler-tick deadline for a finite timeout.
static uint64_t KiQueueTimeoutDeadline(uint64_t timeoutMilliseconds) {
uint64_t now = KiQueueReadSchedulerTick();
uint64_t ticks = KiQueueTimeoutTicks(timeoutMilliseconds);
uint64_t deadline = now + ticks;
return deadline < now ? UINT64_MAX : deadline;
}
// Computes the remaining wait time for a finite timeout deadline.
static uint64_t KiQueueRemainingMilliseconds(uint64_t deadlineTick) {
uint64_t now = KiQueueReadSchedulerTick();
if (deadlineTick <= now)
return 0;
uint64_t ticks = deadlineTick - now;
if (ticks > UINT64_MAX / SCHED_TICK_MS)
return UINT64_MAX;
return ticks * SCHED_TICK_MS;
}
// Waits for a queue state event using either an infinite or finite timeout.
static KSTATUS KiQueueWaitForEvent(QUEUE* queue, EVENT* event, uint64_t timeoutMilliseconds, uint64_t deadlineTick) {
if (!queue || !event)
return STATUS_INVALID_PARAMETER;
uint64_t waitMilliseconds = timeoutMilliseconds;
if (timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE) {
waitMilliseconds = KiQueueRemainingMilliseconds(deadlineTick);
if (waitMilliseconds == 0)
return STATUS_TIMEOUT;
}
EVENT* events[1] = { event };
return KeWaitForMultipleEvents(events, 1, 0, waitMilliseconds, (uint32_t*)0);
}
// Creates a fixed-depth queue with waitable occupancy and capacity events.
KSTATUS KeQueueCreate(QUEUE **out) {
if (!out) return STATUS_INVALID_PARAMETER;
*out = (QUEUE*)0;
QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
if (!q) return STATUS_NO_MEMORY;
q->Header.Destroy = KiQueueDestroy;
q->Lock = (SPINLOCK)SPINLOCK_INIT;
q->Head = q->Tail = q->Count = 0;
LIST_INIT(&q->SendWaiters);
LIST_INIT(&q->RecvWaiters);
KSTATUS s = KeEventCreate(EventManualReset, 0, &q->NotEmptyEvent);
if (!KSUCCESS(s)) {
ObDereferenceObject(&q->Header);
return s;
}
s = KeEventCreate(EventManualReset, 1, &q->NotFullEvent);
if (!KSUCCESS(s)) {
ObDereferenceObject(&q->Header);
return s;
}
*out = q;
return STATUS_SUCCESS;
}
// Implements the send Message operation.
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
// Sends one fixed-size queue message, waiting according to the timeout contract.
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (timeoutMilliseconds != 0 && !KeCanWait()) return STATUS_INVALID_IRQL;
while (true) {
uint64_t deadlineTick = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count < QUEUE_DEPTH) {
uint32_t previousCount = queue->Count;
queue->Ring[queue->Tail] = *msg;
queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH;
queue->Count++;
// Wake a receiver if one is waiting.
if (!LIST_IS_EMPTY(&queue->RecvWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
KiQueueNoteInsertedLocked(queue, previousCount);
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_FULL;
}
KePrepareBlockThread(self, &queue->Header, 0);
ListInsertTail(&queue->SendWaiters, &self->SchedulerLink);
KeSetThreadWaitList(self, &queue->SendWaiters);
KeClearEvent(queue->NotFullEvent);
KeReleaseSpinlock(&queue->Lock, flags);
KeCommitBlockThread(self, 0);
if (timeoutMilliseconds == 0)
return STATUS_WOULD_BLOCK;
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotFullEvent, timeoutMilliseconds, deadlineTick);
if (s == STATUS_TIMEOUT || KFAILED(s))
return s;
}
}
// Implements the receive Message operation.
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
// Receives one fixed-size queue message, waiting according to the timeout contract.
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, uint64_t timeoutMilliseconds) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (timeoutMilliseconds != 0 && !KeCanWait()) return STATUS_INVALID_IRQL;
while (true) {
uint64_t deadlineTick = 0;
if (timeoutMilliseconds != 0 && timeoutMilliseconds != QUEUE_TIMEOUT_INFINITE)
deadlineTick = KiQueueTimeoutDeadline(timeoutMilliseconds);
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count > 0) {
uint32_t previousCount = queue->Count;
*msg = queue->Ring[queue->Head];
queue->Head = (queue->Head + 1) % QUEUE_DEPTH;
queue->Count--;
// Wake a sender if one is waiting.
if (!LIST_IS_EMPTY(&queue->SendWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->SendWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
KiQueueNoteRemovedLocked(queue, previousCount);
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_EMPTY;
}
KePrepareBlockThread(self, &queue->Header, 0);
ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink);
KeSetThreadWaitList(self, &queue->RecvWaiters);
KeClearEvent(queue->NotEmptyEvent);
KeReleaseSpinlock(&queue->Lock, flags);
KeCommitBlockThread(self, 0);
if (timeoutMilliseconds == 0)
return STATUS_WOULD_BLOCK;
KSTATUS s = KiQueueWaitForEvent(queue, queue->NotEmptyEvent, timeoutMilliseconds, deadlineTick);
if (s == STATUS_TIMEOUT || KFAILED(s))
return s;
}
}
code/kernel/ke_sched.c
+7 -11
View File
@@ -3,7 +3,6 @@
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
#pragma intrinsic(_enable)
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_process.h>
@@ -12,6 +11,8 @@
#include <kernel/ke_irql.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_input.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_list.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
@@ -196,7 +197,6 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the arm Post Switch Unlock operation.
static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
// The scheduler lock is intentionally held across KeSwitchContext(). The
@@ -213,7 +213,6 @@ static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
return thread ? thread->SwitchLockCookie : 0x202ULL;
}
// Implements the scheduler Bug operation.
static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
KePanic(KE_PANIC_CODE_SCHEDULER,
@@ -298,8 +297,7 @@ static void KiIdleLoop(void) {
// the idle thread through an unusual switch path must not sleep with
// interrupts disabled, otherwise reschedule/TLB shootdown IPIs can
// be delayed forever.
_enable();
__halt();
HalWaitForInterrupt();
}
}
@@ -312,6 +310,7 @@ void KiInitializeSchedulerGlobal(void) {
KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create idle process, status=%u\n", s);
gIdleProcess->Flags |= KE_PROCESS_FLAG_IDLE | KE_PROCESS_FLAG_TERMINATION_PROTECTED;
PROCESS* systemProc;
s = KeCreateSystemProcess(&systemProc);
@@ -463,7 +462,6 @@ void KeUnreadyThread(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the any Runnable For Cpu Locked operation.
static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
@@ -507,7 +505,6 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
return cpu->IdleThread;
}
// Implements the request Reschedule operation.
void KiRequestReschedule(void) {
KCPU* cpu = KeGetCurrentCpu();
@@ -592,7 +589,6 @@ void KiScheduleFromInterrupt(void) {
KiSchedule();
}
// Implements the scheduler Mark Thread Waiting operation.
void KiSchedulerMarkThreadWaiting(THREAD* thread) {
if (!thread) return;
@@ -624,7 +620,6 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
KeReleaseSpinlock(&gSchedLock, flags);
}
// Completes a wait while the scheduler lock is held.
static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
if (!thread) return;
@@ -666,7 +661,6 @@ static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
KiPokeRunnableCpusLocked(thread);
}
// Wakes a thread after first removing its scheduler-link wait membership.
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
@@ -719,6 +713,7 @@ void KeSchedulerTick(void) {
gSchedStats.Ticks++;
KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
KeInputTimerTick();
int requestReschedule = 0;
@@ -783,11 +778,12 @@ void KeSchedulerTick(void) {
KeReleaseSpinlock(&gSchedLock, flags);
KeTimerTick();
if (requestReschedule)
KiRequestReschedule();
}
// Arms a timeout for the current thread after a wait primitive has prepared it to block.
KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) {
if (milliseconds == 0)
code/kernel/ke_selftest.c
+1 -11
View File
@@ -478,10 +478,6 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
DbgPrint("IRP/APC selftest: PASS\n");
}
// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
static void KiRunPfnDatabaseSelfTest(void) {
MM_PFN_STATS before;
@@ -654,7 +650,6 @@ static void KiRunMemoryMappingSelfTest(void) {
DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
}
// Verifies sparse VAD commitment and NOACCESS protection metadata.
static void KiRunVadSparseSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
@@ -756,7 +751,6 @@ static void KiRunVadSparseSelfTest(void) {
DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
}
// Verifies that writable copy-on-write views split before either alias can modify shared state.
static void KiRunVadCopyOnWriteSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
@@ -872,7 +866,6 @@ static void KiRunVadCopyOnWriteSelfTest(void) {
DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
}
// Verifies that a private resident VAD page can move to swap and fault back in.
static void KiRunVadSwapSelfTest(void) {
if (!MmSwapIsEnabled()) {
@@ -1098,6 +1091,7 @@ static void KiRunAsyncFileIoSelfTest(void) {
wiosb.Status = STATUS_FAILURE;
wiosb.Information = 0;
(void)IoDeleteFile(SYS_SV_LITERAL("asyncio.tmp"));
s = IoCreateFile(SYS_SV_LITERAL("asyncio.tmp"), &wf);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: create write temp failed %u\n", s);
@@ -1161,7 +1155,6 @@ static void KiRunAsyncFileIoSelfTest(void) {
DbgPrint("FILE/ASYNC selftest: PASS\n");
}
// Runs trap Frame Self Test.
static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: begin\n");
@@ -1180,8 +1173,6 @@ static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: PASS\n");
}
// Runs core Hardening Self Tests.
void KiRunCoreHardeningSelfTests(void) {
DbgPrint("CORE selftest: begin\n");
@@ -1253,7 +1244,6 @@ void KiRunCoreHardeningSelfTests(void) {
DbgPrint("CORE selftest: PASS\n");
}
// Runs early Kernel Self Tests.
void KiRunEarlyKernelSelfTests(void) {
KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests();
code/kernel/ke_smp.c
+12 -18
View File
@@ -8,8 +8,6 @@
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <ntlite/status.h>
#include <kernel/ke_smp.h>
#include <kernel/ke_cpu.h>
@@ -254,11 +252,10 @@ static KSTATUS KiBuildApTrampoline(void) {
}
_Noreturn void KiApKernelEntry(KCPU* cpu) {
_disable();
(void)HalDisableInterrupts();
if (!cpu) {
while (1) __halt();
}
if (!cpu)
HalStopProcessor();
// APs start from the SIPI reset state, so do not assume the firmware/BSP
// CR4 feature bits are present. The scheduler and context-switch paths
@@ -313,27 +310,26 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
cpu->Parked = 0;
// Do not arm the AP's periodic LAPIC timer while it is still in
// the raw park loop. Timer IRQs are scheduler-owned events;
// the raw park loop. Timer IRQs are scheduler-owned events;
// the AP may take them only after it has a current idle thread.
HalTimerConfigureCurrentCpu();
_InterlockedExchange(&cpu->SchedulerActive, 1);
_enable();
HalEnableInterrupts();
KeStartInitialThread(idle);
}
}
// The BSP admits parked APs by storing SchedulerCommand and sending a
// wake IPI. Avoid the classic lost wakeup where the command appears
// wake IPI. Avoid the classic lost wakeup where the command appears
// between the poll and HLT: check once with interrupts disabled, then
// STI;HLT as a single sleep transition.
_disable();
uintptr_t interruptFlags = HalDisableInterrupts();
if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) {
_enable();
HalRestoreInterrupts(interruptFlags);
continue;
}
_enable();
__halt();
HalWaitForInterrupt();
}
}
@@ -367,7 +363,6 @@ static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) {
cpu->SchedulerActive = 0;
}
// Implements the ceil Log2 U32 operation.
static uint32_t KiCeilLog2U32(uint32_t value) {
if (value <= 1u)
@@ -591,7 +586,7 @@ void KeBuildProcessorTopology(void) {
// Creates ap Stack.
static KSTATUS KiAllocateApStack(KCPU* cpu) {
// AP kernel stacks do not need physically contiguous memory. Requiring
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmPfnAllocateContiguous(8) made SMP startup depend on a DMA-style run being
// available, which is fragile on real machines and unrelated to AP startup.
size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE;
@@ -612,7 +607,6 @@ static KSTATUS KiAllocateApStack(KCPU* cpu) {
return STATUS_SUCCESS;
}
// Implements the ap Stage Name operation.
static const char* KiApStageName(uint32_t stage) {
switch (stage) {
@@ -806,9 +800,9 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
_ReadWriteBarrier();
_InterlockedExchange(&cpu->SchedulerCommand, 1);
// Parked APs sleep with HLT and have no LAPIC timer armed yet. The
// Parked APs sleep with HLT and have no LAPIC timer armed yet. The
// command store alone is not sufficient; send a fixed-vector IPI only
// to wake the parked loop. ReschedulePending is intentionally left
// to wake the parked loop. ReschedulePending is intentionally left
// clear, so the interrupt exit path will not try to schedule before
// the AP has installed its idle thread.
HalApicSendRescheduleIpi(cpu->LapicId);
code/kernel/ke_switch.asm
+1 -2
View File
@@ -20,7 +20,7 @@ EXTERN KeSetCurrentTssRsp0 : PROC
; -------------------------------------------------------------------------
; void KeSwitchContext(THREAD* from, THREAD* to)
; RCX = from, RDX = to (Microsoft x64)
; RCX = from, RDX = to (Microsoft x64)
; -------------------------------------------------------------------------
KeSwitchContext PROC
mov rax, [rcx + THREAD_FpuContext]
@@ -75,7 +75,6 @@ KeSwitchContext PROC
ret
KeSwitchContext ENDP
; -------------------------------------------------------------------------
; void KeStartInitialThread(THREAD* to)
; RCX = to
code/kernel/ke_syscall.asm
+1 -2
View File
@@ -152,7 +152,7 @@ KeSyscallEntry ENDP
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
; KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; ---------------------------------------------------------------------------
KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
@@ -420,7 +420,6 @@ pf_unhandled_halt:
jmp pf_unhandled_halt
KeFaultPF ENDP
; ---------------------------------------------------------
; Generic CPU exception stubs.
; Each vector gets its own IDT entry and vector number so faults like #UD
code/kernel/ke_syscall.c
+853 -389
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_thread.c
+14 -17
View File
@@ -75,7 +75,7 @@ static void KiThreadDestroy(OBJECT* obj) {
if (remaining == 0) {
// The last thread leaving a process is process exit even if user
// code called ExitThread instead of ExitProcess. KeExitProcess /
// code called ExitThread instead of ExitProcess. KeExitProcess /
// KeTerminateProcess set Terminating earlier; this covers the last
// thread voluntarily exiting without an explicit process-exit call.
_InterlockedExchange(&owner->Terminating, 1);
@@ -151,10 +151,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
return STATUS_NO_MEMORY;
}
thread->UserStackBase = ustack;
// Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
// Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
// nevertheless lets callees immediately use the caller-provided 32-byte
// home/shadow area above RSP, and MSVC commonly emits a first instruction
// like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
// like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
// entry stack as if a CALL with a normal home area had happened:
//
// RSP % 16 == 8 at function entry
@@ -190,12 +190,12 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
uintptr_t* ksp = (uintptr_t *)thread->KernelStackTop;
*(--ksp) = userStackTop;
*(--ksp) = 0x23ULL;
*(--ksp) = userStackTop;
*(--ksp) = 0x202ULL;
*(--ksp) = 0x1BULL;
*(--ksp) = entry;
*(--ksp) = arg;
*(--ksp) = 0x23ULL;
*(--ksp) = userStackTop;
*(--ksp) = 0x202ULL;
*(--ksp) = 0x1BULL;
*(--ksp) = entry;
*(--ksp) = arg;
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
@@ -233,7 +233,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
KeInitThreadFpuContext(thread);
// Hand the creator a short-lived reference before making the thread
// runnable. On SMP a new user thread can run and exit before
// runnable. On SMP a new user thread can run and exit before
// KeCreateThread() returns; without this reference, callers that merely
// read ThreadId for logging or syscall output can race the scheduler's
// dead-thread dereference and touch freed pool.
@@ -332,8 +332,8 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
// Build the initial stack: arg, fn, then KCONTEXT.
uintptr_t* ksp = (uintptr_t*)thread->KernelStackTop;
*(--ksp) = arg; // popped second (-> r12)
*(--ksp) = fn; // popped first (-> rbx)
*(--ksp) = arg; // popped second (-> r12)
*(--ksp) = fn; // popped first (-> rbx)
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT*)ksp;
@@ -351,9 +351,9 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
KeReleaseSpinlock(&owner->HandleLock, flags);
// Hand the creator a short-lived reference before making the kernel
// thread runnable. Kernel workers can run, signal their completion event,
// thread runnable. Kernel workers can run, signal their completion event,
// exit, and be reaped on another CPU before KeCreateKernelThread()'s
// caller drops its returned pointer. The live scheduler reference remains
// caller drops its returned pointer. The live scheduler reference remains
// separate and is released by KeSchedulerPostSwitch() after the dead
// thread has switched away.
if (out) {
@@ -395,7 +395,6 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable)
thread->WaitRundownContext = (void*)0;
}
// Updates thread Wait List.
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
@@ -483,14 +482,12 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
return thread->WaitStatus;
}
// Implements the block Thread operation.
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
KePrepareBlockThread(thread, waitObject, alertable);
return KeCommitBlockThread(thread, alertable);
}
// Implements the unblock Thread With Status operation.
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
code/kernel/ke_timer.c
+219
View File
@@ -0,0 +1,219 @@
// Kernel waitable timer objects. This file owns timer deadlines and timer wait signaling.
#include <stdint.h>
#include <string.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_sched.h>
#include <kernel/ob_object.h>
#define TIMER_SIGNAL_BATCH 32u
static LIST_ENTRY gTimerList;
static SPINLOCK gTimerLock;
static uint8_t gTimerInitialized;
// Initializes the global timer list on first use.
static void KiTimerEnsureInitialized(void) {
if (gTimerInitialized)
return;
gTimerLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&gTimerList);
gTimerInitialized = 1;
}
// Reads the scheduler tick value used by timer deadlines.
static uint64_t KiTimerReadSchedulerTick(void) {
KE_SCHED_STATS stats;
KeGetSchedulerStats(&stats);
return stats.Ticks;
}
// Converts a millisecond interval to at least one scheduler tick.
static uint64_t KiTimerMillisecondsToTicks(uint64_t milliseconds) {
if (milliseconds == 0)
return 0;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
return ticks ? ticks : 1;
}
// Adds a tick interval to the current scheduler tick with saturation.
static uint64_t KiTimerDeadlineFromNow(uint64_t intervalTicks) {
uint64_t now = KiTimerReadSchedulerTick();
uint64_t due = now + intervalTicks;
return due < now ? UINT64_MAX : due;
}
// Removes an active timer from the global deadline list while the timer lock is held.
static uint8_t KiTimerRemoveLocked(KTIMER* timer) {
if (!timer || !timer->Linked)
return 0;
ListRemove(&timer->Link);
timer->Linked = 0;
timer->Active = 0;
timer->DueTick = 0;
timer->PeriodTicks = 0;
return 1;
}
// Implements the timer Destroy operation.
static void KiTimerDestroy(OBJECT* obj) {
KTIMER* timer = (KTIMER*)obj;
if (!timer)
return;
KiTimerEnsureInitialized();
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
(void)KiTimerRemoveLocked(timer);
KeReleaseSpinlock(&gTimerLock, flags);
if (timer->Event) {
ObDereferenceObject(&timer->Event->Header);
timer->Event = (EVENT*)0;
}
}
// Creates an unset waitable timer object.
KSTATUS KeTimerCreate(KTIMER** out) {
if (!out)
return STATUS_INVALID_PARAMETER;
*out = (KTIMER*)0;
KiTimerEnsureInitialized();
KTIMER* timer = (KTIMER*)ObAllocateObject(ObjectTypeTimer, sizeof(KTIMER));
if (!timer)
return STATUS_NO_MEMORY;
timer->Header.Destroy = KiTimerDestroy;
LIST_INIT(&timer->Link);
KSTATUS s = KeEventCreate(EventAutoReset, 0, &timer->Event);
if (!KSUCCESS(s)) {
ObDereferenceObject(&timer->Header);
return s;
}
*out = timer;
return STATUS_SUCCESS;
}
// Arms or rearms a timer with a relative due time and optional period.
KSTATUS KeTimerSet(KTIMER* timer, uint64_t dueTimeMilliseconds, uint64_t periodMilliseconds) {
if (!timer || !timer->Event)
return STATUS_INVALID_PARAMETER;
KiTimerEnsureInitialized();
uint64_t dueTicks = KiTimerMillisecondsToTicks(dueTimeMilliseconds);
uint64_t periodTicks = KiTimerMillisecondsToTicks(periodMilliseconds);
uint8_t dropListReference = 0;
uint8_t addListReference = 0;
uint8_t signalNow = dueTicks == 0;
EVENT* event = timer->Event;
ObReferenceObject(&event->Header);
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
if (timer->Linked) {
if (signalNow && periodTicks == 0) {
dropListReference = KiTimerRemoveLocked(timer);
} else {
timer->DueTick = KiTimerDeadlineFromNow(signalNow ? periodTicks : dueTicks);
timer->PeriodTicks = periodTicks;
timer->Active = 1;
}
} else if (!signalNow || periodTicks != 0) {
timer->DueTick = KiTimerDeadlineFromNow(signalNow ? periodTicks : dueTicks);
timer->PeriodTicks = periodTicks;
timer->Active = 1;
timer->Linked = 1;
ListInsertTail(&gTimerList, &timer->Link);
addListReference = 1;
}
KeReleaseSpinlock(&gTimerLock, flags);
if (addListReference)
ObReferenceObject(&timer->Header);
if (dropListReference)
ObDereferenceObject(&timer->Header);
KeClearEvent(event);
if (signalNow)
KeSignalEvent(event);
ObDereferenceObject(&event->Header);
return STATUS_SUCCESS;
}
// Cancels a timer and clears any retained signal.
KSTATUS KeTimerCancel(KTIMER* timer) {
if (!timer || !timer->Event)
return STATUS_INVALID_PARAMETER;
KiTimerEnsureInitialized();
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
uint8_t hadListReference = KiTimerRemoveLocked(timer);
KeReleaseSpinlock(&gTimerLock, flags);
if (hadListReference)
ObDereferenceObject(&timer->Header);
KeClearEvent(timer->Event);
return STATUS_SUCCESS;
}
// Returns the event that backs timer waits.
EVENT* KeTimerWaitEvent(KTIMER* timer) {
if (!timer)
return (EVENT*)0;
return timer->Event;
}
// Services expired timers from the scheduler tick path.
void KeTimerTick(void) {
KiTimerEnsureInitialized();
EVENT* events[TIMER_SIGNAL_BATCH];
KTIMER* listReferencesToDrop[TIMER_SIGNAL_BATCH];
uint32_t count = 0;
uint32_t dropCount = 0;
uint64_t now = KiTimerReadSchedulerTick();
uintptr_t flags = KeAcquireSpinlock(&gTimerLock);
LIST_ENTRY* node = gTimerList.Fwd;
while (node != &gTimerList && count < TIMER_SIGNAL_BATCH) {
LIST_ENTRY* next = node->Fwd;
KTIMER* timer = CONTAINING_RECORD(node, KTIMER, Link);
if (timer->Active && timer->DueTick <= now) {
if (timer->Event) {
ObReferenceObject(&timer->Event->Header);
events[count++] = timer->Event;
}
if (timer->PeriodTicks != 0) {
uint64_t due = now + timer->PeriodTicks;
timer->DueTick = due < now ? UINT64_MAX : due;
} else {
if (KiTimerRemoveLocked(timer))
listReferencesToDrop[dropCount++] = timer;
}
}
node = next;
}
KeReleaseSpinlock(&gTimerLock, flags);
for (uint32_t i = 0; i < dropCount; i++)
ObDereferenceObject(&listReferencesToDrop[i]->Header);
for (uint32_t i = 0; i < count; i++) {
KeSignalEvent(events[i]);
ObDereferenceObject(&events[i]->Header);
}
}
code/kernel/kmain.c
+10 -20
View File
@@ -3,8 +3,6 @@
#include <stdint.h>
#include <string.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <ntlite/boot_info.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
@@ -175,7 +173,6 @@ static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
}
}
// Phase Helper: Initialize dynamically loaded .sys drivers.
static void KiDriversIniCb(const char* section, const char* key,
const char* val, void* ud) {
@@ -199,7 +196,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
for (uint32_t i = 0; i < bootInfo->ModuleCount; i++) {
BOOT_MODULE* mod = &bootInfo->Modules[i];
// Skip modules that are already mapped (Kernel/HAL)
// Skip modules that are already mapped (Kernel/HAL)
// or have no physical data (invalid).
if (mod->VirtBase != 0 || mod->PhysBase == 0) continue;
@@ -323,7 +320,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
continue;
}
// Runtime modules may also export DDIs to drivers loaded later. Boot
// Runtime modules may also export DDIs to drivers loaded later. Boot
// drivers were already registered above; keep the module model uniform.
LdrRegisterModule(runtimeKeys[i], mappedBase);
@@ -490,9 +487,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Physical & Virtual Memory initialization.
if (!MmInitializePhysical(&bootInfo->MemoryMap, bootInfo->PhysicalMapBase)) {
_disable(); while (true) __halt(); // there's nothing we can do
}
if (!MmInitializePhysical(&bootInfo->MemoryMap, bootInfo->PhysicalMapBase))
HalStopProcessor();
PHYSICAL_ADDRESS pml4 = (PHYSICAL_ADDRESS)__readcr3();
// The early virtual allocator starts small because it only needs bootstrap page tables before the PMM is online.
@@ -523,17 +519,15 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
};
KSTATUS s = HalInitialize(&halParams);
if (!KSUCCESS(s)) {
_disable(); while (true) __halt(); // there's nothing we can do.
}
if (!KSUCCESS(s))
HalStopProcessor();
const HAL_API* hal = HalGetApi();
gKmainHal = hal;
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug);
KeInitDebug(hal, bootInfo->Debug);
KeTimeInitialize(bootInfo, hal);
KeDisplayInitialize(hal);
KeSetDebugConsoleWriter(KeDisplayDebugPrintChar);
// CPU startup intentionally lives after HAL initialization but before IRQs
// and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI;
@@ -595,11 +589,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
DbgPrint("KernelMain: Handing over to scheduler...\n");
// Ignite the system!
_enable();
// The Idle Loop: The CPU rests here whenever all other threads are waiting on events.
while (true) {
__halt();
}
// The bootstrap CPU becomes the scheduler idle context once interrupts are enabled.
while (true)
HalWaitForInterrupt();
}
code/kernel/ldr_image.c
+21 -5
View File
@@ -233,11 +233,31 @@ void LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
// Implements the ldr Register Module operation.
void LdrRegisterModule(const char* name, uintptr_t base) {
if (gModuleCount < MAX_MODULES) {
KiRegisterModuleEntry(&gModules[gModuleCount], name, base, 0);
size_t size = base ? LdrGetImageSize(base) : 0;
KiRegisterModuleEntry(&gModules[gModuleCount], name, base, size);
gModuleCount++;
}
}
// Finds the loaded kernel module that owns an address.
const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut, size_t* sizeOut) {
if (baseOut) *baseOut = 0;
if (sizeOut) *sizeOut = 0;
for (uint32_t i = 0; i < gModuleCount; i++) {
uintptr_t base = gModules[i].Base;
size_t size = gModules[i].Size;
if (!base || !size)
continue;
if (address >= base && address < base + size) {
if (baseOut) *baseOut = base;
if (sizeOut) *sizeOut = size;
return gModules[i].Name;
}
}
return "unknown";
}
// Returns global Module Base.
static uintptr_t KiFindGlobalModuleBase(const char* name) {
return KiFindModuleBaseInTable(gModules, gModuleCount, KiCanonicalModuleName(name));
@@ -327,7 +347,6 @@ size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
return LdrGetImageSize(base);
}
// Releases loaded Image For Process.
static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return;
@@ -450,7 +469,6 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
return s;
}
// Entry point lookup.
uintptr_t LdrGetEntryPoint(uintptr_t base) {
uint32_t pe_offset = *(uint32_t*)(base + 0x3C);
@@ -720,7 +738,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
LdrDbg("LDR: module mapped pid=%u file=%s base=%016llx size=%llx\n",
process->ProcessId, filename, (unsigned long long)imageBase, (unsigned long long)LdrGetMappedImageSize(process, imageBase));
s = LdrResolveImportsForProcess(process, imageBase);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
@@ -733,7 +750,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
return s;
}
s = LdrRegisterProcessModule(process, filename, imageBase);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
code/kernel/ldr_section.c
-1
View File
@@ -268,7 +268,6 @@ static IMAGE_SECTION_OBJECT* KiFindImageSectionLocked(const IO_FILE_IDENTITY* id
return (IMAGE_SECTION_OBJECT*)0;
}
// Verifies that a cached image section is still backed by the same file generation.
static KSTATUS KiValidateImageSectionBackingFile(IMAGE_SECTION_OBJECT* section) {
if (!section || !section->BackingFile)
code/kernel/mm_file_cache.c
+5 -7
View File
@@ -253,7 +253,6 @@ static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOff
return STATUS_NOT_FOUND;
}
static void KiFileCacheAddReadAheadStat(uint64_t* field, uint64_t value) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
*field += value;
@@ -719,7 +718,6 @@ static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry,
return STATUS_SUCCESS;
}
static uint32_t KiFileCacheChooseForegroundBatchPages(void) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
uint64_t resident = gFileCacheResidentPages;
@@ -936,9 +934,11 @@ static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file,
}
memset(page, 0, PAGE_SIZE);
uint32_t validBytes = PAGE_SIZE;
if (oldFileSize > aligned && oldFileSize - aligned < PAGE_SIZE)
validBytes = (uint32_t)(oldFileSize - aligned);
uint32_t validBytes = 0;
if (oldFileSize > aligned) {
uint64_t oldBytes = oldFileSize - aligned;
validBytes = oldBytes < PAGE_SIZE ? (uint32_t)oldBytes : PAGE_SIZE;
}
s = KiFileCacheCompleteLoadingPage(entry, frame, validBytes, outPage, (uint64_t*)0);
if (!KSUCCESS(s))
@@ -967,7 +967,6 @@ static void KiFileCacheMarkPageDirty(FILE_OBJECT* file, uint64_t aligned, uint32
}
}
KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
uint64_t fileSizeBefore, int needExistingBytes,
MM_FILE_CACHE_PAGE* outPage) {
@@ -1180,7 +1179,6 @@ KSTATUS MmFileCacheDropOnePageForPressure(void) {
return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0);
}
KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages) {
if (!file || pages == 0)
return STATUS_INVALID_PARAMETER;
code/kernel/mm_phys.c
-6
View File
@@ -141,7 +141,6 @@ static void KiRegisterManagedRange(PHYSICAL_ADDRESS base,
}
}
// Returns whether the future page-replacement code may consider this frame a candidate.
static int KiPfnEvictableLocked(const MM_PFN* entry) {
if (!entry || entry->PinCount)
@@ -156,8 +155,6 @@ static int KiPfnEvictableLocked(const MM_PFN* entry) {
entry->Usage == MmPfnUseCow;
}
// Returns whether a standby frame can be counted as immediately reclaimable.
static int KiPfnReclaimableLocked(const MM_PFN* entry) {
if (!entry || entry->PinCount || entry->MappedCount)
@@ -395,7 +392,6 @@ __declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame) {
return &gPfnDatabase[pfn];
}
// Returns the direct-map virtual address for a managed physical frame.
__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame) {
if (MmPfnIndex(frame) == MM_PFN_INVALID_INDEX)
@@ -669,8 +665,6 @@ __declspec(dllexport) KSTATUS MmPfnRemoveMapping(PHYSICAL_ADDRESS frame) {
return STATUS_SUCCESS;
}
// Moves a clean, unshared frame to standby so pressure can reclaim it later.
__declspec(dllexport) KSTATUS MmPfnMoveToStandby(PHYSICAL_ADDRESS frame) {
size_t pfn = MmPfnIndex(frame);
code/kernel/mm_pte.c
+2 -7
View File
@@ -77,7 +77,6 @@ static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
return childPhys;
}
// Initializes the page-table manager around the bootloader-created PML4.
void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys) {
gPml4Phys = bootPml4Phys;
@@ -246,7 +245,6 @@ __declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys
return STATUS_SUCCESS;
}
// Removes one 4 KiB leaf PTE from the shared page tables.
__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
KiPteStatInc(&gPteStats.UnmapCalls);
@@ -285,7 +283,6 @@ __declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
return STATUS_SUCCESS;
}
// Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) {
KiPteStatInc(&gPteStats.ProtectCalls);
@@ -346,10 +343,9 @@ __declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uin
return status;
}
// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf. This is
// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf. This is
// used by the pressure trimmer as a second-chance test for clean file-backed
// pages. If the page was accessed recently, reclaim skips it once and flushes
// pages. If the page was accessed recently, reclaim skips it once and flushes
// the TLB entry so future accesses can set PAGE_ACCESSED again.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt) {
if (!gPml4Phys)
@@ -572,4 +568,3 @@ void MmPteQueryStats(MM_PTE_STATS* out) {
*out = gPteStats;
}
code/kernel/mm_swap.c
+1 -1
View File
@@ -93,7 +93,7 @@ KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
// Swap slots are never read before MmSwapPageOut overwrites the whole
// page, so the filesystem may allocate backing clusters without zeroing
// them. Normal files still get zero-on-extend semantics.
// them. Normal files still get zero-on-extend semantics.
file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;
s = IoTruncateFile(file, sizeBytes);
code/kernel/mm_virt.c
+25 -20
View File
@@ -1,6 +1,17 @@
// Kernel virtual-memory manager. This file owns kernel pool allocation, vad bookkeeping, and user-copy validation.
// Page-table construction lives in mm_pte.c so the higher VMM can grow toward VADs, faults, COW, and swap without mixing those policies with raw PTE edits.
// Virtual address layout:
// 0000000000001000-00007FFFFFFFF000 is the lower-half user range.
// 0000100000000000-00007FFFFFFFF000 is the current VAD allocation arena.
// FFFF800000000000-FFFF87FFFFFFFFFF is reserved for kernel images and fixed kernel mappings.
// FFFF880000000000-FFFF8FFFFFFFFFFF is the physical direct map.
// FFFF900000000000-FFFF97FFFFFFFFFF is the PCI MMIO aperture owned by pci.sys.
// FFFFC00000000000-FFFFDFFFFFFFFFFF is the kernel pool.
//
// The current VM model uses one shared PML4, so user pointers are globally
// meaningful while protection and lifetime are still tracked per process VAD.
// Kernel pool:
// A boundary-tag allocator starts at POOL_BASE and commits pages on demand.
@@ -780,7 +791,6 @@ static MM_VAD_STATS gVadStats;
#define MM_VAD_ARENA_BASE 0x0000100000000000ULL
// Initializes the VAD list after the shared page table and kernel pool are ready.
static void KiVadInitialize(void) {
LIST_INIT(&gVadList);
@@ -820,7 +830,6 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
(void)MmPfnPin(gVadZeroPageFrame);
}
// Allocates zeroed kernel-pool memory from the non-user heap.
__declspec(dllexport) void* MmAllocatePool(size_t size) {
return KiHeapAllocEx(&gPoolHeap, size, POOL_TAG_DEFAULT, (uintptr_t)_ReturnAddress());
@@ -1100,11 +1109,11 @@ static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) {
KeYield();
}
// Atomically claims the transient paging bit. This is deliberately not a spinlock:
// Atomically claims the transient paging bit. This is deliberately not a spinlock:
// a page-in/page-out may allocate frames or do filesystem I/O, so waiters must yield
// instead of holding IRQL-raised locks. Without this CAS, two CPUs faulting the same
// instead of holding IRQL-raised locks. Without this CAS, two CPUs faulting the same
// swapped/demand-zero page can both allocate/read/map it; one map fails and rolls
// metadata back to SWAPPED while the other has already published a valid PTE. That
// metadata back to SWAPPED while the other has already published a valid PTE. That
// leaves stale or freed swap slots behind and corrupts user data under pressure.
static int KiVadTryBeginPaging(MM_VAD_PAGE* page) {
if (!page) return 0;
@@ -1172,7 +1181,6 @@ static KSTATUS KiVadFillFileBackedPage(MM_VAD* vad, size_t pageIndex, PHYSICAL_A
return STATUS_SUCCESS;
}
// Materializes one image-backed prototype page or zero-fill page for an image VAD.
static KSTATUS KiVadFillImageBackedPage(MM_VAD* vad, size_t pageIndex, int wantWrite) {
if (!vad || !vad->ImageSection || pageIndex >= vad->PageCount)
@@ -1277,7 +1285,7 @@ retry_after_paging:
if (!KiVadTryBeginPaging(page))
goto retry_after_paging;
// Re-check after claiming the transition bit. Another CPU may have resolved
// Re-check after claiming the transition bit. Another CPU may have resolved
// the page between our initial wait/access check and the CAS above.
s = KiVadCheckPageAccess(page, wantWrite, wantExecute);
if (!KSUCCESS(s)) {
@@ -1906,7 +1914,6 @@ static KSTATUS KiCopyToUserPaged(void* userDst, const void* kernelSrc, size_t si
return KiCopyUserPaged(userDst, kernelSrc, size, 1);
}
// Reserves a VAD in the global user VA arena without committing physical pages.
KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t vadFlags,
@@ -2071,9 +2078,9 @@ KSTATUS MmVadCreateCommitted(PROCESS* process, size_t size, size_t alignment,
return STATUS_SUCCESS;
}
// Creates a demand-paged file-backed user mapping. The VAD owns a reference to
// Creates a demand-paged file-backed user mapping. The VAD owns a reference to
// the FILE_OBJECT so the caller may close the original file handle immediately
// after a view is established. Pages are read from the file on first fault and
// after a view is established. Pages are read from the file on first fault and
// then behave like ordinary private user pages: if memory pressure trims them
// after they are resident, the swap path preserves later private modifications.
KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file,
@@ -2133,7 +2140,6 @@ KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file,
return STATUS_SUCCESS;
}
// Creates an image-backed VAD whose pages fault from a shared image section object.
KSTATUS MmVadCreateImageSectionMapping(PROCESS* process, IMAGE_SECTION_OBJECT* section,
uint64_t pageFlags, uint32_t vadFlags,
@@ -2425,9 +2431,9 @@ KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess,
}
// Returns whether a resident file-backed page can be discarded instead of
// swapped. This is safe only while the page is still clean by construction:
// swapped. This is safe only while the page is still clean by construction:
// read-only mappings cannot have been modified, and the backing file can
// repopulate the page on the next fault. Writable/private file mappings still
// repopulate the page on the next fault. Writable/private file mappings still
// fall back to the swap path because this VMM does not yet track dirty bits.
static int KiVadPageCanDiscardCleanFile(const MM_VAD* vad, const MM_VAD_PAGE* page) {
if (!vad || !page || vad->Type != MmVadMappedFile)
@@ -2463,7 +2469,7 @@ static int KiVadPageCanDiscardCleanFileAfterPagingClaim(const MM_VAD* vad, const
return (page->Flags & MM_VAD_PAGE_PAGING) != 0;
}
// Returns whether a resident VAD page is safe to move to swap. Clean read-only
// Returns whether a resident VAD page is safe to move to swap. Clean read-only
// file-backed pages are intentionally excluded; the pressure trimmer discards
// them and reloads them from the file instead of wasting swap slots.
static int KiVadPageCanSwap(const MM_VAD_PAGE* page) {
@@ -2498,7 +2504,7 @@ static int KiVadPageCanSwapAfterPagingClaim(const MM_VAD_PAGE* page) {
return 1;
}
// Discards one resident clean file-backed page. No swap slot is allocated; the
// Discards one resident clean file-backed page. No swap slot is allocated; the
// VAD keeps its FILE_BACKED state and refaults the page from the file later.
static KSTATUS KiVadDiscardCleanFilePage(MM_VAD* vad, size_t pageIndex, void** baseOut) {
if (baseOut) *baseOut = (void*)0;
@@ -2516,9 +2522,9 @@ static KSTATUS KiVadDiscardCleanFilePage(MM_VAD* vad, size_t pageIndex, void** b
uintptr_t va = vad->Base + pageIndex * PAGE_SIZE;
// Second chance for hot mapped-file pages. Dropping the active video/WAD
// Second chance for hot mapped-file pages. Dropping the active video/WAD
// working set is much worse than swapping cold anonymous pages: it causes
// repeated file reads, page-table churn, and global TLB shootdowns. Clear
// repeated file reads, page-table churn, and global TLB shootdowns. Clear
// the hardware accessed bit and skip the page once; if it stays cold until
// a later pressure scan, it may be discarded then.
if (MmPteTestAndClearAccessed(va))
@@ -2580,7 +2586,7 @@ static KSTATUS KiVadTrimPageToSwap(MM_VAD* vad, size_t pageIndex, void** baseOut
return STATUS_NOT_FOUND;
// Claim the transient paging bit atomically before pinning/removing the
// user PTE. This serializes page-out against another page-out and against
// user PTE. This serializes page-out against another page-out and against
// a concurrent fault on the same VAD page without holding a spinlock across
// pagefile I/O.
if (!KiVadTryBeginPaging(page))
@@ -2655,7 +2661,7 @@ KSTATUS MmVadTrimOnePageForTest(PROCESS* process, void** baseOut) {
// Reclaims one resident user page from any process during allocation pressure.
// Prefer clean file-backed pages because they can be dropped and re-read without
// consuming swap. Fall back to anonymous/private swap only when no such page is
// consuming swap. Fall back to anonymous/private swap only when no such page is
// available.
KSTATUS MmVadTrimOnePageForPressure(void) {
if (!gVadListReady) return STATUS_NOT_SUPPORTED;
@@ -2798,7 +2804,6 @@ KSTATUS MmVadResolveUserPageFault(PROCESS* process, uintptr_t faultAddress, uint
return STATUS_SUCCESS;
}
// Resolves a page fault by looking up the owning VAD in the global user arena.
// The single address space intentionally allows the kernel and other processes
// to touch any lower-half mapping, so fault ownership is derived from the VAD
code/kernel/net_socket.c
+8 -8
View File
@@ -81,7 +81,7 @@ KSTATUS NetSocketCreate(uint32_t domain, uint32_t type, uint32_t protocol, SOCKE
}
// Implements the net Socket Bind operation.
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
KSTATUS NetSocketBind(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
KSTATUS st = NetSocketGetProviderSocket(socket, &provider, &providerSocket);
@@ -97,7 +97,7 @@ KSTATUS NetSocketListen(SOCKET_OBJECT* socket, uint32_t backlog) {
}
// Implements the net Socket Accept operation.
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_IPV4_K* remoteOut) {
KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_SOCKET_ADDRESS_K* remoteOut) {
if (!acceptedOut)
return STATUS_INVALID_PARAMETER;
*acceptedOut = NULL;
@@ -127,7 +127,7 @@ KSTATUS NetSocketAccept(SOCKET_OBJECT* socket, SOCKET_OBJECT** acceptedOut, SYS_
}
// Implements the net Socket Connect operation.
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_IPV4_K* address, size_t addressSize) {
KSTATUS NetSocketConnect(SOCKET_OBJECT* socket, const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
const NET_STACK_PROVIDER* provider = NULL;
void* providerSocket = NULL;
KSTATUS st = NetSocketGetProviderSocket(socket, &provider, &providerSocket);
@@ -165,7 +165,7 @@ KSTATUS NetSocketReceive(SOCKET_OBJECT* socket, void* buffer, size_t length, uin
}
KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length, uint32_t flags,
const SYS_SOCKET_ADDRESS_IPV4_K* remote, size_t remoteSize, size_t* sentOut) {
const SYS_SOCKET_ADDRESS_K* remote, size_t remoteSize, size_t* sentOut) {
if (sentOut)
*sentOut = 0;
if (!buffer && length)
@@ -184,7 +184,7 @@ KSTATUS NetSocketSendTo(SOCKET_OBJECT* socket, const void* buffer, size_t length
}
KSTATUS NetSocketReceiveFrom(SOCKET_OBJECT* socket, void* buffer, size_t length, uint32_t flags,
SYS_SOCKET_ADDRESS_IPV4_K* remoteOut, size_t* receivedOut) {
SYS_SOCKET_ADDRESS_K* remoteOut, size_t* receivedOut) {
if (receivedOut)
*receivedOut = 0;
if (!buffer && length)
@@ -238,8 +238,8 @@ KSTATUS NetSocketPoll(SOCKET_OBJECT* socket, uint32_t events, uint32_t* returned
return KSUCCESS(st) ? provider->Poll(providerSocket, events, returnedEventsOut) : st;
}
// Implements the net Resolve Host Ipv4 operation.
KSTATUS NetResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressOut) {
// Implements the net Resolve Host operation.
KSTATUS NetResolveHost(const char* name, uint32_t family, SYS_SOCKET_ADDRESS_K* addressOut) {
if (!name || !addressOut)
return STATUS_INVALID_PARAMETER;
@@ -247,5 +247,5 @@ KSTATUS NetResolveHostIpv4(const char* name, SYS_SOCKET_ADDRESS_IPV4_K* addressO
if (!provider)
return STATUS_NOT_SUPPORTED;
return provider->ResolveHostIpv4(name, addressOut);
return provider->ResolveHost(name, family, addressOut);
}
code/kernel/net_stack.c
+1 -1
View File
@@ -19,7 +19,7 @@ static KSTATUS NetValidateStackProvider(const NET_STACK_PROVIDER* p) {
!p->Bind || !p->Listen || !p->Accept || !p->Connect ||
!p->Send || !p->Receive || !p->SendTo || !p->ReceiveFrom ||
!p->Shutdown || !p->SetOption || !p->GetOption || !p->Poll ||
!p->ResolveHostIpv4) {
!p->ResolveHost) {
return STATUS_INVALID_PARAMETER;
}
code/kernel/ob_object.c
+1
View File
@@ -26,6 +26,7 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
case ObjectTypeSurface: return MM_POOL_TAG('s', 'u', 'r', 'f');
case ObjectTypeInputQueue: return MM_POOL_TAG('i', 'n', 'p', 'q');
case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' ');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}