maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

isolate POSIX builds from the host UCRT, wire lwext4 and ntfs-3g as ext4.sys and ntfs.sys, split the FAT driver into directory/file/async/driver/mount units, add posixsrv and dwm plus input/console/desktop syscall clients and a pile of new smoke targets, rename serial.sys to cereal.sys, name HDA/USB/lwIP/AHCI workers, harden ATA probing/reset/removal around actual hardware deadlines, absent-channel detection, diagnostics and explicit worker lifetime, redo ACPI S5 shutdown around GAS/MMIO and HW-reduced firmware, scan SSDTs for _S5 and deal with Intel SMI sleep trapping, make serial and SMP timeouts use real monotonic deadlines, add thread names and fix terminal thread teardown, scheduler and APC prepare/commit races, make advisory file locks queryable and preserve split ranges and add mutable file append state, add shared-memory pipes with kernel-assisted wait/wake, introduce capability-scoped input and display managers, split surface map rights into read/write, add explicit unmap and replace present with commit/generation/damage tracking while keeping physical presentation under the display manager, decouple native exception classes from Win32 exception numbers and translate them in kernel32, add opaque process startup data, bump the resource protocol to v3 with namespace directories, symlinks and enumeration and service control to v2 with generic READY handle exports, add the new POSIX errno/status surface and rebuild fds around posixsrv-owned shared open descriptions, descriptor-local flags, exact inheritance snapshots, dup/fcntl/record locks and shared-ring pipes, add waitpid, POSIX credentials/groups/sessions/passwd/group lookups and move sockets onto stable open descriptions, port the shells and apps to the new console/input services and surface commit API, teach init to load services.ini and start/supervise RSSRV, DWM and generic services under restricted capabilities while publishing their endpoints through RSSRV, split RSSRV into namespace, volume, display, session, input, console and resource components and make /SystemRoot a real symlink to the independently published system volume, shift the syscall table again, expand lock/resource/surface/POSIX/service/thread smoke coverage, make docs scream ASK QUESTIONS and DISCUSS at LLMs, replace MIT No Attribution with WTFPL+.

Browse Source
This commit is contained in:
maksw committed 2026-09-16 10:33:20 +02:00
1 parent 3dacbd32c8
commit 73abd6f90d
441 files changed
+187073 -6549
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/hal/hal_main.c
+304 -46
View File
@@ -8,6 +8,7 @@
#include <kernel/hal/hal_timer.h>
#include <string.h>
#include <stdint.h>
#include <stdbool.h>
#include <stdarg.h>
#include <intrin.h>
#include <common/printf.h>
@@ -112,11 +113,26 @@ static HAL_PROCESSOR_TOPOLOGY gCpuTopology;
// ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5).
// Do not guess SLP_TYP on real hardware: these values are firmware-specific.
static uint16_t gPm1aCntBlk = 0;
static uint16_t gPm1bCntBlk = 0;
static uint8_t gSlpTypA = 0;
static uint8_t gSlpTypB = 0;
static int gS5Found = 0;
typedef struct ACPI_FIXED_REGISTER {
uint8_t AddressSpaceId;
uint8_t WidthBytes;
uint16_t Reserved;
uint64_t Address;
} ACPI_FIXED_REGISTER;
static ACPI_FIXED_REGISTER gPm1aCnt;
static ACPI_FIXED_REGISTER gPm1bCnt;
static ACPI_FIXED_REGISTER gSleepControl;
static uint32_t gLegacyPm1aPort = 0;
static uint32_t gLegacyPm1bPort = 0;
static uint32_t gFadtFlags = 0;
static uint32_t gSmiCommandPort = 0;
static uint8_t gAcpiEnableValue = 0;
static uint8_t gSlpTypA = 0;
static uint8_t gSlpTypB = 0;
static int gS5Found = 0;
static int gHwReducedAcpi = 0;
static int gIntelPchPmBase = 0;
int debugEnabled = 0;
int _fltused; // MSVC floating-point helper anchor.
@@ -156,13 +172,21 @@ void* HalPhysicalToVirtual(uint64_t phys) {
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
// systems may put the real DSDT and PM1 control registers in the extended
// X_* fields, so do not rely only on the legacy 32-bit fields.
#define FADT_DSDT_OFFSET 40u
#define FADT_PM1A_CNT_BLK_OFFSET 64u
#define FADT_PM1B_CNT_BLK_OFFSET 68u
#define FADT_X_DSDT_OFFSET 140u
#define FADT_X_PM1A_CNT_BLK_OFFSET 172u
#define FADT_X_PM1B_CNT_BLK_OFFSET 184u
#define FADT_DSDT_OFFSET 40u
#define FADT_SMI_CMD_OFFSET 48u
#define FADT_ACPI_ENABLE_OFFSET 52u
#define FADT_PM1A_CNT_BLK_OFFSET 64u
#define FADT_PM1B_CNT_BLK_OFFSET 68u
#define FADT_PM1_CNT_LEN_OFFSET 89u
#define FADT_CENTURY_OFFSET 108u
#define FADT_FLAGS_OFFSET 112u
#define FADT_X_DSDT_OFFSET 140u
#define FADT_X_PM1A_CNT_BLK_OFFSET 172u
#define FADT_X_PM1B_CNT_BLK_OFFSET 184u
#define FADT_SLEEP_CONTROL_OFFSET 244u
#define FADT_FLAG_HW_REDUCED_ACPI (1u << 20)
#define INTEL_PCH_SMI_EN_OFFSET 0x30u
#define INTEL_PCH_SMI_ON_SLP_EN (1u << 4)
#pragma pack(push, 1)
typedef struct {
@@ -190,19 +214,131 @@ static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off)
return value;
}
// Implements the acpi Read Gas Io Port operation.
static uint16_t AcpiReadGasIoPort(const void* table, uint32_t tableLen, uint32_t off) {
// Decodes one ACPI Generic Address Structure used for fixed power registers.
static int AcpiReadGasRegister(const void* table, uint32_t tableLen, uint32_t off,
ACPI_FIXED_REGISTER* out) {
if (!out || off + sizeof(ACPI_GAS) > tableLen) return 0;
ACPI_GAS gas;
if (off + sizeof(gas) > tableLen) return 0;
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
if (!gas.Address || gas.RegisterBitOffset != 0) return 0;
if (gas.AddressSpaceId != 0x00 && gas.AddressSpaceId != 0x01) return 0;
if (gas.AddressSpaceId == 0x01 && gas.Address > 0xFFFFu) return 0;
// 0x01 == System I/O. HalShutdown currently writes PM1_CNT with outw(),
// so ignore MMIO GAS entries until the HAL grows MMIO register access here.
if (gas.AddressSpaceId != 0x01 || gas.Address == 0 || gas.Address > 0xFFFF)
return 0;
uint8_t widthBytes = 0;
if (gas.AccessSize >= 1 && gas.AccessSize <= 3)
widthBytes = (uint8_t)(1u << (gas.AccessSize - 1u));
else if (gas.RegisterBitWidth)
widthBytes = (uint8_t)((gas.RegisterBitWidth + 7u) / 8u);
if (widthBytes < 2u) widthBytes = 2u;
if (widthBytes > 4u) widthBytes = 4u;
return (uint16_t)gas.Address;
out->AddressSpaceId = gas.AddressSpaceId;
out->WidthBytes = widthBytes;
out->Address = gas.Address;
return 1;
}
// Decodes one byte-wide ACPI GAS used by the HW-reduced Sleep Control register.
static int AcpiReadByteGasRegister(const void* table, uint32_t tableLen, uint32_t off,
ACPI_FIXED_REGISTER* out) {
if (!out || off + sizeof(ACPI_GAS) > tableLen) return 0;
ACPI_GAS gas;
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
if (!gas.Address || gas.RegisterBitOffset != 0 || gas.RegisterBitWidth != 8) return 0;
if (gas.AddressSpaceId != 0x00 && gas.AddressSpaceId != 0x01) return 0;
if (gas.AddressSpaceId == 0x01 && gas.Address > 0xFFFFu) return 0;
out->AddressSpaceId = gas.AddressSpaceId;
out->WidthBytes = 1u;
out->Address = gas.Address;
return 1;
}
// Reads one ACPI fixed register while preserving the address-space distinction.
static uint32_t AcpiReadFixedRegister(const ACPI_FIXED_REGISTER* reg) {
if (!reg || !reg->Address) return 0;
if (reg->AddressSpaceId == 0x01) {
uint16_t port = (uint16_t)reg->Address;
if (reg->WidthBytes <= 1u) return __inbyte(port);
if (reg->WidthBytes == 2u) return __inword(port);
return __indword(port);
}
volatile void* address = HalPhysicalToVirtual(reg->Address);
if (reg->WidthBytes <= 1u) return *(volatile uint8_t*)address;
if (reg->WidthBytes == 2u) return *(volatile uint16_t*)address;
return *(volatile uint32_t*)address;
}
// Writes one ACPI fixed register using the width advertised by firmware.
static void AcpiWriteFixedRegister(const ACPI_FIXED_REGISTER* reg, uint32_t value) {
if (!reg || !reg->Address) return;
if (reg->AddressSpaceId == 0x01) {
uint16_t port = (uint16_t)reg->Address;
if (reg->WidthBytes <= 1u) __outbyte(port, (uint8_t)value);
else if (reg->WidthBytes == 2u) __outword(port, (uint16_t)value);
else __outdword(port, value);
return;
}
volatile void* address = HalPhysicalToVirtual(reg->Address);
if (reg->WidthBytes <= 1u) *(volatile uint8_t*)address = (uint8_t)value;
else if (reg->WidthBytes == 2u) *(volatile uint16_t*)address = (uint16_t)value;
else *(volatile uint32_t*)address = value;
}
// Detects whether PM1 control is using the standard Intel PCH PMBASE layout.
static void AcpiDetectIntelPchPmBase(void) {
gIntelPchPmBase = 0;
if (!gApi.Pci.Read32 || gPm1aCnt.AddressSpaceId != 0x01 ||
gPm1aCnt.Address > 0xFFFFu || (gPm1aCnt.Address & 0x7Fu) != 0x04u) {
return;
}
uint32_t id = gApi.Pci.Read32(0, 0, 31, 0, 0x00u);
uint32_t classRevision = gApi.Pci.Read32(0, 0, 31, 0, 0x08u);
if ((id & 0xFFFFu) == 0x8086u && ((classRevision >> 16) & 0xFFFFu) == 0x0601u)
gIntelPchPmBase = 1;
}
// Disables Intel PCH sleep trapping so SLP_EN reaches the hardware sleep transition.
static void AcpiDisableIntelSleepSmiTrap(void) {
if (!gIntelPchPmBase) return;
uint64_t pmBase = gPm1aCnt.Address - 4u;
if (pmBase > 0xFFFFu - INTEL_PCH_SMI_EN_OFFSET - 3u) return;
uint16_t smiEnablePort = (uint16_t)(pmBase + INTEL_PCH_SMI_EN_OFFSET);
uint32_t smiEnable = __indword(smiEnablePort);
if (!(smiEnable & INTEL_PCH_SMI_ON_SLP_EN)) return;
__outdword(smiEnablePort, smiEnable & ~INTEL_PCH_SMI_ON_SLP_EN);
if (__indword(smiEnablePort) & INTEL_PCH_SMI_ON_SLP_EN)
HalDbgPrint("HAL: Intel PCH refused to clear SMI_ON_SLP_EN.\n");
}
// Requests ACPI mode when firmware left fixed power registers in legacy mode.
static KSTATUS AcpiEnableModeForShutdown(void) {
if (!gPm1aCnt.Address) return NTLSTATUS_NOT_SUPPORTED;
if (AcpiReadFixedRegister(&gPm1aCnt) & 0x0001u) return NTLSTATUS_SUCCESS;
if (!gSmiCommandPort || !gAcpiEnableValue || gSmiCommandPort > 0xFFFFu)
return NTLSTATUS_NOT_SUPPORTED;
__outbyte((uint16_t)gSmiCommandPort, gAcpiEnableValue);
uint64_t startTick = gApi.Timer.QueryTicks ? gApi.Timer.QueryTicks() : 0;
uint64_t deadline = startTick > UINT64_MAX - 3000000ull ? UINT64_MAX : startTick + 3000000ull;
uint32_t fallbackWaits = 0;
for (;;) {
if (AcpiReadFixedRegister(&gPm1aCnt) & 0x0001u) return NTLSTATUS_SUCCESS;
if (gApi.Timer.QueryTicks && gApi.Timer.QueryTicks() >= deadline) return NTLSTATUS_TIMEOUT;
if (++fallbackWaits >= 3000u) return NTLSTATUS_TIMEOUT;
if (gApi.Timer.Stall) gApi.Timer.Stall(1000u);
else _mm_pause();
}
}
static int AmlParsePkgLength(const uint8_t* p, const uint8_t* end,
@@ -302,7 +438,7 @@ static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out)
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and
// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants.
static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
static int ScanAmlForS5(const uint8_t* aml, uint32_t len, const char* source) {
const uint8_t* end = aml + len;
for (const uint8_t* cur = aml; cur < end; cur++) {
@@ -329,12 +465,12 @@ static void ScanDsdtForS5(const uint8_t* aml, uint32_t len) {
gSlpTypB = typB & 0x07;
gS5Found = 1;
HalDbgPrint("HAL: _S5_ SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n",
gSlpTypA, gSlpTypB);
return;
HalDbgPrint("HAL: _S5_ source=%s SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n",
source ? source : "AML", gSlpTypA, gSlpTypB);
return 1;
}
HalDbgPrint("HAL: _S5_ not found in DSDT; ACPI shutdown disabled\n");
return 0;
}
// Validates checksum.
@@ -352,13 +488,50 @@ static void ParseFadt(const ACPISDTHeader* header) {
const void* fadt = header;
uint32_t len = header->Length;
gPm1aCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1A_CNT_BLK_OFFSET);
gPm1bCntBlk = (uint16_t)AcpiReadU32(fadt, len, FADT_PM1B_CNT_BLK_OFFSET);
memset(&gPm1aCnt, 0, sizeof(gPm1aCnt));
memset(&gPm1bCnt, 0, sizeof(gPm1bCnt));
memset(&gSleepControl, 0, sizeof(gSleepControl));
gLegacyPm1aPort = 0;
gLegacyPm1bPort = 0;
gFadtFlags = AcpiReadU32(fadt, len, FADT_FLAGS_OFFSET);
gHwReducedAcpi = (gFadtFlags & FADT_FLAG_HW_REDUCED_ACPI) != 0;
if (!gPm1aCntBlk)
gPm1aCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1A_CNT_BLK_OFFSET);
if (!gPm1bCntBlk)
gPm1bCntBlk = AcpiReadGasIoPort(fadt, len, FADT_X_PM1B_CNT_BLK_OFFSET);
uint8_t pm1CntLength = FADT_PM1_CNT_LEN_OFFSET < len
? *((const uint8_t*)fadt + FADT_PM1_CNT_LEN_OFFSET)
: 2u;
if (pm1CntLength < 2u) pm1CntLength = 2u;
if (pm1CntLength > 4u) pm1CntLength = 4u;
uint32_t legacyPm1a = AcpiReadU32(fadt, len, FADT_PM1A_CNT_BLK_OFFSET);
uint32_t legacyPm1b = AcpiReadU32(fadt, len, FADT_PM1B_CNT_BLK_OFFSET);
gLegacyPm1aPort = legacyPm1a <= 0xFFFFu ? legacyPm1a : 0u;
gLegacyPm1bPort = legacyPm1b <= 0xFFFFu ? legacyPm1b : 0u;
if (legacyPm1a && legacyPm1a <= 0xFFFFu) {
gPm1aCnt.AddressSpaceId = 0x01;
gPm1aCnt.WidthBytes = pm1CntLength;
gPm1aCnt.Address = legacyPm1a;
}
if (legacyPm1b && legacyPm1b <= 0xFFFFu) {
gPm1bCnt.AddressSpaceId = 0x01;
gPm1bCnt.WidthBytes = pm1CntLength;
gPm1bCnt.Address = legacyPm1b;
}
// ACPI 2.0+ GAS fields supersede the legacy blocks when firmware supplies them.
ACPI_FIXED_REGISTER extended;
memset(&extended, 0, sizeof(extended));
if (AcpiReadGasRegister(fadt, len, FADT_X_PM1A_CNT_BLK_OFFSET, &extended))
gPm1aCnt = extended;
memset(&extended, 0, sizeof(extended));
if (AcpiReadGasRegister(fadt, len, FADT_X_PM1B_CNT_BLK_OFFSET, &extended))
gPm1bCnt = extended;
if (gHwReducedAcpi)
(void)AcpiReadByteGasRegister(fadt, len, FADT_SLEEP_CONTROL_OFFSET, &gSleepControl);
gSmiCommandPort = AcpiReadU32(fadt, len, FADT_SMI_CMD_OFFSET);
gAcpiEnableValue = FADT_ACPI_ENABLE_OFFSET < len
? *((const uint8_t*)fadt + FADT_ACPI_ENABLE_OFFSET)
: 0u;
uint64_t dsdtPhys = AcpiReadU64(fadt, len, FADT_X_DSDT_OFFSET);
if (!dsdtPhys)
@@ -369,8 +542,14 @@ static void ParseFadt(const ACPISDTHeader* header) {
centuryRegister = *((const uint8_t*)fadt + FADT_CENTURY_OFFSET);
HalRtcSetCenturyRegister(centuryRegister);
HalDbgPrint("HAL: FADT PM1a_CNT=0x%04x PM1b_CNT=0x%04x DSDT=0x%016llx RTC_CENTURY=0x%02x\n",
gPm1aCntBlk, gPm1bCntBlk, dsdtPhys, centuryRegister);
HalDbgPrint("HAL: FADT flags=0x%08x hw_reduced=%u PM1a_CNT=%u:0x%016llx legacy=0x%04x "
"PM1b_CNT=%u:0x%016llx legacy=0x%04x SLEEP_CTL=%u:0x%016llx "
"SMI=0x%08x DSDT=0x%016llx RTC_CENTURY=0x%02x\n",
gFadtFlags, gHwReducedAcpi ? 1u : 0u,
gPm1aCnt.AddressSpaceId, gPm1aCnt.Address, gLegacyPm1aPort,
gPm1bCnt.AddressSpaceId, gPm1bCnt.Address, gLegacyPm1bPort,
gSleepControl.AddressSpaceId, gSleepControl.Address,
gSmiCommandPort, dsdtPhys, centuryRegister);
if (dsdtPhys) {
const ACPISDTHeader* dsdt = (const ACPISDTHeader*)(dsdtPhys + gPhysBase);
@@ -378,7 +557,7 @@ static void ParseFadt(const ACPISDTHeader* header) {
// Validates checksum.
validate_checksum((void*)dsdt, dsdt->Length)) {
const uint8_t* aml = (const uint8_t*)(dsdt + 1);
ScanDsdtForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader));
(void)ScanAmlForS5(aml, dsdt->Length - (uint32_t)sizeof(ACPISDTHeader), "DSDT");
} else {
HalDbgPrint("HAL: DSDT invalid or checksum failed\n");
}
@@ -514,28 +693,106 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
// HPET base address block is located at offset 44
// (which is 8 bytes past the 36-byte ACPISDTHeader).
gHpetPhys = *(uint64_t*)((uint8_t*)header + 44) & ~0xFFFULL;
} else if (memcmp(header->Signature, "SSDT", 4) == 0 && !gS5Found &&
header->Length >= sizeof(ACPISDTHeader)) {
const uint8_t* aml = (const uint8_t*)(header + 1);
(void)ScanAmlForS5(aml, header->Length - (uint32_t)sizeof(ACPISDTHeader), "SSDT");
}
}
if (!gS5Found)
HalDbgPrint("HAL: _S5_ not found in DSDT or SSDT AML; ACPI S5 unavailable\n");
}
// HalShutdown - ACPI S5 soft power-off. Does not return.
static void HalShutdown(void) {
// Refuse further maskable interrupts before entering firmware power state.
(void)HalDisableInterrupts();
if (!gS5Found || !gPm1aCntBlk) {
// Refuse to guess. Guessing SLP_TYP is exactly how real hardware gets
// put into S3/S4 sleep instead of S5 soft-off.
if (!gS5Found) {
HalDbgPrint("HAL: shutdown unsupported: firmware _S5_ object was not resolved.\n");
HalStopProcessor();
}
// SLP_EN (bit 13) | SLP_TYPx (bits 12:10).
uint16_t valA = (uint16_t)(((uint16_t)gSlpTypA << 10) | (1 << 13));
uint16_t valB = (uint16_t)(((uint16_t)gSlpTypB << 10) | (1 << 13));
if (gHwReducedAcpi) {
if (!gSleepControl.Address) {
HalDbgPrint("HAL: shutdown unsupported: HW-reduced ACPI has no Sleep Control register.\n");
HalStopProcessor();
}
__outword(gPm1aCntBlk, valA);
if (gPm1bCntBlk) __outword(gPm1bCntBlk, valB);
uint32_t sleepValue = ((uint32_t)gSlpTypA << 2) | (1u << 5);
(void)HalDisableInterrupts();
AcpiWriteFixedRegister(&gSleepControl, sleepValue);
} else {
if (!gPm1aCnt.Address) {
HalDbgPrint("HAL: shutdown unsupported: PM1a control register is unavailable.\n");
HalStopProcessor();
}
// Some Intel PCH firmware leaves SMI_ON_SLP_EN armed after boot. When set,
// the chipset traps SLP_EN into SMM and explicitly suppresses the sleep transition.
AcpiDisableIntelSleepSmiTrap();
// Try the fixed-register S5 request before changing ACPI ownership. A late
// SMI_CMD transition can itself enter firmware SMM on machines that boot with SCI_EN clear.
uint32_t valA = AcpiReadFixedRegister(&gPm1aCnt);
valA &= ~(0x7u << 10);
valA |= ((uint32_t)gSlpTypA << 10) | (1u << 13);
uint32_t valB = 0u;
if (gPm1bCnt.Address) {
valB = AcpiReadFixedRegister(&gPm1bCnt);
valB &= ~(0x7u << 10);
valB |= ((uint32_t)gSlpTypB << 10) | (1u << 13);
}
uintptr_t interruptFlags = HalDisableInterrupts();
AcpiWriteFixedRegister(&gPm1aCnt, valA);
if (gPm1bCnt.Address)
AcpiWriteFixedRegister(&gPm1bCnt, valB);
if (gApi.Timer.Stall) gApi.Timer.Stall(50000u);
// If the direct request returned, fall back to the firmware ownership path.
HalRestoreInterrupts(interruptFlags);
(void)AcpiEnableModeForShutdown();
AcpiDisableIntelSleepSmiTrap();
valA = AcpiReadFixedRegister(&gPm1aCnt);
valA &= ~(0x7u << 10);
valA |= ((uint32_t)gSlpTypA << 10) | (1u << 13);
if (gPm1bCnt.Address) {
valB = AcpiReadFixedRegister(&gPm1bCnt);
valB &= ~(0x7u << 10);
valB |= ((uint32_t)gSlpTypB << 10) | (1u << 13);
}
(void)HalDisableInterrupts();
AcpiWriteFixedRegister(&gPm1aCnt, valA);
if (gPm1bCnt.Address)
AcpiWriteFixedRegister(&gPm1bCnt, valB);
if (gApi.Timer.Stall) gApi.Timer.Stall(50000u);
bool selectedPm1aIsLegacy = gPm1aCnt.AddressSpaceId == 0x01 &&
gPm1aCnt.Address == gLegacyPm1aPort;
if (gLegacyPm1aPort && !selectedPm1aIsLegacy) {
uint16_t legacyA = __inword((uint16_t)gLegacyPm1aPort);
legacyA = (uint16_t)((legacyA & ~(0x7u << 10)) |
((uint16_t)gSlpTypA << 10) | (1u << 13));
uint16_t legacyB = 0u;
if (gLegacyPm1bPort) {
legacyB = __inword((uint16_t)gLegacyPm1bPort);
legacyB = (uint16_t)((legacyB & ~(0x7u << 10)) |
((uint16_t)gSlpTypB << 10) | (1u << 13));
}
AcpiDisableIntelSleepSmiTrap();
__outword((uint16_t)gLegacyPm1aPort, legacyA);
if (gLegacyPm1bPort)
__outword((uint16_t)gLegacyPm1bPort, legacyB);
}
}
if (gApi.Timer.Stall) gApi.Timer.Stall(100000u);
HalDbgPrint("HAL: shutdown failed: firmware returned from S5 request.\n");
HalStopProcessor();
}
@@ -616,8 +873,9 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
HalRtcInitialize(&gApi.Rtc);
HalPciInitialize(&gApi.Pci);
gApi.Power.Shutdown = HalShutdown;
gApi.Power.Shutdown = HalShutdown;
AcpiDetectIntelPchPmBase();
gInitialised = 1;
// The 8042 can leave output bytes pending while the keyboard/mouse drivers
code/kernel/hal/hal_serial.c
+22 -3
View File
@@ -6,13 +6,16 @@
#include <kernel/hal/hal_api.h>
#include <kernel/hal/hal_output.h>
#include <kernel/hal/hal_processor.h>
#include <kernel/hal/hal_timer.h>
#define HAL_COM1_BASE 0x03F8u
#define HAL_COM1_DATA_PORT (HAL_COM1_BASE + 0u)
#define HAL_COM1_STATUS_PORT (HAL_COM1_BASE + 5u)
#define HAL_COM1_SCRATCH_PORT (HAL_COM1_BASE + 7u)
#define HAL_COM1_TX_READY 0x20u
#define HAL_SERIAL_TX_SPIN_LIMIT 1000000u
#define HAL_SERIAL_TX_TIMEOUT_US 2000u
#define HAL_SERIAL_TIMER_SPIN_CAP 65536u
#define HAL_SERIAL_EARLY_SPINS 4096u
static volatile long gSerialWriteLock;
static volatile long gSerialPresent;
@@ -30,9 +33,25 @@ static bool HalSerialProbeCom1(void) {
return first == 0x5Au && second == 0xA5u;
}
// Waits for COM1 transmit readiness without allowing broken or removed hardware to stall the machine forever.
// Waits for COM1 transmit readiness with a real microsecond deadline once the HAL timer is live.
static bool HalSerialWaitForTransmit(void) {
for (uint32_t spin = 0; spin < HAL_SERIAL_TX_SPIN_LIMIT; spin++) {
uint64_t start = HalTimerQueryTicks();
if (start != 0) {
uint64_t deadline = start > UINT64_MAX - HAL_SERIAL_TX_TIMEOUT_US
? UINT64_MAX
: start + HAL_SERIAL_TX_TIMEOUT_US;
for (uint32_t spin = 0; spin < HAL_SERIAL_TIMER_SPIN_CAP; ++spin) {
if ((__inbyte(HAL_COM1_STATUS_PORT) & HAL_COM1_TX_READY) != 0)
return true;
if (HalTimerQueryTicks() >= deadline)
return false;
_mm_pause();
}
return false;
}
// Serial is registered before timer calibration, so keep the early path small and finite.
for (uint32_t spin = 0; spin < HAL_SERIAL_EARLY_SPINS; ++spin) {
if ((__inbyte(HAL_COM1_STATUS_PORT) & HAL_COM1_TX_READY) != 0)
return true;
_mm_pause();
code/kernel/hal/hal_timer.c
+5 -2
View File
@@ -42,7 +42,7 @@ static void TimerInterrupt(void) {
}
// Returns ticks.
static uint64_t QueryTicks(void) {
uint64_t HalTimerQueryTicks(void) {
// Prefer HPET when present: it is already a monotonically increasing main
// counter and avoids races between the LAPIC periodic tick and the current
// countdown register.
@@ -53,6 +53,9 @@ static uint64_t QueryTicks(void) {
return seconds * 1000000ull + (rem * 1000000ull) / gHpetHz;
}
// Before LAPIC calibration there is no safe fallback clock to sample.
if (!gLapic || !gApicTicksPerQuantum) return 0;
// Fallback: combine the coarse periodic tick with the current LAPIC
// countdown. Read the coarse tick twice so an interrupt between gTicks and
// LAPIC_TIMER_CUR cannot make time jump backwards by almost one quantum.
@@ -226,7 +229,7 @@ KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* o
// Leave the BSP timer masked until the kernel registers its scheduler
// callback and IRQ route. APs arm their local timer after per-CPU setup.
out->QueryTicks = QueryTicks;
out->QueryTicks = HalTimerQueryTicks;
out->QueryTickFrequency = QueryTickFrequency;
out->Stall = HalTimerStall;
out->HandleInterrupt = TimerInterrupt;
code/kernel/include/kernel/hal/hal_timer.h
+1
View File
@@ -7,4 +7,5 @@
#include <kernel/hal/hal_api.h>
KSTATUS HalTimerInitialize(uintptr_t lapicPhys, uintptr_t hpetPhys, HAL_TIMER* out);
uint64_t HalTimerQueryTicks(void);
void HalTimerStall(uint32_t microseconds);
code/kernel/include/kernel/io_file.h
+10
View File
@@ -8,6 +8,7 @@
#include <kernel/io_async.h>
struct PROCESS;
typedef struct SYS_FILE_LOCK_INFO SYS_FILE_LOCK_INFO;
// Capability-relative opens rooted at an already-open directory.
KSTATUS IoOpenFileAtDirectory(DIR_OBJECT* directory, SYS_STRING_VIEW path, FILE_OBJECT** out);
@@ -50,6 +51,15 @@ __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJ
__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
IO_STATUS_BLOCK* iosbOut);
__declspec(dllexport) KSTATUS IoQueryFileLock(FILE_OBJECT* file,
struct PROCESS* owner,
uint32_t mode,
uint64_t offset,
uint64_t length,
SYS_FILE_LOCK_INFO* infoOut);
__declspec(dllexport) KSTATUS IoSetFileStatus(FILE_OBJECT* file,
uint32_t mask,
uint32_t flags);
__declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file,
struct PROCESS* owner,
uint32_t mode,
code/kernel/include/kernel/io_pipe.h
+34
View File
@@ -0,0 +1,34 @@
// Native anonymous pipe declarations.
#pragma once
#include <stdbool.h>
#include <stdint.h>
#include <kernel/ke_event.h>
#include <ntlite/status.h>
struct PROCESS;
typedef struct PROCESS PROCESS;
typedef struct IO_PIPE_ENDPOINT IO_PIPE_ENDPOINT;
// Creates one shared byte-stream pipe with separate endpoint objects.
KSTATUS IoCreatePipe(IO_PIPE_ENDPOINT** readEndOut, IO_PIPE_ENDPOINT** writeEndOut);
// Maps the pipe's shared data-plane pages into one process.
KSTATUS IoMapPipe(IO_PIPE_ENDPOINT* endpoint, PROCESS* process, void** sharedStateOut);
// Refreshes kernel wait objects after a user-mode pipe state transition.
KSTATUS IoNotifyPipe(IO_PIPE_ENDPOINT* endpoint);
// Queries readiness for poll-style events.
KSTATUS IoPollPipe(IO_PIPE_ENDPOINT* endpoint, uint32_t events, uint32_t* returnedEventsOut);
// Returns the event used by generic wait syscalls for this endpoint.
EVENT* IoPipeWaitEvent(IO_PIPE_ENDPOINT* endpoint);
// Publishes and withdraws one kernel waiter around a generic wait operation.
void IoPipeBeginWait(IO_PIPE_ENDPOINT* endpoint);
void IoPipeEndWait(IO_PIPE_ENDPOINT* endpoint);
// Returns whether this endpoint is the read side of the pipe.
bool IoPipeIsReadEnd(IO_PIPE_ENDPOINT* endpoint);
code/kernel/include/kernel/ke_display.h
+5
View File
@@ -17,6 +17,10 @@
#include <kernel/ob_object.h>
#include <ntlite/ddk/display.h>
typedef struct KE_DISPLAY_MANAGER {
OBJECT Header;
} KE_DISPLAY_MANAGER;
typedef struct DISPLAY_CLOCK_OBJECT {
OBJECT Header;
EVENT* Event;
@@ -28,6 +32,7 @@ typedef struct DISPLAY_CLOCK_OBJECT {
} DISPLAY_CLOCK_OBJECT;
void KeDisplayInitialize(const HAL_API* hal);
KSTATUS KeDisplayReferenceManager(KE_DISPLAY_MANAGER** managerOut);
// Completes every console glyph queued before the call.
KSTATUS KeDisplayFlushConsoleOutput(void);
code/kernel/include/kernel/ke_input.h
+22
View File
@@ -11,6 +11,16 @@
#include <kernel/ke_dpc.h>
typedef struct SYS_GLOBAL_INPUT_STATE SYS_GLOBAL_INPUT_STATE;
typedef struct SYS_INPUT_MANAGER_EVENT SYS_INPUT_MANAGER_EVENT;
typedef struct KE_INPUT_MANAGER_EVENT {
uint32_t Type;
uint64_t Sequence;
union {
INPUT_KEY_EVENT Key;
INPUT_MOUSE_EVENT Mouse;
} Data;
} KE_INPUT_MANAGER_EVENT;
typedef struct INPUT_KEY_QUEUE_ENTRY {
INPUT_KEY Key;
@@ -34,6 +44,12 @@ typedef struct KE_INPUT_CONTEXT {
INPUT_MOUSE_EVENT MouseQueue[128];
volatile int MouseHead;
volatile int MouseTail;
KE_INPUT_MANAGER_EVENT ManagerQueue[256];
volatile int ManagerHead;
volatile int ManagerTail;
volatile long long ManagerSequence;
EVENT* ManagerWaitEvent;
volatile int MouseX;
volatile int MouseY;
volatile int MouseBoundsWidth;
@@ -44,6 +60,10 @@ typedef struct KE_INPUT_CONTEXT {
EVENT* WaitEvent;
} KE_INPUT_CONTEXT;
typedef struct KE_INPUT_MANAGER {
OBJECT Header;
} KE_INPUT_MANAGER;
typedef struct INPUT_QUEUE {
OBJECT Header;
KE_INPUT_CONTEXT* Context;
@@ -51,6 +71,8 @@ typedef struct INPUT_QUEUE {
} INPUT_QUEUE;
void KeInputInitialize(const HAL_API* hal);
KSTATUS KeInputReferenceManager(KE_INPUT_MANAGER** managerOut);
KSTATUS KeInputManagerReadEvent(KE_INPUT_MANAGER* manager, uint64_t timeoutMs, SYS_INPUT_MANAGER_EVENT* eventOut);
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId);
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event);
code/kernel/include/kernel/ke_surface.h
+39
View File
@@ -0,0 +1,39 @@
// Kernel surface declarations. Surfaces are capability-backed shared pixel storage, not display ownership.
#pragma once
#include <stdint.h>
#include <ntlite/status.h>
#include <ntlite/display.h>
#include <kernel/ob_object.h>
struct PROCESS;
struct EVENT;
// Creates one surface storage object and returns its initial object reference.
KSTATUS KeSurfaceCreate(uint32_t width, uint32_t height, uint32_t format, uint32_t flags,
OBJECT** surfaceOut);
// Maps one surface into the calling process's VAD namespace with attenuated page permissions.
KSTATUS KeSurfaceMap(struct PROCESS* process, OBJECT* surface, uint32_t flags,
void** baseOut, uint32_t* pitchBytesOut);
// Removes one mapping only when the supplied VAD is a surface shared view owned by the process.
KSTATUS KeSurfaceUnmap(struct PROCESS* process, void* base);
// Snapshots one surface's storage and commit metadata.
KSTATUS KeSurfaceQuery(OBJECT* surface, SYS_SURFACE_DESC* descOut);
// Announces changed pixels without touching physical scanout.
KSTATUS KeSurfaceCommit(OBJECT* surface, const DISPLAY_RECT* rects, uint32_t rectCount,
uint64_t* generationOut);
// Presents one surface after a caller has separately proved DisplayManager PRESENT authority.
KSTATUS KeSurfacePresent(OBJECT* surface, const DISPLAY_RECT* rects, uint32_t rectCount,
uint64_t* bytesCopiedOut);
// Returns the passive event used to wait for commit-generation changes.
struct EVENT* KeSurfaceWaitEvent(OBJECT* surface);
// Snapshots global surface telemetry.
void KeSurfaceQueryStats(SYS_SURFACE_STATS* statsOut);
code/kernel/include/kernel/ke_thread.h
+13
View File
@@ -108,6 +108,11 @@ struct THREAD {
void* FpuContext;
uint8_t FpuStorage[528];
// Mutable diagnostic description. This is thread-owned state rather than
// process metadata because independent worker threads may carry distinct roles.
SPINLOCK NameLock;
char Name[NTL_THREAD_NAME_MAX];
// Scheduler accounting. QuantaConsumed is kept in the assembly-stable
// portion of THREAD for layout compatibility; the dynamic scheduler uses
// these fields for per-dispatch budget and CPU-burst tracking.
@@ -188,6 +193,11 @@ struct THREAD {
// process memory. Process termination must wait until that ownership has
// returned instead of freeing the thread underneath an active driver.
volatile long TerminationDeferralCount;
// Set once terminal thread teardown begins. Interrupts may continue to
// run, but interrupt-return scheduling must not abandon this kernel stack
// before KeFinalizeThreadExit and process-exit publication complete.
volatile long ExitInProgress;
};
// Creates and queues a user thread. On success, *out receives a referenced
@@ -251,6 +261,7 @@ void KiWakeThreadForUserApc(THREAD* thread);
void KeEnterTerminationDeferral(void);
void KeLeaveTerminationDeferral(void);
int KeThreadTerminationDeferred(const THREAD* thread);
int KeThreadExitInProgress(const THREAD* thread);
// Assembly stub - implemented in ke_switch.asm.
void KeSwitchContext(THREAD* from, THREAD* to);
@@ -258,4 +269,6 @@ __declspec(noreturn) void KeStartInitialThread(THREAD* thread);
void KeInitThreadFpuContext(THREAD* thread);
void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask);
KSTATUS KeSetThreadName(THREAD* thread, const char* name, size_t length);
size_t KeQueryThreadName(THREAD* thread, char* buffer, size_t capacity);
uint64_t KeGetDefaultThreadAffinityMask(void);
code/kernel/include/kernel/ke_userenv.h
+3
View File
@@ -20,6 +20,9 @@
#define SYS_STARTUP_CAP_SERVICE_CONTROL 2u
#define SYS_STARTUP_CAP_SYSTEM_VOLUME_NAMESPACE 3u
#define SYS_STARTUP_CAP_VOLUME_MANAGER 4u
#define SYS_STARTUP_CAP_INPUT_MANAGER 5u
#define SYS_STARTUP_CAP_DISPLAY_MANAGER 6u
#define SYS_STARTUP_CAP_SESSION_CONTROL 7u
#define SYS_STARTUP_CAPABILITY_MAX 32u
typedef struct SYS_STARTUP_CAPABILITY {
code/kernel/include/kernel/mm_virt.h
+2
View File
@@ -34,6 +34,8 @@ typedef enum MM_VAD_TYPE {
#define MM_VAD_ZERO 0x00000004u
#define MM_VAD_PREFETCH 0x00000008u
#define MM_VAD_SHARED_WRITE 0x00000010u
#define MM_VAD_SURFACE 0x00000020u
#define MM_VAD_PIPE 0x00000040u
// Virtual address space constants.
#define USER_BASE 0x0000000000001000ULL
code/kernel/include/kernel/syscall_nr.h
+156 -138
View File
@@ -25,164 +25,182 @@
#define SYS_GET_CURRENT_TID 14
#define SYS_GET_CURRENT_PID 15
#define SYS_SET_THREAD_PRIORITY 16
#define SYS_YIELD 17
#define SYS_SLEEP 18
#define SYS_WAIT_SINGLE 19
#define SYS_WAIT_MANY 20
#define SYS_SET_USER_APC_DISPATCHER 21
#define SYS_APC_RETURN 22
#define SYS_SET_EXCEPTION_DISPATCHER 23
#define SYS_GET_EXCEPTION_DISPATCHER 24
#define SYS_CONTINUE_EXCEPTION 25
#define SYS_RAISE_EXCEPTION 26
#define SYS_SET_THREAD_NAME 17
#define SYS_QUERY_THREAD_NAME 18
#define SYS_YIELD 19
#define SYS_SLEEP 20
#define SYS_WAIT_SINGLE 21
#define SYS_WAIT_MANY 22
#define SYS_SET_USER_APC_DISPATCHER 23
#define SYS_APC_RETURN 24
#define SYS_SET_EXCEPTION_DISPATCHER 25
#define SYS_GET_EXCEPTION_DISPATCHER 26
#define SYS_CONTINUE_EXCEPTION 27
#define SYS_RAISE_EXCEPTION 28
// Virtual memory, mappings, and handles.
#define SYS_ALLOC_VM 27
#define SYS_FREE_VM 28
#define SYS_PROTECT_VM 29
#define SYS_QUERY_VM 30
#define SYS_CLOSE_HANDLE 31
#define SYS_DUPLICATE_HANDLE 32
#define SYS_DERIVE_HANDLE 33
#define SYS_REVOKE_HANDLE 34
#define SYS_DERIVE_SECURITY_CONTEXT 35
#define SYS_QUERY_SECURITY_CONTEXT 36
#define SYS_CREATE_FILE_MAPPING 37
#define SYS_MAP_FILE 38
#define SYS_ALLOC_VM 29
#define SYS_FREE_VM 30
#define SYS_PROTECT_VM 31
#define SYS_QUERY_VM 32
#define SYS_CLOSE_HANDLE 33
#define SYS_DUPLICATE_HANDLE 34
#define SYS_DERIVE_HANDLE 35
#define SYS_REVOKE_HANDLE 36
#define SYS_DERIVE_SECURITY_CONTEXT 37
#define SYS_QUERY_SECURITY_CONTEXT 38
#define SYS_CREATE_FILE_MAPPING 39
#define SYS_MAP_FILE 40
// Synchronization objects.
#define SYS_CREATE_MUTEX 39
#define SYS_TRY_ACQUIRE_MUTEX 40
#define SYS_RELEASE_MUTEX 41
#define SYS_CREATE_SEM 42
#define SYS_TRY_WAIT_SEM 43
#define SYS_SIGNAL_SEM 44
#define SYS_CREATE_EVENT 45
#define SYS_SIGNAL_EVENT 46
#define SYS_CLEAR_EVENT 47
#define SYS_CREATE_MUTEX 41
#define SYS_TRY_ACQUIRE_MUTEX 42
#define SYS_RELEASE_MUTEX 43
#define SYS_CREATE_SEM 44
#define SYS_TRY_WAIT_SEM 45
#define SYS_SIGNAL_SEM 46
#define SYS_CREATE_EVENT 47
#define SYS_SIGNAL_EVENT 48
#define SYS_CLEAR_EVENT 49
// Queues and named queues.
#define SYS_CREATE_QUEUE 48
#define SYS_SEND_QUEUE 49
#define SYS_RECV_QUEUE 50
#define SYS_PUBLISH_QUEUE 51
#define SYS_OPEN_NAMED_QUEUE 52
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 53
#define SYS_UNPUBLISH_QUEUE 54
#define SYS_CREATE_QUEUE 50
#define SYS_SEND_QUEUE 51
#define SYS_RECV_QUEUE 52
#define SYS_PUBLISH_QUEUE 53
#define SYS_OPEN_NAMED_QUEUE 54
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 55
#define SYS_UNPUBLISH_QUEUE 56
// LPC-like Ports and well-known system services.
#define SYS_CREATE_PORT 55
#define SYS_CONNECT_PORT 56
#define SYS_ACCEPT_PORT 57
#define SYS_SEND_PORT 58
#define SYS_RECEIVE_PORT 59
#define SYS_CALL_PORT 60
#define SYS_REPLY_PORT 61
#define SYS_DISCONNECT_PORT 62
#define SYS_REGISTER_SYSTEM_SERVICE 63
#define SYS_UNREGISTER_SYSTEM_SERVICE 64
#define SYS_CONNECT_SYSTEM_SERVICE 65
#define SYS_CREATE_PORT 57
#define SYS_CONNECT_PORT 58
#define SYS_ACCEPT_PORT 59
#define SYS_SEND_PORT 60
#define SYS_RECEIVE_PORT 61
#define SYS_CALL_PORT 62
#define SYS_REPLY_PORT 63
#define SYS_DISCONNECT_PORT 64
#define SYS_REGISTER_SYSTEM_SERVICE 65
#define SYS_UNREGISTER_SYSTEM_SERVICE 66
#define SYS_CONNECT_SYSTEM_SERVICE 67
// Mounted-volume manager control plane.
#define SYS_QUERY_VOLUMES 66
#define SYS_OPEN_VOLUME 67
#define SYS_WAIT_VOLUME_CHANGE 68
#define SYS_QUERY_VOLUMES 68
#define SYS_OPEN_VOLUME 69
#define SYS_WAIT_VOLUME_CHANGE 70
// Files and directories.
#define SYS_OPEN_FILE_AT 69
#define SYS_OPEN_DIR_AT 70
#define SYS_STAT_AT 71
#define SYS_DELETE_AT 72
#define SYS_RENAME_AT 73
#define SYS_CREATE_DIR_AT 74
#define SYS_REMOVE_DIR_AT 75
#define SYS_READ_FILE 76
#define SYS_WRITE_FILE 77
#define SYS_READ_FILE_ASYNC 78
#define SYS_WRITE_FILE_ASYNC 79
#define SYS_CANCEL_IO 80
#define SYS_SEEK_FILE 81
#define SYS_GET_FILE_POS 82
#define SYS_GET_FILE_SIZE 83
#define SYS_GET_FILE_STAT 84
#define SYS_TRUNCATE_FILE 85
#define SYS_FLUSH_FILE 86
#define SYS_LOCK_FILE 87
#define SYS_UNLOCK_FILE 88
#define SYS_READ_DIR 89
#define SYS_OPEN_FILE_AT 71
#define SYS_OPEN_DIR_AT 72
#define SYS_STAT_AT 73
#define SYS_DELETE_AT 74
#define SYS_RENAME_AT 75
#define SYS_CREATE_DIR_AT 76
#define SYS_REMOVE_DIR_AT 77
#define SYS_READ_FILE 78
#define SYS_WRITE_FILE 79
#define SYS_READ_FILE_ASYNC 80
#define SYS_WRITE_FILE_ASYNC 81
#define SYS_CANCEL_IO 82
#define SYS_SEEK_FILE 83
#define SYS_GET_FILE_POS 84
#define SYS_GET_FILE_SIZE 85
#define SYS_GET_FILE_STAT 86
#define SYS_TRUNCATE_FILE 87
#define SYS_FLUSH_FILE 88
#define SYS_LOCK_FILE 89
#define SYS_UNLOCK_FILE 90
#define SYS_QUERY_FILE_LOCK 91
#define SYS_SET_FILE_STATUS 92
#define SYS_CREATE_PIPE 93
#define SYS_NOTIFY_PIPE 94
#define SYS_READ_DIR 95
// Console, display, and surfaces.
#define SYS_WRITE_CONSOLE 90
#define SYS_GET_CONSOLE_INFO 91
#define SYS_SET_CONSOLE_CURSOR 92
#define SYS_FILL_CONSOLE_CHARACTER 93
#define SYS_FILL_CONSOLE_ATTRIBUTE 94
#define SYS_PUT_SCREEN_CHAR 95
#define SYS_CLEAR_SCREEN 96
#define SYS_GET_SCREEN_SIZE 97
#define SYS_QUERY_DISPLAY_INFO 98
#define SYS_OPEN_DISPLAY_CLOCK 99
#define SYS_CREATE_SURFACE 100
#define SYS_MAP_SURFACE 101
#define SYS_QUERY_SURFACE 102
#define SYS_PRESENT_SURFACE 103
#define SYS_WRITE_CONSOLE 96
#define SYS_GET_CONSOLE_INFO 97
#define SYS_SET_CONSOLE_CURSOR 98
#define SYS_FILL_CONSOLE_CHARACTER 99
#define SYS_FILL_CONSOLE_ATTRIBUTE 100
#define SYS_PUT_SCREEN_CHAR 101
#define SYS_CLEAR_SCREEN 102
#define SYS_GET_SCREEN_SIZE 103
#define SYS_QUERY_DISPLAY_INFO 104
#define SYS_QUERY_DISPLAY_MANAGER_INFO 105
#define SYS_PRESENT_DISPLAY_MANAGER_SURFACE 106
#define SYS_SET_DISPLAY_MANAGER_CURSOR 107
#define SYS_WRITE_DISPLAY_MANAGER_CONSOLE 108
#define SYS_QUERY_DISPLAY_MANAGER_CONSOLE_INFO 109
#define SYS_SET_DISPLAY_MANAGER_CONSOLE_CURSOR 110
#define SYS_FILL_DISPLAY_MANAGER_CONSOLE_CHARACTER 111
#define SYS_FILL_DISPLAY_MANAGER_CONSOLE_ATTRIBUTE 112
#define SYS_PUT_DISPLAY_MANAGER_CONSOLE_CHARACTER 113
#define SYS_CLEAR_DISPLAY_MANAGER_CONSOLE 114
#define SYS_OPEN_DISPLAY_CLOCK 115
#define SYS_CREATE_SURFACE 116
#define SYS_MAP_SURFACE 117
#define SYS_UNMAP_SURFACE 118
#define SYS_QUERY_SURFACE 119
#define SYS_COMMIT_SURFACE 120
// Native render audio.
#define SYS_AUDIO_ENUMERATE_ENDPOINTS 104
#define SYS_AUDIO_OPEN_STREAM 105
#define SYS_AUDIO_WRITE 106
#define SYS_AUDIO_ARM_WAIT 107
#define SYS_AUDIO_QUERY_STREAM 108
#define SYS_AUDIO_START 109
#define SYS_AUDIO_PAUSE 110
#define SYS_AUDIO_RESUME 111
#define SYS_AUDIO_DRAIN 112
#define SYS_AUDIO_STOP 113
#define SYS_AUDIO_RESET 114
#define SYS_AUDIO_SET_VOLUME 115
#define SYS_AUDIO_SET_MASTER_VOLUME 116
#define SYS_AUDIO_ENUMERATE_ENDPOINTS 121
#define SYS_AUDIO_OPEN_STREAM 122
#define SYS_AUDIO_WRITE 123
#define SYS_AUDIO_ARM_WAIT 124
#define SYS_AUDIO_QUERY_STREAM 125
#define SYS_AUDIO_START 126
#define SYS_AUDIO_PAUSE 127
#define SYS_AUDIO_RESUME 128
#define SYS_AUDIO_DRAIN 129
#define SYS_AUDIO_STOP 130
#define SYS_AUDIO_RESET 131
#define SYS_AUDIO_SET_VOLUME 132
#define SYS_AUDIO_SET_MASTER_VOLUME 133
// Input.
#define SYS_POLL_KEYBOARD 117
#define SYS_IS_KEY_DOWN 118
#define SYS_READ_KEYBOARD_CHAR 119
#define SYS_READ_KEY 120
#define SYS_GET_KEY_MODIFIERS 121
#define SYS_QUERY_GLOBAL_INPUT_STATE 122
#define SYS_OPEN_INPUT_QUEUE 123
#define SYS_READ_INPUT_EVENT 124
#define SYS_PEEK_INPUT_EVENT 125
#define SYS_PEEK_MOUSE 126
#define SYS_READ_MOUSE 127
#define SYS_GET_MOUSE_POS 128
#define SYS_READ_INPUT_MANAGER_EVENT 134
#define SYS_POLL_KEYBOARD 135
#define SYS_IS_KEY_DOWN 136
#define SYS_READ_KEYBOARD_CHAR 137
#define SYS_READ_KEY 138
#define SYS_GET_KEY_MODIFIERS 139
#define SYS_QUERY_GLOBAL_INPUT_STATE 140
#define SYS_OPEN_INPUT_QUEUE 141
#define SYS_READ_INPUT_EVENT 142
#define SYS_PEEK_INPUT_EVENT 143
#define SYS_PEEK_MOUSE 144
#define SYS_READ_MOUSE 145
#define SYS_GET_MOUSE_POS 146
// Time, debug, and system information.
#define SYS_CREATE_TIMER 129
#define SYS_SET_TIMER 130
#define SYS_CANCEL_TIMER 131
#define SYS_GET_TICK 132
#define SYS_GET_TICK_FREQUENCY 133
#define SYS_GET_SYSTEM_TIME 134
#define SYS_PRINT_DEBUG 135
#define SYS_QUERY_TRAP_FRAME 136
#define SYS_GET_MEMORY_STATUS 137
#define SYS_QUERY_KERNEL_STATS 138
#define SYS_CREATE_TIMER 147
#define SYS_SET_TIMER 148
#define SYS_CANCEL_TIMER 149
#define SYS_GET_TICK 150
#define SYS_GET_TICK_FREQUENCY 151
#define SYS_GET_SYSTEM_TIME 152
#define SYS_PRINT_DEBUG 153
#define SYS_QUERY_TRAP_FRAME 154
#define SYS_GET_MEMORY_STATUS 155
#define SYS_QUERY_KERNEL_STATS 156
// Native sockets and polling.
#define SYS_CREATE_SOCKET 139
#define SYS_SOCKET_IO_ASYNC 140
#define SYS_BIND_SOCKET 141
#define SYS_LISTEN_SOCKET 142
#define SYS_ACCEPT_SOCKET 143
#define SYS_CONNECT_SOCKET 144
#define SYS_SEND_SOCKET 145
#define SYS_RECV_SOCKET 146
#define SYS_SHUTDOWN_SOCKET 147
#define SYS_SEND_TO_SOCKET 148
#define SYS_RECV_FROM_SOCKET 149
#define SYS_RESOLVE_HOST 150
#define SYS_SET_SOCKET_OPTION 151
#define SYS_GET_SOCKET_OPTION 152
#define SYS_POLL 153
#define SYS_CREATE_SOCKET 157
#define SYS_SOCKET_IO_ASYNC 158
#define SYS_BIND_SOCKET 159
#define SYS_LISTEN_SOCKET 160
#define SYS_ACCEPT_SOCKET 161
#define SYS_CONNECT_SOCKET 162
#define SYS_SEND_SOCKET 163
#define SYS_RECV_SOCKET 164
#define SYS_SHUTDOWN_SOCKET 165
#define SYS_SEND_TO_SOCKET 166
#define SYS_RECV_FROM_SOCKET 167
#define SYS_RESOLVE_HOST 168
#define SYS_SET_SOCKET_OPTION 169
#define SYS_GET_SOCKET_OPTION 170
#define SYS_POLL 171
#define SYS_MAX 154
#define SYS_MAX 172
code/kernel/io_file.c
+193 -19
View File
@@ -44,6 +44,7 @@ typedef struct IO_FILE_LOCK_ENTRY {
LIST_ENTRY Link;
IO_FILE_IDENTITY Identity;
PROCESS* Owner;
uint32_t OwnerProcessId;
uint64_t Offset;
uint64_t Length;
uint32_t Mode;
@@ -102,6 +103,41 @@ static bool KiIoFileLockRangesOverlap(uint64_t aOffset, uint64_t aLength,
return aOffset < bEnd && bOffset < aEnd;
}
// Converts an exclusive range end back to the public lock-length representation.
static uint64_t KiIoFileLockLengthFromEnd(uint64_t offset, uint64_t end) {
if (end == UINT64_MAX)
return SYS_FILE_LOCK_TO_EOF;
return end - offset;
}
// Initializes one file-lock entry from an existing owner's identity.
static void KiIoInitializeFileLockEntry(IO_FILE_LOCK_ENTRY* entry,
const IO_FILE_IDENTITY* identity,
PROCESS* owner,
uint32_t mode,
uint64_t offset,
uint64_t length) {
memset(entry, 0, sizeof(*entry));
LIST_INIT(&entry->Link);
entry->Identity = *identity;
entry->Owner = owner;
entry->OwnerProcessId = owner->ProcessId;
entry->Offset = offset;
entry->Length = length;
entry->Mode = mode;
}
// Frees every detached file-lock entry linked on a private list.
static void KiIoFreeDetachedFileLocks(LIST_ENTRY* head) {
if (!head)
return;
while (head->Fwd != head) {
LIST_ENTRY* node = ListPopHead(head);
IO_FILE_LOCK_ENTRY* entry = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
MmFreePool(entry);
}
}
// Tests whether one existing lock conflicts with a requested lock.
static bool KiIoFileLockConflicts(const IO_FILE_LOCK_ENTRY* lock,
const IO_FILE_IDENTITY* identity,
@@ -302,7 +338,8 @@ static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "io.file.async");
s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker,
0,
&parameters,
@@ -686,7 +723,7 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
size_t remaining = (size_t)(fileSize - offset);
if (size > remaining) size = remaining;
if ((file->Flags & IO_FILE_FLAG_NO_CACHE) || size < 512)
if (file->Flags & IO_FILE_FLAG_NO_CACHE)
return IoVolumeRead(file->Volume, file->FileCtx, offset, buf, size, bytesRead);
size_t total = 0;
@@ -881,6 +918,63 @@ __declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDE
return IoVolumeQueryIdentity(file->Volume, file->FileCtx, out);
}
// Queries the first conflicting advisory lock held by another process.
__declspec(dllexport) KSTATUS IoQueryFileLock(FILE_OBJECT* file, PROCESS* owner, uint32_t mode,
uint64_t offset, uint64_t length,
SYS_FILE_LOCK_INFO* infoOut) {
if (!file || !owner || !infoOut)
return NTLSTATUS_INVALID_PARAMETER;
if (mode != SYS_FILE_LOCK_SHARED && mode != SYS_FILE_LOCK_EXCLUSIVE)
return NTLSTATUS_INVALID_PARAMETER;
if (length == 0)
return NTLSTATUS_INVALID_PARAMETER;
IO_FILE_IDENTITY identity;
KSTATUS status = IoQueryFileIdentity(file, &identity);
if (!KSUCCESS(status))
return status;
KiIoEnsureFileLockState();
memset(infoOut, 0, sizeof(*infoOut));
infoOut->Size = sizeof(*infoOut);
infoOut->Mode = SYS_FILE_LOCK_NONE;
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
LIST_ENTRY* head = &gFileLocks;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (!KiIoFileLockConflicts(lock, &identity, owner, mode, offset, length))
continue;
infoOut->Mode = lock->Mode;
infoOut->OwnerProcessId = lock->OwnerProcessId;
infoOut->Offset = lock->Offset;
infoOut->Length = lock->Length;
break;
}
KeReleaseSpinlock(&gFileLockLock, flags);
return NTLSTATUS_SUCCESS;
}
// Changes mutable file-object status flags without changing handle authority.
__declspec(dllexport) KSTATUS IoSetFileStatus(FILE_OBJECT* file, uint32_t mask, uint32_t flags) {
if (!file)
return NTLSTATUS_INVALID_PARAMETER;
if (mask & ~SYS_FILE_STATUS_APPEND)
return NTLSTATUS_INVALID_PARAMETER;
if (flags & ~mask)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t lockFlags = KeAcquireSpinlock(&file->Lock);
if (mask & SYS_FILE_STATUS_APPEND) {
if (flags & SYS_FILE_STATUS_APPEND)
file->Flags |= IO_FILE_FLAG_APPEND;
else
file->Flags &= ~IO_FILE_FLAG_APPEND;
}
KeReleaseSpinlock(&file->Lock, lockFlags);
return NTLSTATUS_SUCCESS;
}
// Acquires an advisory process-owned byte-range lock for one file identity.
__declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint32_t mode,
uint64_t offset, uint64_t length,
@@ -905,27 +999,73 @@ __declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint
memset(&waiter, 0, sizeof(waiter));
while (true) {
IO_FILE_LOCK_ENTRY* entry = (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k'));
if (!entry) {
IO_FILE_LOCK_ENTRY* entry =
(IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k'));
IO_FILE_LOCK_ENTRY* split =
(IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*split), MM_POOL_TAG('f', 'l', 'c', 'k'));
if (!entry || !split) {
if (entry) MmFreePool(entry);
if (split) MmFreePool(split);
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return NTLSTATUS_NO_MEMORY;
}
LIST_ENTRY detached;
LIST_INIT(&detached);
bool changedOwnedRange = false;
bool splitUsed = false;
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
if (!KiIoFileLockHasConflictLocked(&identity, owner, mode, offset, length)) {
entry->Identity = identity;
entry->Owner = owner;
entry->Offset = offset;
entry->Length = length;
entry->Mode = mode;
LIST_INIT(&entry->Link);
uint64_t requestedEnd = KiIoFileLockEnd(offset, length);
LIST_ENTRY* head = &gFileLocks;
LIST_ENTRY* node = head->Fwd;
while (node != head) {
LIST_ENTRY* next = node->Fwd;
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (lock->Owner != owner || !KiIoSameFileIdentity(&lock->Identity, &identity) ||
!KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length)) {
node = next;
continue;
}
uint64_t lockStart = lock->Offset;
uint64_t lockEnd = KiIoFileLockEnd(lock->Offset, lock->Length);
changedOwnedRange = true;
if (lockStart < offset && requestedEnd < lockEnd) {
lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
KiIoInitializeFileLockEntry(split, &identity, owner, lock->Mode,
requestedEnd,
KiIoFileLockLengthFromEnd(requestedEnd, lockEnd));
ListInsertTail(&gFileLocks, &split->Link);
splitUsed = true;
} else if (lockStart < offset) {
lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
} else if (requestedEnd < lockEnd) {
lock->Offset = requestedEnd;
lock->Length = KiIoFileLockLengthFromEnd(requestedEnd, lockEnd);
} else {
ListRemove(&lock->Link);
ListInsertTail(&detached, &lock->Link);
}
node = next;
}
KiIoInitializeFileLockEntry(entry, &identity, owner, mode, offset, length);
ListInsertTail(&gFileLocks, &entry->Link);
if (changedOwnedRange)
KiIoSignalFileLockWaitersLocked(&identity);
KeReleaseSpinlock(&gFileLockLock, flags);
if (!splitUsed)
MmFreePool(split);
KiIoFreeDetachedFileLocks(&detached);
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
return NTLSTATUS_SUCCESS;
}
KeReleaseSpinlock(&gFileLockLock, flags);
MmFreePool(entry);
MmFreePool(split);
if (timeoutMilliseconds == 0) {
if (waitEvent) ObDereferenceObject(&waitEvent->Header);
@@ -971,7 +1111,7 @@ __declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint
}
}
// Releases this process's overlapping advisory locks for one file identity.
// Releases this process's requested advisory byte range while preserving outside fragments.
__declspec(dllexport) KSTATUS IoUnlockFile(FILE_OBJECT* file, PROCESS* owner, uint64_t offset, uint64_t length) {
if (!file || !owner || length == 0) return NTLSTATUS_INVALID_PARAMETER;
@@ -981,27 +1121,61 @@ __declspec(dllexport) KSTATUS IoUnlockFile(FILE_OBJECT* file, PROCESS* owner, ui
KiIoEnsureFileLockState();
bool released = false;
IO_FILE_LOCK_ENTRY* split =
(IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*split), MM_POOL_TAG('f', 'l', 'c', 'k'));
LIST_ENTRY detached;
LIST_INIT(&detached);
bool changed = false;
bool splitUsed = false;
uint64_t requestedEnd = KiIoFileLockEnd(offset, length);
uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
LIST_ENTRY* head = &gFileLocks;
LIST_ENTRY* node = head->Fwd;
while (node != head) {
LIST_ENTRY* next = node->Fwd;
IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
if (lock->Owner == owner && KiIoSameFileIdentity(&lock->Identity, &identity) &&
KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length)) {
ListRemove(&lock->Link);
MmFreePool(lock);
released = true;
if (lock->Owner != owner || !KiIoSameFileIdentity(&lock->Identity, &identity) ||
!KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length)) {
node = next;
continue;
}
uint64_t lockStart = lock->Offset;
uint64_t lockEnd = KiIoFileLockEnd(lock->Offset, lock->Length);
if (lockStart < offset && requestedEnd < lockEnd) {
if (!split) {
KeReleaseSpinlock(&gFileLockLock, flags);
KiIoFreeDetachedFileLocks(&detached);
return NTLSTATUS_NO_MEMORY;
}
lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
KiIoInitializeFileLockEntry(split, &identity, owner, lock->Mode,
requestedEnd,
KiIoFileLockLengthFromEnd(requestedEnd, lockEnd));
ListInsertTail(&gFileLocks, &split->Link);
splitUsed = true;
} else if (lockStart < offset) {
lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
} else if (requestedEnd < lockEnd) {
lock->Offset = requestedEnd;
lock->Length = KiIoFileLockLengthFromEnd(requestedEnd, lockEnd);
} else {
ListRemove(&lock->Link);
ListInsertTail(&detached, &lock->Link);
}
changed = true;
node = next;
}
if (released)
if (changed)
KiIoSignalFileLockWaitersLocked(&identity);
KeReleaseSpinlock(&gFileLockLock, flags);
return released ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND;
if (split && !splitUsed)
MmFreePool(split);
KiIoFreeDetachedFileLocks(&detached);
return changed ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND;
}
// Releases every advisory file lock owned by one exiting process.
code/kernel/io_pipe.c
+321
View File
@@ -0,0 +1,321 @@
// Kernel pipe object lifetime, shared backing, readiness, and wait integration.
#include <string.h>
#include <intrin.h>
#include <kernel/io_pipe.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/mm_pool.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <ntlite/syscall.h>
#define IO_PIPE_STATE_TAG MM_POOL_TAG('p', 'i', 'p', 's')
#define IO_PIPE_FRAME_TAG MM_POOL_TAG('p', 'i', 'p', 'f')
typedef enum IO_PIPE_DIRECTION {
IoPipeReadEnd = 1,
IoPipeWriteEnd = 2,
} IO_PIPE_DIRECTION;
typedef struct IO_PIPE_STATE {
volatile long RefCount;
SPINLOCK Lock;
EVENT* ReadEvent;
EVENT* WriteEvent;
PHYSICAL_ADDRESS* Frames;
size_t PageCount;
MM_KERNEL_VIEW* KernelView;
SYS_PIPE_SHARED_STATE* Shared;
} IO_PIPE_STATE;
struct IO_PIPE_ENDPOINT {
OBJECT Header;
IO_PIPE_STATE* State;
IO_PIPE_DIRECTION Direction;
};
// Returns the current buffered-byte count when the shared counters are sane.
static bool KiPipeUsedBytes(const SYS_PIPE_SHARED_STATE* shared, uint64_t* usedOut) {
if (!shared || !usedOut || shared->Magic != SYS_PIPE_SHARED_MAGIC ||
shared->Version != SYS_PIPE_SHARED_VERSION ||
shared->Capacity != SYS_PIPE_BUFFER_CAPACITY ||
shared->HeaderBytes != SYS_PIPE_SHARED_HEADER_BYTES)
return false;
uint64_t readPosition = shared->ReadPosition;
uint64_t writePosition = shared->WritePosition;
uint64_t used = writePosition - readPosition;
if (used > SYS_PIPE_BUFFER_CAPACITY)
return false;
*usedOut = used;
return true;
}
// Refreshes level-triggered endpoint events while the state lock is held.
static void KiPipeRefreshEventsLocked(IO_PIPE_STATE* state) {
if (!state || !state->Shared)
return;
SYS_PIPE_SHARED_STATE* shared = state->Shared;
uint64_t used = 0;
if (!KiPipeUsedBytes(shared, &used)) {
KeSignalEvent(state->ReadEvent);
KeSignalEvent(state->WriteEvent);
return;
}
if (used != 0 || shared->WriterOpen == 0)
KeSignalEvent(state->ReadEvent);
else
KeClearEvent(state->ReadEvent);
uint64_t freeBytes = SYS_PIPE_BUFFER_CAPACITY - used;
if (shared->ReaderOpen == 0 || freeBytes >= SYS_PIPE_ATOMIC_WRITE_MAX)
KeSignalEvent(state->WriteEvent);
else
KeClearEvent(state->WriteEvent);
}
// Releases shared pipe storage after both endpoint objects are gone.
static void KiPipeStateRelease(IO_PIPE_STATE* state) {
if (!state || _InterlockedDecrement(&state->RefCount) != 0)
return;
if (state->KernelView) {
MmDestroyKernelView(state->KernelView);
state->KernelView = (MM_KERNEL_VIEW*)0;
state->Shared = (SYS_PIPE_SHARED_STATE*)0;
}
if (state->ReadEvent)
ObDereferenceObject(&state->ReadEvent->Header);
if (state->WriteEvent)
ObDereferenceObject(&state->WriteEvent->Header);
if (state->Frames) {
for (size_t i = 0; i < state->PageCount; ++i)
if (state->Frames[i])
(void)MmPfnRelease(state->Frames[i]);
MmFreePool(state->Frames);
}
MmFreePool(state);
}
// Destroys one endpoint and publishes final-reader/final-writer state.
static void KiPipeEndpointDestroy(OBJECT* object) {
if (!object)
return;
IO_PIPE_ENDPOINT* endpoint = CONTAINING_RECORD(object, IO_PIPE_ENDPOINT, Header);
IO_PIPE_STATE* state = endpoint->State;
endpoint->State = (IO_PIPE_STATE*)0;
if (!state)
return;
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (state->Shared) {
if (endpoint->Direction == IoPipeReadEnd)
state->Shared->ReaderOpen = 0;
else
state->Shared->WriterOpen = 0;
_ReadWriteBarrier();
KiPipeRefreshEventsLocked(state);
}
KeReleaseSpinlock(&state->Lock, flags);
KiPipeStateRelease(state);
}
// Allocates one endpoint object for existing shared state.
static IO_PIPE_ENDPOINT* KiPipeCreateEndpoint(IO_PIPE_STATE* state, IO_PIPE_DIRECTION direction) {
IO_PIPE_ENDPOINT* endpoint =
(IO_PIPE_ENDPOINT*)ObAllocateObject(ObjectTypePipeEndpoint, sizeof(IO_PIPE_ENDPOINT));
if (!endpoint)
return (IO_PIPE_ENDPOINT*)0;
endpoint->Header.Destroy = KiPipeEndpointDestroy;
endpoint->State = state;
endpoint->Direction = direction;
return endpoint;
}
// Allocates the physical pages and stable kernel view for one pipe data plane.
static KSTATUS KiPipeCreateBacking(IO_PIPE_STATE* state) {
size_t payloadSize = sizeof(SYS_PIPE_SHARED_STATE);
size_t pageCount = (payloadSize + PAGE_SIZE - 1u) / PAGE_SIZE;
state->Frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS),
IO_PIPE_FRAME_TAG);
if (!state->Frames)
return NTLSTATUS_NO_MEMORY;
memset(state->Frames, 0, pageCount * sizeof(PHYSICAL_ADDRESS));
state->PageCount = pageCount;
for (size_t i = 0; i < pageCount; ++i) {
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseAnonymous);
if (!frame)
return NTLSTATUS_NO_MEMORY;
state->Frames[i] = frame;
void* page = MmPfnToVirtual(frame);
if (page)
memset(page, 0, PAGE_SIZE);
}
KSTATUS status = MmCreateKernelViewFromFrames(state->Frames, state->PageCount,
&state->KernelView);
if (!KSUCCESS(status))
return status;
state->Shared = (SYS_PIPE_SHARED_STATE*)MmKernelViewBase(state->KernelView);
if (!state->Shared)
return NTLSTATUS_FAILURE;
state->Shared->Magic = SYS_PIPE_SHARED_MAGIC;
state->Shared->Version = SYS_PIPE_SHARED_VERSION;
state->Shared->Capacity = SYS_PIPE_BUFFER_CAPACITY;
state->Shared->HeaderBytes = SYS_PIPE_SHARED_HEADER_BYTES;
state->Shared->ReaderOpen = 1;
state->Shared->WriterOpen = 1;
return NTLSTATUS_SUCCESS;
}
// Creates one anonymous pipe with kernel-owned endpoints and a user-mapped data plane.
KSTATUS IoCreatePipe(IO_PIPE_ENDPOINT** readEndOut, IO_PIPE_ENDPOINT** writeEndOut) {
if (!readEndOut || !writeEndOut)
return NTLSTATUS_INVALID_PARAMETER;
*readEndOut = (IO_PIPE_ENDPOINT*)0;
*writeEndOut = (IO_PIPE_ENDPOINT*)0;
IO_PIPE_STATE* state = (IO_PIPE_STATE*)MmAllocatePoolTag(sizeof(*state), IO_PIPE_STATE_TAG);
if (!state)
return NTLSTATUS_NO_MEMORY;
memset(state, 0, sizeof(*state));
state->RefCount = 2;
state->Lock = (SPINLOCK)SPINLOCK_INIT;
KSTATUS status = KeEventCreate(EventManualReset, 0, &state->ReadEvent);
if (KSUCCESS(status))
status = KeEventCreate(EventManualReset, 1, &state->WriteEvent);
if (KSUCCESS(status))
status = KiPipeCreateBacking(state);
if (!KSUCCESS(status)) {
state->RefCount = 1;
KiPipeStateRelease(state);
return status;
}
IO_PIPE_ENDPOINT* readEnd = KiPipeCreateEndpoint(state, IoPipeReadEnd);
IO_PIPE_ENDPOINT* writeEnd = KiPipeCreateEndpoint(state, IoPipeWriteEnd);
if (!readEnd || !writeEnd) {
if (readEnd)
ObDereferenceObject(&readEnd->Header);
else
KiPipeStateRelease(state);
if (writeEnd)
ObDereferenceObject(&writeEnd->Header);
else
KiPipeStateRelease(state);
return NTLSTATUS_NO_MEMORY;
}
*readEndOut = readEnd;
*writeEndOut = writeEnd;
return NTLSTATUS_SUCCESS;
}
// Maps the shared pipe data plane into one process-owned VAD.
KSTATUS IoMapPipe(IO_PIPE_ENDPOINT* endpoint, PROCESS* process, void** sharedStateOut) {
if (!endpoint || !process || !sharedStateOut || !endpoint->State)
return NTLSTATUS_INVALID_PARAMETER;
*sharedStateOut = (void*)0;
IO_PIPE_STATE* state = endpoint->State;
KSTATUS status = MmVadCreateSharedViewFromFrames(process, state->Frames, state->PageCount,
sizeof(SYS_PIPE_SHARED_STATE),
PAGE_PRESENT | PAGE_USER | PAGE_WRITE | PAGE_NX,
MM_VAD_PIPE, MmVadShared, sharedStateOut);
if (!KSUCCESS(status))
return status;
status = MmVadProtect(process, *sharedStateOut, SYS_PIPE_SHARED_HEADER_BYTES,
PAGE_PRESENT | PAGE_USER | PAGE_NX);
if (!KSUCCESS(status)) {
(void)MmVadDestroy(process, *sharedStateOut);
*sharedStateOut = (void*)0;
}
return status;
}
// Refreshes readiness after a user-mode ring transition.
KSTATUS IoNotifyPipe(IO_PIPE_ENDPOINT* endpoint) {
if (!endpoint || !endpoint->State)
return NTLSTATUS_INVALID_PARAMETER;
IO_PIPE_STATE* state = endpoint->State;
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
KiPipeRefreshEventsLocked(state);
KeReleaseSpinlock(&state->Lock, flags);
return NTLSTATUS_SUCCESS;
}
// Queries readiness for poll-style events.
KSTATUS IoPollPipe(IO_PIPE_ENDPOINT* endpoint, uint32_t events, uint32_t* returnedEventsOut) {
if (!endpoint || !returnedEventsOut || !endpoint->State || !endpoint->State->Shared)
return NTLSTATUS_INVALID_PARAMETER;
SYS_PIPE_SHARED_STATE* shared = endpoint->State->Shared;
uint64_t used = 0;
if (!KiPipeUsedBytes(shared, &used))
return NTLSTATUS_FAILURE;
uint32_t returned = 0;
if (endpoint->Direction == IoPipeReadEnd) {
if ((events & SYS_POLL_READ) && (used != 0 || shared->WriterOpen == 0))
returned |= SYS_POLL_READ;
if (shared->WriterOpen == 0)
returned |= SYS_POLL_HANGUP;
} else {
if (shared->ReaderOpen == 0) {
returned |= SYS_POLL_ERROR | SYS_POLL_HANGUP;
} else if ((events & SYS_POLL_WRITE) && used < SYS_PIPE_BUFFER_CAPACITY) {
returned |= SYS_POLL_WRITE;
}
}
*returnedEventsOut = returned;
return NTLSTATUS_SUCCESS;
}
// Returns the endpoint event used by generic waits.
EVENT* IoPipeWaitEvent(IO_PIPE_ENDPOINT* endpoint) {
if (!endpoint || !endpoint->State)
return (EVENT*)0;
return endpoint->Direction == IoPipeReadEnd
? endpoint->State->ReadEvent
: endpoint->State->WriteEvent;
}
// Publishes one kernel waiter before entering the generic event wait engine.
void IoPipeBeginWait(IO_PIPE_ENDPOINT* endpoint) {
if (!endpoint || !endpoint->State || !endpoint->State->Shared)
return;
SYS_PIPE_SHARED_STATE* shared = endpoint->State->Shared;
if (endpoint->Direction == IoPipeReadEnd)
_InterlockedIncrement(&shared->ReadWaiters);
else
_InterlockedIncrement(&shared->WriteWaiters);
uintptr_t flags = KeAcquireSpinlock(&endpoint->State->Lock);
KiPipeRefreshEventsLocked(endpoint->State);
KeReleaseSpinlock(&endpoint->State->Lock, flags);
}
// Withdraws one kernel waiter after the generic event wait completes.
void IoPipeEndWait(IO_PIPE_ENDPOINT* endpoint) {
if (!endpoint || !endpoint->State || !endpoint->State->Shared)
return;
SYS_PIPE_SHARED_STATE* shared = endpoint->State->Shared;
if (endpoint->Direction == IoPipeReadEnd)
_InterlockedDecrement(&shared->ReadWaiters);
else
_InterlockedDecrement(&shared->WriteWaiters);
}
// Returns whether this endpoint is the read side of the pipe.
bool IoPipeIsReadEnd(IO_PIPE_ENDPOINT* endpoint) {
return endpoint && endpoint->Direction == IoPipeReadEnd;
}
code/kernel/ke_apc.c
+14 -2
View File
@@ -2,6 +2,7 @@
#include <kernel/ke_apc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
@@ -109,7 +110,7 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
return false;
THREAD* thread = apc->Thread;
if (thread->State == ThreadDead)
if (thread->State == ThreadDead || KeThreadExitInProgress(thread))
return false;
if (_InterlockedCompareExchange(&apc->Inserted, 1, 0) != 0)
@@ -192,12 +193,23 @@ void KiDeliverKernelApcs(void) {
if (!thread)
return;
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KCPU* cpu = KeGetCurrentCpu();
if (KeGetCurrentIrql() != PASSIVE_LEVEL ||
(__readeflags() & 0x200u) == 0 ||
!cpu || cpu->InterruptDepth != 0)
return;
if (thread->ApcDeliveryActive || thread->KernelApcDisable != 0 || !thread->KernelApcPending)
return;
// A thread may already have marked itself waiting while it is still
// executing in the prepare-to-commit window. Delivering an APC there can
// consume the pending APC without completing an alertable wait, after
// which KeCommitBlockThread() would sleep forever. Wait commit owns APC
// interruption while the thread is logically waiting.
if (thread->State == ThreadWaiting)
return;
thread->ApcDeliveryActive = 1;
while (true) {
code/kernel/ke_audio.c
+1 -1
View File
@@ -1397,7 +1397,7 @@ static KSTATUS KiAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* reg
ObReferenceObject(&endpoint->Header);
const KE_THREAD_CREATE_PARAMETERS mixerParameters =
KE_THREAD_CREATE_REALTIME(KE_AUDIO_MIXER_REALTIME_PRIORITY);
KE_THREAD_CREATE_NAMED_REALTIME(KE_AUDIO_MIXER_REALTIME_PRIORITY, "audio.mixer");
status = KeCreateKernelThread((uintptr_t)KiAudioMixerThread,
(uintptr_t)endpoint,
&mixerParameters,
code/kernel/ke_display.c
+11 -1
View File
@@ -65,6 +65,7 @@ typedef struct BOOTFB_CONTEXT {
} BOOTFB_CONTEXT;
static const HAL_API* gDisplayHal;
static KE_DISPLAY_MANAGER gDisplayManager;
static DISPLAY_DEVICE* gDisplayList;
static DISPLAY_DEVICE* gPrimaryDisplay;
static SPINLOCK gDisplayLock = SPINLOCK_INIT;
@@ -1236,8 +1237,17 @@ static void KiDisplayRegisterOutputEndpoint(void) {
}
}
// Implements the display Initialize operation.
// References the singleton display-manager authority object used for bootstrap delegation.
KSTATUS KeDisplayReferenceManager(KE_DISPLAY_MANAGER** managerOut) {
if (!managerOut) return NTLSTATUS_INVALID_PARAMETER;
*managerOut = &gDisplayManager;
ObReferenceObject(&gDisplayManager.Header);
return NTLSTATUS_SUCCESS;
}
// Initializes the kernel display manager and bootstrap framebuffer provider.
void KeDisplayInitialize(const HAL_API* hal) {
ObInitializeObjectHeader(&gDisplayManager.Header, ObjectTypeDisplayManager);
gDisplayHal = hal;
gDisplayList = NULL;
gPrimaryDisplay = NULL;
code/kernel/ke_exception.c
+6 -3
View File
@@ -129,8 +129,10 @@ static KSTATUS KiExceptionUnhandledExitStatus(const SYS_EXCEPTION_RECORD* record
return NTLSTATUS_PROCESS_PAGE_FAULT;
case SYS_EXCEPTION_CODE_DATATYPE_MISALIGNMENT:
return NTLSTATUS_PROCESS_ALIGNMENT_FAULT;
case SYS_EXCEPTION_CODE_USER_RAISED:
return NTLSTATUS_PROCESS_USER_EXCEPTION;
default:
return (KSTATUS)record->Code;
return NTLSTATUS_PROCESS_UNKNOWN_FAULT;
}
}
@@ -146,7 +148,7 @@ static uint32_t KiExceptionCodeFromVector(uint32_t vector) {
case 13: return SYS_EXCEPTION_CODE_PRIVILEGED_INSTRUCTION;
case 14: return SYS_EXCEPTION_CODE_ACCESS_VIOLATION;
case 17: return SYS_EXCEPTION_CODE_DATATYPE_MISALIGNMENT;
default: return SYS_EXCEPTION_CODE_NTLITE_UNKNOWN;
default: return SYS_EXCEPTION_CODE_UNKNOWN;
}
}
@@ -519,7 +521,8 @@ uint64_t KeRaiseUserException(const SYS_RAISE_EXCEPTION_REQUEST* request, const
memset(&record, 0, sizeof(record));
record.Size = sizeof(record);
record.Version = SYS_EXCEPTION_ABI_VERSION;
record.Code = request->Code;
record.Code = SYS_EXCEPTION_CODE_USER_RAISED;
record.UserCode = request->UserCode;
record.Flags = SYS_EXCEPTION_FLAG_FIRST_CHANCE;
if (request->Flags & SYS_RAISE_EXCEPTION_FLAG_NONCONTINUABLE)
record.Flags |= SYS_EXCEPTION_FLAG_NONCONTINUABLE;
code/kernel/ke_handle.c
+15 -2
View File
@@ -236,7 +236,8 @@ SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type) {
SYS_PROCESS_RIGHT_TERMINATE | SYS_PROCESS_RIGHT_INSERT_HANDLE |
SYS_PROCESS_RIGHT_BUILD;
case ObjectTypeThread:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_THREAD_RIGHT_RESUME;
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_THREAD_RIGHT_RESUME |
SYS_THREAD_RIGHT_SET_INFORMATION;
case ObjectTypeMutex:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_MUTEX_RIGHT_RELEASE;
case ObjectTypeSemaphore:
@@ -261,10 +262,15 @@ SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type) {
SYS_SOCKET_RIGHT_ACCEPT | SYS_SOCKET_RIGHT_SEND |
SYS_SOCKET_RIGHT_RECEIVE | SYS_SOCKET_RIGHT_SHUTDOWN |
SYS_SOCKET_RIGHT_POLL | SYS_SOCKET_RIGHT_SET_OPTION;
case ObjectTypePipeEndpoint:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT |
SYS_PIPE_RIGHT_READ | SYS_PIPE_RIGHT_WRITE | SYS_PIPE_RIGHT_POLL;
case ObjectTypeFileMapping:
return management | SYS_RIGHT_QUERY | SYS_FILE_MAPPING_RIGHT_MAP;
case ObjectTypeSurface:
return management | SYS_RIGHT_QUERY | SYS_SURFACE_RIGHT_MAP | SYS_SURFACE_RIGHT_PRESENT;
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT |
SYS_SURFACE_RIGHT_MAP_READ | SYS_SURFACE_RIGHT_MAP_WRITE |
SYS_SURFACE_RIGHT_COMMIT;
case ObjectTypeInputQueue:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_INPUT_QUEUE_RIGHT_READ;
case ObjectTypeTimer:
@@ -289,6 +295,13 @@ SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type) {
return management | SYS_RIGHT_QUERY |
SYS_VOLUME_MANAGER_RIGHT_ENUMERATE | SYS_VOLUME_MANAGER_RIGHT_OPEN |
SYS_VOLUME_MANAGER_RIGHT_WAIT;
case ObjectTypeInputManager:
return management | SYS_RIGHT_QUERY | SYS_INPUT_MANAGER_RIGHT_READ |
SYS_INPUT_MANAGER_RIGHT_CONFIGURE;
case ObjectTypeDisplayManager:
return management | SYS_RIGHT_QUERY | SYS_DISPLAY_MANAGER_RIGHT_ENUMERATE |
SYS_DISPLAY_MANAGER_RIGHT_MODESET | SYS_DISPLAY_MANAGER_RIGHT_PRESENT |
SYS_DISPLAY_MANAGER_RIGHT_CURSOR | SYS_DISPLAY_MANAGER_RIGHT_CONSOLE;
case ObjectTypeDevice:
case ObjectTypeDriver:
case ObjectTypeUsbDevice:
code/kernel/ke_input.c
+97 -2
View File
@@ -19,6 +19,8 @@
#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u)
#define MOUSE_QUEUE_SIZE 128u
#define MOUSE_QUEUE_MASK (MOUSE_QUEUE_SIZE - 1u)
#define MANAGER_QUEUE_SIZE 256u
#define MANAGER_QUEUE_MASK (MANAGER_QUEUE_SIZE - 1u)
#define MAX_INPUT_KEYS 256u
#define KEY_REPEAT_DELAY_US 500000ull
#define KEY_REPEAT_PERIOD_US 33000ull
@@ -30,6 +32,7 @@ typedef struct KEY_CHAR_PAIR {
} KEY_CHAR_PAIR;
static KE_INPUT_CONTEXT gDefaultInputContext;
static KE_INPUT_MANAGER gInputManager;
static KE_ID_ALLOCATOR gInputDeviceIds = KE_ID_ALLOCATOR_INIT(1u);
static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key);
@@ -64,6 +67,25 @@ static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Updates the waitable manager event to match queued physical input.
static void KiInputRefreshManagerEvent(KE_INPUT_CONTEXT* context) {
if (!context->ManagerWaitEvent) return;
if (context->ManagerHead != context->ManagerTail) KeSignalEvent(context->ManagerWaitEvent);
else KeClearEvent(context->ManagerWaitEvent);
}
// Queues one normalized physical event for the delegated input manager.
static void KiInputQueueManagerEvent(KE_INPUT_CONTEXT* context, const KE_INPUT_MANAGER_EVENT* event) {
if (!context || !event) return;
int next = (context->ManagerTail + 1) & (int)MANAGER_QUEUE_MASK;
if (next == context->ManagerHead) return;
KE_INPUT_MANAGER_EVENT queued = *event;
queued.Sequence = (uint64_t)_InterlockedIncrement64(&context->ManagerSequence);
context->ManagerQueue[context->ManagerTail] = queued;
context->ManagerTail = next;
KiInputRefreshManagerEvent(context);
}
// Updates the waitable keyboard event to match queued keyboard input.
static void KiInputRefreshKeyboardEvent(KE_INPUT_CONTEXT* context) {
if (!context->WaitEvent) return;
@@ -208,6 +230,7 @@ static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key) {
// Implements the input Initialize operation.
void KeInputInitialize(const HAL_API* hal) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
ObInitializeObjectHeader(&gInputManager.Header, ObjectTypeInputManager);
context->Hal = hal;
memset((void*)context->KeyDown, 0, sizeof(context->KeyDown));
context->Modifiers = 0;
@@ -216,6 +239,8 @@ void KeInputInitialize(const HAL_API* hal) {
context->RepeatKey = INPUT_KEY_NONE;
context->RepeatNextTick = 0;
context->MouseHead = context->MouseTail = 0;
context->ManagerHead = context->ManagerTail = 0;
context->ManagerSequence = 0;
context->MouseBoundsWidth = 1920;
context->MouseBoundsHeight = 1080;
context->MouseWheel = 0;
@@ -227,7 +252,13 @@ void KeInputInitialize(const HAL_API* hal) {
if (!KSUCCESS(s))
DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
}
if (!context->ManagerWaitEvent) {
KSTATUS s = KeEventCreate(EventManualReset, 0, &context->ManagerWaitEvent);
if (!KSUCCESS(s))
DbgPrint("Input: manager wait event create failed st=0x%x\n", s);
}
KiInputRefreshKeyboardEvent(context);
KiInputRefreshManagerEvent(context);
int w = 1920, h = 1080;
DISPLAY_MODE mode;
@@ -243,6 +274,48 @@ void KeInputInitialize(const HAL_API* hal) {
DbgPrint("Input: kernel input core initialized (%dx%d)\n", w, h);
}
// References the singleton input-manager authority object used for bootstrap delegation.
KSTATUS KeInputReferenceManager(KE_INPUT_MANAGER** managerOut) {
if (!managerOut) return NTLSTATUS_INVALID_PARAMETER;
*managerOut = &gInputManager;
ObReferenceObject(&gInputManager.Header);
return NTLSTATUS_SUCCESS;
}
// Reads one normalized physical event from the singleton manager stream.
KSTATUS KeInputManagerReadEvent(KE_INPUT_MANAGER* manager, uint64_t timeoutMs, SYS_INPUT_MANAGER_EVENT* eventOut) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!manager || manager != &gInputManager || !eventOut)
return NTLSTATUS_INVALID_PARAMETER;
for (;;) {
if (context->ManagerHead != context->ManagerTail) {
KE_INPUT_MANAGER_EVENT event = context->ManagerQueue[context->ManagerHead];
context->ManagerHead = (context->ManagerHead + 1) & (int)MANAGER_QUEUE_MASK;
KiInputRefreshManagerEvent(context);
memset(eventOut, 0, sizeof(*eventOut));
eventOut->Size = sizeof(*eventOut);
eventOut->Type = event.Type;
eventOut->Sequence = event.Sequence;
if (event.Type == SYS_INPUT_MANAGER_EVENT_KEY)
eventOut->Data.Key = event.Data.Key;
else if (event.Type == SYS_INPUT_MANAGER_EVENT_MOUSE)
eventOut->Data.Mouse = event.Data.Mouse;
else
return NTLSTATUS_FAILURE;
return NTLSTATUS_SUCCESS;
}
if (timeoutMs == 0) return NTLSTATUS_WOULD_BLOCK;
if (!context->ManagerWaitEvent) return NTLSTATUS_FAILURE;
EVENT* waitEvent = context->ManagerWaitEvent;
uint32_t selected = 0;
KSTATUS status = KeWaitForMultipleEvents(&waitEvent, 1, 0, timeoutMs, &selected);
if (!KSUCCESS(status)) return status;
}
}
// Implements the input Register Device operation.
__declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t flags, uint32_t* outDeviceId) {
(void)flags;
@@ -261,6 +334,12 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
INPUT_KEY key = event->Key;
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return;
KE_INPUT_MANAGER_EVENT managerEvent;
memset(&managerEvent, 0, sizeof(managerEvent));
managerEvent.Type = SYS_INPUT_MANAGER_EVENT_KEY;
managerEvent.Data.Key = *event;
KiInputQueueManagerEvent(context, &managerEvent);
bool wasDown = context->KeyDown[key] != 0;
bool pressed = event->Pressed;
@@ -302,7 +381,6 @@ static bool KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int nx, int ny
context->MouseX = nx;
context->MouseY = ny;
context->MouseButtons = buttons;
KeDisplaySetCursorPosition(nx, ny);
return changed;
}
@@ -311,6 +389,12 @@ __declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
KE_INPUT_MANAGER_EVENT managerEvent;
memset(&managerEvent, 0, sizeof(managerEvent));
managerEvent.Type = SYS_INPUT_MANAGER_EVENT_MOUSE;
managerEvent.Data.Mouse = *event;
KiInputQueueManagerEvent(context, &managerEvent);
int oldX = context->MouseX;
int oldY = context->MouseY;
int nx = oldX + event->dx;
@@ -337,6 +421,18 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
int oldX = context->MouseX;
int oldY = context->MouseY;
INPUT_MOUSE_EVENT managerMouse;
managerMouse.dx = event->x - oldX;
managerMouse.dy = event->y - oldY;
managerMouse.wheel = event->wheel;
managerMouse.buttons = event->buttons;
KE_INPUT_MANAGER_EVENT managerEvent;
memset(&managerEvent, 0, sizeof(managerEvent));
managerEvent.Type = SYS_INPUT_MANAGER_EVENT_MOUSE;
managerEvent.Data.Mouse = managerMouse;
KiInputQueueManagerEvent(context, &managerEvent);
bool changed = KiInputCommitMousePosition(context, event->x, event->y, event->buttons);
if (event->wheel != 0) {
context->MouseWheel += event->wheel;
@@ -539,7 +635,6 @@ void KeInputMouseSetPosition(int x, int y) {
bool changed = (x != context->MouseX) || (y != context->MouseY);
context->MouseX = x;
context->MouseY = y;
KeDisplaySetCursorPosition(x, y);
if (changed) KiInputBumpGeneration(context);
}
code/kernel/ke_irq.c
+9 -1
View File
@@ -134,6 +134,7 @@ void KiExitInterrupt(void) {
if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
THREAD* current = cpu->CurrentThread;
if (current && !current->IsIdleThread && current->Owner &&
!KeThreadExitInProgress(current) &&
KeIsProcessTerminating(current->Owner) &&
!KeThreadTerminationDeferred(current)) {
KeExitThread(current, current->Owner->ExitCode);
@@ -141,7 +142,14 @@ void KiExitInterrupt(void) {
}
KiDrainDpcQueue();
if (KeGetCurrentIrql() == PASSIVE_LEVEL)
// KeExitThread owns a terminal stack/rundown interval. A timer or IPI
// may interrupt it, but switching away here would abandon teardown
// before the process ExitEvent is necessarily published. The explicit
// KeYield at the end of KeExitThread performs the one terminal switch.
current = cpu->CurrentThread;
if (KeGetCurrentIrql() == PASSIVE_LEVEL &&
(!current || !KeThreadExitInProgress(current)))
KiScheduleFromInterrupt();
}
}
code/kernel/ke_irql.c
+3 -1
View File
@@ -4,6 +4,7 @@
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_apc.h>
#include <intrin.h>
// Implements the irql Name operation.
__declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
@@ -60,7 +61,8 @@ __declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
cpu->CurrentIrql = oldIrql;
if (oldIrql == PASSIVE_LEVEL && currentIrql != PASSIVE_LEVEL)
if (oldIrql == PASSIVE_LEVEL && currentIrql != PASSIVE_LEVEL &&
(__readeflags() & 0x200u) != 0 && cpu->InterruptDepth == 0)
KiDeliverKernelApcs();
}
code/kernel/ke_process.c
+2
View File
@@ -229,6 +229,7 @@ static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
case ObjectTypeSurface: return SYS_THREAD_WAIT_SURFACE;
case ObjectTypeTimer: return SYS_THREAD_WAIT_TIMER;
case ObjectTypeDisplayClock: return SYS_THREAD_WAIT_DISPLAY_CLOCK;
case ObjectTypePipeEndpoint: return SYS_THREAD_WAIT_PIPE;
default: return SYS_THREAD_WAIT_UNKNOWN;
}
}
@@ -273,6 +274,7 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
: SYS_THREAD_WAIT_NONE;
info->WaitStatus = (uint32_t)snapshot.WaitStatus;
info->KernelApcDisable = thread->KernelApcDisable;
(void)KeQueryThreadName(thread, info->Name, sizeof(info->Name));
if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT;
if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE;
code/kernel/ke_sched.c
+13 -8
View File
@@ -794,6 +794,9 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
idle->Teb = (void*)0;
idle->TebSize = 0;
idle->ExitEvent = (EVENT*)0;
idle->NameLock = (SPINLOCK)SPINLOCK_INIT;
memset(idle->Name, 0, sizeof(idle->Name));
memcpy(idle->Name, "idle", sizeof("idle"));
KiInitializeApcState(idle);
idle->TrapFrame = (KTRAP_FRAME*)0;
LIST_INIT(&idle->ProcessLink);
@@ -1071,14 +1074,11 @@ static void KiSchedule(void) {
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiArmPostSwitchUnlock(cur, flags);
// The currently running thread is not present in a run queue. If it is
// still runnable and nobody else is ready, keep executing it. Otherwise a
// lone CPU-bound thread gets alternated with the idle thread on every timer
// tick, effectively turning a periodic preemption check into a 50% throttle.
if (cur != idle && KiThreadIsKilledLocked(cur)) {
cur->State = ThreadDead;
}
// The scheduler never converts process termination directly into
// ThreadDead. Terminal state owns rundown/lifetime publication and is
// entered only through KeExitThread at a safe execution boundary.
// If the current thread is still runnable and nobody else is ready, keep
// executing it rather than alternating with idle on every timer tick.
if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableForCpuLocked(cpu)) {
if (cur->QuantumRemaining == 0)
KiRefillQuantumLocked(cur);
@@ -1379,6 +1379,11 @@ static int KiRunLocalSchedulerTickLocked(KCPU* cpu, THREAD* cur, THREAD* idle) {
if (cur == idle) {
if (KiAnyRunnableForCpuLocked(cpu))
requestReschedule = 1;
} else if (KeThreadExitInProgress(cur)) {
// Terminal teardown is interruptible but not preemptible. Its explicit
// final KeYield performs the permanent switch after completion state is
// published.
requestReschedule = 0;
} else if (KiThreadIsKilledLocked(cur)) {
requestReschedule = 1;
} else {
code/kernel/ke_selftest.c
+69 -442
View File
@@ -3,6 +3,7 @@
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_apc.h>
@@ -28,460 +29,83 @@
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>
// Kernel APC self-test.
typedef struct KI_APC_SELFTEST_STATE {
EVENT* ReadyEvent;
EVENT* GoEvent;
EVENT* DoneEvent;
EVENT* FinishedEvent;
THREAD* WorkerThread;
volatile int SelfDelivered;
volatile int DeferredDelivered;
volatile int CrossDelivered;
} KI_APC_SELFTEST_STATE;
static void KiApcSelfTestKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("APC selftest: kernel routine apc=0x%016llX normal=0x%016llX ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)apc,
(uint64_t)*normalRoutine,
(uint64_t)*normalContext,
(uint64_t)*systemArgument1,
(uint64_t)*systemArgument2,
KeGetCurrentIrql());
}
static void KiApcSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
volatile int* counter = (volatile int*)normalContext;
DbgPrint("APC selftest: normal routine ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)normalContext,
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
// Focused APC wait-commit race self-test. User-mode APC return and async I/O
// behavior are covered by executable smoke tests; keep only the scheduler race
// that cannot be constructed through the public ABI.
// Verifies that one APC normal routine executes in interruptible PASSIVE thread context.
static void KiAssertApcNormalContext(const char* name) {
KCPU* cpu = KeGetCurrentCpu();
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: normal routine ran above PASSIVE_LEVEL\n");
(*counter)++;
KePanic(KE_PANIC_CODE_SELFTEST, "%s ran above PASSIVE_LEVEL\n", name);
if ((__readeflags() & 0x200u) == 0)
KePanic(KE_PANIC_CODE_SELFTEST, "%s ran with interrupts disabled\n", name);
if (!cpu || cpu->InterruptDepth != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "%s ran inside interrupt context depth=%u\n",
name, cpu ? cpu->InterruptDepth : 0xFFFFFFFFu);
}
static void KiApcSelfTestCrossThreadRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext;
DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread APC ran above PASSIVE_LEVEL\n");
st->CrossDelivered++;
KeSignalEvent(st->DoneEvent);
}
// Implements the apc Self Test Worker operation.
static void KiApcSelfTestWorker(void* arg) {
KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)arg;
DbgPrint("APC selftest: worker started, signaling ready\n");
KeSignalEvent(st->ReadyEvent);
DbgPrint("APC selftest: worker waiting for go event\n");
KSTATUS s = KeWaitForEvent(st->GoEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker wait failed status=%d\n", s);
// KeCommitBlockThread(..., alertable) should deliver pending kernel APCs before this
// wait returns. If that hook is broken, DoneEvent will never be signaled.
DbgPrint("APC selftest: worker resumed after go event, cross=%d\n",
st->CrossDelivered);
if (st->CrossDelivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker resumed but APC was not delivered\n");
KeSignalEvent(st->FinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
typedef struct KI_APC_ALERTABLE_SELFTEST_STATE {
EVENT* ReadyEvent;
EVENT* WaitEvent;
EVENT* FinishedEvent;
THREAD* WorkerThread;
volatile int ApcDelivered;
volatile KSTATUS WaitStatus;
} KI_APC_ALERTABLE_SELFTEST_STATE;
static void KiApcAlertableSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_APC_ALERTABLE_SELFTEST_STATE* st =
(KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext;
DbgPrint("APC selftest: alertable APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC ran above PASSIVE_LEVEL\n");
st->ApcDelivered++;
}
// Implements the apc Alertable Self Test Worker operation.
static void KiApcAlertableSelfTestWorker(void* arg) {
KI_APC_ALERTABLE_SELFTEST_STATE* st =
(KI_APC_ALERTABLE_SELFTEST_STATE*)arg;
DbgPrint("APC selftest: alertable worker ready, entering alertable wait\n");
KeSignalEvent(st->ReadyEvent);
KSTATUS s = KeWaitForEvent(st->WaitEvent, 1);
st->WaitStatus = s;
DbgPrint("APC selftest: alertable wait returned status=%u delivered=%d\n",
s, st->ApcDelivered);
if (s != NTLSTATUS_KERNEL_APC)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable wait returned wrong status=%u\n", s);
if (st->ApcDelivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC delivery count wrong=%d\n",
st->ApcDelivered);
KeSignalEvent(st->FinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
// Runs kernel Apc Self Test.
static void KiRunKernelApcSelfTest(void) {
DbgPrint("APC selftest: begin\n");
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: called above PASSIVE_LEVEL\n");
THREAD* self = KeGetCurrentThread();
if (!self)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: no current thread\n");
// Test 1: queue APC to current thread at PASSIVE_LEVEL.
{
KAPC apc;
volatile int count = 0;
KeInitializeApc(&apc,
self,
KiApcSelfTestKernelRoutine,
KiApcSelfTestNormalRoutine,
(void*)&count);
if (!KeInsertQueueApc(&apc, (void*)0x11111111, (void*)0x22222222))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: self queue failed\n");
if (count != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: self APC was not delivered immediately, count=%d\n", count);
DbgPrint("APC selftest: self APC passed\n");
}
// Test 2: disable kernel APCs, queue, then re-enable.
{
KAPC apc;
volatile int count = 0;
KeInitializeApc(&apc,
self,
KiApcSelfTestKernelRoutine,
KiApcSelfTestNormalRoutine,
(void*)&count);
KeEnterCriticalRegion();
if (!KeInsertQueueApc(&apc, (void*)0x33333333, (void*)0x44444444))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: deferred queue failed\n");
if (count != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: APC delivered while critical region held, count=%d\n", count);
KeLeaveCriticalRegion();
if (count != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: deferred APC was not delivered after leave, count=%d\n", count);
DbgPrint("APC selftest: critical-region deferral passed\n");
}
// Test 3: queue APC to another blocked kernel thread, then wake it.
{
KI_APC_SELFTEST_STATE st;
KAPC apc;
memset(&st, 0, sizeof(st));
KSTATUS s;
s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: ReadyEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.GoEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: GoEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.DoneEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: DoneEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: FinishedEvent create failed %d\n", s);
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker,
(uintptr_t)&st,
&workerParameters,
&st.WorkerThread);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker create failed %d\n", s);
s = KeWaitForEvent(st.ReadyEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: ready wait failed %d\n", s);
DbgPrint("APC selftest: worker ready, queueing cross-thread APC\n");
KeInitializeApc(&apc,
st.WorkerThread,
KiApcSelfTestKernelRoutine,
KiApcSelfTestCrossThreadRoutine,
(void*)&st);
if (!KeInsertQueueApc(&apc, (void*)0x55555555, (void*)0x66666666))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread queue failed\n");
if (st.CrossDelivered != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread APC delivered before target resumed\n");
KeSignalEvent(st.GoEvent);
s = KeWaitForEvent(st.DoneEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: done wait failed %d\n", s);
if (st.CrossDelivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread delivery count wrong, count=%d\n",
st.CrossDelivered);
// The APC signals DoneEvent before the worker returns from the wait.
// Wait for an explicit worker-owned event before dropping stack-backed
// state and event references. THREAD.ExitEvent is not part of this
// kernel-thread contract on all builds.
s = KeWaitForEvent(st.FinishedEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker finished wait failed %d\n", s);
DbgPrint("APC selftest: cross-thread APC passed\n");
ObDereferenceObject(&st.ReadyEvent->Header);
ObDereferenceObject(&st.GoEvent->Header);
ObDereferenceObject(&st.DoneEvent->Header);
ObDereferenceObject(&st.FinishedEvent->Header);
ObDereferenceObject(&st.WorkerThread->Header);
}
// Test 4: queue APC to a thread blocked in an alertable wait.
// The APC itself must wake the waiter; no event signal is sent.
{
KI_APC_ALERTABLE_SELFTEST_STATE st;
KAPC apc;
memset(&st, 0, sizeof(st));
st.WaitStatus = NTLSTATUS_PENDING;
KSTATUS s;
s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert ReadyEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.WaitEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert WaitEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert FinishedEvent create failed %d\n", s);
const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker,
(uintptr_t)&st,
&workerParameters,
&st.WorkerThread);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable worker create failed %d\n", s);
s = KeWaitForEvent(st.ReadyEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert ready wait failed %d\n", s);
DbgPrint("APC selftest: queueing APC to alertable waiter\n");
KeInitializeApc(&apc,
st.WorkerThread,
KiApcSelfTestKernelRoutine,
KiApcAlertableSelfTestNormalRoutine,
(void*)&st);
if (!KeInsertQueueApc(&apc, (void*)(unsigned long long)0x77777777, (void*)(unsigned long long)0x88888888))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC queue failed\n");
s = KeWaitForEvent(st.FinishedEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert finished wait failed %d\n", s);
if (st.WaitStatus != NTLSTATUS_KERNEL_APC || st.ApcDelivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable result bad status=%u delivered=%d\n",
st.WaitStatus, st.ApcDelivered);
DbgPrint("APC selftest: alertable wait passed\n");
ObDereferenceObject(&st.ReadyEvent->Header);
ObDereferenceObject(&st.WaitEvent->Header);
ObDereferenceObject(&st.FinishedEvent->Header);
ObDereferenceObject(&st.WorkerThread->Header);
}
DbgPrint("APC selftest: PASS\n");
}
// IRP + kernel APC self-test.
typedef struct KI_IRP_APC_SELFTEST_STATE {
EVENT* CompletionEvent;
DEVICE_OBJECT* DiskDevice;
IRP Irp;
KAPC Apc;
uint8_t* Buffer;
volatile int ApcDelivered;
volatile KSTATUS ApcStatus;
volatile size_t ApcInformation;
} KI_IRP_APC_SELFTEST_STATE;
static void KiIrpApcSelfTestKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("IRP/APC selftest: kernel routine apc=0x%016llX irp=0x%016llX status=%u irql=%u\n",
(uint64_t)apc,
(uint64_t)*systemArgument1,
(uint32_t)(uintptr_t)*systemArgument2,
KeGetCurrentIrql());
// Leaves the APC's normal routine and arguments unchanged.
static void KiApcWaitRaceKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
(void)apc;
(void)normalRoutine;
(void)normalContext;
(void)systemArgument1;
(void)systemArgument2;
}
static void KiIrpApcSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_IRP_APC_SELFTEST_STATE* st = (KI_IRP_APC_SELFTEST_STATE*)normalContext;
IRP* irp = (IRP*)systemArgument1;
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
DbgPrint("IRP/APC selftest: APC delivered irp=0x%016llX status=%u info=%llu irql=%u\n",
(uint64_t)irp,
status,
(uint64_t)(irp ? irp->Information : 0),
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: normal routine ran above PASSIVE_LEVEL\n");
if (irp != &st->Irp)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: APC arg1 was not the submitted IRP\n");
st->ApcDelivered++;
st->ApcStatus = status;
st->ApcInformation = irp->Information;
// Records delivery for the prepared-wait race probe.
static void KiApcWaitRaceNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
(void)systemArgument1;
(void)systemArgument2;
KiAssertApcNormalContext("APC wait-race normal routine");
(*(volatile int*)normalContext)++;
}
// Runs irp Kernel Apc Self Test.
static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
DbgPrint("IRP/APC selftest: begin\n");
// Reproduces the prepare-to-commit APC race deterministically.
static void KiRunApcWaitRaceSelfTest(void) {
DbgPrint("APC wait-race selftest: begin\n");
(void)hal;
if (!IoDiskDevicePresent(0)) {
DbgPrint("IRP/APC selftest: SKIP, drive 0 not present\n");
return;
}
THREAD* self = KeGetCurrentThread();
if (!self || KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: invalid execution context\n");
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: called above PASSIVE_LEVEL\n");
EVENT* waitEvent = (EVENT*)0;
KAPC apc;
volatile int delivered = 0;
KSTATUS status = KeEventCreate(EventAutoReset, 0, &waitEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: event create failed %u\n", status);
KI_IRP_APC_SELFTEST_STATE st;
memset(&st, 0, sizeof(st));
KeInitializeApc(&apc, self, KiApcWaitRaceKernelRoutine,
KiApcWaitRaceNormalRoutine, (void*)&delivered);
KSTATUS s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: completion event create failed %u\n", s);
uintptr_t flags = KeAcquireSpinlock(&waitEvent->Lock);
if (!KeInsertQueueApc(&apc, (void*)(uintptr_t)0x99999999u,
(void*)(uintptr_t)0xAAAAAAAAu))
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: queue failed\n");
if (delivered != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: delivered above PASSIVE_LEVEL\n");
st.Buffer = (uint8_t*)MmAllocatePool(512);
if (!st.Buffer)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: no buffer\n");
memset(st.Buffer, 0xCC, 512);
KePrepareBlockThread(self, &waitEvent->Header, 1);
KeListInsertTailChecked(&waitEvent->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &waitEvent->WaitList);
KeReleaseSpinlock(&waitEvent->Lock, flags);
KeInitializeApc(&st.Apc,
KeGetCurrentThread(),
KiIrpApcSelfTestKernelRoutine,
KiIrpApcSelfTestNormalRoutine,
(void*)&st);
status = KeCommitBlockThread(self, 1);
if (status != NTLSTATUS_KERNEL_APC || delivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"APC wait-race selftest: bad result status=%u delivered=%d\n",
status, delivered);
st.DiskDevice = IoReferenceDiskDevice(0);
if (!st.DiskDevice)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: drive 0 disappeared before submit\n");
IoBuildReadWriteIrp(&st.Irp, IRP_MJ_READ, 0, 512, st.Buffer);
st.Irp.CompletionEvent = st.CompletionEvent;
st.Irp.CompletionApc = &st.Apc;
st.Irp.CompletionFn = (IRP_COMPLETION_FN)0;
st.Irp.CompletionCtx = (void*)0;
DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n");
s = IoSubmitIrp(st.DiskDevice, &st.Irp);
if (s != NTLSTATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: IoSubmitIrp returned %u\n", s);
s = KeWaitForEvent(st.CompletionEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: completion wait failed %u\n", s);
// The wait return path should have delivered the pending kernel APC.
if (st.ApcDelivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: APC was not delivered after I/O wait, count=%d\n",
st.ApcDelivered);
if (!KSUCCESS(st.Irp.Status) || !KSUCCESS(st.ApcStatus))
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: I/O failed irp=%u apc=%u info=%llu\n",
st.Irp.Status, st.ApcStatus, (uint64_t)st.Irp.Information);
if (st.Irp.Information != 512 || st.ApcInformation != 512)
KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: wrong byte count irp=%llu apc=%llu\n",
(uint64_t)st.Irp.Information, (uint64_t)st.ApcInformation);
DbgPrint("IRP/APC selftest: first bytes %02X %02X %02X %02X, tail %02X %02X\n",
st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
st.Buffer[510], st.Buffer[511]);
MmFreePool(st.Buffer);
if (st.DiskDevice) IoDereferenceDeviceObject(st.DiskDevice);
ObDereferenceObject(&st.CompletionEvent->Header);
DbgPrint("IRP/APC selftest: PASS\n");
ObDereferenceObject(&waitEvent->Header);
DbgPrint("APC wait-race selftest: PASS\n");
}
// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
@@ -1510,7 +1134,8 @@ static void KiRunSynchronizationTimeoutSelfTest(void) {
"SYNC/TIMEOUT selftest: owner acquire failed status=%u\n", status);
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_REALTIME(12);
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_REALTIME(12, "selftest.sync.timeout");
status = KeCreateKernelThread((uintptr_t)KiSynchronizationTimeoutWorker,
(uintptr_t)&state,
&parameters,
@@ -1691,7 +1316,8 @@ static void KiRunRealtimeSchedulerSelfTest(void) {
"SCHED selftest: owner acquire failed status=%u\n", status);
THREAD* chainWorker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS chainParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
const KE_THREAD_CREATE_PARAMETERS chainParameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "selftest.sched.chain");
status = KeCreateKernelThread((uintptr_t)KiSchedulerChainWorker,
(uintptr_t)&state,
&chainParameters,
@@ -1713,7 +1339,8 @@ static void KiRunRealtimeSchedulerSelfTest(void) {
"SCHED selftest: mutex cycle was not rejected status=%u\n", status);
THREAD* realtimeWorker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS realtimeParameters = KE_THREAD_CREATE_REALTIME(16);
const KE_THREAD_CREATE_PARAMETERS realtimeParameters =
KE_THREAD_CREATE_NAMED_REALTIME(16, "selftest.sched.rt");
status = KeCreateKernelThread((uintptr_t)KiSchedulerRealtimeWorker,
(uintptr_t)&state,
&realtimeParameters,
@@ -1895,10 +1522,11 @@ void KiRunEarlyKernelSelfTests(void) {
// Runs kernel Self Tests.
void KiRunKernelSelfTests(const HAL_API* hal) {
(void)hal;
DbgPrint("Kernel selftests: begin\n");
KiRunSynchronizationTimeoutSelfTest();
KiRunRealtimeSchedulerSelfTest();
KiRunKernelApcSelfTest();
KiRunApcWaitRaceSelfTest();
KiRunPfnDatabaseSelfTest();
KiRunDmaSelfTest();
KiRunMemoryMappingSelfTest();
@@ -1907,7 +1535,6 @@ void KiRunKernelSelfTests(const HAL_API* hal) {
KiRunVadSwapSelfTest();
KiRunStringViewSelfTest();
KiRunVolumeRelativePathSelfTest();
KiRunIrpKernelApcSelfTest(hal);
KiRunAsyncFileIoSelfTest();
KiRunTrapFrameSelfTest();
DbgPrint("Kernel selftests: PASS\n");
code/kernel/ke_smp.c
+31 -15
View File
@@ -28,8 +28,8 @@
#define AP_TRAMPOLINE_FALLBACK_MIN 0x00010000ULL
#define AP_TRAMPOLINE_BOOT_OFF 0x0800u
#define AP_STACK_PAGES 8u
#define AP_START_TIMEOUT_SPINS 20000000u
#define AP_SCHED_TIMEOUT_SPINS 20000000u
#define AP_START_TIMEOUT_US 1000000ull
#define AP_SCHED_TIMEOUT_US 1000000ull
#define AP_CR4_PAE 0x00000020u
#define AP_CR4_OSFXSR 0x00000200u
@@ -50,6 +50,18 @@
#define AP_STATE_KERNEL_PARKED 4u
#define AP_STATE_FAILED 0xFFFFFFFFu
// Returns the HAL monotonic counter in microseconds for pre-scheduler hardware deadlines.
static uint64_t KiSmpQueryTicks(void) {
const HAL_API* hal = KeGetHal();
return hal && hal->Timer.QueryTicks ? hal->Timer.QueryTicks() : 0;
}
// Returns a saturated deadline from the current HAL monotonic counter.
static uint64_t KiSmpDeadlineAfter(uint64_t timeoutUs) {
uint64_t now = KiSmpQueryTicks();
return now > UINT64_MAX - timeoutUs ? UINT64_MAX : now + timeoutUs;
}
#define AP_STAGE_NONE 0x00000000u
#define AP_STAGE_LONG64 0x4C4D3634u // 'LM64'
#define AP_STAGE_ENTRY 0x454E5452u // 'ENTR'
@@ -748,16 +760,16 @@ KSTATUS KeStartApplicationProcessors(void) {
}
uint64_t lastHeartbeat = gApBootBlock->Heartbeat;
uint64_t startDeadline = KiSmpDeadlineAfter(AP_START_TIMEOUT_US);
int parked = 0;
for (uint32_t spin = 0; spin < AP_START_TIMEOUT_SPINS; spin++) {
for (;;) {
if (gApBootBlock->State == AP_STATE_KERNEL_PARKED) {
parked = 1;
break;
}
if ((spin & 0xFFFFu) == 0) {
uint64_t hb = gApBootBlock->Heartbeat;
lastHeartbeat = hb;
}
lastHeartbeat = gApBootBlock->Heartbeat;
if (KiSmpQueryTicks() >= startDeadline)
break;
_mm_pause();
}
@@ -823,11 +835,14 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
KeSendProcessorInterrupt(cpu->HardwareId, KE_VECTOR_RESCHEDULE_IPI);
int active = 0;
for (uint32_t spin = 0; spin < AP_SCHED_TIMEOUT_SPINS; spin++) {
uint64_t schedulerDeadline = KiSmpDeadlineAfter(AP_SCHED_TIMEOUT_US);
for (;;) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuSchedulerActive) {
active = 1;
break;
}
if (KiSmpQueryTicks() >= schedulerDeadline)
break;
_mm_pause();
}
@@ -849,7 +864,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
}
#define SMP_AP_WORKER_ROUNDS 64u
#define SMP_AP_WORKER_TIMEOUT_SPINS 4000000u
#define SMP_AP_WORKER_TIMEOUT_US 2000000ull
#define SMP_AP_TEST_MAGIC 0x524B575041504D53ULL // 'SMPAPWKR'
typedef struct SMP_AP_WORK_TEST SMP_AP_WORK_TEST;
@@ -933,13 +948,13 @@ static void KiSmpApControlledWorker(void* context) {
// Waits for for Worker Counter.
static int KiWaitForWorkerCounter(volatile long* counter, long expected, const char* what) {
for (uint32_t spin = 0; spin < SMP_AP_WORKER_TIMEOUT_SPINS; spin++) {
uint64_t deadline = KiSmpDeadlineAfter(SMP_AP_WORKER_TIMEOUT_US);
for (;;) {
if (_InterlockedCompareExchange(counter, 0, 0) >= expected)
return 1;
if ((spin & 0x3FFu) == 0)
KeYield();
else
_mm_pause();
if (KiSmpQueryTicks() >= deadline)
break;
KeYield();
}
DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%ld got=%ld\n",
what ? what : "counter", expected, _InterlockedCompareExchange(counter, 0, 0));
@@ -981,7 +996,8 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
cpuIndex, st);
}
KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
KE_THREAD_CREATE_PARAMETERS workerParameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "selftest.smp");
workerParameters.AffinityMask = 1ULL << cpuIndex;
st = KeCreateKernelThread((uintptr_t)KiSmpApControlledWorker,
(uintptr_t)arg,
code/kernel/ke_surface.c
+439
View File
@@ -0,0 +1,439 @@
// Kernel surface storage. This file owns shared backing frames, mappings, commit generations, and privileged presentation.
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#include <kernel/ke_surface.h>
#include <kernel/ke_process.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_event.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_display.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_pool.h>
#include <ntlite/syscall.h>
typedef struct KE_SURFACE {
OBJECT Header;
uint32_t Width;
uint32_t Height;
uint32_t PitchBytes;
uint32_t Format;
uint32_t Flags;
size_t PayloadSize;
size_t PageCount;
uint8_t StatsRegistered;
uint8_t LastDamageValid;
uint16_t Reserved;
uint32_t Reserved2;
uint64_t MapCount;
uint64_t ContentGeneration;
uint64_t CommitCount;
uint64_t CommitRectCount;
uint64_t CommitClippedRectCount;
uint64_t CommitEmptyClipCount;
DISPLAY_RECT LastDamageBounds;
PHYSICAL_ADDRESS* Frames;
MM_KERNEL_VIEW* KernelView;
void* KernelBase;
EVENT* CommitEvent;
MUTEX* Lock;
} KE_SURFACE;
static SPINLOCK gSurfaceStatsLock = SPINLOCK_INIT;
static SYS_SURFACE_STATS gSurfaceStats;
// Multiplies two sizes while rejecting overflow.
static int KiSurfaceMulSize(size_t a, size_t b, size_t* out) {
if (!out || (a != 0 && b > SIZE_MAX / a)) return 0;
*out = a * b;
return 1;
}
// Aligns one size upward while rejecting wraparound.
static size_t KiSurfaceAlignUp(size_t value, size_t alignment) {
if (!alignment || (alignment & (alignment - 1u))) return 0;
size_t mask = alignment - 1u;
if (value > SIZE_MAX - mask) return 0;
return (value + mask) & ~mask;
}
// Converts a generic object reference to a surface only when its type is correct.
static KE_SURFACE* KiSurfaceFromObject(OBJECT* object) {
if (!object || object->Type != ObjectTypeSurface) return (KE_SURFACE*)0;
return CONTAINING_RECORD(object, KE_SURFACE, Header);
}
// Registers a surface in the global telemetry snapshot.
static void KiSurfaceStatsRegister(KE_SURFACE* surface) {
if (!surface || surface->StatsRegistered) return;
uintptr_t irq = KeAcquireSpinlock(&gSurfaceStatsLock);
gSurfaceStats.CreatedSurfaces++;
gSurfaceStats.LiveSurfaces++;
gSurfaceStats.LivePages += surface->PageCount;
gSurfaceStats.LiveBytes += surface->PayloadSize;
surface->StatsRegistered = 1;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
// Removes a surface from the global telemetry snapshot.
static void KiSurfaceStatsUnregister(KE_SURFACE* surface) {
if (!surface || !surface->StatsRegistered) return;
uintptr_t irq = KeAcquireSpinlock(&gSurfaceStatsLock);
gSurfaceStats.DestroyedSurfaces++;
if (gSurfaceStats.LiveSurfaces) gSurfaceStats.LiveSurfaces--;
gSurfaceStats.LivePages = gSurfaceStats.LivePages >= surface->PageCount
? gSurfaceStats.LivePages - surface->PageCount : 0;
gSurfaceStats.LiveBytes = gSurfaceStats.LiveBytes >= surface->PayloadSize
? gSurfaceStats.LiveBytes - surface->PayloadSize : 0;
surface->StatsRegistered = 0;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
// Records one successful surface mapping operation.
static void KiSurfaceStatsRecordMap(KE_SURFACE* surface) {
uintptr_t irq = KeAcquireSpinlock(&gSurfaceStatsLock);
surface->MapCount++;
gSurfaceStats.MapCalls++;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
// Records one commit after rectangle validation and clipping.
static void KiSurfaceStatsRecordCommit(uint64_t inputRects, uint64_t clippedRects, uint64_t emptyClips) {
uintptr_t irq = KeAcquireSpinlock(&gSurfaceStatsLock);
gSurfaceStats.CommitCalls++;
gSurfaceStats.CommitRectInputs += inputRects;
gSurfaceStats.CommitClippedRects += clippedRects;
gSurfaceStats.CommitEmptyClips += emptyClips;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
// Records one presentation performed through DisplayManager authority.
static void KiSurfaceStatsRecordPresent(uint64_t bytesCopied) {
uintptr_t irq = KeAcquireSpinlock(&gSurfaceStatsLock);
gSurfaceStats.PhysicalPresentCalls++;
gSurfaceStats.PhysicalPresentBytesCopied += bytesCopied;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
// Releases surface storage. User VADs retain their own backing-frame references until unmapped.
static void KiSurfaceDestroy(OBJECT* object) {
KE_SURFACE* surface = KiSurfaceFromObject(object);
if (!surface) return;
KiSurfaceStatsUnregister(surface);
if (surface->KernelView) {
MmDestroyKernelView(surface->KernelView);
surface->KernelView = (MM_KERNEL_VIEW*)0;
surface->KernelBase = (void*)0;
}
if (surface->CommitEvent) {
ObDereferenceObject(&surface->CommitEvent->Header);
surface->CommitEvent = (EVENT*)0;
}
if (surface->Lock) {
ObDereferenceObject(&surface->Lock->Header);
surface->Lock = (MUTEX*)0;
}
if (surface->Frames) {
for (size_t i = 0; i < surface->PageCount; ++i)
if (surface->Frames[i]) (void)MmPfnRelease(surface->Frames[i]);
MmFreePool(surface->Frames);
surface->Frames = (PHYSICAL_ADDRESS*)0;
}
}
// Clips one rectangle to the surface extent.
static int KiSurfaceClipRect(const KE_SURFACE* surface, const DISPLAY_RECT* input, DISPLAY_RECT* output) {
if (!surface || !output) return 0;
DISPLAY_RECT r;
if (input) {
r = *input;
} else {
r.X = 0;
r.Y = 0;
r.W = (int32_t)surface->Width;
r.H = (int32_t)surface->Height;
}
if (r.W <= 0 || r.H <= 0) return 0;
int64_t x0 = r.X;
int64_t y0 = r.Y;
int64_t x1 = x0 + r.W;
int64_t y1 = y0 + r.H;
if (x0 < 0) x0 = 0;
if (y0 < 0) y0 = 0;
if (x1 > surface->Width) x1 = surface->Width;
if (y1 > surface->Height) y1 = surface->Height;
if (x0 >= x1 || y0 >= y1) return 0;
output->X = (int32_t)x0;
output->Y = (int32_t)y0;
output->W = (int32_t)(x1 - x0);
output->H = (int32_t)(y1 - y0);
return 1;
}
// Expands one damage bound to include another clipped rectangle.
static void KiSurfaceUnionRect(DISPLAY_RECT* bounds, const DISPLAY_RECT* rect) {
int32_t x0 = bounds->X < rect->X ? bounds->X : rect->X;
int32_t y0 = bounds->Y < rect->Y ? bounds->Y : rect->Y;
int32_t bx1 = bounds->X + bounds->W;
int32_t by1 = bounds->Y + bounds->H;
int32_t rx1 = rect->X + rect->W;
int32_t ry1 = rect->Y + rect->H;
int32_t x1 = bx1 > rx1 ? bx1 : rx1;
int32_t y1 = by1 > ry1 ? by1 : ry1;
bounds->X = x0;
bounds->Y = y0;
bounds->W = x1 - x0;
bounds->H = y1 - y0;
}
// Clips one kernel-captured rectangle list. A zero count means the whole surface.
static KSTATUS KiSurfaceCaptureRects(KE_SURFACE* surface, const DISPLAY_RECT* rects, uint32_t rectCount,
DISPLAY_RECT* clippedOut, uint32_t* clippedCountOut,
uint64_t* emptyOut) {
if (!surface || !clippedOut || !clippedCountOut || !emptyOut)
return NTLSTATUS_INVALID_PARAMETER;
if (rectCount > SYS_SURFACE_MAX_RECTS || (rectCount && !rects))
return NTLSTATUS_INVALID_PARAMETER;
DISPLAY_RECT whole = {0, 0, (int32_t)surface->Width, (int32_t)surface->Height};
if (!rectCount) {
rects = &whole;
rectCount = 1;
}
uint32_t clippedCount = 0;
uint64_t empty = 0;
for (uint32_t i = 0; i < rectCount; ++i) {
DISPLAY_RECT clipped;
if (!KiSurfaceClipRect(surface, &rects[i], &clipped)) {
empty++;
continue;
}
clippedOut[clippedCount++] = clipped;
}
*clippedCountOut = clippedCount;
*emptyOut = empty;
return NTLSTATUS_SUCCESS;
}
// Creates one surface storage object and returns its initial object reference.
KSTATUS KeSurfaceCreate(uint32_t width, uint32_t height, uint32_t format, uint32_t flags,
OBJECT** surfaceOut) {
if (!surfaceOut) return NTLSTATUS_INVALID_PARAMETER;
*surfaceOut = (OBJECT*)0;
if (!width || !height || width > SYS_SURFACE_MAX_DIMENSION || height > SYS_SURFACE_MAX_DIMENSION)
return NTLSTATUS_INVALID_PARAMETER;
if (format != SYS_SURFACE_FORMAT_XRGB8888 || flags != SYS_SURFACE_FLAG_NONE)
return NTLSTATUS_NOT_SUPPORTED;
size_t rowBytes = 0;
if (!KiSurfaceMulSize((size_t)width, sizeof(uint32_t), &rowBytes))
return NTLSTATUS_INVALID_PARAMETER;
size_t pitchBytes = KiSurfaceAlignUp(rowBytes, 64u);
if (!pitchBytes || pitchBytes < rowBytes) return NTLSTATUS_INVALID_PARAMETER;
size_t payloadSize = 0;
if (!KiSurfaceMulSize(pitchBytes, (size_t)height, &payloadSize) || payloadSize > SYS_SURFACE_MAX_BYTES)
return NTLSTATUS_INVALID_PARAMETER;
size_t pageCount = (payloadSize + PAGE_SIZE - 1u) / PAGE_SIZE;
if (!pageCount || pageCount > SIZE_MAX / sizeof(PHYSICAL_ADDRESS))
return NTLSTATUS_INVALID_PARAMETER;
KE_SURFACE* surface = (KE_SURFACE*)ObAllocateObject(ObjectTypeSurface, sizeof(KE_SURFACE));
if (!surface) return NTLSTATUS_NO_MEMORY;
surface->Header.Destroy = KiSurfaceDestroy;
surface->Width = width;
surface->Height = height;
surface->PitchBytes = (uint32_t)pitchBytes;
surface->Format = format;
surface->Flags = flags;
surface->PayloadSize = payloadSize;
surface->PageCount = pageCount;
surface->Frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS),
MM_POOL_TAG('s', 'f', 'r', 'm'));
if (surface->Frames)
memset(surface->Frames, 0, pageCount * sizeof(PHYSICAL_ADDRESS));
KSTATUS status = KeMutexCreate(&surface->Lock);
if (KSUCCESS(status)) status = KeEventCreate(EventAutoReset, 0, &surface->CommitEvent);
if (!surface->Frames || !KSUCCESS(status)) {
ObDereferenceObject(&surface->Header);
return !surface->Frames ? NTLSTATUS_NO_MEMORY : status;
}
for (size_t i = 0; i < pageCount; ++i) {
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseSurface);
if (!frame) {
ObDereferenceObject(&surface->Header);
return NTLSTATUS_NO_MEMORY;
}
surface->Frames[i] = frame;
void* kva = MmPfnToVirtual(frame);
if (kva) memset(kva, 0, PAGE_SIZE);
}
status = MmCreateKernelViewFromFrames(surface->Frames, surface->PageCount, &surface->KernelView);
if (!KSUCCESS(status)) {
ObDereferenceObject(&surface->Header);
return status;
}
surface->KernelBase = MmKernelViewBase(surface->KernelView);
KiSurfaceStatsRegister(surface);
*surfaceOut = &surface->Header;
return NTLSTATUS_SUCCESS;
}
// Maps one surface into the calling process's VAD namespace with attenuated page permissions.
KSTATUS KeSurfaceMap(PROCESS* process, OBJECT* object, uint32_t flags,
void** baseOut, uint32_t* pitchBytesOut) {
KE_SURFACE* surface = KiSurfaceFromObject(object);
if (!process || !surface || !baseOut || !pitchBytesOut || !(flags & SYS_SURFACE_MAP_READ) ||
(flags & ~(SYS_SURFACE_MAP_READ | SYS_SURFACE_MAP_WRITE)))
return NTLSTATUS_INVALID_PARAMETER;
uint64_t pageFlags = PAGE_PRESENT | PAGE_USER | PAGE_NX;
if (flags & SYS_SURFACE_MAP_WRITE) pageFlags |= PAGE_WRITE;
KSTATUS status = MmVadCreateSharedViewFromFrames(process, surface->Frames, surface->PageCount,
surface->PayloadSize, pageFlags,
MM_VAD_SURFACE, MmVadShared, baseOut);
if (KSUCCESS(status)) {
*pitchBytesOut = surface->PitchBytes;
KiSurfaceStatsRecordMap(surface);
}
return status;
}
// Removes one mapping only when the supplied VAD is a surface shared view owned by the process.
KSTATUS KeSurfaceUnmap(PROCESS* process, void* base) {
if (!process || !base) return NTLSTATUS_INVALID_PARAMETER;
void* vadBase = (void*)0;
size_t vadSize = 0;
uint64_t pageFlags = 0;
MM_VAD_TYPE type = MmVadGeneric;
uint32_t vadFlags = 0;
KSTATUS status = MmVadQuery(process, base, &vadBase, &vadSize, &pageFlags, &type, &vadFlags);
(void)vadSize;
(void)pageFlags;
if (!KSUCCESS(status)) return status;
if (vadBase != base || type != MmVadShared || !(vadFlags & MM_VAD_SURFACE))
return NTLSTATUS_INVALID_PARAMETER;
return MmVadDestroy(process, base);
}
// Snapshots one surface's storage and commit metadata.
KSTATUS KeSurfaceQuery(OBJECT* object, SYS_SURFACE_DESC* descOut) {
KE_SURFACE* surface = KiSurfaceFromObject(object);
if (!surface || !descOut) return NTLSTATUS_INVALID_PARAMETER;
memset(descOut, 0, sizeof(*descOut));
KSTATUS status = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE);
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return status;
descOut->Width = surface->Width;
descOut->Height = surface->Height;
descOut->PitchBytes = surface->PitchBytes;
descOut->Format = surface->Format;
descOut->PayloadBytes = surface->PayloadSize;
descOut->PageCount = surface->PageCount;
descOut->ContentGeneration = surface->ContentGeneration;
descOut->CommitCount = surface->CommitCount;
descOut->CommitRectCount = surface->CommitRectCount;
descOut->CommitClippedRectCount = surface->CommitClippedRectCount;
descOut->CommitEmptyClipCount = surface->CommitEmptyClipCount;
descOut->LastDamageBounds = surface->LastDamageBounds;
descOut->LastDamageValid = surface->LastDamageValid;
KeReleaseMutex(surface->Lock);
uintptr_t irq = KeAcquireSpinlock(&gSurfaceStatsLock);
descOut->MapCount = surface->MapCount;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
return NTLSTATUS_SUCCESS;
}
// Announces changed pixels without touching physical scanout.
KSTATUS KeSurfaceCommit(OBJECT* object, const DISPLAY_RECT* rects, uint32_t rectCount,
uint64_t* generationOut) {
KE_SURFACE* surface = KiSurfaceFromObject(object);
if (!surface) return NTLSTATUS_INVALID_PARAMETER;
uint32_t requestedCount = rectCount ? rectCount : 1u;
DISPLAY_RECT clipped[SYS_SURFACE_MAX_RECTS];
uint32_t clippedCount = 0;
uint64_t emptyCount = 0;
KSTATUS status = KiSurfaceCaptureRects(surface, rects, rectCount, clipped, &clippedCount, &emptyCount);
if (!KSUCCESS(status)) return status;
status = KeAcquireMutex(surface->Lock, KE_WAIT_INFINITE);
if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return status;
surface->ContentGeneration++;
surface->CommitCount++;
surface->CommitRectCount += requestedCount;
surface->CommitClippedRectCount += clippedCount;
surface->CommitEmptyClipCount += emptyCount;
surface->LastDamageValid = clippedCount ? 1u : 0u;
if (clippedCount) {
surface->LastDamageBounds = clipped[0];
for (uint32_t i = 1; i < clippedCount; ++i)
KiSurfaceUnionRect(&surface->LastDamageBounds, &clipped[i]);
} else {
memset(&surface->LastDamageBounds, 0, sizeof(surface->LastDamageBounds));
}
uint64_t generation = surface->ContentGeneration;
KeReleaseMutex(surface->Lock);
KiSurfaceStatsRecordCommit(requestedCount, clippedCount, emptyCount);
KeSignalEvent(surface->CommitEvent);
if (generationOut) *generationOut = generation;
return NTLSTATUS_SUCCESS;
}
// Presents one surface after a caller has separately proved DisplayManager PRESENT authority.
KSTATUS KeSurfacePresent(OBJECT* object, const DISPLAY_RECT* rects, uint32_t rectCount,
uint64_t* bytesCopiedOut) {
KE_SURFACE* surface = KiSurfaceFromObject(object);
if (!surface || !bytesCopiedOut) return NTLSTATUS_INVALID_PARAMETER;
DISPLAY_RECT clipped[SYS_SURFACE_MAX_RECTS];
uint32_t clippedCount = 0;
uint64_t emptyCount = 0;
KSTATUS status = KiSurfaceCaptureRects(surface, rects, rectCount, clipped, &clippedCount, &emptyCount);
(void)emptyCount;
if (!KSUCCESS(status)) return status;
if (!surface->KernelBase) return NTLSTATUS_DEVICE_ERROR;
const uint8_t* sourceBase = (const uint8_t*)surface->KernelBase;
uint64_t bytesCopied = 0;
for (uint32_t i = 0; i < clippedCount; ++i) {
const DISPLAY_RECT* rect = &clipped[i];
const uint32_t* source = (const uint32_t*)(sourceBase +
(size_t)(uint32_t)rect->Y * surface->PitchBytes +
(size_t)(uint32_t)rect->X * sizeof(uint32_t));
status = KeDisplayPresentPitched(rect->X, rect->Y, rect->W, rect->H, source, surface->PitchBytes);
if (!KSUCCESS(status)) return status;
bytesCopied += (uint64_t)(uint32_t)rect->W * (uint64_t)(uint32_t)rect->H * sizeof(uint32_t);
}
*bytesCopiedOut = bytesCopied;
KiSurfaceStatsRecordPresent(bytesCopied);
return NTLSTATUS_SUCCESS;
}
// Returns the passive event used to wait for commit-generation changes.
EVENT* KeSurfaceWaitEvent(OBJECT* object) {
KE_SURFACE* surface = KiSurfaceFromObject(object);
return surface ? surface->CommitEvent : (EVENT*)0;
}
// Snapshots global surface telemetry.
void KeSurfaceQueryStats(SYS_SURFACE_STATS* statsOut) {
if (!statsOut) return;
uintptr_t irq = KeAcquireSpinlock(&gSurfaceStatsLock);
*statsOut = gSurfaceStats;
statsOut->Size = sizeof(*statsOut);
statsOut->Version = SYS_STATS_VERSION;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
code/kernel/ke_syscall.asm
+15
View File
@@ -7,6 +7,7 @@
EXTERN KeSyscallDispatch:PROC
EXTERN KiEnterTrapFrame:PROC
EXTERN KiExitTrapFrame:PROC
EXTERN KiDeliverKernelApcs:PROC
EXTERN KeFaultDispatchFrame:PROC
KTRAP_FRAME_SIZE EQU 240
@@ -112,6 +113,12 @@ KeSyscallEntry PROC
call KiEnterTrapFrame
add rsp, 32
; The interrupt gate protects the ring transition and SWAPGS/trap-frame
; construction. Once the thread context is fully established, syscalls are
; ordinary interruptible PASSIVE_LEVEL execution rather than one giant
; interrupt-disabled critical section.
sti
; Re-order original user arguments into KeSyscallDispatch(nr,a0..a3).
; Original registers were saved in the trap frame before any C calls.
sub rsp, 48
@@ -127,6 +134,14 @@ KeSyscallEntry PROC
mov [rbx + TF_Rax], rax
; Syscall return is a scheduler-safe kernel APC delivery boundary. Keep IF
; set while normal routines run, then close the tiny machine-exit window
; before committing the editable IRET frame and restoring user GS.
sub rsp, 32
call KiDeliverKernelApcs
add rsp, 32
cli
sub rsp, 32
mov rcx, rbx
call KiExitTrapFrame
code/kernel/ke_syscall.c
+604 -504
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_thread.c
+109 -3
View File
@@ -200,6 +200,25 @@ static void KiThreadDestroy(OBJECT* obj) {
}
}
// Validates one optional kernel-supplied thread description and returns its length.
static KSTATUS KiValidateThreadName(const char* name, size_t* lengthOut) {
if (!lengthOut) return NTLSTATUS_INVALID_PARAMETER;
if (!name) {
*lengthOut = 0;
return NTLSTATUS_SUCCESS;
}
size_t length = 0;
while (length < NTL_THREAD_NAME_MAX && name[length])
length++;
if (length >= NTL_THREAD_NAME_MAX)
return NTLSTATUS_INVALID_PARAMETER;
*lengthOut = length;
return NTLSTATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) {
if (!owner || !parameters || !out) return NTLSTATUS_INVALID_PARAMETER;
@@ -208,6 +227,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
(parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED))
return NTLSTATUS_INVALID_PARAMETER;
size_t initialNameLength = 0;
KSTATUS nameStatus = KiValidateThreadName(parameters->Name, &initialNameLength);
if (!KSUCCESS(nameStatus)) return nameStatus;
uint8_t initiallySuspended =
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
@@ -259,6 +282,11 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->UserExceptionFrameUserAddress = 0;
memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
thread->TerminationDeferralCount = 0;
thread->ExitInProgress = 0;
thread->NameLock = (SPINLOCK)SPINLOCK_INIT;
memset(thread->Name, 0, sizeof(thread->Name));
if (initialNameLength)
memcpy(thread->Name, parameters->Name, initialNameLength);
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
@@ -414,18 +442,42 @@ int KeThreadTerminationDeferred(const THREAD* thread) {
0) > 0;
}
// Returns whether terminal teardown owns the current thread stack.
int KeThreadExitInProgress(const THREAD* thread) {
return thread && _InterlockedCompareExchange((volatile long*)&thread->ExitInProgress,
0,
0) != 0;
}
// Implements the exit Thread operation.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
(void)code;
if (!thread) return;
KiRundownApcState(thread);
KiMutexRundownThread(thread);
// Terminal teardown must run exactly once and must retain this kernel
// stack until thread/process completion is fully published. Hardware
// interrupts remain enabled, but interrupt-return scheduling observes
// ExitInProgress and leaves the thread on-CPU until the final KeYield().
if (_InterlockedCompareExchange(&thread->ExitInProgress, 1, 0) != 0)
KePanic(KE_PANIC_CODE_SCHEDULER,
"recursive thread exit: thread=0x%016llX tid=%u\n",
(uint64_t)thread,
thread->ThreadId);
// Reject new APCs as soon as teardown owns the thread. The scheduler must
// not interpret ThreadDead as permission to switch away until the explicit
// terminal yield below.
thread->State = ThreadDead;
KiRundownApcState(thread);
KiMutexRundownThread(thread);
KeFinalizeThreadExit(thread);
KeYield();
KePanic(KE_PANIC_CODE_SCHEDULER,
"dead thread resumed after terminal yield: thread=0x%016llX tid=%u\n",
(uint64_t)thread,
thread->ThreadId);
}
// KeCreateKernelThread
@@ -463,6 +515,10 @@ static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
return NTLSTATUS_INVALID_PARAMETER;
}
size_t initialNameLength = 0;
KSTATUS nameStatus = KiValidateThreadName(parameters->Name, &initialNameLength);
if (!KSUCCESS(nameStatus)) return nameStatus;
uint8_t initiallySuspended =
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
@@ -514,6 +570,11 @@ static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
thread->UserExceptionFrameUserAddress = 0;
memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
thread->TerminationDeferralCount = 0;
thread->ExitInProgress = 0;
thread->NameLock = (SPINLOCK)SPINLOCK_INIT;
memset(thread->Name, 0, sizeof(thread->Name));
if (initialNameLength)
memcpy(thread->Name, parameters->Name, initialNameLength);
thread->UserStackBase = (void*)0; // no user stack
thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx
thread->TebSize = 0;
@@ -571,6 +632,37 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
return KiCreateKernelThread(fn, arg, parameters, out);
}
// Replaces one thread's diagnostic description.
KSTATUS KeSetThreadName(THREAD* thread, const char* name, size_t length) {
if (!thread || (length && !name) || length >= NTL_THREAD_NAME_MAX)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&thread->NameLock);
if (length)
memcpy(thread->Name, name, length);
thread->Name[length] = '\0';
KeReleaseSpinlock(&thread->NameLock, flags);
return NTLSTATUS_SUCCESS;
}
// Queries one stable snapshot of a mutable thread description.
size_t KeQueryThreadName(THREAD* thread, char* buffer, size_t capacity) {
if (!thread) return 0;
uintptr_t flags = KeAcquireSpinlock(&thread->NameLock);
size_t length = 0;
while (length < NTL_THREAD_NAME_MAX && thread->Name[length])
length++;
if (buffer && capacity) {
size_t copyLength = length < capacity - 1u ? length : capacity - 1u;
if (copyLength)
memcpy(buffer, thread->Name, copyLength);
buffer[copyLength] = '\0';
}
KeReleaseSpinlock(&thread->NameLock, flags);
return length;
}
// Prepares block Thread.
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
// Called with the synchronization object's spinlock held, so the caller is
@@ -640,11 +732,25 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
return status;
}
// A kernel APC can be queued before the waiter finishes publishing its
// alertable state. In that race the queueing CPU cannot wake the wait, and
// APC delivery is deliberately deferred while State is ThreadWaiting.
// Consume the prepared wait here before the thread can actually sleep.
if (alertable && thread->KernelApcPending && thread->KernelApcDisable == 0) {
if (KeAbortThreadWait(thread, NTLSTATUS_KERNEL_APC)) {
thread->AlertableWait = 0;
KiDeliverKernelApcs();
return NTLSTATUS_KERNEL_APC;
}
}
// If a user APC was already pending before the thread fully blocked,
// consume the alertable wait immediately instead of going to sleep.
if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC))
if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC)) {
thread->AlertableWait = 0;
return NTLSTATUS_USER_APC;
}
}
KiSchedulerNoteVoluntaryBlock(thread);
code/kernel/ke_work.c
+2 -1
View File
@@ -122,7 +122,8 @@ KSTATUS KeStartWorkQueue(void) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "work.queue");
s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker,
0,
&parameters,
code/kernel/kmain.c
+44 -13
View File
@@ -433,12 +433,14 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
return 0;
}
// Bootstrap init receives read-only system-volume authority, separate namespace-seeding
// authority, and the volume-manager capability used to supervise RSSRV publication.
// The bootstrap service manager receives only the capabilities required to construct
// the initial user-mode resource plane. Its PID is deliberately not part of this ABI.
IO_VOLUME* systemVolume = (IO_VOLUME*)0;
SYS_HANDLE systemVolumeHandle = SYS_HANDLE_INVALID;
SYS_HANDLE namespaceVolumeHandle = SYS_HANDLE_INVALID;
SYS_HANDLE volumeManagerHandle = SYS_HANDLE_INVALID;
SYS_HANDLE inputManagerHandle = SYS_HANDLE_INVALID;
SYS_HANDLE displayManagerHandle = SYS_HANDLE_INVALID;
s = KeReferenceSystemRootVolume(&systemVolume);
if (KSUCCESS(s)) {
s = KeCreateHandle(proc, &systemVolume->Header,
@@ -464,18 +466,46 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
ObDereferenceObject(&volumeManager->Header);
}
}
if (KSUCCESS(s)) {
KE_INPUT_MANAGER* inputManager = (KE_INPUT_MANAGER*)0;
s = KeInputReferenceManager(&inputManager);
if (KSUCCESS(s)) {
s = KeCreateHandle(proc, &inputManager->Header,
SYS_RIGHT_DUPLICATE | SYS_RIGHT_TRANSFER | SYS_RIGHT_QUERY |
SYS_INPUT_MANAGER_RIGHT_READ | SYS_INPUT_MANAGER_RIGHT_CONFIGURE,
&inputManagerHandle);
ObDereferenceObject(&inputManager->Header);
}
}
if (KSUCCESS(s)) {
KE_DISPLAY_MANAGER* displayManager = (KE_DISPLAY_MANAGER*)0;
s = KeDisplayReferenceManager(&displayManager);
if (KSUCCESS(s)) {
// Init must be able to attenuate this manager again when RSSRV publishes
// narrower display capabilities such as /Device/Display0.
s = KeCreateHandle(proc, &displayManager->Header,
SYS_RIGHT_DUPLICATE | SYS_RIGHT_DERIVE | SYS_RIGHT_TRANSFER | SYS_RIGHT_QUERY |
SYS_DISPLAY_MANAGER_RIGHT_ENUMERATE | SYS_DISPLAY_MANAGER_RIGHT_MODESET |
SYS_DISPLAY_MANAGER_RIGHT_PRESENT | SYS_DISPLAY_MANAGER_RIGHT_CURSOR |
SYS_DISPLAY_MANAGER_RIGHT_CONSOLE,
&displayManagerHandle);
ObDereferenceObject(&displayManager->Header);
}
}
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to create bootstrap startup capabilities. Status: %d\n", s);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
}
SYS_STARTUP_CAPABILITY startupCapabilities[3] = {
SYS_STARTUP_CAPABILITY startupCapabilities[5] = {
{ SYS_STARTUP_CAP_SYSTEM_VOLUME, systemVolumeHandle },
{ SYS_STARTUP_CAP_SYSTEM_VOLUME_NAMESPACE, namespaceVolumeHandle },
{ SYS_STARTUP_CAP_VOLUME_MANAGER, volumeManagerHandle },
{ SYS_STARTUP_CAP_INPUT_MANAGER, inputManagerHandle },
{ SYS_STARTUP_CAP_DISPLAY_MANAGER, displayManagerHandle },
};
s = KeSetProcessStartupCapabilities(proc, startupCapabilities, 3);
s = KeSetProcessStartupCapabilities(proc, startupCapabilities, 5);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to install startup capabilities. Status: %d\n", s);
IoCloseFile(f);
@@ -513,7 +543,8 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
// Create the main thread last. KeCreateThread queues it before returning.
const KE_THREAD_CREATE_PARAMETERS threadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
const KE_THREAD_CREATE_PARAMETERS threadParameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, proc->Name);
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, &threadParameters, &thread);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
@@ -549,20 +580,20 @@ void SystemInitThread(void* arg) {
if (args->BootConfig.Debug & KE_DEBUG_SELFTEST)
KiRunKernelSelfTests(args->Hal);
// Launch User-Mode init.exe.
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->BootConfig.CommandLine);
if (!shellProc) {
// Launch the bootstrap service manager. Its allocated PID is not a system contract.
PROCESS* initProcess = KiLaunchUserProcess("system/init.exe", args->BootConfig.CommandLine);
if (!initProcess) {
DbgPrint("SystemInitThread: Failed to start system/init.exe.\n");
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n");
}
KeWaitForEvent(shellProc->ExitEvent, 0);
int exitCode = shellProc->ExitCode;
KeWaitForEvent(initProcess->ExitEvent, 0);
int exitCode = initProcess->ExitCode;
if (exitCode != 0) {
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init.exe exited with code %d.\n", exitCode);
}
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
ObDereferenceObject(&shellProc->Header);
ObDereferenceObject(&initProcess->Header);
if (args->BootConfig.Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
KSTATUS logStatus = KeFlushDebugLogToSystemRoot("kernel.log");
@@ -805,7 +836,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
// Spawn the Initialization Thread. No creator reference is needed; the
// scheduler owns the live thread lifetime.
const KE_THREAD_CREATE_PARAMETERS initThreadParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
const KE_THREAD_CREATE_PARAMETERS initThreadParameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "system-init");
KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
(uintptr_t)initArgs,
&initThreadParameters, (THREAD**)0);
@@ -813,7 +845,6 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st);
}
DbgPrint("KernelMain: Handing over to scheduler...\n");
// The bootstrap CPU becomes the scheduler idle context once interrupts are enabled.
code/kernel/mm_file_cache.c
+2 -1
View File
@@ -375,7 +375,8 @@ static KSTATUS KiFileCacheEnsureWritebackStarted(void) {
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gWritebackSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "mm.writeback");
s = KeCreateKernelThread((uintptr_t)KiFileCacheWritebackWorker,
0,
&parameters,
code/kernel/net_socket.c
+2 -1
View File
@@ -426,7 +426,8 @@ static KSTATUS NetEnsureAsyncSocketWorkerStarted(void) {
KSTATUS status = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncSocketSemaphore);
if (KSUCCESS(status)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "net.socket.async");
status = KeCreateKernelThread((uintptr_t)NetAsyncSocketWorker,
0,
&parameters,
code/kernel/ob_object.c
+3
View File
@@ -36,6 +36,9 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
case ObjectTypePort: return MM_POOL_TAG('p', 'o', 'r', 't');
case ObjectTypeVolume: return MM_POOL_TAG('v', 'o', 'l', 'm');
case ObjectTypeVolumeManager: return MM_POOL_TAG('v', 'm', 'g', 'r');
case ObjectTypeInputManager: return MM_POOL_TAG('i', 'm', 'g', 'r');
case ObjectTypeDisplayManager: return MM_POOL_TAG('d', 'm', 'g', 'r');
case ObjectTypePipeEndpoint: return MM_POOL_TAG('p', 'i', 'p', 'e');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}