maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clean up and standardize types across the codebase, unify atomic operations, improve UTF-8 handling throughout the system, remove obsolete input and named queue interfaces, redesign process preparation and creation, tighten IPC and service boundaries, improve error handling and reliability, minor documentation cleanup, fix doom surfaces, fix panic in ini parsing.

Browse Source
This commit is contained in:
maksw committed 2026-10-12 18:17:43 +02:00
1 parent 899225f551
commit c0ad63c27f
226 files changed
+6562 -4147
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

OVMF/OVMF_VARS_4M_d.fd
BIN
View File
Binary file not shown.
code/build/build.c
+4 -8
View File
@@ -78,7 +78,7 @@ void create_layout() {
"obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping", "obj\\doom", "obj\\video", "obj\\memtop",
"obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\security_smoke", "obj\\port_ipc_smoke", "obj\\resource_service_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\input_service_smoke", "obj\\console_service_smoke", "obj\\desktop_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\process_exit_multithread_smoke", "obj\\thread_name_smoke", "obj\\process_builder_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_userspace_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\security_smoke", "obj\\port_ipc_smoke", "obj\\resource_service_smoke", "obj\\anonymous_queue_smoke", "obj\\input_service_smoke", "obj\\console_service_smoke", "obj\\desktop_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\process_exit_multithread_smoke", "obj\\thread_name_smoke", "obj\\process_builder_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_userspace_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\all_tests", "obj\\posix_service_smoke", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_commit", "obj\\surface_cross_process_smoke", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\posix_runtime_smoke", "obj\\posix_fd_smoke", "obj\\posix_process_smoke", "obj\\posix_signal_smoke", "obj\\posix_pipe_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke"
};
@@ -119,7 +119,7 @@ bool clean_build_object_directories(void) {
"obj\\init", "obj\\rssrv", "obj\\posixsrv", "obj\\dwm", "obj\\shell_win32", "obj\\shell_posix", "obj\\shell_native",
"obj\\async_apc", "obj\\async_cancel", "obj\\apc_return_smoke", "obj\\edit", "obj\\httpget",
"obj\\tcpserve", "obj\\udp", "obj\\resolve", "obj\\ping",
"obj\\doom", "obj\\video", "obj\\memtop", "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\security_smoke", "obj\\port_ipc_smoke", "obj\\resource_service_smoke", "obj\\named_queue_lifetime_smoke", "obj\\input_api_smoke", "obj\\input_model_smoke", "obj\\input_service_smoke", "obj\\console_service_smoke", "obj\\desktop_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\process_exit_multithread_smoke", "obj\\thread_name_smoke", "obj\\process_builder_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_userspace_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\doom", "obj\\video", "obj\\memtop", "obj\\file_stat_smoke", "obj\\handle_dup_smoke", "obj\\security_smoke", "obj\\port_ipc_smoke", "obj\\resource_service_smoke", "obj\\anonymous_queue_smoke", "obj\\input_service_smoke", "obj\\console_service_smoke", "obj\\desktop_smoke", "obj\\timer_smoke", "obj\\stats_abi_smoke", "obj\\realpath_smoke", "obj\\file_lock_smoke", "obj\\process_protection_smoke", "obj\\process_exit_multithread_smoke", "obj\\thread_name_smoke", "obj\\process_builder_smoke", "obj\\socket_addr_smoke", "obj\\syscall_contract_smoke", "obj\\would_block_smoke", "obj\\audio_stream_smoke", "obj\\winmm_waveout_smoke", "obj\\win32_condition_variable_smoke", "obj\\win32_mutex_smoke", "obj\\win32_thread_suspend_smoke", "obj\\win32_module_lookup_smoke", "obj\\cwd_userspace_smoke", "obj\\mmap_smoke", "obj\\pcache_smoke", "obj\\pcache_pressure", "obj\\dirty_writeback", "obj\\shared_mmap",
"obj\\pcache_invalidate", "obj\\pcache_mmap_read_reuse", "obj\\pcache_multithread",
"obj\\all_tests", "obj\\posix_service_smoke", "obj\\surface_smoke", "obj\\surface_invalid_rect", "obj\\surface_multithread_commit", "obj\\surface_cross_process_smoke", "obj\\net_irp_smoke", "obj\\module_load_smoke", "obj\\posix_module_smoke", "obj\\posix_runtime_smoke", "obj\\posix_fd_smoke", "obj\\posix_process_smoke", "obj\\posix_signal_smoke", "obj\\posix_pipe_smoke", "obj\\exception_dispatcher_smoke", "obj\\native_exception_runtime_smoke", "obj\\native_raise_exception_smoke", "obj\\exception_smoke", "obj\\win32_exception_smoke", "obj\\win32_seh_smoke"
};
@@ -477,9 +477,7 @@ int main(int argc, char** argv) {
{"tests\\security_smoke", NATIVE_EXE_LIBS_LITERAL, "security_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\port_ipc_smoke", NATIVE_EXE_LIBS_LITERAL, "port_ipc_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\resource_service_smoke", NATIVE_EXE_LIBS_LITERAL, "resource_service_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\named_queue_lifetime_smoke", NATIVE_EXE_LIBS_LITERAL, "named_queue_lifetime_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\input_api_smoke", NATIVE_EXE_LIBS_LITERAL, "input_api_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\input_model_smoke", NATIVE_EXE_LIBS_LITERAL, "input_model_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\anonymous_queue_smoke", NATIVE_EXE_LIBS_LITERAL, "anonymous_queue_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\input_service_smoke", NATIVE_EXE_LIBS_LITERAL, "input_service_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\console_service_smoke", NATIVE_EXE_LIBS_LITERAL, "console_service_smoke", NATIVE_EXE_ENTRY_LITERAL},
{"tests\\desktop_smoke", NATIVE_EXE_LIBS_LITERAL, "desktop_smoke", NATIVE_EXE_ENTRY_LITERAL},
@@ -637,9 +635,7 @@ int main(int argc, char** argv) {
"bin\\security_smoke.exe",
"bin\\port_ipc_smoke.exe",
"bin\\resource_service_smoke.exe",
"bin\\named_queue_lifetime_smoke.exe",
"bin\\input_api_smoke.exe",
"bin\\input_model_smoke.exe",
"bin\\anonymous_queue_smoke.exe",
"bin\\input_service_smoke.exe",
"bin\\console_service_smoke.exe",
"bin\\desktop_smoke.exe",
code/common/include/ntlite/internal/session_control.h
+7 -2
View File
@@ -3,10 +3,11 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/input.h>
#include <ntlite/status.h>
#define NTL_SESSION_DWM_PROTOCOL_VERSION 1u
#define NTL_SESSION_DWM_PROTOCOL_VERSION 2u
#define NTL_SESSION_DWM_FLAG_NONE 0x00000000u
#define NTL_SESSION_DWM_OP_SET_KEYBOARD_FOCUS 1u
@@ -23,7 +24,7 @@ typedef struct NTL_SESSION_DWM_REQUEST {
uint32_t Version;
uint32_t Operation;
uint32_t Flags;
uint32_t Enabled;
bool Enabled;
uint32_t Reserved[3];
} NTL_SESSION_DWM_REQUEST;
@@ -46,3 +47,7 @@ typedef struct NTL_SESSION_DWM_EVENT {
INPUT_KEY_RECORD Key;
INPUT_MOUSE_EVENT Mouse;
} NTL_SESSION_DWM_EVENT;
// RSSRV and DWM must agree on the private session-control payload.
_Static_assert(sizeof(NTL_SESSION_DWM_REQUEST) == 32u, "NTL_SESSION_DWM_REQUEST size changed");
_Static_assert(offsetof(NTL_SESSION_DWM_REQUEST, Enabled) == 16u, "NTL_SESSION_DWM_REQUEST Enabled offset changed");
code/drivers/audio/hda/hda.h
+4 -4
View File
@@ -207,8 +207,8 @@ typedef struct HDA_OUTPUT_STREAM {
uint8_t NextCompletionBdl;
bool Prepared;
bool Running;
volatile long PendingPeriodInterrupts;
volatile long PendingDmaErrors;
volatile int32_t PendingPeriodInterrupts;
volatile int32_t PendingDmaErrors;
HDA_OUTPUT_RING Rings[HDA_MAX_LOGICAL_ENDPOINTS];
uint8_t ActiveRingIndex;
IO_DMA_BUFFER* BdlDma;
@@ -249,8 +249,8 @@ struct HDA_CONTROLLER {
bool SurpriseRemoved;
SPINLOCK StateLock;
KDPC Dpc;
volatile long IsrActive;
volatile long DpcActive;
volatile int32_t IsrActive;
volatile int32_t DpcActive;
IO_DMA_ADAPTER* DmaAdapter;
IO_DMA_BUFFER* CorbDma;
IO_DMA_BUFFER* RirbDma;
code/drivers/audio/hda/hda_codec.c
+5 -4
View File
@@ -1,5 +1,6 @@
// Generic Intel HDA codec discovery and analog render routing.
#include <stdbool.h>
#include "hda.h"
#include <limits.h>
@@ -211,9 +212,9 @@ struct HDA_CODEC_STATE {
EVENT* PollFinished;
THREAD* WorkThread;
THREAD* PollThread;
volatile long WorkRunning;
volatile long PollRunning;
volatile long ShuttingDown;
volatile int32_t WorkRunning;
volatile int32_t PollRunning;
volatile int32_t ShuttingDown;
uint64_t PendingUnsolicitedTags;
bool PollRequested;
bool HavePollingPins;
@@ -1586,7 +1587,7 @@ static void HdaCodecEndpointStop(void* providerContext) {
// Stores or applies one endpoint hardware gain and mute request.
static KSTATUS HdaCodecEndpointSetGain(void* providerContext,
int32_t gainMilliDb,
uint8_t muted) {
bool muted) {
HDA_LOGICAL_ENDPOINT* endpoint = (HDA_LOGICAL_ENDPOINT*)providerContext;
if (!endpoint || !endpoint->State || !endpoint->HasHardwareGain)
return NTLSTATUS_INVALID_PARAMETER;
code/drivers/audio/hda/hda_transport.c
+6 -4
View File
@@ -1,5 +1,7 @@
// Intel High Definition Audio controller transport, command rings, output DMA, and recovery.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include "hda.h"
#include <intrin.h>
@@ -1079,7 +1081,7 @@ uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller) {
void HdaHandleInterrupt(HDA_CONTROLLER* controller) {
if (!controller)
return;
_InterlockedIncrement(&controller->IsrActive);
NtlAtomicIncrement32(&controller->IsrActive);
if (!controller->Started || controller->Removed || controller->SurpriseRemoved)
goto finished;
@@ -1137,7 +1139,7 @@ void HdaHandleInterrupt(HDA_CONTROLLER* controller) {
(void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0);
finished:
_InterlockedDecrement(&controller->IsrActive);
NtlAtomicDecrement32(&controller->IsrActive);
}
// Promotes an error to stream or controller recovery using the one-second threshold.
@@ -1166,7 +1168,7 @@ void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* system
HDA_CONTROLLER* controller = (HDA_CONTROLLER*)context;
if (!controller)
return;
_InterlockedIncrement(&controller->DpcActive);
NtlAtomicIncrement32(&controller->DpcActive);
if (controller->Removed || controller->SurpriseRemoved)
goto finished;
@@ -1253,7 +1255,7 @@ void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* system
}
finished:
_InterlockedDecrement(&controller->DpcActive);
NtlAtomicDecrement32(&controller->DpcActive);
}
// Recovers one isolated stream error without disturbing the codec topology.
code/drivers/bus/usb/usb.h
+2 -2
View File
@@ -636,7 +636,7 @@ struct USB_STORAGE_EXTENSION {
uint32_t BlockSize;
uint64_t BlockCount;
USB_STORAGE_FLUSH_CAPABILITY FlushCapability;
volatile long IoBusy;
volatile int32_t IoBusy;
};
struct USB_STORAGE_DISK_EXTENSION {
@@ -723,7 +723,7 @@ typedef struct XHCI_DEVICE {
bool MassRemoveLockReady;
bool MassSurpriseRemoved;
bool MassRemoved;
volatile long MassIoBusy;
volatile int32_t MassIoBusy;
uint64_t InputContextDeviceAddress;
uint64_t DeviceContextDeviceAddress;
code/drivers/bus/usb/usb_hid.c
+7 -7
View File
@@ -77,9 +77,9 @@ void XhciProcessBootKeyboardReport(XHCI_DEVICE* dev, const uint8_t* report, uint
void XhciProcessBootMouseReport(XHCI_DEVICE* dev, const uint8_t* report, uint16_t len) {
if (len < 3) return;
int dx = (int)(int8_t)report[1];
int dy = (int)(int8_t)report[2];
int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0;
int32_t dx = (int32_t)(int8_t)report[1];
int32_t dy = (int32_t)(int8_t)report[2];
int32_t wheel = (len >= 4) ? (int32_t)(int8_t)report[3] : 0;
uint8_t buttons = report[0] & 0x07u;
if (dx || dy || wheel || buttons != dev->HidLastButtons) {
@@ -397,9 +397,9 @@ static void UsbHidProcessBootKeyboardReport(USB_HID_EXTENSION* ext, const uint8_
static void UsbHidProcessBootMouseReport(USB_HID_EXTENSION* ext, const uint8_t* report, uint16_t len) {
if (!ext || len < 3) return;
int dx = (int)(int8_t)report[1];
int dy = (int)(int8_t)report[2];
int wheel = (len >= 4) ? (int)(int8_t)report[3] : 0;
int32_t dx = (int32_t)(int8_t)report[1];
int32_t dy = (int32_t)(int8_t)report[2];
int32_t wheel = (len >= 4) ? (int32_t)(int8_t)report[3] : 0;
uint8_t buttons = report[0] & 0x07u;
if (dx || dy || wheel || buttons != ext->LastButtons) {
@@ -421,7 +421,7 @@ static void UsbHidProcessQemuTabletReport(USB_HID_EXTENSION* ext, const uint8_t*
uint8_t buttons = report[0] & 0x07u;
uint16_t x = (uint16_t)report[1] | ((uint16_t)report[2] << 8);
uint16_t y = (uint16_t)report[3] | ((uint16_t)report[4] << 8);
int wheel = (len >= 6) ? (int)(int8_t)report[5] : 0;
int32_t wheel = (len >= 6) ? (int32_t)(int8_t)report[5] : 0;
if (x != ext->LastAbsoluteX || y != ext->LastAbsoluteY ||
wheel || buttons != ext->LastButtons) {
code/drivers/bus/usb/usb_storage.c
+5 -2
View File
@@ -5,6 +5,9 @@
// interface stack and issues BOT transfers through USB URBs. The xHCI HCD
// still backs the bulk URBs with the existing per-device rings; later stages
// will make endpoint rings fully pipe-owned and asynchronous.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <intrin.h>
#include "usb.h"
DRIVER_OBJECT gUsbStorageDriver;
@@ -464,14 +467,14 @@ static KSTATUS UsbStorageSynchronizeCache(USB_STORAGE_EXTENSION* storage) {
void UsbStorageAcquireIo(USB_STORAGE_EXTENSION* storage) {
if (!storage) return;
XHCI_CONTROLLER* xhci = storage->Device ? (XHCI_CONTROLLER*)storage->Device->Hcd->ControllerContext : (XHCI_CONTROLLER*)0;
while (_InterlockedCompareExchange(&storage->IoBusy, 1, 0) != 0) {
while (NtlAtomicCompareExchange32(&storage->IoBusy, 1, 0) != 0) {
if (xhci) UsbDelayUs(xhci->Ddk, 10);
}
}
// Implements the storage Release Io operation.
void UsbStorageReleaseIo(USB_STORAGE_EXTENSION* storage) {
if (storage) _InterlockedExchange(&storage->IoBusy, 0);
if (storage) NtlAtomicExchange32(&storage->IoBusy, 0);
}
// Implements the storage Dispatch Read Write operation.
code/drivers/display/vmwaresvga/vmwaresvga.c
+45 -44
View File
@@ -1,5 +1,6 @@
// VMware SVGA-II PCI display provider. This file owns adapter setup, FIFO command submission, framebuffer presentation, and cursor integration.
#include <stdbool.h>
#include "vmwaresvga.h"
const NTL_DDK* gVmwSvgaDdk;
@@ -211,7 +212,7 @@ static void VmwSvgaWakeFifo(VMWSVGA_DEVICE* v) {
VmwSvgaWriteReg(v, SVGA_REG_SYNC, 1u);
}
static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels);
static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels);
// Implements the premultiply Argb For Bgra operation.
static uint32_t VmwSvgaPremultiplyArgbForBgra(uint32_t argb) {
@@ -275,7 +276,7 @@ static void VmwSvgaCursorSetVisible(VMWSVGA_DEVICE* v, bool visible) {
}
// Implements the cursor Set Position operation.
static void VmwSvgaCursorSetPosition(VMWSVGA_DEVICE* v, int x, int y) {
static void VmwSvgaCursorSetPosition(VMWSVGA_DEVICE* v, int32_t x, int32_t y) {
if (!v || !v->HardwareCursor)
return;
v->CursorX = x;
@@ -284,7 +285,7 @@ static void VmwSvgaCursorSetPosition(VMWSVGA_DEVICE* v, int x, int y) {
}
// Implements the define Alpha Cursor operation.
static KSTATUS VmwSvgaDefineAlphaCursor(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
static KSTATUS VmwSvgaDefineAlphaCursor(VMWSVGA_DEVICE* v, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
KSTATUS st = VmwSvgaFifoWrite(v, SVGA_CMD_DEFINE_ALPHA_CURSOR);
if (!KSUCCESS(st)) return st;
st = VmwSvgaFifoWrite(v, 0u); // cursor id, reserved
@@ -298,8 +299,8 @@ static KSTATUS VmwSvgaDefineAlphaCursor(VMWSVGA_DEVICE* v, int width, int height
st = VmwSvgaFifoWrite(v, (uint32_t)height);
if (!KSUCCESS(st)) return st;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
for (int32_t y = 0; y < height; y++) {
for (int32_t x = 0; x < width; x++) {
uint32_t p = argbPixels[(size_t)y * (size_t)width + (size_t)x];
st = VmwSvgaFifoWrite(v, VmwSvgaPremultiplyArgbForBgra(p));
if (!KSUCCESS(st)) return st;
@@ -310,9 +311,9 @@ static KSTATUS VmwSvgaDefineAlphaCursor(VMWSVGA_DEVICE* v, int width, int height
}
static uint32_t VmwSvgaPackMonoCursorDword(const uint8_t* bits,
int width,
int height,
int y,
int32_t width,
int32_t height,
int32_t y,
uint32_t dwordIndex) {
// SVGA classic cursor scanlines are padded to 32 bits. QEMU stores
@@ -343,7 +344,7 @@ static uint32_t VmwSvgaPackMonoCursorDword(const uint8_t* bits,
}
// Implements the classic Cursor Width operation.
static int VmwSvgaClassicCursorWidth(int width) {
static int32_t VmwSvgaClassicCursorWidth(int32_t width) {
// QEMU's VMware SVGA implementation reads classic cursor masks as
// SVGA dword-padded scanlines, then passes the raw byte stream to
@@ -358,9 +359,9 @@ static int VmwSvgaClassicCursorWidth(int width) {
}
static void VmwSvgaBuildMonoCursorBits(const uint32_t* argbPixels,
int sourceWidth,
int encodedWidth,
int height,
int32_t sourceWidth,
int32_t encodedWidth,
int32_t height,
uint8_t* andBits,
uint8_t* xorBits) {
@@ -372,8 +373,8 @@ static void VmwSvgaBuildMonoCursorBits(const uint32_t* argbPixels,
// This intentionally discards antialiasing, but avoids QEMU/GTK's
// fragile 32bpp classic host-cursor path.
for (int y = 0; y < height; y++) {
for (int x = 0; x < encodedWidth; x++) {
for (int32_t y = 0; y < height; y++) {
for (int32_t x = 0; x < encodedWidth; x++) {
size_t out = (size_t)y * (size_t)encodedWidth + (size_t)x;
if (x >= sourceWidth) {
andBits[out] = 1u;
@@ -400,8 +401,8 @@ static void VmwSvgaBuildMonoCursorBits(const uint32_t* argbPixels,
}
// Implements the define Classic Cursor operation.
static KSTATUS VmwSvgaDefineClassicCursor(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
int encodedWidth = VmwSvgaClassicCursorWidth(width);
static KSTATUS VmwSvgaDefineClassicCursor(VMWSVGA_DEVICE* v, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
int32_t encodedWidth = VmwSvgaClassicCursorWidth(width);
uint32_t dwordsPerRow = ((uint32_t)encodedWidth + 31u) / 32u;
size_t pixelCount = (size_t)encodedWidth * (size_t)height;
@@ -438,14 +439,14 @@ static KSTATUS VmwSvgaDefineClassicCursor(VMWSVGA_DEVICE* v, int width, int heig
st = VmwSvgaFifoWrite(v, 1u); // XOR mask depth
if (!KSUCCESS(st)) goto done;
for (int y = 0; y < height; y++) {
for (int32_t y = 0; y < height; y++) {
for (uint32_t d = 0; d < dwordsPerRow; d++) {
st = VmwSvgaFifoWrite(v, VmwSvgaPackMonoCursorDword(andBits, encodedWidth, height, y, d));
if (!KSUCCESS(st)) goto done;
}
}
for (int y = 0; y < height; y++) {
for (int32_t y = 0; y < height; y++) {
for (uint32_t d = 0; d < dwordsPerRow; d++) {
st = VmwSvgaFifoWrite(v, VmwSvgaPackMonoCursorDword(xorBits, encodedWidth, height, y, d));
if (!KSUCCESS(st)) goto done;
@@ -461,7 +462,7 @@ done:
}
// Implements the upload Cursor Image operation.
static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
if (!v || !v->HardwareCursor || !argbPixels || width <= 0 || height <= 0)
return NTLSTATUS_INVALID_PARAMETER;
if (width > 256 || height > 256 || hotX < 0 || hotY < 0 || hotX >= width || hotY >= height)
@@ -479,7 +480,7 @@ static KSTATUS VmwSvgaUploadCursorImage(VMWSVGA_DEVICE* v, int width, int height
}
// Implements the define Cursor Image operation.
static KSTATUS VmwSvgaDefineCursorImage(VMWSVGA_DEVICE* v, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
static KSTATUS VmwSvgaDefineCursorImage(VMWSVGA_DEVICE* v, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
if (!v || !v->HardwareCursor || !argbPixels || width <= 0 || height <= 0)
return NTLSTATUS_INVALID_PARAMETER;
if (width > 256 || height > 256 || hotX < 0 || hotY < 0 || hotX >= width || hotY >= height)
@@ -568,7 +569,7 @@ static KSTATUS VmwSvgaDefineGmrfb(VMWSVGA_DEVICE* v) {
}
// Blits gmrfb To Screen.
static KSTATUS VmwSvgaBlitGmrfbToScreen(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
static KSTATUS VmwSvgaBlitGmrfbToScreen(VMWSVGA_DEVICE* v, int32_t x, int32_t y, int32_t w, int32_t h) {
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
return NTLSTATUS_SUCCESS;
@@ -659,7 +660,7 @@ static KSTATUS VmwSvgaInitializeScreenObject(VMWSVGA_DEVICE* v) {
}
// Implements the legacy Update operation.
static KSTATUS VmwSvgaLegacyUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
static KSTATUS VmwSvgaLegacyUpdate(VMWSVGA_DEVICE* v, int32_t x, int32_t y, int32_t w, int32_t h) {
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
return NTLSTATUS_SUCCESS;
@@ -680,7 +681,7 @@ static KSTATUS VmwSvgaLegacyUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h
}
// Updates the requested value.
static KSTATUS VmwSvgaUpdate(VMWSVGA_DEVICE* v, int x, int y, int w, int h) {
static KSTATUS VmwSvgaUpdate(VMWSVGA_DEVICE* v, int32_t x, int32_t y, int32_t w, int32_t h) {
DISPLAY_RECT r = { x, y, w, h };
if (!VmwSvgaClipRect(v, &r))
return NTLSTATUS_SUCCESS;
@@ -841,7 +842,7 @@ static void VmwSvgaCopyToShadowAndFramebuffer(uint32_t* shadow,
}
// Implements the display pitched blit operation.
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels, uint32_t pitchBytes) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
@@ -857,7 +858,7 @@ static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, i
int srcOffsetX = r.X - x;
int srcOffsetY = r.Y - y;
size_t bytesPerRow = (size_t)r.W * sizeof(uint32_t);
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
uint32_t* dst = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
@@ -868,10 +869,10 @@ static KSTATUS VmwSvgaDisplayBlit(DISPLAY_DEVICE* device, int x, int y, int w, i
// Presents one pitched rectangle without a shadow-to-framebuffer second pass.
static KSTATUS VmwSvgaDisplayPresentPitched(DISPLAY_DEVICE* device,
int x,
int y,
int w,
int h,
int32_t x,
int32_t y,
int32_t w,
int32_t h,
const uint32_t* pixels,
uint32_t pitchBytes) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
@@ -894,7 +895,7 @@ static KSTATUS VmwSvgaDisplayPresentPitched(DISPLAY_DEVICE* device,
v->Stats.ScanoutBytes += bytes;
v->Stats.BytesCopied += bytes * 2u;
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
const uint8_t* sourceRow = (const uint8_t*)pixels +
(size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
@@ -914,7 +915,7 @@ static KSTATUS VmwSvgaDisplayPresentPitched(DISPLAY_DEVICE* device,
}
// Implements the display Fill Rect operation.
static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -935,7 +936,7 @@ static KSTATUS VmwSvgaDisplayFillRect(DISPLAY_DEVICE* device, int x, int y, int
}
// Implements the display Copy Rect operation.
static KSTATUS VmwSvgaDisplayCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h) {
static KSTATUS VmwSvgaDisplayCopyRect(DISPLAY_DEVICE* device, int32_t dstX, int32_t dstY, int32_t srcX, int32_t srcY, int32_t w, int32_t h) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -956,13 +957,13 @@ static KSTATUS VmwSvgaDisplayCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY
size_t rowBytes = (size_t)w * sizeof(uint32_t);
if (dstY > srcY) {
for (int row = h - 1; row >= 0; row--) {
for (int32_t row = h - 1; row >= 0; row--) {
uint32_t* dst = v->Backbuffer + ((size_t)(dstY + row) * v->Stride) + dstX;
uint32_t* src = v->Backbuffer + ((size_t)(srcY + row) * v->Stride) + srcX;
memmove(dst, src, rowBytes);
}
} else {
for (int row = 0; row < h; row++) {
for (int32_t row = 0; row < h; row++) {
uint32_t* dst = v->Backbuffer + ((size_t)(dstY + row) * v->Stride) + dstX;
uint32_t* src = v->Backbuffer + ((size_t)(srcY + row) * v->Stride) + srcX;
memmove(dst, src, rowBytes);
@@ -972,7 +973,7 @@ static KSTATUS VmwSvgaDisplayCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY
}
// Implements the display Read Rect operation.
static KSTATUS VmwSvgaDisplayReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride) {
static KSTATUS VmwSvgaDisplayReadRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t* outPixels, uint32_t outStride) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->Backbuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -986,7 +987,7 @@ static KSTATUS VmwSvgaDisplayReadRect(DISPLAY_DEVICE* device, int x, int y, int
int dstOffsetX = r.X - x;
int dstOffsetY = r.Y - y;
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
const uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
uint32_t* dst = outPixels + ((size_t)(dstOffsetY + row) * outStride) + dstOffsetX;
memcpy(dst, src, (size_t)r.W * sizeof(uint32_t));
@@ -1015,7 +1016,7 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT*
v->Stats.BytesCopied += bytes;
v->Stats.ScanoutBytes += bytes;
size_t pixelCount = (size_t)r.W;
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
uint32_t* dst = (uint32_t*)(uintptr_t)(v->Framebuffer +
((size_t)(r.Y + row) * v->BytesPerLine) +
((size_t)r.X * sizeof(uint32_t)));
@@ -1046,7 +1047,7 @@ static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) {
}
// Implements the display Set Cursor Image operation.
static KSTATUS VmwSvgaDisplaySetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
static KSTATUS VmwSvgaDisplaySetCursorImage(DISPLAY_DEVICE* device, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v || !v->HardwareCursor)
return NTLSTATUS_NOT_SUPPORTED;
@@ -1054,7 +1055,7 @@ static KSTATUS VmwSvgaDisplaySetCursorImage(DISPLAY_DEVICE* device, int width, i
}
// Implements the display Set Cursor Position operation.
static KSTATUS VmwSvgaDisplaySetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
static KSTATUS VmwSvgaDisplaySetCursorPosition(DISPLAY_DEVICE* device, int32_t x, int32_t y) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
if (!v)
return NTLSTATUS_INVALID_PARAMETER;
@@ -1073,7 +1074,7 @@ static void VmwSvgaRestoreCursor(VMWSVGA_DEVICE* v) {
DISPLAY_RECT r = { v->CursorDrawX, v->CursorDrawY, VMWSVGA_CURSOR_W, VMWSVGA_CURSOR_H };
if (VmwSvgaClipRect(v, &r)) {
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
volatile uint8_t* dstBytes = v->Framebuffer + ((size_t)(r.Y + row) * v->BytesPerLine) + ((size_t)r.X * sizeof(uint32_t));
uint32_t* src = v->Backbuffer + ((size_t)(r.Y + row) * v->Stride) + r.X;
memcpy((void*)dstBytes, src, (size_t)r.W * sizeof(uint32_t));
@@ -1091,17 +1092,17 @@ static void VmwSvgaDrawCursor(VMWSVGA_DEVICE* v) {
if (!v || !v->Backbuffer || !v->Framebuffer)
return;
int x = v->CursorX;
int y = v->CursorY;
int32_t x = v->CursorX;
int32_t y = v->CursorY;
v->CursorDrawX = x;
v->CursorDrawY = y;
DISPLAY_RECT dirty = { x, y, VMWSVGA_CURSOR_W, VMWSVGA_CURSOR_H };
for (int row = 0; row < VMWSVGA_CURSOR_H; row++) {
for (int32_t row = 0; row < VMWSVGA_CURSOR_H; row++) {
int fy = y + row;
if ((unsigned)fy >= v->Height)
continue;
for (int col = 0; col < VMWSVGA_CURSOR_W; col++) {
for (int32_t col = 0; col < VMWSVGA_CURSOR_W; col++) {
int fx = x + col;
if ((unsigned)fx >= v->Width)
continue;
code/drivers/display/vmwaresvga/vmwaresvga.h
+8 -8
View File
@@ -192,18 +192,18 @@ typedef struct VMWSVGA_DEVICE {
bool CursorVisible;
uint32_t* CursorImageArgb;
size_t CursorImageBytes;
int CursorImageWidth;
int CursorImageHeight;
int CursorHotX;
int CursorHotY;
int32_t CursorImageWidth;
int32_t CursorImageHeight;
int32_t CursorHotX;
int32_t CursorHotY;
bool Started;
bool Removed;
bool SurpriseRemoved;
int CursorX;
int CursorY;
int CursorDrawX;
int CursorDrawY;
int32_t CursorX;
int32_t CursorY;
int32_t CursorDrawX;
int32_t CursorDrawY;
bool CursorDrawn;
DISPLAY_STATS Stats;
code/drivers/fs/ext4/ext4_driver.c
+5 -5
View File
@@ -838,11 +838,11 @@ static KSTATUS Ext4ReadDir(IO_VOLUME_PROVIDER* provider, void* dirCtx, KDIR_ENTR
}
memset(out, 0, sizeof(*out));
size_t nameLength = entry->name_length;
if (nameLength >= SYS_DIR_NAME_MAX)
nameLength = SYS_DIR_NAME_MAX - 1;
memcpy(out->Name, entry->name, nameLength);
out->Name[nameLength] = '\0';
// ext4 allows arbitrary non-NUL filename bytes. Project malformed UTF-8
// to U+FFFD only for enumeration; keep raw bytes for ext4 metadata lookups.
// The displayed name is not a reversible path for malformed disk names.
(void)SysSvCopyUtf8WithReplacement(
SysSvFromParts(entry->name, entry->name_length), out->Name, sizeof(out->Name));
out->Attr = entry->inode_type == EXT4_DE_DIR
? SYS_FILE_ATTR_DIRECTORY
: SYS_FILE_ATTR_ARCHIVE;
code/drivers/fs/fat/fat.c
+18 -17
View File
@@ -1,5 +1,6 @@
// FAT allocation, cache, and metadata core shared by the filesystem operation translation units.
#include <stdbool.h>
#include "fat_internal.h"
// FAT sector cache (fixes #1 and #5)
@@ -169,7 +170,7 @@ uint32_t FatEocMark(FAT_VOL* v) {
return v && v->Variant == FatVariant16 ? FAT16_EOC_MARK : FAT32_EOC_MARK;
}
int FatIsDataCluster(FAT_VOL* v, uint32_t cluster) {
bool FatIsDataCluster(FAT_VOL* v, uint32_t cluster) {
return v && cluster >= 2u && cluster < FatEocThreshold(v);
}
@@ -185,49 +186,49 @@ void FatSetDirentFirstCluster(FAT_VOL* v, DIRENT* d, uint32_t cluster) {
d->FirstClusterHi = (v && v->Variant == FatVariant32) ? (uint16_t)(cluster >> 16) : 0u;
}
int FatIsFixedRoot(FAT_VOL* v, uint32_t dirCluster) {
bool FatIsFixedRoot(FAT_VOL* v, uint32_t dirCluster) {
return v && v->Variant == FatVariant16 && dirCluster == FAT_FIXED_ROOT_CLUSTER;
}
uint64_t FatClusterToLba(FAT_VOL* v, uint32_t cluster);
uint32_t FatEntry(FAT_VOL* v, uint32_t cluster);
int FatDirectoryFirstSegment(FAT_VOL* v, uint32_t dirCluster,
bool FatDirectoryFirstSegment(FAT_VOL* v, uint32_t dirCluster,
uint32_t* cursor, uint64_t* lba,
uint32_t* sectors) {
if (!v || !cursor || !lba || !sectors) return 0;
if (!v || !cursor || !lba || !sectors) return false;
if (FatIsFixedRoot(v, dirCluster)) {
if (!v->RootDirSectors) return 0;
if (!v->RootDirSectors) return false;
*cursor = 0;
*lba = v->RootDirStart;
*sectors = 1;
return 1;
return true;
}
if (!FatIsDataCluster(v, dirCluster)) return 0;
if (!FatIsDataCluster(v, dirCluster)) return false;
*cursor = dirCluster;
*lba = FatClusterToLba(v, dirCluster);
*sectors = v->SectorsPerCluster;
return 1;
return true;
}
int FatDirectoryNextSegment(FAT_VOL* v, uint32_t dirCluster,
bool FatDirectoryNextSegment(FAT_VOL* v, uint32_t dirCluster,
uint32_t* cursor, uint64_t* lba,
uint32_t* sectors) {
if (!v || !cursor || !lba || !sectors) return 0;
if (!v || !cursor || !lba || !sectors) return false;
if (FatIsFixedRoot(v, dirCluster)) {
uint32_t next = *cursor + 1u;
if (next >= v->RootDirSectors) return 0;
if (next >= v->RootDirSectors) return false;
*cursor = next;
*lba = v->RootDirStart + next;
*sectors = 1;
return 1;
return true;
}
uint32_t next = FatEntry(v, *cursor);
if (!FatIsDataCluster(v, next)) return 0;
if (!FatIsDataCluster(v, next)) return false;
*cursor = next;
*lba = FatClusterToLba(v, next);
*sectors = v->SectorsPerCluster;
return 1;
return true;
}
// Implements the cluster To Lba operation.
@@ -314,7 +315,7 @@ KSTATUS FatWriteEntry(FAT_VOL* v, uint32_t cluster, uint32_t value) {
}
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan).
uint32_t FatAllocClusterEx(FAT_VOL* v, int zeroCluster) {
uint32_t FatAllocClusterEx(FAT_VOL* v, bool zeroCluster) {
if (v->TotalClusters == 0) return 0;
uint32_t total = v->TotalClusters;
@@ -353,7 +354,7 @@ uint32_t FatAllocCluster(FAT_VOL* v) {
return FatAllocClusterEx(v, 1);
}
uint32_t FatAppendClusterEx(FAT_VOL* v, uint32_t lastCluster, int zeroCluster) {
uint32_t FatAppendClusterEx(FAT_VOL* v, uint32_t lastCluster, bool zeroCluster) {
uint32_t newCluster = FatAllocClusterEx(v, zeroCluster);
if (!newCluster) return 0;
if (!KSUCCESS(FatWriteEntry(v, lastCluster, newCluster))) return 0;
@@ -390,7 +391,7 @@ void FatFreeClusterChain(FAT_VOL* v, uint32_t start) {
// FAT timestamp and DIRENT metadata helpers.
// Implements the fat Is Leap Year operation.
static int FatIsLeapYear(uint32_t year) {
static bool FatIsLeapYear(uint32_t year) {
return ((year % 4u) == 0u && (year % 100u) != 0u) || ((year % 400u) == 0u);
}
code/drivers/fs/fat/fat_directory.c
+99 -98
View File
@@ -1,18 +1,19 @@
// FAT name handling, path traversal, and directory mutation.
#include <stdbool.h>
#include "fat_internal.h"
// 8.3 name matching / assembly.
static int MatchRaw83(const char dirName[8], const char dirExt[3],
static bool MatchRaw83(const char dirName[8], const char dirExt[3],
const char wantName[8], const char wantExt[3]) {
return memcmp(dirName, wantName, 8) == 0 &&
memcmp(dirExt, wantExt, 3) == 0;
}
static char FatUpperChar(char c);
static int FatIsLowerChar(char c);
static int FatIsValidShortChar(char c);
static bool FatIsLowerChar(char c);
static bool FatIsValidShortChar(char c);
// Implements the make83 Name V operation.
static void Make83NameV(SYS_STRING_VIEW src, char name8[8], char ext3[3]) {
@@ -53,40 +54,40 @@ static char FatUpperChar(char c) {
}
// Implements the fat Is Lower Char operation.
static int FatIsLowerChar(char c) {
static bool FatIsLowerChar(char c) {
return c >= 'a' && c <= 'z';
}
// Implements the fat Is Valid Short Char operation.
static int FatIsValidShortChar(char c) {
static bool FatIsValidShortChar(char c) {
c = FatUpperChar(c);
if (c >= 'A' && c <= 'Z') return 1;
if (c >= '0' && c <= '9') return 1;
if (c >= 'A' && c <= 'Z') return true;
if (c >= '0' && c <= '9') return true;
switch (c) {
case '$': case '%': case '\'': case '-': case '_': case '@':
case '~': case '`': case '!': case '(': case ')': case '{':
case '}': case '^': case '#': case '&':
return 1;
return true;
default:
return 0;
return false;
}
}
// True when the user spelling can refer to a normal FAT 8.3 entry. This is
// intentionally structural and case-insensitive: "doom1.wad" must still open
// an existing short entry named DOOM1.WAD. Leading-dot names are not 8.3.
static int FatNameIsStructurallyShort83V(SYS_STRING_VIEW name) {
if (!SysSvIsValid(name) || name.Length == 0) return 0;
if (name.Data[0] == '.') return 0;
static bool FatNameIsStructurallyShort83V(SYS_STRING_VIEW name) {
if (!SysSvIsValid(name) || name.Length == 0) return false;
if (name.Data[0] == '.') return false;
size_t dot = (size_t)-1;
for (size_t i = 0; i < name.Length; i++) {
char c = name.Data[i];
if (c == '.') {
if (dot != (size_t)-1) return 0;
if (dot != (size_t)-1) return false;
dot = i;
} else if (!FatIsValidShortChar(c)) {
return 0;
return false;
}
}
@@ -96,17 +97,17 @@ static int FatNameIsStructurallyShort83V(SYS_STRING_VIEW name) {
}
// Implements the fat Component Has Lower V operation.
static int FatComponentHasLowerV(SYS_STRING_VIEW name, size_t start, size_t end) {
static bool FatComponentHasLowerV(SYS_STRING_VIEW name, size_t start, size_t end) {
for (size_t i = start; i < end; i++)
if (FatIsLowerChar(name.Data[i])) return 1;
return 0;
if (FatIsLowerChar(name.Data[i])) return true;
return false;
}
// Implements the fat Component Has Upper V operation.
static int FatComponentHasUpperV(SYS_STRING_VIEW name, size_t start, size_t end) {
static bool FatComponentHasUpperV(SYS_STRING_VIEW name, size_t start, size_t end) {
for (size_t i = start; i < end; i++)
if (name.Data[i] >= 'A' && name.Data[i] <= 'Z') return 1;
return 0;
if (name.Data[i] >= 'A' && name.Data[i] <= 'Z') return true;
return false;
}
static void FatSplitShortNameV(SYS_STRING_VIEW name, size_t* baseEnd,
@@ -128,19 +129,19 @@ static void FatSplitShortNameV(SYS_STRING_VIEW name, size_t* baseEnd,
// Creation uses LFN only when the spelling cannot be represented by a short
// entry, or when mixed-case 8.3 spelling would be lost. Plain lowercase
// 8.3 names are represented with FAT's NT case bits, not VFAT LFN entries.
static int FatNameNeedsLfnForCreateV(SYS_STRING_VIEW name) {
if (!FatNameIsStructurallyShort83V(name)) return 1;
static bool FatNameNeedsLfnForCreateV(SYS_STRING_VIEW name) {
if (!FatNameIsStructurallyShort83V(name)) return true;
size_t baseEnd, extStart, extEnd;
FatSplitShortNameV(name, &baseEnd, &extStart, &extEnd);
int baseLower = FatComponentHasLowerV(name, 0, baseEnd);
int baseUpper = FatComponentHasUpperV(name, 0, baseEnd);
int extLower = FatComponentHasLowerV(name, extStart, extEnd);
int extUpper = FatComponentHasUpperV(name, extStart, extEnd);
bool baseLower = FatComponentHasLowerV(name, 0, baseEnd);
bool baseUpper = FatComponentHasUpperV(name, 0, baseEnd);
bool extLower = FatComponentHasLowerV(name, extStart, extEnd);
bool extUpper = FatComponentHasUpperV(name, extStart, extEnd);
if ((baseLower && baseUpper) || (extLower && extUpper)) return 1;
return 0;
if ((baseLower && baseUpper) || (extLower && extUpper)) return true;
return false;
}
// Implements the fat Short Name Case Flags V operation.
@@ -211,15 +212,15 @@ static void MakeLegacyBrokenDot83(const char* src, char name8[8], char ext3[3])
ext3[ei++] = FatUpperChar(*p);
}
static int ShortNameExistsRaw(FAT_VOL* v, uint32_t dirCluster,
static bool ShortNameExistsRaw(FAT_VOL* v, uint32_t dirCluster,
const char name8[8], const char ext3[3]) {
uint8_t* clusterBuf = (uint8_t*)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return 1; // fail closed; caller should pick another alias
if (!clusterBuf) return true; // fail closed; caller should pick another alias
uint32_t cursor; uint64_t lba; uint32_t sectors;
if (!FatDirectoryFirstSegment(v, dirCluster, &cursor, &lba, &sectors)) {
MmFreePool(clusterBuf);
return 0;
return false;
}
do {
@@ -228,17 +229,17 @@ static int ShortNameExistsRaw(FAT_VOL* v, uint32_t dirCluster,
uint32_t entries = (sectors * v->BytesPerSector) / sizeof(DIRENT);
DIRENT* dir = (DIRENT*)clusterBuf;
for (uint32_t i = 0; i < entries; i++) {
if (dir[i].Name[0] == 0x00) { MmFreePool(clusterBuf); return 0; }
if (dir[i].Name[0] == 0x00) { MmFreePool(clusterBuf); return false; }
if ((uint8_t)dir[i].Name[0] == 0xE5 || dir[i].Attr == ATTR_LFN) continue;
if (MatchRaw83(dir[i].Name, dir[i].Ext, name8, ext3)) {
MmFreePool(clusterBuf);
return 1;
return true;
}
}
} while (FatDirectoryNextSegment(v, dirCluster, &cursor, &lba, &sectors));
MmFreePool(clusterBuf);
return 0;
return false;
}
// Implements the fat Alias Append Char operation.
@@ -310,7 +311,7 @@ static void MakeAliasAttempt(const char stem[9], const char extIn[3],
memcpy(name8 + keep, suffix, suffixLen);
}
static int MakeUniqueShortAliasV(FAT_VOL* v, uint32_t dirCluster,
static bool MakeUniqueShortAliasV(FAT_VOL* v, uint32_t dirCluster,
SYS_STRING_VIEW longName,
char name8[8], char ext3[3]) {
char stem[9], extCandidate[3];
@@ -318,9 +319,9 @@ static int MakeUniqueShortAliasV(FAT_VOL* v, uint32_t dirCluster,
for (uint32_t attempt = 1; attempt < 1000000; attempt++) {
MakeAliasAttempt(stem, extCandidate, attempt, name8, ext3);
if (!ShortNameExistsRaw(v, dirCluster, name8, ext3)) return 1;
if (!ShortNameExistsRaw(v, dirCluster, name8, ext3)) return true;
}
return 0;
return false;
}
// Creates legacy Broken Dot83 V.
@@ -400,44 +401,44 @@ void FatLfnAppend(char* lfn, const LFN_ENTRY* e) {
}
// Directory search and view-native path traversal.
static int FatIsPathSep(char c) {
static bool FatIsPathSep(char c) {
return c == '\\' || c == '/';
}
// Tests whether a non-empty FAT path view points at plausible kernel-owned storage.
static int FatStringViewDataIsPlausible(SYS_STRING_VIEW view) {
if (!SysSvIsValid(view)) return 0;
if (view.Length == 0) return 1;
static bool FatStringViewDataIsPlausible(SYS_STRING_VIEW view) {
if (!SysSvIsValid(view)) return false;
if (view.Length == 0) return true;
uintptr_t data = (uintptr_t)view.Data;
return data >= 0x10000u;
}
// Implements the name View Is Valid operation.
static int FatNameViewIsValid(SYS_STRING_VIEW name) {
if (!FatStringViewDataIsPlausible(name)) return 0;
if (name.Length == 0 || name.Length >= FAT_MAX_LFN) return 0;
static bool FatNameViewIsValid(SYS_STRING_VIEW name) {
if (!FatStringViewDataIsPlausible(name)) return false;
if (name.Length == 0 || name.Length >= FAT_MAX_LFN) return false;
for (size_t i = 0; i < name.Length; i++) {
char c = name.Data[i];
if (c == '\0' || FatIsPathSep(c)) return 0;
if (c == '\0' || FatIsPathSep(c)) return false;
}
return 1;
return true;
}
// Implements the lfn Match V operation.
static int LfnMatchV(const char* lfn, SYS_STRING_VIEW want) {
if (!lfn || !SysSvIsValid(want)) return 0;
static bool LfnMatchV(const char* lfn, SYS_STRING_VIEW want) {
if (!lfn || !SysSvIsValid(want)) return false;
size_t i = 0;
while (lfn[i] && i < want.Length) {
char a = lfn[i], b = want.Data[i];
if (a >= 'A' && a <= 'Z') a += 32;
if (b >= 'A' && b <= 'Z') b += 32;
if (a != b) return 0;
if (a != b) return false;
i++;
}
return lfn[i] == '\0' && i == want.Length;
}
static int MatchName83V(const char* dirName, const char* dirExt,
static bool MatchName83V(const char* dirName, const char* dirExt,
SYS_STRING_VIEW want) {
char buf[13];
int n = 0;
@@ -449,14 +450,14 @@ static int MatchName83V(const char* dirName, const char* dirExt,
buf[n++] = (char)(dirExt[i] >= 'A' && dirExt[i] <= 'Z' ? dirExt[i]+32 : dirExt[i]);
}
if ((size_t)n != want.Length) return 0;
if ((size_t)n != want.Length) return false;
for (int i = 0; i < n; i++) {
char a = buf[i], b = want.Data[i];
if (a >= 'A' && a <= 'Z') a += 32;
if (b >= 'A' && b <= 'Z') b += 32;
if (a != b) return 0;
if (a != b) return false;
}
return 1;
return true;
}
// Returns the path cache canonical byte used for hashes and comparisons.
@@ -719,21 +720,21 @@ bool FatBuildReadRun(FAT_FILE* f, uint32_t logicalIndex,
return true;
}
static int SearchDirV(FAT_VOL* v, uint32_t dirCluster, SYS_STRING_VIEW name,
static bool SearchDirV(FAT_VOL* v, uint32_t dirCluster, SYS_STRING_VIEW name,
uint32_t* outCluster, uint32_t* outSize, uint8_t* outAttr,
uint64_t* outDirentLba, uint32_t* outDirentOffset) {
if (!FatNameViewIsValid(name)) return 0;
if (!FatNameViewIsValid(name)) return false;
uint8_t* clusterBuf = (uint8_t*)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return 0;
if (!clusterBuf) return false;
char lfn[FAT_MAX_LFN]; int hasLfn = 0; memset(lfn, 0, sizeof(lfn));
char lfn[FAT_MAX_LFN]; bool hasLfn = false; memset(lfn, 0, sizeof(lfn));
char wantName83[8], wantExt83[3];
int allowShortMatch = FatNameIsStructurallyShort83V(name);
bool allowShortMatch = FatNameIsStructurallyShort83V(name);
Make83NameV(name, wantName83, wantExt83);
uint64_t clustersRead = 0;
uint64_t entriesScanned = 0;
int found = 0;
bool found = false;
uint32_t cursor; uint64_t lba; uint32_t sectors;
if (!FatDirectoryFirstSegment(v, dirCluster, &cursor, &lba, &sectors))
@@ -749,16 +750,16 @@ static int SearchDirV(FAT_VOL* v, uint32_t dirCluster, SYS_STRING_VIEW name,
for (uint32_t i = 0; i < entries; i++) {
entriesScanned++;
if (dir[i].Name[0] == 0x00) goto done;
if ((uint8_t)dir[i].Name[0] == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; }
if (dir[i].Attr == ATTR_LFN) { FatLfnAppend(lfn, (LFN_ENTRY*)&dir[i]); hasLfn = 1; continue; }
if ((uint8_t)dir[i].Name[0] == 0xE5) { hasLfn = false; memset(lfn, 0, sizeof(lfn)); continue; }
if (dir[i].Attr == ATTR_LFN) { FatLfnAppend(lfn, (LFN_ENTRY*)&dir[i]); hasLfn = true; continue; }
int match = 0;
bool match = false;
if (hasLfn && lfn[0]) match = LfnMatchV(lfn, name);
if (!match && allowShortMatch)
match = MatchName83V(dir[i].Name, dir[i].Ext, name);
if (!match && allowShortMatch)
match = MatchRaw83(dir[i].Name, dir[i].Ext, wantName83, wantExt83);
hasLfn = 0; memset(lfn, 0, sizeof(lfn));
hasLfn = false; memset(lfn, 0, sizeof(lfn));
if (match) {
*outCluster = FatDirentFirstCluster(v, &dir[i]);
@@ -769,7 +770,7 @@ static int SearchDirV(FAT_VOL* v, uint32_t dirCluster, SYS_STRING_VIEW name,
if (outDirentLba) *outDirentLba = lba + eb / v->BytesPerSector;
if (outDirentOffset) *outDirentOffset = (uint32_t)(eb % v->BytesPerSector);
}
found = 1;
found = true;
goto done;
}
}
@@ -781,22 +782,22 @@ done:
return found;
}
static int FatPathNextComponent(SYS_STRING_VIEW path, size_t* cursor,
static bool FatPathNextComponent(SYS_STRING_VIEW path, size_t* cursor,
SYS_STRING_VIEW* component) {
if (!cursor || !component || !SysSvIsValid(path)) return 0;
if (!cursor || !component || !SysSvIsValid(path)) return false;
size_t pos = *cursor;
while (pos < path.Length && FatIsPathSep(path.Data[pos])) pos++;
if (pos >= path.Length) {
*cursor = pos;
*component = SYS_SV_NULL;
return 0;
return false;
}
size_t start = pos;
while (pos < path.Length && !FatIsPathSep(path.Data[pos])) pos++;
*cursor = pos;
*component = SysSvFromParts(path.Data + start, pos - start);
return 1;
return true;
}
KSTATUS FatLookupPathV(FAT_VOL* v, SYS_STRING_VIEW path,
@@ -884,20 +885,20 @@ static KSTATUS SplitPathV(SYS_STRING_VIEW path, SYS_STRING_VIEW* parent,
}
// Find a free DIRENT slot.
static int FindFreeDirent(FAT_VOL* v, uint32_t dirCluster,
static bool FindFreeDirent(FAT_VOL* v, uint32_t dirCluster,
uint64_t* outLba, uint32_t* outOffset) {
uint8_t* clusterBuf = (uint8_t*)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return 0;
if (!clusterBuf) return false;
uint32_t cursor; uint64_t lba; uint32_t sectors;
if (!FatDirectoryFirstSegment(v, dirCluster, &cursor, &lba, &sectors)) {
MmFreePool(clusterBuf);
return 0;
return false;
}
do {
if (!KSUCCESS(FatReadSectors(v, lba, sectors, clusterBuf))) {
MmFreePool(clusterBuf); return 0;
MmFreePool(clusterBuf); return false;
}
uint32_t entries = (sectors * v->BytesPerSector) / sizeof(DIRENT);
DIRENT* dir = (DIRENT*)clusterBuf;
@@ -907,46 +908,46 @@ static int FindFreeDirent(FAT_VOL* v, uint32_t dirCluster,
uint64_t eb = (uint64_t)i * sizeof(DIRENT);
*outLba = lba + eb / v->BytesPerSector;
*outOffset = (uint32_t)(eb % v->BytesPerSector);
MmFreePool(clusterBuf); return 1;
MmFreePool(clusterBuf); return true;
}
}
} while (FatDirectoryNextSegment(v, dirCluster, &cursor, &lba, &sectors));
if (FatIsFixedRoot(v, dirCluster)) {
MmFreePool(clusterBuf);
return 0;
return false;
}
uint32_t newCluster = FatAppendCluster(v, cursor);
if (newCluster) {
*outLba = FatClusterToLba(v, newCluster); *outOffset = 0;
MmFreePool(clusterBuf); return 1;
MmFreePool(clusterBuf); return true;
}
MmFreePool(clusterBuf);
return 0;
return false;
}
// Find a contiguous run of free directory entries.
// VFAT LFN creation requires N long-name entries to immediately precede the
// final short entry, so the run must live inside one readable/writeable segment.
static int FindFreeDirentRun(FAT_VOL* v, uint32_t dirCluster, uint32_t needed,
static bool FindFreeDirentRun(FAT_VOL* v, uint32_t dirCluster, uint32_t needed,
uint64_t* outLba, uint32_t* outFirstIndex,
uint32_t* outSectors) {
if (!needed) return 0;
if (!needed) return false;
uint8_t* clusterBuf = (uint8_t*)MmAllocatePool(v->BytesPerCluster);
if (!clusterBuf) return 0;
if (!clusterBuf) return false;
uint32_t cursor; uint64_t lba; uint32_t sectors;
if (!FatDirectoryFirstSegment(v, dirCluster, &cursor, &lba, &sectors)) {
MmFreePool(clusterBuf);
return 0;
return false;
}
do {
if (!KSUCCESS(FatReadSectors(v, lba, sectors, clusterBuf))) {
MmFreePool(clusterBuf);
return 0;
return false;
}
uint32_t entries = (sectors * v->BytesPerSector) / sizeof(DIRENT);
@@ -963,7 +964,7 @@ static int FindFreeDirentRun(FAT_VOL* v, uint32_t dirCluster, uint32_t needed,
*outFirstIndex = runStart;
*outSectors = sectors;
MmFreePool(clusterBuf);
return 1;
return true;
}
} else {
runLen = 0;
@@ -973,7 +974,7 @@ static int FindFreeDirentRun(FAT_VOL* v, uint32_t dirCluster, uint32_t needed,
if (FatIsFixedRoot(v, dirCluster)) {
MmFreePool(clusterBuf);
return 0;
return false;
}
uint32_t newCluster = FatAppendCluster(v, cursor);
@@ -982,11 +983,11 @@ static int FindFreeDirentRun(FAT_VOL* v, uint32_t dirCluster, uint32_t needed,
*outFirstIndex = 0;
*outSectors = v->SectorsPerCluster;
MmFreePool(clusterBuf);
return 1;
return true;
}
MmFreePool(clusterBuf);
return 0;
return false;
}
// Finds the parent directory and final component for a create or delete path.
@@ -1039,7 +1040,7 @@ KSTATUS FatCreateDirent(FAT_VOL* v, SYS_STRING_VIEW path,
}
}
int useLfn = FatNameNeedsLfnForCreateV(filenameView);
bool useLfn = FatNameNeedsLfnForCreateV(filenameView);
char shortName[8], shortExt[3];
if (useLfn) {
s = DeleteLegacyBrokenDotAliasV(v, parentCluster, filenameView);
@@ -1152,17 +1153,17 @@ KSTATUS FatInitializeDirectoryCluster(FAT_VOL* v, uint32_t dirCluster,
}
// Tests whether a FAT directory entry is one of the synthetic dot entries.
static int FatIsDotEntry(const DIRENT* d) {
if (!d || !(d->Attr & ATTR_DIRECTORY)) return 0;
if (d->Name[0] != '.') return 0;
static bool FatIsDotEntry(const DIRENT* d) {
if (!d || !(d->Attr & ATTR_DIRECTORY)) return false;
if (d->Name[0] != '.') return false;
if (d->Name[1] == ' ')
return 1;
return true;
return d->Name[1] == '.' && d->Name[2] == ' ';
}
// Tests whether a directory contains any real child entries.
KSTATUS FatDirectoryIsEmpty(FAT_VOL* v, uint32_t dirCluster,
int* emptyOut) {
bool* emptyOut) {
if (!v || !emptyOut)
return NTLSTATUS_INVALID_PARAMETER;
@@ -1209,12 +1210,12 @@ KSTATUS FatDeleteDirentRunV(FAT_VOL* v, uint32_t parentCluster,
if (!clusterBuf) return NTLSTATUS_NO_MEMORY;
char lfn[FAT_MAX_LFN];
int hasLfn = 0;
bool hasLfn = false;
uint32_t lfnStart = 0;
memset(lfn, 0, sizeof(lfn));
char wantName83[8], wantExt83[3];
int allowShortMatch = FatNameIsStructurallyShort83V(name);
bool allowShortMatch = FatNameIsStructurallyShort83V(name);
Make83NameV(name, wantName83, wantExt83);
uint32_t cursor; uint64_t lba; uint32_t sectors;
@@ -1229,20 +1230,20 @@ KSTATUS FatDeleteDirentRunV(FAT_VOL* v, uint32_t parentCluster,
uint32_t entries = (sectors * v->BytesPerSector) / sizeof(DIRENT);
DIRENT* dir = (DIRENT*)clusterBuf;
hasLfn = 0;
hasLfn = false;
memset(lfn, 0, sizeof(lfn));
for (uint32_t i = 0; i < entries; i++) {
uint8_t first = (uint8_t)dir[i].Name[0];
if (first == 0x00) { MmFreePool(clusterBuf); return NTLSTATUS_NOT_FOUND; }
if (first == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; }
if (first == 0xE5) { hasLfn = false; memset(lfn, 0, sizeof(lfn)); continue; }
if (dir[i].Attr == ATTR_LFN) {
if (!hasLfn) lfnStart = i;
FatLfnAppend(lfn, (LFN_ENTRY*)&dir[i]);
hasLfn = 1;
hasLfn = true;
continue;
}
int match = 0;
bool match = false;
if (hasLfn && lfn[0]) match = LfnMatchV(lfn, name);
if (!match && allowShortMatch)
match = MatchName83V(dir[i].Name, dir[i].Ext, name);
@@ -1259,7 +1260,7 @@ KSTATUS FatDeleteDirentRunV(FAT_VOL* v, uint32_t parentCluster,
return s;
}
hasLfn = 0;
hasLfn = false;
memset(lfn, 0, sizeof(lfn));
}
} while (FatDirectoryNextSegment(v, parentCluster, &cursor, &lba, &sectors));
code/drivers/fs/fat/fat_driver.c
+13 -12
View File
@@ -1,5 +1,6 @@
// FAT mount discovery, BPB validation, and driver lifetime.
#include <stdbool.h>
#include "fat_internal.h"
// FAT BPB validation.
@@ -12,18 +13,18 @@ typedef struct FAT_BPB_LAYOUT {
uint32_t TotalClusters;
} FAT_BPB_LAYOUT;
static int FatComputeBpbLayout(const BPB* bpb, FAT_BPB_LAYOUT* out) {
if (!bpb || !out) return 0;
static bool FatComputeBpbLayout(const BPB* bpb, FAT_BPB_LAYOUT* out) {
if (!bpb || !out) return false;
memset(out, 0, sizeof(*out));
if (bpb->BytesPerSector != 512) return 0;
if (bpb->SectorsPerCluster == 0) return 0;
if (bpb->NumFats == 0) return 0;
if (bpb->ReservedSectors == 0) return 0;
if (bpb->BytesPerSector != 512) return false;
if (bpb->SectorsPerCluster == 0) return false;
if (bpb->NumFats == 0) return false;
if (bpb->ReservedSectors == 0) return false;
uint32_t totalSectors = bpb->TotalSectors16 ? bpb->TotalSectors16 : bpb->TotalSectors32;
uint32_t fatSize = bpb->FatSize16 ? bpb->FatSize16 : bpb->FatSize32;
if (!totalSectors || !fatSize) return 0;
if (!totalSectors || !fatSize) return false;
uint32_t rootDirSectors = ((uint32_t)bpb->RootEntryCount * sizeof(DIRENT) +
(uint32_t)bpb->BytesPerSector - 1u) /
@@ -31,7 +32,7 @@ static int FatComputeBpbLayout(const BPB* bpb, FAT_BPB_LAYOUT* out) {
uint32_t metadataSectors = (uint32_t)bpb->ReservedSectors +
(uint32_t)bpb->NumFats * fatSize +
rootDirSectors;
if (totalSectors <= metadataSectors) return 0;
if (totalSectors <= metadataSectors) return false;
uint32_t dataSectors = totalSectors - metadataSectors;
uint32_t totalClusters = dataSectors / bpb->SectorsPerCluster;
@@ -44,10 +45,10 @@ static int FatComputeBpbLayout(const BPB* bpb, FAT_BPB_LAYOUT* out) {
variant = FatVariant32;
if (variant == FatVariant16) {
if (bpb->FatSize16 == 0 || bpb->RootEntryCount == 0) return 0;
if (bpb->FatSize16 == 0 || bpb->RootEntryCount == 0) return false;
} else if (variant == FatVariant32) {
if (bpb->FatSize16 != 0 || bpb->RootEntryCount != 0) return 0;
if (bpb->FatSize32 == 0 || bpb->RootCluster < 2) return 0;
if (bpb->FatSize16 != 0 || bpb->RootEntryCount != 0) return false;
if (bpb->FatSize32 == 0 || bpb->RootCluster < 2) return false;
}
out->Variant = variant;
@@ -56,7 +57,7 @@ static int FatComputeBpbLayout(const BPB* bpb, FAT_BPB_LAYOUT* out) {
out->RootDirSectors = rootDirSectors;
out->DataSectors = dataSectors;
out->TotalClusters = totalClusters;
return 1;
return true;
}
code/drivers/fs/fat/fat_file.c
+2 -1
View File
@@ -1,5 +1,6 @@
// FAT synchronous file and directory operations.
#include <stdbool.h>
#include "fat_internal.h"
// Open / Close.
@@ -323,7 +324,7 @@ KSTATUS FatRemoveDir(IO_VOLUME_PROVIDER* fs, SYS_STRING_VIEW path) {
if (!direntLba) return NTLSTATUS_ACCESS_DENIED;
if (!(attr & ATTR_DIRECTORY)) return NTLSTATUS_ACCESS_DENIED;
int empty = 0;
bool empty = false;
s = FatDirectoryIsEmpty(v, cluster, &empty);
if (!KSUCCESS(s)) return s;
if (!empty) return NTLSTATUS_ACCESS_DENIED;
code/drivers/fs/fat/fat_internal.h
+8 -7
View File
@@ -2,6 +2,7 @@
#pragma once
#include <stdbool.h>
#include "fat.h"
#include "fat_disk.h"
@@ -99,21 +100,21 @@ KSTATUS FatReadSectors(FAT_VOL* volume, uint64_t lba, uint32_t count, void* buff
KSTATUS FatWriteSectors(FAT_VOL* volume, uint64_t lba, uint32_t count, const void* buffer);
const char* FatVariantName(FAT_VARIANT variant);
uint32_t FatEocMark(FAT_VOL* volume);
int FatIsDataCluster(FAT_VOL* volume, uint32_t cluster);
bool FatIsDataCluster(FAT_VOL* volume, uint32_t cluster);
uint32_t FatDirentFirstCluster(FAT_VOL* volume, const DIRENT* dirent);
void FatSetDirentFirstCluster(FAT_VOL* volume, DIRENT* dirent, uint32_t cluster);
int FatIsFixedRoot(FAT_VOL* volume, uint32_t dirCluster);
int FatDirectoryFirstSegment(FAT_VOL* volume, uint32_t dirCluster,
bool FatIsFixedRoot(FAT_VOL* volume, uint32_t dirCluster);
bool FatDirectoryFirstSegment(FAT_VOL* volume, uint32_t dirCluster,
uint32_t* cursorOut, uint64_t* lbaOut, uint32_t* sectorsOut);
int FatDirectoryNextSegment(FAT_VOL* volume, uint32_t dirCluster,
bool FatDirectoryNextSegment(FAT_VOL* volume, uint32_t dirCluster,
uint32_t* cursorInOut, uint64_t* lbaOut, uint32_t* sectorsOut);
uint64_t FatClusterToLba(FAT_VOL* volume, uint32_t cluster);
uint32_t FatEntry(FAT_VOL* volume, uint32_t cluster);
KSTATUS FatWriteEntry(FAT_VOL* volume, uint32_t cluster, uint32_t value);
void FatFlushCache(FAT_VOL* volume);
uint32_t FatAllocClusterEx(FAT_VOL* volume, int zeroCluster);
uint32_t FatAllocClusterEx(FAT_VOL* volume, bool zeroCluster);
uint32_t FatAllocCluster(FAT_VOL* volume);
uint32_t FatAppendClusterEx(FAT_VOL* volume, uint32_t lastCluster, int zeroCluster);
uint32_t FatAppendClusterEx(FAT_VOL* volume, uint32_t lastCluster, bool zeroCluster);
uint32_t FatAppendCluster(FAT_VOL* volume, uint32_t lastCluster);
uint32_t FatGetChainEnd(FAT_VOL* volume, uint32_t start, uint32_t* countOut);
void FatFreeClusterChain(FAT_VOL* volume, uint32_t start);
@@ -145,7 +146,7 @@ KSTATUS FatCreateDirent(FAT_VOL* volume, SYS_STRING_VIEW path,
uint64_t* direntLbaOut, uint32_t* direntOffsetOut);
KSTATUS FatInitializeDirectoryCluster(FAT_VOL* volume, uint32_t dirCluster,
uint32_t parentCluster);
KSTATUS FatDirectoryIsEmpty(FAT_VOL* volume, uint32_t dirCluster, int* emptyOut);
KSTATUS FatDirectoryIsEmpty(FAT_VOL* volume, uint32_t dirCluster, bool* emptyOut);
KSTATUS FatDeleteDirentRunV(FAT_VOL* volume, uint32_t parentCluster,
SYS_STRING_VIEW name);
void FatCacheInvalidate(void);
code/drivers/input/ps2/ps2.c
+2 -2
View File
@@ -216,8 +216,8 @@ static void Ps2MouseIsr(void) {
if (flags & 0xC0u)
return;
int dx = (int)gMousePacket[1] - ((flags & 0x10u) ? 256 : 0);
int dy = (int)gMousePacket[2] - ((flags & 0x20u) ? 256 : 0);
int32_t dx = (int32_t)gMousePacket[1] - ((flags & 0x10u) ? 256 : 0);
int32_t dy = (int32_t)gMousePacket[2] - ((flags & 0x20u) ? 256 : 0);
dy = -dy;
int next = (gMouseDpcTail + 1) & (int)MOUSE_DPC_QUEUE_MASK;
code/drivers/input/ps2/ps2.h
+2 -2
View File
@@ -33,8 +33,8 @@
#define MOUSE_DPC_QUEUE_MASK (MOUSE_DPC_QUEUE_SIZE - 1u)
typedef struct PS2_MOUSE_PACKET {
int dx;
int dy;
int32_t dx;
int32_t dy;
uint8_t buttons;
} PS2_MOUSE_PACKET;
code/drivers/net/lwip/lwip_stack.c
+29 -28
View File
@@ -1,5 +1,6 @@
// lwIP stack provider glue. This file bridges the kernel network-device model into lwIP.
#include <ntlite/atomic.h>
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
@@ -77,7 +78,7 @@ typedef struct LWIP_ATTACH_CTX {
typedef struct LWIP_RESOLVE_CONTEXT {
SEMAPHORE* Done;
volatile long References;
volatile int32_t References;
err_t Result;
u8_t DnsType;
ip_addr_t Address;
@@ -167,12 +168,12 @@ static KSTATUS LwipErrToStatus(err_t err) {
}
// Implements the ascii Is Digit operation.
static int LwipAsciiIsDigit(char c) {
static bool LwipAsciiIsDigit(char c) {
return c >= '0' && c <= '9';
}
// Parses ipv4 Part.
static int LwipParseIpv4Part(const char** cursor, uint8_t* out) {
static bool LwipParseIpv4Part(const char** cursor, uint8_t* out) {
const char* p = *cursor;
uint32_t value = 0;
uint32_t digits = 0;
@@ -180,37 +181,37 @@ static int LwipParseIpv4Part(const char** cursor, uint8_t* out) {
while (LwipAsciiIsDigit(*p)) {
value = value * 10u + (uint32_t)(*p - '0');
if (value > 255u)
return 0;
return false;
p++;
digits++;
}
if (digits == 0)
return 0;
return false;
*out = (uint8_t)value;
*cursor = p;
return 1;
return true;
}
// Parses ipv4.
static int LwipParseIpv4(const char* text, ip4_addr_t* out) {
static bool LwipParseIpv4(const char* text, ip4_addr_t* out) {
if (!text || !out)
return 0;
return false;
const char* p = text;
uint8_t a = 0, b = 0, c = 0, d = 0;
if (!LwipParseIpv4Part(&p, &a) || *p++ != '.') return 0;
if (!LwipParseIpv4Part(&p, &b) || *p++ != '.') return 0;
if (!LwipParseIpv4Part(&p, &c) || *p++ != '.') return 0;
if (!LwipParseIpv4Part(&p, &d) || *p != '\0') return 0;
if (!LwipParseIpv4Part(&p, &a) || *p++ != '.') return false;
if (!LwipParseIpv4Part(&p, &b) || *p++ != '.') return false;
if (!LwipParseIpv4Part(&p, &c) || *p++ != '.') return false;
if (!LwipParseIpv4Part(&p, &d) || *p != '\0') return false;
IP4_ADDR(out, a, b, c, d);
return 1;
return true;
}
static KSTATUS LwipGetSettingIpv4(const char* section, const char* key,
ip4_addr_t* out, int required) {
ip4_addr_t* out, bool required) {
char value[LWIP_SETTINGS_VALUE_MAX];
KSTATUS st = KeGetSettingString(section, key, value, sizeof(value));
if (st == NTLSTATUS_NOT_FOUND && !required)
@@ -225,7 +226,7 @@ static KSTATUS LwipGetSettingIpv4(const char* section, const char* key,
}
static KSTATUS LwipGetSettingIpv6(const char* section, const char* key,
ip6_addr_t* out, int required) {
ip6_addr_t* out, bool required) {
char value[LWIP_SETTINGS_VALUE_MAX];
KSTATUS st = KeGetSettingString(section, key, value, sizeof(value));
if (st == NTLSTATUS_NOT_FOUND && !required)
@@ -308,13 +309,13 @@ static KSTATUS LwipLoadConfigFromSection(const char* section, LWIP_NETIF_CONFIG*
if (strcmp(mode, "static") == 0) {
cfg->UseDhcp = 0;
st = LwipGetSettingIpv4(section, "address", &cfg->Address, 1);
st = LwipGetSettingIpv4(section, "address", &cfg->Address, true);
if (!KSUCCESS(st)) return st;
st = LwipGetSettingIpv4(section, "netmask", &cfg->Netmask, 1);
st = LwipGetSettingIpv4(section, "netmask", &cfg->Netmask, true);
if (!KSUCCESS(st)) return st;
st = LwipGetSettingIpv4(section, "gateway", &cfg->Gateway, 1);
st = LwipGetSettingIpv4(section, "gateway", &cfg->Gateway, true);
if (!KSUCCESS(st)) return st;
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns4, 1);
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns4, true);
if (!KSUCCESS(st)) return st;
cfg->HasDns4 = 1;
return LwipLoadIpv6Config(section, cfg);
@@ -322,7 +323,7 @@ static KSTATUS LwipLoadConfigFromSection(const char* section, LWIP_NETIF_CONFIG*
if (strcmp(mode, "dhcp") == 0) {
cfg->UseDhcp = 1;
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns4, 0);
st = LwipGetSettingIpv4(section, "dns", &cfg->Dns4, false);
if (KSUCCESS(st))
cfg->HasDns4 = 1;
else if (st != NTLSTATUS_NOT_FOUND)
@@ -449,7 +450,7 @@ static KSTATUS LwipSocketAllocateWithConn(uint32_t domain, uint32_t type, uint32
// Returns whether the IRP cancel token has been set by SysCancelIo.
static int LwipSocketCancelRequested(volatile uint8_t* cancelRequested) {
static bool LwipSocketCancelRequested(volatile uint8_t* cancelRequested) {
return cancelRequested && *cancelRequested;
}
@@ -552,22 +553,22 @@ static KSTATUS LwipSocketWaitAccept(LWIP_PROVIDER_SOCKET* s,
}
}
static int LwipSocketAddressValid(const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
static bool LwipSocketAddressValid(const SYS_SOCKET_ADDRESS_K* address, size_t addressSize) {
if (!address || addressSize != sizeof(SYS_SOCKET_ADDRESS_K))
return 0;
return false;
if (address->Size != sizeof(SYS_SOCKET_ADDRESS_K))
return 0;
return false;
return address->Family == SYS_AF_INET || address->Family == SYS_AF_INET6;
}
static int LwipSocketFamilyMatches(const LWIP_PROVIDER_SOCKET* s, const SYS_SOCKET_ADDRESS_K* address) {
static bool LwipSocketFamilyMatches(const LWIP_PROVIDER_SOCKET* s, const SYS_SOCKET_ADDRESS_K* address) {
if (!s || !address)
return 0;
return false;
if (s->Domain == SYS_AF_INET)
return address->Family == SYS_AF_INET;
if (s->Domain == SYS_AF_INET6)
return address->Family == SYS_AF_INET6;
return 0;
return false;
}
// Implements the socket Make Ip operation.
@@ -1613,7 +1614,7 @@ static void LwipResolveRelease(LWIP_RESOLVE_CONTEXT* context) {
if (!context)
return;
if (_InterlockedDecrement(&context->References) != 0)
if (NtlAtomicDecrement32(&context->References) != 0)
return;
if (context->Done)
code/drivers/storage/ahci/ahci.c
+8 -8
View File
@@ -266,7 +266,7 @@ static void AhciIsr(void) {
if (!pending)
continue;
for (int i = 0; i < ctl->driveCount; i++) {
for (uint32_t i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
uint32_t portBit = (1u << drv->portNo);
if (!(pending & portBit)) continue;
@@ -676,7 +676,7 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_SURPRISE_REMOVAL:
ctl->surpriseRemoved = true;
device->PnpState = DevicePnpSurpriseRemoved;
for (int i = 0; i < ctl->driveCount; i++) {
for (uint32_t i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
drv->surpriseRemoved = true;
if (drv->deviceObject)
@@ -689,7 +689,7 @@ static KSTATUS AhciDispatchControllerPnp(DEVICE_OBJECT* device, IRP* irp) {
case IRP_MN_REMOVE_DEVICE:
ctl->removed = true;
device->PnpState = DevicePnpDeleted;
for (int i = 0; i < ctl->driveCount; i++) {
for (uint32_t i = 0; i < ctl->driveCount; i++) {
AHCI_DRIVE* drv = &ctl->drives[i];
if (drv->deviceObject) {
(void)IoUnregisterDiskDevice(drv->deviceObject->u.Disk.DiskIndex);
@@ -974,7 +974,7 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
DbgPrint("AHCI: port %u: link up, initializing...\n", portno);
int driveIdx = ctl->driveCount;
uint32_t driveIdx = ctl->driveCount;
AHCI_DRIVE* drv = &ctl->drives[driveIdx];
memset(drv, 0, sizeof(*drv));
drv->magic = AHCI_DRIVE_MAGIC;
@@ -1119,11 +1119,11 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
ctl->pciInfo.Address.Function);
if (!ctl->pollingMode) {
DbgPrint("AHCI: ISR registered on IRQ %u (%s); controller drives=%d\n",
(unsigned)ctl->irq, ctl->usingMsi ? "MSI" : "INTx", ctl->driveCount);
DbgPrint("AHCI: ISR registered on IRQ %u (%s); controller drives=%u\n",
(unsigned)ctl->irq, ctl->usingMsi ? "MSI" : "INTx", (unsigned)ctl->driveCount);
} else {
DbgPrint("AHCI: controller started in polling mode; controller drives=%d\n",
ctl->driveCount);
DbgPrint("AHCI: controller started in polling mode; controller drives=%u\n",
(unsigned)ctl->driveCount);
}
ctl->started = true;
ctl->fdo->PnpState = DevicePnpStarted;
code/drivers/storage/ahci/ahci.h
+1 -1
View File
@@ -378,7 +378,7 @@ struct AHCI_CONTROLLER {
bool pollingMode;
AHCI_DRIVE drives[MAX_AHCI_DRIVES];
int driveCount;
uint32_t driveCount;
bool started;
bool surpriseRemoved;
code/kernel/hal/hal_main.c
+36 -36
View File
@@ -13,7 +13,7 @@
#include <intrin.h>
#include <common/printf.h>
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int32_t w, int32_t h, int32_t stride,
HAL_SCREEN* out);
void HalPciInitialize(HAL_PCI* out);
void HalRtcInitialize(HAL_RTC* out);
@@ -215,15 +215,15 @@ static uint64_t AcpiReadU64(const void* table, uint32_t tableLen, uint32_t off)
}
// Decodes one ACPI Generic Address Structure used for fixed power registers.
static int AcpiReadGasRegister(const void* table, uint32_t tableLen, uint32_t off,
static bool AcpiReadGasRegister(const void* table, uint32_t tableLen, uint32_t off,
ACPI_FIXED_REGISTER* out) {
if (!out || off + sizeof(ACPI_GAS) > tableLen) return 0;
if (!out || off + sizeof(ACPI_GAS) > tableLen) return false;
ACPI_GAS gas;
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
if (!gas.Address || gas.RegisterBitOffset != 0) return 0;
if (gas.AddressSpaceId != 0x00 && gas.AddressSpaceId != 0x01) return 0;
if (gas.AddressSpaceId == 0x01 && gas.Address > 0xFFFFu) return 0;
if (!gas.Address || gas.RegisterBitOffset != 0) return false;
if (gas.AddressSpaceId != 0x00 && gas.AddressSpaceId != 0x01) return false;
if (gas.AddressSpaceId == 0x01 && gas.Address > 0xFFFFu) return false;
uint8_t widthBytes = 0;
if (gas.AccessSize >= 1 && gas.AccessSize <= 3)
@@ -236,24 +236,24 @@ static int AcpiReadGasRegister(const void* table, uint32_t tableLen, uint32_t of
out->AddressSpaceId = gas.AddressSpaceId;
out->WidthBytes = widthBytes;
out->Address = gas.Address;
return 1;
return true;
}
// Decodes one byte-wide ACPI GAS used by the HW-reduced Sleep Control register.
static int AcpiReadByteGasRegister(const void* table, uint32_t tableLen, uint32_t off,
static bool AcpiReadByteGasRegister(const void* table, uint32_t tableLen, uint32_t off,
ACPI_FIXED_REGISTER* out) {
if (!out || off + sizeof(ACPI_GAS) > tableLen) return 0;
if (!out || off + sizeof(ACPI_GAS) > tableLen) return false;
ACPI_GAS gas;
memcpy(&gas, (const uint8_t*)table + off, sizeof(gas));
if (!gas.Address || gas.RegisterBitOffset != 0 || gas.RegisterBitWidth != 8) return 0;
if (gas.AddressSpaceId != 0x00 && gas.AddressSpaceId != 0x01) return 0;
if (gas.AddressSpaceId == 0x01 && gas.Address > 0xFFFFu) return 0;
if (!gas.Address || gas.RegisterBitOffset != 0 || gas.RegisterBitWidth != 8) return false;
if (gas.AddressSpaceId != 0x00 && gas.AddressSpaceId != 0x01) return false;
if (gas.AddressSpaceId == 0x01 && gas.Address > 0xFFFFu) return false;
out->AddressSpaceId = gas.AddressSpaceId;
out->WidthBytes = 1u;
out->Address = gas.Address;
return 1;
return true;
}
// Reads one ACPI fixed register while preserving the address-space distinction.
@@ -341,9 +341,9 @@ static KSTATUS AcpiEnableModeForShutdown(void) {
}
}
static int AmlParsePkgLength(const uint8_t* p, const uint8_t* end,
static bool AmlParsePkgLength(const uint8_t* p, const uint8_t* end,
const uint8_t** body, const uint8_t** pkgEnd) {
if (p >= end) return 0;
if (p >= end) return false;
const uint8_t* start = p;
uint8_t first = *p++;
@@ -355,21 +355,21 @@ static int AmlParsePkgLength(const uint8_t* p, const uint8_t* end,
} else {
length = first & 0x0F;
for (uint32_t i = 0; i < follow; i++) {
if (p >= end) return 0;
if (p >= end) return false;
length |= ((uint32_t)*p++) << (4 + i * 8);
}
}
if (start + length < p || start + length > end) return 0;
if (start + length < p || start + length > end) return false;
*body = p;
*pkgEnd = start + length;
return 1;
return true;
}
static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
int* lastSegmentIsS5) {
bool* lastSegmentIsS5) {
uint32_t segments = 1;
*lastSegmentIsS5 = 0;
*lastSegmentIsS5 = false;
// Optional root or parent prefixes: '\\', '^', '^^', ...
while (p < end && (*p == 0x5C || *p == 0x5E)) p++;
@@ -389,7 +389,7 @@ static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
for (uint32_t s = 0; s < segments; s++) {
if (p + 4 > end) return NULL;
if (s == segments - 1 && memcmp(p, "_S5_", 4) == 0)
*lastSegmentIsS5 = 1;
*lastSegmentIsS5 = true;
p += 4;
}
@@ -397,9 +397,9 @@ static const uint8_t* AmlParseNameString(const uint8_t* p, const uint8_t* end,
}
// Implements the aml Read Integer operation.
static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
static bool AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out) {
const uint8_t* p = *pp;
if (p >= end) return 0;
if (p >= end) return false;
switch (*p++) {
case 0x00: // ZeroOp
@@ -409,43 +409,43 @@ static int AmlReadInteger(const uint8_t** pp, const uint8_t* end, uint8_t* out)
*out = 1;
break;
case 0x0A: // BytePrefix
if (p >= end) return 0;
if (p >= end) return false;
*out = *p++;
break;
case 0x0B: // WordPrefix
if (p + 2 > end) return 0;
if (p + 2 > end) return false;
*out = p[0];
p += 2;
break;
case 0x0C: // DWordPrefix
if (p + 4 > end) return 0;
if (p + 4 > end) return false;
*out = p[0];
p += 4;
break;
case 0x0E: // QWordPrefix
if (p + 8 > end) return 0;
if (p + 8 > end) return false;
*out = p[0];
p += 8;
break;
default:
return 0;
return false;
}
*pp = p;
return 1;
return true;
}
// Scan DSDT AML for the _S5_ Name object and extract SLP_TYP values. Real
// firmware commonly encodes the name as "\\_S5_" instead of just "_S5_", and
// package integers may be ZeroOp/OneOp/Byte/Word/DWord/QWord constants.
static int ScanAmlForS5(const uint8_t* aml, uint32_t len, const char* source) {
static bool ScanAmlForS5(const uint8_t* aml, uint32_t len, const char* source) {
const uint8_t* end = aml + len;
for (const uint8_t* cur = aml; cur < end; cur++) {
if (*cur != 0x08) continue; // NameOp
const uint8_t* p = cur + 1;
int isS5 = 0;
bool isS5 = false;
p = AmlParseNameString(p, end, &isS5);
if (!p || !isS5 || p >= end || *p != 0x12) continue; // PackageOp
@@ -467,14 +467,14 @@ static int ScanAmlForS5(const uint8_t* aml, uint32_t len, const char* source) {
HalDbgPrint("HAL: _S5_ source=%s SLP_TYP_A=0x%02x SLP_TYP_B=0x%02x\n",
source ? source : "AML", gSlpTypA, gSlpTypB);
return 1;
return true;
}
return 0;
return false;
}
// Validates checksum.
static int validate_checksum(void* table, uint32_t length) {
static bool validate_checksum(void* table, uint32_t length) {
uint8_t sum = 0;
uint8_t* bytes = (uint8_t*)table;
for (uint32_t i = 0; i < length; i++) {
@@ -627,7 +627,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
ACPISDTHeader* sdtHeader = NULL;
uint32_t count = 0;
int isXsdt = 0;
bool isXsdt = false;
// Safely find and validate the XSDT (ACPI 2.0+).
if (rsdp->Revision >= 2 && rsdp->XsdtAddress != 0) {
@@ -639,7 +639,7 @@ static void WalkAcpi(uintptr_t rsdpPhys) {
sdtHeader = &xsdt->Header;
count = (xsdt->Header.Length - sizeof(ACPISDTHeader)) / 8;
isXsdt = 1;
isXsdt = true;
}
}
code/kernel/hal/hal_output.c
+26 -26
View File
@@ -1,5 +1,6 @@
// HAL output router. This file fans typed text streams out to registered physical and runtime endpoints.
#include <ntlite/atomic.h>
#include <stdint.h>
#include <string.h>
#include <intrin.h>
@@ -10,26 +11,26 @@
typedef struct HAL_OUTPUT_SLOT {
HAL_OUTPUT_ENDPOINT Endpoint;
volatile long Published;
volatile int32_t Published;
} HAL_OUTPUT_SLOT;
static HAL_OUTPUT_SLOT gOutputSlots[HAL_OUTPUT_ENDPOINT_CAPACITY];
static volatile long gOutputRegistrationLock;
static volatile long long gOutputRecordCount;
static volatile long long gOutputByteCount;
static volatile long long gOutputEndpointWriteCount;
static volatile long long gOutputEmergencyRecordCount;
static volatile long gOutputEndpointCount;
static volatile int32_t gOutputRegistrationLock;
static volatile int64_t gOutputRecordCount;
static volatile int64_t gOutputByteCount;
static volatile int64_t gOutputEndpointWriteCount;
static volatile int64_t gOutputEmergencyRecordCount;
static volatile int32_t gOutputEndpointCount;
// Acquires the endpoint registration lock.
static void HalOutputAcquireRegistrationLock(void) {
while (_InterlockedCompareExchange(&gOutputRegistrationLock, 1, 0) != 0)
while (NtlAtomicCompareExchange32(&gOutputRegistrationLock, 1, 0) != 0)
_mm_pause();
}
// Releases the endpoint registration lock.
static void HalOutputReleaseRegistrationLock(void) {
_InterlockedExchange(&gOutputRegistrationLock, 0);
NtlAtomicExchange32(&gOutputRegistrationLock, 0);
}
// Registers one output endpoint in the fixed endpoint table.
@@ -42,13 +43,13 @@ static KSTATUS HalOutputRegisterEndpoint(const HAL_OUTPUT_ENDPOINT* endpoint, ui
HalOutputAcquireRegistrationLock();
for (uint32_t i = 0; i < HAL_OUTPUT_ENDPOINT_CAPACITY; i++) {
HAL_OUTPUT_SLOT* slot = &gOutputSlots[i];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) != 0)
if (NtlAtomicLoad32(&slot->Published) != 0)
continue;
slot->Endpoint = *endpoint;
_ReadWriteBarrier();
_InterlockedExchange(&slot->Published, 1);
_InterlockedIncrement(&gOutputEndpointCount);
NtlAtomicExchange32(&slot->Published, 1);
NtlAtomicIncrement32(&gOutputEndpointCount);
*endpointIdOut = i;
status = NTLSTATUS_SUCCESS;
break;
@@ -64,10 +65,10 @@ static KSTATUS HalOutputSetEndpointStreams(uint32_t endpointId, uint32_t streamM
return NTLSTATUS_INVALID_PARAMETER;
HAL_OUTPUT_SLOT* slot = &gOutputSlots[endpointId];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) == 0)
if (NtlAtomicLoad32(&slot->Published) == 0)
return NTLSTATUS_NOT_FOUND;
_InterlockedExchange((volatile long*)&slot->Endpoint.StreamMask, (long)streamMask);
NtlAtomicExchangeU32(&slot->Endpoint.StreamMask, streamMask);
return NTLSTATUS_SUCCESS;
}
@@ -78,18 +79,17 @@ static void HalOutputWrite(const HAL_OUTPUT_RECORD* record) {
(record->Stream & (record->Stream - 1u)) != 0)
return;
_InterlockedIncrement64(&gOutputRecordCount);
_InterlockedExchangeAdd64(&gOutputByteCount, (long long)record->Length);
NtlAtomicIncrement64(&gOutputRecordCount);
NtlAtomicExchangeAdd64(&gOutputByteCount, (int64_t)record->Length);
if (record->Flags & HAL_OUTPUT_WRITE_EMERGENCY)
_InterlockedIncrement64(&gOutputEmergencyRecordCount);
NtlAtomicIncrement64(&gOutputEmergencyRecordCount);
for (uint32_t i = 0; i < HAL_OUTPUT_ENDPOINT_CAPACITY; i++) {
HAL_OUTPUT_SLOT* slot = &gOutputSlots[i];
if (_InterlockedCompareExchange(&slot->Published, 0, 0) == 0)
if (NtlAtomicLoad32(&slot->Published) == 0)
continue;
uint32_t streams = (uint32_t)_InterlockedCompareExchange(
(volatile long*)&slot->Endpoint.StreamMask, 0, 0);
uint32_t streams = NtlAtomicLoadU32(&slot->Endpoint.StreamMask);
if ((streams & record->Stream) == 0)
continue;
if ((record->Flags & HAL_OUTPUT_WRITE_EMERGENCY) &&
@@ -102,7 +102,7 @@ static void HalOutputWrite(const HAL_OUTPUT_RECORD* record) {
continue;
write(context, record);
_InterlockedIncrement64(&gOutputEndpointWriteCount);
NtlAtomicIncrement64(&gOutputEndpointWriteCount);
}
}
@@ -112,11 +112,11 @@ static void HalOutputQueryStats(HAL_OUTPUT_STATS* statsOut) {
return;
memset(statsOut, 0, sizeof(*statsOut));
statsOut->RecordCount = (uint64_t)_InterlockedCompareExchange64(&gOutputRecordCount, 0, 0);
statsOut->ByteCount = (uint64_t)_InterlockedCompareExchange64(&gOutputByteCount, 0, 0);
statsOut->EndpointWriteCount = (uint64_t)_InterlockedCompareExchange64(&gOutputEndpointWriteCount, 0, 0);
statsOut->EmergencyRecordCount = (uint64_t)_InterlockedCompareExchange64(&gOutputEmergencyRecordCount, 0, 0);
statsOut->EndpointCount = (uint32_t)_InterlockedCompareExchange(&gOutputEndpointCount, 0, 0);
statsOut->RecordCount = (uint64_t)NtlAtomicLoad64(&gOutputRecordCount);
statsOut->ByteCount = (uint64_t)NtlAtomicLoad64(&gOutputByteCount);
statsOut->EndpointWriteCount = (uint64_t)NtlAtomicLoad64(&gOutputEndpointWriteCount);
statsOut->EmergencyRecordCount = (uint64_t)NtlAtomicLoad64(&gOutputEmergencyRecordCount);
statsOut->EndpointCount = (uint32_t)NtlAtomicLoad32(&gOutputEndpointCount);
}
// Initializes the fixed HAL output endpoint table.
code/kernel/hal/hal_screen.c
+25 -25
View File
@@ -26,12 +26,12 @@ static int gCursorX = 0, gCursorY = 0;
#include <kernel/hal/font8x16.h> // see font8x16.h
// Internals.
static inline uint32_t* ScreenBackbufferPixel(int x, int y) {
static inline uint32_t* ScreenBackbufferPixel(int32_t x, int32_t y) {
return gBackbuffer ? (gBackbuffer + y * gStride + x) : (gFb + y * gStride + x);
}
// Updates pixel Raw.
static inline void PutPixelRaw(int x, int y, uint32_t color) {
static inline void PutPixelRaw(int32_t x, int32_t y, uint32_t color) {
if ((unsigned)x >= (unsigned)gWidth || (unsigned)y >= (unsigned)gHeight)
return;
@@ -60,26 +60,26 @@ static void ScreenFlushRows(int startY, int rows) {
// HAL_SCREEN implementations.
static void ScreenClear(uint32_t color) {
for (int y = 0; y < gHeight; y++)
for (int x = 0; x < gWidth; x++)
for (int32_t y = 0; y < gHeight; y++)
for (int32_t x = 0; x < gWidth; x++)
ScreenBackbufferPixel(x, y)[0] = color;
ScreenFlushRows(0, gHeight);
if (!gBackbuffer) {
// ScreenFlushRows is a no-op without a backbuffer.
for (int y = 0; y < gHeight; y++)
for (int x = 0; x < gWidth; x++)
for (int32_t y = 0; y < gHeight; y++)
for (int32_t x = 0; x < gWidth; x++)
gFb[y * gStride + x] = color;
}
gCursorX = 0; gCursorY = 0;
}
// Implements the screen Put Pixel operation.
static void ScreenPutPixel(int x, int y, uint32_t color) {
static void ScreenPutPixel(int32_t x, int32_t y, uint32_t color) {
PutPixelRaw(x, y, color);
}
// Implements the screen Blit operation.
static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
static void ScreenBlit(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels) {
// Calculate clipping bounds.
int startX = (x < 0) ? 0 : x;
int startY = (y < 0) ? 0 : y;
@@ -101,7 +101,7 @@ static void ScreenBlit(int x, int y, int w, int h, const uint32_t* pixels) {
// Calculate bytes to copy per row.
size_t bytesPerRow = (size_t)drawW * sizeof(uint32_t);
for (int row = 0; row < drawH; row++) {
for (int32_t row = 0; row < drawH; row++) {
uint32_t* shadow = ScreenBackbufferPixel(startX, startY + row);
uint32_t* fb = gFb + ((startY + row) * gStride) + startX;
const uint32_t* src = pixels + ((srcOffsetY + row) * w) + srcOffsetX;
@@ -128,7 +128,7 @@ static KSTATUS ScreenGetFramebuffer(HAL_FRAMEBUFFER_INFO* outInfo) {
}
// Implements the screen Scroll operation.
static void ScreenScroll(int lines) {
static void ScreenScroll(int32_t lines) {
if (lines <= 0)
return;
if (lines >= gHeight) {
@@ -145,8 +145,8 @@ static void ScreenScroll(int lines) {
// Fill the revealed rows with the console background. Do not memset here:
// the background color is not zero.
uint32_t* fill = shadow + (gHeight - lines) * gStride;
for (int y = 0; y < lines; y++) {
for (int x = 0; x < gWidth; x++)
for (int32_t y = 0; y < lines; y++) {
for (int32_t x = 0; x < gWidth; x++)
fill[y * gStride + x] = SCREEN_DEFAULT_BG;
}
@@ -154,11 +154,11 @@ static void ScreenScroll(int lines) {
}
// Implements the screen Put Char operation.
static void ScreenPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
static void ScreenPutChar(char c, int32_t x, int32_t y, uint32_t fg, uint32_t bg) {
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16);
for (int row = 0; row < 16; row++) {
for (int32_t row = 0; row < 16; row++) {
uint8_t bits = glyph[row];
for (int col = 0; col < 8; col++) {
for (int32_t col = 0; col < 8; col++) {
uint32_t color = (bits & (0x80u >> col)) ? fg : bg;
PutPixelRaw(x + col, y + row, color);
}
@@ -234,10 +234,10 @@ static int gCursorDrawX = -1; // -1 = cursor not yet drawn
static int gCursorDrawY = -1;
// Save the background rectangle [x,y,CURSOR_W,CURSOR_H] from the framebuffer.
static void CursorSaveBg(int x, int y) {
for (int row = 0; row < CURSOR_H; row++) {
static void CursorSaveBg(int32_t x, int32_t y) {
for (int32_t row = 0; row < CURSOR_H; row++) {
int fy = y + row;
for (int col = 0; col < CURSOR_W; col++) {
for (int32_t col = 0; col < CURSOR_W; col++) {
int fx = x + col;
uint32_t px = 0;
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight)
@@ -248,10 +248,10 @@ static void CursorSaveBg(int x, int y) {
}
// Restore the background previously saved by CursorSaveBg.
static void CursorRestoreBg(int x, int y) {
for (int row = 0; row < CURSOR_H; row++) {
static void CursorRestoreBg(int32_t x, int32_t y) {
for (int32_t row = 0; row < CURSOR_H; row++) {
int fy = y + row;
for (int col = 0; col < CURSOR_W; col++) {
for (int32_t col = 0; col < CURSOR_W; col++) {
int fx = x + col;
if ((unsigned)fx < (unsigned)gWidth && (unsigned)fy < (unsigned)gHeight) {
uint32_t px = gBackbuffer
@@ -287,13 +287,13 @@ static inline uint32_t BlendPixel(uint32_t dst, uint32_t src) {
// Draw the arrow sprite at (x,y), saving the background first.
// Called from hal_mouse.c via extern declaration.
void HalScreenDrawCursor(int x, int y) {
void HalScreenDrawCursor(int32_t x, int32_t y) {
CursorSaveBg(x, y);
gCursorDrawX = x;
gCursorDrawY = y;
for (int row = 0; row < CURSOR_H; row++) {
for (int32_t row = 0; row < CURSOR_H; row++) {
int fy = y + row;
for (int col = 0; col < CURSOR_W; col++) {
for (int32_t col = 0; col < CURSOR_W; col++) {
uint32_t src = gCursorPixels[row][col];
if ((src >> 24) == 0) continue; // fully transparent, skip
int fx = x + col;
@@ -341,7 +341,7 @@ void HalScreenRegisterOutputEndpoint(HAL_OUTPUT_ROUTER* output) {
}
// Populate the HAL_SCREEN vtable.
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int w, int h, int stride,
void HalScreenInitialize(void* (*AllocatePool)(size_t size), uintptr_t fb, int32_t w, int32_t h, int32_t stride,
HAL_SCREEN* out) {
gFb = (uint32_t*)fb;
gWidth = w;
code/kernel/hal/hal_serial.c
+7 -6
View File
@@ -1,5 +1,6 @@
// HAL serial output endpoint. This file owns polling COM1 writes and subscribes serial to HAL text streams.
#include <ntlite/atomic.h>
#include <stdint.h>
#include <stdbool.h>
#include <intrin.h>
@@ -17,8 +18,8 @@
#define HAL_SERIAL_TIMER_SPIN_CAP 65536u
#define HAL_SERIAL_EARLY_SPINS 4096u
static volatile long gSerialWriteLock;
static volatile long gSerialPresent;
static volatile int32_t gSerialWriteLock;
static volatile int32_t gSerialPresent;
// Probes the 16550 scratch register without relying on line-status values from an absent port.
static bool HalSerialProbeCom1(void) {
@@ -69,20 +70,20 @@ static bool HalSerialWriteByte(uint8_t value) {
// Acquires the normal serial writer lock.
static void HalSerialAcquireWriteLock(void) {
while (_InterlockedCompareExchange(&gSerialWriteLock, 1, 0) != 0)
while (NtlAtomicCompareExchange32(&gSerialWriteLock, 1, 0) != 0)
_mm_pause();
}
// Releases the normal serial writer lock.
static void HalSerialReleaseWriteLock(void) {
_InterlockedExchange(&gSerialWriteLock, 0);
NtlAtomicExchange32(&gSerialWriteLock, 0);
}
// Writes one output record through polling COM1.
static void HalSerialWriteRecord(void* context, const HAL_OUTPUT_RECORD* record) {
(void)context;
if (!record || !record->Buffer ||
_InterlockedCompareExchange(&gSerialPresent, 0, 0) == 0)
NtlAtomicLoad32(&gSerialPresent) == 0)
return;
bool emergency = (record->Flags & HAL_OUTPUT_WRITE_EMERGENCY) != 0;
@@ -102,7 +103,7 @@ static void HalSerialWriteRecord(void* context, const HAL_OUTPUT_RECORD* record)
}
if (!responsive)
_InterlockedExchange(&gSerialPresent, 0);
NtlAtomicExchange32(&gSerialPresent, 0);
if (!emergency) {
HalSerialReleaseWriteLock();
code/kernel/include/kernel/hal/hal_api.h
+11 -10
View File
@@ -2,6 +2,7 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <ntlite/status.h>
@@ -17,9 +18,9 @@ typedef struct HAL_PROCESSOR_INFO {
uint32_t FirmwareId;
uint32_t HardwareId;
uint32_t Flags;
uint8_t Enabled;
uint8_t IsBootProcessor;
uint8_t Startable;
bool Enabled;
bool IsBootProcessor;
bool Startable;
uint8_t Reserved;
} HAL_PROCESSOR_INFO;
@@ -56,17 +57,17 @@ typedef struct HAL_FRAMEBUFFER_INFO {
typedef struct HAL_SCREEN {
void (*Clear)(uint32_t color);
void (*PutPixel)(int x, int y, uint32_t color);
void (*Blit)(int x, int y, int w, int h, const uint32_t* pixels);
void (*GetSize)(int* width, int* height);
void (*PutPixel)(int32_t x, int32_t y, uint32_t color);
void (*Blit)(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels);
void (*GetSize)(int32_t* width, int32_t* height);
KSTATUS (*GetFramebuffer)(HAL_FRAMEBUFFER_INFO* outInfo);
void (*Scroll)(int lines);
void (*PutChar)(char c, int x, int y, uint32_t fg, uint32_t bg);
void (*Scroll)(int32_t lines);
void (*PutChar)(char c, int32_t x, int32_t y, uint32_t fg, uint32_t bg);
// Writes only to the boot framebuffer text console. Stream routing belongs to HAL_OUTPUT_ROUTER.
void (*PrintChar)(char c, uint32_t fg, uint32_t bg);
// Legacy software cursor hooks kept for early-boot/panic fallback only.
// Normal input cursor ownership lives in ke_display.
void (*DrawCursor)(int x, int y);
void (*DrawCursor)(int32_t x, int32_t y);
void (*EraseCursor)(void);
} HAL_SCREEN;
@@ -219,7 +220,7 @@ typedef struct HAL_INIT_PARAMS {
uint64_t PhysicalWindowBase;
// Debug.
int Debug;
bool Debug;
void* (*AllocatePool)(size_t size);
} HAL_INIT_PARAMS;
code/kernel/include/kernel/io_volume.h
+2 -1
View File
@@ -2,6 +2,7 @@
#pragma once
#include <stdbool.h>
#include <ntlite/ddk/file.h>
#include <kernel/ke_list.h>
#include <ntlite/ddi/io_driver.h>
@@ -41,7 +42,7 @@ struct IO_VOLUME {
uint32_t InstanceId;
uint32_t State;
uint32_t RemovalReason;
uint8_t RegistryLinked;
bool RegistryLinked;
IO_VOLUME_SOURCE Source;
DEVICE_OBJECT* BackingDevice;
IO_VOLUME_PROVIDER Provider;
code/kernel/include/kernel/ke_apc.h
+1 -1
View File
@@ -65,7 +65,7 @@ struct KAPC {
void* NormalContext;
void* SystemArgument1;
void* SystemArgument2;
volatile long Inserted;
volatile int32_t Inserted;
};
void KiInitializeApcState(THREAD* thread);
code/kernel/include/kernel/ke_audio.h
+5 -5
View File
@@ -59,10 +59,10 @@ typedef struct KE_AUDIO_ENDPOINT_INFO {
IO_AUDIO_FORMAT HardwareFormat;
uint32_t PeriodCount;
uint32_t FramesPerPeriod;
uint8_t Available;
uint8_t IsDefault;
uint8_t HardwareRunning;
uint8_t MasterMuted;
bool Available;
bool IsDefault;
bool HardwareRunning;
bool MasterMuted;
int32_t MasterHardwareGainMilliDb;
float MasterRequestedVolume;
uint32_t StreamCount;
@@ -108,7 +108,7 @@ typedef struct KE_AUDIO_STREAM_INFO {
KSTATUS FailureStatus;
float Volume;
float Pan;
uint8_t Muted;
bool Muted;
} KE_AUDIO_STREAM_INFO;
// Initializes endpoint registry and mixer ownership before drivers load.
code/kernel/include/kernel/ke_cpu.h
+4 -4
View File
@@ -37,8 +37,8 @@ typedef struct KCPU {
uint32_t FirmwareId;
uint32_t BootProcessorIndex;
volatile long State;
uint8_t IsBootProcessor;
volatile int32_t State;
bool IsBootProcessor;
// CPUID/MADT-derived SMT topology. PackageId/CoreId/ThreadId are decoded
// from the processor hardware ID using CPUID leaf 0x1F/0x0B when available, with a
@@ -54,7 +54,7 @@ typedef struct KCPU {
uint32_t PhysicalCoreIndex;
uint64_t CoreSiblingMask;
uint64_t PackageMask;
uint8_t IsPrimaryThread;
bool IsPrimaryThread;
uint8_t SmtThreadCount;
uint16_t TopologyFlags;
@@ -67,7 +67,7 @@ typedef struct KCPU {
KIRQL CurrentIrql;
uint32_t InterruptDepth;
volatile long ReschedulePending;
volatile int32_t ReschedulePending;
TSS64 Tss;
uint64_t Gdt[KE_GDT_ENTRIES];
code/kernel/include/kernel/ke_debug.h
+4 -3
View File
@@ -4,6 +4,7 @@
#define KE_DEBUG_H
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
@@ -16,8 +17,8 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...);
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len);
KSTATUS KeFlushDebugLogToSystemRoot(const char* relativePath);
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...);
__declspec(dllexport) int KeHandleNmiIpi(void);
__declspec(dllexport) _Noreturn void KePanic(int32_t code, const char* fmt, ...);
__declspec(dllexport) bool KeHandleNmiIpi(void);
// Fault dispatch - called from the IDT fault stubs (ke_syscall.asm).
@@ -57,7 +58,7 @@ typedef struct KFAULT_FRAME {
__declspec(dllexport) KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame);
// Compatibility wrapper for any older assembly/object still calling this.
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
__declspec(dllexport) _Noreturn void KeFaultDispatch(uint32_t vector, uintptr_t rip,
uintptr_t errCode, uintptr_t cr2,
uintptr_t cs);
code/kernel/include/kernel/ke_display.h
+2 -1
View File
@@ -2,6 +2,7 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <kernel/hal/hal_api.h>
#ifndef DISPLAY_API
@@ -28,7 +29,7 @@ typedef struct DISPLAY_CLOCK_OBJECT {
uint64_t PeriodTicks;
uint64_t NextTick;
uint64_t LastFrameGeneration;
uint8_t Linked;
bool Linked;
} DISPLAY_CLOCK_OBJECT;
void KeDisplayInitialize(const HAL_API* hal);
code/kernel/include/kernel/ke_id.h
+2 -2
View File
@@ -6,10 +6,10 @@
#include <ntlite/status.h>
typedef struct KE_ID_ALLOCATOR {
volatile long long Next;
volatile int64_t Next;
} KE_ID_ALLOCATOR;
#define KE_ID_ALLOCATOR_INIT(first_) { (long long)(uint32_t)(first_) }
#define KE_ID_ALLOCATOR_INIT(first_) { (int64_t)(uint32_t)(first_) }
// Allocates one 32-bit identifier and never reuses an issued value during the boot.
KSTATUS KeAllocateId32(KE_ID_ALLOCATOR* allocator, uint32_t* idOut);
code/kernel/include/kernel/ke_input.h
+10 -62
View File
@@ -8,9 +8,7 @@
#include <ntlite/status.h>
#include <kernel/ob_object.h>
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
typedef struct SYS_GLOBAL_INPUT_STATE SYS_GLOBAL_INPUT_STATE;
typedef struct SYS_INPUT_MANAGER_EVENT SYS_INPUT_MANAGER_EVENT;
typedef struct KE_INPUT_MANAGER_EVENT {
@@ -22,54 +20,23 @@ typedef struct KE_INPUT_MANAGER_EVENT {
} Data;
} KE_INPUT_MANAGER_EVENT;
typedef struct INPUT_KEY_QUEUE_ENTRY {
INPUT_KEY Key;
char Character;
uint32_t Modifiers;
} INPUT_KEY_QUEUE_ENTRY;
typedef struct KE_INPUT_CONTEXT {
const HAL_API* Hal;
volatile uint8_t KeyDown[256];
volatile uint32_t Modifiers;
volatile long long Generation;
INPUT_KEY_QUEUE_ENTRY KeyQueue[256];
volatile int KeyHead;
volatile int KeyTail;
volatile INPUT_KEY RepeatKey;
volatile uint64_t RepeatNextTick;
KDPC RepeatDpc;
INPUT_MOUSE_EVENT MouseQueue[128];
volatile int MouseHead;
volatile int MouseTail;
KE_INPUT_MANAGER_EVENT ManagerQueue[256];
volatile int ManagerHead;
volatile int ManagerTail;
volatile long long ManagerSequence;
volatile int32_t ManagerHead;
volatile int32_t ManagerTail;
volatile int64_t ManagerSequence;
EVENT* ManagerWaitEvent;
volatile int MouseX;
volatile int MouseY;
volatile int MouseBoundsWidth;
volatile int MouseBoundsHeight;
volatile int MouseWheel;
volatile uint8_t MouseButtons;
EVENT* WaitEvent;
volatile int32_t MouseX;
volatile int32_t MouseY;
volatile int32_t MouseBoundsWidth;
volatile int32_t MouseBoundsHeight;
} KE_INPUT_CONTEXT;
typedef struct KE_INPUT_MANAGER {
OBJECT Header;
} KE_INPUT_MANAGER;
typedef struct INPUT_QUEUE {
OBJECT Header;
KE_INPUT_CONTEXT* Context;
EVENT* WaitEvent;
} INPUT_QUEUE;
void KeInputInitialize(const HAL_API* hal);
KSTATUS KeInputReferenceManager(KE_INPUT_MANAGER** managerOut);
KSTATUS KeInputManagerReadEvent(KE_INPUT_MANAGER* manager, uint64_t timeoutMs, SYS_INPUT_MANAGER_EVENT* eventOut);
@@ -78,26 +45,7 @@ __declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t f
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event);
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event);
__declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE_EVENT* event);
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons);
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int32_t wheel, uint8_t buttons);
INPUT_KEY KeInputKbPoll(void);
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs);
void KeInputTimerTick(void);
bool KeInputIsKeyDown(INPUT_KEY key);
bool KeInputPeekEvent(INPUT_KEY_RECORD* record);
bool KeInputReadEvent(INPUT_KEY_RECORD* record);
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out);
bool KeInputQueuePeekEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record);
bool KeInputQueueReadEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record);
KSTATUS KeInputQueueWaitKey(INPUT_QUEUE* queue, uint8_t alertable, uint64_t timeoutMs);
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue);
INPUT_KEY KeInputReadKey(void);
char KeInputReadChar(void);
uint32_t KeInputGetModifiers(void);
KSTATUS KeInputQueryGlobalState(SYS_GLOBAL_INPUT_STATE* out);
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event);
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event);
void KeInputMouseGetPosition(int* x, int* y);
void KeInputMouseSetPosition(int x, int y);
void KeInputMouseSetBounds(int w, int h);
void KeInputMouseSetPosition(int32_t x, int32_t y);
void KeInputMouseSetBounds(int32_t w, int32_t h);
code/kernel/include/kernel/ke_irql.h
+2 -1
View File
@@ -6,6 +6,7 @@
// The level constants are DDK-visible because current synchronization and IRP
// layouts expose KIRQL. Kernel routines remain declared here.
#include <stdbool.h>
#include <ntlite/ddk/irql.h>
// Return the current processor's software IRQL.
@@ -24,7 +25,7 @@ __declspec(dllexport) void KeAssertIrql(KIRQL expected);
__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql);
// Return nonzero when the current context may block.
__declspec(dllexport) int KeCanWait(void);
__declspec(dllexport) bool KeCanWait(void);
// Return a diagnostic name for an IRQL.
__declspec(dllexport) const char* KeIrqlName(KIRQL irql);
code/kernel/include/kernel/ke_mutex.h
+4 -3
View File
@@ -1,6 +1,7 @@
// Kernel mutex declarations shared by implementation files and kernel clients.
#pragma once
#include <stdbool.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
@@ -18,8 +19,8 @@ typedef struct MUTEX {
LIST_ENTRY OwnerLink;
KE_SCHEDULING_CLASS DonationClass;
uint32_t DonationPriority;
uint8_t HasDonation;
uint8_t Abandoned;
bool HasDonation;
bool Abandoned;
} MUTEX;
// Tests whether a mutex acquisition transferred ownership to the caller.
@@ -31,7 +32,7 @@ __declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex, uint64_t timeoutMilli
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex);
// Removes one blocked waiter while preserving priority-inheritance state.
int KiMutexAbortWait(MUTEX* mutex, struct THREAD* thread);
bool KiMutexAbortWait(MUTEX* mutex, struct THREAD* thread);
// Releases every mutex owned by a terminating thread as abandoned.
void KiMutexRundownThread(struct THREAD* thread);
code/kernel/include/kernel/ke_named_queue.h
-12
View File
@@ -1,12 +0,0 @@
// Named queue registry hooks shared by process handle cleanup and syscalls.
#pragma once
#include <stdint.h>
#include <ntlite/handle.h>
struct PROCESS;
struct OBJECT;
// Removes registry entries owned by the given process handle/object pair.
void KeNamedQueueUnpublishHandle(struct PROCESS* process, SYS_HANDLE handle, struct OBJECT* object);
code/kernel/include/kernel/ke_process.h
+8 -7
View File
@@ -11,6 +11,7 @@
#include <kernel/ke_handle.h>
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/string_view.h>
#include <kernel/ke_userenv.h>
@@ -57,14 +58,14 @@ typedef struct PROCESS {
struct SECURITY_CONTEXT* SecurityContext;
int32_t ExitCode;
volatile long Terminating;
volatile long Exited;
volatile long ResourcesReleased;
volatile int32_t Terminating;
volatile int32_t Exited;
volatile int32_t ResourcesReleased;
// Userspace image construction is authorized by SYS_PROCESS_RIGHT_BUILD.
// BuildOperations closes the race between memory construction and the one-way start transition.
volatile long BuildState;
volatile long BuildOperations;
volatile int32_t BuildState;
volatile int32_t BuildOperations;
// Main executable image. Used for cleanup.
uintptr_t ImageBase;
@@ -124,8 +125,8 @@ void KeStopProcessConstruction(PROCESS* process);
KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode);
void KeCompleteProcessExitIfLastThread(PROCESS* process);
int KeIsProcessTerminating(PROCESS* process);
int KeIsProcessTerminationProtected(PROCESS* process);
bool KeIsProcessTerminating(PROCESS* process);
bool KeIsProcessTerminationProtected(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int32_t* exitCodeOut);
void KeQueryExceptionStats(struct SYS_EXCEPTION_STATS* statsOut);
code/kernel/include/kernel/ke_sched.h
+6 -5
View File
@@ -1,6 +1,7 @@
// Kernel ke sched declarations shared by implementation files and kernel clients.
#pragma once
#include <stdbool.h>
// The scheduler combines dynamic normal queues with static real-time queues.
// Per-processor timer IRQs drive KeSchedulerTick.
#include <kernel/ke_thread.h>
@@ -55,8 +56,8 @@ typedef struct KE_THREAD_SCHEDULER_SNAPSHOT {
uint32_t WaitObjectType;
KSTATUS WaitStatus;
struct MUTEX* WaitingMutex;
uint8_t Queued;
uint8_t AlertableWait;
bool Queued;
bool AlertableWait;
} KE_THREAD_SCHEDULER_SNAPSHOT;
// Copies scheduler-owned thread state into a stable diagnostic snapshot.
@@ -64,7 +65,7 @@ void KiSchedulerSnapshotThread(THREAD* thread, KE_THREAD_SCHEDULER_SNAPSHOT* sna
// Selects waiters and refreshes donations while the mutex lock is held.
struct THREAD* KiSchedulerSelectMutexWaiter(struct MUTEX* mutex);
int KiSchedulerRefreshMutexDonation(struct MUTEX* mutex);
bool KiSchedulerRefreshMutexDonation(struct MUTEX* mutex);
// Updates one thread's normal-class base priority.
KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority);
@@ -82,7 +83,7 @@ struct MUTEX* KiSchedulerFirstOwnedMutex(THREAD* thread);
void KiSchedulerNotePriorityPropagation(uint32_t depth);
// Recomputes effective rank and optionally returns a referenced upstream mutex.
int KiSchedulerRefreshEffectivePriority(THREAD* thread, struct MUTEX** waitingMutexOut);
bool KiSchedulerRefreshEffectivePriority(THREAD* thread, struct MUTEX** waitingMutexOut);
// Called once by KernelMain before any threads are created.
void KeInitializeScheduler(void);
@@ -107,7 +108,7 @@ void KeSchedulerTick(void);
// is cheap and can be called from interrupt context. KiScheduleFromInterrupt
// consumes the pending request and performs the context switch, if needed.
void KiRequestReschedule(void);
int KiIsReschedulePending(void);
bool KiIsReschedulePending(void);
void KiScheduleFromInterrupt(void);
void KiSchedulerNoteVoluntaryBlock(THREAD* thread);
void KiSchedulerMarkThreadWaiting(THREAD* thread);
code/kernel/include/kernel/ke_thread.h
+16 -15
View File
@@ -6,6 +6,7 @@
// ke_thread.h - thread model
// A THREAD is the unit of scheduling. Every thread belongs to exactly one process.
#include <stdint.h>
#include <stdbool.h>
#include <kernel/ob_object.h>
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
@@ -131,8 +132,8 @@ struct THREAD {
// Scheduler hardening state. Queued means membership in a scheduler ready
// queue only; wait-object membership is tracked by WaitListHead. Idle
// threads are CPU-local and never enter a global ready queue.
uint8_t Queued;
uint8_t IsIdleThread;
bool Queued;
bool IsIdleThread;
uint16_t SchedulerFlags;
uint32_t LastCpu;
uint64_t AffinityMask;
@@ -143,9 +144,9 @@ struct THREAD {
// FpuContext.
KSTATUS WaitStatus;
LIST_ENTRY* WaitListHead;
uint8_t AlertableWait;
bool AlertableWait;
LIST_ENTRY TimeoutLink;
uint8_t TimeoutArmed;
bool TimeoutArmed;
uint64_t TimeoutTick;
KSTATUS TimeoutStatus;
@@ -157,9 +158,9 @@ struct THREAD {
// when that thread reaches PASSIVE_LEVEL with APC delivery enabled.
LIST_ENTRY ApcQueue;
SPINLOCK ApcLock;
volatile long KernelApcPending;
volatile int32_t KernelApcPending;
int32_t KernelApcDisable;
uint8_t ApcDeliveryActive;
bool ApcDeliveryActive;
// Non-NULL while executing on a syscall/interrupt/fault entry frame.
// The pointed-to frame lives on this thread's kernel stack.
@@ -170,10 +171,10 @@ struct THREAD {
// user-mode dispatcher stub.
LIST_ENTRY UserApcQueue;
SPINLOCK UserApcLock;
volatile long UserApcPending;
volatile int32_t UserApcPending;
uintptr_t UserApcDispatcher;
uint8_t UserApcDeliverable;
uint8_t UserApcDeliveryActive;
bool UserApcDeliverable;
bool UserApcDeliveryActive;
uintptr_t UserApcFrameUserAddress;
uint64_t UserApcReturnToken;
uint64_t UserApcSavedRip;
@@ -192,12 +193,12 @@ struct THREAD {
// Synchronous kernel work can temporarily retain caller-owned stack or
// process memory. Process termination must wait until that ownership has
// returned instead of freeing the thread underneath an active driver.
volatile long TerminationDeferralCount;
volatile int32_t TerminationDeferralCount;
// Set once terminal thread teardown begins. Interrupts may continue to
// run, but interrupt-return scheduling must not abandon this kernel stack
// before KeFinalizeThreadExit and process-exit publication complete.
volatile long ExitInProgress;
volatile int32_t ExitInProgress;
};
// Creates and queues a user thread. On success, *out receives a referenced
@@ -224,7 +225,7 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD** out);
// Mark a thread dead and deschedule it permanently.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code);
__declspec(dllexport) void KeExitThread(THREAD* thread, int32_t code);
// Publishes completion, releases per-thread user memory, and removes one dead
// thread from its process live-thread set. Retained handles remain waitable.
@@ -251,7 +252,7 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status);
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status);
int KeAbortThreadWait(THREAD* thread, KSTATUS status);
bool KeAbortThreadWait(THREAD* thread, KSTATUS status);
void KeUnblockThread(THREAD* thread);
KSTATUS KeAlertThread(THREAD* thread);
void KiWakeThreadForKernelApc(THREAD* thread);
@@ -260,8 +261,8 @@ void KiWakeThreadForUserApc(THREAD* thread);
// Defers process termination while kernel work owns caller-backed storage.
void KeEnterTerminationDeferral(void);
void KeLeaveTerminationDeferral(void);
int KeThreadTerminationDeferred(const THREAD* thread);
int KeThreadExitInProgress(const THREAD* thread);
bool KeThreadTerminationDeferred(const THREAD* thread);
bool KeThreadExitInProgress(const THREAD* thread);
// Assembly stub - implemented in ke_switch.asm.
void KeSwitchContext(THREAD* from, THREAD* to);
code/kernel/include/kernel/ke_timer.h
+3 -2
View File
@@ -3,6 +3,7 @@
#pragma once
// ke_timer.h - waitable one-shot and periodic timers.
#include <stdint.h>
#include <stdbool.h>
#include <kernel/ke_event.h>
#include <kernel/ke_list.h>
#include <kernel/ke_spinlock.h>
@@ -15,8 +16,8 @@ typedef struct KTIMER {
LIST_ENTRY Link;
uint64_t DueMicroseconds;
uint64_t PeriodMicroseconds;
uint8_t Active;
uint8_t Linked;
bool Active;
bool Linked;
} KTIMER;
// Creates an unset waitable timer object.
code/kernel/include/kernel/ke_trap.h
+2 -1
View File
@@ -8,6 +8,7 @@
// and KiExitTrapFrame commits those values back to the hardware IRET slots.
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/status.h>
#define KTRAP_FRAME_MAGIC 0x4650545241544B31ULL
@@ -80,7 +81,7 @@ typedef struct SYS_TRAP_FRAME_INFO_K {
void KiEnterTrapFrame(KTRAP_FRAME* frame);
void KiExitTrapFrame(KTRAP_FRAME* frame);
KTRAP_FRAME* KeGetCurrentTrapFrame(void);
int KeTrapFrameFromUser(const KTRAP_FRAME* frame);
bool KeTrapFrameFromUser(const KTRAP_FRAME* frame);
void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3);
KSTATUS KeQueryCurrentTrapFrame(SYS_TRAP_FRAME_INFO_K* out);
code/kernel/include/kernel/ke_work.h
+1 -1
View File
@@ -15,7 +15,7 @@ typedef struct KWORK_ITEM {
LIST_ENTRY Link;
KWORK_ROUTINE Routine;
void* Context;
volatile long Queued;
volatile int32_t Queued;
} KWORK_ITEM;
void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context);
code/kernel/include/kernel/mm_file_cache.h
+2 -1
View File
@@ -4,6 +4,7 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <ntlite/status.h>
@@ -59,7 +60,7 @@ void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page);
// Returns a retained coherent cache page suitable for a shared writable mapping.
KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
uint64_t fileSizeBefore, int needExistingBytes,
uint64_t fileSizeBefore, bool needExistingBytes,
MM_FILE_CACHE_PAGE* outPage);
// Marks one coherent cache page dirty after a shared writable mapping is written.
code/kernel/include/kernel/mm_pte.h
+4 -3
View File
@@ -4,6 +4,7 @@
#include <ntlite/ddk/memory.h>
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <ntlite/status.h>
@@ -87,10 +88,10 @@ __declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uin
// Tests and clears the hardware accessed bit on one mapped 4 KiB leaf.
// Returns nonzero when the page had been accessed and was given a second chance.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt);
__declspec(dllexport) bool MmPteTestAndClearAccessed(uintptr_t virt);
// Tests and clears the hardware dirty bit on one mapped 4 KiB leaf.
__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt);
__declspec(dllexport) bool MmPteTestAndClearDirty(uintptr_t virt);
// Translates a virtual address through the current shared page tables.
__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt);
@@ -106,7 +107,7 @@ void MmPteFlushLocal(uintptr_t virt);
void MmPteFlushKernelAddress(uintptr_t virt);
// Handles the fixed-vector NMI used for kernel TLB shootdowns.
int MmPteHandleShootdownNmi(void);
bool MmPteHandleShootdownNmi(void);
// Returns page-table operation counters used by memory telemetry.
void MmPteQueryStats(MM_PTE_STATS* out);
code/kernel/include/kernel/mm_swap.h
+3 -2
View File
@@ -3,12 +3,13 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <kernel/mm_phys.h>
typedef struct MM_SWAP_STATS {
uint64_t Enabled;
bool Enabled;
uint64_t TotalSlots;
uint64_t UsedSlots;
uint64_t FreeSlots;
@@ -23,7 +24,7 @@ typedef struct MM_SWAP_STATS {
} MM_SWAP_STATS;
KSTATUS MmSwapInitialize(const char* relativePath, uint64_t sizeBytes);
int MmSwapIsEnabled(void);
bool MmSwapIsEnabled(void);
KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut);
KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame);
void MmSwapFreeSlot(uint32_t slot);
code/kernel/include/kernel/mm_virt.h
+2 -1
View File
@@ -4,6 +4,7 @@
// mm_virt.h - virtual memory manager (VMM)
// Manages the single shared PML4, kernel pool, and VAD-backed user VM ownership.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pte.h>
@@ -92,7 +93,7 @@ __declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
// Locks a user buffer for asynchronous kernel access. The containing VADs stay
// alive and protected, and resident PFNs stay pinned until the buffer is
// unlocked. WriteAccess describes kernel writes to the user buffer.
__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, bool writeAccess, MM_LOCKED_USER_BUFFER** out);
__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
size_t MmLockedUserBufferPageCount(const MM_LOCKED_USER_BUFFER* buffer);
PHYSICAL_ADDRESS MmLockedUserBufferPageFrame(const MM_LOCKED_USER_BUFFER* buffer, size_t index);
code/kernel/include/kernel/syscall_nr.h
+106 -122
View File
@@ -68,145 +68,129 @@
#define SYS_SIGNAL_EVENT 53
#define SYS_CLEAR_EVENT 54
// Queues and named queues.
// Anonymous queues.
#define SYS_CREATE_QUEUE 55
#define SYS_SEND_QUEUE 56
#define SYS_RECV_QUEUE 57
#define SYS_PUBLISH_QUEUE 58
#define SYS_OPEN_NAMED_QUEUE 59
#define SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS 60
#define SYS_UNPUBLISH_QUEUE 61
// LPC-like Ports and well-known system services.
#define SYS_CREATE_PORT 62
#define SYS_CONNECT_PORT 63
#define SYS_ACCEPT_PORT 64
#define SYS_SEND_PORT 65
#define SYS_RECEIVE_PORT 66
#define SYS_CALL_PORT 67
#define SYS_REPLY_PORT 68
#define SYS_DISCONNECT_PORT 69
#define SYS_OPEN_PORT_PEER_PROCESS 70
#define SYS_REGISTER_SYSTEM_SERVICE 71
#define SYS_UNREGISTER_SYSTEM_SERVICE 72
#define SYS_CONNECT_SYSTEM_SERVICE 73
#define SYS_CREATE_PORT 58
#define SYS_CONNECT_PORT 59
#define SYS_ACCEPT_PORT 60
#define SYS_SEND_PORT 61
#define SYS_RECEIVE_PORT 62
#define SYS_CALL_PORT 63
#define SYS_REPLY_PORT 64
#define SYS_DISCONNECT_PORT 65
#define SYS_OPEN_PORT_PEER_PROCESS 66
#define SYS_REGISTER_SYSTEM_SERVICE 67
#define SYS_UNREGISTER_SYSTEM_SERVICE 68
#define SYS_CONNECT_SYSTEM_SERVICE 69
// Mounted-volume manager control plane.
#define SYS_QUERY_VOLUMES 74
#define SYS_OPEN_VOLUME 75
#define SYS_WAIT_VOLUME_CHANGE 76
#define SYS_QUERY_VOLUMES 70
#define SYS_OPEN_VOLUME 71
#define SYS_WAIT_VOLUME_CHANGE 72
// Files and directories.
#define SYS_OPEN_FILE_AT 77
#define SYS_OPEN_DIR_AT 78
#define SYS_STAT_AT 79
#define SYS_DELETE_AT 80
#define SYS_RENAME_AT 81
#define SYS_CREATE_DIR_AT 82
#define SYS_REMOVE_DIR_AT 83
#define SYS_READ_FILE 84
#define SYS_WRITE_FILE 85
#define SYS_READ_FILE_ASYNC 86
#define SYS_WRITE_FILE_ASYNC 87
#define SYS_CANCEL_IO 88
#define SYS_SEEK_FILE 89
#define SYS_GET_FILE_POS 90
#define SYS_GET_FILE_SIZE 91
#define SYS_GET_FILE_STAT 92
#define SYS_TRUNCATE_FILE 93
#define SYS_FLUSH_FILE 94
#define SYS_LOCK_FILE 95
#define SYS_UNLOCK_FILE 96
#define SYS_QUERY_FILE_LOCK 97
#define SYS_SET_FILE_STATUS 98
#define SYS_CREATE_PIPE 99
#define SYS_NOTIFY_PIPE 100
#define SYS_READ_DIR 101
#define SYS_OPEN_FILE_AT 73
#define SYS_OPEN_DIR_AT 74
#define SYS_STAT_AT 75
#define SYS_DELETE_AT 76
#define SYS_RENAME_AT 77
#define SYS_CREATE_DIR_AT 78
#define SYS_REMOVE_DIR_AT 79
#define SYS_READ_FILE 80
#define SYS_WRITE_FILE 81
#define SYS_READ_FILE_ASYNC 82
#define SYS_WRITE_FILE_ASYNC 83
#define SYS_CANCEL_IO 84
#define SYS_SEEK_FILE 85
#define SYS_GET_FILE_POS 86
#define SYS_GET_FILE_SIZE 87
#define SYS_GET_FILE_STAT 88
#define SYS_TRUNCATE_FILE 89
#define SYS_FLUSH_FILE 90
#define SYS_LOCK_FILE 91
#define SYS_UNLOCK_FILE 92
#define SYS_QUERY_FILE_LOCK 93
#define SYS_SET_FILE_STATUS 94
#define SYS_CREATE_PIPE 95
#define SYS_NOTIFY_PIPE 96
#define SYS_READ_DIR 97
// Console, display, and surfaces.
#define SYS_WRITE_CONSOLE 102
#define SYS_GET_CONSOLE_INFO 103
#define SYS_SET_CONSOLE_CURSOR 104
#define SYS_FILL_CONSOLE_CHARACTER 105
#define SYS_FILL_CONSOLE_ATTRIBUTE 106
#define SYS_PUT_SCREEN_CHAR 107
#define SYS_CLEAR_SCREEN 108
#define SYS_GET_SCREEN_SIZE 109
#define SYS_QUERY_DISPLAY_INFO 110
#define SYS_QUERY_DISPLAY_MANAGER_INFO 111
#define SYS_PRESENT_DISPLAY_MANAGER_SURFACE 112
#define SYS_SET_DISPLAY_MANAGER_CURSOR 113
#define SYS_WRITE_DISPLAY_MANAGER_CONSOLE 114
#define SYS_QUERY_DISPLAY_MANAGER_CONSOLE_INFO 115
#define SYS_SET_DISPLAY_MANAGER_CONSOLE_CURSOR 116
#define SYS_FILL_DISPLAY_MANAGER_CONSOLE_CHARACTER 117
#define SYS_FILL_DISPLAY_MANAGER_CONSOLE_ATTRIBUTE 118
#define SYS_PUT_DISPLAY_MANAGER_CONSOLE_CHARACTER 119
#define SYS_CLEAR_DISPLAY_MANAGER_CONSOLE 120
#define SYS_OPEN_DISPLAY_CLOCK 121
#define SYS_CREATE_SURFACE 122
#define SYS_MAP_SURFACE 123
#define SYS_UNMAP_SURFACE 124
#define SYS_QUERY_SURFACE 125
#define SYS_COMMIT_SURFACE 126
#define SYS_WRITE_CONSOLE 98
#define SYS_GET_CONSOLE_INFO 99
#define SYS_SET_CONSOLE_CURSOR 100
#define SYS_FILL_CONSOLE_CHARACTER 101
#define SYS_FILL_CONSOLE_ATTRIBUTE 102
#define SYS_PUT_SCREEN_CHAR 103
#define SYS_CLEAR_SCREEN 104
#define SYS_GET_SCREEN_SIZE 105
#define SYS_QUERY_DISPLAY_INFO 106
#define SYS_QUERY_DISPLAY_MANAGER_INFO 107
#define SYS_PRESENT_DISPLAY_MANAGER_SURFACE 108
#define SYS_SET_DISPLAY_MANAGER_CURSOR 109
#define SYS_WRITE_DISPLAY_MANAGER_CONSOLE 110
#define SYS_QUERY_DISPLAY_MANAGER_CONSOLE_INFO 111
#define SYS_SET_DISPLAY_MANAGER_CONSOLE_CURSOR 112
#define SYS_FILL_DISPLAY_MANAGER_CONSOLE_CHARACTER 113
#define SYS_FILL_DISPLAY_MANAGER_CONSOLE_ATTRIBUTE 114
#define SYS_PUT_DISPLAY_MANAGER_CONSOLE_CHARACTER 115
#define SYS_CLEAR_DISPLAY_MANAGER_CONSOLE 116
#define SYS_OPEN_DISPLAY_CLOCK 117
#define SYS_CREATE_SURFACE 118
#define SYS_MAP_SURFACE 119
#define SYS_UNMAP_SURFACE 120
#define SYS_QUERY_SURFACE 121
#define SYS_COMMIT_SURFACE 122
// Native render audio.
#define SYS_AUDIO_ENUMERATE_ENDPOINTS 127
#define SYS_AUDIO_OPEN_STREAM 128
#define SYS_AUDIO_WRITE 129
#define SYS_AUDIO_ARM_WAIT 130
#define SYS_AUDIO_QUERY_STREAM 131
#define SYS_AUDIO_START 132
#define SYS_AUDIO_PAUSE 133
#define SYS_AUDIO_RESUME 134
#define SYS_AUDIO_DRAIN 135
#define SYS_AUDIO_STOP 136
#define SYS_AUDIO_RESET 137
#define SYS_AUDIO_SET_VOLUME 138
#define SYS_AUDIO_SET_MASTER_VOLUME 139
#define SYS_AUDIO_ENUMERATE_ENDPOINTS 123
#define SYS_AUDIO_OPEN_STREAM 124
#define SYS_AUDIO_WRITE 125
#define SYS_AUDIO_ARM_WAIT 126
#define SYS_AUDIO_QUERY_STREAM 127
#define SYS_AUDIO_START 128
#define SYS_AUDIO_PAUSE 129
#define SYS_AUDIO_RESUME 130
#define SYS_AUDIO_DRAIN 131
#define SYS_AUDIO_STOP 132
#define SYS_AUDIO_RESET 133
#define SYS_AUDIO_SET_VOLUME 134
#define SYS_AUDIO_SET_MASTER_VOLUME 135
// Input.
#define SYS_READ_INPUT_MANAGER_EVENT 140
#define SYS_POLL_KEYBOARD 141
#define SYS_IS_KEY_DOWN 142
#define SYS_READ_KEYBOARD_CHAR 143
#define SYS_READ_KEY 144
#define SYS_GET_KEY_MODIFIERS 145
#define SYS_QUERY_GLOBAL_INPUT_STATE 146
#define SYS_OPEN_INPUT_QUEUE 147
#define SYS_READ_INPUT_EVENT 148
#define SYS_PEEK_INPUT_EVENT 149
#define SYS_PEEK_MOUSE 150
#define SYS_READ_MOUSE 151
#define SYS_GET_MOUSE_POS 152
#define SYS_READ_INPUT_MANAGER_EVENT 136
// Time, debug, and system information.
#define SYS_CREATE_TIMER 153
#define SYS_SET_TIMER 154
#define SYS_CANCEL_TIMER 155
#define SYS_GET_TICK 156
#define SYS_GET_TICK_FREQUENCY 157
#define SYS_GET_SYSTEM_TIME 158
#define SYS_PRINT_DEBUG 159
#define SYS_QUERY_TRAP_FRAME 160
#define SYS_GET_MEMORY_STATUS 161
#define SYS_QUERY_KERNEL_STATS 162
#define SYS_CREATE_TIMER 137
#define SYS_SET_TIMER 138
#define SYS_CANCEL_TIMER 139
#define SYS_GET_TICK 140
#define SYS_GET_TICK_FREQUENCY 141
#define SYS_GET_SYSTEM_TIME 142
#define SYS_PRINT_DEBUG 143
#define SYS_QUERY_TRAP_FRAME 144
#define SYS_GET_MEMORY_STATUS 145
#define SYS_QUERY_KERNEL_STATS 146
// Native sockets and polling.
#define SYS_CREATE_SOCKET 163
#define SYS_SOCKET_IO_ASYNC 164
#define SYS_BIND_SOCKET 165
#define SYS_LISTEN_SOCKET 166
#define SYS_ACCEPT_SOCKET 167
#define SYS_CONNECT_SOCKET 168
#define SYS_SEND_SOCKET 169
#define SYS_RECV_SOCKET 170
#define SYS_SHUTDOWN_SOCKET 171
#define SYS_SEND_TO_SOCKET 172
#define SYS_RECV_FROM_SOCKET 173
#define SYS_RESOLVE_HOST 174
#define SYS_SET_SOCKET_OPTION 175
#define SYS_GET_SOCKET_OPTION 176
#define SYS_POLL 177
#define SYS_CREATE_SOCKET 147
#define SYS_SOCKET_IO_ASYNC 148
#define SYS_BIND_SOCKET 149
#define SYS_LISTEN_SOCKET 150
#define SYS_ACCEPT_SOCKET 151
#define SYS_CONNECT_SOCKET 152
#define SYS_SEND_SOCKET 153
#define SYS_RECV_SOCKET 154
#define SYS_SHUTDOWN_SOCKET 155
#define SYS_SEND_TO_SOCKET 156
#define SYS_RECV_FROM_SOCKET 157
#define SYS_RESOLVE_HOST 158
#define SYS_SET_SOCKET_OPTION 159
#define SYS_GET_SOCKET_OPTION 160
#define SYS_POLL 161
#define SYS_MAX 178
#define SYS_MAX 162
code/kernel/io_device.c
+8 -5
View File
@@ -1,5 +1,7 @@
// I/O manager. This file owns driver objects, device stacks, PnP dispatch, and the disk namespace.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
@@ -87,6 +89,7 @@ __declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
// Registers driver Object.
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
if (!driver) return NTLSTATUS_INVALID_PARAMETER;
if (!SysSvIsUtf8Text(driver->Name)) return NTLSTATUS_INVALID_PARAMETER;
if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
return NTLSTATUS_NO_MEMORY;
@@ -193,7 +196,7 @@ __declspec(dllexport) KSTATUS IoCreatePhysicalDevice(DRIVER_OBJECT* busDriver,
if (idCount && ids) {
for (size_t i = 0; i < idCount; i++) {
if (!SysSvIsValid(ids[i].Id) || ids[i].Id.Length >= IO_MAX_DEVICE_ID_CHARS) {
if (!SysSvIsUtf8Text(ids[i].Id) || ids[i].Id.Length >= IO_MAX_DEVICE_ID_CHARS) {
IoDeleteDevice(pdo);
return NTLSTATUS_INVALID_PARAMETER;
}
@@ -255,7 +258,7 @@ __declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
__declspec(dllexport) KSTATUS IoSetDeviceName(DEVICE_OBJECT* device,
SYS_STRING_VIEW name) {
if (!device || !SysSvIsValid(name))
if (!device || !SysSvIsUtf8Text(name))
return NTLSTATUS_INVALID_PARAMETER;
if (name.Length >= IO_MAX_DEVICE_NAME_CHARS)
return NTLSTATUS_BUFFER_TOO_SMALL;
@@ -397,7 +400,7 @@ __declspec(dllexport) KSTATUS IoForwardIrpToLower(DEVICE_OBJECT* device, IRP* ir
if (currentDevice == device) {
cur->DeviceObject = (DEVICE_OBJECT*)0;
_InterlockedDecrement(&device->OutstandingIrps);
NtlAtomicDecrement32(&device->OutstandingIrps);
IoDereferenceDeviceObject(device);
}
@@ -693,13 +696,13 @@ __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
ios->DeviceObject = device;
IoReferenceDeviceObject(device);
_InterlockedIncrement(&device->OutstandingIrps);
NtlAtomicIncrement32(&device->OutstandingIrps);
KSTATUS st = dispatch(device, irp);
if (st != NTLSTATUS_PENDING) {
if (ios->DeviceObject == device) {
ios->DeviceObject = (DEVICE_OBJECT*)0;
_InterlockedDecrement(&device->OutstandingIrps);
NtlAtomicDecrement32(&device->OutstandingIrps);
IoDereferenceDeviceObject(device);
}
}
code/kernel/io_dma.c
+82 -80
View File
@@ -1,5 +1,7 @@
// Kernel DMA core. This file owns device adapters, common buffers, pinned memory, streaming mappings, and bounce copies.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <intrin.h>
#include <stdint.h>
#include <stddef.h>
@@ -21,8 +23,8 @@
#define IO_DMA_MAP_TAG MM_POOL_TAG('d', 'm', 'a', 'm')
struct IO_DMA_ADAPTER {
volatile long RefCount;
volatile long Removed;
volatile int32_t RefCount;
volatile int32_t Removed;
SPINLOCK Lock;
DEVICE_OBJECT* Device;
IO_DMA_REQUIREMENTS Requirements;
@@ -44,7 +46,7 @@ typedef enum MM_PINNED_MEMORY_ORIGIN {
} MM_PINNED_MEMORY_ORIGIN;
struct MM_PINNED_MEMORY {
volatile long RefCount;
volatile int32_t RefCount;
MM_PINNED_MEMORY_ORIGIN Origin;
IO_DMA_DIRECTION Direction;
void* Address;
@@ -63,38 +65,38 @@ struct IO_DMA_MAPPING {
size_t Length;
IO_DMA_DIRECTION Direction;
uint32_t SegmentCount;
int DeviceOwnsMapping;
bool DeviceOwnsMapping;
IO_DMA_SEGMENT Segments[1];
};
typedef struct IO_DMA_INTERNAL_STATS {
volatile long long AdaptersCreated;
volatile long long AdaptersDestroyed;
volatile long long AdaptersInUse;
volatile long long CommonBuffersAllocated;
volatile long long CommonBuffersFreed;
volatile long long CommonBuffersInUse;
volatile long long CommonBufferBytesInUse;
volatile long long PinnedKernelRanges;
volatile long long PinnedUserRanges;
volatile long long PinnedRangesReleased;
volatile long long PinnedRangesInUse;
volatile long long PinnedPagesInUse;
volatile long long StreamingMappings;
volatile long long StreamingMappingsUnmapped;
volatile long long StreamingMappingsInUse;
volatile long long DirectMappings;
volatile long long BounceMappings;
volatile long long BounceBytesCopiedToDevice;
volatile long long BounceBytesCopiedToCpu;
volatile long long FailedAllocations;
volatile long long FailedMappings;
volatile int64_t AdaptersCreated;
volatile int64_t AdaptersDestroyed;
volatile int64_t AdaptersInUse;
volatile int64_t CommonBuffersAllocated;
volatile int64_t CommonBuffersFreed;
volatile int64_t CommonBuffersInUse;
volatile int64_t CommonBufferBytesInUse;
volatile int64_t PinnedKernelRanges;
volatile int64_t PinnedUserRanges;
volatile int64_t PinnedRangesReleased;
volatile int64_t PinnedRangesInUse;
volatile int64_t PinnedPagesInUse;
volatile int64_t StreamingMappings;
volatile int64_t StreamingMappingsUnmapped;
volatile int64_t StreamingMappingsInUse;
volatile int64_t DirectMappings;
volatile int64_t BounceMappings;
volatile int64_t BounceBytesCopiedToDevice;
volatile int64_t BounceBytesCopiedToCpu;
volatile int64_t FailedAllocations;
volatile int64_t FailedMappings;
} IO_DMA_INTERNAL_STATS;
static IO_DMA_INTERNAL_STATS gIoDmaStats;
// Returns whether a size is a nonzero power of two.
static int IoDmaIsPowerOfTwo(size_t value) {
static bool IoDmaIsPowerOfTwo(size_t value) {
return value && ((value & (value - 1)) == 0);
}
@@ -125,7 +127,7 @@ static KSTATUS IoDmaNormalizeRequirements(const IO_DMA_REQUIREMENTS* source,
}
// Returns whether two normalized requirement sets are identical.
static int IoDmaRequirementsEqual(const IO_DMA_REQUIREMENTS* left,
static bool IoDmaRequirementsEqual(const IO_DMA_REQUIREMENTS* left,
const IO_DMA_REQUIREMENTS* right) {
return left && right &&
left->MaximumDeviceAddress == right->MaximumDeviceAddress &&
@@ -136,37 +138,37 @@ static int IoDmaRequirementsEqual(const IO_DMA_REQUIREMENTS* left,
}
// Returns whether device removal has closed the adapter to new ownership.
static int IoDmaAdapterRemoved(const IO_DMA_ADAPTER* adapter) {
static bool IoDmaAdapterRemoved(const IO_DMA_ADAPTER* adapter) {
if (!adapter)
return 1;
return _InterlockedCompareExchange((volatile long*)&adapter->Removed, 0, 0) != 0;
return true;
return NtlAtomicLoad32(&adapter->Removed) != 0;
}
// Adds one reference to an adapter that is already stabilized by its owner.
static void IoDmaReferenceAdapter(IO_DMA_ADAPTER* adapter) {
if (adapter)
_InterlockedIncrement(&adapter->RefCount);
NtlAtomicIncrement32(&adapter->RefCount);
}
// Adds a reference only while the adapter still accepts new DMA ownership.
static int IoDmaTryReferenceAdapter(IO_DMA_ADAPTER* adapter) {
static bool IoDmaTryReferenceAdapter(IO_DMA_ADAPTER* adapter) {
if (!adapter)
return 0;
return false;
uintptr_t flags = KeAcquireSpinlock(&adapter->Lock);
if (adapter->Removed) {
KeReleaseSpinlock(&adapter->Lock, flags);
return 0;
return false;
}
_InterlockedIncrement(&adapter->RefCount);
NtlAtomicIncrement32(&adapter->RefCount);
KeReleaseSpinlock(&adapter->Lock, flags);
return 1;
return true;
}
// Releases one adapter reference and destroys the detached adapter at zero.
static void IoDmaPutAdapterImpl(IO_DMA_ADAPTER* adapter) {
if (!adapter)
return;
long remaining = _InterlockedDecrement(&adapter->RefCount);
int32_t remaining = NtlAtomicDecrement32(&adapter->RefCount);
if (remaining > 0)
return;
if (remaining < 0)
@@ -177,8 +179,8 @@ static void IoDmaPutAdapterImpl(IO_DMA_ADAPTER* adapter) {
KePanic(KE_PANIC_CODE_GENERIC,
"DMA: attached adapter reached zero references adapter=0x%016llX\n",
(unsigned long long)(uintptr_t)adapter);
_InterlockedIncrement64(&gIoDmaStats.AdaptersDestroyed);
_InterlockedDecrement64(&gIoDmaStats.AdaptersInUse);
NtlAtomicIncrement64(&gIoDmaStats.AdaptersDestroyed);
NtlAtomicDecrement64(&gIoDmaStats.AdaptersInUse);
MmFreePool(adapter);
}
@@ -217,7 +219,7 @@ static KSTATUS IoDmaGetAdapterImpl(DEVICE_OBJECT* device,
IO_DMA_ADAPTER* candidate = (IO_DMA_ADAPTER*)MmAllocatePoolTag(sizeof(*candidate),
IO_DMA_ADAPTER_TAG);
if (!candidate) {
_InterlockedIncrement64(&gIoDmaStats.FailedAllocations);
NtlAtomicIncrement64(&gIoDmaStats.FailedAllocations);
return NTLSTATUS_NO_MEMORY;
}
memset(candidate, 0, sizeof(*candidate));
@@ -249,8 +251,8 @@ static KSTATUS IoDmaGetAdapterImpl(DEVICE_OBJECT* device,
device->DmaAdapter = candidate;
KeReleaseSpinlock(&device->Lock, flags);
_InterlockedIncrement64(&gIoDmaStats.AdaptersCreated);
_InterlockedIncrement64(&gIoDmaStats.AdaptersInUse);
NtlAtomicIncrement64(&gIoDmaStats.AdaptersCreated);
NtlAtomicIncrement64(&gIoDmaStats.AdaptersInUse);
*outAdapter = candidate;
return NTLSTATUS_SUCCESS;
}
@@ -324,7 +326,7 @@ static KSTATUS IoDmaAllocateCommonBufferInternal(IO_DMA_ADAPTER* adapter,
allocationBoundary,
MmPfnUseDma);
if (!physical) {
_InterlockedIncrement64(&gIoDmaStats.FailedAllocations);
NtlAtomicIncrement64(&gIoDmaStats.FailedAllocations);
return NTLSTATUS_NO_MEMORY;
}
@@ -332,7 +334,7 @@ static KSTATUS IoDmaAllocateCommonBufferInternal(IO_DMA_ADAPTER* adapter,
if (!buffer) {
for (size_t i = 0; i < pageCount; ++i)
(void)MmPfnFree(physical + i * PAGE_SIZE);
_InterlockedIncrement64(&gIoDmaStats.FailedAllocations);
NtlAtomicIncrement64(&gIoDmaStats.FailedAllocations);
return NTLSTATUS_NO_MEMORY;
}
@@ -348,7 +350,7 @@ static KSTATUS IoDmaAllocateCommonBufferInternal(IO_DMA_ADAPTER* adapter,
MmFreePool(buffer);
for (size_t i = 0; i < pageCount; ++i)
(void)MmPfnFree(physical + i * PAGE_SIZE);
_InterlockedIncrement64(&gIoDmaStats.FailedAllocations);
NtlAtomicIncrement64(&gIoDmaStats.FailedAllocations);
return NTLSTATUS_FAILURE;
}
@@ -359,9 +361,9 @@ static KSTATUS IoDmaAllocateCommonBufferInternal(IO_DMA_ADAPTER* adapter,
(void)MmPfnFree(physical + i * PAGE_SIZE);
return NTLSTATUS_OBJECT_DEAD;
}
_InterlockedIncrement64(&gIoDmaStats.CommonBuffersAllocated);
_InterlockedIncrement64(&gIoDmaStats.CommonBuffersInUse);
_InterlockedExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, (long long)allocationSize);
NtlAtomicIncrement64(&gIoDmaStats.CommonBuffersAllocated);
NtlAtomicIncrement64(&gIoDmaStats.CommonBuffersInUse);
NtlAtomicExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, (int64_t)allocationSize);
*outBuffer = buffer;
return NTLSTATUS_SUCCESS;
}
@@ -389,9 +391,9 @@ static void IoDmaFreeCommonBufferImpl(IO_DMA_BUFFER* buffer) {
for (size_t i = 0; i < pageCount; ++i)
(void)MmPfnFree(physical + i * PAGE_SIZE);
_InterlockedIncrement64(&gIoDmaStats.CommonBuffersFreed);
_InterlockedDecrement64(&gIoDmaStats.CommonBuffersInUse);
_InterlockedExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, -(long long)allocationSize);
NtlAtomicIncrement64(&gIoDmaStats.CommonBuffersFreed);
NtlAtomicDecrement64(&gIoDmaStats.CommonBuffersInUse);
NtlAtomicExchangeAdd64(&gIoDmaStats.CommonBufferBytesInUse, -(int64_t)allocationSize);
IoDmaPutAdapterImpl(adapter);
}
@@ -411,7 +413,7 @@ static size_t IoDmaGetCommonBufferSizeImpl(const IO_DMA_BUFFER* buffer) {
}
// Returns whether a pinned access mode can satisfy a mapping direction.
static int IoDmaDirectionAllows(IO_DMA_DIRECTION pinned, IO_DMA_DIRECTION mapped) {
static bool IoDmaDirectionAllows(IO_DMA_DIRECTION pinned, IO_DMA_DIRECTION mapped) {
if (pinned == IoDmaBidirectional)
return mapped == IoDmaToDevice || mapped == IoDmaFromDevice || mapped == IoDmaBidirectional;
return pinned == mapped;
@@ -432,7 +434,7 @@ static MM_PINNED_MEMORY* IoDmaAllocatePinnedMemory(size_t pageCount) {
static void IoDmaReleasePinnedMemory(MM_PINNED_MEMORY* memory) {
if (!memory)
return;
long remaining = _InterlockedDecrement(&memory->RefCount);
int32_t remaining = NtlAtomicDecrement32(&memory->RefCount);
if (remaining > 0)
return;
if (remaining < 0)
@@ -446,16 +448,16 @@ static void IoDmaReleasePinnedMemory(MM_PINNED_MEMORY* memory) {
for (size_t i = 0; i < memory->PageCount; ++i)
(void)MmPfnUnpin(memory->Frames[i]);
}
_InterlockedIncrement64(&gIoDmaStats.PinnedRangesReleased);
_InterlockedDecrement64(&gIoDmaStats.PinnedRangesInUse);
_InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, -(long long)memory->PageCount);
NtlAtomicIncrement64(&gIoDmaStats.PinnedRangesReleased);
NtlAtomicDecrement64(&gIoDmaStats.PinnedRangesInUse);
NtlAtomicExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, -(int64_t)memory->PageCount);
MmFreePool(memory);
}
// Adds one mapping reference to pinned memory.
static void IoDmaReferencePinnedMemory(MM_PINNED_MEMORY* memory) {
if (memory)
_InterlockedIncrement(&memory->RefCount);
NtlAtomicIncrement32(&memory->RefCount);
}
// Pins a kernel virtual range for later DMA mapping.
@@ -480,7 +482,7 @@ static KSTATUS IoDmaPinKernelMemoryImpl(void* address,
MM_PINNED_MEMORY* memory = IoDmaAllocatePinnedMemory(pageCount);
if (!memory) {
_InterlockedIncrement64(&gIoDmaStats.FailedAllocations);
NtlAtomicIncrement64(&gIoDmaStats.FailedAllocations);
return NTLSTATUS_NO_MEMORY;
}
@@ -515,9 +517,9 @@ static KSTATUS IoDmaPinKernelMemoryImpl(void* address,
return status;
}
_InterlockedIncrement64(&gIoDmaStats.PinnedKernelRanges);
_InterlockedIncrement64(&gIoDmaStats.PinnedRangesInUse);
_InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (long long)pageCount);
NtlAtomicIncrement64(&gIoDmaStats.PinnedKernelRanges);
NtlAtomicIncrement64(&gIoDmaStats.PinnedRangesInUse);
NtlAtomicExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (int64_t)pageCount);
*outMemory = memory;
return NTLSTATUS_SUCCESS;
}
@@ -549,7 +551,7 @@ static KSTATUS IoDmaPinUserMemoryImpl(struct PROCESS* process,
MM_PINNED_MEMORY* memory = IoDmaAllocatePinnedMemory(pageCount);
if (!memory) {
MmUnlockUserBuffer(locked);
_InterlockedIncrement64(&gIoDmaStats.FailedAllocations);
NtlAtomicIncrement64(&gIoDmaStats.FailedAllocations);
return NTLSTATUS_NO_MEMORY;
}
@@ -564,9 +566,9 @@ static KSTATUS IoDmaPinUserMemoryImpl(struct PROCESS* process,
for (size_t i = 0; i < pageCount; ++i)
memory->Frames[i] = MmLockedUserBufferPageFrame(locked, i);
_InterlockedIncrement64(&gIoDmaStats.PinnedUserRanges);
_InterlockedIncrement64(&gIoDmaStats.PinnedRangesInUse);
_InterlockedExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (long long)pageCount);
NtlAtomicIncrement64(&gIoDmaStats.PinnedUserRanges);
NtlAtomicIncrement64(&gIoDmaStats.PinnedRangesInUse);
NtlAtomicExchangeAdd64(&gIoDmaStats.PinnedPagesInUse, (int64_t)pageCount);
*outMemory = memory;
return NTLSTATUS_SUCCESS;
}
@@ -842,7 +844,7 @@ static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter,
if (!KSUCCESS(status)) {
if (status != NTLSTATUS_NOT_SUPPORTED || (flags & IO_DMA_MAP_NO_BOUNCE)) {
_InterlockedIncrement64(&gIoDmaStats.FailedMappings);
NtlAtomicIncrement64(&gIoDmaStats.FailedMappings);
return status;
}
@@ -852,7 +854,7 @@ static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter,
0,
&bounce);
if (!KSUCCESS(status)) {
_InterlockedIncrement64(&gIoDmaStats.FailedMappings);
NtlAtomicIncrement64(&gIoDmaStats.FailedMappings);
return status;
}
status = IoDmaBuildContiguousSegments(bounce->DeviceAddress,
@@ -862,7 +864,7 @@ static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter,
&segmentCount);
if (!KSUCCESS(status)) {
IoDmaFreeCommonBufferImpl(bounce);
_InterlockedIncrement64(&gIoDmaStats.FailedMappings);
NtlAtomicIncrement64(&gIoDmaStats.FailedMappings);
return status;
}
}
@@ -870,7 +872,7 @@ static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter,
IO_DMA_MAPPING* mapping = IoDmaAllocateMapping(segmentCount);
if (!mapping) {
IoDmaFreeCommonBufferImpl(bounce);
_InterlockedIncrement64(&gIoDmaStats.FailedMappings);
NtlAtomicIncrement64(&gIoDmaStats.FailedMappings);
return NTLSTATUS_NO_MEMORY;
}
@@ -891,7 +893,7 @@ static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter,
if (!KSUCCESS(status)) {
MmFreePool(mapping);
IoDmaFreeCommonBufferImpl(bounce);
_InterlockedIncrement64(&gIoDmaStats.FailedMappings);
NtlAtomicIncrement64(&gIoDmaStats.FailedMappings);
return status;
}
@@ -909,12 +911,12 @@ static KSTATUS IoDmaMapPinnedMemoryImpl(IO_DMA_ADAPTER* adapter,
}
IoDmaReferencePinnedMemory(memory);
_InterlockedIncrement64(&gIoDmaStats.StreamingMappings);
_InterlockedIncrement64(&gIoDmaStats.StreamingMappingsInUse);
NtlAtomicIncrement64(&gIoDmaStats.StreamingMappings);
NtlAtomicIncrement64(&gIoDmaStats.StreamingMappingsInUse);
if (bounce)
_InterlockedIncrement64(&gIoDmaStats.BounceMappings);
NtlAtomicIncrement64(&gIoDmaStats.BounceMappings);
else
_InterlockedIncrement64(&gIoDmaStats.DirectMappings);
NtlAtomicIncrement64(&gIoDmaStats.DirectMappings);
*outMapping = mapping;
return NTLSTATUS_SUCCESS;
}
@@ -934,8 +936,8 @@ static KSTATUS IoDmaSynchronizeForDeviceImpl(IO_DMA_MAPPING* mapping) {
mapping->Length);
if (!KSUCCESS(status))
return status;
_InterlockedExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToDevice,
(long long)mapping->Length);
NtlAtomicExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToDevice,
(int64_t)mapping->Length);
}
_ReadWriteBarrier();
_mm_mfence();
@@ -961,8 +963,8 @@ static KSTATUS IoDmaSynchronizeForCpuImpl(IO_DMA_MAPPING* mapping) {
mapping->Length);
if (!KSUCCESS(status))
return status;
_InterlockedExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToCpu,
(long long)mapping->Length);
NtlAtomicExchangeAdd64(&gIoDmaStats.BounceBytesCopiedToCpu,
(int64_t)mapping->Length);
}
mapping->DeviceOwnsMapping = 0;
return NTLSTATUS_SUCCESS;
@@ -989,8 +991,8 @@ static void IoDmaUnmapMemoryImpl(IO_DMA_MAPPING* mapping) {
IoDmaFreeCommonBufferImpl(bounce);
IoDmaReleasePinnedMemory(memory);
IoDmaPutAdapterImpl(adapter);
_InterlockedIncrement64(&gIoDmaStats.StreamingMappingsUnmapped);
_InterlockedDecrement64(&gIoDmaStats.StreamingMappingsInUse);
NtlAtomicIncrement64(&gIoDmaStats.StreamingMappingsUnmapped);
NtlAtomicDecrement64(&gIoDmaStats.StreamingMappingsInUse);
}
// Returns the number of device segments in a streaming mapping.
code/kernel/io_file.c
+5 -4
View File
@@ -1,5 +1,6 @@
// Kernel file I/O layer. This file owns file handles, filesystem dispatch, and asynchronous request plumbing.
#include <stdbool.h>
#include <string.h>
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
@@ -58,13 +59,13 @@ typedef struct IO_FILE_LOCK_WAITER {
uint64_t Length;
uint32_t Mode;
EVENT* Event;
uint8_t Armed;
bool Armed;
} IO_FILE_LOCK_WAITER;
static SPINLOCK gFileLockLock = SPINLOCK_INIT;
static LIST_ENTRY gFileLocks;
static LIST_ENTRY gFileLockWaiters;
static uint8_t gFileLocksInitialized;
static bool gFileLocksInitialized;
// Initializes the global advisory file-lock lists on first use.
static void KiIoEnsureFileLockState(void) {
@@ -545,8 +546,8 @@ __declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJE
// Validates one canonical native path relative to an already-authoritative root capability.
// Native relative paths use '/' only and never carry drive/CWD or parent-traversal semantics.
static KSTATUS KiValidateNativeRelativePath(SYS_STRING_VIEW path, bool allowEmpty) {
if (!SysSvIsValid(path)) return NTLSTATUS_INVALID_PARAMETER;
if (SysSvContainsByte(path, '\0') || SysSvContainsByte(path, '\\'))
if (!SysSvIsUtf8Text(path)) return NTLSTATUS_INVALID_PARAMETER;
if (SysSvContainsByte(path, '\\'))
return NTLSTATUS_INVALID_PARAMETER;
if (path.Length == 0)
return allowEmpty ? NTLSTATUS_SUCCESS : NTLSTATUS_INVALID_PARAMETER;
code/kernel/io_pipe.c
+8 -6
View File
@@ -1,5 +1,7 @@
// Kernel pipe object lifetime, shared backing, readiness, and wait integration.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <kernel/io_pipe.h>
@@ -20,7 +22,7 @@ typedef enum IO_PIPE_DIRECTION {
} IO_PIPE_DIRECTION;
typedef struct IO_PIPE_STATE {
volatile long RefCount;
volatile int32_t RefCount;
SPINLOCK Lock;
EVENT* ReadEvent;
EVENT* WriteEvent;
@@ -80,7 +82,7 @@ static void KiPipeRefreshEventsLocked(IO_PIPE_STATE* state) {
// Releases shared pipe storage after both endpoint objects are gone.
static void KiPipeStateRelease(IO_PIPE_STATE* state) {
if (!state || _InterlockedDecrement(&state->RefCount) != 0)
if (!state || NtlAtomicDecrement32(&state->RefCount) != 0)
return;
if (state->KernelView) {
@@ -295,9 +297,9 @@ void IoPipeBeginWait(IO_PIPE_ENDPOINT* endpoint) {
return;
SYS_PIPE_SHARED_STATE* shared = endpoint->State->Shared;
if (endpoint->Direction == IoPipeReadEnd)
_InterlockedIncrement(&shared->ReadWaiters);
NtlAtomicIncrement32(&shared->ReadWaiters);
else
_InterlockedIncrement(&shared->WriteWaiters);
NtlAtomicIncrement32(&shared->WriteWaiters);
uintptr_t flags = KeAcquireSpinlock(&endpoint->State->Lock);
KiPipeRefreshEventsLocked(endpoint->State);
@@ -310,9 +312,9 @@ void IoPipeEndWait(IO_PIPE_ENDPOINT* endpoint) {
return;
SYS_PIPE_SHARED_STATE* shared = endpoint->State->Shared;
if (endpoint->Direction == IoPipeReadEnd)
_InterlockedDecrement(&shared->ReadWaiters);
NtlAtomicDecrement32(&shared->ReadWaiters);
else
_InterlockedDecrement(&shared->WriteWaiters);
NtlAtomicDecrement32(&shared->WriteWaiters);
}
// Returns whether this endpoint is the read side of the pipe.
code/kernel/io_volume.c
+3 -2
View File
@@ -1,5 +1,6 @@
// Mounted-volume core. This file owns volume identity, provider lifetime, and dead-volume dispatch semantics.
#include <stdbool.h>
#include <string.h>
#include <kernel/io_volume.h>
#include <kernel/ke_id.h>
@@ -13,7 +14,7 @@
static KE_ID_ALLOCATOR gVolumeIds = KE_ID_ALLOCATOR_INIT(1);
static SPINLOCK gVolumeRegistryLock = SPINLOCK_INIT;
static LIST_ENTRY gVolumeRegistry;
static uint8_t gVolumeRegistryInitialized;
static bool gVolumeRegistryInitialized;
static SPINLOCK gVolumeStatsLock = SPINLOCK_INIT;
static IO_VOLUME_STATS gVolumeStats;
static IO_VOLUME_MANAGER_STATS gVolumeManagerStats;
@@ -21,7 +22,7 @@ static EVENT* gVolumeChangeEvent;
static SPINLOCK gVolumeChangeLock = SPINLOCK_INIT;
static uint64_t gVolumeGeneration;
static IO_VOLUME_MANAGER gVolumeManagerObject;
static uint8_t gVolumeManagerObjectInitialized;
static bool gVolumeManagerObjectInitialized;
static void KiIoVolumeAddStat(uint64_t* field, int64_t delta);
code/kernel/ke_apc.c
+5 -3
View File
@@ -1,5 +1,7 @@
// Kernel APC infrastructure. This file owns APC queuing, delivery, and user-mode return transitions.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_cpu.h>
@@ -19,7 +21,7 @@
#define KI_RFLAGS_NT 0x4000ULL
#define KI_RFLAGS_VM 0x20000ULL
static volatile long long gUserApcReturnTokenSeed;
static volatile int64_t gUserApcReturnTokenSeed;
// Returns whether an address is in the canonical user half accepted by IRET.
static bool KiUserAddressValid(uint64_t address) {
@@ -49,7 +51,7 @@ static KSTATUS KiValidateUserApcResumeState(uint64_t rip, uint64_t rsp, uint64_t
// Allocates a nonzero token for one active user APC return frame.
static uint64_t KiNextUserApcReturnToken(THREAD* thread, uintptr_t frameAddress) {
uint64_t seed = (uint64_t)_InterlockedIncrement64(&gUserApcReturnTokenSeed);
uint64_t seed = (uint64_t)NtlAtomicIncrement64(&gUserApcReturnTokenSeed);
uint64_t token = seed ^ (uint64_t)(uintptr_t)thread ^ (uint64_t)frameAddress ^ SYS_USER_APC_FRAME_MAGIC;
if (token == 0)
token = SYS_USER_APC_FRAME_MAGIC;
@@ -113,7 +115,7 @@ __declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
if (thread->State == ThreadDead || KeThreadExitInProgress(thread))
return false;
if (_InterlockedCompareExchange(&apc->Inserted, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&apc->Inserted, 1, 0) != 0)
return false;
apc->SystemArgument1 = systemArgument1;
code/kernel/ke_audio.c
+74 -69
View File
@@ -1,5 +1,7 @@
// Kernel audio core. This file owns endpoint identity, logical streams, mixing, and provider timing handoff.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <intrin.h>
#include <stdint.h>
#include <stddef.h>
@@ -52,8 +54,8 @@ struct KE_AUDIO_STREAM {
LIST_ENTRY EndpointLink;
IO_AUDIO_ENDPOINT* Endpoint;
KE_AUDIO_STREAM_STATE State;
volatile long ProcessHandleCount;
volatile long CloseRequested;
volatile int32_t ProcessHandleCount;
volatile int32_t CloseRequested;
IO_AUDIO_FORMAT Format;
uint8_t* Ring;
uint32_t FrameBytes;
@@ -82,7 +84,7 @@ struct KE_AUDIO_STREAM {
uint64_t WaitDiscontinuityGeneration;
float Volume;
float Pan;
uint8_t Muted;
bool Muted;
float CurrentLeftGain;
float CurrentRightGain;
float TargetLeftGain;
@@ -106,10 +108,10 @@ struct IO_AUDIO_ENDPOINT {
IO_AUDIO_ENDPOINT_KIND Kind;
KE_AUDIO_ENDPOINT_STATE State;
uint32_t Capabilities;
uint8_t Available;
uint8_t Registered;
uint8_t HardwareRunning;
uint8_t SkipProviderStop;
bool Available;
bool Registered;
bool HardwareRunning;
bool SkipProviderStop;
IO_AUDIO_REMOVAL_MODE RemovalMode;
DEVICE_OBJECT* Device;
void* ProviderContext;
@@ -133,7 +135,7 @@ struct IO_AUDIO_ENDPOINT {
float MasterGainStep;
uint32_t MasterRampRemaining;
uint64_t MasterGainGeneration;
uint8_t MasterMuted;
bool MasterMuted;
KSTATUS FailureStatus;
uint64_t PeriodsMixed;
uint64_t PeriodsCompleted;
@@ -151,23 +153,23 @@ struct IO_AUDIO_ENDPOINT {
};
typedef struct KE_AUDIO_CORE_STATS {
volatile long long EndpointsRegistered;
volatile long long EndpointsUnregistered;
volatile long long StreamsCreated;
volatile long long StreamsDestroyed;
volatile long long Starts;
volatile long long Stops;
volatile long long Pauses;
volatile long long Resumes;
volatile long long Drains;
volatile long long Resets;
volatile int64_t EndpointsRegistered;
volatile int64_t EndpointsUnregistered;
volatile int64_t StreamsCreated;
volatile int64_t StreamsDestroyed;
volatile int64_t Starts;
volatile int64_t Stops;
volatile int64_t Pauses;
volatile int64_t Resumes;
volatile int64_t Drains;
volatile int64_t Resets;
} KE_AUDIO_CORE_STATS;
typedef struct KE_AUDIO_LIFECYCLE_SNAPSHOT {
KE_AUDIO_ENDPOINT_STATE State;
KSTATUS FailureStatus;
IO_AUDIO_REMOVAL_MODE RemovalMode;
uint8_t SkipProviderStop;
bool SkipProviderStop;
} KE_AUDIO_LIFECYCLE_SNAPSHOT;
static SPINLOCK gAudioRegistryLock = SPINLOCK_INIT;
@@ -175,7 +177,7 @@ static LIST_ENTRY gAudioEndpointList;
static uint32_t gAudioEndpointCount;
static KE_ID_ALLOCATOR gAudioEndpointIds = KE_ID_ALLOCATOR_INIT(1u);
static uint32_t gAudioDefaultEndpointId;
static uint8_t gAudioInitialized;
static bool gAudioInitialized;
static KE_AUDIO_CORE_STATS gAudioCoreStats;
// Stops on one violated internal audio invariant.
@@ -205,7 +207,7 @@ static uint64_t KiAudioQueryTicks(void) {
}
// Returns whether one float is finite without depending on a runtime math library.
static int KiAudioFloatIsFinite(float value) {
static bool KiAudioFloatIsFinite(float value) {
union {
float Float;
uint32_t Bits;
@@ -333,12 +335,12 @@ static uint32_t KiAudioFrameBytes(const IO_AUDIO_FORMAT* format) {
}
// Validates one logical source format accepted by the first mixer contract.
static int KiAudioValidateSourceFormat(const IO_AUDIO_FORMAT* format) {
if (!format) return 0;
static bool KiAudioValidateSourceFormat(const IO_AUDIO_FORMAT* format) {
if (!format) return false;
if (format->ChannelCount != 1 && format->ChannelCount != 2)
return 0;
return false;
if (format->SampleFormat != IoAudioSampleS16)
return 0;
return false;
return format->SampleRate == 11025u ||
format->SampleRate == 22050u ||
format->SampleRate == 44100u ||
@@ -392,10 +394,10 @@ static KSTATUS KiAudioValidateRegistration(
}
// Returns whether one endpoint lifecycle transition is valid.
static int KiAudioEndpointTransitionAllowed(KE_AUDIO_ENDPOINT_STATE oldState,
static bool KiAudioEndpointTransitionAllowed(KE_AUDIO_ENDPOINT_STATE oldState,
KE_AUDIO_ENDPOINT_STATE newState) {
if (oldState == newState)
return 1;
return true;
switch (oldState) {
case KeAudioEndpointStarting:
return newState == KeAudioEndpointReady ||
@@ -408,9 +410,9 @@ static int KiAudioEndpointTransitionAllowed(KE_AUDIO_ENDPOINT_STATE oldState,
case KeAudioEndpointStopping:
return newState == KeAudioEndpointRemoved;
case KeAudioEndpointRemoved:
return 0;
return false;
default:
return 0;
return false;
}
}
@@ -471,7 +473,7 @@ static IO_AUDIO_ENDPOINT* KiAudioReferenceEndpointById(uint32_t endpointId) {
// Signals one stream when an armed progress condition or terminal state is present.
static void KiAudioRefreshStreamEventLocked(KE_AUDIO_STREAM* stream) {
uint32_t writableFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames;
int signaled = stream->State == KeAudioStreamFailed ||
bool signaled = stream->State == KeAudioStreamFailed ||
stream->State == KeAudioStreamClosed;
if ((stream->WaitMask & KE_AUDIO_WAIT_WRITABLE) &&
writableFrames >= stream->WaitWritableFrames)
@@ -525,21 +527,21 @@ static void KiAudioAdjustActiveStreams(IO_AUDIO_ENDPOINT* endpoint, int delta) {
}
// Returns whether one stream state contributes to the active hardware clock.
static int KiAudioStreamStateIsActive(KE_AUDIO_STREAM_STATE state) {
static bool KiAudioStreamStateIsActive(KE_AUDIO_STREAM_STATE state) {
return state == KeAudioStreamRunning || state == KeAudioStreamDraining;
}
// Returns whether every frame targeted by one drain has reached a terminal disposition.
static int KiAudioDrainIsCompleteLocked(const KE_AUDIO_STREAM* stream) {
static bool KiAudioDrainIsCompleteLocked(const KE_AUDIO_STREAM* stream) {
return stream->BufferedFrames == 0 &&
stream->FramesPresented + stream->FramesDiscarded >= stream->FramesSubmitted;
}
// Returns whether one logical stream transition is valid.
static int KiAudioStreamTransitionAllowed(KE_AUDIO_STREAM_STATE oldState,
static bool KiAudioStreamTransitionAllowed(KE_AUDIO_STREAM_STATE oldState,
KE_AUDIO_STREAM_STATE newState) {
if (oldState == newState)
return 1;
return true;
if (newState == KeAudioStreamFailed || newState == KeAudioStreamClosed)
return oldState != KeAudioStreamClosed;
switch (oldState) {
@@ -561,7 +563,7 @@ static int KiAudioStreamTransitionAllowed(KE_AUDIO_STREAM_STATE oldState,
case KeAudioStreamFailed:
case KeAudioStreamClosed:
default:
return 0;
return false;
}
}
@@ -571,8 +573,8 @@ static void KiAudioSetStreamStateLocked(KE_AUDIO_STREAM* stream,
if (!KiAudioStreamTransitionAllowed(stream->State, newState))
KiAudioBug("KiAudioSetStreamStateLocked", "invalid stream transition",
((uint64_t)stream->State << 32) | (uint32_t)newState);
int oldActive = KiAudioStreamStateIsActive(stream->State);
int newActive = KiAudioStreamStateIsActive(newState);
bool oldActive = KiAudioStreamStateIsActive(stream->State);
bool newActive = KiAudioStreamStateIsActive(newState);
stream->State = newState;
if (oldActive != newActive)
KiAudioAdjustActiveStreams(stream->Endpoint, newActive ? 1 : -1);
@@ -656,8 +658,8 @@ static void KiAudioDecodeSourceFrame(const KE_AUDIO_STREAM* stream,
}
// Applies one final-handle close request while the stream mutex serializes state.
static int KiAudioApplyCloseRequestLocked(KE_AUDIO_STREAM* stream) {
if (!_InterlockedCompareExchange(&stream->CloseRequested, 0, 0) ||
static bool KiAudioApplyCloseRequestLocked(KE_AUDIO_STREAM* stream) {
if (!NtlAtomicLoad32(&stream->CloseRequested) ||
stream->State == KeAudioStreamClosed)
return stream->State == KeAudioStreamClosed;
@@ -673,7 +675,7 @@ static int KiAudioApplyCloseRequestLocked(KE_AUDIO_STREAM* stream) {
stream->ConsecutiveUnderrunPeriods = 0;
stream->DiscontinuityGeneration++;
KiAudioRefreshStreamEventLocked(stream);
return 1;
return true;
}
// Mixes one logical stream into one endpoint period.
@@ -803,7 +805,7 @@ static void KiAudioWriteOutputPeriod(IO_AUDIO_ENDPOINT* endpoint,
}
// Reserves one free hardware period for passive-level mixing.
static int KiAudioReservePeriod(IO_AUDIO_ENDPOINT* endpoint,
static bool KiAudioReservePeriod(IO_AUDIO_ENDPOINT* endpoint,
uint32_t* periodIndexOut,
uint64_t* generationOut) {
uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock);
@@ -816,10 +818,10 @@ static int KiAudioReservePeriod(IO_AUDIO_ENDPOINT* endpoint,
*periodIndexOut = i;
*generationOut = endpoint->Periods[i].Generation;
KeReleaseSpinlock(&endpoint->NotifyLock, flags);
return 1;
return true;
}
KeReleaseSpinlock(&endpoint->NotifyLock, flags);
return 0;
return false;
}
// Discards one contribution array after its mixed source frames cannot be presented.
@@ -914,7 +916,7 @@ static int KiAudioMixOnePeriod(IO_AUDIO_ENDPOINT* endpoint) {
}
// Applies one hardware-completed period to per-stream presentation counters.
static int KiAudioReapOneCompletedPeriod(IO_AUDIO_ENDPOINT* endpoint) {
static bool KiAudioReapOneCompletedPeriod(IO_AUDIO_ENDPOINT* endpoint) {
uint32_t periodIndex = 0;
KE_AUDIO_PERIOD_STATE* period = (KE_AUDIO_PERIOD_STATE*)0;
uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock);
@@ -928,7 +930,7 @@ static int KiAudioReapOneCompletedPeriod(IO_AUDIO_ENDPOINT* endpoint) {
}
KeReleaseSpinlock(&endpoint->NotifyLock, flags);
if (!period)
return 0;
return false;
for (uint32_t i = 0; i < period->ContributionCount; i++) {
KE_AUDIO_PERIOD_CONTRIBUTION* contribution = &period->Contributions[i];
@@ -952,7 +954,7 @@ static int KiAudioReapOneCompletedPeriod(IO_AUDIO_ENDPOINT* endpoint) {
period->ContributionCount = 0;
period->State = KE_AUDIO_PERIOD_STATE_FREE;
KeReleaseSpinlock(&endpoint->NotifyLock, flags);
return 1;
return true;
}
// Counts periods ready for hardware consumption.
@@ -1029,7 +1031,7 @@ static void KiAudioDiscardPeriodContributions(KE_AUDIO_PERIOD_STATE* period) {
static void KiAudioStopHardware(IO_AUDIO_ENDPOINT* endpoint, int hardwareAccessible) {
KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioStopHardware");
uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock);
int wasRunning = endpoint->HardwareRunning;
bool wasRunning = endpoint->HardwareRunning != 0;
endpoint->HardwareRunning = 0;
endpoint->IdleDeadlineTicks = 0;
KeReleaseSpinlock(&endpoint->NotifyLock, flags);
@@ -1140,7 +1142,7 @@ static void KiAudioSnapshotLifecycle(IO_AUDIO_ENDPOINT* endpoint,
}
// Processes all currently available period and lifecycle work.
static int KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint,
static bool KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint,
uint64_t* waitMillisecondsOut) {
*waitMillisecondsOut = (uint64_t)-1;
while (KiAudioReapOneCompletedPeriod(endpoint)) {
@@ -1152,7 +1154,7 @@ static int KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint,
if (lifecycle.State == KeAudioEndpointStopping ||
lifecycle.State == KeAudioEndpointFailed ||
lifecycle.State == KeAudioEndpointRemoved) {
int accessible = !lifecycle.SkipProviderStop &&
bool accessible = !lifecycle.SkipProviderStop &&
lifecycle.RemovalMode != IoAudioSurpriseRemoval;
KiAudioStopHardware(endpoint, accessible);
if (lifecycle.State == KeAudioEndpointRemoved && KSUCCESS(failure))
@@ -1160,12 +1162,12 @@ static int KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint,
if (KSUCCESS(failure))
failure = NTLSTATUS_DEVICE_ERROR;
KiAudioFailAllStreams(endpoint, failure);
return 0;
return false;
}
uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock);
uint32_t active = endpoint->ActiveStreamCount;
int running = endpoint->HardwareRunning;
bool running = endpoint->HardwareRunning != 0;
uint64_t idleDeadline = endpoint->IdleDeadlineTicks;
KeReleaseSpinlock(&endpoint->NotifyLock, flags);
@@ -1182,7 +1184,7 @@ static int KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint,
KiAudioSelectDefaultEndpointLocked();
KeReleaseSpinlock(&gAudioRegistryLock, registryFlags);
KeSignalEvent(endpoint->WorkEvent);
return 1;
return true;
}
}
}
@@ -1221,7 +1223,7 @@ static int KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint,
*waitMillisecondsOut = 1;
}
}
return 1;
return true;
}
// Runs one endpoint's real-time mixer and provider lifecycle loop.
@@ -1275,7 +1277,7 @@ void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream) {
if (!stream)
return;
long handleCount = _InterlockedIncrement(&stream->ProcessHandleCount);
int32_t handleCount = NtlAtomicIncrement32(&stream->ProcessHandleCount);
if (handleCount <= 0)
KiAudioBug("KeAudioStreamHandleOpened", "invalid process handle count", handleCount);
}
@@ -1285,13 +1287,13 @@ void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream) {
if (!stream)
return;
long handleCount = _InterlockedDecrement(&stream->ProcessHandleCount);
int32_t handleCount = NtlAtomicDecrement32(&stream->ProcessHandleCount);
if (handleCount < 0)
KiAudioBug("KeAudioStreamHandleClosed", "process handle underflow", handleCount);
if (handleCount != 0)
return;
_InterlockedExchange(&stream->CloseRequested, 1);
NtlAtomicExchange32(&stream->CloseRequested, 1);
KeSignalEvent(stream->WaitEvent);
if (stream->Endpoint)
KeSignalEvent(stream->Endpoint->WorkEvent);
@@ -1315,7 +1317,7 @@ static void KiAudioStreamDestroy(OBJECT* object) {
(uint64_t)(uintptr_t)stream->Lock->Owner);
if (!KiAudioStreamTransitionAllowed(stream->State, KeAudioStreamClosed))
KiAudioBug("KiAudioStreamDestroy", "invalid terminal stream state", stream->State);
int wasActive = KiAudioStreamStateIsActive(stream->State);
bool wasActive = KiAudioStreamStateIsActive(stream->State);
stream->State = KeAudioStreamClosed;
if (wasActive)
KiAudioAdjustActiveStreams(endpoint, -1);
@@ -1328,7 +1330,7 @@ static void KiAudioStreamDestroy(OBJECT* object) {
ObDereferenceObject(&stream->Lock->Header);
if (endpoint)
ObDereferenceObject(&endpoint->Header);
_InterlockedIncrement64(&gAudioCoreStats.StreamsDestroyed);
NtlAtomicIncrement64(&gAudioCoreStats.StreamsDestroyed);
}
// Registers one hardware-provider endpoint.
@@ -1440,7 +1442,7 @@ static KSTATUS KiAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* reg
KiAudioSelectDefaultEndpointLocked();
KeReleaseSpinlock(&gAudioRegistryLock, flags);
_InterlockedIncrement64(&gAudioCoreStats.EndpointsRegistered);
NtlAtomicIncrement64(&gAudioCoreStats.EndpointsRegistered);
*outEndpoint = endpoint;
return NTLSTATUS_SUCCESS;
}
@@ -1485,7 +1487,7 @@ static void KiAudioDdkUnregisterEndpoint(IO_AUDIO_ENDPOINT* endpoint,
ObDereferenceObject(&endpoint->MixerThread->Header);
endpoint->MixerThread = (THREAD*)0;
_InterlockedIncrement64(&gAudioCoreStats.EndpointsUnregistered);
NtlAtomicIncrement64(&gAudioCoreStats.EndpointsUnregistered);
ObDereferenceObject(&endpoint->Header);
ObDereferenceObject(&endpoint->Header);
}
@@ -1544,7 +1546,7 @@ static void KiAudioDdkNotifyPeriod(IO_AUDIO_ENDPOINT* endpoint,
}
// Updates one endpoint's jack-derived availability and default selection.
static void KiAudioDdkSetEndpointAvailable(IO_AUDIO_ENDPOINT* endpoint, uint8_t available) {
static void KiAudioDdkSetEndpointAvailable(IO_AUDIO_ENDPOINT* endpoint, bool available) {
if (!endpoint || !(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE))
KiAudioBug("KiAudioDdkSetEndpointAvailable", "invalid availability update", 0);
uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock);
@@ -1757,7 +1759,7 @@ KSTATUS KeAudioCreateStream(uint32_t endpointId,
KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags);
KeReleaseSpinlock(&gAudioRegistryLock, registryFlags);
_InterlockedIncrement64(&gAudioCoreStats.StreamsCreated);
NtlAtomicIncrement64(&gAudioCoreStats.StreamsCreated);
*outStream = stream;
return NTLSTATUS_SUCCESS;
}
@@ -1839,7 +1841,7 @@ KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream,
stream->FramesSubmitted += toWrite;
*writtenFramesOut = toWrite;
KiAudioRefreshStreamEventLocked(stream);
int active = KiAudioStreamStateIsActive(stream->State);
bool active = KiAudioStreamStateIsActive(stream->State);
KiAudioReleaseMutex(stream->Lock);
if (active)
@@ -1862,7 +1864,7 @@ KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream) {
stream->FailureStatus = NTLSTATUS_SUCCESS;
KiAudioRefreshStreamEventLocked(stream);
KiAudioReleaseMutex(stream->Lock);
_InterlockedIncrement64(&gAudioCoreStats.Starts);
NtlAtomicIncrement64(&gAudioCoreStats.Starts);
return NTLSTATUS_SUCCESS;
}
@@ -1880,7 +1882,7 @@ KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream) {
KiAudioSetStreamStateLocked(stream, KeAudioStreamPaused);
KiAudioRefreshStreamEventLocked(stream);
KiAudioReleaseMutex(stream->Lock);
_InterlockedIncrement64(&gAudioCoreStats.Pauses);
NtlAtomicIncrement64(&gAudioCoreStats.Pauses);
return NTLSTATUS_SUCCESS;
}
@@ -1898,7 +1900,7 @@ KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream) {
KiAudioSetStreamStateLocked(stream, KeAudioStreamRunning);
KiAudioRefreshStreamEventLocked(stream);
KiAudioReleaseMutex(stream->Lock);
_InterlockedIncrement64(&gAudioCoreStats.Resumes);
NtlAtomicIncrement64(&gAudioCoreStats.Resumes);
return NTLSTATUS_SUCCESS;
}
@@ -1921,7 +1923,7 @@ KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream) {
KiAudioRefreshStreamEventLocked(stream);
KiAudioReleaseMutex(stream->Lock);
KeSignalEvent(stream->Endpoint->WorkEvent);
_InterlockedIncrement64(&gAudioCoreStats.Drains);
NtlAtomicIncrement64(&gAudioCoreStats.Drains);
return NTLSTATUS_SUCCESS;
}
@@ -1945,7 +1947,7 @@ KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream) {
KiAudioRefreshStreamEventLocked(stream);
KiAudioReleaseMutex(stream->Lock);
KeSignalEvent(stream->Endpoint->WorkEvent);
_InterlockedIncrement64(&gAudioCoreStats.Stops);
NtlAtomicIncrement64(&gAudioCoreStats.Stops);
return NTLSTATUS_SUCCESS;
}
@@ -1979,7 +1981,7 @@ KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream) {
KiAudioRefreshStreamEventLocked(stream);
KiAudioReleaseMutex(stream->Lock);
KeSignalEvent(stream->Endpoint->WorkEvent);
_InterlockedIncrement64(&gAudioCoreStats.Resets);
NtlAtomicIncrement64(&gAudioCoreStats.Resets);
return NTLSTATUS_SUCCESS;
}
@@ -2027,7 +2029,9 @@ KSTATUS KeAudioSetMasterVolume(uint32_t endpointId, float volume, bool muted) {
if (!endpoint)
return NTLSTATUS_NOT_FOUND;
DbgPrint("audiodiag: master endpoint=%u waiting control lock\n", endpointId);
KiAudioAcquireMutex(endpoint->ControlLock, "KeAudioSetMasterVolume");
DbgPrint("audiodiag: master endpoint=%u acquired control lock\n", endpointId);
float softwareGain = muted ? 0.0f
: KiAudioSoftwareGainForVolume(endpoint, volume);
uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock);
@@ -2058,6 +2062,7 @@ KSTATUS KeAudioSetMasterVolume(uint32_t endpointId, float volume, bool muted) {
KeReleaseSpinlock(&gAudioRegistryLock, registryFlags);
KiAudioReleaseMutex(endpoint->ControlLock);
DbgPrint("audiodiag: master endpoint=%u update committed\n", endpointId);
KeSignalEvent(endpoint->WorkEvent);
ObDereferenceObject(&endpoint->Header);
return NTLSTATUS_SUCCESS;
code/kernel/ke_boot_config.c
+9 -1
View File
@@ -29,8 +29,16 @@ static bool KiBootConfigSpanEquals(const char* begin, const char* end, const cha
return begin == end && *text == '\0';
}
// Parses one unsigned decimal or 0x-prefixed hexadecimal value without accepting trailing junk.
// Parses an unsigned numeric configuration value with an optional inline comment.
static KSTATUS KiBootConfigParseUInt32(const char* begin, const char* end, uint32_t* valueOut) {
KiBootConfigTrim(&begin, &end);
// Numeric debug values support inline comments without changing command-line text semantics.
for (const char* p = begin; p < end; ++p) {
if (*p == ';' || *p == '#') {
end = p;
break;
}
}
KiBootConfigTrim(&begin, &end);
if (begin == end || !valueOut)
return NTLSTATUS_INVALID_PARAMETER;
code/kernel/ke_cpu.c
+7 -5
View File
@@ -1,5 +1,7 @@
// CPU-local kernel state. This file owns bootstrap and per-CPU bookkeeping shared by scheduler and SMP code.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <intrin.h>
@@ -44,8 +46,8 @@ KCPU gCpus[KE_MAX_CPUS];
uint32_t gCpuCount = 0;
static uint8_t gBootstrapCpuStack[BOOTSTRAP_CPU_STACK_SIZE];
static volatile long gKernelGsReady = 0;
static volatile long gWriteCombiningAvailable = 0;
static volatile int32_t gKernelGsReady = 0;
static volatile int32_t gWriteCombiningAvailable = 0;
extern KCPU* KeReadCurrentCpuFromGs(void);
@@ -103,7 +105,7 @@ void KeInitializeMemoryTypes(void) {
uint32_t variableCount = (uint32_t)(__readmsr(IA32_MTRRCAP) & MTRR_CAP_VARIABLE_COUNT);
if (KiMemoryTypePolicyInstalled(variableCount)) {
_InterlockedExchange(&gWriteCombiningAvailable, 1);
NtlAtomicExchange32(&gWriteCombiningAvailable, 1);
return;
}
@@ -139,12 +141,12 @@ void KeInitializeMemoryTypes(void) {
if (!installed)
KeStopProcessor();
_InterlockedExchange(&gWriteCombiningAvailable, 1);
NtlAtomicExchange32(&gWriteCombiningAvailable, 1);
}
// Reports whether PAT write-combining is available on the current platform.
bool KeWriteCombiningAvailable(void) {
return _InterlockedCompareExchange(&gWriteCombiningAvailable, 0, 0) != 0;
return NtlAtomicLoad32(&gWriteCombiningAvailable) != 0;
}
// Loads kernel Gs Base.
code/kernel/ke_debug.c
+39 -38
View File
@@ -1,5 +1,6 @@
// Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.
#include <ntlite/atomic.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
@@ -36,7 +37,7 @@ static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile long gDebugLogFlushing;
static volatile int32_t gDebugLogFlushing;
// Appends bytes to the in-memory debug log while its lock is held.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
@@ -97,13 +98,13 @@ static void KiOutputVPrintf(uint32_t stream, uint32_t flags,
KiOutputWrite(stream, flags, foreground, background, line, length);
}
static volatile long gPanicInProgress;
static volatile int32_t gPanicInProgress;
static volatile uintptr_t gPanicOwnerToken;
static volatile long gPanicStoppedCpuCount;
static volatile int32_t gPanicStoppedCpuCount;
// Permanently stops a secondary processor that entered an active panic.
static _Noreturn void KiPanicSecondaryHalt(void) {
_InterlockedIncrement(&gPanicStoppedCpuCount);
NtlAtomicIncrement32(&gPanicStoppedCpuCount);
KeStopProcessor();
}
@@ -119,12 +120,12 @@ static uintptr_t KiPanicCurrentToken(void) {
}
// Returns whether a fatal report is active.
static int KiPanicIsActive(void) {
return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
static bool KiPanicIsActive(void) {
return NtlAtomicLoad32(&gPanicInProgress) != 0;
}
// Returns whether the current processor owns the active fatal report.
static int KiPanicCurrentCpuOwnsReport(void) {
static bool KiPanicCurrentCpuOwnsReport(void) {
uintptr_t token = KiPanicCurrentToken();
return token != 0 && gPanicOwnerToken == token;
}
@@ -133,7 +134,7 @@ static int KiPanicCurrentCpuOwnsReport(void) {
static void KiBeginFatalReport(void) {
(void)KeDisableInterrupts();
uintptr_t token = KiPanicCurrentToken();
if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) == 0) {
if (NtlAtomicCompareExchange32(&gPanicInProgress, 1, 0) == 0) {
gPanicOwnerToken = token;
return;
}
@@ -142,14 +143,14 @@ static void KiBeginFatalReport(void) {
}
// Handles NMI-based panic stops and TLB shootdowns.
__declspec(dllexport) int KeHandleNmiIpi(void) {
__declspec(dllexport) bool KeHandleNmiIpi(void) {
if (KiPanicIsActive())
KiPanicSecondaryHalt();
if (MmPteHandleShootdownNmi())
return 1;
return true;
return 0;
return false;
}
// Initializes debug logging and initial output subscriptions.
@@ -208,7 +209,7 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!panic) {
uintptr_t lockFlags = KeAcquireSpinlock(&gDebugPrintLock);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
if (!NtlAtomicLoad32(&gDebugLogFlushing))
KiDebugLogAppendLocked(line, length);
KeReleaseSpinlock(&gDebugPrintLock, lockFlags);
}
@@ -249,7 +250,7 @@ KSTATUS KeFlushDebugLogToSystemRoot(const char* relativePath) {
if (!KeCanWait())
return NTLSTATUS_INVALID_IRQL;
if (_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&gDebugLogFlushing, 1, 0) != 0)
return NTLSTATUS_PENDING;
size_t used;
@@ -264,7 +265,7 @@ KSTATUS KeFlushDebugLogToSystemRoot(const char* relativePath) {
FILE_OBJECT* file = (FILE_OBJECT*)0;
KSTATUS st = KeCreateSystemRootFile(relativePath, &file);
if (!KSUCCESS(st)) {
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
NtlAtomicExchange32(&gDebugLogFlushing, 0);
return st;
}
@@ -292,7 +293,7 @@ KSTATUS KeFlushDebugLogToSystemRoot(const char* relativePath) {
st = IoFlushFile(file);
IoCloseFile(file);
_InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
NtlAtomicExchange32(&gDebugLogFlushing, 0);
return st;
}
@@ -318,7 +319,7 @@ static void KiPanicPrintf(const char* format, ...) {
}
// Stops the system after reporting a fatal kernel error.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
__declspec(dllexport) _Noreturn void KePanic(int32_t code, const char* fmt, ...) {
KiBeginFatalReport();
// Stop every other CPU before writing the final report. Panic-safe endpoints
@@ -443,9 +444,9 @@ static void KiPrintAddressValue(PROCESS* proc, uintptr_t addr) {
}
// Checks whether a kernel virtual range is mapped before diagnostic reads.
static int KiKernelRangeReadable(uintptr_t addr, size_t size) {
if (!addr || size == 0) return 0;
if (size - 1u > (size_t)(UINTPTR_MAX - addr)) return 0;
static bool KiKernelRangeReadable(uintptr_t addr, size_t size) {
if (!addr || size == 0) return false;
if (size - 1u > (size_t)(UINTPTR_MAX - addr)) return false;
uintptr_t start = addr & PAGE_MASK;
uintptr_t last = (addr + size - 1u) & PAGE_MASK;
@@ -454,41 +455,41 @@ static int KiKernelRangeReadable(uintptr_t addr, size_t size) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
if (!KSUCCESS(MmPteQueryPageFor(MmPteQueryDebug, page, &phys, &flags)))
return 0;
return false;
if ((flags & PAGE_PRESENT) == 0)
return 0;
return false;
if (page == last)
break;
if (page > UINTPTR_MAX - PAGE_SIZE)
return 0;
return false;
}
return 1;
return true;
}
// Reads diagnostic memory from either user space or a mapped kernel range.
static int KiReadDiagnosticMemory(uintptr_t addr, void* out, size_t size) {
if (!out || !addr || size == 0) return 0;
static bool KiReadDiagnosticMemory(uintptr_t addr, void* out, size_t size) {
if (!out || !addr || size == 0) return false;
if (addr >= KERNEL_BASE) {
if (!KiKernelRangeReadable(addr, size))
return 0;
return false;
memcpy(out, (const void*)addr, size);
return 1;
return true;
}
if (!KSUCCESS(MmProbeUserRead((const void*)addr, size)))
return 0;
return false;
memcpy(out, (const void*)addr, size);
return 1;
return true;
}
// Reads one diagnostic uint64_t.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
static bool KiUserReadU64(uintptr_t addr, uint64_t* out) {
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
// Reads one diagnostic byte.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
static bool KiUserReadU8(uintptr_t addr, uint8_t* out) {
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
@@ -518,7 +519,7 @@ static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
}
// Reads one diagnostic int32_t.
static int KiUserReadI32(uintptr_t addr, int32_t* out) {
static bool KiUserReadI32(uintptr_t addr, int32_t* out) {
return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}
@@ -585,11 +586,11 @@ static void KiDumpModules(PROCESS* proc) {
#if 0
// Returns whether a range is inside the current kernel stack.
static int KiKernelStackContains(const THREAD* thread, uintptr_t addr, size_t size) {
static bool KiKernelStackContains(const THREAD* thread, uintptr_t addr, size_t size) {
if (!thread || !thread->KernelStackBase || !thread->KernelStackTop || !addr || size == 0)
return 0;
return false;
if (size - 1u > (size_t)(UINTPTR_MAX - addr))
return 0;
return false;
uintptr_t base = (uintptr_t)thread->KernelStackBase;
uintptr_t top = thread->KernelStackTop;
@@ -598,9 +599,9 @@ static int KiKernelStackContains(const THREAD* thread, uintptr_t addr, size_t si
}
// Returns whether the fault frame is using the current kernel stack.
static int KiFaultFrameOnKernelStack(const THREAD* thread, const KFAULT_FRAME* f) {
static bool KiFaultFrameOnKernelStack(const THREAD* thread, const KFAULT_FRAME* f) {
if (!f || !thread)
return 0;
return false;
return KiKernelStackContains(thread, (uintptr_t)f->Rsp, sizeof(uint64_t));
}
@@ -896,7 +897,7 @@ __declspec(dllexport) KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) {
// The asm stubs collect the CPU exception frame values and call here.
// KeFaultDispatchFrame owns the mode split and default unhandled policy.
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
__declspec(dllexport) _Noreturn void KeFaultDispatch(uint32_t vector, uintptr_t rip,
uintptr_t errCode, uintptr_t cr2,
uintptr_t cs) {
KFAULT_FRAME frame;
code/kernel/ke_display.c
+105 -104
View File
@@ -1,5 +1,6 @@
// Kernel display manager. This file owns provider registration, framebuffer routing, dirty rectangles, text console state, and cursor delegation.
#include <ntlite/atomic.h>
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
@@ -72,12 +73,12 @@ static SPINLOCK gDisplayLock = SPINLOCK_INIT;
static MUTEX* gDisplayProviderMutex;
static MUTEX* gDisplayOutputMutex;
static KWORK_ITEM gDisplayCursorWorkItem;
static volatile long gDisplayCursorWorkInitialized;
static volatile int32_t gDisplayCursorWorkInitialized;
static bool gDisplayTransitionInProgress;
static uint64_t gDisplayGeneration;
static LIST_ENTRY gDisplayClockList;
static SPINLOCK gDisplayClockLock = SPINLOCK_INIT;
static volatile long gDisplayClockInitialized;
static volatile int32_t gDisplayClockInitialized;
static uint64_t gDisplayFrameGeneration;
static DISPLAY_DEVICE gBootFbDevice;
@@ -106,27 +107,27 @@ static int gConsoleY;
static uint16_t gConsoleAttribute;
static char gConsoleCells[DISPLAY_CONSOLE_MAX_CELLS];
static uint8_t gConsoleAttrs[DISPLAY_CONSOLE_MAX_CELLS];
static volatile long gDisplayConsoleSuspended;
static volatile int32_t gDisplayConsoleSuspended;
static SPINLOCK gDisplayOutputLock = SPINLOCK_INIT;
static DISPLAY_OUTPUT_GLYPH* gDisplayOutputBuffer;
static size_t gDisplayOutputRead;
static size_t gDisplayOutputWrite;
static size_t gDisplayOutputUsed;
static KWORK_ITEM gDisplayOutputWorkItem;
static volatile long gDisplayOutputWorkInitialized;
static volatile int32_t gDisplayOutputWorkInitialized;
static KSTATUS KiDisplayPrintCharInternal(char c, uint32_t fg, uint32_t bg);
static KSTATUS KiDisplayPrintGlyphsInternal(const DISPLAY_OUTPUT_GLYPH* glyphs, size_t count);
static KSTATUS BootFbGetMode(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color);
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h);
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride);
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels, uint32_t pitchBytes);
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color);
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int32_t dstX, int32_t dstY, int32_t srcX, int32_t srcY, int32_t w, int32_t h);
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t* outPixels, uint32_t outStride);
static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects, size_t rectCount);
static KSTATUS BootFbSync(DISPLAY_DEVICE* device);
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels);
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y);
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels);
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int32_t x, int32_t y);
static KSTATUS BootFbShowCursor(DISPLAY_DEVICE* device, bool show);
static KSTATUS BootFbQueryStats(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats);
@@ -235,7 +236,7 @@ static KSTATUS KiDisplayFallbackPrintChar(char c, uint32_t fg, uint32_t bg) {
}
static void KiDisplayQueueCursorWork(void) {
if (!_InterlockedCompareExchange(&gDisplayCursorWorkInitialized, 1, 1))
if (!NtlAtomicCompareExchange32(&gDisplayCursorWorkInitialized, 1, 1))
return;
if (KeIsWorkQueueStarted()) {
(void)KeQueueWorkItem(&gDisplayCursorWorkItem);
@@ -285,8 +286,8 @@ static void KiDisplayCursorWorker(void* context) {
DISPLAY_DEVICE* device = (DISPLAY_DEVICE*)0;
const DISPLAY_DEVICE_OPS* ops = (const DISPLAY_DEVICE_OPS*)0;
uint32_t caps = 0;
int x = 0;
int y = 0;
int32_t x = 0;
int32_t y = 0;
bool visible = false;
bool drawn = false;
bool suspended = false;
@@ -497,7 +498,7 @@ static DISPLAY_RECT KiDisplayUnionDirtyRects(const DISPLAY_RECT* a, const DISPLA
return result;
}
static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int32_t x, int32_t y, int32_t w, int32_t h) {
if (!mode || gDirtyAll || w <= 0 || h <= 0)
return;
@@ -533,13 +534,13 @@ static void KiDisplayAddDirtyForModeLocked(const DISPLAY_MODE* mode, int x, int
gDirtyRects[gDirtyRectCount++] = rect;
}
static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
static void KiDisplayRecordBlitLocked(const DISPLAY_MODE* mode, int32_t x, int32_t y, int32_t w, int32_t h) {
gDisplayStats.BlitCount++;
gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
KiDisplayAddDirtyForModeLocked(mode, x, y, w, h);
}
static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int32_t x, int32_t y, int32_t w, int32_t h) {
if (!mode || w <= 0 || h <= 0)
return;
@@ -557,13 +558,13 @@ static void KiDisplayRecordDirectPresentLocked(const DISPLAY_MODE* mode, int x,
gDisplayStats.BytesCopied += bytes;
}
static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int x, int y, int w, int h) {
static void KiDisplayRecordFillLocked(const DISPLAY_MODE* mode, int32_t x, int32_t y, int32_t w, int32_t h) {
gDisplayStats.FillRectCount++;
gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
KiDisplayAddDirtyForModeLocked(mode, x, y, w, h);
}
static void KiDisplayRecordCopyLocked(const DISPLAY_MODE* mode, int dstX, int dstY, int w, int h) {
static void KiDisplayRecordCopyLocked(const DISPLAY_MODE* mode, int32_t dstX, int32_t dstY, int32_t w, int32_t h) {
gDisplayStats.CopyRectCount++;
gDisplayStats.BytesCopied += (uint64_t)(uint32_t)w * (uint64_t)(uint32_t)h * sizeof(uint32_t);
KiDisplayAddDirtyForModeLocked(mode, dstX, dstY, w, h);
@@ -656,8 +657,8 @@ static void KiDisplayRestoreCursorAfterDrawSnapshot(DISPLAY_DEVICE* device,
if (!device || !ops || !ops->ShowCursor || !(caps & DISPLAY_CAP_SOFTWARE_CURSOR))
return;
int x = 0;
int y = 0;
int32_t x = 0;
int32_t y = 0;
bool shouldShow = false;
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
shouldShow = (device == gPrimaryDisplay) &&
@@ -719,9 +720,9 @@ static KSTATUS KiDisplayGetProviderSnapshot(DISPLAY_DEVICE** outDevice,
static void KiDisplayBuildGlyphPixels(char c, uint32_t fg, uint32_t bg, uint32_t* outPixels) {
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u);
for (int row = 0; row < FONT_H; row++) {
for (int32_t row = 0; row < FONT_H; row++) {
uint8_t bits = glyph[row];
for (int col = 0; col < FONT_W; col++)
for (int32_t col = 0; col < FONT_W; col++)
outPixels[row * FONT_W + col] = (bits & (0x80u >> col)) ? fg : bg;
}
}
@@ -730,8 +731,8 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
const DISPLAY_DEVICE_OPS* ops,
const DISPLAY_MODE* mode,
char c,
int x,
int y,
int32_t x,
int32_t y,
uint32_t fg,
uint32_t bg) {
if (!device || !ops || !ops->Blit || !mode)
@@ -753,10 +754,10 @@ static KSTATUS KiDisplayBlitGlyphSnapshot(DISPLAY_DEVICE* device,
static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device,
const DISPLAY_DEVICE_OPS* ops,
const DISPLAY_MODE* mode,
int x,
int y,
int w,
int h,
int32_t x,
int32_t y,
int32_t w,
int32_t h,
const uint32_t* pixels,
uint32_t pitchBytes) {
if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
@@ -781,10 +782,10 @@ static KSTATUS KiDisplayDrawBlitPitchedSnapshot(DISPLAY_DEVICE* device,
static KSTATUS KiDisplayFillRectSnapshot(DISPLAY_DEVICE* device,
const DISPLAY_DEVICE_OPS* ops,
const DISPLAY_MODE* mode,
int x,
int y,
int w,
int h,
int32_t x,
int32_t y,
int32_t w,
int32_t h,
uint32_t color) {
if (w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -804,12 +805,12 @@ static KSTATUS KiDisplayFillRectSnapshot(DISPLAY_DEVICE* device,
static KSTATUS KiDisplayCopyRectSnapshot(DISPLAY_DEVICE* device,
const DISPLAY_DEVICE_OPS* ops,
const DISPLAY_MODE* mode,
int dstX,
int dstY,
int srcX,
int srcY,
int w,
int h) {
int32_t dstX,
int32_t dstY,
int32_t srcX,
int32_t srcY,
int32_t w,
int32_t h) {
if (w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
if (!device || !ops || !ops->CopyRect || !mode)
@@ -844,7 +845,7 @@ static void KiDisplayDirtyAll(void) {
}
// Implements the display Add Dirty operation.
static void KiDisplayAddDirty(int x, int y, int w, int h) {
static void KiDisplayAddDirty(int32_t x, int32_t y, int32_t w, int32_t h) {
if (!KiDisplayHasDevice() || gDirtyAll || w <= 0 || h <= 0)
return;
@@ -946,7 +947,7 @@ static KSTATUS KiDisplayEndDraw(void) {
}
// Implements the display Draw Blit No Cursor operation.
static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint32_t* pixels) {
static KSTATUS KiDisplayDrawBlitNoCursor(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels) {
if (!pixels || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->Blit)
@@ -962,7 +963,7 @@ static KSTATUS KiDisplayDrawBlitNoCursor(int x, int y, int w, int h, const uint3
}
// Implements the display Fill Rect No Cursor operation.
static KSTATUS KiDisplayFillRectNoCursor(int x, int y, int w, int h, uint32_t color) {
static KSTATUS KiDisplayFillRectNoCursor(int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) {
if (w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->FillRect)
@@ -978,7 +979,7 @@ static KSTATUS KiDisplayFillRectNoCursor(int x, int y, int w, int h, uint32_t co
}
// Implements the display Copy Rect No Cursor operation.
static KSTATUS KiDisplayCopyRectNoCursor(int dstX, int dstY, int srcX, int srcY, int w, int h) {
static KSTATUS KiDisplayCopyRectNoCursor(int32_t dstX, int32_t dstY, int32_t srcX, int32_t srcY, int32_t w, int32_t h) {
if (w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
if (!KiDisplayHasDevice() || !gPrimaryDisplay->Ops->CopyRect)
@@ -994,7 +995,7 @@ static KSTATUS KiDisplayCopyRectNoCursor(int dstX, int dstY, int srcX, int srcY,
}
// Implements the display Console Index operation.
static size_t KiDisplayConsoleIndex(int col, int row) {
static size_t KiDisplayConsoleIndex(int32_t col, int32_t row) {
return (size_t)row * DISPLAY_CONSOLE_MAX_COLS + (size_t)col;
}
@@ -1054,8 +1055,8 @@ static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines)
return;
if (cellLines >= rows) {
for (int y = 0; y < rows; y++) {
for (int x = 0; x < cols; x++) {
for (int32_t y = 0; y < rows; y++) {
for (int32_t x = 0; x < cols; x++) {
size_t i = KiDisplayConsoleIndex(x, y);
gConsoleCells[i] = ' ';
gConsoleAttrs[i] = (uint8_t)gConsoleAttribute;
@@ -1064,8 +1065,8 @@ static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines)
return;
}
for (int y = 0; y < rows - cellLines; y++) {
for (int x = 0; x < cols; x++) {
for (int32_t y = 0; y < rows - cellLines; y++) {
for (int32_t x = 0; x < cols; x++) {
size_t dst = KiDisplayConsoleIndex(x, y);
size_t src = KiDisplayConsoleIndex(x, y + cellLines);
gConsoleCells[dst] = gConsoleCells[src];
@@ -1073,8 +1074,8 @@ static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines)
}
}
for (int y = rows - cellLines; y < rows; y++) {
for (int x = 0; x < cols; x++) {
for (int32_t y = rows - cellLines; y < rows; y++) {
for (int32_t x = 0; x < cols; x++) {
size_t i = KiDisplayConsoleIndex(x, y);
gConsoleCells[i] = ' ';
gConsoleAttrs[i] = (uint8_t)gConsoleAttribute;
@@ -1083,13 +1084,13 @@ static void KiDisplayConsoleScrollCellsLocked(int cols, int rows, int cellLines)
}
// Implements the display Draw Char Pixels Locked operation.
static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int x, int y, uint32_t fg, uint32_t bg) {
static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int32_t x, int32_t y, uint32_t fg, uint32_t bg) {
uint32_t glyphPixels[FONT_W * FONT_H];
const uint8_t* glyph = Font8x16 + ((uint8_t)c * 16u);
for (int row = 0; row < FONT_H; row++) {
for (int32_t row = 0; row < FONT_H; row++) {
uint8_t bits = glyph[row];
for (int col = 0; col < FONT_W; col++)
for (int32_t col = 0; col < FONT_W; col++)
glyphPixels[row * FONT_W + col] = (bits & (0x80u >> col)) ? fg : bg;
}
@@ -1097,7 +1098,7 @@ static KSTATUS KiDisplayDrawCharPixelsLocked(char c, int x, int y, uint32_t fg,
}
// Implements the display Draw Console Cell Locked operation.
static KSTATUS KiDisplayDrawConsoleCellLocked(int col, int row, char c, uint16_t attr, bool updateCell) {
static KSTATUS KiDisplayDrawConsoleCellLocked(int32_t col, int32_t row, char c, uint16_t attr, bool updateCell) {
if (updateCell) {
size_t i = KiDisplayConsoleIndex(col, row);
gConsoleCells[i] = c;
@@ -1177,7 +1178,7 @@ KSTATUS KeDisplayFlushConsoleOutput(void) {
// Queues the runtime display output worker when passive work is available.
static void KiDisplayQueueOutputWorker(void) {
if (_InterlockedCompareExchange(&gDisplayOutputWorkInitialized, 1, 1) &&
if (NtlAtomicCompareExchange32(&gDisplayOutputWorkInitialized, 1, 1) &&
KeIsWorkQueueStarted()) {
(void)KeQueueWorkItem(&gDisplayOutputWorkItem);
}
@@ -1271,15 +1272,15 @@ void KeDisplayInitialize(const HAL_API* hal) {
gDisplayClockLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&gDisplayClockList);
gDisplayFrameGeneration = 0;
_InterlockedExchange(&gDisplayClockInitialized, 1);
NtlAtomicExchange32(&gDisplayClockInitialized, 1);
if (!gDisplayProviderMutex)
(void)KeMutexCreate(&gDisplayProviderMutex);
if (!gDisplayOutputMutex)
(void)KeMutexCreate(&gDisplayOutputMutex);
KeInitializeWorkItem(&gDisplayCursorWorkItem, KiDisplayCursorWorker, 0);
_InterlockedExchange(&gDisplayCursorWorkInitialized, 1);
NtlAtomicExchange32(&gDisplayCursorWorkInitialized, 1);
KeInitializeWorkItem(&gDisplayOutputWorkItem, KiDisplayOutputWorker, 0);
_InterlockedExchange(&gDisplayOutputWorkInitialized, 1);
NtlAtomicExchange32(&gDisplayOutputWorkInitialized, 1);
gDisplayOutputRead = 0;
gDisplayOutputWrite = 0;
gDisplayOutputUsed = 0;
@@ -1291,7 +1292,7 @@ void KeDisplayInitialize(const HAL_API* hal) {
gConsoleX = 0;
gConsoleY = 0;
KiDisplayConsoleResetCellsLocked();
_InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, 0);
NtlAtomicExchange32(&gDisplayConsoleSuspended, 0);
memset(&gBootFb, 0, sizeof(gBootFb));
memset(&gBootFbDevice, 0, sizeof(gBootFbDevice));
@@ -1353,7 +1354,7 @@ DISPLAY_API KSTATUS KeDisplayOpenClock(uint64_t periodMilliseconds, DISPLAY_CLOC
return NTLSTATUS_INVALID_PARAMETER;
*outClock = (DISPLAY_CLOCK_OBJECT*)0;
if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
if (!NtlAtomicCompareExchange32(&gDisplayClockInitialized, 1, 1))
return NTLSTATUS_NOT_SUPPORTED;
uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
@@ -1429,7 +1430,7 @@ static void KiDisplaySignalFrameGeneration(uint64_t generation) {
// Publishes a frame-ready event from the current primary display provider.
DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device) {
if (!device || !_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
if (!device || !NtlAtomicCompareExchange32(&gDisplayClockInitialized, 1, 1))
return;
uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
@@ -1449,7 +1450,7 @@ DISPLAY_API void KeDisplayNotifyFrame(DISPLAY_DEVICE* device) {
}
DISPLAY_API void KeDisplayTick(void) {
if (!_InterlockedCompareExchange(&gDisplayClockInitialized, 1, 1))
if (!NtlAtomicCompareExchange32(&gDisplayClockInitialized, 1, 1))
return;
uintptr_t displayFlags = KeAcquireSpinlock(&gDisplayLock);
@@ -1555,12 +1556,12 @@ static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE
}
DISPLAY_API void KeDisplaySuspendConsole(bool suspend) {
_InterlockedExchange((volatile long*)&gDisplayConsoleSuspended, suspend ? 1 : 0);
NtlAtomicExchange32(&gDisplayConsoleSuspended, suspend ? 1 : 0);
}
// Implements the display Output Suspended operation.
static bool KiDisplayOutputSuspended(void) {
return _InterlockedCompareExchange((volatile long*)&gDisplayConsoleSuspended, 0, 0) != 0;
return NtlAtomicLoad32(&gDisplayConsoleSuspended) != 0;
}
DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) {
@@ -1676,7 +1677,7 @@ DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode) {
return s;
}
DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight) {
DISPLAY_API KSTATUS KeDisplayGetSize(int32_t* outWidth, int32_t* outHeight) {
DISPLAY_MODE mode;
KSTATUS s = KeDisplayGetMode(&mode);
if (!KSUCCESS(s))
@@ -1770,7 +1771,7 @@ DISPLAY_API KSTATUS KeDisplayQueryInfo(KE_DISPLAY_INFO* outInfo) {
return NTLSTATUS_SUCCESS;
}
DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels) {
DISPLAY_API KSTATUS KeDisplayBlit(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels) {
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
if (!KSUCCESS(s))
@@ -1792,7 +1793,7 @@ DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pi
return s;
}
DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
DISPLAY_API KSTATUS KeDisplayBlitPitched(int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels, uint32_t pitchBytes) {
if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
return NTLSTATUS_INVALID_PARAMETER;
@@ -1816,10 +1817,10 @@ DISPLAY_API KSTATUS KeDisplayBlitPitched(int x, int y, int w, int h, const uint3
return s;
}
DISPLAY_API KSTATUS KeDisplayPresentPitched(int x,
int y,
int w,
int h,
DISPLAY_API KSTATUS KeDisplayPresentPitched(int32_t x,
int32_t y,
int32_t w,
int32_t h,
const uint32_t* pixels,
uint32_t pitchBytes) {
if (!pixels || w <= 0 || h <= 0 || pitchBytes < (uint32_t)w * sizeof(uint32_t))
@@ -1857,7 +1858,7 @@ DISPLAY_API KSTATUS KeDisplayPresentPitched(int x,
return s;
}
DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color) {
DISPLAY_API KSTATUS KeDisplayFillRect(int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) {
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
if (!KSUCCESS(s))
@@ -1908,7 +1909,7 @@ DISPLAY_API KSTATUS KeDisplayClear(uint32_t color) {
return s;
}
DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h) {
DISPLAY_API KSTATUS KeDisplayCopyRect(int32_t dstX, int32_t dstY, int32_t srcX, int32_t srcY, int32_t w, int32_t h) {
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
if (!KSUCCESS(s))
@@ -1929,7 +1930,7 @@ DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, in
return s;
}
DISPLAY_API KSTATUS KeDisplayScroll(int lines) {
DISPLAY_API KSTATUS KeDisplayScroll(int32_t lines) {
if (lines <= 0)
return NTLSTATUS_SUCCESS;
@@ -1972,7 +1973,7 @@ DISPLAY_API KSTATUS KeDisplayScroll(int lines) {
return s;
}
DISPLAY_API KSTATUS KeDisplayPutChar(char c, int x, int y, uint32_t fg, uint32_t bg) {
DISPLAY_API KSTATUS KeDisplayPutChar(char c, int32_t x, int32_t y, uint32_t fg, uint32_t bg) {
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
if (!KSUCCESS(s))
@@ -2189,7 +2190,7 @@ DISPLAY_API KSTATUS KeDisplayGetConsoleInfo(KE_CONSOLE_INFO* outInfo) {
return NTLSTATUS_SUCCESS;
}
DISPLAY_API KSTATUS KeDisplaySetConsoleCursor(int col, int row) {
DISPLAY_API KSTATUS KeDisplaySetConsoleCursor(int32_t col, int32_t row) {
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
if (!KSUCCESS(s))
@@ -2226,7 +2227,7 @@ DISPLAY_API KSTATUS KeDisplaySetConsoleCursor(int col, int row) {
return NTLSTATUS_SUCCESS;
}
DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int col, int row, size_t length, size_t* writtenOut) {
DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int32_t col, int32_t row, size_t length, size_t* writtenOut) {
if (writtenOut) *writtenOut = 0;
bool providerHeld = false;
@@ -2258,8 +2259,8 @@ DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int col, int row, size_t l
(void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
size_t written = 0;
int x = col;
int y = row;
int32_t x = col;
int32_t y = row;
while (written < length && y < rows) {
uint16_t attr = DISPLAY_CONSOLE_DEFAULT_ATTR;
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
@@ -2295,7 +2296,7 @@ DISPLAY_API KSTATUS KeDisplayFillConsoleChars(char c, int col, int row, size_t l
return KSUCCESS(s) ? fs : s;
}
DISPLAY_API KSTATUS KeDisplayFillConsoleAttrs(uint16_t attr, int col, int row, size_t length, size_t* writtenOut) {
DISPLAY_API KSTATUS KeDisplayFillConsoleAttrs(uint16_t attr, int32_t col, int32_t row, size_t length, size_t* writtenOut) {
if (writtenOut) *writtenOut = 0;
bool providerHeld = false;
@@ -2327,8 +2328,8 @@ DISPLAY_API KSTATUS KeDisplayFillConsoleAttrs(uint16_t attr, int col, int row, s
(void)KiDisplayHideCursorForDrawSnapshot(device, ops, caps);
size_t written = 0;
int x = col;
int y = row;
int32_t x = col;
int32_t y = row;
while (written < length && y < rows) {
char ch = ' ';
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
@@ -2403,7 +2404,7 @@ DISPLAY_API KSTATUS KeDisplayPresent(const DISPLAY_RECT* rects, size_t rectCount
return s;
}
DISPLAY_API KSTATUS KeDisplaySetCursorImage(int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
DISPLAY_API KSTATUS KeDisplaySetCursorImage(int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
bool providerHeld = false;
KSTATUS s = KiDisplayAcquireProviderMutex(&providerHeld);
if (!KSUCCESS(s))
@@ -2424,7 +2425,7 @@ DISPLAY_API KSTATUS KeDisplaySetCursorImage(int width, int height, int hotX, int
return s;
}
DISPLAY_API KSTATUS KeDisplaySetCursorPosition(int x, int y) {
DISPLAY_API KSTATUS KeDisplaySetCursorPosition(int32_t x, int32_t y) {
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
gCursorX = x;
gCursorY = y;
@@ -2588,7 +2589,7 @@ static void BootFbCopyToShadowAndScanout(uint32_t* shadow,
}
// Implements the boot framebuffer pitched blit operation.
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, const uint32_t* pixels, uint32_t pitchBytes) {
static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, const uint32_t* pixels, uint32_t pitchBytes) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !pixels || w <= 0 || h <= 0 ||
pitchBytes < (uint32_t)w * sizeof(uint32_t))
@@ -2610,7 +2611,7 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co
fb->Stats.BlitCount++;
fb->Stats.BytesCopied += (uint64_t)(endX - startX) * (uint64_t)(endY - startY) * sizeof(uint32_t);
for (int row = 0; row < endY - startY; row++) {
for (int32_t row = 0; row < endY - startY; row++) {
uint32_t* dst = dstBase + ((size_t)(startY + row) * fb->Stride) + startX;
const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* src = (const uint32_t*)sourceRow + srcOffsetX;
@@ -2621,10 +2622,10 @@ static KSTATUS BootFbBlit(DISPLAY_DEVICE* device, int x, int y, int w, int h, co
// Presents one pitched source rectangle without a shadow-to-scanout second pass.
static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
int x,
int y,
int w,
int h,
int32_t x,
int32_t y,
int32_t w,
int32_t h,
const uint32_t* pixels,
uint32_t pitchBytes) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
@@ -2651,7 +2652,7 @@ static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
fb->Stats.ScanoutBytes += bytes;
fb->Stats.BytesCopied += fb->Backbuffer ? bytes * 2u : bytes;
for (int row = 0; row < endY - startY; row++) {
for (int32_t row = 0; row < endY - startY; row++) {
const uint8_t* sourceRow = (const uint8_t*)pixels + (size_t)(srcOffsetY + row) * pitchBytes;
const uint32_t* source = (const uint32_t*)sourceRow + srcOffsetX;
uint32_t* scanout = fb->Framebuffer + ((size_t)(startY + row) * fb->Stride) + startX;
@@ -2668,7 +2669,7 @@ static KSTATUS BootFbPresentPitched(DISPLAY_DEVICE* device,
}
// Implements the boot Fb Fill Rect operation.
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t color) {
static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t color) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -2694,7 +2695,7 @@ static KSTATUS BootFbFillRect(DISPLAY_DEVICE* device, int x, int y, int w, int h
}
// Implements the boot Fb Copy Rect operation.
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int srcX, int srcY, int w, int h) {
static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int32_t dstX, int32_t dstY, int32_t srcX, int32_t srcY, int32_t w, int32_t h) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -2717,13 +2718,13 @@ static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int sr
size_t rowBytes = (size_t)w * sizeof(uint32_t);
if (dstY > srcY) {
for (int row = h - 1; row >= 0; row--) {
for (int32_t row = h - 1; row >= 0; row--) {
uint32_t* dst = base + ((size_t)(dstY + row) * fb->Stride) + dstX;
uint32_t* src = base + ((size_t)(srcY + row) * fb->Stride) + srcX;
memmove(dst, src, rowBytes);
}
} else {
for (int row = 0; row < h; row++) {
for (int32_t row = 0; row < h; row++) {
uint32_t* dst = base + ((size_t)(dstY + row) * fb->Stride) + dstX;
uint32_t* src = base + ((size_t)(srcY + row) * fb->Stride) + srcX;
memmove(dst, src, rowBytes);
@@ -2733,7 +2734,7 @@ static KSTATUS BootFbCopyRect(DISPLAY_DEVICE* device, int dstX, int dstY, int sr
}
// Implements the boot Fb Read Rect operation.
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h, uint32_t* outPixels, uint32_t outStride) {
static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int32_t x, int32_t y, int32_t w, int32_t h, uint32_t* outPixels, uint32_t outStride) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (!fb || !fb->Framebuffer || !outPixels || outStride == 0 || w <= 0 || h <= 0)
return NTLSTATUS_INVALID_PARAMETER;
@@ -2749,7 +2750,7 @@ static KSTATUS BootFbReadRect(DISPLAY_DEVICE* device, int x, int y, int w, int h
int dstOffsetX = r.X - x;
int dstOffsetY = r.Y - y;
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
const uint32_t* src = srcBase + ((size_t)(r.Y + row) * fb->Stride) + r.X;
uint32_t* dst = outPixels + ((size_t)(dstOffsetY + row) * outStride) + dstOffsetX;
memcpy(dst, src, (size_t)r.W * sizeof(uint32_t));
@@ -2782,7 +2783,7 @@ static KSTATUS BootFbPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT* rects,
fb->Stats.BytesCopied += bytes;
fb->Stats.ScanoutBytes += bytes;
size_t pixelCount = (size_t)r.W;
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
BootFbCopyToScanout(dst, src, pixelCount);
@@ -2800,7 +2801,7 @@ static KSTATUS BootFbSync(DISPLAY_DEVICE* device) {
}
// Implements the boot Fb Set Cursor Image operation.
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int height, int hotX, int hotY, const uint32_t* argbPixels) {
static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int32_t width, int32_t height, int32_t hotX, int32_t hotY, const uint32_t* argbPixels) {
(void)device;
(void)width;
(void)height;
@@ -2811,7 +2812,7 @@ static KSTATUS BootFbSetCursorImage(DISPLAY_DEVICE* device, int width, int heigh
}
// Implements the boot Fb Set Cursor Position operation.
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int x, int y) {
static KSTATUS BootFbSetCursorPosition(DISPLAY_DEVICE* device, int32_t x, int32_t y) {
BOOTFB_CONTEXT* fb = BootFbContext(device);
if (fb) fb->Stats.CursorMoveCount++;
(void)x;
@@ -2846,7 +2847,7 @@ static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
DISPLAY_RECT r = { gCursorDrawX, gCursorDrawY, DISPLAY_CURSOR_W, DISPLAY_CURSOR_H };
DISPLAY_MODE mode = { fb->Width, fb->Height, fb->Stride, DisplayPixelFormatX8R8G8B8 };
if (KiDisplayClipRect(&mode, &r)) {
for (int row = 0; row < r.H; row++) {
for (int32_t row = 0; row < r.H; row++) {
uint32_t* dst = fb->Framebuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
uint32_t* src = fb->Backbuffer + ((size_t)(r.Y + row) * fb->Stride) + r.X;
memcpy(dst, src, (size_t)r.W * sizeof(uint32_t));
@@ -2859,18 +2860,18 @@ static void BootFbRestoreCursor(BOOTFB_CONTEXT* fb) {
}
// Implements the boot Fb Draw Cursor operation.
static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int x, int y) {
static void BootFbDrawCursor(BOOTFB_CONTEXT* fb, int32_t x, int32_t y) {
if (!fb || !fb->Framebuffer || !fb->Backbuffer)
return;
gCursorDrawX = x;
gCursorDrawY = y;
for (int row = 0; row < DISPLAY_CURSOR_H; row++) {
for (int32_t row = 0; row < DISPLAY_CURSOR_H; row++) {
int fy = y + row;
if ((unsigned)fy >= fb->Height)
continue;
for (int col = 0; col < DISPLAY_CURSOR_W; col++) {
for (int32_t col = 0; col < DISPLAY_CURSOR_W; col++) {
int fx = x + col;
if ((unsigned)fx >= fb->Width)
continue;
code/kernel/ke_dpc.c
+6 -4
View File
@@ -1,5 +1,7 @@
// Deferred procedure call infrastructure. This file owns DPC queuing and execution outside hard IRQ context.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_list.h>
#include <kernel/ke_irql.h>
@@ -8,8 +10,8 @@
static LIST_ENTRY gDpcQueue;
static SPINLOCK gDpcLock;
static volatile long gDpcInitialized = 0;
static volatile long gDpcActive = 0;
static volatile int32_t gDpcInitialized = 0;
static volatile int32_t gDpcActive = 0;
// Initializes dpc System.
void KeInitializeDpcSystem(void) {
@@ -52,7 +54,7 @@ __declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
// Coalesce repeated inserts of the same KDPC. The first insert owns the
// argument pair until the DPC is popped by the dispatcher.
if (_InterlockedCompareExchange(&dpc->Inserted, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&dpc->Inserted, 1, 0) != 0)
return false;
dpc->SystemArgument1 = systemArgument1;
@@ -74,7 +76,7 @@ void KiDrainDpcQueue(void) {
if (KeGetCurrentIrql() > DISPATCH_LEVEL)
return;
if (_InterlockedCompareExchange(&gDpcActive, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&gDpcActive, 1, 0) != 0)
return;
KIRQL oldIrql = KeGetCurrentIrql();
code/kernel/ke_event.c
+13 -12
View File
@@ -1,5 +1,6 @@
// Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <intrin.h>
@@ -20,14 +21,14 @@ typedef struct KE_EVENT_WAIT_BLOCK {
KE_EVENT_MULTI_WAIT_CONTEXT* Context;
EVENT* Event;
uint32_t Index;
uint8_t Armed;
bool Armed;
} KE_EVENT_WAIT_BLOCK;
struct KE_EVENT_MULTI_WAIT_CONTEXT {
THREAD* Thread;
KE_EVENT_WAIT_BLOCK* Blocks;
uint32_t Count;
volatile long Completed;
volatile int32_t Completed;
KSTATUS Status;
uint32_t SelectedIndex;
};
@@ -47,7 +48,7 @@ static void KiCompleteMultiWaitBlockLocked(KE_EVENT_WAIT_BLOCK* block) {
if (!ctx)
return;
if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&ctx->Completed, 1, 0) != 0)
return;
ctx->Status = NTLSTATUS_SUCCESS;
@@ -81,7 +82,7 @@ static void KiRundownMultiWait(THREAD* thread, void* context, KSTATUS status) {
if (!ctx)
return;
if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) == 0) {
if (NtlAtomicCompareExchange32(&ctx->Completed, 1, 0) == 0) {
ctx->Status = status;
ctx->SelectedIndex = 0;
}
@@ -96,7 +97,7 @@ static void KiRundownMultiWait(THREAD* thread, void* context, KSTATUS status) {
}
// Creates event.
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT** out) {
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, bool initialState, EVENT** out) {
if (!out) return NTLSTATUS_INVALID_PARAMETER;
EVENT* e = (EVENT*)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
if (!e) return NTLSTATUS_NO_MEMORY;
@@ -145,7 +146,7 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
LIST_ENTRY* node = event->MultiWaitList.Fwd;
KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link);
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
if (ctx && _InterlockedCompareExchange(&ctx->Completed, 0, 0) == 0) {
if (ctx && NtlAtomicLoad32(&ctx->Completed) == 0) {
event->Signaled = 0;
KiCompleteMultiWaitBlockLocked(block);
KeReleaseSpinlock(&event->Lock, flags);
@@ -182,7 +183,7 @@ __declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event) {
}
// Waits for Event.
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, bool alertable) {
if (!event) return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
if (alertable && KeAreKernelApcsDisabled()) return NTLSTATUS_INVALID_PARAMETER;
@@ -204,7 +205,7 @@ __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
// Waits for any of a set of events, returning the selected index.
__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count,
uint8_t alertable, uint64_t timeoutMs,
bool alertable, uint64_t timeoutMs,
uint32_t* signaledIndexOut) {
if (!events || count == 0 || count > SYS_WAIT_MANY_MAX_HANDLES)
return NTLSTATUS_INVALID_PARAMETER;
@@ -255,18 +256,18 @@ __declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t c
for (uint32_t i = 0; i < count; i++) {
EVENT* event = events[i];
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0 && event->Signaled) {
if (NtlAtomicLoad32(&ctx.Completed) == 0 && event->Signaled) {
if (event->Type == EventAutoReset)
event->Signaled = 0;
ctx.Status = NTLSTATUS_SUCCESS;
ctx.SelectedIndex = i;
_InterlockedExchange(&ctx.Completed, 1);
NtlAtomicExchange32(&ctx.Completed, 1);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadFromWaitListWithStatus(ctx.Thread, NTLSTATUS_SUCCESS);
break;
}
if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0) {
if (NtlAtomicLoad32(&ctx.Completed) == 0) {
stackBlocks[i].Armed = 1;
KeListInsertTailChecked(&event->MultiWaitList, &stackBlocks[i].Link);
}
@@ -274,7 +275,7 @@ __declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t c
}
KSTATUS waitStatus = KeCommitBlockThread(ctx.Thread, alertable);
if (_InterlockedCompareExchange(&ctx.Completed, 1, 0) == 0)
if (NtlAtomicCompareExchange32(&ctx.Completed, 1, 0) == 0)
ctx.Status = waitStatus;
KiCleanupMultiWaitBlocks(&ctx);
code/kernel/ke_hal.c
+2 -1
View File
@@ -1,5 +1,6 @@
// Kernel HAL binding and generic platform-operation wrappers.
#include <stdbool.h>
#include <kernel/ke_hal.h>
static const HAL_API* gHal;
@@ -18,7 +19,7 @@ KSTATUS KeBindBootstrapHal(void) {
}
// Returns whether the initialized table contains every mandatory kernel platform service.
static int KiHalRuntimeTableValid(const HAL_API* hal) {
static bool KiHalRuntimeTableValid(const HAL_API* hal) {
return hal && hal->Size == sizeof(HAL_API) && hal->Version == HAL_API_VERSION &&
hal->Processor.DisableInterrupts && hal->Processor.RestoreInterrupts &&
hal->Processor.EnableInterrupts && hal->Processor.WaitForInterrupt &&
code/kernel/ke_handle.c
+13 -16
View File
@@ -1,10 +1,11 @@
// Kernel capability handles. This file owns process-local handle identity, grant delegation, and revocation.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <kernel/ke_handle.h>
#include <kernel/ke_process.h>
#include <kernel/ke_named_queue.h>
#include <kernel/ke_debug.h>
#include <kernel/io_file.h>
#include <kernel/ke_audio.h>
@@ -14,8 +15,8 @@
#include <ntlite/port.h>
typedef struct CAPABILITY_GRANT {
volatile long RefCount;
volatile long Revoked;
volatile int32_t RefCount;
volatile int32_t Revoked;
uint32_t Depth;
SYS_RIGHTS MaximumAccess;
OBJECT* Target;
@@ -62,7 +63,7 @@ static KE_HANDLE_ENTRY* KiFindHandleLocked(KE_HANDLE_TABLE* table, SYS_HANDLE ha
// Acquires one grant reference while its owner still stabilizes the pointer.
static void KiReferenceGrant(CAPABILITY_GRANT* grant) {
if (!grant) return;
long refs = _InterlockedIncrement(&grant->RefCount);
int32_t refs = NtlAtomicIncrement32(&grant->RefCount);
if (refs <= 1) {
KePanic(KE_PANIC_CODE_GENERIC,
"capability: invalid grant reference grant=0x%016llX refs=%ld\n",
@@ -74,7 +75,7 @@ static void KiReferenceGrant(CAPABILITY_GRANT* grant) {
// Releases one grant and recursively drops its immutable parent chain.
static void KiDereferenceGrant(CAPABILITY_GRANT* grant) {
while (grant) {
long refs = _InterlockedDecrement(&grant->RefCount);
int32_t refs = NtlAtomicDecrement32(&grant->RefCount);
if (refs > 0) return;
if (refs < 0) {
KePanic(KE_PANIC_CODE_GENERIC,
@@ -92,17 +93,17 @@ static void KiDereferenceGrant(CAPABILITY_GRANT* grant) {
}
// Returns whether a grant and every ancestor remain unrevoked.
static int KiGrantIsActive(const CAPABILITY_GRANT* grant) {
static bool KiGrantIsActive(const CAPABILITY_GRANT* grant) {
const CAPABILITY_GRANT* current = grant;
uint32_t depth = 0;
while (current) {
if (_InterlockedCompareExchange((volatile long*)&current->Revoked, 0, 0) != 0)
return 0;
if (NtlAtomicLoad32(&current->Revoked) != 0)
return false;
current = current->Parent;
if (++depth > KE_CAPABILITY_MAX_DEPTH)
return 0;
return false;
}
return 1;
return true;
}
// Creates a root grant and takes the target object's lifetime reference.
@@ -246,7 +247,7 @@ SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type) {
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_EVENT_RIGHT_SIGNAL;
case ObjectTypeQueue:
return management | SYS_RIGHT_QUERY | SYS_QUEUE_RIGHT_SEND |
SYS_QUEUE_RIGHT_RECEIVE | SYS_QUEUE_RIGHT_PUBLISH;
SYS_QUEUE_RIGHT_RECEIVE;
case ObjectTypeFile:
return management | SYS_RIGHT_QUERY | SYS_FILE_RIGHT_READ_DATA |
SYS_FILE_RIGHT_WRITE_DATA | SYS_FILE_RIGHT_LOCK |
@@ -271,8 +272,6 @@ SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type) {
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT |
SYS_SURFACE_RIGHT_MAP_READ | SYS_SURFACE_RIGHT_MAP_WRITE |
SYS_SURFACE_RIGHT_COMMIT;
case ObjectTypeInputQueue:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_INPUT_QUEUE_RIGHT_READ;
case ObjectTypeTimer:
return management | SYS_RIGHT_QUERY | SYS_RIGHT_WAIT | SYS_TIMER_RIGHT_SET;
case ObjectTypeDisplayClock:
@@ -395,7 +394,6 @@ KSTATUS KeCloseHandle(PROCESS* process, SYS_HANDLE handle) {
OBJECT* object = entry->Grant->Target;
if (object && object->Type == ObjectTypeFile)
closeStatus = IoFlushFile((FILE_OBJECT*)object);
KeNamedQueueUnpublishHandle(process, handle, object);
KiHandleClosed(object);
KiDereferenceGrant(entry->Grant);
MmFreePool(entry);
@@ -533,7 +531,7 @@ KSTATUS KeRevokeHandle(PROCESS* process, SYS_HANDLE handle) {
return NTLSTATUS_CAPABILITY_REVOKED;
}
_InterlockedExchange(&entry->Grant->Revoked, 1);
NtlAtomicExchange32(&entry->Grant->Revoked, 1);
KeReleaseSpinlock(&table->Lock, flags);
return NTLSTATUS_SUCCESS;
}
@@ -757,7 +755,6 @@ void KeRundownHandleTable(PROCESS* process) {
if (!entry) break;
OBJECT* object = entry->Grant->Target;
KeNamedQueueUnpublishHandle(process, entry->Handle, object);
KiHandleClosed(object);
KiDereferenceGrant(entry->Grant);
MmFreePool(entry);
code/kernel/ke_id.c
+5 -4
View File
@@ -1,5 +1,6 @@
// Kernel monotonic identifier allocator. Public identities stay 32-bit while the internal cursor detects exhaustion without wrapping.
#include <ntlite/atomic.h>
#include <intrin.h>
#include <kernel/ke_id.h>
@@ -9,12 +10,12 @@ KSTATUS KeAllocateId32(KE_ID_ALLOCATOR* allocator, uint32_t* idOut) {
return NTLSTATUS_INVALID_PARAMETER;
while (1) {
long long next = _InterlockedCompareExchange64(&allocator->Next, 0, 0);
if ((unsigned long long)next > 0xFFFFFFFFULL)
int64_t next = NtlAtomicLoad64(&allocator->Next);
if ((uint64_t)next > UINT32_MAX)
return NTLSTATUS_ID_EXHAUSTED;
long long desired = next + 1;
if (_InterlockedCompareExchange64(&allocator->Next, desired, next) == next) {
int64_t desired = next + 1;
if (NtlAtomicCompareExchange64(&allocator->Next, desired, next) == next) {
*idOut = (uint32_t)next;
return NTLSTATUS_SUCCESS;
}
code/kernel/ke_input.c
+26 -485
View File
@@ -1,5 +1,6 @@
// Kernel input core. This file owns canonical keyboard and mouse state shared by input drivers and syscalls.
#include <ntlite/atomic.h>
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
@@ -11,62 +12,16 @@
#include <kernel/ke_debug.h>
#include <kernel/ke_display.h>
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
#include <kernel/ob_object.h>
#include <ntlite/syscall.h>
#define KEY_QUEUE_SIZE 256u
#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u)
#define MOUSE_QUEUE_SIZE 128u
#define MOUSE_QUEUE_MASK (MOUSE_QUEUE_SIZE - 1u)
#define MANAGER_QUEUE_SIZE 256u
#define MANAGER_QUEUE_MASK (MANAGER_QUEUE_SIZE - 1u)
#define MAX_INPUT_KEYS 256u
#define KEY_REPEAT_DELAY_US 500000ull
#define KEY_REPEAT_PERIOD_US 33000ull
typedef struct KEY_CHAR_PAIR {
INPUT_KEY key;
char normal;
char shifted;
} KEY_CHAR_PAIR;
static KE_INPUT_CONTEXT gDefaultInputContext;
static KE_INPUT_MANAGER gInputManager;
static KE_ID_ALLOCATOR gInputDeviceIds = KE_ID_ALLOCATOR_INIT(1u);
static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key);
static void KiInputServiceRepeat(KE_INPUT_CONTEXT* context);
// Advances the physical input snapshot generation after visible state changes.
static void KiInputBumpGeneration(KE_INPUT_CONTEXT* context) {
_InterlockedIncrement64(&context->Generation);
}
static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_1, '1', '!' }, { INPUT_KEY_2, '2', '@' },
{ INPUT_KEY_3, '3', '#' }, { INPUT_KEY_4, '4', '$' },
{ INPUT_KEY_5, '5', '%' }, { INPUT_KEY_6, '6', '^' },
{ INPUT_KEY_7, '7', '&' }, { INPUT_KEY_8, '8', '*' },
{ INPUT_KEY_9, '9', '(' }, { INPUT_KEY_0, '0', ')' },
{ INPUT_KEY_MINUS, '-', '_' }, { INPUT_KEY_EQUAL, '=', '+' },
{ INPUT_KEY_LEFT_BRACKET, '[', '{' }, { INPUT_KEY_RIGHT_BRACKET, ']', '}' },
{ INPUT_KEY_BACKSLASH, '\\', '|' }, { INPUT_KEY_SEMICOLON, ';', ':' },
{ INPUT_KEY_APOSTROPHE, '\'', '"' }, { INPUT_KEY_GRAVE, '`', '~' },
{ INPUT_KEY_COMMA, ',', '<' }, { INPUT_KEY_PERIOD, '.', '>' },
{ INPUT_KEY_SLASH, '/', '?' },
{ INPUT_KEY_SPACE, ' ', ' ' }, { INPUT_KEY_TAB, '\t', '\t' },
{ INPUT_KEY_ENTER, '\n', '\n' }, { INPUT_KEY_BACKSPACE, '\b', '\b' },
{ INPUT_KEY_KP_DIVIDE, '/', '/' }, { INPUT_KEY_KP_MULTIPLY, '*', '*' },
{ INPUT_KEY_KP_MINUS, '-', '-' }, { INPUT_KEY_KP_PLUS, '+', '+' },
{ INPUT_KEY_KP_ENTER, '\n', '\n' }, { INPUT_KEY_KP_1, '1', '1' },
{ INPUT_KEY_KP_2, '2', '2' }, { INPUT_KEY_KP_3, '3', '3' },
{ INPUT_KEY_KP_4, '4', '4' }, { INPUT_KEY_KP_5, '5', '5' },
{ INPUT_KEY_KP_6, '6', '6' }, { INPUT_KEY_KP_7, '7', '7' },
{ INPUT_KEY_KP_8, '8', '8' }, { INPUT_KEY_KP_9, '9', '9' },
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Updates the waitable manager event to match queued physical input.
static void KiInputRefreshManagerEvent(KE_INPUT_CONTEXT* context) {
if (!context->ManagerWaitEvent) return;
@@ -80,187 +35,30 @@ static void KiInputQueueManagerEvent(KE_INPUT_CONTEXT* context, const KE_INPUT_M
int next = (context->ManagerTail + 1) & (int)MANAGER_QUEUE_MASK;
if (next == context->ManagerHead) return;
KE_INPUT_MANAGER_EVENT queued = *event;
queued.Sequence = (uint64_t)_InterlockedIncrement64(&context->ManagerSequence);
queued.Sequence = (uint64_t)NtlAtomicIncrement64(&context->ManagerSequence);
context->ManagerQueue[context->ManagerTail] = queued;
context->ManagerTail = next;
KiInputRefreshManagerEvent(context);
}
// Updates the waitable keyboard event to match queued keyboard input.
static void KiInputRefreshKeyboardEvent(KE_INPUT_CONTEXT* context) {
if (!context->WaitEvent) return;
if (context->KeyHead != context->KeyTail) KeSignalEvent(context->WaitEvent);
else KeClearEvent(context->WaitEvent);
}
// Queues a keyboard event and wakes waiters when the event was accepted.
static void KiInputQueueKey(KE_INPUT_CONTEXT* context, INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (context->KeyTail + 1) & (int)KEY_QUEUE_MASK;
if (next == context->KeyHead) return;
context->KeyQueue[context->KeyTail].Key = key;
context->KeyQueue[context->KeyTail].Character = ch;
context->KeyQueue[context->KeyTail].Modifiers = context->Modifiers;
context->KeyTail = next;
KiInputRefreshKeyboardEvent(context);
}
// Implements the input Queue Has Space operation.
static bool KiInputQueueHasSpace(KE_INPUT_CONTEXT* context) {
int next = (context->KeyTail + 1) & (int)KEY_QUEUE_MASK;
return next != context->KeyHead;
}
// Implements the input Key Is Repeatable operation.
static bool KiInputKeyIsRepeatable(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return false;
switch (key) {
case INPUT_KEY_LEFT_SHIFT:
case INPUT_KEY_RIGHT_SHIFT:
case INPUT_KEY_LEFT_CTRL:
case INPUT_KEY_RIGHT_CTRL:
case INPUT_KEY_LEFT_ALT:
case INPUT_KEY_RIGHT_ALT:
case INPUT_KEY_LEFT_GUI:
case INPUT_KEY_RIGHT_GUI:
case INPUT_KEY_CAPS_LOCK:
case INPUT_KEY_NUM_LOCK:
return false;
default:
return true;
}
}
// Implements the input Now Ticks operation.
static uint64_t KiInputNowTicks(KE_INPUT_CONTEXT* context) {
if (context->Hal && context->Hal->Timer.QueryTicks)
return context->Hal->Timer.QueryTicks();
return 0;
}
// Implements the input Start Repeat operation.
static void KiInputStartRepeat(KE_INPUT_CONTEXT* context, INPUT_KEY key, uint64_t now) {
if (!KiInputKeyIsRepeatable(key)) return;
context->RepeatKey = key;
context->RepeatNextTick = now + KEY_REPEAT_DELAY_US;
}
// Implements the input Stop Repeat operation.
static void KiInputStopRepeat(KE_INPUT_CONTEXT* context, INPUT_KEY key) {
if (context->RepeatKey == key) {
context->RepeatKey = INPUT_KEY_NONE;
context->RepeatNextTick = 0;
}
}
// Implements the input Service Repeat operation.
static void KiInputServiceRepeat(KE_INPUT_CONTEXT* context) {
INPUT_KEY key = context->RepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !context->KeyDown[key]) {
context->RepeatKey = INPUT_KEY_NONE;
context->RepeatNextTick = 0;
return;
}
uint64_t now = KiInputNowTicks(context);
if (now == 0 || now < context->RepeatNextTick)
return;
if (!KiInputQueueHasSpace(context)) {
context->RepeatNextTick = now + KEY_REPEAT_PERIOD_US;
return;
}
KiInputQueueKey(context, key, KiInputKeyToChar(context, key));
context->RepeatNextTick = now + KEY_REPEAT_PERIOD_US;
}
// Implements the input Repeat Dpc Routine operation.
static void KiInputRepeatDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)systemArgument1;
(void)systemArgument2;
KiInputServiceRepeat((KE_INPUT_CONTEXT*)context);
}
// Implements the input Set Modifier operation.
static void KiInputSetModifier(KE_INPUT_CONTEXT* context, INPUT_KEY key, bool pressed) {
uint32_t bit = 0;
switch (key) {
case INPUT_KEY_LEFT_SHIFT: bit = INPUT_MOD_LSHIFT; break;
case INPUT_KEY_RIGHT_SHIFT: bit = INPUT_MOD_RSHIFT; break;
case INPUT_KEY_LEFT_CTRL: bit = INPUT_MOD_LCTRL; break;
case INPUT_KEY_RIGHT_CTRL: bit = INPUT_MOD_RCTRL; break;
case INPUT_KEY_LEFT_ALT: bit = INPUT_MOD_LALT; break;
case INPUT_KEY_RIGHT_ALT: bit = INPUT_MOD_RALT; break;
case INPUT_KEY_LEFT_GUI: bit = INPUT_MOD_LGUI; break;
case INPUT_KEY_RIGHT_GUI: bit = INPUT_MOD_RGUI; break;
default: break;
}
if (bit) {
if (pressed) context->Modifiers |= bit;
else context->Modifiers &= ~bit;
}
if (pressed && key == INPUT_KEY_CAPS_LOCK) context->Modifiers ^= INPUT_MOD_CAPS_LOCK;
if (pressed && key == INPUT_KEY_NUM_LOCK) context->Modifiers ^= INPUT_MOD_NUM_LOCK;
}
// Implements the input Key To Char operation.
static char KiInputKeyToChar(KE_INPUT_CONTEXT* context, INPUT_KEY key) {
if (key >= INPUT_KEY_A && key <= INPUT_KEY_Z) {
bool shifted = (context->Modifiers & INPUT_MOD_SHIFT) != 0;
bool caps = (context->Modifiers & INPUT_MOD_CAPS_LOCK) != 0;
char c = (char)('a' + (key - INPUT_KEY_A));
if (shifted ^ caps) c = (char)(c - 'a' + 'A');
return c;
}
bool shifted = (context->Modifiers & INPUT_MOD_SHIFT) != 0;
for (size_t i = 0; i < sizeof(gCharMap) / sizeof(gCharMap[0]); i++) {
if (gCharMap[i].key == key)
return shifted ? gCharMap[i].shifted : gCharMap[i].normal;
}
return 0;
}
// Implements the input Initialize operation.
void KeInputInitialize(const HAL_API* hal) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
ObInitializeObjectHeader(&gInputManager.Header, ObjectTypeInputManager);
context->Hal = hal;
memset((void*)context->KeyDown, 0, sizeof(context->KeyDown));
context->Modifiers = 0;
context->Generation = 0;
context->KeyHead = context->KeyTail = 0;
context->RepeatKey = INPUT_KEY_NONE;
context->RepeatNextTick = 0;
context->MouseHead = context->MouseTail = 0;
context->ManagerHead = context->ManagerTail = 0;
context->ManagerSequence = 0;
context->MouseBoundsWidth = 1920;
context->MouseBoundsHeight = 1080;
context->MouseWheel = 0;
context->MouseButtons = 0;
KeInitializeDpc(&context->RepeatDpc, KiInputRepeatDpcRoutine, context);
if (!context->WaitEvent) {
KSTATUS s = KeEventCreate(EventManualReset, 0, &context->WaitEvent);
if (!KSUCCESS(s))
DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
}
if (!context->ManagerWaitEvent) {
KSTATUS s = KeEventCreate(EventManualReset, 0, &context->ManagerWaitEvent);
if (!KSUCCESS(s))
DbgPrint("Input: manager wait event create failed st=0x%x\n", s);
}
KiInputRefreshKeyboardEvent(context);
KiInputRefreshManagerEvent(context);
int w = 1920, h = 1080;
int32_t w = 1920, h = 1080;
DISPLAY_MODE mode;
if (KSUCCESS(KeDisplayGetMode(&mode))) {
w = (int)mode.Width;
@@ -332,43 +130,19 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
INPUT_KEY key = event->Key;
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return;
if (key == INPUT_KEY_NONE || key >= 256u) return;
KE_INPUT_MANAGER_EVENT managerEvent;
memset(&managerEvent, 0, sizeof(managerEvent));
// Forward physical transitions without interpreting key state or repeat policy.
KE_INPUT_MANAGER_EVENT managerEvent = {0};
managerEvent.Type = SYS_INPUT_MANAGER_EVENT_KEY;
managerEvent.Data.Key = *event;
KiInputQueueManagerEvent(context, &managerEvent);
bool wasDown = context->KeyDown[key] != 0;
bool pressed = event->Pressed;
if (pressed) {
// The input core owns typematic repeat. Treat repeated hardware make
// events as idempotent so PS/2 typematic and USB HID share one policy.
if (wasDown)
return;
context->KeyDown[key] = 1u;
KiInputSetModifier(context, key, true);
KiInputBumpGeneration(context);
KiInputQueueKey(context, key, KiInputKeyToChar(context, key));
KiInputStartRepeat(context, key, KiInputNowTicks(context));
} else {
if (!wasDown)
return;
context->KeyDown[key] = 0u;
KiInputSetModifier(context, key, false);
KiInputBumpGeneration(context);
KiInputStopRepeat(context, key);
}
}
// Implements the input Commit Mouse Position operation.
static bool KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int nx, int ny, uint8_t buttons) {
int bw = context->MouseBoundsWidth;
int bh = context->MouseBoundsHeight;
static void KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int32_t nx, int32_t ny) {
int32_t bw = context->MouseBoundsWidth;
int32_t bh = context->MouseBoundsHeight;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
@@ -377,11 +151,8 @@ static bool KiInputCommitMousePosition(KE_INPUT_CONTEXT* context, int nx, int ny
if (nx >= bw) nx = bw - 1;
if (ny >= bh) ny = bh - 1;
bool changed = (nx != context->MouseX) || (ny != context->MouseY) || (buttons != context->MouseButtons);
context->MouseX = nx;
context->MouseY = ny;
context->MouseButtons = buttons;
return changed;
}
// Implements the input Submit Mouse operation.
@@ -395,23 +166,12 @@ __declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event) {
managerEvent.Data.Mouse = *event;
KiInputQueueManagerEvent(context, &managerEvent);
int oldX = context->MouseX;
int oldY = context->MouseY;
int nx = oldX + event->dx;
int ny = oldY + event->dy;
int32_t oldX = context->MouseX;
int32_t oldY = context->MouseY;
int32_t nx = oldX + event->dx;
int32_t ny = oldY + event->dy;
bool changed = KiInputCommitMousePosition(context, nx, ny, event->buttons);
if (event->wheel != 0) {
context->MouseWheel += event->wheel;
changed = true;
}
if (changed) KiInputBumpGeneration(context);
int next = (context->MouseTail + 1) & (int)MOUSE_QUEUE_MASK;
if (next != context->MouseHead) {
context->MouseQueue[context->MouseTail] = *event;
context->MouseTail = next;
}
KiInputCommitMousePosition(context, nx, ny);
}
// Implements the input Submit Mouse Absolute operation.
@@ -419,8 +179,8 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return;
int oldX = context->MouseX;
int oldY = context->MouseY;
int32_t oldX = context->MouseX;
int32_t oldY = context->MouseY;
INPUT_MOUSE_EVENT managerMouse;
managerMouse.dx = event->x - oldX;
@@ -433,213 +193,42 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
managerEvent.Data.Mouse = managerMouse;
KiInputQueueManagerEvent(context, &managerEvent);
bool changed = KiInputCommitMousePosition(context, event->x, event->y, event->buttons);
if (event->wheel != 0) {
context->MouseWheel += event->wheel;
changed = true;
}
if (changed) KiInputBumpGeneration(context);
INPUT_MOUSE_EVENT rel;
rel.dx = context->MouseX - oldX;
rel.dy = context->MouseY - oldY;
rel.wheel = event->wheel;
rel.buttons = event->buttons;
int next = (context->MouseTail + 1) & (int)MOUSE_QUEUE_MASK;
if (next != context->MouseHead) {
context->MouseQueue[context->MouseTail] = rel;
context->MouseTail = next;
}
KiInputCommitMousePosition(context, event->x, event->y);
}
// Implements the input Submit Mouse Absolute Scaled operation.
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons) {
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int32_t wheel, uint8_t buttons) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
int bw = context->MouseBoundsWidth;
int bh = context->MouseBoundsHeight;
int32_t bw = context->MouseBoundsWidth;
int32_t bh = context->MouseBoundsHeight;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
INPUT_MOUSE_ABSOLUTE_EVENT ev;
ev.x = maxX ? (int)(((uint64_t)rawX * (uint64_t)(bw - 1)) / (uint64_t)maxX) : 0;
ev.y = maxY ? (int)(((uint64_t)rawY * (uint64_t)(bh - 1)) / (uint64_t)maxY) : 0;
ev.x = maxX ? (int32_t)(((uint64_t)rawX * (uint64_t)(bw - 1)) / (uint64_t)maxX) : 0;
ev.y = maxY ? (int32_t)(((uint64_t)rawY * (uint64_t)(bh - 1)) / (uint64_t)maxY) : 0;
ev.wheel = wheel;
ev.buttons = buttons;
KeInputSubmitMouseAbsolute(&ev);
}
// Returns the next queued key without consuming it.
INPUT_KEY KeInputKbPoll(void) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (context->KeyHead == context->KeyTail) KiInputServiceRepeat(context);
return (context->KeyHead == context->KeyTail) ? INPUT_KEY_NONE : context->KeyQueue[context->KeyHead].Key;
}
// Waits for keyboard input in one context or until the timeout expires.
static KSTATUS KiInputWaitKey(KE_INPUT_CONTEXT* context, uint8_t alertable, uint64_t timeoutMs) {
if (!context)
return NTLSTATUS_INVALID_PARAMETER;
KiInputServiceRepeat(context);
if (context->KeyHead != context->KeyTail)
return NTLSTATUS_SUCCESS;
if (!context->WaitEvent)
return NTLSTATUS_FAILURE;
while (true) {
EVENT* event = context->WaitEvent;
uint32_t selected = 0;
KSTATUS s = KeWaitForMultipleEvents(&event, 1, alertable, timeoutMs, &selected);
if (!KSUCCESS(s))
return s;
KiInputServiceRepeat(context);
if (context->KeyHead != context->KeyTail)
return NTLSTATUS_SUCCESS;
}
}
// Waits for keyboard input in the default input context.
KSTATUS KeInputWaitKey(uint8_t alertable, uint64_t timeoutMs) {
return KiInputWaitKey(&gDefaultInputContext, alertable, timeoutMs);
}
// Services typematic repeat from the scheduler timer bottom half.
void KeInputTimerTick(void) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
INPUT_KEY key = context->RepeatKey;
if (!KiInputKeyIsRepeatable(key) || key >= MAX_INPUT_KEYS || !context->KeyDown[key])
return;
uint64_t now = KiInputNowTicks(context);
if (now != 0 && now >= context->RepeatNextTick)
KeInsertQueueDpc(&context->RepeatDpc, (void*)0, (void*)0);
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
return context->KeyDown[key] != 0;
}
// Peeks one keyboard event from an input context without consuming it.
static bool KiInputPeekEvent(KE_INPUT_CONTEXT* context, INPUT_KEY_RECORD* record) {
if (!context || !record) return false;
if (context->KeyHead == context->KeyTail) KiInputServiceRepeat(context);
if (context->KeyHead == context->KeyTail) return false;
INPUT_KEY_QUEUE_ENTRY entry = context->KeyQueue[context->KeyHead];
record->Key = entry.Key;
record->Char = entry.Character;
record->Modifiers = entry.Modifiers;
return true;
}
// Reads one keyboard event from an input context.
static bool KiInputReadEvent(KE_INPUT_CONTEXT* context, INPUT_KEY_RECORD* record) {
if (!KiInputPeekEvent(context, record)) return false;
context->KeyHead = (context->KeyHead + 1) & (int)KEY_QUEUE_MASK;
KiInputRefreshKeyboardEvent(context);
return true;
}
// Peeks one keyboard event from the default input context.
bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
return KiInputPeekEvent(&gDefaultInputContext, record);
}
// Reads one keyboard event from the default input context.
bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
return KiInputReadEvent(&gDefaultInputContext, record);
}
// Implements the input Read Key operation.
INPUT_KEY KeInputReadKey(void) {
INPUT_KEY_RECORD record;
return KeInputReadEvent(&record) ? record.Key : INPUT_KEY_NONE;
}
// Implements the input Read Char operation.
char KeInputReadChar(void) {
INPUT_KEY_RECORD record;
return KeInputReadEvent(&record) ? record.Char : 0;
}
// Implements the input Get Modifiers operation.
uint32_t KeInputGetModifiers(void) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
return context->Modifiers;
}
// Snapshots the current physical global input state without consuming events.
KSTATUS KeInputQueryGlobalState(SYS_GLOBAL_INPUT_STATE* out) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!out) return NTLSTATUS_INVALID_PARAMETER;
out->Size = sizeof(*out);
out->Flags = SYS_GLOBAL_INPUT_STATE_FLAG_NONE;
out->Generation = (uint64_t)context->Generation;
out->Modifiers = context->Modifiers;
memset(out->KeyDown, 0, sizeof(out->KeyDown));
for (uint32_t key = 0; key < MAX_INPUT_KEYS; key++) {
if (context->KeyDown[key])
out->KeyDown[key >> 3] |= (uint8_t)(1u << (key & 7u));
}
out->MouseX = context->MouseX;
out->MouseY = context->MouseY;
out->MouseWheel = context->MouseWheel;
out->MouseButtons = context->MouseButtons;
return NTLSTATUS_SUCCESS;
}
// Implements the input Mouse Peek operation.
bool KeInputMousePeek(INPUT_MOUSE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!event) return false;
if (context->MouseHead == context->MouseTail) return false;
*event = context->MouseQueue[context->MouseHead];
return true;
}
// Implements the input Mouse Read operation.
bool KeInputMouseRead(INPUT_MOUSE_EVENT* event) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!KeInputMousePeek(event)) return false;
context->MouseHead = (context->MouseHead + 1) & (int)MOUSE_QUEUE_MASK;
return true;
}
// Implements the input Mouse Get Position operation.
void KeInputMouseGetPosition(int* x, int* y) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (x) *x = context->MouseX;
if (y) *y = context->MouseY;
}
// Implements the input Mouse Set Position operation.
void KeInputMouseSetPosition(int x, int y) {
void KeInputMouseSetPosition(int32_t x, int32_t y) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
int bw = context->MouseBoundsWidth;
int bh = context->MouseBoundsHeight;
int32_t bw = context->MouseBoundsWidth;
int32_t bh = context->MouseBoundsHeight;
if (bw < 1) bw = 1;
if (bh < 1) bh = 1;
if (x < 0) x = 0;
if (y < 0) y = 0;
if (x >= bw) x = bw - 1;
if (y >= bh) y = bh - 1;
bool changed = (x != context->MouseX) || (y != context->MouseY);
context->MouseX = x;
context->MouseY = y;
if (changed) KiInputBumpGeneration(context);
}
// Implements the input Mouse Set Bounds operation.
void KeInputMouseSetBounds(int w, int h) {
void KeInputMouseSetBounds(int32_t w, int32_t h) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (w < 1) w = 1;
if (h < 1) h = 1;
@@ -647,51 +236,3 @@ void KeInputMouseSetBounds(int w, int h) {
context->MouseBoundsHeight = h;
KeInputMouseSetPosition(context->MouseX, context->MouseY);
}
// Destroys an input queue handle.
static void KiInputQueueDestroy(OBJECT* object) {
INPUT_QUEUE* queue = CONTAINING_RECORD(object, INPUT_QUEUE, Header);
if (queue->WaitEvent) {
ObDereferenceObject(&queue->WaitEvent->Header);
queue->WaitEvent = (EVENT*)0;
}
queue->Context = (KE_INPUT_CONTEXT*)0;
}
// Opens the default keyboard input queue as a waitable object.
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
KE_INPUT_CONTEXT* context = &gDefaultInputContext;
if (!out) return NTLSTATUS_INVALID_PARAMETER;
*out = (INPUT_QUEUE*)0;
if (!context->WaitEvent) return NTLSTATUS_FAILURE;
INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
if (!queue) return NTLSTATUS_NO_MEMORY;
queue->Header.Destroy = KiInputQueueDestroy;
queue->Context = context;
ObReferenceObject(&context->WaitEvent->Header);
queue->WaitEvent = context->WaitEvent;
*out = queue;
return NTLSTATUS_SUCCESS;
}
// Peeks one keyboard event from the queue's owning context.
bool KeInputQueuePeekEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record) {
return queue ? KiInputPeekEvent(queue->Context, record) : false;
}
// Reads one keyboard event from the queue's owning context.
bool KeInputQueueReadEvent(INPUT_QUEUE* queue, INPUT_KEY_RECORD* record) {
return queue ? KiInputReadEvent(queue->Context, record) : false;
}
// Waits for keyboard input in the queue's owning context.
KSTATUS KeInputQueueWaitKey(INPUT_QUEUE* queue, uint8_t alertable, uint64_t timeoutMs) {
return queue ? KiInputWaitKey(queue->Context, alertable, timeoutMs)
: NTLSTATUS_INVALID_PARAMETER;
}
// Returns the event that wakes when keyboard input can be read.
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
return queue ? queue->WaitEvent : (EVENT*)0;
}
code/kernel/ke_irp.c
+6 -4
View File
@@ -1,5 +1,7 @@
// I/O request packet infrastructure. This file owns IRP dispatch, queuing, cancellation, and remove-lock helpers.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
#include <kernel/ke_irp.h>
@@ -95,7 +97,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
KeReleaseSpinlock(&lock->Lock, flags);
return NTLSTATUS_OBJECT_DEAD;
}
_InterlockedIncrement(&lock->IoCount);
NtlAtomicIncrement32(&lock->IoCount);
KeReleaseSpinlock(&lock->Lock, flags);
return NTLSTATUS_SUCCESS;
}
@@ -104,7 +106,7 @@ __declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
(void)irp;
if (!lock) return;
long left = _InterlockedDecrement(&lock->IoCount);
int32_t left = NtlAtomicDecrement32(&lock->IoCount);
if (left == 0 && lock->Removing && lock->RemovedEvent)
KeSignalEvent(lock->RemovedEvent);
}
@@ -116,7 +118,7 @@ __declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP*
uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
lock->Removing = true;
long left = _InterlockedDecrement(&lock->IoCount); // release sentinel
int32_t left = NtlAtomicDecrement32(&lock->IoCount); // release sentinel
bool done = (left == 0);
KeReleaseSpinlock(&lock->Lock, flags);
@@ -297,7 +299,7 @@ __declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
if (dev) {
_InterlockedDecrement(&dev->OutstandingIrps);
NtlAtomicDecrement32(&dev->OutstandingIrps);
IoDereferenceDeviceObject(dev);
ios->DeviceObject = (DEVICE_OBJECT*)0;
}
code/kernel/ke_irql.c
+2 -1
View File
@@ -1,5 +1,6 @@
// Kernel IRQL model. This file owns the software interrupt-priority state used by IRQ and scheduler code.
#include <stdbool.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
@@ -87,6 +88,6 @@ __declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
}
// Checks whether wait.
__declspec(dllexport) int KeCanWait(void) {
__declspec(dllexport) bool KeCanWait(void) {
return KeGetCurrentIrql() == PASSIVE_LEVEL;
}
code/kernel/ke_mutex.c
+7 -7
View File
@@ -36,8 +36,8 @@ static void KiMutexReferenceOwnerLocked(MUTEX* mutex, THREAD* owner) {
}
// Tests whether one scheduler link is currently attached to a list.
static int KiMutexWaiterIsLinked(const THREAD* thread) {
if (!thread) return 0;
static bool KiMutexWaiterIsLinked(const THREAD* thread) {
if (!thread) return false;
return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
thread->SchedulerLink.Bwd != &thread->SchedulerLink;
}
@@ -48,7 +48,7 @@ static void KiMutexPropagatePriority(THREAD* thread) {
for (uint32_t depth = 1; thread && depth <= KE_PRIORITY_INHERITANCE_MAX_DEPTH; depth++) {
MUTEX* waitingMutex = (MUTEX*)0;
int changed = KiSchedulerRefreshEffectivePriority(thread, &waitingMutex);
bool changed = KiSchedulerRefreshEffectivePriority(thread, &waitingMutex);
if (!changed) {
if (releaseThread) ObDereferenceObject(&thread->Header);
return;
@@ -223,8 +223,8 @@ __declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
}
// Removes one blocked mutex waiter during thread-wait rundown.
int KiMutexAbortWait(MUTEX* mutex, THREAD* thread) {
if (!mutex || !thread) return 0;
bool KiMutexAbortWait(MUTEX* mutex, THREAD* thread) {
if (!mutex || !thread) return false;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (thread->WaitingMutex != mutex ||
@@ -232,7 +232,7 @@ int KiMutexAbortWait(MUTEX* mutex, THREAD* thread) {
thread->WaitListHead != &mutex->WaitList ||
!KiMutexWaiterIsLinked(thread)) {
KeReleaseSpinlock(&mutex->Lock, flags);
return 0;
return false;
}
KeListRemoveChecked(&thread->SchedulerLink);
@@ -245,7 +245,7 @@ int KiMutexAbortWait(MUTEX* mutex, THREAD* thread) {
KiMutexPropagatePriority(owner);
if (owner) ObDereferenceObject(&owner->Header);
KiMutexPropagatePriority(thread);
return 1;
return true;
}
// Abandons one mutex owned by a terminating thread.
code/kernel/ke_port.c
+48 -47
View File
@@ -1,5 +1,6 @@
// Kernel Port IPC mechanism. Ports provide copied messaging, synchronous calls, waitability, and capability transfer.
#include <ntlite/atomic.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
@@ -67,17 +68,17 @@ struct PORT_RECEIVE_WAIT {
SYS_HANDLE Handles[SYS_PORT_MAX_TRANSFER_HANDLES];
uint32_t HandleCount;
KSTATUS Status;
volatile long Claimed;
volatile long Completed;
volatile int32_t Claimed;
volatile int32_t Completed;
};
struct PORT_CALL {
LIST_ENTRY Link;
volatile long RefCount;
volatile int32_t RefCount;
PORT_CONNECTION* Connection;
uint32_t MessageId;
uint32_t CallerSide;
volatile long State;
volatile int32_t State;
struct PROCESS* CallerProcess;
EVENT* CompletionEvent;
MM_LOCKED_USER_BUFFER* LockedReplyPayload;
@@ -91,7 +92,7 @@ struct PORT_CALL {
};
struct PORT_CONNECTION {
volatile long RefCount;
volatile int32_t RefCount;
SPINLOCK Lock;
struct PORT* Endpoints[2];
struct PROCESS* PeerProcesses[2];
@@ -133,22 +134,22 @@ struct PORT {
static KE_ID_ALLOCATOR gPortMessageIds = KE_ID_ALLOCATOR_INIT(1u);
typedef struct PORT_COUNTERS {
volatile long long ConnectionsCreated;
volatile long long DirectDeliveries;
volatile long long DirectBytes;
volatile long long QueuedMessages;
volatile long long QueuedBytes;
volatile long long QueuedDeliveries;
volatile long long RepliesCompleted;
volatile long long Disconnects;
volatile long long ProcessQuotaRejects;
volatile int64_t ConnectionsCreated;
volatile int64_t DirectDeliveries;
volatile int64_t DirectBytes;
volatile int64_t QueuedMessages;
volatile int64_t QueuedBytes;
volatile int64_t QueuedDeliveries;
volatile int64_t RepliesCompleted;
volatile int64_t Disconnects;
volatile int64_t ProcessQuotaRejects;
} PORT_COUNTERS;
static PORT_COUNTERS gPortCounters;
// Reads one 64-bit diagnostic counter atomically without changing it.
static uint64_t KiPortReadCounter(volatile long long* counter) {
return (uint64_t)_InterlockedCompareExchange64(counter, 0, 0);
static uint64_t KiPortReadCounter(volatile int64_t* counter) {
return (uint64_t)NtlAtomicLoad64(counter);
}
// Copies current Port transport diagnostics for the kernel stats syscall.
@@ -194,7 +195,7 @@ static KSTATUS KiPortChargeProcess(PROCESS* process, uint32_t payloadBytes, uint
handleCount > PORT_PROCESS_HANDLE_LIMIT - process->PortOutstandingHandles) {
KeReleaseSpinlock(&process->PortQuotaLock, flags);
ObDereferenceObject(&process->Header);
_InterlockedIncrement64(&gPortCounters.ProcessQuotaRejects);
NtlAtomicIncrement64(&gPortCounters.ProcessQuotaRejects);
return NTLSTATUS_LIMIT_REACHED;
}
process->PortOutstandingMessages++;
@@ -236,7 +237,7 @@ static KSTATUS KiPortChargeCall(PROCESS* process) {
if (process->PortOutstandingCalls >= PORT_PROCESS_CALL_LIMIT) {
KeReleaseSpinlock(&process->PortQuotaLock, flags);
ObDereferenceObject(&process->Header);
_InterlockedIncrement64(&gPortCounters.ProcessQuotaRejects);
NtlAtomicIncrement64(&gPortCounters.ProcessQuotaRejects);
return NTLSTATUS_LIMIT_REACHED;
}
process->PortOutstandingCalls++;
@@ -264,7 +265,7 @@ static void KiPortReleaseCallCharge(PROCESS* process) {
static void KiPortDereferenceConnection(PORT_CONNECTION* connection) {
if (!connection)
return;
long refs = _InterlockedDecrement(&connection->RefCount);
int32_t refs = NtlAtomicDecrement32(&connection->RefCount);
if (refs > 0)
return;
if (refs < 0) {
@@ -418,7 +419,7 @@ static KSTATUS KiPortCreateMessage(PROCESS* senderProcess,
message->Info.HandleCount = transferCount;
message->Call = call;
if (call)
_InterlockedIncrement(&call->RefCount);
NtlAtomicIncrement32(&call->RefCount);
KSTATUS status = KiPortCaptureHandles(senderProcess,
transfers,
@@ -529,10 +530,10 @@ static KSTATUS KiPortDeliverDirect(PORT* targetEndpoint,
receiver->Info = message->Info;
receiver->Status = status;
if (KSUCCESS(status)) {
_InterlockedIncrement64(&gPortCounters.DirectDeliveries);
_InterlockedExchangeAdd64(&gPortCounters.DirectBytes, (long long)message->Info.PayloadLength);
NtlAtomicIncrement64(&gPortCounters.DirectDeliveries);
NtlAtomicExchangeAdd64(&gPortCounters.DirectBytes, (int64_t)message->Info.PayloadLength);
}
_InterlockedExchange(&receiver->Completed, 1);
NtlAtomicExchange32(&receiver->Completed, 1);
KeSignalEvent(receiver->CompletionEvent);
return status;
}
@@ -543,7 +544,7 @@ static void KiPortCompleteCallError(PORT_CALL* call, KSTATUS status) {
return;
call->Status = status;
call->ReplyHandleCount = 0;
_InterlockedExchange(&call->State, PORT_CALL_COMPLETED);
NtlAtomicExchange32(&call->State, PORT_CALL_COMPLETED);
KeSignalEvent(call->CompletionEvent);
}
@@ -586,7 +587,7 @@ static void KiPortMarkDisconnected(PORT* endpoint, bool removeEndpointPointer) {
uintptr_t flags = KeAcquireSpinlock(&connection->Lock);
if (!connection->Disconnected[side]) {
connection->Disconnected[side] = 1;
_InterlockedIncrement64(&gPortCounters.Disconnects);
NtlAtomicIncrement64(&gPortCounters.Disconnects);
}
if (removeEndpointPointer && connection->Endpoints[side] == endpoint)
connection->Endpoints[side] = (PORT*)0;
@@ -594,9 +595,9 @@ static void KiPortMarkDisconnected(PORT* endpoint, bool removeEndpointPointer) {
while (!LIST_IS_EMPTY(&endpoint->u.Endpoint.WaitingReceivers)) {
LIST_ENTRY* link = KeListPopHeadChecked(&endpoint->u.Endpoint.WaitingReceivers);
PORT_RECEIVE_WAIT* waiter = CONTAINING_RECORD(link, PORT_RECEIVE_WAIT, Link);
if (_InterlockedCompareExchange(&waiter->Claimed, 1, 0) == 0) {
if (NtlAtomicCompareExchange32(&waiter->Claimed, 1, 0) == 0) {
waiter->Status = NTLSTATUS_PORT_DISCONNECTED;
_InterlockedExchange(&waiter->Completed, 1);
NtlAtomicExchange32(&waiter->Completed, 1);
KeSignalEvent(waiter->CompletionEvent);
}
}
@@ -618,7 +619,7 @@ static void KiPortMarkDisconnected(PORT* endpoint, bool removeEndpointPointer) {
while (callLink != callHead) {
PORT_CALL* call = CONTAINING_RECORD(callLink, PORT_CALL, Link);
callLink = callLink->Fwd;
long state = _InterlockedCompareExchange(&call->State, 0, 0);
int32_t state = NtlAtomicLoad32(&call->State);
if (state == PORT_CALL_REPLYING || state == PORT_CALL_COMPLETED || state == PORT_CALL_CANCELLED)
continue;
PORT_MESSAGE* queuedMessage = KiPortRemoveQueuedCallLocked(connection, call);
@@ -626,7 +627,7 @@ static void KiPortMarkDisconnected(PORT* endpoint, bool removeEndpointPointer) {
KeListInsertTailChecked(&abandonedMessages, &queuedMessage->Link);
KeListRemoveChecked(&call->Link);
call->Status = NTLSTATUS_PORT_DISCONNECTED;
_InterlockedExchange(&call->State, PORT_CALL_COMPLETED);
NtlAtomicExchange32(&call->State, PORT_CALL_COMPLETED);
KeListInsertTailChecked(&completedCalls, &call->Link);
}
@@ -886,7 +887,7 @@ KSTATUS KePortConnect(PORT* listener,
}
KeListInsertTailChecked(&listener->u.Listener.Pending, &pending->Link);
listener->u.Listener.PendingCount++;
_InterlockedIncrement64(&gPortCounters.ConnectionsCreated);
NtlAtomicIncrement64(&gPortCounters.ConnectionsCreated);
KeSignalEvent(listener->u.Listener.WaitEvent);
KeReleaseSpinlock(&listener->u.Listener.Lock, flags);
@@ -999,7 +1000,7 @@ static KSTATUS KiPortSubmitMessage(PORT* endpoint,
if (payloadLength > candidate->PayloadCapacity ||
message->CapturedCount > candidate->HandleCapacity)
continue;
if (_InterlockedCompareExchange(&candidate->Claimed, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&candidate->Claimed, 1, 0) != 0)
continue;
receiver = candidate;
KeListRemoveChecked(&receiver->Link);
@@ -1007,7 +1008,7 @@ static KSTATUS KiPortSubmitMessage(PORT* endpoint,
}
if (receiver) {
if (message->Call)
_InterlockedExchange(&message->Call->State, PORT_CALL_DELIVERED);
NtlAtomicExchange32(&message->Call->State, PORT_CALL_DELIVERED);
if (!ObTryReferenceObject(&peer->Header)) {
receiver->Claimed = 0;
KeListInsertHeadChecked(&peer->u.Endpoint.WaitingReceivers, &receiver->Link);
@@ -1052,10 +1053,10 @@ static KSTATUS KiPortSubmitMessage(PORT* endpoint,
peer->u.Endpoint.IncomingCount++;
peer->u.Endpoint.IncomingBytes += message->Info.PayloadLength;
peer->u.Endpoint.IncomingHandles += message->CapturedCount;
_InterlockedIncrement64(&gPortCounters.QueuedMessages);
_InterlockedExchangeAdd64(&gPortCounters.QueuedBytes, (long long)message->Info.PayloadLength);
NtlAtomicIncrement64(&gPortCounters.QueuedMessages);
NtlAtomicExchangeAdd64(&gPortCounters.QueuedBytes, (int64_t)message->Info.PayloadLength);
if (message->Call)
_InterlockedExchange(&message->Call->State, PORT_CALL_PENDING);
NtlAtomicExchange32(&message->Call->State, PORT_CALL_PENDING);
KeSignalEvent(peer->u.Endpoint.WaitEvent);
if (peer->u.Endpoint.IncomingCount >= SYS_PORT_QUEUE_MESSAGE_LIMIT ||
peer->u.Endpoint.IncomingBytes >= SYS_PORT_QUEUE_BYTE_LIMIT ||
@@ -1160,7 +1161,7 @@ static KSTATUS KiPortDeliverQueued(PORT* endpoint,
PORT_CONNECTION* connection = endpoint->u.Endpoint.Connection;
bool completeCall = false;
uintptr_t flags = KeAcquireSpinlock(&connection->Lock);
long callState = _InterlockedCompareExchange(&call->State, 0, 0);
int32_t callState = NtlAtomicLoad32(&call->State);
if (callState != PORT_CALL_CANCELLED && callState != PORT_CALL_COMPLETED) {
if (call->Link.Fwd != &call->Link && call->Link.Bwd != &call->Link)
KeListRemoveChecked(&call->Link);
@@ -1176,7 +1177,7 @@ static KSTATUS KiPortDeliverQueued(PORT* endpoint,
if (message->CapturedCount)
memcpy(handlesOut, handles, sizeof(SYS_HANDLE) * message->CapturedCount);
*infoOut = message->Info;
_InterlockedIncrement64(&gPortCounters.QueuedDeliveries);
NtlAtomicIncrement64(&gPortCounters.QueuedDeliveries);
return NTLSTATUS_SUCCESS;
}
@@ -1246,7 +1247,7 @@ KSTATUS KePortReceive(PORT* endpoint,
KeListRemoveChecked(&message->Link);
KiPortNoteDequeuedLocked(endpoint, message);
if (message->Call)
_InterlockedExchange(&message->Call->State, PORT_CALL_DELIVERED);
NtlAtomicExchange32(&message->Call->State, PORT_CALL_DELIVERED);
KeReleaseSpinlock(&connection->Lock, flags);
result = KiPortDeliverQueued(endpoint,
@@ -1292,7 +1293,7 @@ KSTATUS KePortReceive(PORT* endpoint,
(uint32_t*)0);
flags = KeAcquireSpinlock(&connection->Lock);
if (_InterlockedCompareExchange(&wait.Completed, 0, 0) != 0) {
if (NtlAtomicLoad32(&wait.Completed) != 0) {
if (wait.Link.Fwd != &wait.Link || wait.Link.Bwd != &wait.Link)
KeListRemoveChecked(&wait.Link);
if (endpoint->u.Endpoint.IncomingCount == 0 &&
@@ -1308,7 +1309,7 @@ KSTATUS KePortReceive(PORT* endpoint,
break;
}
if (_InterlockedCompareExchange(&wait.Claimed, 0, 0) != 0) {
if (NtlAtomicLoad32(&wait.Claimed) != 0) {
KeReleaseSpinlock(&connection->Lock, flags);
EVENT* claimedEvents[1] = { wait.CompletionEvent };
(void)KeWaitForMultipleEvents(claimedEvents,
@@ -1422,7 +1423,7 @@ static KSTATUS KiPortCreateCall(PORT* endpoint,
static void KiPortReleaseCall(PORT_CALL* call) {
if (!call)
return;
long refs = _InterlockedDecrement(&call->RefCount);
int32_t refs = NtlAtomicDecrement32(&call->RefCount);
if (refs > 0)
return;
if (refs < 0) {
@@ -1530,7 +1531,7 @@ KSTATUS KePortCall(PORT* endpoint,
if (!KSUCCESS(waitStatus)) {
PORT_MESSAGE* queuedMessage = (PORT_MESSAGE*)0;
uintptr_t flags = KeAcquireSpinlock(&call->Connection->Lock);
long state = _InterlockedCompareExchange(&call->State, 0, 0);
int32_t state = NtlAtomicLoad32(&call->State);
if (state == PORT_CALL_REPLYING) {
KeReleaseSpinlock(&call->Connection->Lock, flags);
(void)KeWaitForMultipleEvents(events, 1, 0, SYS_PORT_TIMEOUT_INFINITE, (uint32_t*)0);
@@ -1539,7 +1540,7 @@ KSTATUS KePortCall(PORT* endpoint,
} else {
if (call->Link.Fwd != &call->Link && call->Link.Bwd != &call->Link)
KeListRemoveChecked(&call->Link);
_InterlockedExchange(&call->State, PORT_CALL_CANCELLED);
NtlAtomicExchange32(&call->State, PORT_CALL_CANCELLED);
queuedMessage = KiPortRemoveQueuedCallLocked(call->Connection, call);
KeReleaseSpinlock(&call->Connection->Lock, flags);
if (queuedMessage)
@@ -1602,7 +1603,7 @@ KSTATUS KePortReply(PORT* endpoint,
}
}
if (!call || _InterlockedCompareExchange(&call->State, 0, 0) != PORT_CALL_DELIVERED) {
if (!call || NtlAtomicLoad32(&call->State) != PORT_CALL_DELIVERED) {
KeReleaseSpinlock(&connection->Lock, flags);
for (uint32_t i = 0; i < transferCount; ++i)
KeReleaseCapturedHandle(captured[i]);
@@ -1615,7 +1616,7 @@ KSTATUS KePortReply(PORT* endpoint,
return NTLSTATUS_BUFFER_TOO_SMALL;
}
_InterlockedExchange(&call->State, PORT_CALL_REPLYING);
NtlAtomicExchange32(&call->State, PORT_CALL_REPLYING);
KeListRemoveChecked(&call->Link);
KeReleaseSpinlock(&connection->Lock, flags);
@@ -1649,9 +1650,9 @@ KSTATUS KePortReply(PORT* endpoint,
}
if (KSUCCESS(status))
_InterlockedIncrement64(&gPortCounters.RepliesCompleted);
NtlAtomicIncrement64(&gPortCounters.RepliesCompleted);
call->Status = status;
_InterlockedExchange(&call->State, PORT_CALL_COMPLETED);
NtlAtomicExchange32(&call->State, PORT_CALL_COMPLETED);
KeSignalEvent(call->CompletionEvent);
return status;
}
code/kernel/ke_process.c
+27 -19
View File
@@ -1,5 +1,7 @@
// Kernel process model. This file owns process lifetime, thread membership, and address-space metadata.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <string.h>
#include <kernel/ke_idt.h>
#include <kernel/ke_hal.h>
@@ -145,8 +147,8 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
KeReleaseSpinlock(&p->ThreadLock, threadFlags);
uint32_t handleCount = KeGetHandleCount(p);
int32_t exitCode = p->ExitCode;
long terminating = p->Terminating;
long exited = p->Exited;
int32_t terminating = p->Terminating;
int32_t exited = p->Exited;
info->ThreadCount = threadCount;
info->HandleCount = handleCount;
@@ -170,7 +172,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
} else if (terminating) {
info->State = SYS_PROCESS_STATE_TERMINATING;
info->Flags |= SYS_PROCESS_FLAG_TERMINATING;
} else if (_InterlockedCompareExchange(&p->BuildState, 0, 0) != KE_PROCESS_BUILD_STARTED) {
} else if (NtlAtomicLoad32(&p->BuildState) != KE_PROCESS_BUILD_STARTED) {
info->State = SYS_PROCESS_STATE_CONSTRUCTING;
info->Flags |= SYS_PROCESS_FLAG_CONSTRUCTING;
} else {
@@ -298,7 +300,7 @@ KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, s
// both observe a process whose threads are already gone.
static void KiProcessRundownResources(PROCESS* proc) {
if (!proc) return;
if (_InterlockedExchange(&proc->ResourcesReleased, 1) != 0)
if (NtlAtomicExchange32(&proc->ResourcesReleased, 1) != 0)
return;
IoReleaseFileLocksForProcess(proc);
@@ -377,6 +379,9 @@ KSTATUS KeCreateProcess(const char* name, SECURITY_CONTEXT* securityContext, PRO
KiProcessInit();
if (!name || !securityContext || !out) return NTLSTATUS_INVALID_PARAMETER;
size_t nameLength = strlen(name);
if (!SysSvIsUtf8Text(SysSvFromParts(name, nameLength)))
return NTLSTATUS_INVALID_PARAMETER;
uint32_t processId = 0;
KSTATUS idStatus = KeAllocateId32(&gProcessIds, &processId);
@@ -391,8 +396,14 @@ KSTATUS KeCreateProcess(const char* name, SECURITY_CONTEXT* securityContext, PRO
proc->Flags = 0;
proc->SecurityContext = securityContext;
ObReferenceObject(&securityContext->Header);
strncpy(proc->Name, name, PROCESS_NAME_MAX - 1);
proc->Name[PROCESS_NAME_MAX - 1] = '\0';
// Truncate the diagnostic name on a scalar boundary, never inside UTF-8.
size_t nameCopy = nameLength < PROCESS_NAME_MAX ? nameLength : PROCESS_NAME_MAX - 1u;
if (nameCopy < nameLength) {
while (nameCopy && (((uint8_t)name[nameCopy] & 0xC0u) == 0x80u))
--nameCopy;
}
memcpy(proc->Name, name, nameCopy);
proc->Name[nameCopy] = '\0';
LIST_INIT(&proc->ThreadList);
proc->ThreadCount = 0;
@@ -463,8 +474,7 @@ static KSTATUS KiCopyEnvironmentStringView(PROCESS* proc,
*baseOut = 0;
*sizeOut = 0;
if (!SysSvIsValid(src)) return NTLSTATUS_INVALID_PARAMETER;
if (SysSvContainsByte(src, '\0')) return NTLSTATUS_INVALID_PARAMETER;
if (!SysSvIsUtf8Text(src)) return NTLSTATUS_INVALID_PARAMETER;
if (src.Length == (size_t)-1) return NTLSTATUS_BUFFER_TOO_SMALL;
size_t len = src.Length + 1;
@@ -509,9 +519,7 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
SYS_STRING_VIEW imagePath,
SYS_STRING_VIEW commandLine) {
if (!proc || imagePath.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
if (!SysSvIsValid(imagePath) || !SysSvIsValid(commandLine))
return NTLSTATUS_INVALID_PARAMETER;
if (SysSvContainsByte(imagePath, '\0') || SysSvContainsByte(commandLine, '\0'))
if (!SysSvIsUtf8Text(imagePath) || !SysSvIsUtf8Text(commandLine))
return NTLSTATUS_INVALID_PARAMETER;
SYS_STRING_VIEW finalCommandLine = commandLine.Length ? commandLine : imagePath;
@@ -594,19 +602,19 @@ KSTATUS KeSetProcessLoaderData(PROCESS* proc, const void* loaderData) {
// Stops new process-construction operations. Active builders defer final teardown until they leave.
void KeStopProcessConstruction(PROCESS* process) {
if (!process) return;
_InterlockedExchange(&process->BuildState, KE_PROCESS_BUILD_DEAD);
NtlAtomicExchange32(&process->BuildState, KE_PROCESS_BUILD_DEAD);
}
// Process exit / termination.
void KeCompleteProcessExitIfLastThread(PROCESS* process) {
if (!process) return;
if (_InterlockedCompareExchange(&process->BuildOperations, 0, 0) != 0) return;
if (NtlAtomicLoad32(&process->BuildOperations) != 0) return;
uintptr_t flags = KeAcquireSpinlock(&process->ThreadLock);
uint32_t threadCount = process->ThreadCount;
KeReleaseSpinlock(&process->ThreadLock, flags);
if (threadCount == 0 && _InterlockedExchange(&process->Exited, 1) == 0) {
if (threadCount == 0 && NtlAtomicExchange32(&process->Exited, 1) == 0) {
KiProcessRundownResources(process);
if (process->ExitEvent) KeSignalEvent(process->ExitEvent);
}
@@ -624,9 +632,9 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
if (KeIsProcessTerminationProtected(process)) return NTLSTATUS_ACCESS_DENIED;
process->ExitCode = exitCode;
_InterlockedExchange(&process->Terminating, 1);
NtlAtomicExchange32(&process->Terminating, 1);
KeStopProcessConstruction(process);
if (_InterlockedCompareExchange(&process->BuildOperations, 0, 0) != 0)
if (NtlAtomicLoad32(&process->BuildOperations) != 0)
return NTLSTATUS_SUCCESS;
uintptr_t flags = KeAcquireSpinlock(&process->ThreadLock);
@@ -655,7 +663,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
KCPU* cpu = &gCpus[t->RunningCpu];
if (cpu->State == KeCpuOnline) {
_InterlockedExchange(&cpu->ReschedulePending, 1);
NtlAtomicExchange32(&cpu->ReschedulePending, 1);
KeSendProcessorInterrupt(cpu->HardwareId, KE_VECTOR_RESCHEDULE_IPI);
}
}
@@ -667,12 +675,12 @@ KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
}
// Checks whether process Terminating.
int KeIsProcessTerminating(PROCESS* process) {
bool KeIsProcessTerminating(PROCESS* process) {
return process && process->Terminating != 0;
}
// Returns whether process termination is forbidden.
int KeIsProcessTerminationProtected(PROCESS* process) {
bool KeIsProcessTerminationProtected(PROCESS* process) {
return process && (process->Flags & KE_PROCESS_FLAG_TERMINATION_PROTECTED) != 0;
}
code/kernel/ke_sched.c
+52 -53
View File
@@ -1,5 +1,6 @@
// Kernel scheduler. This file owns run queues, dispatch decisions, preemption, and CPU-local scheduling state.
#include <ntlite/atomic.h>
#include <stddef.h>
#include <stdbool.h>
#include <intrin.h>
@@ -15,7 +16,6 @@
#include <kernel/ke_irql.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_input.h>
#include <kernel/ke_timer.h>
#include <kernel/ke_display.h>
#include <kernel/ke_list.h>
@@ -35,7 +35,7 @@ static KE_SCHED_STATS gSchedStats;
static PROCESS* gIdleProcess;
// Tests whether one scheduling rank outranks another.
static int KiRankOutranks(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority,
static bool KiRankOutranks(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority,
KE_SCHEDULING_CLASS rightClass, uint32_t rightPriority) {
if (leftClass != rightClass)
return leftClass == KeSchedulingRealtime;
@@ -43,15 +43,15 @@ static int KiRankOutranks(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority,
}
// Tests whether two scheduling ranks are equal.
static int KiRanksEqual(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority,
static bool KiRanksEqual(KE_SCHEDULING_CLASS leftClass, uint32_t leftPriority,
KE_SCHEDULING_CLASS rightClass, uint32_t rightPriority) {
return leftClass == rightClass && leftPriority == rightPriority;
}
// Compares two threads by effective scheduling rank.
static int KiSchedulerThreadOutranks(const THREAD* left, const THREAD* right) {
if (!left) return 0;
if (!right) return 1;
static bool KiSchedulerThreadOutranks(const THREAD* left, const THREAD* right) {
if (!left) return false;
if (!right) return true;
return KiRankOutranks(left->EffectiveSchedulingClass, left->EffectivePriority,
right->EffectiveSchedulingClass, right->EffectivePriority);
}
@@ -97,8 +97,8 @@ THREAD* KiSchedulerSelectMutexWaiter(MUTEX* mutex) {
}
// Refreshes one mutex donation while the caller holds the mutex lock.
int KiSchedulerRefreshMutexDonation(MUTEX* mutex) {
if (!mutex) return 0;
bool KiSchedulerRefreshMutexDonation(MUTEX* mutex) {
if (!mutex) return false;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
THREAD* highest = (THREAD*)0;
@@ -122,7 +122,7 @@ int KiSchedulerRefreshMutexDonation(MUTEX* mutex) {
mutex->HasDonation = 0;
}
int changed = oldHasDonation != mutex->HasDonation ||
bool changed = oldHasDonation != mutex->HasDonation ||
oldClass != mutex->DonationClass ||
oldPriority != mutex->DonationPriority;
KeReleaseSpinlock(&gSchedLock, flags);
@@ -138,32 +138,32 @@ static uint64_t KiDefaultThreadAffinityMask(void) {
}
// Implements the thread Is Killed Locked operation.
static int KiThreadIsKilledLocked(const THREAD* thread) {
static bool KiThreadIsKilledLocked(const THREAD* thread) {
return thread && !thread->IsIdleThread && thread->Owner &&
KeIsProcessTerminating(thread->Owner) &&
!KeThreadTerminationDeferred(thread);
}
// Implements the thread Can Run On Cpu Locked operation.
static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
if (!thread || !cpu) return 0;
static bool KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
if (!thread || !cpu) return false;
if (thread->IsIdleThread) return thread == cpu->IdleThread;
if (cpu->CpuIndex >= 64) return 0;
if (cpu->CpuIndex >= 64) return false;
return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
}
// Implements the cpu Is Idle Locked operation.
static int KiCpuIsIdleLocked(const KCPU* cpu) {
static bool KiCpuIsIdleLocked(const KCPU* cpu) {
if (!cpu || cpu->State != KeCpuOnline)
return 0;
return false;
THREAD* cur = cpu->CurrentThread;
return (!cur || cur == cpu->IdleThread || cur->State != ThreadRunning);
}
// Implements the any Idle Primary For Thread Locked operation.
static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
static bool KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
if (!thread || thread->IsIdleThread)
return 0;
return false;
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
@@ -174,17 +174,17 @@ static int KiAnyIdlePrimaryForThreadLocked(const THREAD* thread) {
if (!KiCpuIsIdleLocked(cpu))
continue;
if (KiThreadCanRunOnCpuLocked(thread, cpu))
return 1;
return true;
}
return 0;
return false;
}
// Checks whether defer To Idle Primary Locked.
static int KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
static bool KiShouldDeferToIdlePrimaryLocked(KCPU* cpu, const THREAD* thread) {
if (!cpu || !thread || thread->IsIdleThread)
return 0;
return false;
if (cpu->IsPrimaryThread || cpu->SmtThreadCount <= 1)
return 0;
return false;
// Do not fill a secondary SMT sibling while any primary hardware thread
// that can run this thread is idle. KiPokeRunnableCpusLocked targets those
@@ -198,7 +198,7 @@ static void KiRequestRescheduleCpuLocked(KCPU* target) {
if (!target || target->State != KeCpuOnline)
return;
_InterlockedExchange(&target->ReschedulePending, 1);
NtlAtomicExchange32(&target->ReschedulePending, 1);
KCPU* self = KeGetCurrentCpu();
if (self && target->CpuIndex != self->CpuIndex) {
@@ -389,8 +389,8 @@ static void KiInsertReadyLocked(THREAD* thread) {
}
// Tests whether any ready thread outranks one running thread on a CPU.
static int KiAnyRunnableOutrankingThreadForCpuLocked(KCPU* cpu, const THREAD* current) {
if (!cpu || !current) return 0;
static bool KiAnyRunnableOutrankingThreadForCpuLocked(KCPU* cpu, const THREAD* current) {
if (!cpu || !current) return false;
for (int p = (int)KE_REALTIME_PRIORITY_MAX; p >= 0; p--) {
LIST_ENTRY* head = &gRealtimeRunQueue[p];
@@ -399,7 +399,7 @@ static int KiAnyRunnableOutrankingThreadForCpuLocked(KCPU* cpu, const THREAD* cu
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
if (KiSchedulerThreadOutranks(thread, current))
return 1;
return true;
}
}
@@ -410,16 +410,16 @@ static int KiAnyRunnableOutrankingThreadForCpuLocked(KCPU* cpu, const THREAD* cu
if (!KiThreadCanRunOnCpuLocked(thread, cpu))
continue;
if (KiSchedulerThreadOutranks(thread, current))
return 1;
return true;
}
}
return 0;
return false;
}
// Tests whether a real-time peer at the current or higher priority is ready.
static int KiAnyRealtimePeerForCpuLocked(KCPU* cpu, uint32_t minimumPriority) {
if (!cpu) return 0;
static bool KiAnyRealtimePeerForCpuLocked(KCPU* cpu, uint32_t minimumPriority) {
if (!cpu) return false;
if (minimumPriority > KE_REALTIME_PRIORITY_MAX)
minimumPriority = KE_REALTIME_PRIORITY_MAX;
@@ -428,10 +428,10 @@ static int KiAnyRealtimePeerForCpuLocked(KCPU* cpu, uint32_t minimumPriority) {
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (KiThreadCanRunOnCpuLocked(thread, cpu))
return 1;
return true;
}
}
return 0;
return false;
}
// Computes one thread's effective rank from its base rank and mutex donations.
@@ -458,13 +458,13 @@ static void KiComputeEffectiveRankLocked(THREAD* thread,
}
// Applies one effective-rank change and preserves ready-queue ordering.
static int KiApplyEffectiveRankLocked(THREAD* thread,
static bool KiApplyEffectiveRankLocked(THREAD* thread,
KE_SCHEDULING_CLASS schedulingClass,
uint32_t priority) {
KE_SCHEDULING_CLASS oldClass = thread->EffectiveSchedulingClass;
uint32_t oldPriority = thread->EffectivePriority;
if (KiRanksEqual(oldClass, oldPriority, schedulingClass, priority))
return 0;
return false;
int wasQueued = thread->Queued != 0;
if (wasQueued) {
@@ -485,12 +485,12 @@ static int KiApplyEffectiveRankLocked(THREAD* thread,
KiRequestRescheduleCpuLocked(cpu);
}
return 1;
return true;
}
// Recomputes effective rank from current base rank and owned mutexes.
int KiSchedulerRefreshEffectivePriority(THREAD* thread, MUTEX** waitingMutexOut) {
if (!thread) return 0;
bool KiSchedulerRefreshEffectivePriority(THREAD* thread, MUTEX** waitingMutexOut) {
if (!thread) return false;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KE_SCHEDULING_CLASS oldClass = thread->EffectiveSchedulingClass;
@@ -498,7 +498,7 @@ int KiSchedulerRefreshEffectivePriority(THREAD* thread, MUTEX** waitingMutexOut)
KE_SCHEDULING_CLASS schedulingClass;
uint32_t priority;
KiComputeEffectiveRankLocked(thread, &schedulingClass, &priority);
int changed = KiApplyEffectiveRankLocked(thread, schedulingClass, priority);
bool changed = KiApplyEffectiveRankLocked(thread, schedulingClass, priority);
if (changed) {
if (KiRankOutranks(schedulingClass, priority, oldClass, oldPriority))
gSchedStats.PriorityInheritanceBoosts++;
@@ -692,7 +692,7 @@ KSTATUS KiSchedulerSetNormalPriority(THREAD* thread, THREAD_PRIORITY priority) {
return NTLSTATUS_SUCCESS;
}
static int KiAnyRunnableForCpuLocked(KCPU* cpu);
static bool KiAnyRunnableForCpuLocked(KCPU* cpu);
// Implements the any Runnable For Current Cpu operation.
static int KiAnyRunnableForCurrentCpu(void) {
@@ -856,7 +856,7 @@ void KeInitializeScheduler(void) {
KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create CPU%u idle thread, status=%u\n",
(unsigned)cpu->CpuIndex, s);
_InterlockedExchange(&cpu->State, KeCpuOnline);
NtlAtomicExchange32(&cpu->State, KeCpuOnline);
}
// Safely release the lock and clean up any dead threads natively post-switch.
@@ -980,7 +980,7 @@ void KeUnreadyThread(THREAD* thread) {
}
// Implements the any Runnable For Cpu Locked operation.
static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
static bool KiAnyRunnableForCpuLocked(KCPU* cpu) {
if (!cpu) cpu = KeGetCurrentCpu();
for (uint32_t p = 0; p < KE_REALTIME_PRIORITY_COUNT; p++) {
@@ -991,7 +991,7 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
continue;
if (KiShouldDeferToIdlePrimaryLocked(cpu, thread))
continue;
return 1;
return true;
}
}
@@ -1003,10 +1003,10 @@ static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
continue;
if (KiShouldDeferToIdlePrimaryLocked(cpu, thread))
continue;
return 1;
return true;
}
}
return 0;
return false;
}
// Removes the first eligible thread from one ready queue.
@@ -1053,13 +1053,13 @@ static THREAD* KiPickNextForCpu(KCPU* cpu) {
void KiRequestReschedule(void) {
KCPU* cpu = KeGetCurrentCpu();
if (cpu)
_InterlockedExchange(&cpu->ReschedulePending, 1);
NtlAtomicExchange32(&cpu->ReschedulePending, 1);
}
// Checks whether reschedule Pending.
int KiIsReschedulePending(void) {
bool KiIsReschedulePending(void) {
KCPU* cpu = KeGetCurrentCpu();
return cpu && _InterlockedCompareExchange(&cpu->ReschedulePending, 0, 0) != 0;
return cpu && NtlAtomicLoad32(&cpu->ReschedulePending) != 0;
}
// Implements the schedule operation.
@@ -1069,7 +1069,7 @@ static void KiSchedule(void) {
THREAD* idle = cpu->IdleThread;
if (!cur) return;
_InterlockedExchange(&cpu->ReschedulePending, 0);
NtlAtomicExchange32(&cpu->ReschedulePending, 0);
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiArmPostSwitchUnlock(cur, flags);
@@ -1296,14 +1296,14 @@ static THREAD* KiTakeExpiredTimeoutLocked(uint64_t nowTick, KSTATUS* statusOut)
}
// Reports whether one deadline is ready while the scheduler lock is held.
static int KiHasExpiredTimeoutLocked(uint64_t nowTick) {
static bool KiHasExpiredTimeoutLocked(uint64_t nowTick) {
LIST_ENTRY* head = &gTimeoutList;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, TimeoutLink);
if (thread->TimeoutTick <= nowTick)
return 1;
return true;
}
return 0;
return false;
}
// Resolves expired deadlines at passive level through each object's abort path.
@@ -1326,7 +1326,7 @@ static void KiTimeoutWorkRoutine(void* context) {
}
// Checks whether this CPU owns global scheduler tick duties.
static int KiCpuOwnsGlobalSchedulerTick(const KCPU* cpu) {
static bool KiCpuOwnsGlobalSchedulerTick(const KCPU* cpu) {
// Periodic timer interrupts are per-CPU, but sleep deadlines, waitable timers,
// input repeat, and priority aging are global scheduler services. Keep the
// global clock single-sourced on the boot processor so enabling AP timers does not
@@ -1367,7 +1367,6 @@ static void KiAgeReadyThreadsLocked(void) {
// Runs scheduler work that must be single-sourced for the whole system.
static int KiRunGlobalSchedulerTickLocked(void) {
gSchedStats.Ticks++;
KeInputTimerTick();
KiAgeReadyThreadsLocked();
return KiHasExpiredTimeoutLocked(gSchedStats.Ticks);
}
code/kernel/ke_settings.c
+4 -3
View File
@@ -1,5 +1,6 @@
// Kernel settings reader. This file performs a direct lookup in the simple system settings file.
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
@@ -12,7 +13,7 @@
#define KE_SETTINGS_MAX_FILE_BYTES (64u * 1024u)
// Returns whether one settings.ini character is horizontal whitespace.
static int KeSettingSpace(char c) {
static bool KeSettingSpace(char c) {
return c == ' ' || c == '\t' || c == '\r';
}
@@ -25,10 +26,10 @@ static void KeSettingTrim(const char** begin, const char** end) {
}
// Returns whether one immutable span exactly matches a NUL-terminated string.
static int KeSettingSpanEquals(const char* begin, const char* end, const char* text) {
static bool KeSettingSpanEquals(const char* begin, const char* end, const char* text) {
while (begin < end && *text) {
if (*begin++ != *text++)
return 0;
return false;
}
return begin == end && *text == '\0';
}
code/kernel/ke_smp.c
+53 -52
View File
@@ -4,6 +4,7 @@
// The kernel asks the HAL for platform processor identities, prepares the x86
// bootstrap state, asks the HAL to start each processor, and waits for it to
// enter KiApKernelEntry() and park in kernel-owned state.
#include <ntlite/atomic.h>
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
@@ -323,12 +324,12 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
KeLoadIdtForCurrentCpu();
if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_IDT;
_InterlockedExchange(&cpu->State, KeCpuStarted);
NtlAtomicExchange32(&cpu->State, KeCpuStarted);
cpu->CurrentIrql = PASSIVE_LEVEL;
cpu->InterruptDepth = 0;
cpu->ReschedulePending = 0;
_InterlockedExchange(&cpu->State, KeCpuParked);
NtlAtomicExchange32(&cpu->State, KeCpuParked);
if (gApBootBlock) {
gApBootBlock->Stage = AP_STAGE_PARK;
gApBootBlock->State = AP_STATE_KERNEL_PARKED;
@@ -337,7 +338,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
while (true) {
cpu->SchedulerTicks++;
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
if (NtlAtomicLoad32(&cpu->State) == KeCpuAdmissionRequested) {
THREAD* idle = cpu->IdleThread;
if (idle) {
cpu->CurrentThread = idle;
@@ -349,7 +350,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
// a current idle thread before they may be delivered.
KeConfigureCurrentProcessorTimer();
_InterlockedExchange(&cpu->State, KeCpuSchedulerActive);
NtlAtomicExchange32(&cpu->State, KeCpuSchedulerActive);
KeEnableInterrupts();
KeStartInitialThread(idle);
}
@@ -360,7 +361,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
// changes between the poll and HLT: check once with interrupts
// disabled, then use STI;HLT as a single sleep transition.
uintptr_t interruptFlags = KeDisableInterrupts();
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
if (NtlAtomicLoad32(&cpu->State) == KeCpuAdmissionRequested) {
KeRestoreInterrupts(interruptFlags);
continue;
}
@@ -742,7 +743,7 @@ KSTATUS KeStartApplicationProcessors(void) {
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u hardware=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
i, cpu->HardwareId, st);
_InterlockedExchange(&cpu->State, KeCpuAbsent);
NtlAtomicExchange32(&cpu->State, KeCpuAbsent);
continue;
}
@@ -755,7 +756,7 @@ KSTATUS KeStartApplicationProcessors(void) {
if (!KSUCCESS(st)) {
DbgPrint("SMP: AP%u hardware=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
i, cpu->HardwareId, st);
_InterlockedExchange(&cpu->State, KeCpuAbsent);
NtlAtomicExchange32(&cpu->State, KeCpuAbsent);
continue;
}
@@ -777,11 +778,11 @@ KSTATUS KeStartApplicationProcessors(void) {
DbgPrint("SMP: AP%u failed to reach kernel park state hardware=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
i, cpu->HardwareId, gApBootBlock->State, gApBootBlock->Stage,
KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
_InterlockedExchange(&cpu->State, KeCpuAbsent);
NtlAtomicExchange32(&cpu->State, KeCpuAbsent);
continue;
}
if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
if (NtlAtomicLoad32(&cpu->State) != KeCpuParked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u boot block parked without CPU state transition state=%ld\n",
i, cpu->State);
@@ -812,7 +813,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
KCPU* cpu = &gCpus[i];
if (cpu->State == KeCpuAbsent) continue;
if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
if (NtlAtomicLoad32(&cpu->State) != KeCpuParked) {
KePanic(KE_PANIC_CODE_INIT,
"SMP: AP%u cannot enter scheduler from state=%ld\n",
i, cpu->State);
@@ -824,7 +825,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
return st;
KiSmpPoolCheckpoint("smp after AP idle create", i);
_InterlockedExchange(&cpu->State, KeCpuAdmissionRequested);
NtlAtomicExchange32(&cpu->State, KeCpuAdmissionRequested);
KiSmpPoolCheckpoint("smp before AP wake", i);
// Parked secondary processors sleep with no periodic timer armed yet. The
@@ -834,11 +835,11 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
// the AP has installed its idle thread.
KeSendProcessorInterrupt(cpu->HardwareId, KE_VECTOR_RESCHEDULE_IPI);
int active = 0;
bool active = false;
uint64_t schedulerDeadline = KiSmpDeadlineAfter(AP_SCHED_TIMEOUT_US);
for (;;) {
if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuSchedulerActive) {
active = 1;
if (NtlAtomicLoad32(&cpu->State) == KeCpuSchedulerActive) {
active = true;
break;
}
if (KiSmpQueryTicks() >= schedulerDeadline)
@@ -852,7 +853,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
i, cpu->HardwareId, cpu->State, cpu->SchedulerTicks);
}
_InterlockedExchange(&cpu->State, KeCpuOnline);
NtlAtomicExchange32(&cpu->State, KeCpuOnline);
KiSmpPoolCheckpoint("smp after AP scheduler active", i);
admitted++;
if (KiSmpDiagnosticsEnabled())
@@ -875,11 +876,11 @@ typedef struct SMP_AP_WORKER_ARG {
uint32_t WorkerIndex;
struct SMP_AP_WORK_TEST* Test;
EVENT* StepEvent;
volatile long Ready;
volatile long WaitArmed;
volatile long Progress;
volatile long Done;
volatile long Failure;
volatile int32_t Ready;
volatile int32_t WaitArmed;
volatile int32_t Progress;
volatile int32_t Done;
volatile int32_t Failure;
uint32_t WrongCpu;
uint32_t WrongRunningCpu;
} SMP_AP_WORKER_ARG;
@@ -887,10 +888,10 @@ typedef struct SMP_AP_WORKER_ARG {
struct SMP_AP_WORK_TEST {
uint64_t Magic;
uint32_t WorkerCount;
volatile long ReadyCount;
volatile long DoneCount;
volatile long FailureCount;
volatile long CpuHits[KE_MAX_CPUS];
volatile int32_t ReadyCount;
volatile int32_t DoneCount;
volatile int32_t FailureCount;
volatile int32_t CpuHits[KE_MAX_CPUS];
SMP_AP_WORKER_ARG Workers[KE_MAX_CPUS];
};
@@ -901,14 +902,14 @@ static void KiSmpApControlledWorker(void* context) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%016llX\n", (uint64_t)arg);
}
_InterlockedExchange(&arg->Ready, 1);
_InterlockedIncrement(&arg->Test->ReadyCount);
NtlAtomicExchange32(&arg->Ready, 1);
NtlAtomicIncrement32(&arg->Test->ReadyCount);
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
_InterlockedIncrement(&arg->WaitArmed);
NtlAtomicIncrement32(&arg->WaitArmed);
KSTATUS waitStatus = KeWaitForEvent(arg->StepEvent, 0);
if (!KSUCCESS(waitStatus)) {
_InterlockedExchange(&arg->Failure, 1);
NtlAtomicExchange32(&arg->Failure, 1);
break;
}
@@ -916,49 +917,49 @@ static void KiSmpApControlledWorker(void* context) {
THREAD* self = KeGetCurrentThread();
if (!cpu || cpu->CpuIndex != arg->CpuIndex) {
arg->WrongCpu = cpu ? cpu->CpuIndex : KE_INVALID_CPU;
_InterlockedExchange(&arg->Failure, 1);
NtlAtomicExchange32(&arg->Failure, 1);
break;
}
if (!self || self->RunningCpu != cpu->CpuIndex) {
arg->WrongRunningCpu = self ? self->RunningCpu : KE_INVALID_CPU;
_InterlockedExchange(&arg->Failure, 1);
NtlAtomicExchange32(&arg->Failure, 1);
break;
}
if (self && (self->AffinityMask & (1ULL << cpu->CpuIndex)) == 0) {
arg->WrongCpu = cpu->CpuIndex;
_InterlockedExchange(&arg->Failure, 1);
NtlAtomicExchange32(&arg->Failure, 1);
break;
}
if (cpu->CpuIndex < KE_MAX_CPUS)
_InterlockedIncrement(&arg->Test->CpuHits[cpu->CpuIndex]);
NtlAtomicIncrement32(&arg->Test->CpuHits[cpu->CpuIndex]);
_InterlockedIncrement(&arg->Progress);
NtlAtomicIncrement32(&arg->Progress);
if ((round & 3u) == 0)
KeYield();
}
if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) {
_InterlockedIncrement(&arg->Test->FailureCount);
if (NtlAtomicLoad32(&arg->Failure) != 0) {
NtlAtomicIncrement32(&arg->Test->FailureCount);
}
_InterlockedExchange(&arg->Done, 1);
_InterlockedIncrement(&arg->Test->DoneCount);
NtlAtomicExchange32(&arg->Done, 1);
NtlAtomicIncrement32(&arg->Test->DoneCount);
}
// Waits for for Worker Counter.
static int KiWaitForWorkerCounter(volatile long* counter, long expected, const char* what) {
static bool KiWaitForWorkerCounter(volatile int32_t* counter, int32_t expected, const char* what) {
uint64_t deadline = KiSmpDeadlineAfter(SMP_AP_WORKER_TIMEOUT_US);
for (;;) {
if (_InterlockedCompareExchange(counter, 0, 0) >= expected)
return 1;
if (NtlAtomicLoad32(counter) >= expected)
return true;
if (KiSmpQueryTicks() >= deadline)
break;
KeYield();
}
DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%ld got=%ld\n",
what ? what : "counter", expected, _InterlockedCompareExchange(counter, 0, 0));
return 0;
DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%d got=%ld\n",
what ? what : "counter", expected, NtlAtomicLoad32(counter));
return false;
}
// Runs application Processor Scheduler Self Tests.
@@ -1019,7 +1020,7 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
return NTLSTATUS_SUCCESS;
}
if (!KiWaitForWorkerCounter(&test->ReadyCount, (long)workerCount, "AP worker ready")) {
if (!KiWaitForWorkerCounter(&test->ReadyCount, (int32_t)workerCount, "AP worker ready")) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers failed to become ready ready=%ld workers=%u\n",
test->ReadyCount, workerCount);
}
@@ -1027,7 +1028,7 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
for (uint32_t w = 0; w < workerCount; w++) {
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
if (!KiWaitForWorkerCounter(&arg->WaitArmed, (long)(round + 1), "AP worker wait-armed")) {
if (!KiWaitForWorkerCounter(&arg->WaitArmed, (int32_t)(round + 1), "AP worker wait-armed")) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u did not arm wait for round %u armed=%ld progress=%ld\n",
arg->CpuIndex, round, arg->WaitArmed, arg->Progress);
@@ -1037,13 +1038,13 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
for (uint32_t w = 0; w < workerCount; w++) {
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
if (!KiWaitForWorkerCounter(&arg->Progress, (long)(round + 1), "AP worker progress")) {
if (!KiWaitForWorkerCounter(&arg->Progress, (int32_t)(round + 1), "AP worker progress")) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u no progress round=%u progress=%ld failure=%ld wrongCpu=%u wrongRunningCpu=%u\n",
arg->CpuIndex, round, arg->Progress, arg->Failure,
arg->WrongCpu, arg->WrongRunningCpu);
}
if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) {
if (NtlAtomicLoad32(&arg->Failure) != 0) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u worker failed round=%u wrongCpu=%u wrongRunningCpu=%u progress=%ld\n",
arg->CpuIndex, round, arg->WrongCpu, arg->WrongRunningCpu, arg->Progress);
@@ -1051,19 +1052,19 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
}
}
if (!KiWaitForWorkerCounter(&test->DoneCount, (long)workerCount, "AP worker done")) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%ld workers=%u\n",
if (!KiWaitForWorkerCounter(&test->DoneCount, (int32_t)workerCount, "AP worker done")) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%d workers=%u\n",
test->DoneCount, workerCount);
}
if (_InterlockedCompareExchange(&test->FailureCount, 0, 0) != 0) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%ld\n",
if (NtlAtomicLoad32(&test->FailureCount) != 0) {
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%d\n",
test->FailureCount);
}
for (uint32_t w = 0; w < workerCount; w++) {
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
long hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0;
int32_t hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0;
if (hits <= 0) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u recorded no CPU hits\n", arg->CpuIndex);
code/kernel/ke_surface.c
+12 -12
View File
@@ -23,8 +23,8 @@ typedef struct KE_SURFACE {
uint32_t Flags;
size_t PayloadSize;
size_t PageCount;
uint8_t StatsRegistered;
uint8_t LastDamageValid;
bool StatsRegistered;
bool LastDamageValid;
uint16_t Reserved;
uint32_t Reserved2;
uint64_t MapCount;
@@ -45,10 +45,10 @@ static SPINLOCK gSurfaceStatsLock = SPINLOCK_INIT;
static SYS_SURFACE_STATS gSurfaceStats;
// Multiplies two sizes while rejecting overflow.
static int KiSurfaceMulSize(size_t a, size_t b, size_t* out) {
if (!out || (a != 0 && b > SIZE_MAX / a)) return 0;
static bool KiSurfaceMulSize(size_t a, size_t b, size_t* out) {
if (!out || (a != 0 && b > SIZE_MAX / a)) return false;
*out = a * b;
return 1;
return true;
}
// Aligns one size upward while rejecting wraparound.
@@ -73,7 +73,7 @@ static void KiSurfaceStatsRegister(KE_SURFACE* surface) {
gSurfaceStats.LiveSurfaces++;
gSurfaceStats.LivePages += surface->PageCount;
gSurfaceStats.LiveBytes += surface->PayloadSize;
surface->StatsRegistered = 1;
surface->StatsRegistered = true;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
@@ -87,7 +87,7 @@ static void KiSurfaceStatsUnregister(KE_SURFACE* surface) {
? gSurfaceStats.LivePages - surface->PageCount : 0;
gSurfaceStats.LiveBytes = gSurfaceStats.LiveBytes >= surface->PayloadSize
? gSurfaceStats.LiveBytes - surface->PayloadSize : 0;
surface->StatsRegistered = 0;
surface->StatsRegistered = false;
KeReleaseSpinlock(&gSurfaceStatsLock, irq);
}
@@ -145,8 +145,8 @@ static void KiSurfaceDestroy(OBJECT* object) {
}
// Clips one rectangle to the surface extent.
static int KiSurfaceClipRect(const KE_SURFACE* surface, const DISPLAY_RECT* input, DISPLAY_RECT* output) {
if (!surface || !output) return 0;
static bool KiSurfaceClipRect(const KE_SURFACE* surface, const DISPLAY_RECT* input, DISPLAY_RECT* output) {
if (!surface || !output) return false;
DISPLAY_RECT r;
if (input) {
r = *input;
@@ -156,7 +156,7 @@ static int KiSurfaceClipRect(const KE_SURFACE* surface, const DISPLAY_RECT* inpu
r.W = (int32_t)surface->Width;
r.H = (int32_t)surface->Height;
}
if (r.W <= 0 || r.H <= 0) return 0;
if (r.W <= 0 || r.H <= 0) return false;
int64_t x0 = r.X;
int64_t y0 = r.Y;
@@ -166,13 +166,13 @@ static int KiSurfaceClipRect(const KE_SURFACE* surface, const DISPLAY_RECT* inpu
if (y0 < 0) y0 = 0;
if (x1 > surface->Width) x1 = surface->Width;
if (y1 > surface->Height) y1 = surface->Height;
if (x0 >= x1 || y0 >= y1) return 0;
if (x0 >= x1 || y0 >= y1) return false;
output->X = (int32_t)x0;
output->Y = (int32_t)y0;
output->W = (int32_t)(x1 - x0);
output->H = (int32_t)(y1 - y0);
return 1;
return true;
}
// Expands one damage bound to include another clipped rectangle.
code/kernel/ke_syscall.c
+34 -563
View File
@@ -24,6 +24,8 @@
// call: the object model already handled it; the dedicated syscall was only
// adding a footgun where callers could bypass the unified refcount path.
#include <ntlite/atomic.h>
#include <intrin.h>
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
@@ -43,7 +45,6 @@
#include <kernel/ke_spinlock.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_exception.h>
#include <kernel/ke_named_queue.h>
#ifndef PROCESS_LAUNCH_DEBUG
#define PROCESS_LAUNCH_DEBUG 0
@@ -83,7 +84,7 @@ static const HAL_API* gHal;
typedef struct SYS_IO_STATUS_BLOCK_K {
volatile uint32_t Status;
volatile KSTATUS Status;
volatile uint64_t Information;
} SYS_IO_STATUS_BLOCK_K;
@@ -259,7 +260,7 @@ static KSTATUS KiImportUserTextView(SYS_STRING_VIEW userView,
bool nulTerminate) {
KSTATUS s = KiImportUserStringView(userView, dst, dstSize, out, nulTerminate);
if (!KSUCCESS(s)) return s;
if (out && SysSvContainsByte(*out, '\0')) return NTLSTATUS_INVALID_PARAMETER;
if (!out || !SysSvIsUtf8Text(*out)) return NTLSTATUS_INVALID_PARAMETER;
return NTLSTATUS_SUCCESS;
}
@@ -300,11 +301,11 @@ static KSTATUS KiCopyInt32ToUser(int32_t* valueOut, int32_t value) {
}
// Implements the mul Size operation.
static int KiMulSize(size_t a, size_t b, size_t* out) {
if (!out) return 0;
if (a && b > ((size_t)-1) / a) return 0;
static bool KiMulSize(size_t a, size_t b, size_t* out) {
if (!out) return false;
if (a && b > ((size_t)-1) / a) return false;
*out = a * b;
return 1;
return true;
}
static size_t KiAlignUpSize(size_t value, size_t alignment) {
@@ -554,9 +555,6 @@ static KSTATUS KiHandleToWaitEvent(PROCESS* proc, SYS_HANDLE handle, EVENT** eve
case ObjectTypeAsyncIo:
event = IoGetAsyncRequestWaitEvent((IO_ASYNC_REQUEST_OBJECT*)obj);
break;
case ObjectTypeInputQueue:
event = KeInputQueueWaitEvent(CONTAINING_RECORD(obj, INPUT_QUEUE, Header));
break;
case ObjectTypeTimer:
event = KeTimerWaitEvent(CONTAINING_RECORD(obj, KTIMER, Header));
break;
@@ -698,29 +696,6 @@ static uint64_t SyscallWaitMany(const SYS_WAIT_MANY_REQUEST* request) {
return s;
}
// Implements SYS_OPEN_INPUT_QUEUE.
static uint64_t SyscallOpenInputQueue(uint32_t flags, SYS_HANDLE* handleOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (flags != 0) return NTLSTATUS_INVALID_PARAMETER;
if (!handleOut) return NTLSTATUS_INVALID_PARAMETER;
INPUT_QUEUE* queue = (INPUT_QUEUE*)0;
KSTATUS s = KeInputOpenQueue(&queue);
if (!KSUCCESS(s)) return s;
SYS_HANDLE handle = SYS_HANDLE_INVALID;
s = KiAllocHandleForCurrentWithRights(&queue->Header, KeGetValidHandleRights(ObjectTypeInputQueue), &handle);
if (!KSUCCESS(s)) return s;
s = KiCopyHandleToUser(handleOut, handle);
if (!KSUCCESS(s)) {
PROCESS* proc = KeGetCurrentProcess();
if (proc) KeCloseHandle(proc, handle);
}
return s;
}
// Returns the current thread ID.
static uint64_t SyscallGetCurrentTid(void) {
THREAD* t = KeGetCurrentThread();
@@ -2503,7 +2478,7 @@ static KSTATUS KiConsoleQueryInfo(SYS_CONSOLE_INFO* infoOut) {
}
// Updates the legacy text-console cursor after authorization.
static KSTATUS KiConsoleSetCursor(int x, int y) {
static KSTATUS KiConsoleSetCursor(int32_t x, int32_t y) {
if (x < 0 || y < 0) return NTLSTATUS_INVALID_PARAMETER;
return KeDisplaySetConsoleCursor(x, y);
}
@@ -2560,18 +2535,18 @@ static uint64_t SyscallClearScreen(uint32_t color) {
}
// Implements SYS_GET_SCREEN_SIZE.
static uint64_t SyscallGetScreenSize(int* widthOut, int* heightOut) {
static uint64_t SyscallGetScreenSize(int32_t* widthOut, int32_t* heightOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_CONSOLE_ACCESS);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
int w = 0, h = 0;
int32_t w = 0, h = 0;
KSTATUS gs = KeDisplayGetSize(&w, &h);
if (!KSUCCESS(gs)) return gs;
if (widthOut) {
KSTATUS s = KiCopyIntToUser(widthOut, w);
KSTATUS s = KiCopyInt32ToUser(widthOut, w);
if (!KSUCCESS(s)) return s;
}
if (heightOut) {
KSTATUS s = KiCopyIntToUser(heightOut, h);
KSTATUS s = KiCopyInt32ToUser(heightOut, h);
if (!KSUCCESS(s)) return s;
}
return NTLSTATUS_SUCCESS;
@@ -2592,7 +2567,7 @@ static uint64_t SyscallGetConsoleInfo(SYS_CONSOLE_INFO* infoOut) {
}
// Implements SYS_SET_CONSOLE_CURSOR through the legacy ambient console privilege.
static uint64_t SyscallSetConsoleCursor(int x, int y) {
static uint64_t SyscallSetConsoleCursor(int32_t x, int32_t y) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_CONSOLE_ACCESS);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
return KiConsoleSetCursor(x, y);
@@ -2612,19 +2587,6 @@ static uint64_t SyscallFillConsoleAttribute(const SYS_FILL_CONSOLE_ATTRIBUTE_REQ
return KiConsoleFillAttribute(request);
}
static KSTATUS KiReferenceInputQueueHandle(SYS_HANDLE handle, INPUT_QUEUE** queueOut) {
if (!queueOut) return NTLSTATUS_INVALID_PARAMETER;
*queueOut = (INPUT_QUEUE*)0;
if (handle == 0) return NTLSTATUS_SUCCESS;
KSTATUS handleStatus = NTLSTATUS_SUCCESS;
OBJECT* object = KiHandleToObject(handle, ObjectTypeInputQueue,
SYS_INPUT_QUEUE_RIGHT_READ, &handleStatus);
if (!object) return handleStatus;
*queueOut = CONTAINING_RECORD(object, INPUT_QUEUE, Header);
return NTLSTATUS_SUCCESS;
}
// Implements the capability-scoped raw input-manager stream used by RSSRV.
static uint64_t SyscallReadInputManagerEvent(const SYS_INPUT_MANAGER_READ_REQUEST* request) {
SYS_INPUT_MANAGER_READ_REQUEST req;
@@ -2645,209 +2607,6 @@ static uint64_t SyscallReadInputManagerEvent(const SYS_INPUT_MANAGER_READ_REQUES
return CopyToUser(req.EventOut, &event, sizeof(event));
}
// Copies one keyboard input record to user mode.
static KSTATUS KiCopyInputRecordToUser(INPUT_KEY_RECORD* userRecord, const INPUT_KEY_RECORD* record) {
if (!userRecord || !record) return NTLSTATUS_INVALID_PARAMETER;
return CopyToUser(userRecord, record, sizeof(*record));
}
// Imports and validates a keyboard input event request.
static KSTATUS KiCopyInputEventRequest(const SYS_INPUT_EVENT_REQUEST* userRequest, SYS_INPUT_EVENT_REQUEST* req) {
KSTATUS s = KiCopySizedUserRequest(userRequest, req, sizeof(*req));
if (!KSUCCESS(s)) return s;
if (req->Flags != SYS_INPUT_EVENT_FLAG_NONE) return NTLSTATUS_INVALID_PARAMETER;
if (!req->RecordOut) return NTLSTATUS_INVALID_PARAMETER;
return NTLSTATUS_SUCCESS;
}
// Implements SYS_POLL_KEYBOARD.
static uint64_t SyscallKbPoll(INPUT_KEY* keyOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (!keyOut) return NTLSTATUS_INVALID_PARAMETER;
INPUT_KEY key = KeInputKbPoll();
KSTATUS s = CopyToUser(keyOut, &key, sizeof(key));
if (!KSUCCESS(s)) return s;
return (key == INPUT_KEY_NONE) ? NTLSTATUS_WOULD_BLOCK : NTLSTATUS_SUCCESS;
}
// Implements SYS_IS_KEY_DOWN.
static uint64_t SyscallKbIsKeyDown(INPUT_KEY key, bool* downOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (!downOut) return NTLSTATUS_INVALID_PARAMETER;
bool down = KeInputIsKeyDown((INPUT_KEY)key);
return CopyToUser(downOut, &down, sizeof(down));
}
// Implements SYS_READ_KEYBOARD_CHAR.
static uint64_t SyscallKbReadChar(char* charOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (!charOut) return NTLSTATUS_INVALID_PARAMETER;
while (true) {
char ch = KeInputReadChar();
if (ch)
return CopyToUser(charOut, &ch, sizeof(ch));
KSTATUS s = KeInputWaitKey(0, SYS_WAIT_TIMEOUT_INFINITE);
if (!KSUCCESS(s))
return s;
}
}
// Implements SYS_READ_KEY.
static uint64_t SyscallKbReadKey(INPUT_KEY* keyOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (!keyOut) return NTLSTATUS_INVALID_PARAMETER;
while (true) {
INPUT_KEY key = KeInputReadKey();
if (key != INPUT_KEY_NONE)
return CopyToUser(keyOut, &key, sizeof(key));
KSTATUS s = KeInputWaitKey(0, SYS_WAIT_TIMEOUT_INFINITE);
if (!KSUCCESS(s))
return s;
}
}
// SYS_GET_KEY_MODIFIERS -> uint32_t INPUT_MOD_* flags.
static uint64_t SyscallKbGetModifiers(void) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
return (uint64_t)KeInputGetModifiers();
}
// Implements SYS_QUERY_GLOBAL_INPUT_STATE.
static uint64_t SyscallQueryGlobalInputState(SYS_GLOBAL_INPUT_STATE* stateOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
SYS_GLOBAL_INPUT_STATE state;
KSTATUS s = KiCopySizedUserRequest(stateOut, &state, sizeof(state));
if (!KSUCCESS(s)) return s;
if (state.Flags != SYS_GLOBAL_INPUT_STATE_FLAG_NONE) return NTLSTATUS_INVALID_PARAMETER;
s = KeInputQueryGlobalState(&state);
if (!KSUCCESS(s)) return s;
return CopyToUser(stateOut, &state, sizeof(state));
}
// Implements SYS_READ_INPUT_EVENT.
static uint64_t SyscallKbReadInputEvent(const SYS_INPUT_EVENT_REQUEST* request) {
SYS_INPUT_EVENT_REQUEST req;
KSTATUS s = KiCopyInputEventRequest(request, &req);
if (!KSUCCESS(s)) return s;
if (req.QueueHandle == SYS_HANDLE_INVALID) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
}
INPUT_QUEUE* queue = (INPUT_QUEUE*)0;
s = KiReferenceInputQueueHandle(req.QueueHandle, &queue);
if (!KSUCCESS(s)) return s;
INPUT_KEY_RECORD record;
while (true) {
bool available = queue ? KeInputQueueReadEvent(queue, &record)
: KeInputReadEvent(&record);
if (available) {
s = KiCopyInputRecordToUser(req.RecordOut, &record);
break;
}
if (req.TimeoutMilliseconds == 0) {
s = NTLSTATUS_WOULD_BLOCK;
break;
}
s = queue ? KeInputQueueWaitKey(queue, 0, req.TimeoutMilliseconds)
: KeInputWaitKey(0, req.TimeoutMilliseconds);
if (!KSUCCESS(s))
break;
}
if (queue) ObDereferenceObject(&queue->Header);
return s;
}
// Implements SYS_PEEK_INPUT_EVENT.
static uint64_t SyscallKbPeekInputEvent(const SYS_INPUT_EVENT_REQUEST* request) {
SYS_INPUT_EVENT_REQUEST req;
KSTATUS s = KiCopyInputEventRequest(request, &req);
if (!KSUCCESS(s)) return s;
if (req.QueueHandle == SYS_HANDLE_INVALID) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
}
INPUT_QUEUE* queue = (INPUT_QUEUE*)0;
s = KiReferenceInputQueueHandle(req.QueueHandle, &queue);
if (!KSUCCESS(s)) return s;
INPUT_KEY_RECORD record;
while (true) {
bool available = queue ? KeInputQueuePeekEvent(queue, &record)
: KeInputPeekEvent(&record);
if (available) {
s = KiCopyInputRecordToUser(req.RecordOut, &record);
break;
}
if (req.TimeoutMilliseconds == 0) {
s = NTLSTATUS_WOULD_BLOCK;
break;
}
s = queue ? KeInputQueueWaitKey(queue, 0, req.TimeoutMilliseconds)
: KeInputWaitKey(0, req.TimeoutMilliseconds);
if (!KSUCCESS(s))
break;
}
if (queue) ObDereferenceObject(&queue->Header);
return s;
}
// Implements SYS_PEEK_MOUSE.
static uint64_t SyscallMousePeek(INPUT_MOUSE_EVENT* eventOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (!eventOut) return NTLSTATUS_INVALID_PARAMETER;
INPUT_MOUSE_EVENT event;
if (!KeInputMousePeek(&event)) return NTLSTATUS_WOULD_BLOCK;
return CopyToUser(eventOut, &event, sizeof(event));
}
// Implements SYS_READ_MOUSE.
static uint64_t SyscallMouseRead(INPUT_MOUSE_EVENT* eventOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (!eventOut) return NTLSTATUS_INVALID_PARAMETER;
INPUT_MOUSE_EVENT event;
if (!KeInputMouseRead(&event)) return NTLSTATUS_WOULD_BLOCK;
return CopyToUser(eventOut, &event, sizeof(event));
}
// Implements SYS_GET_MOUSE_POS.
static uint64_t SyscallMouseGetPos(int* xOut, int* yOut) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_GLOBAL_INPUT);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
if (!xOut || !yOut) return NTLSTATUS_INVALID_PARAMETER;
int x = 0, y = 0;
KeInputMouseGetPosition(&x, &y);
KSTATUS s = KiCopyIntToUser(xOut, x);
if (!KSUCCESS(s)) return s;
return KiCopyIntToUser(yOut, y);
}
// Returns monotonic ticks since boot.
static uint64_t SyscallGetTick(void) {
return gHal ? gHal->Timer.QueryTicks() : 0;
@@ -2919,19 +2678,19 @@ static uint64_t SyscallQueryTrapFrame(SYS_TRAP_FRAME_INFO* infoOut) {
static KSTATUS KiBeginProcessBuildOperation(PROCESS* process) {
if (!process) return NTLSTATUS_INVALID_PARAMETER;
for (;;) {
if (_InterlockedCompareExchange(&process->BuildState, 0, 0) != KE_PROCESS_BUILD_CONSTRUCTING)
if (NtlAtomicLoad32(&process->BuildState) != KE_PROCESS_BUILD_CONSTRUCTING)
return NTLSTATUS_ACCESS_DENIED;
_InterlockedIncrement(&process->BuildOperations);
if (_InterlockedCompareExchange(&process->BuildState, 0, 0) == KE_PROCESS_BUILD_CONSTRUCTING)
NtlAtomicIncrement32(&process->BuildOperations);
if (NtlAtomicLoad32(&process->BuildState) == KE_PROCESS_BUILD_CONSTRUCTING)
return NTLSTATUS_SUCCESS;
_InterlockedDecrement(&process->BuildOperations);
NtlAtomicDecrement32(&process->BuildOperations);
}
}
// Ends one process-construction memory operation.
static void KiEndProcessBuildOperation(PROCESS* process) {
if (!process) return;
long remaining = _InterlockedDecrement(&process->BuildOperations);
int32_t remaining = NtlAtomicDecrement32(&process->BuildOperations);
if (remaining == 0 && KeIsProcessTerminating(process))
(void)KeTerminateProcess(process, process->ExitCode);
}
@@ -3094,20 +2853,20 @@ static uint64_t SyscallStartProcess(const SYS_START_PROCESS_REQUEST* request) {
int buildActive = 0;
status = KiBeginProcessBuildOperation(child);
if (!KSUCCESS(status)) {
int terminating = KeIsProcessTerminating(child);
bool terminating = KeIsProcessTerminating(child);
ObDereferenceObject(&child->Header);
return terminating ? NTLSTATUS_OBJECT_DEAD : status;
}
buildActive = 1;
THREAD* thread = (THREAD*)0;
if (_InterlockedCompareExchange(&child->BuildState, KE_PROCESS_BUILD_STARTING,
if (NtlAtomicCompareExchange32(&child->BuildState, KE_PROCESS_BUILD_STARTING,
KE_PROCESS_BUILD_CONSTRUCTING) != KE_PROCESS_BUILD_CONSTRUCTING) {
status = KeIsProcessTerminating(child) ? NTLSTATUS_OBJECT_DEAD : NTLSTATUS_ACCESS_DENIED;
goto done;
}
if (_InterlockedCompareExchange(&child->BuildOperations, 0, 0) != 1) {
(void)_InterlockedCompareExchange(&child->BuildState, KE_PROCESS_BUILD_CONSTRUCTING,
if (NtlAtomicLoad32(&child->BuildOperations) != 1) {
(void)NtlAtomicCompareExchange32(&child->BuildState, KE_PROCESS_BUILD_CONSTRUCTING,
KE_PROCESS_BUILD_STARTING);
status = NTLSTATUS_WOULD_BLOCK;
goto done;
@@ -3169,7 +2928,7 @@ static uint64_t SyscallStartProcess(const SYS_START_PROCESS_REQUEST* request) {
parameters.Flags = KE_THREAD_CREATE_FLAG_SUSPENDED;
status = KeCreateThread(child, req.EntryPoint, 0, &parameters, &thread);
if (KSUCCESS(status)) {
if (_InterlockedCompareExchange(&child->BuildState, KE_PROCESS_BUILD_STARTED,
if (NtlAtomicCompareExchange32(&child->BuildState, KE_PROCESS_BUILD_STARTED,
KE_PROCESS_BUILD_STARTING) != KE_PROCESS_BUILD_STARTING) {
(void)KeCancelSuspendedThread(thread);
status = NTLSTATUS_OBJECT_DEAD;
@@ -3184,7 +2943,7 @@ static uint64_t SyscallStartProcess(const SYS_START_PROCESS_REQUEST* request) {
child->ImageSize = 0;
(void)KeSetProcessStartupCapabilities(child, (const SYS_STARTUP_CAPABILITY*)0, 0);
(void)KeSetProcessLoaderData(child, (const void*)0);
(void)_InterlockedCompareExchange(&child->BuildState, KE_PROCESS_BUILD_CONSTRUCTING,
(void)NtlAtomicCompareExchange32(&child->BuildState, KE_PROCESS_BUILD_CONSTRUCTING,
KE_PROCESS_BUILD_STARTING);
goto done;
}
@@ -3841,221 +3600,6 @@ static uint64_t SyscallWaitVolumeChange(SYS_HANDLE managerHandle,
return IoWaitForVolumeChange(observedGeneration, timeoutMilliseconds);
}
// Named Queue Registry
// A flat, fixed-size table of { name -> QUEUE* } pairs protected by a
// spinlock. The registry holds one reference to each published queue for
// the lifetime of the entry, so the queue object cannot be destroyed while
// it is registered even if every process closes its own handle to it.
// Limits: NAMED_QUEUE_MAX names, each up to NAMED_QUEUE_NAME_MAX bytes.
// These are generous for the intended usage (a handful of system servers).
#define NAMED_QUEUE_MAX 16
#define NAMED_QUEUE_NAME_MAX 64
typedef struct {
char Name[NAMED_QUEUE_NAME_MAX];
size_t NameLength;
QUEUE* Queue; // NULL = slot free; non-NULL = occupied
uint32_t PublisherPid; // ProcessId of whoever called SysPublishQueue
SYS_HANDLE PublisherHandle; // Publisher-owned handle that keeps the name live
} NAMED_QUEUE_ENTRY;
static NAMED_QUEUE_ENTRY gNamedQueues[NAMED_QUEUE_MAX];
static SPINLOCK gNamedQueueLock;
static int KiNamedQueueNameEquals(const NAMED_QUEUE_ENTRY* entry,
SYS_STRING_VIEW name) {
if (!entry || !entry->Queue) return 0;
return SysSvEqual(SysSvFromParts(entry->Name, entry->NameLength), name);
}
static int KiNamedQueueOwnedByHandle(const NAMED_QUEUE_ENTRY* entry,
const PROCESS* process,
SYS_HANDLE handle,
const OBJECT* object) {
if (!entry || !entry->Queue || !process || !object)
return 0;
return entry->PublisherPid == process->ProcessId &&
entry->PublisherHandle == handle &&
&entry->Queue->Header == object;
}
// Clears a named queue slot and returns the registry reference for later release.
static QUEUE* KiNamedQueueClearSlotLocked(NAMED_QUEUE_ENTRY* entry) {
if (!entry || !entry->Queue)
return (QUEUE*)0;
QUEUE* queue = entry->Queue;
memset(entry, 0, sizeof(*entry));
return queue;
}
// Removes one published queue name owned by the current process.
static KSTATUS KiNamedQueueUnpublishByName(SYS_STRING_VIEW nameView) {
PROCESS* process = KeGetCurrentProcess();
if (!process)
return NTLSTATUS_FAILURE;
QUEUE* queue = (QUEUE*)0;
uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
if (KiNamedQueueNameEquals(&gNamedQueues[i], nameView)) {
if (gNamedQueues[i].PublisherPid != process->ProcessId) {
KeReleaseSpinlock(&gNamedQueueLock, flags);
return NTLSTATUS_ACCESS_DENIED;
}
queue = KiNamedQueueClearSlotLocked(&gNamedQueues[i]);
break;
}
}
KeReleaseSpinlock(&gNamedQueueLock, flags);
if (!queue)
return NTLSTATUS_NOT_FOUND;
ObDereferenceObject(&queue->Header);
return NTLSTATUS_SUCCESS;
}
// Removes published names whose owner handle is being closed.
void KeNamedQueueUnpublishHandle(PROCESS* process, SYS_HANDLE handle, OBJECT* object) {
if (!process || !object || object->Type != ObjectTypeQueue)
return;
while (true) {
QUEUE* queue = (QUEUE*)0;
uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
if (KiNamedQueueOwnedByHandle(&gNamedQueues[i], process, handle, object)) {
queue = KiNamedQueueClearSlotLocked(&gNamedQueues[i]);
break;
}
}
KeReleaseSpinlock(&gNamedQueueLock, flags);
if (!queue)
break;
ObDereferenceObject(&queue->Header);
}
}
// Implements SYS_PUBLISH_QUEUE.
// Registers the queue identified by QueueHandle under Name.
// The published name is owned by the publishing process and handle. Closing that
// handle or exiting the process withdraws every name published through it.
static uint64_t SyscallPublishQueue(const SYS_PUBLISH_QUEUE_REQUEST* request) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_NAMED_QUEUE_PUBLISH);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
SYS_PUBLISH_QUEUE_REQUEST req;
KSTATUS s = KiCopySizedUserRequest(request, &req, sizeof(req));
if (!KSUCCESS(s)) return s;
char name[NAMED_QUEUE_NAME_MAX];
SYS_STRING_VIEW nameView;
s = KiImportUserTextView(req.Name, name, sizeof(name), &nameView, false);
if (!KSUCCESS(s)) return s;
if (nameView.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
KSTATUS handleStatus = NTLSTATUS_SUCCESS;
QUEUE* q = (QUEUE*)KiHandleToObject(req.QueueHandle, ObjectTypeQueue, SYS_QUEUE_RIGHT_PUBLISH, &handleStatus);
if (!q) return handleStatus;
uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
if (KiNamedQueueNameEquals(&gNamedQueues[i], nameView)) {
KeReleaseSpinlock(&gNamedQueueLock, flags);
ObDereferenceObject(&q->Header);
return NTLSTATUS_ALREADY_EXISTS;
}
}
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
if (!gNamedQueues[i].Queue) {
memcpy(gNamedQueues[i].Name, name, nameView.Length);
gNamedQueues[i].NameLength = nameView.Length;
gNamedQueues[i].Queue = q;
gNamedQueues[i].PublisherPid = KeGetCurrentProcess()->ProcessId;
gNamedQueues[i].PublisherHandle = req.QueueHandle;
KeReleaseSpinlock(&gNamedQueueLock, flags);
return NTLSTATUS_SUCCESS;
}
}
KeReleaseSpinlock(&gNamedQueueLock, flags);
ObDereferenceObject(&q->Header);
return NTLSTATUS_NO_MEMORY;
}
// Implements SYS_UNPUBLISH_QUEUE.
// Withdraws a published queue name owned by the calling process.
static uint64_t SyscallUnpublishQueue(const SYS_UNPUBLISH_QUEUE_REQUEST* request) {
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_NAMED_QUEUE_PUBLISH);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
SYS_UNPUBLISH_QUEUE_REQUEST req;
KSTATUS s = KiCopySizedUserRequest(request, &req, sizeof(req));
if (!KSUCCESS(s)) return s;
char name[NAMED_QUEUE_NAME_MAX];
SYS_STRING_VIEW nameView;
s = KiImportUserTextView(req.Name, name, sizeof(name), &nameView, false);
if (!KSUCCESS(s)) return s;
if (nameView.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
return KiNamedQueueUnpublishByName(nameView);
}
// Implements SYS_OPEN_NAMED_QUEUE.
// Looks up 'name' in the registry and allocates a handle to the queue in the
// calling process's handle table. Returns NTLSTATUS_NOT_FOUND if no match.
// Callers that race against the publisher should retry on NOT_FOUND.
static uint64_t SyscallOpenNamedQueue(const SYS_OPEN_NAMED_QUEUE_REQUEST* request) {
SYS_OPEN_NAMED_QUEUE_REQUEST req;
KSTATUS s = KiCopySizedUserRequest(request, &req, sizeof(req));
if (!KSUCCESS(s)) return s;
if (!req.HandleOut) return NTLSTATUS_INVALID_PARAMETER;
char name[NAMED_QUEUE_NAME_MAX];
SYS_STRING_VIEW nameView;
s = KiImportUserTextView(req.Name, name, sizeof(name), &nameView, false);
if (!KSUCCESS(s)) return s;
if (nameView.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
s = MmProbeUserWrite(req.HandleOut, sizeof(SYS_HANDLE));
if (!KSUCCESS(s)) return s;
uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
QUEUE* q = NULL;
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
if (KiNamedQueueNameEquals(&gNamedQueues[i], nameView)) {
q = gNamedQueues[i].Queue;
if (q && !ObTryReferenceObject(&q->Header))
q = (QUEUE*)0;
break;
}
}
KeReleaseSpinlock(&gNamedQueueLock, flags);
if (!q) return NTLSTATUS_NOT_FOUND;
SYS_HANDLE handle = SYS_HANDLE_INVALID;
s = KiAllocHandleForCurrentWithRights(&q->Header, KeGetValidHandleRights(ObjectTypeQueue), &handle);
if (!KSUCCESS(s)) return s;
s = KiCopyHandleToUser(req.HandleOut, handle);
if (!KSUCCESS(s)) {
PROCESS* proc = KeGetCurrentProcess();
if (proc) KeCloseHandle(proc, handle);
return s;
}
return NTLSTATUS_SUCCESS;
}
// Implements SYS_DUPLICATE_HANDLE.
// Duplicates a handle from the caller into the target process table.
@@ -4240,65 +3784,6 @@ static uint64_t SyscallOpenProcessSecurityContext(SYS_HANDLE processHandle,
return status;
}
// Implements SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS.
// Opens a process handle for the process that published a named queue.
static uint64_t SyscallOpenNamedQueueOwnerProcess(const SYS_NAMED_QUEUE_OWNER_REQUEST* request) {
SYS_NAMED_QUEUE_OWNER_REQUEST req;
KSTATUS s = KiCopySizedUserRequest(request, &req, sizeof(req));
if (!KSUCCESS(s)) return s;
if (!req.ProcessHandleOut || req.DesiredAccess == 0) return NTLSTATUS_INVALID_PARAMETER;
SYS_RIGHTS validAccess = KeGetValidHandleRights(ObjectTypeProcess);
if ((req.DesiredAccess & ~validAccess) != 0) return NTLSTATUS_INVALID_PARAMETER;
KSTATUS privilegeStatus = KiRequirePrivilege(SYS_PRIV_PROCESS_OPEN_ANY);
if (!KSUCCESS(privilegeStatus)) return privilegeStatus;
char name[NAMED_QUEUE_NAME_MAX];
SYS_STRING_VIEW nameView;
s = KiImportUserTextView(req.Name, name, sizeof(name), &nameView, false);
if (!KSUCCESS(s)) return s;
if (nameView.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
s = MmProbeUserWrite(req.ProcessHandleOut, sizeof(SYS_HANDLE));
if (!KSUCCESS(s)) return s;
uint32_t pid = 0;
uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
for (int i = 0; i < NAMED_QUEUE_MAX; i++) {
if (KiNamedQueueNameEquals(&gNamedQueues[i], nameView)) {
pid = gNamedQueues[i].PublisherPid;
break;
}
}
KeReleaseSpinlock(&gNamedQueueLock, flags);
if (!pid) return NTLSTATUS_NOT_FOUND;
PROCESS* owner = KeGetProcessById(pid);
if (!owner) return NTLSTATUS_NOT_FOUND;
PROCESS* caller = KeGetCurrentProcess();
if (!caller) {
ObDereferenceObject(&owner->Header);
return NTLSTATUS_FAILURE;
}
SYS_HANDLE handle = SYS_HANDLE_INVALID;
s = KeCreateHandle(caller,
&owner->Header,
req.DesiredAccess,
&handle);
ObDereferenceObject(&owner->Header);
if (!KSUCCESS(s)) return s;
s = KiCopyHandleToUser(req.ProcessHandleOut, handle);
if (!KSUCCESS(s)) {
KeCloseHandle(caller, handle);
return s;
}
return NTLSTATUS_SUCCESS;
}
// Terminates the process referenced by one process handle.
static uint64_t SyscallKillProcess(SYS_HANDLE processHandle, int32_t exitCode) {
KSTATUS handleStatus = NTLSTATUS_SUCCESS;
@@ -5108,13 +4593,13 @@ static uint64_t SyscallGetSocketOption(SYS_HANDLE socket, const SYS_SOCKET_OPTIO
}
// Polls deadline Expired.
static int KiPollDeadlineExpired(uint64_t start, uint32_t timeoutMs) {
static bool KiPollDeadlineExpired(uint64_t start, uint32_t timeoutMs) {
if (timeoutMs == 0)
return 1;
return true;
if (timeoutMs == SYS_POLL_TIMEOUT_INFINITE)
return 0;
return false;
if (!gHal)
return 1;
return true;
return (gHal->Timer.QueryTicks() - start) >= ((uint64_t)timeoutMs * 1000u);
}
@@ -5788,7 +5273,7 @@ static uint64_t SyscallQueryDisplayManagerConsoleInfo(SYS_HANDLE managerHandle,
}
// Updates the console cursor through an explicitly delegated display-manager capability.
static uint64_t SyscallSetDisplayManagerConsoleCursor(SYS_HANDLE managerHandle, int x, int y) {
static uint64_t SyscallSetDisplayManagerConsoleCursor(SYS_HANDLE managerHandle, int32_t x, int32_t y) {
KSTATUS handleStatus = NTLSTATUS_SUCCESS;
KE_DISPLAY_MANAGER* manager = KiReferenceDisplayManagerConsole(managerHandle, &handleStatus);
if (!manager) return handleStatus;
@@ -6659,7 +6144,8 @@ static uint64_t SyscallAudioSetVolume(const SYS_AUDIO_SET_VOLUME_REQUEST* userRe
KSTATUS status = KiCopySizedUserRequest(userRequest, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Flags != 0 || !request.StreamHandle ||
request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY ||
((const uint8_t*)&request)[offsetof(SYS_AUDIO_SET_VOLUME_REQUEST, Muted)] > 1u ||
request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY ||
request.PanQ16 < SYS_AUDIO_PAN_Q16_LEFT ||
request.PanQ16 > SYS_AUDIO_PAN_Q16_RIGHT)
return NTLSTATUS_INVALID_PARAMETER;
@@ -6682,7 +6168,8 @@ static uint64_t SyscallAudioSetMasterVolume(const SYS_AUDIO_SET_MASTER_VOLUME_RE
KSTATUS status = KiCopySizedUserRequest(userRequest, &request, sizeof(request));
if (!KSUCCESS(status)) return status;
if (request.Flags != 0 || !request.EndpointId ||
request.Muted > 1u || request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY)
((const uint8_t*)&request)[offsetof(SYS_AUDIO_SET_MASTER_VOLUME_REQUEST, Muted)] > 1u ||
request.VolumeQ16 > SYS_AUDIO_GAIN_Q16_UNITY)
return NTLSTATUS_INVALID_PARAMETER;
return KeAudioSetMasterVolume(request.EndpointId,
@@ -6774,10 +6261,6 @@ static const SYSCALL_ENTRY gSyscallTable[SYS_MAX] = {
[SYS_CREATE_QUEUE] = SYSCALL_ENTRY1(SyscallQueueCreate),
[SYS_SEND_QUEUE] = SYSCALL_ENTRY1(SyscallQueueSend),
[SYS_RECV_QUEUE] = SYSCALL_ENTRY1(SyscallQueueRecv),
[SYS_PUBLISH_QUEUE] = SYSCALL_ENTRY1(SyscallPublishQueue),
[SYS_UNPUBLISH_QUEUE] = SYSCALL_ENTRY1(SyscallUnpublishQueue),
[SYS_OPEN_NAMED_QUEUE] = SYSCALL_ENTRY1(SyscallOpenNamedQueue),
[SYS_OPEN_NAMED_QUEUE_OWNER_PROCESS] = SYSCALL_ENTRY1(SyscallOpenNamedQueueOwnerProcess),
// LPC-like Ports and well-known system services.
[SYS_CREATE_PORT] = SYSCALL_ENTRY1(SyscallCreatePort),
@@ -6869,18 +6352,6 @@ static const SYSCALL_ENTRY gSyscallTable[SYS_MAX] = {
// Input.
[SYS_READ_INPUT_MANAGER_EVENT] = SYSCALL_ENTRY1(SyscallReadInputManagerEvent),
[SYS_POLL_KEYBOARD] = SYSCALL_ENTRY1(SyscallKbPoll),
[SYS_IS_KEY_DOWN] = SYSCALL_ENTRY2(SyscallKbIsKeyDown),
[SYS_READ_KEYBOARD_CHAR] = SYSCALL_ENTRY1(SyscallKbReadChar),
[SYS_READ_KEY] = SYSCALL_ENTRY1(SyscallKbReadKey),
[SYS_GET_KEY_MODIFIERS] = SYSCALL_ENTRY0(SyscallKbGetModifiers),
[SYS_QUERY_GLOBAL_INPUT_STATE] = SYSCALL_ENTRY1(SyscallQueryGlobalInputState),
[SYS_OPEN_INPUT_QUEUE] = SYSCALL_ENTRY2(SyscallOpenInputQueue),
[SYS_READ_INPUT_EVENT] = SYSCALL_ENTRY1(SyscallKbReadInputEvent),
[SYS_PEEK_INPUT_EVENT] = SYSCALL_ENTRY1(SyscallKbPeekInputEvent),
[SYS_PEEK_MOUSE] = SYSCALL_ENTRY1(SyscallMousePeek),
[SYS_READ_MOUSE] = SYSCALL_ENTRY1(SyscallMouseRead),
[SYS_GET_MOUSE_POS] = SYSCALL_ENTRY2(SyscallMouseGetPos),
// Time, debug, and system information.
[SYS_CREATE_TIMER] = SYSCALL_ENTRY1(SyscallCreateTimer),
code/kernel/ke_thread.c
+29 -27
View File
@@ -1,5 +1,8 @@
// Thread manager. This file owns thread allocation, kernel/user startup contexts, and lifecycle transitions.
#include <ntlite/atomic.h>
#include <ntlite/string_view.h>
#include <stdbool.h>
#include <string.h>
#include <kernel/ke_list.h>
#include <stddef.h>
@@ -127,7 +130,7 @@ void KeFinalizeThreadExit(THREAD* thread) {
if (remaining == 0) {
// ThreadCount represents live process membership, not retained thread
// handles. Dead THREAD objects may remain waitable after process exit.
_InterlockedExchange(&owner->Terminating, 1);
NtlAtomicExchange32(&owner->Terminating, 1);
KeStopProcessConstruction(owner);
KeCompleteProcessExitIfLastThread(owner);
}
@@ -215,6 +218,8 @@ static KSTATUS KiValidateThreadName(const char* name, size_t* lengthOut) {
if (length >= NTL_THREAD_NAME_MAX)
return NTLSTATUS_INVALID_PARAMETER;
if (!SysSvIsUtf8Text(SysSvFromParts(name, length)))
return NTLSTATUS_INVALID_PARAMETER;
*lengthOut = length;
return NTLSTATUS_SUCCESS;
}
@@ -413,7 +418,7 @@ void KeEnterTerminationDeferral(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread || thread->IsIdleThread)
return;
_InterlockedIncrement(&thread->TerminationDeferralCount);
NtlAtomicIncrement32(&thread->TerminationDeferralCount);
}
// Leaves a process-termination deferral region.
@@ -423,7 +428,7 @@ void KeLeaveTerminationDeferral(void) {
if (!thread || thread->IsIdleThread)
return;
long remaining = _InterlockedDecrement(&thread->TerminationDeferralCount);
int32_t remaining = NtlAtomicDecrement32(&thread->TerminationDeferralCount);
if (remaining < 0) {
KePanic(KE_PANIC_CODE_SCHEDULER,
"thread termination deferral underflow: thread=0x%016llX tid=%u\n",
@@ -436,21 +441,17 @@ void KeLeaveTerminationDeferral(void) {
}
// Returns whether one thread still owns termination-sensitive kernel work.
int KeThreadTerminationDeferred(const THREAD* thread) {
return thread && _InterlockedCompareExchange((volatile long*)&thread->TerminationDeferralCount,
0,
0) > 0;
bool KeThreadTerminationDeferred(const THREAD* thread) {
return thread && NtlAtomicLoad32(&thread->TerminationDeferralCount) > 0;
}
// Returns whether terminal teardown owns the current thread stack.
int KeThreadExitInProgress(const THREAD* thread) {
return thread && _InterlockedCompareExchange((volatile long*)&thread->ExitInProgress,
0,
0) != 0;
bool KeThreadExitInProgress(const THREAD* thread) {
return thread && NtlAtomicLoad32(&thread->ExitInProgress) != 0;
}
// Implements the exit Thread operation.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
__declspec(dllexport) void KeExitThread(THREAD* thread, int32_t code) {
(void)code;
if (!thread) return;
@@ -458,7 +459,7 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
// stack until thread/process completion is fully published. Hardware
// interrupts remain enabled, but interrupt-return scheduling observes
// ExitInProgress and leaves the thread on-CPU until the final KeYield().
if (_InterlockedCompareExchange(&thread->ExitInProgress, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&thread->ExitInProgress, 1, 0) != 0)
KePanic(KE_PANIC_CODE_SCHEDULER,
"recursive thread exit: thread=0x%016llX tid=%u\n",
(uint64_t)thread,
@@ -634,7 +635,8 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
// Replaces one thread's diagnostic description.
KSTATUS KeSetThreadName(THREAD* thread, const char* name, size_t length) {
if (!thread || (length && !name) || length >= NTL_THREAD_NAME_MAX)
if (!thread || length >= NTL_THREAD_NAME_MAX ||
!SysSvIsUtf8Text(SysSvFromParts(name, length)))
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&thread->NameLock);
@@ -777,28 +779,28 @@ void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
}
// Returns whether the scheduler link still belongs to an object wait list.
static int KiThreadWaitLinkIsLinked(const THREAD* thread) {
if (!thread) return 0;
static bool KiThreadWaitLinkIsLinked(const THREAD* thread) {
if (!thread) return false;
return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
thread->SchedulerLink.Bwd != &thread->SchedulerLink;
}
// Removes a thread from one wait list while that wait object's lock is held.
static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
if (!thread || !waitListHead) return 0;
static bool KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
if (!thread || !waitListHead) return false;
if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead)
return 0;
return false;
// The signaling side pops the wait-list entry before it wakes the thread.
// In that handoff window the scheduler link is already self-linked; abort
// must not remove it again or turn a successful mutex/semaphore handoff
// into an aborted wait for a thread that may already own the object.
if (!KiThreadWaitLinkIsLinked(thread))
return 0;
return false;
KeListRemoveChecked(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
return 1;
return true;
}
// Removes a thread from an object-owned wait queue using that object's lock.
@@ -837,9 +839,9 @@ static int KiAbortObjectWait(THREAD* thread) {
// unsignaled after the foreground thread has faulted. Object wait-list removal
// must use the object's own lock; otherwise signal/abort races can leave stale
// scheduler links in an event, mutex, or semaphore wait queue.
int KeAbortThreadWait(THREAD* thread, KSTATUS status) {
if (!thread) return 0;
if (thread->State != ThreadWaiting) return 0;
bool KeAbortThreadWait(THREAD* thread, KSTATUS status) {
if (!thread) return false;
if (thread->State != ThreadWaiting) return false;
if (thread->WaitRundownRoutine) {
KeRundownThreadWait(thread, status);
@@ -848,18 +850,18 @@ int KeAbortThreadWait(THREAD* thread, KSTATUS status) {
if (KiAbortObjectWait(thread)) {
KeUnblockThreadWithStatus(thread, status);
return 1;
return true;
}
// A signaler that already dequeued an object waiter owns completion. Let
// that handoff finish rather than returning timeout after ownership or a
// semaphore count has already transferred to this thread.
if (thread->WaitObject)
return 0;
return false;
// Delays and auxiliary scheduler waits have no object-owned queue.
KeUnblockThreadFromWaitListWithStatus(thread, status);
return 1;
return true;
}
// Implements the unblock Thread operation.
code/kernel/ke_timer.c
+2 -1
View File
@@ -1,6 +1,7 @@
// Kernel waitable timer objects. This file owns timer deadlines and timer wait signaling.
#include <stdint.h>
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <string.h>
#include <kernel/ke_timer.h>
@@ -11,7 +12,7 @@
static LIST_ENTRY gTimerList;
static SPINLOCK gTimerLock;
static uint8_t gTimerInitialized;
static bool gTimerInitialized;
// Initializes the global timer list on first use.
static void KiTimerEnsureInitialized(void) {
code/kernel/ke_trap.c
+3 -2
View File
@@ -1,5 +1,6 @@
// Trap-frame tracking. This file owns current-thread trap snapshots and validation for user diagnostics.
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <kernel/ke_trap.h>
@@ -45,8 +46,8 @@ STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousFrame, 224);
_Static_assert(sizeof(KTRAP_FRAME) == 240, "KTRAP_FRAME size must match ke_syscall.asm");
// Implements the trap Frame From User operation.
int KeTrapFrameFromUser(const KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) return 0;
bool KeTrapFrameFromUser(const KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) return false;
return (frame->Cs & 3u) == 3u;
}
code/kernel/ke_work.c
+13 -12
View File
@@ -1,6 +1,7 @@
// Kernel passive-level work queue. This file owns the generic mechanism used
// to move ISR/DPC-originated work to a sleepable system thread context.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <stddef.h>
@@ -19,8 +20,8 @@ static SPINLOCK gWorkQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gWorkQueueList;
static EVENT* gWorkQueueEvent;
static KDPC gWorkQueueWakeDpc;
static volatile long gWorkQueueListReady;
static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started.
static volatile int32_t gWorkQueueListReady;
static volatile int32_t gWorkQueueStartState; // 0=not started, 1=starting, 2=started.
// Signals the passive worker after an enqueue from hardware interrupt context.
static void KiWorkQueueWakeDpc(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
@@ -57,7 +58,7 @@ static void KiWorkQueueWorker(void* arg) {
// Allow the item to be queued again while it is running. This keeps
// level-triggered users such as cursor updates naturally coalesced
// but does not lose an update that arrives during the callback.
_InterlockedExchange(&item->Queued, 0);
NtlAtomicExchange32(&item->Queued, 0);
if (item->Routine)
item->Routine(item->Context);
}
@@ -74,18 +75,18 @@ void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context
}
bool KeIsWorkQueueStarted(void) {
return _InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 2;
return NtlAtomicCompareExchange32(&gWorkQueueStartState, 2, 2) == 2;
}
bool KeQueueWorkItem(KWORK_ITEM* item) {
if (!item || !item->Routine)
return false;
if (_InterlockedCompareExchange(&item->Queued, 1, 0) != 0)
if (NtlAtomicCompareExchange32(&item->Queued, 1, 0) != 0)
return false;
if (!_InterlockedCompareExchange(&gWorkQueueListReady, 1, 1)) {
_InterlockedExchange(&item->Queued, 0);
if (!NtlAtomicCompareExchange32(&gWorkQueueListReady, 1, 1)) {
NtlAtomicExchange32(&item->Queued, 0);
return false;
}
@@ -106,18 +107,18 @@ bool KeQueueWorkItem(KWORK_ITEM* item) {
}
KSTATUS KeStartWorkQueue(void) {
long state = _InterlockedCompareExchange(&gWorkQueueStartState, 1, 0);
int32_t state = NtlAtomicCompareExchange32(&gWorkQueueStartState, 1, 0);
if (state == 2)
return NTLSTATUS_SUCCESS;
if (state == 1) {
while (_InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 1)
while (NtlAtomicCompareExchange32(&gWorkQueueStartState, 2, 2) == 1)
KeYield();
return KeIsWorkQueueStarted() ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
}
LIST_INIT(&gWorkQueueList);
KeInitializeDpc(&gWorkQueueWakeDpc, KiWorkQueueWakeDpc, (void*)0);
_InterlockedExchange(&gWorkQueueListReady, 1);
NtlAtomicExchange32(&gWorkQueueListReady, 1);
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
if (KSUCCESS(s)) {
@@ -133,10 +134,10 @@ KSTATUS KeStartWorkQueue(void) {
}
if (KSUCCESS(s)) {
_InterlockedExchange(&gWorkQueueStartState, 2);
NtlAtomicExchange32(&gWorkQueueStartState, 2);
KeSignalEvent(gWorkQueueEvent);
} else {
_InterlockedExchange(&gWorkQueueStartState, 0);
NtlAtomicExchange32(&gWorkQueueStartState, 0);
}
return s;
}
code/kernel/kmain.c
+24 -23
View File
@@ -1,5 +1,6 @@
// Kernel entry point. This file connects boot information to core subsystem initialization and scheduler start.
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <intrin.h>
@@ -128,7 +129,7 @@ typedef KSTATUS (*DRIVER_ENTRY_FN)(const NTL_DDK* ddk, NTL_DRIVER_INIT* init);
static const HAL_API* gKmainHal;
// Implements the page In Bootloader Region operation.
static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
static bool KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) {
const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i];
if (r->Type != MemoryBootloader) continue;
@@ -136,10 +137,10 @@ static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS
uintptr_t start = r->Base & PAGE_MASK;
uintptr_t end = (r->Base + r->Length + PAGE_SIZE - 1) & PAGE_MASK;
if ((uintptr_t)page >= start && (uintptr_t)page < end) {
return 1;
return true;
}
}
return 0;
return false;
}
// Reclaims one exact page-backed blob after its boot consumer has copied or materialized it.
@@ -170,7 +171,7 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
}
// Returns whether one drivers.ini line character is horizontal whitespace.
static int KiDriverConfigSpace(char c) {
static bool KiDriverConfigSpace(char c) {
return c == ' ' || c == '\t' || c == '\r';
}
@@ -183,10 +184,10 @@ static void KiDriverConfigTrim(const char** begin, const char** end) {
}
// Returns whether one immutable drivers.ini span exactly matches an ASCII string.
static int KiDriverConfigSpanEquals(const char* begin, const char* end, const char* text) {
static bool KiDriverConfigSpanEquals(const char* begin, const char* end, const char* text) {
while (begin < end && *text) {
if (*begin++ != *text++)
return 0;
return false;
}
return begin == end && *text == '\0';
}
@@ -605,55 +606,55 @@ void SystemInitThread(void* arg) {
}
// Returns whether one page-backed boot allocation is aligned, nonempty, and does not overflow.
static int KiValidateBootPageRange(uintptr_t physicalBase, uint32_t pageCount) {
static bool KiValidateBootPageRange(uintptr_t physicalBase, uint32_t pageCount) {
if (!physicalBase || (physicalBase & (PAGE_SIZE - 1u)) != 0 || pageCount == 0)
return 0;
return false;
uint64_t byteCount = (uint64_t)pageCount * PAGE_SIZE;
return byteCount != 0 && physicalBase <= UINTPTR_MAX - byteCount;
}
// Returns whether one page-backed raw boot file is fully contained in its recorded allocation.
static int KiValidateBootFile(const BOOT_FILE* file) {
static bool KiValidateBootFile(const BOOT_FILE* file) {
if (!file || !KiValidateBootPageRange(file->PhysicalBase, file->PageCount) || file->Length == 0)
return 0;
return false;
uint64_t capacity = (uint64_t)file->PageCount * PAGE_SIZE;
if ((uint64_t)file->Length + 1u > capacity || file->Name[0] == '\0')
return 0;
return false;
return memchr(file->Name, '\0', sizeof(file->Name)) != NULL;
}
// Returns whether one fixed core-image descriptor fits inside its page-backed physical allocation.
static int KiValidateBootImage(const BOOT_IMAGE* image, int requireEntry) {
static bool KiValidateBootImage(const BOOT_IMAGE* image, int requireEntry) {
if (!image || !image->VirtualBase || (image->VirtualBase & (PAGE_SIZE - 1u)) != 0 ||
!KiValidateBootPageRange(image->PhysicalBase, image->PageCount) || !image->ImageSize) {
return 0;
return false;
}
uint64_t capacity = (uint64_t)image->PageCount * PAGE_SIZE;
if (image->ImageSize > capacity || image->VirtualBase > UINTPTR_MAX - image->ImageSize)
return 0;
return false;
if (requireEntry && image->EntryRva >= image->ImageSize)
return 0;
return 1;
return false;
return true;
}
// Validates the exact bootloader/kernel handoff contract before dereferencing variable physical blobs.
static int KiValidateBootInfo(const BOOT_INFO* bootInfo) {
static bool KiValidateBootInfo(const BOOT_INFO* bootInfo) {
if (!bootInfo || bootInfo->Magic != BOOT_INFO_MAGIC ||
bootInfo->Version != BOOT_INFO_VERSION || bootInfo->Size != sizeof(BOOT_INFO) ||
bootInfo->MemoryMap.Magic != MEMORY_MAP_MAGIC ||
bootInfo->MemoryMap.RegionCount > MEMORY_MAX_REGIONS ||
bootInfo->PhysicalMapBase == 0) {
return 0;
return false;
}
uint64_t bootIniCapacity = (uint64_t)bootInfo->BootIni.PageCount * PAGE_SIZE;
if (!KiValidateBootPageRange(bootInfo->BootIni.PhysicalBase, bootInfo->BootIni.PageCount) ||
bootInfo->BootIni.Length == 0 || (uint64_t)bootInfo->BootIni.Length + 1u > bootIniCapacity ||
bootInfo->BootIni.PhysicalBase > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
return 0;
return false;
}
uint64_t driverListCapacity = (uint64_t)bootInfo->BootDrivers.PageCount * PAGE_SIZE;
@@ -661,7 +662,7 @@ static int KiValidateBootInfo(const BOOT_INFO* bootInfo) {
if (!KiValidateBootPageRange(bootInfo->BootDrivers.FilesPhysical, bootInfo->BootDrivers.PageCount) ||
bootInfo->BootDrivers.Count == 0 || driverListBytes > driverListCapacity ||
bootInfo->BootDrivers.FilesPhysical > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
return 0;
return false;
}
const BOOT_FILE* bootFiles =
@@ -669,16 +670,16 @@ static int KiValidateBootInfo(const BOOT_INFO* bootInfo) {
for (uint32_t index = 0; index < bootInfo->BootDrivers.Count; ++index) {
if (!KiValidateBootFile(&bootFiles[index]) ||
bootFiles[index].PhysicalBase > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
return 0;
return false;
}
}
if (!KiValidateBootImage(&bootInfo->KernelImage, 1) ||
!KiValidateBootImage(&bootInfo->HalImage, 0)) {
return 0;
return false;
}
return 1;
return true;
}
// Main Kernel Entry.
code/kernel/ldr_image.c
+2 -1
View File
@@ -188,7 +188,8 @@ static KSTATUS KiRegisterGlobalModule(const char* name, uintptr_t base, size_t s
size_t nameLength = 0;
while (nameLength < LDR_KERNEL_MODULE_NAME_MAX && name[nameLength]) nameLength++;
if (nameLength == 0 || nameLength >= LDR_KERNEL_MODULE_NAME_MAX)
if (nameLength == 0 || nameLength >= LDR_KERNEL_MODULE_NAME_MAX ||
!SysSvIsUtf8Text(SysSvFromParts(name, nameLength)))
return NTLSTATUS_INVALID_PARAMETER;
MODULE_ENTRY* entry = (MODULE_ENTRY*)MmAllocatePool(sizeof(MODULE_ENTRY));
code/kernel/mm_file_cache.c
+14 -13
View File
@@ -5,6 +5,7 @@
// truth. Dirty pages are flushed explicitly, by close, or by the background
// writeback worker under pressure.
#include <stdbool.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
@@ -36,7 +37,7 @@ typedef struct MM_FILE_CACHE_ENTRY {
uint8_t Referenced;
uint8_t Standby;
uint8_t Loading;
uint8_t Invalidated;
bool Invalidated;
uint8_t Dirty;
uint8_t Writeback;
uint64_t DirtySequence;
@@ -53,7 +54,7 @@ static uint64_t gFileCacheMaxResidentPages;
static MM_FILE_CACHE_STATS gFileCacheStats;
static int KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry);
static bool KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry);
static SPINLOCK gWritebackLock = SPINLOCK_INIT;
static SEMAPHORE* gWritebackSemaphore;
@@ -149,12 +150,12 @@ static void KiFileCacheRemoveEntryLocked(uint32_t bucket, MM_FILE_CACHE_ENTRY* p
KiFileCacheSetDirtyLocked(e, 0);
}
static int KiFileCacheIsReclaimCandidateLocked(MM_FILE_CACHE_ENTRY* e, FILE_OBJECT* skipFile, uint64_t skipOffset) {
if (!e) return 0;
static bool KiFileCacheIsReclaimCandidateLocked(MM_FILE_CACHE_ENTRY* e, FILE_OBJECT* skipFile, uint64_t skipOffset) {
if (!e) return false;
if (skipFile && e->File == skipFile && e->Offset == skipOffset)
return 0;
return false;
if (e->Loading || e->Invalidated || !e->Frame || e->Dirty || e->Writeback)
return 0;
return false;
MM_PFN* pfn = MmPfnGet(e->Frame);
return pfn && pfn->PinCount == 0 && pfn->MappedCount == 0 && pfn->RefCount == 1;
@@ -491,19 +492,19 @@ static void KiFileCacheFreeLoadingEntry(MM_FILE_CACHE_ENTRY* entry) {
KiFileCacheFreeEntry(entry);
}
static int KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry) {
static bool KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry) {
if (!entry)
return 0;
return false;
uint32_t bucket = KiFileCacheHash(entry->File, entry->Offset);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[bucket]; e; prev = e, e = e->Next) {
if (e == entry) {
KiFileCacheRemoveEntryLocked(bucket, prev, e);
return 1;
return true;
}
}
return 0;
return false;
}
static KSTATUS KiFileCacheReserveLoadingPage(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_ENTRY** outEntry) {
@@ -536,7 +537,7 @@ static void KiFileCacheCancelLoadingPage(MM_FILE_CACHE_ENTRY* entry) {
return;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
int unlinked = KiFileCacheUnlinkEntryLocked(entry);
bool unlinked = KiFileCacheUnlinkEntryLocked(entry);
KeReleaseSpinlock(&gFileCacheLock, flags);
if (unlinked)
@@ -655,7 +656,7 @@ static KSTATUS KiFileCacheLoadPage(FILE_OBJECT* file,
static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file,
uint64_t aligned,
uint64_t oldFileSize,
int needExistingBytes,
bool needExistingBytes,
MM_FILE_CACHE_PAGE* outPage) {
if (!file || !outPage)
return NTLSTATUS_INVALID_PARAMETER;
@@ -728,7 +729,7 @@ static void KiFileCacheMarkPageDirty(FILE_OBJECT* file, uint64_t aligned, uint32
}
KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
uint64_t fileSizeBefore, int needExistingBytes,
uint64_t fileSizeBefore, bool needExistingBytes,
MM_FILE_CACHE_PAGE* outPage) {
if (!outPage)
return NTLSTATUS_INVALID_PARAMETER;
code/kernel/mm_phys.c
+11 -10
View File
@@ -4,6 +4,7 @@
// but callers now get typed, reference-counted frames instead of anonymous
// bitmap bits.
#include <stdbool.h>
#include <kernel/mm_pfn.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
@@ -61,12 +62,12 @@ static inline PHYSICAL_ADDRESS KiAlignDown(PHYSICAL_ADDRESS value) {
}
// Returns whether a boot memory type describes RAM that the PFN database should track.
static int KiMemoryTypeManaged(MEMORY_TYPE type) {
static bool KiMemoryTypeManaged(MEMORY_TYPE type) {
return type == MemoryUsable || type == MemoryBootloader || type == MemoryKernel;
}
// Returns whether an allocation may consume the emergency kernel reserve.
static int KiPfnUsageMayUseReserve(MM_PFN_USAGE usage) {
static bool KiPfnUsageMayUseReserve(MM_PFN_USAGE usage) {
return usage == MmPfnUseKernel ||
usage == MmPfnUsePageTable ||
usage == MmPfnUseDma ||
@@ -142,11 +143,11 @@ static void KiRegisterManagedRange(PHYSICAL_ADDRESS base,
}
// Returns whether the future page-replacement code may consider this frame a candidate.
static int KiPfnEvictableLocked(const MM_PFN* entry) {
static bool KiPfnEvictableLocked(const MM_PFN* entry) {
if (!entry || entry->PinCount)
return 0;
return false;
if (entry->State != MmPfnStateActive && entry->State != MmPfnStateStandby && entry->State != MmPfnStateModified)
return 0;
return false;
return entry->Usage == MmPfnUseUser ||
entry->Usage == MmPfnUseCleanFileCache ||
entry->Usage == MmPfnUseImage ||
@@ -156,18 +157,18 @@ static int KiPfnEvictableLocked(const MM_PFN* entry) {
}
// Returns whether a standby frame can be counted as immediately reclaimable.
static int KiPfnReclaimableLocked(const MM_PFN* entry) {
static bool KiPfnReclaimableLocked(const MM_PFN* entry) {
if (!entry || entry->PinCount || entry->MappedCount)
return 0;
return false;
if (entry->State != MmPfnStateStandby || entry->RefCount != 1)
return 0;
return false;
return entry->Usage == MmPfnUseCleanFileCache || entry->Usage == MmPfnUseImage;
}
// Returns true when a PFN can satisfy a new allocation.
static int KiPfnAvailableLocked(size_t pfn) {
static bool KiPfnAvailableLocked(size_t pfn) {
if (pfn >= gMaxPfn)
return 0;
return false;
MM_PFN* entry = &gPfnDatabase[pfn];
return entry->State == MmPfnStateFree || entry->State == MmPfnStateZeroed;
}
code/kernel/mm_pool.c
+9 -8
View File
@@ -1,5 +1,6 @@
// Kernel pool allocator and corruption diagnostics.
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
@@ -163,9 +164,9 @@ static _Noreturn void KiPoolPanic(const char* reason, HEAP* h, BLOCK_HDR* blk, c
}
// Tests whether an allocator header pointer is inside the committed heap.
static int KiPoolHeaderInCommittedHeap(const HEAP* h, const BLOCK_HDR* blk) {
static bool KiPoolHeaderInCommittedHeap(const HEAP* h, const BLOCK_HDR* blk) {
if (!h || !blk)
return 0;
return false;
uintptr_t addr = (uintptr_t)blk;
return addr >= h->Start && addr < h->Brk;
}
@@ -250,9 +251,9 @@ static void KiFreeListInsertHeadLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller
static void KiFreeListRemoveLocked(HEAP* h, BLOCK_HDR* blk, uintptr_t caller);
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
static bool KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (h->Brk + grow > h->Limit) return 0;
if (h->Brk + grow > h->Limit) return false;
uintptr_t start = h->Brk;
uintptr_t mappedEnd = start;
@@ -260,12 +261,12 @@ static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame) {
KiHeapRollbackGrow(start, mappedEnd);
return 0;
return false;
}
if (!KSUCCESS(MmPteMapPage(va, frame, h->MapFlags, MmPteMapFailIfPresent))) {
MmPfnFree(frame);
KiHeapRollbackGrow(start, mappedEnd);
return 0;
return false;
}
mappedEnd = va + PAGE_SIZE;
memset((void*)va, 0, PAGE_SIZE);
@@ -294,7 +295,7 @@ static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = BLOCK_STATE_FREE;
KiHdrToFtr(prev)->Size = prev->Size;
return 1;
return true;
}
}
}
@@ -323,7 +324,7 @@ static int KiHeapGrow(HEAP* h, size_t need, uintptr_t caller) {
KiHdrToFtr(blk)->Size = blk->Size;
KiFreeListInsertHeadLocked(h, blk, caller);
return 1;
return true;
}
// Validates a free-list node before the allocator mutates list links.
code/kernel/mm_pte.c
+24 -22
View File
@@ -1,5 +1,7 @@
// Kernel page-table manager. This file owns x86-64 PTE manipulation, TLB invalidation, and shared-CR3 activation.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
@@ -27,9 +29,9 @@ static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
static PHYSICAL_ADDRESS gPml4Phys = 0;
static SPINLOCK gPageTableLock = SPINLOCK_INIT;
static volatile long gTlbShootdownLock = 0;
static volatile long gTlbShootdownActive = 0;
static volatile long gTlbShootdownAcks = 0;
static volatile int32_t gTlbShootdownLock = 0;
static volatile int32_t gTlbShootdownActive = 0;
static volatile int32_t gTlbShootdownAcks = 0;
static uintptr_t gTlbShootdownVirt = 0;
static MM_PTE_STATS gPteStats;
@@ -113,7 +115,7 @@ static void KiAcquireTlbShootdownLock(void) {
// A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
// concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is
// meant to avoid.
while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0)
while (NtlAtomicCompareExchange32(&gTlbShootdownLock, 1, 0) != 0)
_mm_pause();
_ReadWriteBarrier();
}
@@ -121,7 +123,7 @@ static void KiAcquireTlbShootdownLock(void) {
// Releases the shootdown state lock after all shared fields are stable.
static void KiReleaseTlbShootdownLock(void) {
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownLock, 0);
NtlAtomicExchange32(&gTlbShootdownLock, 0);
}
// Flushes one kernel mapping on every online CPU that shares this PML4.
@@ -139,11 +141,11 @@ void MmPteFlushKernelAddress(uintptr_t virt) {
KiAcquireTlbShootdownLock();
long targets = 0;
int32_t targets = 0;
gTlbShootdownVirt = virt;
_InterlockedExchange(&gTlbShootdownAcks, 0);
NtlAtomicExchange32(&gTlbShootdownAcks, 0);
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownActive, 1);
NtlAtomicExchange32(&gTlbShootdownActive, 1);
_ReadWriteBarrier();
for (uint32_t i = 0; i < gCpuCount; i++) {
@@ -159,7 +161,7 @@ void MmPteFlushKernelAddress(uintptr_t virt) {
if (!targets) {
KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets);
_InterlockedExchange(&gTlbShootdownActive, 0);
NtlAtomicExchange32(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
return;
@@ -168,13 +170,13 @@ void MmPteFlushKernelAddress(uintptr_t virt) {
KiPteStatInc(&gPteStats.ShootdownRequests);
for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets)
if (NtlAtomicLoad32(&gTlbShootdownAcks) >= targets)
break;
_mm_pause();
}
long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0);
_InterlockedExchange(&gTlbShootdownActive, 0);
int32_t acks = NtlAtomicLoad32(&gTlbShootdownAcks);
NtlAtomicExchange32(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
@@ -188,20 +190,20 @@ void MmPteFlushKernelAddress(uintptr_t virt) {
}
// Handles an incoming shootdown request on the current CPU.
static int KiHandleTlbShootdown(void) {
if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
return 0;
static bool KiHandleTlbShootdown(void) {
if (NtlAtomicLoad32(&gTlbShootdownActive) == 0)
return false;
uintptr_t va = gTlbShootdownVirt;
_ReadWriteBarrier();
MmPteFlushLocal(va);
_ReadWriteBarrier();
_InterlockedIncrement(&gTlbShootdownAcks);
return 1;
NtlAtomicIncrement32(&gTlbShootdownAcks);
return true;
}
// Handles the NMI vector reserved for TLB shootdowns.
int MmPteHandleShootdownNmi(void) {
bool MmPteHandleShootdownNmi(void) {
return KiHandleTlbShootdown();
}
@@ -363,9 +365,9 @@ __declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uin
// used by the pressure trimmer as a second-chance test for clean file-backed
// pages. If the page was accessed recently, reclaim skips it once and flushes
// the TLB entry so future accesses can set PAGE_ACCESSED again.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt) {
__declspec(dllexport) bool MmPteTestAndClearAccessed(uintptr_t virt) {
if (!gPml4Phys)
return 0;
return false;
uintptr_t va = virt & PAGE_MASK;
int wasAccessed = 0;
@@ -399,9 +401,9 @@ out:
}
// Tests and clears the hardware dirty bit on a mapped 4 KiB leaf.
__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt) {
__declspec(dllexport) bool MmPteTestAndClearDirty(uintptr_t virt) {
if (!gPml4Phys)
return 0;
return false;
uintptr_t va = virt & PAGE_MASK;
int wasDirty = 0;
code/kernel/mm_swap.c
+7 -6
View File
@@ -3,6 +3,7 @@
// Kernel memory and filesystem code remain non-pageable, so this layer is safe
// to use before the VMM has direct block-extents for pageout.
#include <stdbool.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
@@ -21,14 +22,14 @@ typedef struct MM_SWAP_STATE {
uint32_t SlotCount;
uint32_t UsedSlots;
uint32_t HighWaterSlots;
int Enabled;
bool Enabled;
MM_SWAP_STATS Stats;
} MM_SWAP_STATE;
static MM_SWAP_STATE gSwap = { .Lock = SPINLOCK_INIT };
// Returns whether a swap slot is marked in use.
static int KiSwapTestSlot(uint32_t slot) {
static bool KiSwapTestSlot(uint32_t slot) {
return (gSwap.Bitmap[slot >> 3] & (uint8_t)(1u << (slot & 7))) != 0;
}
@@ -133,7 +134,7 @@ KSTATUS MmSwapInitialize(const char* relativePath, uint64_t sizeBytes) {
gSwap.HighWaterSlots = 0;
gSwap.Enabled = 1;
memset(&gSwap.Stats, 0, sizeof(gSwap.Stats));
gSwap.Stats.Enabled = 1;
gSwap.Stats.Enabled = true;
gSwap.Stats.TotalSlots = slotCount;
gSwap.Stats.FreeSlots = slotCount;
KeReleaseSpinlock(&gSwap.Lock, flags);
@@ -146,7 +147,7 @@ KSTATUS MmSwapInitialize(const char* relativePath, uint64_t sizeBytes) {
}
// Returns whether swap is available for pageout.
int MmSwapIsEnabled(void) {
bool MmSwapIsEnabled(void) {
return gSwap.Enabled;
}
@@ -193,7 +194,7 @@ KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame) {
return NTLSTATUS_NOT_SUPPORTED;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
int valid = slot < gSwap.SlotCount && KiSwapTestSlot(slot);
bool valid = slot < gSwap.SlotCount && KiSwapTestSlot(slot);
KeReleaseSpinlock(&gSwap.Lock, flags);
if (!valid) {
gSwap.Stats.PageInFailures++;
@@ -234,7 +235,7 @@ void MmSwapQueryStats(MM_SWAP_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
MM_SWAP_STATS stats = gSwap.Stats;
stats.Enabled = gSwap.Enabled ? 1 : 0;
stats.Enabled = gSwap.Enabled;
stats.TotalSlots = gSwap.SlotCount;
stats.UsedSlots = gSwap.UsedSlots;
stats.FreeSlots = gSwap.SlotCount >= gSwap.UsedSlots ? gSwap.SlotCount - gSwap.UsedSlots : 0;
code/kernel/mm_virt.c
+60 -58
View File
@@ -16,6 +16,8 @@
// User mappings:
// VADs separate reservation from commitment. Committed pages become resident
// when a user fault or kernel prefault path materializes them.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
@@ -42,12 +44,12 @@ static PHYSICAL_ADDRESS gVadZeroPageFrame;
// User/kernel copy boundary helpers.
// Implements the user Range Valid operation.
static int KiUserRangeValid(uintptr_t addr, size_t size) {
if (size == 0) return 1;
if (addr < USER_BASE) return 0;
if (addr >= USER_LIMIT) return 0;
if (size > (size_t)(USER_LIMIT - addr)) return 0;
return 1;
static bool KiUserRangeValid(uintptr_t addr, size_t size) {
if (size == 0) return true;
if (addr < USER_BASE) return false;
if (addr >= USER_LIMIT) return false;
if (size > (size_t)(USER_LIMIT - addr)) return false;
return true;
}
// Probes user Range and materializes demand-zero pages before the kernel copies through it.
@@ -300,7 +302,7 @@ static inline size_t KiAlignUpSize(size_t v, size_t align) {
}
// Returns whether two half-open address ranges intersect.
static int KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) {
static bool KiRangesOverlap(uintptr_t aBase, size_t aSize, uintptr_t bBase, size_t bSize) {
uintptr_t aEnd = aBase + aSize;
uintptr_t bEnd = bBase + bSize;
return aBase < bEnd && bBase < aEnd;
@@ -376,19 +378,19 @@ static MM_VAD* KiFindReservedVadContainingLocked(PROCESS* process, uintptr_t bas
}
// Returns whether every page in the aligned range is committed in the VAD metadata.
static int KiVadRangeCommitted(const MM_VAD* vad, uintptr_t alignedStart, uintptr_t alignedEnd) {
static bool KiVadRangeCommitted(const MM_VAD* vad, uintptr_t alignedStart, uintptr_t alignedEnd) {
if (!vad || alignedStart < vad->Base || alignedEnd > vad->Base + vad->PayloadSize)
return 0;
return false;
size_t first = KiVadPageIndex(vad, alignedStart);
size_t count = KiVadRangePageCount(alignedStart, alignedEnd);
if (first > vad->PageCount || count > vad->PageCount - first)
return 0;
return false;
for (size_t i = 0; i < count; ++i) {
if (vad->Pages[first + i].State != MmVadPageCommitted)
return 0;
return false;
}
return 1;
return true;
}
static uint64_t KiVadHardwarePageFlags(const MM_VAD_PAGE* page);
@@ -447,7 +449,7 @@ static void KiVadSyncSharedFilePageBeforeRelease(MM_VAD* vad, size_t index) {
return;
int wasDirty = MmPteTestAndClearDirty(va);
int wasWriteEnabled = (page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0;
bool wasWriteEnabled = (page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0;
if (wasDirty || wasWriteEnabled)
(void)KiVadMarkSharedFilePageDirty(vad, index, 0);
}
@@ -544,7 +546,7 @@ static KSTATUS KiVadCheckPageAccess(const MM_VAD_PAGE* page, int wantWrite, int
// concurrent faults from treating half-transitioned metadata as either resident or
// fully swapped.
static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) {
while (page && (_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0) &
while (page && (NtlAtomicLoadU32(&page->Flags) &
MM_VAD_PAGE_PAGING))
KeYield();
}
@@ -555,18 +557,18 @@ static void KiVadWaitForPagingComplete(MM_VAD_PAGE* page) {
// swapped/demand-zero page can both allocate/read/map it; one map fails and rolls
// metadata back to SWAPPED while the other has already published a valid PTE. That
// leaves stale or freed swap slots behind and corrupts user data under pressure.
static int KiVadTryBeginPaging(MM_VAD_PAGE* page) {
if (!page) return 0;
static bool KiVadTryBeginPaging(MM_VAD_PAGE* page) {
if (!page) return false;
while (true) {
uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0);
uint32_t oldFlags = (uint32_t)NtlAtomicLoadU32(&page->Flags);
if (oldFlags & MM_VAD_PAGE_PAGING)
return 0;
return false;
uint32_t newFlags = oldFlags | MM_VAD_PAGE_PAGING;
if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags,
(long)newFlags,
(long)oldFlags) == oldFlags)
return 1;
if ((uint32_t)NtlAtomicCompareExchangeU32(&page->Flags,
newFlags,
oldFlags) == oldFlags)
return true;
_mm_pause();
}
}
@@ -575,11 +577,11 @@ static void KiVadEndPaging(MM_VAD_PAGE* page) {
if (!page) return;
while (true) {
uint32_t oldFlags = (uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags, 0, 0);
uint32_t oldFlags = (uint32_t)NtlAtomicLoadU32(&page->Flags);
uint32_t newFlags = oldFlags & ~MM_VAD_PAGE_PAGING;
if ((uint32_t)_InterlockedCompareExchange((volatile long*)&page->Flags,
(long)newFlags,
(long)oldFlags) == oldFlags)
if ((uint32_t)NtlAtomicCompareExchangeU32(&page->Flags,
newFlags,
oldFlags) == oldFlags)
return;
_mm_pause();
}
@@ -1201,7 +1203,7 @@ static void KiVadRemoveUserBufferPageLock(PROCESS* process, MM_VAD* vad) {
__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process,
void* userAddress,
size_t size,
int writeAccess,
bool writeAccess,
MM_LOCKED_USER_BUFFER** out) {
if (!out)
return NTLSTATUS_INVALID_PARAMETER;
@@ -1939,73 +1941,73 @@ KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess,
// read-only mappings cannot have been modified, and the backing file can
// repopulate the page on the next fault. Writable/private file mappings still
// fall back to the swap path because this VMM does not yet track dirty bits.
static int KiVadPageCanDiscardCleanFile(const MM_VAD* vad, const MM_VAD_PAGE* page) {
static bool KiVadPageCanDiscardCleanFile(const MM_VAD* vad, const MM_VAD_PAGE* page) {
if (!vad || !page || vad->Type != MmVadMappedFile)
return 0;
return false;
if (!vad->FileObject || page->State != MmVadPageCommitted)
return 0;
return false;
if (!page->Resident || !page->Frame || !page->PageFlags)
return 0;
return false;
if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) == 0)
return 0;
return false;
if (page->PageFlags & PAGE_WRITE)
return 0;
return false;
if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME |
MM_VAD_PAGE_SWAPPED | MM_VAD_PAGE_PAGING))
return 0;
return 1;
return false;
return true;
}
static int KiVadPageCanDiscardCleanFileAfterPagingClaim(const MM_VAD* vad, const MM_VAD_PAGE* page) {
static bool KiVadPageCanDiscardCleanFileAfterPagingClaim(const MM_VAD* vad, const MM_VAD_PAGE* page) {
if (!vad || !page || vad->Type != MmVadMappedFile)
return 0;
return false;
if (!vad->FileObject || page->State != MmVadPageCommitted)
return 0;
return false;
if (!page->Resident || !page->Frame || !page->PageFlags)
return 0;
return false;
if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) == 0)
return 0;
return false;
if (page->PageFlags & PAGE_WRITE)
return 0;
return false;
if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME |
MM_VAD_PAGE_SWAPPED))
return 0;
return false;
return (page->Flags & MM_VAD_PAGE_PAGING) != 0;
}
// Returns whether a resident VAD page is safe to move to swap. Clean read-only
// file-backed pages are intentionally excluded; the pressure trimmer discards
// them and reloads them from the file instead of wasting swap slots.
static int KiVadPageCanSwap(const MM_VAD_PAGE* page) {
static bool KiVadPageCanSwap(const MM_VAD_PAGE* page) {
if (!page || page->State != MmVadPageCommitted)
return 0;
return false;
if (!page->Resident || !page->Frame || !page->PageFlags)
return 0;
return false;
if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) && ((page->PageFlags & PAGE_WRITE) == 0))
return 0;
return false;
if (page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE)
return 0;
return false;
if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME |
MM_VAD_PAGE_SWAPPED | MM_VAD_PAGE_PAGING))
return 0;
return 1;
return false;
return true;
}
static int KiVadPageCanSwapAfterPagingClaim(const MM_VAD_PAGE* page) {
static bool KiVadPageCanSwapAfterPagingClaim(const MM_VAD_PAGE* page) {
if (!page || page->State != MmVadPageCommitted)
return 0;
return false;
if (!page->Resident || !page->Frame || !page->PageFlags)
return 0;
return false;
if ((page->Flags & MM_VAD_PAGE_FILE_BACKED) && ((page->PageFlags & PAGE_WRITE) == 0))
return 0;
return false;
if (page->Flags & MM_VAD_PAGE_FILE_SHARED_WRITE)
return 0;
return false;
if (page->Flags & (MM_VAD_PAGE_COW | MM_VAD_PAGE_ZERO_PAGE | MM_VAD_PAGE_SHARED_FRAME |
MM_VAD_PAGE_SWAPPED))
return 0;
return false;
if ((page->Flags & MM_VAD_PAGE_PAGING) == 0)
return 0;
return 1;
return false;
return true;
}
// Discards one resident clean file-backed page. No swap slot is allocated; the
@@ -2215,7 +2217,7 @@ KSTATUS MmVadSyncFileMappings(FILE_OBJECT* file) {
uintptr_t va = vad->Base + i * PAGE_SIZE;
int wasDirty = MmPteTestAndClearDirty(va);
int wasWriteEnabled = (page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0;
bool wasWriteEnabled = (page->Flags & MM_VAD_PAGE_FILE_DIRTY_TRACK) == 0;
if (wasDirty || wasWriteEnabled)
(void)KiVadMarkSharedFilePageDirty(vad, i, 0);
code/kernel/ob_object.c
+9 -9
View File
@@ -1,5 +1,6 @@
// Object manager. This file owns reference-counted kernel objects and their terminal lifetime transitions.
#include <ntlite/atomic.h>
#include <stdbool.h>
#include <string.h>
#include <intrin.h>
@@ -26,7 +27,6 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
case ObjectTypeUsbPipe: return MM_POOL_TAG('u', 's', 'b', 'p');
case ObjectTypeFileMapping: return MM_POOL_TAG('m', 'a', 'p', 'f');
case ObjectTypeSurface: return MM_POOL_TAG('s', 'u', 'r', 'f');
case ObjectTypeInputQueue: return MM_POOL_TAG('i', 'n', 'p', 'q');
case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
case ObjectTypeTimer: return MM_POOL_TAG('t', 'm', 'r', ' ');
case ObjectTypeDisplayClock: return MM_POOL_TAG('d', 'c', 'l', 'k');
@@ -73,16 +73,16 @@ __declspec(dllexport) bool ObTryReferenceObject(OBJECT* object) {
return false;
while (true) {
if (_InterlockedCompareExchange(&object->LifetimeState, 0, 0) != ObjectAlive)
if (NtlAtomicLoad32(&object->LifetimeState) != ObjectAlive)
return false;
if ((_InterlockedCompareExchange(&object->Flags, 0, 0) & OBJECT_FLAG_INSERTED) == 0)
if ((NtlAtomicLoadU32(&object->Flags) & OBJECT_FLAG_INSERTED) == 0)
return false;
long refs = _InterlockedCompareExchange(&object->RefCount, 0, 0);
int32_t refs = NtlAtomicLoad32(&object->RefCount);
if (refs <= 0 || refs == 0x7fffffffL)
return false;
if (_InterlockedCompareExchange(&object->RefCount, refs + 1, refs) == refs)
if (NtlAtomicCompareExchange32(&object->RefCount, refs + 1, refs) == refs)
return true;
}
}
@@ -108,7 +108,7 @@ __declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
if (!object)
return;
long refs = _InterlockedDecrement(&object->RefCount);
int32_t refs = NtlAtomicDecrement32(&object->RefCount);
if (refs > 0)
return;
if (refs < 0) {
@@ -121,7 +121,7 @@ __declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
(unsigned)object->Flags);
}
if (_InterlockedCompareExchange(&object->LifetimeState, ObjectDying, ObjectAlive) != ObjectAlive) {
if (NtlAtomicCompareExchange32(&object->LifetimeState, ObjectDying, ObjectAlive) != ObjectAlive) {
KePanic(KE_PANIC_CODE_GENERIC,
"object: duplicate terminal transition object=0x%016llX type=%u state=%ld flags=0x%08x\n",
(uint64_t)(uintptr_t)object,
@@ -130,10 +130,10 @@ __declspec(dllexport) void ObDereferenceObject(OBJECT* object) {
(unsigned)object->Flags);
}
_InterlockedExchange(&object->Flags, OBJECT_FLAG_UNLINKED);
NtlAtomicExchangeU32(&object->Flags, OBJECT_FLAG_UNLINKED);
if (object->Destroy)
object->Destroy(object);
_InterlockedExchange(&object->LifetimeState, ObjectDead);
NtlAtomicExchange32(&object->LifetimeState, ObjectDead);
MmFreePool(object);
}
code/public/freestanding/intrin.h
+17
View File
@@ -2,20 +2,37 @@
#pragma once
#include <stdint.h>
unsigned __int64 __readgsqword(unsigned long offset);
long _InterlockedCompareExchange(long volatile* destination, long exchange, long comparand);
long _InterlockedExchange(long volatile* target, long value);
long _InterlockedIncrement(long volatile* addend);
long _InterlockedDecrement(long volatile* addend);
__int64 _InterlockedCompareExchange64(__int64 volatile* destination, __int64 exchange, __int64 comparand);
__int64 _InterlockedIncrement64(__int64 volatile* addend);
__int64 _InterlockedDecrement64(__int64 volatile* addend);
__int64 _InterlockedExchangeAdd64(__int64 volatile* addend, __int64 value);
void* _InterlockedCompareExchangePointer(void* volatile* destination, void* exchange, void* comparand);
void* _InterlockedExchangePointer(void* volatile* target, void* value);
void _ReadWriteBarrier(void);
void _mm_pause(void);
_Static_assert(sizeof(long) == sizeof(int32_t), "MSVC long must be 32-bit");
_Static_assert(sizeof(__int64) == sizeof(int64_t), "MSVC __int64 must be 64-bit");
#ifdef _MSC_VER
#pragma intrinsic(__readgsqword)
#pragma intrinsic(_InterlockedCompareExchange)
#pragma intrinsic(_InterlockedExchange)
#pragma intrinsic(_InterlockedIncrement)
#pragma intrinsic(_InterlockedDecrement)
#pragma intrinsic(_InterlockedCompareExchange64)
#pragma intrinsic(_InterlockedIncrement64)
#pragma intrinsic(_InterlockedDecrement64)
#pragma intrinsic(_InterlockedExchangeAdd64)
#pragma intrinsic(_InterlockedCompareExchangePointer)
#pragma intrinsic(_InterlockedExchangePointer)
#pragma intrinsic(_ReadWriteBarrier)
#pragma intrinsic(_mm_pause)
#endif
code/public/freestanding/stddef.h
+4
View File
@@ -10,5 +10,9 @@ typedef __int64 ptrdiff_t;
#endif
#ifndef offsetof
#if defined(__clang__) || defined(__GNUC__)
#define offsetof(type, member) __builtin_offsetof(type, member)
#else
#define offsetof(type, member) ((size_t)&(((type*)0)->member))
#endif
#endif
code/public/ntlite/atomic.h
+98
View File
@@ -0,0 +1,98 @@
// Fixed-width atomic operations for nt-lite-owned storage.
#pragma once
#include <stdint.h>
#include <intrin.h>
_Static_assert(sizeof(long) == sizeof(int32_t), "MSVC long must be 32-bit");
_Static_assert(sizeof(__int64) == sizeof(int64_t), "MSVC __int64 must be 64-bit");
// Atomically compares and exchanges a signed 32-bit value.
static __forceinline int32_t NtlAtomicCompareExchange32(volatile int32_t* destination,
int32_t exchange,
int32_t comparand) {
return (int32_t)_InterlockedCompareExchange((volatile long*)destination,
(long)exchange,
(long)comparand);
}
// Atomically loads a signed 32-bit value.
static __forceinline int32_t NtlAtomicLoad32(const volatile int32_t* source) {
return (int32_t)_InterlockedCompareExchange((volatile long*)source, 0, 0);
}
// Atomically compares and exchanges an unsigned 32-bit value.
static __forceinline uint32_t NtlAtomicCompareExchangeU32(volatile uint32_t* destination,
uint32_t exchange,
uint32_t comparand) {
return (uint32_t)_InterlockedCompareExchange((volatile long*)destination,
(long)exchange,
(long)comparand);
}
// Atomically loads an unsigned 32-bit value.
static __forceinline uint32_t NtlAtomicLoadU32(const volatile uint32_t* source) {
return (uint32_t)_InterlockedCompareExchange((volatile long*)source, 0, 0);
}
// Atomically exchanges an unsigned 32-bit value.
static __forceinline uint32_t NtlAtomicExchangeU32(volatile uint32_t* destination, uint32_t value) {
return (uint32_t)_InterlockedExchange((volatile long*)destination, (long)value);
}
// Atomically exchanges a signed 32-bit value.
static __forceinline int32_t NtlAtomicExchange32(volatile int32_t* destination, int32_t value) {
return (int32_t)_InterlockedExchange((volatile long*)destination, (long)value);
}
// Atomically increments a signed 32-bit value.
static __forceinline int32_t NtlAtomicIncrement32(volatile int32_t* destination) {
return (int32_t)_InterlockedIncrement((volatile long*)destination);
}
// Atomically decrements a signed 32-bit value.
static __forceinline int32_t NtlAtomicDecrement32(volatile int32_t* destination) {
return (int32_t)_InterlockedDecrement((volatile long*)destination);
}
// Atomically compares and exchanges a signed 64-bit value.
static __forceinline int64_t NtlAtomicCompareExchange64(volatile int64_t* destination,
int64_t exchange,
int64_t comparand) {
return (int64_t)_InterlockedCompareExchange64((volatile __int64*)destination,
(__int64)exchange,
(__int64)comparand);
}
// Atomically loads a signed 64-bit value.
static __forceinline int64_t NtlAtomicLoad64(const volatile int64_t* source) {
return (int64_t)_InterlockedCompareExchange64((volatile __int64*)source, 0, 0);
}
// Atomically increments a signed 64-bit value.
static __forceinline int64_t NtlAtomicIncrement64(volatile int64_t* destination) {
return (int64_t)_InterlockedIncrement64((volatile __int64*)destination);
}
// Atomically decrements a signed 64-bit value.
static __forceinline int64_t NtlAtomicDecrement64(volatile int64_t* destination) {
return (int64_t)_InterlockedDecrement64((volatile __int64*)destination);
}
// Atomically adds to a signed 64-bit value and returns the previous value.
static __forceinline int64_t NtlAtomicExchangeAdd64(volatile int64_t* destination, int64_t value) {
return (int64_t)_InterlockedExchangeAdd64((volatile __int64*)destination, (__int64)value);
}
// Atomically compares and exchanges a pointer value.
static __forceinline void* NtlAtomicCompareExchangePointer(void* volatile* destination,
void* exchange,
void* comparand) {
return _InterlockedCompareExchangePointer((void* volatile*)destination, exchange, comparand);
}
// Atomically exchanges a pointer value.
static __forceinline void* NtlAtomicExchangePointer(void* volatile* destination, void* value) {
return _InterlockedExchangePointer((void* volatile*)destination, value);
}
code/public/ntlite/audio.h
+6 -5
View File
@@ -3,10 +3,11 @@
#pragma once
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/status.h>
#include <ntlite/handle.h>
#define SYS_AUDIO_ABI_VERSION 4u
#define SYS_AUDIO_ABI_VERSION 5u
#define SYS_AUDIO_MAX_ENDPOINTS 32u
#define SYS_AUDIO_STREAM_CAPACITY_FRAMES 4096u
@@ -64,7 +65,7 @@ typedef struct SYS_AUDIO_ENDPOINT_INFO {
uint32_t PeriodCount;
uint32_t FramesPerPeriod;
uint32_t MasterVolumeQ16;
uint32_t MasterMuted;
bool MasterMuted;
KSTATUS FailureStatus;
uint32_t Reserved;
} SYS_AUDIO_ENDPOINT_INFO;
@@ -157,7 +158,7 @@ typedef struct SYS_AUDIO_STREAM_INFO {
uint64_t DiscontinuityGeneration;
uint32_t VolumeQ16;
int32_t PanQ16;
uint8_t Muted;
bool Muted;
uint8_t ReservedBytes[7];
} SYS_AUDIO_STREAM_INFO;
@@ -167,7 +168,7 @@ typedef struct SYS_AUDIO_SET_VOLUME_REQUEST {
uint32_t Size;
uint32_t Flags;
SYS_HANDLE StreamHandle;
uint32_t Muted;
bool Muted;
uint32_t VolumeQ16;
int32_t PanQ16;
} SYS_AUDIO_SET_VOLUME_REQUEST;
@@ -177,7 +178,7 @@ typedef struct SYS_AUDIO_SET_MASTER_VOLUME_REQUEST {
uint32_t Size;
uint32_t Flags;
uint32_t EndpointId;
uint32_t Muted;
bool Muted;
uint32_t VolumeQ16;
} SYS_AUDIO_SET_MASTER_VOLUME_REQUEST;
code/public/ntlite/boot_info.h
+14
View File
@@ -2,6 +2,7 @@
#pragma once
#include <stddef.h>
#include <stdint.h>
#include <ntlite/memory_map.h>
@@ -97,3 +98,16 @@ typedef struct BOOT_INFO {
// Firmware wall-clock snapshot taken before ExitBootServices.
BOOT_TIME BootTime;
} BOOT_INFO;
// BOOT_INFO is an x64 firmware-to-kernel binary handoff, not a private C structure.
_Static_assert(sizeof(uintptr_t) == 8u, "BOOT_INFO requires x64 pointers");
_Static_assert(sizeof(BOOT_TIME) == 20u, "BOOT_TIME ABI size changed");
_Static_assert(sizeof(BOOT_BLOB) == 16u, "BOOT_BLOB ABI size changed");
_Static_assert(sizeof(BOOT_FILE) == 80u, "BOOT_FILE ABI size changed");
_Static_assert(sizeof(BOOT_FILE_LIST) == 16u, "BOOT_FILE_LIST ABI size changed");
_Static_assert(sizeof(BOOT_IMAGE) == 32u, "BOOT_IMAGE ABI size changed");
_Static_assert(offsetof(BOOT_INFO, MemoryMap) == 16u, "BOOT_INFO MemoryMap offset changed");
_Static_assert(offsetof(BOOT_INFO, KernelImage) == 6200u, "BOOT_INFO KernelImage offset changed");
_Static_assert(offsetof(BOOT_INFO, BootDrivers) == 6280u, "BOOT_INFO BootDrivers offset changed");
_Static_assert(offsetof(BOOT_INFO, BootTime) == 6320u, "BOOT_INFO BootTime offset changed");
_Static_assert(sizeof(BOOT_INFO) == 6344u, "BOOT_INFO ABI size changed");
code/public/ntlite/console_service.h
+17 -6
View File
@@ -55,7 +55,7 @@ typedef struct NTL_CONSOLE_REQUEST {
int32_t Y;
} Cursor;
struct {
char Character;
char Character; // One legacy ASCII display cell, never one UTF-8 code point.
uint8_t Reserved[3];
int32_t X;
int32_t Y;
@@ -69,7 +69,7 @@ typedef struct NTL_CONSOLE_REQUEST {
uint32_t Length;
} FillAttribute;
struct {
char Character;
char Character; // One legacy ASCII display cell, never one UTF-8 code point.
uint8_t Reserved[3];
int32_t X;
int32_t Y;
@@ -93,6 +93,14 @@ typedef struct NTL_CONSOLE_REPLY {
NTL_CONSOLE_INFO Info;
} NTL_CONSOLE_REPLY;
// RSSRV console IPC layout must agree with every native console client.
_Static_assert(sizeof(NTL_CONSOLE_INFO) == 48u, "NTL_CONSOLE_INFO ABI size changed");
_Static_assert(offsetof(NTL_CONSOLE_INFO, Generation) == 40u, "NTL_CONSOLE_INFO generation offset changed");
_Static_assert(sizeof(NTL_CONSOLE_REQUEST) == 544u, "NTL_CONSOLE_REQUEST ABI size changed");
_Static_assert(offsetof(NTL_CONSOLE_REQUEST, Data) == 24u, "NTL_CONSOLE_REQUEST payload offset changed");
_Static_assert(sizeof(NTL_CONSOLE_REPLY) == 80u, "NTL_CONSOLE_REPLY ABI size changed");
_Static_assert(offsetof(NTL_CONSOLE_REPLY, Status) == 16u, "NTL_CONSOLE_REPLY status offset changed");
#if defined(NTL_SYSCALL_STATIC)
#define NTL_CONSOLE_API
#elif defined(NTL_SYSCALL_BUILD)
@@ -105,12 +113,14 @@ NTL_CONSOLE_API KSTATUS NtlConsoleConnect(SYS_HANDLE* sessionOut);
NTL_CONSOLE_API KSTATUS NtlConsoleClose(SYS_HANDLE session);
NTL_CONSOLE_API KSTATUS NtlConsoleWrite(SYS_HANDLE session, const void* data, size_t length, size_t* writtenOut);
NTL_CONSOLE_API KSTATUS NtlConsoleQueryInfo(SYS_HANDLE session, NTL_CONSOLE_INFO* infoOut);
NTL_CONSOLE_API KSTATUS NtlConsoleSetCursor(SYS_HANDLE session, int x, int y);
NTL_CONSOLE_API KSTATUS NtlConsoleFillCharacter(SYS_HANDLE session, char character, int x, int y,
NTL_CONSOLE_API KSTATUS NtlConsoleSetCursor(SYS_HANDLE session, int32_t x, int32_t y);
// Legacy cell operations accept ASCII glyph codes only. Unicode text uses
// NtlConsoleWrite's byte stream until the RSSRV text renderer is upgraded.
NTL_CONSOLE_API KSTATUS NtlConsoleFillCharacter(SYS_HANDLE session, char character, int32_t x, int32_t y,
size_t length, size_t* writtenOut);
NTL_CONSOLE_API KSTATUS NtlConsoleFillAttribute(SYS_HANDLE session, uint16_t attribute, int x, int y,
NTL_CONSOLE_API KSTATUS NtlConsoleFillAttribute(SYS_HANDLE session, uint16_t attribute, int32_t x, int32_t y,
size_t length, size_t* writtenOut);
NTL_CONSOLE_API KSTATUS NtlConsolePutCharacter(SYS_HANDLE session, char character, int x, int y,
NTL_CONSOLE_API KSTATUS NtlConsolePutCharacter(SYS_HANDLE session, char character, int32_t x, int32_t y,
uint32_t foreground, uint32_t background);
NTL_CONSOLE_API KSTATUS NtlConsoleClear(SYS_HANDLE session, uint32_t color);
NTL_CONSOLE_API KSTATUS NtlConsoleReadInput(SYS_HANDLE session, uint64_t timeoutMilliseconds,
@@ -118,4 +128,5 @@ NTL_CONSOLE_API KSTATUS NtlConsoleReadInput(SYS_HANDLE session, uint64_t timeout
NTL_CONSOLE_API KSTATUS NtlConsolePeekInput(SYS_HANDLE session, uint64_t timeoutMilliseconds,
INPUT_KEY_RECORD* recordOut);
NTL_CONSOLE_API KSTATUS NtlConsoleFlushInput(SYS_HANDLE session);
// ASCII-only legacy key convenience function; use NtlConsoleReadInput for key events.
NTL_CONSOLE_API KSTATUS NtlConsoleReadCharacter(SYS_HANDLE session, char* characterOut);
code/public/ntlite/ddi/display.h
+10 -10
View File
@@ -24,26 +24,26 @@ typedef struct DISPLAY_DEVICE_OPS {
KSTATUS (*GetMode)(DISPLAY_DEVICE* device, DISPLAY_MODE* outMode);
KSTATUS (*Blit)(DISPLAY_DEVICE* device,
int x, int y, int w, int h,
int32_t x, int32_t y, int32_t w, int32_t h,
const uint32_t* pixels,
uint32_t pitchBytes);
KSTATUS (*PresentPitched)(DISPLAY_DEVICE* device,
int x, int y, int w, int h,
int32_t x, int32_t y, int32_t w, int32_t h,
const uint32_t* pixels,
uint32_t pitchBytes);
KSTATUS (*FillRect)(DISPLAY_DEVICE* device,
int x, int y, int w, int h,
int32_t x, int32_t y, int32_t w, int32_t h,
uint32_t color);
KSTATUS (*CopyRect)(DISPLAY_DEVICE* device,
int dstX, int dstY,
int srcX, int srcY,
int w, int h);
int32_t dstX, int32_t dstY,
int32_t srcX, int32_t srcY,
int32_t w, int32_t h);
KSTATUS (*ReadRect)(DISPLAY_DEVICE* device,
int x, int y, int w, int h,
int32_t x, int32_t y, int32_t w, int32_t h,
uint32_t* outPixels, uint32_t outStride);
KSTATUS (*Present)(DISPLAY_DEVICE* device,
@@ -53,11 +53,11 @@ typedef struct DISPLAY_DEVICE_OPS {
KSTATUS (*Sync)(DISPLAY_DEVICE* device);
KSTATUS (*SetCursorImage)(DISPLAY_DEVICE* device,
int width, int height,
int hotX, int hotY,
int32_t width, int32_t height,
int32_t hotX, int32_t hotY,
const uint32_t* argbPixels);
KSTATUS (*SetCursorPosition)(DISPLAY_DEVICE* device, int x, int y);
KSTATUS (*SetCursorPosition)(DISPLAY_DEVICE* device, int32_t x, int32_t y);
KSTATUS (*ShowCursor)(DISPLAY_DEVICE* device, bool show);
KSTATUS (*QueryStats)(DISPLAY_DEVICE* device, DISPLAY_STATS* outStats);
code/public/ntlite/ddi/input_device.h
+1 -1
View File
@@ -10,4 +10,4 @@ __declspec(dllexport) KSTATUS KeInputRegisterDevice(const char* name, uint32_t f
__declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event);
__declspec(dllexport) void KeInputSubmitMouse(const INPUT_MOUSE_EVENT* event);
__declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE_EVENT* event);
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons);
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int32_t wheel, uint8_t buttons);
code/public/ntlite/ddi/io_driver.h
+1 -1
View File
@@ -97,7 +97,7 @@ struct DEVICE_OBJECT {
uint32_t AlignmentRequirement;
SPINLOCK Lock;
volatile long OutstandingIrps;
volatile int32_t OutstandingIrps;
bool DeletePending;
SYS_STRING_VIEW Name;
code/public/ntlite/ddk/async.h
+2 -1
View File
@@ -4,6 +4,7 @@
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/status.h>
#include <ntlite/ddk/list.h>
#include <ntlite/ddk/object.h>
@@ -36,7 +37,7 @@ struct IO_ASYNC_REQUEST_OBJECT {
OBJECT Header;
LIST_ENTRY Link;
IO_ASYNC_REQUEST_STATE State;
uint8_t CancelRequested;
bool CancelRequested;
struct EVENT* WaitEvent;
struct EVENT* CompletionEvent;
struct KAPC* CompletionApc;
code/public/ntlite/ddk/audio.h
+4 -3
View File
@@ -4,6 +4,7 @@
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include <ntlite/status.h>
struct DEVICE_OBJECT;
@@ -52,7 +53,7 @@ typedef struct IO_AUDIO_PROVIDER_OPERATIONS {
// Stops DMA before unregister permits provider storage to be released.
void (*Stop)(void* providerContext);
// Applies endpoint gain and mute only with IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN.
KSTATUS (*SetGain)(void* providerContext, int32_t gainMilliDb, uint8_t muted);
KSTATUS (*SetGain)(void* providerContext, int32_t gainMilliDb, bool muted);
} IO_AUDIO_PROVIDER_OPERATIONS;
// Registers one fixed-format render ring owned by a hardware provider.
@@ -61,7 +62,7 @@ typedef struct IO_AUDIO_ENDPOINT_REGISTRATION {
struct DEVICE_OBJECT* Device;
IO_AUDIO_ENDPOINT_KIND Kind;
uint32_t Capabilities;
uint8_t InitiallyAvailable;
bool InitiallyAvailable;
void* ProviderContext;
IO_AUDIO_PROVIDER_OPERATIONS Operations;
void* PeriodBuffer;
@@ -96,7 +97,7 @@ typedef struct NTL_DDK_AUDIO {
uint64_t timestampTicks);
// Updates jack-derived endpoint availability without changing endpoint identity.
void (*SetEndpointAvailable)(IO_AUDIO_ENDPOINT* endpoint, uint8_t available);
void (*SetEndpointAvailable)(IO_AUDIO_ENDPOINT* endpoint, bool available);
// Fails an endpoint and wakes every stream owner.
void (*ReportEndpointFailure)(IO_AUDIO_ENDPOINT* endpoint,
Loaded 100 of 226 files, more files were not shown because too many files have changed in this diff. Show more