maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implement proper IRQL, DPCs, kernel and user APCs, alertable waits, async file I/O with cancellation, trap frames, selftests, a real driver/device model, native async FAT32, VFAT dotfile fixes, syscall ABI cleanup, async_apc and async_cancel tests, POSIX multi-volume paths, and pimp the readme.

Browse Source
This commit is contained in:
maksw committed 2026-05-19 20:56:15 +02:00
1 parent f5fd7bdd75
commit 529f816166
75 files changed
+9969 -5675
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/boot_info.h
+1 -1
View File
@@ -40,5 +40,5 @@ typedef struct BOOT_INFO {
uintptr_t PhysicalMapBase;
/* From boot.ini */
uint32_t DebugEnabled;
uint32_t Debug;
} BOOT_INFO;
code/kernel/io_device.c
+114
View File
@@ -0,0 +1,114 @@
// io_device.c - kernel driver/device dispatch and disk registry
#include <string.h>
#include <io_driver.h>
#include <ke_irp.h>
#include <mm_virt.h>
#define MAX_DISK_DEVICES 8
static DEVICE_OBJECT* gDiskDevices[MAX_DISK_DEVICES];
__declspec(dllexport) KSTATUS IoCreateDevice(DRIVER_OBJECT* driver,
size_t extensionSize,
DEVICE_OBJECT** outDevice) {
if (!driver || !outDevice) return STATUS_INVALID_PARAMETER;
size_t total = sizeof(DEVICE_OBJECT) + extensionSize;
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)MmAllocatePool(total);
if (!dev) return STATUS_NO_MEMORY;
memset(dev, 0, total);
dev->DriverObject = driver;
if (extensionSize)
dev->DeviceExtension = (void*)((uint8_t*)dev + sizeof(DEVICE_OBJECT));
dev->AlignmentRequirement = 1;
dev->DiskIndex = 0xFF;
*outDevice = dev;
return STATUS_SUCCESS;
}
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
MmFreePool(device);
}
__declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
DEVICE_OBJECT* device) {
if (driveIndex >= MAX_DISK_DEVICES || !device || !device->DriverObject)
return STATUS_INVALID_PARAMETER;
if (gDiskDevices[driveIndex])
return STATUS_ALREADY_EXISTS;
device->Flags |= DEVICE_OBJECT_DISK;
device->DiskIndex = driveIndex;
gDiskDevices[driveIndex] = device;
return STATUS_SUCCESS;
}
__declspec(dllexport) DEVICE_OBJECT* IoGetDiskDevice(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES) return (DEVICE_OBJECT*)0;
return gDiskDevices[driveIndex];
}
__declspec(dllexport) bool IoDiskDevicePresent(uint8_t driveIndex) {
return IoGetDiskDevice(driveIndex) != (DEVICE_OBJECT*)0;
}
__declspec(dllexport) uint64_t IoDiskSectorCount(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoGetDiskDevice(driveIndex);
return dev ? dev->SectorCount : 0;
}
__declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
if (!device || !device->DriverObject || !irp)
return STATUS_INVALID_PARAMETER;
IO_MAJOR_FUNCTION major = (irp->Flags & IRP_FLAG_WRITE) ? IRP_MJ_WRITE : IRP_MJ_READ;
if (major >= IRP_MJ_MAXIMUM_FUNCTION)
return STATUS_INVALID_PARAMETER;
PDRIVER_DISPATCH dispatch = device->DriverObject->MajorFunction[major];
if (!dispatch)
return STATUS_NOT_SUPPORTED;
return dispatch(device, irp);
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
void* buffer) {
if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
memset(&irp, 0, sizeof(irp));
irp.DriveIndex = driveIndex;
irp.Flags = 0;
irp.Lba = lba;
irp.SectorCount = count;
irp.Buffer = buffer;
irp.Status = STATUS_FAILURE;
irp.Information = 0;
return IoSubmitIrpSync(&irp);
}
__declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
const void* buffer) {
if (!buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
memset(&irp, 0, sizeof(irp));
irp.DriveIndex = driveIndex;
irp.Flags = IRP_FLAG_WRITE;
irp.Lba = lba;
irp.SectorCount = count;
irp.Buffer = (void*)buffer;
irp.Status = STATUS_FAILURE;
irp.Information = 0;
return IoSubmitIrpSync(&irp);
}
code/kernel/io_file.c
+500 -52
View File
@@ -6,6 +6,12 @@
#include <mm_virt.h>
#include <ob_object.h>
#include <ke_userenv.h>
#include <ke_irql.h>
#include <ke_event.h>
#include <ke_apc.h>
#include <ke_semaphore.h>
#include <ke_thread.h>
#include <ke_sched.h>
#define MAX_DRIVES 8
@@ -17,11 +23,389 @@ typedef struct {
static VOLUME_ENTRY gVolumes[MAX_DRIVES];
// ---------------------------------------------------------------------------
// Current Working Directory (drive 0 relative, no trailing backslash)
// e.g. "" = root, "apps" = root\apps, "apps\\data" = nested
// Async file I/O worker
// ---------------------------------------------------------------------------
typedef enum IO_ASYNC_FILE_OP {
IoAsyncFileRead = 0,
IoAsyncFileWrite = 1,
} IO_ASYNC_FILE_OP;
typedef enum IO_ASYNC_FILE_STATE {
IoAsyncStateQueued = 0,
IoAsyncStateRunning = 1,
IoAsyncStateCompleted = 2,
IoAsyncStateCancelled = 3,
} IO_ASYNC_FILE_STATE;
struct IO_ASYNC_FILE_REQUEST_OBJECT {
OBJECT Header; // ObjectTypeAsyncIo
LIST_ENTRY Link; // protected by gAsyncFileQueueLock
IO_ASYNC_FILE_OP Op;
IO_ASYNC_FILE_STATE State;
uint8_t CancelRequested;
FILE_OBJECT* File; // referenced until object destruction
EVENT* CompletionEvent; // optional, referenced until object destruction
KAPC* CompletionApc; // caller-owned APC storage
IO_STATUS_BLOCK* Iosb; // caller/user-owned completion block
void* Buffer; // caller/user-owned; must remain valid until completion/cancel
size_t Size;
uint64_t Offset;
};
static SPINLOCK gAsyncFileInitLock = SPINLOCK_INIT;
static int gAsyncFileInitState; // 0=off, 1=starting, 2=ready
static SPINLOCK gAsyncFileQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gAsyncFileQueue;
static SEMAPHORE* gAsyncFileSemaphore;
static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header);
if (req->CompletionEvent) {
ObDereferenceObject(&req->CompletionEvent->Header);
req->CompletionEvent = (EVENT*)0;
}
if (req->File) {
ObDereferenceObject(&req->File->Header);
req->File = (FILE_OBJECT*)0;
}
}
static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
KSTATUS status,
size_t information) {
if (!req) return;
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
req->State = (status == STATUS_CANCELLED) ? IoAsyncStateCancelled : IoAsyncStateCompleted;
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
if (req->Iosb) {
req->Iosb->Status = status;
req->Iosb->Information = information;
}
// Match IRP completion ordering: queue APC first, then signal the event.
if (req->CompletionApc) {
if (!KeInsertQueueApc(req->CompletionApc, req->Iosb, (void*)(uintptr_t)status)) {
if (req->CompletionApc->RundownRoutine)
req->CompletionApc->RundownRoutine(req->CompletionApc);
}
}
if (req->CompletionEvent)
KeSignalEvent(req->CompletionEvent);
}
static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
LIST_ENTRY* node = ListPopHead(&gAsyncFileQueue);
if (!node) {
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
return (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
}
IO_ASYNC_FILE_REQUEST_OBJECT* req =
CONTAINING_RECORD(node, IO_ASYNC_FILE_REQUEST_OBJECT, Link);
if (req->CancelRequested) {
req->State = IoAsyncStateCancelled;
} else {
req->State = IoAsyncStateRunning;
}
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
return req;
}
static void KiIoAsyncFileWorker(void* arg) {
(void)arg;
for (;;) {
KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore);
if (!KSUCCESS(s)) {
KeYield();
continue;
}
IO_ASYNC_FILE_REQUEST_OBJECT* req = KiIoAsyncFileDequeue();
if (!req) continue;
if (req->CancelRequested || req->State == IoAsyncStateCancelled) {
KiIoCompleteAsyncFileRequest(req, STATUS_CANCELLED, 0);
ObDereferenceObject(&req->Header); // queue/worker reference
continue;
}
size_t done = 0;
if (req->Op == IoAsyncFileRead) {
s = IoReadFileAt(req->File, req->Offset, req->Buffer,
req->Size, &done);
} else {
s = IoWriteFileAt(req->File, req->Offset, req->Buffer,
req->Size, &done);
}
if (req->CancelRequested) {
// The operation may already have reached the filesystem. The
// completion contract still reports cancellation so waiters/APCs do
// not mistake a cancelled request for useful data.
s = STATUS_CANCELLED;
done = 0;
}
KiIoCompleteAsyncFileRequest(req, s, done);
ObDereferenceObject(&req->Header); // queue/worker reference
}
}
static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileInitLock);
if (gAsyncFileInitState == 2) {
KeReleaseSpinlock(&gAsyncFileInitLock, flags);
return STATUS_SUCCESS;
}
if (gAsyncFileInitState == 1) {
KeReleaseSpinlock(&gAsyncFileInitLock, flags);
while (gAsyncFileInitState == 1)
KeYield();
return (gAsyncFileInitState == 2) ? STATUS_SUCCESS : STATUS_FAILURE;
}
gAsyncFileInitState = 1;
KeReleaseSpinlock(&gAsyncFileInitLock, flags);
LIST_INIT(&gAsyncFileQueue);
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker,
0,
PriorityNormal,
&worker);
// Keep the creator reference: this worker is intentionally persistent
// for the lifetime of the kernel.
(void)worker;
}
flags = KeAcquireSpinlock(&gAsyncFileInitLock);
gAsyncFileInitState = KSUCCESS(s) ? 2 : 0;
KeReleaseSpinlock(&gAsyncFileInitLock, flags);
return s;
}
static void KiIoNativeFsAsyncComplete(void* completeContext,
KSTATUS status,
size_t information) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
(IO_ASYNC_FILE_REQUEST_OBJECT*)completeContext;
if (!req) return;
if (req->CancelRequested) {
status = STATUS_CANCELLED;
information = 0;
}
KiIoCompleteAsyncFileRequest(req, status, information);
ObDereferenceObject(&req->Header); // native filesystem provider reference
}
static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
FILE_OBJECT* file,
void* buffer,
size_t size,
uint64_t offset,
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
if (!file || !iosb) return STATUS_INVALID_PARAMETER;
if (!buffer && size) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
iosb->Status = STATUS_PENDING;
iosb->Information = 0;
IO_ASYNC_FILE_REQUEST_OBJECT* req =
(IO_ASYNC_FILE_REQUEST_OBJECT*)ObAllocateObject(ObjectTypeAsyncIo,
sizeof(IO_ASYNC_FILE_REQUEST_OBJECT));
if (!req) {
iosb->Status = STATUS_NO_MEMORY;
return STATUS_NO_MEMORY;
}
req->Header.Destroy = KiIoAsyncFileRequestDestroy;
LIST_INIT(&req->Link);
req->Op = op;
req->State = IoAsyncStateQueued;
req->CancelRequested = 0;
req->File = file;
req->CompletionEvent = completionEvent;
req->CompletionApc = completionApc;
req->Iosb = iosb;
req->Buffer = buffer;
req->Size = size;
req->Offset = offset;
ObReferenceObject(&file->Header);
if (completionEvent)
ObReferenceObject(&completionEvent->Header);
IO_FS_ASYNC_COMPLETE complete = KiIoNativeFsAsyncComplete;
KSTATUS ns = STATUS_NOT_SUPPORTED;
bool hasNative = (op == IoAsyncFileRead) ? (file->Fs->ReadAsync != 0)
: (file->Fs->WriteAsync != 0);
if (hasNative) {
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
req->State = IoAsyncStateRunning;
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
ObReferenceObject(&req->Header); // native filesystem provider reference
if (op == IoAsyncFileRead) {
ns = file->Fs->ReadAsync(file->Fs, file->FileCtx, offset, buffer,
size, complete, req);
} else {
ns = file->Fs->WriteAsync(file->Fs, file->FileCtx, offset, buffer,
size, complete, req);
}
if (ns == STATUS_PENDING) {
if (requestOut) {
*requestOut = req;
} else {
ObDereferenceObject(&req->Header);
}
return STATUS_PENDING;
}
ObDereferenceObject(&req->Header); // provider rejected synchronously
iosb->Status = ns;
ObDereferenceObject(&req->Header); // creator reference
return ns;
}
KSTATUS s = KiIoEnsureAsyncFileIoStarted();
if (!KSUCCESS(s)) {
iosb->Status = s;
ObDereferenceObject(&req->Header);
return s;
}
// Queue ownership keeps the request alive until the worker completes or a
// successful queued cancellation completes it from IoCancelAsyncFileRequest.
ObReferenceObject(&req->Header);
uintptr_t qflags = KeAcquireSpinlock(&gAsyncFileQueueLock);
ListInsertTail(&gAsyncFileQueue, &req->Link);
KeReleaseSpinlock(&gAsyncFileQueueLock, qflags);
KeSignalSemaphore(gAsyncFileSemaphore);
if (requestOut) {
*requestOut = req; // creator reference returned to caller
} else {
ObDereferenceObject(&req->Header);
}
return STATUS_PENDING;
}
__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf,
size_t size, uint64_t offset,
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset,
iosb, completionEvent, completionApc,
requestOut);
}
__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
size_t size, uint64_t offset,
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset,
iosb, completionEvent, completionApc,
requestOut);
}
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? &req->Header : (OBJECT*)0;
}
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
if (!req) return STATUS_INVALID_PARAMETER;
IO_ASYNC_FILE_STATE state;
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
state = req->State;
if (state == IoAsyncStateQueued) {
ListRemove(&req->Link);
req->CancelRequested = 1;
req->State = IoAsyncStateCancelled;
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
KiIoCompleteAsyncFileRequest(req, STATUS_CANCELLED, 0);
ObDereferenceObject(&req->Header); // queue reference removed above
return STATUS_SUCCESS;
}
if (state == IoAsyncStateRunning) {
req->CancelRequested = 1;
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
return STATUS_PENDING;
}
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
return STATUS_OBJECT_DEAD;
}
__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
IO_STATUS_BLOCK* iosbOut) {
if (!req || !iosbOut) return STATUS_INVALID_PARAMETER;
if (req->Iosb) {
iosbOut->Status = req->Iosb->Status;
iosbOut->Information = req->Iosb->Information;
} else {
iosbOut->Status = STATUS_FAILURE;
iosbOut->Information = 0;
}
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// Current Working Directory
// Stored as (CurrentDrive, drive-relative path with no trailing backslash).
// Exported as native "N:/path" so personalities can round-trip volumes.
// ---------------------------------------------------------------------------
#define CWD_MAX 512
static size_t KiFormatCwd(uint8_t drive, const char* rel, char* buf, size_t size) {
if (!buf || size == 0) return 0;
if (drive > 7) drive = 0;
if (!rel) rel = "";
size_t relLen = strlen(rel);
size_t needed = 3 + relLen + 1;
if (needed > size) return needed;
buf[0] = (char)('0' + drive);
buf[1] = ':';
buf[2] = '/';
memcpy(buf + 3, rel, relLen);
buf[3 + relLen] = '\0';
return needed;
}
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
if (!buf || size == 0) return STATUS_INVALID_PARAMETER;
@@ -29,24 +413,26 @@ __declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
if (!proc) return STATUS_ACCESS_DENIED;
uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
size_t cwdLen = strlen(proc->CurrentDirectory);
if (cwdLen + 1 > size) {
KeReleaseSpinlock(&proc->CwdLock, flags);
return STATUS_BUFFER_TOO_SMALL;
}
memcpy(buf, proc->CurrentDirectory, cwdLen + 1);
size_t needed = KiFormatCwd(proc->CurrentDrive, proc->CurrentDirectory, buf, size);
KeReleaseSpinlock(&proc->CwdLock, flags);
return STATUS_SUCCESS;
return needed && needed <= size ? STATUS_SUCCESS : STATUS_BUFFER_TOO_SMALL;
}
// Resolve a possibly-relative path component against the CWD.
// Writes the fully-resolved, drive-stripped relative path into 'resolved'
// (caller supplies buf of at least CWD_MAX bytes).
// Returns the drive index.
static uint8_t ResolvePath(const char* path, char* resolved, size_t resolvedSize) {
// Strip explicit drive prefix "N:\"
static uint8_t KiResolvePath(const char* path, char* resolved, size_t resolvedSize) {
uint8_t drive = 0;
PROCESS* proc = KeGetCurrentProcess();
if (proc) {
uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
drive = proc->CurrentDrive;
KeReleaseSpinlock(&proc->CwdLock, flags);
}
// Strip explicit drive prefix "N:\" or "N:/".
bool fromDriveRoot = false;
if (path[0] >= '0' && path[0] <= '7' &&
path[1] == ':' && (path[2] == '\\' || path[2] == '/')) {
@@ -124,14 +510,15 @@ static uint8_t ResolvePath(const char* path, char* resolved, size_t resolvedSize
__declspec(dllexport) KSTATUS IoSetCwd(const char* path) {
if (!path) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
PROCESS* proc = KeGetCurrentProcess();
if (!proc) return STATUS_ACCESS_DENIED;
// Resolve the new path (this also validates ".." navigation).
// Note: ResolvePath already uses proc->CurrentDirectory internally!
// Note: KiResolvePath already uses proc->CurrentDirectory internally!
char resolved[CWD_MAX];
uint8_t drive = ResolvePath(path, resolved, sizeof(resolved));
uint8_t drive = KiResolvePath(path, resolved, sizeof(resolved));
// Validate: try to open the directory to confirm it actually exists on disk
IO_FILESYSTEM* fs = gVolumes[drive].Fs;
@@ -148,16 +535,15 @@ __declspec(dllexport) KSTATUS IoSetCwd(const char* path) {
if (len + 1 > sizeof(proc->CurrentDirectory)) return STATUS_BUFFER_TOO_SMALL;
uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
proc->CurrentDrive = drive;
memcpy(proc->CurrentDirectory, resolved, len + 1);
// Keep the user-visible PEB mirror in sync. The PEB's CWD buffer is a
// fixed SYS_CWD_MAX allocation made at process creation, so the pointer
// remains stable for personalities that cache SYS_PEB*.
if (proc->UserCwdBase && proc->UserCwdSize) {
size_t copyLen = len;
if (copyLen + 1 > proc->UserCwdSize) copyLen = proc->UserCwdSize - 1;
memcpy((void*)proc->UserCwdBase, resolved, copyLen);
((char*)proc->UserCwdBase)[copyLen] = '\0';
size_t formatted = KiFormatCwd(drive, resolved, (char*)proc->UserCwdBase, proc->UserCwdSize);
size_t copyLen = (formatted && formatted <= proc->UserCwdSize) ? formatted - 1 : proc->UserCwdSize - 1;
SYS_PEB* peb = (SYS_PEB*)proc->UserPebBase;
if (peb && peb->Magic == SYS_PEB_MAGIC) {
@@ -189,7 +575,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
// ---------------------------------------------------------------------------
// Destructor
// ---------------------------------------------------------------------------
static void FileDestroy(OBJECT* obj) {
static void KiFileDestroy(OBJECT* obj) {
FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx);
}
@@ -199,9 +585,10 @@ static void FileDestroy(OBJECT* obj) {
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out) {
if (!path || !out) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
char relPath[CWD_MAX];
uint8_t drive = ResolvePath(path, relPath, sizeof(relPath));
uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath));
IO_FILESYSTEM* fs = gVolumes[drive].Fs;
if (!fs) return STATUS_NOT_FOUND;
@@ -213,7 +600,7 @@ __declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out) {
FILE_OBJECT* f = (FILE_OBJECT *)ObAllocateObject(ObjectTypeFile /* reuse slot */,
sizeof(FILE_OBJECT));
if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; }
f->Header.Destroy = FileDestroy;
f->Header.Destroy = KiFileDestroy;
f->Fs = fs;
f->FileCtx = fileCtx;
f->Size = fs->GetSize(fs, fileCtx);
@@ -224,20 +611,69 @@ __declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out) {
}
// ---------------------------------------------------------------------------
// IoReadFile - sequential read at current offset
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, size_t size, size_t* bytesRead) {
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
void* buf, size_t size,
size_t* bytesRead) {
if (bytesRead) *bytesRead = 0;
if (!file || !buf || !bytesRead) return STATUS_INVALID_PARAMETER;
if (file->Offset >= file->Size || size == 0) return STATUS_SUCCESS;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (size == 0) return STATUS_SUCCESS;
// Clamp to remaining bytes.
size_t remaining = (size_t)(file->Size - file->Offset);
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t fileSize = file->Size;
KeReleaseSpinlock(&file->Lock, flags);
if (offset >= fileSize) return STATUS_SUCCESS;
size_t remaining = (size_t)(fileSize - offset);
if (size > remaining) size = remaining;
KSTATUS s = file->Fs->Read(file->Fs, file->FileCtx,
file->Offset, buf, size, bytesRead);
if (KSUCCESS(s)) file->Offset += *bytesRead;
return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead);
}
__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
const void* buf, size_t size,
size_t* bytesWritten) {
if (bytesWritten) *bytesWritten = 0;
if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (!file->Fs->Write) return STATUS_NOT_SUPPORTED;
if (size == 0) return STATUS_SUCCESS;
size_t written = 0;
KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t end = offset + written;
if (end > file->Size) file->Size = end;
KeReleaseSpinlock(&file->Lock, flags);
}
*bytesWritten = written;
return s;
}
// ---------------------------------------------------------------------------
// IoReadFile - sequential read at current offset
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead) {
if (bytesRead) *bytesRead = 0;
if (!file || !buf || !bytesRead) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t offset = file->Offset;
KeReleaseSpinlock(&file->Lock, flags);
KSTATUS s = IoReadFileAt(file, offset, buf, size, bytesRead);
if (KSUCCESS(s)) {
flags = KeAcquireSpinlock(&file->Lock);
file->Offset = offset + *bytesRead;
KeReleaseSpinlock(&file->Lock, flags);
}
return s;
}
@@ -246,19 +682,19 @@ __declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf, size_t si
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
size_t size, size_t* bytesWritten) {
if (bytesWritten) *bytesWritten = 0;
if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER;
if (!file->Fs->Write) return STATUS_NOT_SUPPORTED;
size_t written = 0;
KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx,
file->Offset, buf, size, &written);
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t offset = file->Offset;
KeReleaseSpinlock(&file->Lock, flags);
KSTATUS s = IoWriteFileAt(file, offset, buf, size, bytesWritten);
if (KSUCCESS(s)) {
file->Offset += written;
// Keep the cached size in sync if the file grew.
if (file->Offset > file->Size)
file->Size = file->Offset;
flags = KeAcquireSpinlock(&file->Lock);
file->Offset = offset + *bytesWritten;
KeReleaseSpinlock(&file->Lock, flags);
}
*bytesWritten = written;
return s;
}
@@ -267,9 +703,10 @@ __declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoCreateFile(const char* path, FILE_OBJECT** out) {
if (!path || !out) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
char relPath[CWD_MAX];
uint8_t drive = ResolvePath(path, relPath, sizeof(relPath));
uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath));
IO_FILESYSTEM* fs = gVolumes[drive].Fs;
if (!fs) return STATUS_NOT_FOUND;
@@ -282,9 +719,10 @@ __declspec(dllexport) KSTATUS IoCreateFile(const char* path, FILE_OBJECT** out)
FILE_OBJECT* f = (FILE_OBJECT*)ObAllocateObject(ObjectTypeFile,
sizeof(FILE_OBJECT));
if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; }
f->Header.Destroy = FileDestroy;
f->Header.Destroy = KiFileDestroy;
f->Fs = fs;
f->FileCtx = fileCtx;
f->Lock = (SPINLOCK)SPINLOCK_INIT;
f->Size = 0;
f->Offset = 0;
@@ -297,9 +735,10 @@ __declspec(dllexport) KSTATUS IoCreateFile(const char* path, FILE_OBJECT** out)
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoDeleteFile(const char* path) {
if (!path) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
char relPath[CWD_MAX];
uint8_t drive = ResolvePath(path, relPath, sizeof(relPath));
uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath));
IO_FILESYSTEM* fs = gVolumes[drive].Fs;
if (!fs) return STATUS_NOT_FOUND;
@@ -310,11 +749,12 @@ __declspec(dllexport) KSTATUS IoDeleteFile(const char* path) {
__declspec(dllexport) KSTATUS IoRenameFile(const char* oldPath, const char* newPath) {
if (!oldPath || !newPath) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
char oldRel[CWD_MAX];
char newRel[CWD_MAX];
uint8_t oldDrive = ResolvePath(oldPath, oldRel, sizeof(oldRel));
uint8_t newDrive = ResolvePath(newPath, newRel, sizeof(newRel));
uint8_t oldDrive = KiResolvePath(oldPath, oldRel, sizeof(oldRel));
uint8_t newDrive = KiResolvePath(newPath, newRel, sizeof(newRel));
if (oldDrive != newDrive) return STATUS_NOT_SUPPORTED;
IO_FILESYSTEM* fs = gVolumes[oldDrive].Fs;
@@ -326,9 +766,10 @@ __declspec(dllexport) KSTATUS IoRenameFile(const char* oldPath, const char* newP
__declspec(dllexport) KSTATUS IoCreateDir(const char* path) {
if (!path) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
char relPath[CWD_MAX];
uint8_t drive = ResolvePath(path, relPath, sizeof(relPath));
uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath));
IO_FILESYSTEM* fs = gVolumes[drive].Fs;
if (!fs) return STATUS_NOT_FOUND;
@@ -339,9 +780,10 @@ __declspec(dllexport) KSTATUS IoCreateDir(const char* path) {
__declspec(dllexport) KSTATUS IoRemoveDir(const char* path) {
if (!path) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
char relPath[CWD_MAX];
uint8_t drive = ResolvePath(path, relPath, sizeof(relPath));
uint8_t drive = KiResolvePath(path, relPath, sizeof(relPath));
IO_FILESYSTEM* fs = gVolumes[drive].Fs;
if (!fs) return STATUS_NOT_FOUND;
@@ -355,6 +797,7 @@ __declspec(dllexport) KSTATUS IoRemoveDir(const char* path) {
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize) {
if (!file) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (!file->Fs->Truncate) return STATUS_NOT_SUPPORTED;
KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize);
@@ -370,12 +813,18 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) {
if (!file) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
file->Offset = offset;
KeReleaseSpinlock(&file->Lock, flags);
return STATUS_SUCCESS;
}
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
return file ? file->Size : 0;
if (!file) return 0;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t size = file->Size;
KeReleaseSpinlock(&file->Lock, flags);
return size;
}
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
@@ -386,13 +835,14 @@ __declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
// Directory enumeration
// ---------------------------------------------------------------------------
static void DirDestroy(OBJECT* obj) {
static void KiDirDestroy(OBJECT* obj) {
DIR_OBJECT* d = (DIR_OBJECT*)obj; // Header is first member
if (d->Fs && d->DirCtx) d->Fs->CloseDir(d->Fs, d->DirCtx);
}
__declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) {
if (!out) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
// NULL or empty path means the CWD itself
char relPath[CWD_MAX];
@@ -404,13 +854,10 @@ __declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) {
uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
memcpy(relPath, proc->CurrentDirectory, strlen(proc->CurrentDirectory) + 1);
drive = proc->CurrentDrive;
KeReleaseSpinlock(&proc->CwdLock, flags);
// Note: For now, this assumes drive 0. If you add multi-drive support
// later, you should add a CurrentDrive field to the PROCESS struct.
drive = 0;
} else {
drive = ResolvePath(dirPath, relPath, sizeof(relPath));
drive = KiResolvePath(dirPath, relPath, sizeof(relPath));
}
IO_FILESYSTEM* fs = gVolumes[drive].Fs;
@@ -424,7 +871,7 @@ __declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) {
DIR_OBJECT* d = (DIR_OBJECT*)ObAllocateObject(ObjectTypeDir, sizeof(DIR_OBJECT));
if (!d) { fs->CloseDir(fs, dirCtx); return STATUS_NO_MEMORY; }
d->Header.Destroy = DirDestroy;
d->Header.Destroy = KiDirDestroy;
d->Fs = fs;
d->DirCtx = dirCtx;
@@ -434,6 +881,7 @@ __declspec(dllexport) KSTATUS IoOpenDir(const char* dirPath, DIR_OBJECT** out) {
__declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) {
if (!dir || !entryOut) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (!dir->Fs->ReadDir) return STATUS_NOT_SUPPORTED;
return dir->Fs->ReadDir(dir->Fs, dir->DirCtx, entryOut);
}
code/kernel/io_file.h
+199 -149
View File
@@ -1,149 +1,199 @@
#pragma once
// io_file.h - VFS layer
// Filesystem drivers register an IO_FILESYSTEM vtable via IoRegisterFilesystem.
// Kernel and user code open files through IoOpenFile.
#include <stdint.h>
#include <stddef.h>
#include <ke_status.h>
#include <ob_object.h>
// ---------------------------------------------------------------------------
// KDIR_ENTRY - one directory entry returned by ReadDir / SYS_READ_DIR
// Attribute flags match FAT32/Win32: ATTR_DIRECTORY=0x10, ATTR_ARCHIVE=0x20
// ---------------------------------------------------------------------------
#define KDIR_NAME_MAX 260
typedef struct KDIR_ENTRY {
char Name[KDIR_NAME_MAX]; // null-terminated filename
uint8_t Attr; // FAT attribute byte (0x10=dir, 0x20=file)
uint32_t FileSize; // 0 for directories
} KDIR_ENTRY;
// ---------------------------------------------------------------------------
// Filesystem driver vtable - filled in by fat32.sys (or any future fs driver)
// ---------------------------------------------------------------------------
typedef struct IO_FILESYSTEM {
// Open a file by absolute path (e.g. "apps\\shell.exe").
// Returns an opaque per-file driver cookie in* fileCtx.
KSTATUS (*Open)(struct IO_FILESYSTEM* fs,
const char* path, void **fileCtx);
// Read up to 'size' bytes at 'offset' into 'buf'.
// Sets* bytesRead to actual bytes copied.
KSTATUS (*Read)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t offset, void* buf,
size_t size, size_t* bytesRead);
KSTATUS (*Write)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t offset, const void* buf,
size_t size, size_t* bytesWritten);
// Create a new file at 'path'; returns an open fileCtx on success.
KSTATUS (*Create)(struct IO_FILESYSTEM* fs, const char* path,
void** fileCtx);
// Delete the file at 'path'.
KSTATUS (*Delete)(struct IO_FILESYSTEM* fs, const char* path);
// Rename or move a file/directory within the same filesystem.
KSTATUS (*Rename)(struct IO_FILESYSTEM* fs,
const char* oldPath, const char* newPath);
// Create or remove directories.
KSTATUS (*CreateDir)(struct IO_FILESYSTEM* fs, const char* path);
KSTATUS (*RemoveDir)(struct IO_FILESYSTEM* fs, const char* path);
// Truncate or extend the file to exactly 'newSize' bytes.
KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t newSize);
// Return the file size in bytes.
uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Release any resources the driver allocated for fileCtx.
void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Optional driver-private data.
void* DriverCtx;
// ---------------------------------------------------------------------------
// Directory enumeration (optional - may be NULL if not supported)
// ---------------------------------------------------------------------------
// Open a directory iterator for 'dirPath' (relative or drive-prefixed).
// Returns an opaque per-iteration driver cookie in *dirCtx.
KSTATUS (*OpenDir)(struct IO_FILESYSTEM* fs,
const char* dirPath, void** dirCtx);
// Advance to the next entry.
// Fills 'out' with name, attributes, and size.
// Returns STATUS_NO_MORE_FILES when the directory is exhausted.
KSTATUS (*ReadDir)(struct IO_FILESYSTEM* fs, void* dirCtx,
struct KDIR_ENTRY* out);
// Release resources allocated by OpenDir.
void (*CloseDir)(struct IO_FILESYSTEM* fs, void* dirCtx);
} IO_FILESYSTEM;
// ---------------------------------------------------------------------------
// FILE_OBJECT - the kernel object returned to callers
// ---------------------------------------------------------------------------
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
void* FileCtx; // driver cookie
uint64_t Size;
uint64_t Offset; // current position for sequential reads
} FILE_OBJECT;
// ---------------------------------------------------------------------------
// DIR_OBJECT - kernel object for an open directory iterator
// ---------------------------------------------------------------------------
typedef struct DIR_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
void* DirCtx; // driver cookie (opaque iteration state)
} DIR_OBJECT;
typedef struct IO_VOLUME_PARAMS {
uint8_t DiskIndex; // which disk in HAL_DISK table
uint64_t PartitionLba; // starting LBA of this partition
uint64_t PartitionSize; // sectors, for bounds checking
} IO_VOLUME_PARAMS;
// ---------------------------------------------------------------------------
// Registration - called by fat32.sys from DriverEntry
// ---------------------------------------------------------------------------
// driveIndex matches the index passed to IoRegisterDisk.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol);
// ---------------------------------------------------------------------------
// Kernel-side API
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out);
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead);
__declspec(dllexport) KSTATUS IoWriteFile (FILE_OBJECT* file, const void* buf,
size_t size, size_t* bytesWritten);
__declspec(dllexport) KSTATUS IoCreateFile (const char* path, FILE_OBJECT** out);
__declspec(dllexport) KSTATUS IoDeleteFile (const char* path);
__declspec(dllexport) KSTATUS IoRenameFile (const char* oldPath, const char* newPath);
__declspec(dllexport) KSTATUS IoCreateDir (const char* path);
__declspec(dllexport) KSTATUS IoRemoveDir (const char* path);
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize);
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// ---------------------------------------------------------------------------
// Directory enumeration API
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoOpenDir (const char* dirPath, DIR_OBJECT** out);
__declspec(dllexport) KSTATUS IoReadDir (DIR_OBJECT* dir, KDIR_ENTRY* entryOut);
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir);
// ---------------------------------------------------------------------------
// Current Working Directory
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoSetCwd(const char* path);
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size);
#pragma once
// io_file.h - VFS layer
// Filesystem drivers register an IO_FILESYSTEM vtable via IoRegisterFilesystem.
// Kernel and user code open files through IoOpenFile.
#include <stdint.h>
#include <stddef.h>
#include <ke_status.h>
#include <ob_object.h>
#include <ke_spinlock.h>
// ---------------------------------------------------------------------------
// KDIR_ENTRY - one directory entry returned by ReadDir / SYS_READ_DIR
// Attribute flags match FAT32/Win32: ATTR_DIRECTORY=0x10, ATTR_ARCHIVE=0x20
// ---------------------------------------------------------------------------
#define KDIR_NAME_MAX 260
typedef struct KDIR_ENTRY {
char Name[KDIR_NAME_MAX]; // null-terminated filename
uint8_t Attr; // FAT attribute byte (0x10=dir, 0x20=file)
uint32_t FileSize; // 0 for directories
} KDIR_ENTRY;
// Native asynchronous filesystem completion. Filesystems that provide
// ReadAsync/WriteAsync accept the operation, return STATUS_PENDING, and later
// call Complete(CompleteContext, status, information). Filesystems without
// these callbacks fall back to the generic VFS async worker.
typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
KSTATUS status,
size_t information);
// ---------------------------------------------------------------------------
// Filesystem driver vtable - filled in by fat32.sys (or any future fs driver)
// ---------------------------------------------------------------------------
typedef struct IO_FILESYSTEM {
// Open a file by absolute path (e.g. "apps\\shell.exe").
// Returns an opaque per-file driver cookie in* fileCtx.
KSTATUS (*Open)(struct IO_FILESYSTEM* fs,
const char* path, void **fileCtx);
// Read up to 'size' bytes at 'offset' into 'buf'.
// Sets* bytesRead to actual bytes copied.
KSTATUS (*Read)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t offset, void* buf,
size_t size, size_t* bytesRead);
KSTATUS (*Write)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t offset, const void* buf,
size_t size, size_t* bytesWritten);
KSTATUS (*ReadAsync)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t offset, void* buf, size_t size,
IO_FS_ASYNC_COMPLETE complete,
void* completeContext);
KSTATUS (*WriteAsync)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t offset, const void* buf, size_t size,
IO_FS_ASYNC_COMPLETE complete,
void* completeContext);
// Create a new file at 'path'; returns an open fileCtx on success.
KSTATUS (*Create)(struct IO_FILESYSTEM* fs, const char* path,
void** fileCtx);
// Delete the file at 'path'.
KSTATUS (*Delete)(struct IO_FILESYSTEM* fs, const char* path);
// Rename or move a file/directory within the same filesystem.
KSTATUS (*Rename)(struct IO_FILESYSTEM* fs,
const char* oldPath, const char* newPath);
// Create or remove directories.
KSTATUS (*CreateDir)(struct IO_FILESYSTEM* fs, const char* path);
KSTATUS (*RemoveDir)(struct IO_FILESYSTEM* fs, const char* path);
// Truncate or extend the file to exactly 'newSize' bytes.
KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t newSize);
// Return the file size in bytes.
uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Release any resources the driver allocated for fileCtx.
void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Optional driver-private data.
void* DriverCtx;
// ---------------------------------------------------------------------------
// Directory enumeration (optional - may be NULL if not supported)
// ---------------------------------------------------------------------------
// Open a directory iterator for 'dirPath' (relative or drive-prefixed).
// Returns an opaque per-iteration driver cookie in *dirCtx.
KSTATUS (*OpenDir)(struct IO_FILESYSTEM* fs,
const char* dirPath, void** dirCtx);
// Advance to the next entry.
// Fills 'out' with name, attributes, and size.
// Returns STATUS_NO_MORE_FILES when the directory is exhausted.
KSTATUS (*ReadDir)(struct IO_FILESYSTEM* fs, void* dirCtx,
struct KDIR_ENTRY* out);
// Release resources allocated by OpenDir.
void (*CloseDir)(struct IO_FILESYSTEM* fs, void* dirCtx);
} IO_FILESYSTEM;
// ---------------------------------------------------------------------------
// IO_STATUS_BLOCK - completion status visible to async callers
// ---------------------------------------------------------------------------
typedef struct IO_STATUS_BLOCK {
volatile KSTATUS Status;
volatile size_t Information;
} IO_STATUS_BLOCK;
struct EVENT;
struct KAPC;
typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
// ---------------------------------------------------------------------------
// FILE_OBJECT - the kernel object returned to callers
// ---------------------------------------------------------------------------
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
void* FileCtx; // driver cookie
SPINLOCK Lock; // protects cached Size/Offset updates
uint64_t Size;
uint64_t Offset; // current position for sequential reads
} FILE_OBJECT;
// ---------------------------------------------------------------------------
// DIR_OBJECT - kernel object for an open directory iterator
// ---------------------------------------------------------------------------
typedef struct DIR_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
void* DirCtx; // driver cookie (opaque iteration state)
} DIR_OBJECT;
typedef struct IO_VOLUME_PARAMS {
uint8_t DiskIndex; // registered disk DEVICE_OBJECT index
uint64_t PartitionLba; // starting LBA of this partition
uint64_t PartitionSize; // sectors, for bounds checking
} IO_VOLUME_PARAMS;
// ---------------------------------------------------------------------------
// Registration - called by fat32.sys from DriverEntry
// ---------------------------------------------------------------------------
// driveIndex matches the index passed to IoRegisterDiskDevice.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESYSTEM* fs, const IO_VOLUME_PARAMS *vol);
// ---------------------------------------------------------------------------
// Kernel-side API
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoOpenFile(const char* path, FILE_OBJECT **out);
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead);
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
void* buf, size_t size, size_t* bytesRead);
__declspec(dllexport) KSTATUS IoWriteFile (FILE_OBJECT* file, const void* buf,
size_t size, size_t* bytesWritten);
__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
const void* buf, size_t size,
size_t* bytesWritten);
__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf,
size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
struct EVENT* completionEvent, struct KAPC* completionApc,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
struct EVENT* completionEvent, struct KAPC* completionApc,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
IO_STATUS_BLOCK* iosbOut);
__declspec(dllexport) KSTATUS IoCreateFile (const char* path, FILE_OBJECT** out);
__declspec(dllexport) KSTATUS IoDeleteFile (const char* path);
__declspec(dllexport) KSTATUS IoRenameFile (const char* oldPath, const char* newPath);
__declspec(dllexport) KSTATUS IoCreateDir (const char* path);
__declspec(dllexport) KSTATUS IoRemoveDir (const char* path);
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize);
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// ---------------------------------------------------------------------------
// Directory enumeration API
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoOpenDir (const char* dirPath, DIR_OBJECT** out);
__declspec(dllexport) KSTATUS IoReadDir (DIR_OBJECT* dir, KDIR_ENTRY* entryOut);
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir);
// ---------------------------------------------------------------------------
// Current Working Directory
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS IoSetCwd(const char* path);
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size);
code/kernel/ke_apc.c
+335
View File
@@ -0,0 +1,335 @@
// ke_apc.c - kernel APC infrastructure
#include <ke_apc.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ke_spinlock.h>
#include <ke_irql.h>
#include <ke_trap.h>
#include <mm_virt.h>
#include <string.h>
#include <intrin.h>
void KiInitializeApcState(THREAD* thread) {
if (!thread) return;
LIST_INIT(&thread->ApcQueue);
thread->ApcLock = (SPINLOCK)SPINLOCK_INIT;
thread->KernelApcPending = 0;
thread->KernelApcDisable = 0;
thread->ApcDeliveryActive = 0;
LIST_INIT(&thread->UserApcQueue);
thread->UserApcLock = (SPINLOCK)SPINLOCK_INIT;
thread->UserApcPending = 0;
thread->UserApcDispatcher = 0;
thread->UserApcDeliverable = 0;
thread->UserApcDeliveryActive = 0;
}
__declspec(dllexport) void KeInitializeApc(KAPC* apc,
THREAD* thread,
PKKERNEL_ROUTINE kernelRoutine,
PKNORMAL_ROUTINE normalRoutine,
void* normalContext) {
if (!apc) return;
LIST_INIT(&apc->Link);
apc->Thread = thread;
apc->KernelRoutine = kernelRoutine;
apc->NormalRoutine = normalRoutine;
apc->RundownRoutine = (PKRUNDOWN_ROUTINE)0;
apc->NormalContext = normalContext;
apc->SystemArgument1 = (void*)0;
apc->SystemArgument2 = (void*)0;
apc->Inserted = 0;
}
__declspec(dllexport) void KeSetApcRundownRoutine(KAPC* apc,
PKRUNDOWN_ROUTINE rundownRoutine) {
if (!apc) return;
apc->RundownRoutine = rundownRoutine;
}
__declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
void* systemArgument1,
void* systemArgument2) {
if (!apc || !apc->Thread)
return false;
THREAD* thread = apc->Thread;
if (thread->State == ThreadDead)
return false;
if (_InterlockedCompareExchange(&apc->Inserted, 1, 0) != 0)
return false;
apc->SystemArgument1 = systemArgument1;
apc->SystemArgument2 = systemArgument2;
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
ListInsertTail(&thread->ApcQueue, &apc->Link);
thread->KernelApcPending = 1;
uint8_t wakeAlertable = (thread->State == ThreadWaiting &&
thread->AlertableWait &&
thread->KernelApcDisable == 0);
KeReleaseSpinlock(&thread->ApcLock, flags);
// If the target opted into an alertable wait, make the APC itself the
// wait completion. The target thread will deliver the APC before
// KeWaitForEvent(..., true) returns STATUS_KERNEL_APC.
if (wakeAlertable)
KiWakeThreadForKernelApc(thread);
// If the caller queued an APC to itself at PASSIVE_LEVEL, try to deliver
// immediately. Cross-thread APCs run the next time the target thread
// returns to PASSIVE_LEVEL.
if (thread == KeGetCurrentThread())
KiDeliverKernelApcs();
return true;
}
__declspec(dllexport) void KeEnterCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
thread->KernelApcDisable++;
KeReleaseSpinlock(&thread->ApcLock, flags);
}
__declspec(dllexport) void KeLeaveCriticalRegion(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
if (thread->KernelApcDisable > 0)
thread->KernelApcDisable--;
int deliver = (thread->KernelApcDisable == 0 && thread->KernelApcPending);
KeReleaseSpinlock(&thread->ApcLock, flags);
if (deliver)
KiDeliverKernelApcs();
}
__declspec(dllexport) bool KeAreKernelApcsDisabled(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return true;
return thread->KernelApcDisable != 0;
}
static KAPC* KiPopKernelApc(THREAD* thread) {
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
LIST_ENTRY* node = ListPopHead(&thread->ApcQueue);
if (LIST_IS_EMPTY(&thread->ApcQueue))
thread->KernelApcPending = 0;
KeReleaseSpinlock(&thread->ApcLock, flags);
if (!node)
return (KAPC*)0;
return CONTAINING_RECORD(node, KAPC, Link);
}
void KiDeliverKernelApcs(void) {
THREAD* thread = KeGetCurrentThread();
if (!thread)
return;
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
return;
if (thread->ApcDeliveryActive || thread->KernelApcDisable != 0 || !thread->KernelApcPending)
return;
thread->ApcDeliveryActive = 1;
for (;;) {
if (KeGetCurrentIrql() != PASSIVE_LEVEL || thread->KernelApcDisable != 0)
break;
KAPC* apc = KiPopKernelApc(thread);
if (!apc)
break;
apc->Inserted = 0;
PKNORMAL_ROUTINE normalRoutine = apc->NormalRoutine;
void* normalContext = apc->NormalContext;
void* arg1 = apc->SystemArgument1;
void* arg2 = apc->SystemArgument2;
if (apc->KernelRoutine)
apc->KernelRoutine(apc,
(void**)&normalRoutine,
&normalContext,
&arg1,
&arg2);
// After the kernel routine has had a chance to adjust the callout,
// stop touching APC storage before invoking the normal routine. The
// normal routine is allowed to free the APC's containing allocation.
apc->SystemArgument1 = (void*)0;
apc->SystemArgument2 = (void*)0;
if (normalRoutine && KeGetCurrentIrql() == PASSIVE_LEVEL)
normalRoutine(normalContext, arg1, arg2);
}
thread->ApcDeliveryActive = 0;
}
__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher) {
THREAD* thread = KeGetCurrentThread();
if (!thread) return STATUS_FAILURE;
if (!dispatcher) return STATUS_INVALID_PARAMETER;
thread->UserApcDispatcher = (uintptr_t)dispatcher;
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
void* routine,
void* normalContext,
void* systemArgument1,
void* systemArgument2) {
if (!thread || !routine)
return STATUS_INVALID_PARAMETER;
if (thread->State == ThreadDead)
return STATUS_OBJECT_DEAD;
KUSER_APC* apc = (KUSER_APC*)MmAllocatePool(sizeof(KUSER_APC));
if (!apc)
return STATUS_NO_MEMORY;
memset(apc, 0, sizeof(*apc));
LIST_INIT(&apc->Link);
apc->Routine = routine;
apc->NormalContext = normalContext;
apc->SystemArgument1 = systemArgument1;
apc->SystemArgument2 = systemArgument2;
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
ListInsertTail(&thread->UserApcQueue, &apc->Link);
thread->UserApcPending = 1;
uint8_t wakeAlertable = (thread->State == ThreadWaiting &&
thread->AlertableWait &&
thread->UserApcDispatcher != 0);
KeReleaseSpinlock(&thread->UserApcLock, flags);
if (wakeAlertable)
KiWakeThreadForUserApc(thread);
return STATUS_SUCCESS;
}
static KUSER_APC* KiPopUserApc(THREAD* thread) {
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue);
if (LIST_IS_EMPTY(&thread->UserApcQueue))
thread->UserApcPending = 0;
KeReleaseSpinlock(&thread->UserApcLock, flags);
if (!node)
return (KUSER_APC*)0;
return CONTAINING_RECORD(node, KUSER_APC, Link);
}
void KiCheckForUserApcDelivery(KTRAP_FRAME* frame) {
if (!frame || !KeTrapFrameFromUser(frame))
return;
THREAD* thread = KeGetCurrentThread();
if (!thread)
return;
if (!thread->UserApcDeliverable || thread->UserApcDeliveryActive ||
!thread->UserApcPending || !thread->UserApcDispatcher)
return;
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
return;
KUSER_APC* apc = KiPopUserApc(thread);
if (!apc)
return;
uintptr_t oldRsp = (uintptr_t)frame->Rsp;
uintptr_t newRsp = oldRsp - sizeof(SYS_USER_APC_FRAME_K);
newRsp &= ~(uintptr_t)0xFULL;
newRsp -= 8; // make injected dispatcher entry look like a normal call target
SYS_USER_APC_FRAME_K* userFrame = (SYS_USER_APC_FRAME_K*)newRsp;
userFrame->Magic = SYS_USER_APC_FRAME_MAGIC;
userFrame->OriginalRip = frame->Rip;
userFrame->OriginalRsp = frame->Rsp;
userFrame->OriginalRflags = frame->Rflags;
userFrame->OriginalRax = STATUS_USER_APC;
userFrame->Routine = (uint64_t)(uintptr_t)apc->Routine;
userFrame->NormalContext = (uint64_t)(uintptr_t)apc->NormalContext;
userFrame->SystemArgument1 = (uint64_t)(uintptr_t)apc->SystemArgument1;
userFrame->SystemArgument2 = (uint64_t)(uintptr_t)apc->SystemArgument2;
frame->Rip = thread->UserApcDispatcher;
frame->Rsp = newRsp;
frame->Rax = STATUS_USER_APC;
thread->UserApcDeliverable = 0;
thread->UserApcDeliveryActive = 1;
MmFreePool(apc);
}
void KiRundownApcState(THREAD* thread) {
if (!thread) return;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&thread->ApcLock);
LIST_ENTRY* node = ListPopHead(&thread->ApcQueue);
if (LIST_IS_EMPTY(&thread->ApcQueue))
thread->KernelApcPending = 0;
KeReleaseSpinlock(&thread->ApcLock, flags);
if (!node) break;
KAPC* apc = CONTAINING_RECORD(node, KAPC, Link);
apc->Inserted = 0;
if (apc->RundownRoutine)
apc->RundownRoutine(apc);
}
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&thread->UserApcLock);
LIST_ENTRY* node = ListPopHead(&thread->UserApcQueue);
if (LIST_IS_EMPTY(&thread->UserApcQueue))
thread->UserApcPending = 0;
KeReleaseSpinlock(&thread->UserApcLock, flags);
if (!node) break;
KUSER_APC* apc = CONTAINING_RECORD(node, KUSER_APC, Link);
MmFreePool(apc);
}
thread->UserApcDispatcher = 0;
thread->UserApcDeliverable = 0;
thread->UserApcDeliveryActive = 0;
}
__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* userFrame) {
if (!userFrame || userFrame->Magic != SYS_USER_APC_FRAME_MAGIC)
return STATUS_INVALID_PARAMETER;
THREAD* thread = KeGetCurrentThread();
if (!thread)
return STATUS_FAILURE;
KTRAP_FRAME* frame = KeGetCurrentTrapFrame();
if (!frame || !KeTrapFrameFromUser(frame) || frame->Kind != KTRAP_KIND_SYSCALL)
return STATUS_FAILURE;
frame->Rip = userFrame->OriginalRip;
frame->Rsp = userFrame->OriginalRsp;
frame->Rflags = userFrame->OriginalRflags;
frame->Rax = userFrame->OriginalRax;
thread->UserApcDeliveryActive = 0;
thread->UserApcDeliverable = 0;
return userFrame->OriginalRax;
}
code/kernel/ke_apc.h
+102
View File
@@ -0,0 +1,102 @@
#pragma once
// ke_apc.h - kernel APC infrastructure
//
// Kernel APCs are per-thread callbacks that run in the target thread's
// context when that thread reaches PASSIVE_LEVEL and APC delivery is enabled.
// They are the first APC layer; user APCs will build on this once trap-frame
// return-to-user delivery exists.
#include <stdint.h>
#include <stdbool.h>
#include <ke_list.h>
#include <ke_status.h>
struct THREAD;
typedef struct THREAD THREAD;
typedef struct KTRAP_FRAME KTRAP_FRAME;
typedef struct KAPC KAPC;
typedef void (*PKKERNEL_ROUTINE)(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2);
typedef void (*PKNORMAL_ROUTINE)(void* normalContext,
void* systemArgument1,
void* systemArgument2);
typedef void (*PKRUNDOWN_ROUTINE)(KAPC* apc);
typedef struct KUSER_APC KUSER_APC;
typedef struct SYS_USER_APC_FRAME_K {
uint64_t Magic;
uint64_t OriginalRip;
uint64_t OriginalRsp;
uint64_t OriginalRflags;
uint64_t OriginalRax;
uint64_t Routine;
uint64_t NormalContext;
uint64_t SystemArgument1;
uint64_t SystemArgument2;
} SYS_USER_APC_FRAME_K;
#define SYS_USER_APC_FRAME_MAGIC 0x3143504152555355ULL /* "USURAPC1" little-ish */
struct KUSER_APC {
LIST_ENTRY Link;
void* Routine;
void* NormalContext;
void* SystemArgument1;
void* SystemArgument2;
};
struct KAPC {
LIST_ENTRY Link;
THREAD* Thread;
PKKERNEL_ROUTINE KernelRoutine;
PKNORMAL_ROUTINE NormalRoutine;
PKRUNDOWN_ROUTINE RundownRoutine;
void* NormalContext;
void* SystemArgument1;
void* SystemArgument2;
volatile long Inserted;
};
void KiInitializeApcState(THREAD* thread);
__declspec(dllexport) void KeInitializeApc(KAPC* apc,
THREAD* thread,
PKKERNEL_ROUTINE kernelRoutine,
PKNORMAL_ROUTINE normalRoutine,
void* normalContext);
__declspec(dllexport) void KeSetApcRundownRoutine(KAPC* apc,
PKRUNDOWN_ROUTINE rundownRoutine);
__declspec(dllexport) bool KeInsertQueueApc(KAPC* apc,
void* systemArgument1,
void* systemArgument2);
__declspec(dllexport) void KeEnterCriticalRegion(void);
__declspec(dllexport) void KeLeaveCriticalRegion(void);
__declspec(dllexport) bool KeAreKernelApcsDisabled(void);
// Internal dispatcher hook. Called when a thread returns to PASSIVE_LEVEL,
// after context switches, and after waits complete.
void KiDeliverKernelApcs(void);
__declspec(dllexport) KSTATUS KeSetUserApcDispatcher(void* dispatcher);
__declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
void* routine,
void* normalContext,
void* systemArgument1,
void* systemArgument2);
__declspec(dllexport) uint64_t KeUserApcReturn(const SYS_USER_APC_FRAME_K* frame);
void KiRundownApcState(THREAD* thread);
// Called from KiExitTrapFrame before committing the editable user return state.
void KiCheckForUserApcDelivery(KTRAP_FRAME* frame);
code/kernel/ke_debug.c
+187 -187
View File
@@ -1,187 +1,187 @@
// ke_debug.c - debugging utilities
#include <stdarg.h>
#include <stdint.h>
#include <string.h>
#include <hal_api.h>
#include <ke_spinlock.h>
#include <ke_process.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <printf.h>
int _fltused; // fuckass
int gDebugEnabled = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
void kernel_putchar(char c) {
PrintChar(c, 0x00FF00, 0x000000);
}
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) {
gDebugEnabled = v;
PrintChar = PrintPtr;
printf_set_putchar(kernel_putchar);
}
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled) return;
static SPINLOCK sl = {0};
uintptr_t flags = KeAcquireSpinlock(&sl);
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
KeReleaseSpinlock(&sl, flags);
}
// KePanic
__declspec(dllexport) _Noreturn void KePanic(const char* fmt, ...) {
// Silence! Stop the world!
_disable();
// Print a bright-red banner so the framebuffer is unmistakable.
// We call PrintChar directly so we can set a red foreground.
// (DbgPrint uses green; we want this visually distinct.)
static const char banner[] = "\n*** KERNEL PANIC ***\n";
for (const char* p = banner; *p; p++)
PrintChar(*p, 0xFF2222, 0x000000);
// Now print the caller's message through the normal vprintf path.
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
// Die!
while (true) __halt();
}
// KePrintConsole - write user-visible console text in white-on-black.
// Used by SYS_WRITE_CONSOLE so user programs get a distinct color from
// kernel debug output (green) and panic output (red).
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
if (!buf || !PrintChar) return;
for (size_t i = 0; i < len; i++)
PrintChar(buf[i], 0xFFFFFF, 0x000000);
}
// ---------------------------------------------------------------------------
// KeBugCheckProcess - user-mode fault handler
//
// Resolves the faulting RIP to a module name, prints a yellow crash banner,
// kills the offending process, then exits the current thread so the scheduler
// can continue running everything else.
// ---------------------------------------------------------------------------
// Return the short exception name for a vector number.
static const char* FaultVectorName(int vector) {
switch (vector) {
case 0: return "#DE Divide Error";
case 13: return "#GP General Protection";
case 14: return "#PF Page Fault";
default: return "Unknown Exception";
}
}
// Walk the process module list to find which image owns rip.
// Returns the module name string, or "unknown" if rip is outside all regions.
static const char* ResolveModule(PROCESS* proc, uintptr_t rip) {
// Check the main executable image first.
if (proc->ImageBase && proc->ImageSize &&
rip >= proc->ImageBase && rip < proc->ImageBase + proc->ImageSize)
return proc->Name;
// Walk the per-process module table (DLLs / shared libraries).
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
uintptr_t base = proc->Modules[i].Base;
size_t size = proc->Modules[i].Size;
if (base && size && rip >= base && rip < base + size)
return proc->Modules[i].Name;
}
return "unknown";
}
static void PrintCrashChar(char c) {
PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic
}
_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
THREAD* thread = KeGetCurrentThread();
PROCESS* proc = thread ? thread->Owner : (PROCESS*)0;
// Snapshot everything we need before touching process state.
const char* faultName = FaultVectorName(vector);
const char* procName = proc ? proc->Name : "unknown";
uint32_t pid = proc ? proc->ProcessId : 0;
const char* modName = proc ? ResolveModule(proc, rip) : "unknown";
// Amber crash banner.
static const char banner[] = "\n*** APPLICATION CRASH ***\n";
for (const char* p = banner; *p; p++)
PrintCrashChar(*p);
printf_set_putchar(PrintCrashChar);
if (vector == 14)
printf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
procName, pid, faultName,
modName,
(unsigned long long)rip,
(unsigned long long)errCode,
(unsigned long long)cr2);
else
printf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx\n",
procName, pid, faultName,
modName,
(unsigned long long)rip,
(unsigned long long)errCode);
// Restore normal (green) output for subsequent kernel messages.
printf_set_putchar(kernel_putchar);
// Kill the process and exit this thread.
// KeExitThread sets ThreadDead then calls KeYield; the scheduler will
// never return here, so the process crash cannot affect anything else.
if (proc) KeExitProcess(proc, -1);
if (thread) KeExitThread(thread, -1);
// Unreachable - KeExitThread never returns for a dead thread.
while (1) __halt();
}
// ---------------------------------------------------------------------------
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
//
// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and
// call here. We check the CS RPL to decide: kernel fault → KePanic (halt),
// user fault → KeBugCheckProcess (kill process, resume scheduler).
// ---------------------------------------------------------------------------
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
uintptr_t errCode, uintptr_t cr2,
uintptr_t cs) {
if ((cs & 3) == 0) {
// Kernel-mode fault: unrecoverable.
switch (vector) {
case 0:
KePanic("PANIC: #DE Divide Error | RIP: %016llx\n",
(unsigned long long)rip);
case 13:
KePanic("PANIC: #GP General Protection | RIP: %016llx | Code: %04llx\n",
(unsigned long long)rip, (unsigned long long)errCode);
case 14:
KePanic("PANIC: #PF Page Fault | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
(unsigned long long)rip, (unsigned long long)errCode,
(unsigned long long)cr2);
default:
KePanic("PANIC: Exception #%d | RIP: %016llx\n",
vector, (unsigned long long)rip);
}
}
// User-mode fault: recoverable - kill the process, resume the scheduler.
KeBugCheckProcess(vector, rip, errCode, cr2);
}
// ke_debug.c - debugging utilities
#include <stdarg.h>
#include <stdint.h>
#include <string.h>
#include <hal_api.h>
#include <ke_spinlock.h>
#include <ke_process.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <printf.h>
int _fltused; // fuckass
int gDebugEnabled = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
void KiKernelPutChar(char c) {
PrintChar(c, 0x00FF00, 0x000000);
}
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), int v) {
gDebugEnabled = v;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
}
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (!gDebugEnabled) return;
static SPINLOCK sl = {0};
uintptr_t flags = KeAcquireSpinlock(&sl);
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
KeReleaseSpinlock(&sl, flags);
}
// KePanic
__declspec(dllexport) _Noreturn void KePanic(const char* fmt, ...) {
// Silence! Stop the world!
_disable();
// Print a bright-red banner so the framebuffer is unmistakable.
// We call PrintChar directly so we can set a red foreground.
// (DbgPrint uses green; we want this visually distinct.)
static const char banner[] = "\n*** KERNEL PANIC ***\n";
for (const char* p = banner; *p; p++)
PrintChar(*p, 0xFF2222, 0x000000);
// Now print the caller's message through the normal vprintf path.
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
// Die!
while (true) __halt();
}
// KePrintConsole - write user-visible console text in white-on-black.
// Used by SYS_WRITE_CONSOLE so user programs get a distinct color from
// kernel debug output (green) and panic output (red).
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
if (!buf || !PrintChar) return;
for (size_t i = 0; i < len; i++)
PrintChar(buf[i], 0xFFFFFF, 0x000000);
}
// ---------------------------------------------------------------------------
// KeBugCheckProcess - user-mode fault handler
//
// Resolves the faulting RIP to a module name, prints a yellow crash banner,
// kills the offending process, then exits the current thread so the scheduler
// can continue running everything else.
// ---------------------------------------------------------------------------
// Return the short exception name for a vector number.
static const char* KiFaultVectorName(int vector) {
switch (vector) {
case 0: return "#DE Divide Error";
case 13: return "#GP General Protection";
case 14: return "#PF Page Fault";
default: return "Unknown Exception";
}
}
// Walk the process module list to find which image owns rip.
// Returns the module name string, or "unknown" if rip is outside all regions.
static const char* KiResolveModule(PROCESS* proc, uintptr_t rip) {
// Check the main executable image first.
if (proc->ImageBase && proc->ImageSize &&
rip >= proc->ImageBase && rip < proc->ImageBase + proc->ImageSize)
return proc->Name;
// Walk the per-process module table (DLLs / shared libraries).
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
uintptr_t base = proc->Modules[i].Base;
size_t size = proc->Modules[i].Size;
if (base && size && rip >= base && rip < base + size)
return proc->Modules[i].Name;
}
return "unknown";
}
static void KiPrintCrashChar(char c) {
PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic
}
_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
THREAD* thread = KeGetCurrentThread();
PROCESS* proc = thread ? thread->Owner : (PROCESS*)0;
// Snapshot everything we need before touching process state.
const char* faultName = KiFaultVectorName(vector);
const char* procName = proc ? proc->Name : "unknown";
uint32_t pid = proc ? proc->ProcessId : 0;
const char* modName = proc ? KiResolveModule(proc, rip) : "unknown";
// Amber crash banner.
static const char banner[] = "\n*** APPLICATION CRASH ***\n";
for (const char* p = banner; *p; p++)
KiPrintCrashChar(*p);
printf_set_putchar(KiPrintCrashChar);
if (vector == 14)
printf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
procName, pid, faultName,
modName,
(unsigned long long)rip,
(unsigned long long)errCode,
(unsigned long long)cr2);
else
printf("Process: %s (PID %u) | Fault: %s\n"
"Module: %s | RIP: %016llx | Code: %04llx\n",
procName, pid, faultName,
modName,
(unsigned long long)rip,
(unsigned long long)errCode);
// Restore normal (green) output for subsequent kernel messages.
printf_set_putchar(KiKernelPutChar);
// Kill the process and exit this thread.
// KeExitThread sets ThreadDead then calls KeYield; the scheduler will
// never return here, so the process crash cannot affect anything else.
if (proc) KeExitProcess(proc, -1);
if (thread) KeExitThread(thread, -1);
// Unreachable - KeExitThread never returns for a dead thread.
while (1) __halt();
}
// ---------------------------------------------------------------------------
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
//
// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and
// call here. We check the CS RPL to decide: kernel fault → KePanic (halt),
// user fault → KeBugCheckProcess (kill process, resume scheduler).
// ---------------------------------------------------------------------------
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
uintptr_t errCode, uintptr_t cr2,
uintptr_t cs) {
if ((cs & 3) == 0) {
// Kernel-mode fault: unrecoverable.
switch (vector) {
case 0:
KePanic("PANIC: #DE Divide Error | RIP: %016llx\n",
(unsigned long long)rip);
case 13:
KePanic("PANIC: #GP General Protection | RIP: %016llx | Code: %04llx\n",
(unsigned long long)rip, (unsigned long long)errCode);
case 14:
KePanic("PANIC: #PF Page Fault | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
(unsigned long long)rip, (unsigned long long)errCode,
(unsigned long long)cr2);
default:
KePanic("PANIC: Exception #%d | RIP: %016llx\n",
vector, (unsigned long long)rip);
}
}
// User-mode fault: recoverable - kill the process, resume the scheduler.
KeBugCheckProcess(vector, rip, errCode, cr2);
}
code/kernel/ke_dpc.c
+104
View File
@@ -0,0 +1,104 @@
// ke_dpc.c - deferred procedure calls
#include <ke_dpc.h>
#include <ke_irql.h>
#include <ke_debug.h>
#include <intrin.h>
static LIST_ENTRY gDpcQueue;
static SPINLOCK gDpcLock;
static volatile long gDpcInitialized = 0;
static volatile long gDpcActive = 0;
void KeInitializeDpcSystem(void) {
LIST_INIT(&gDpcQueue);
gDpcLock = (SPINLOCK)SPINLOCK_INIT;
gDpcActive = 0;
gDpcInitialized = 1;
}
static void KiEnsureDpcInitialized(void) {
if (gDpcInitialized)
return;
// Single-CPU bring-up path: this is only a safety net for early callers.
KeInitializeDpcSystem();
}
__declspec(dllexport) void KeInitializeDpc(KDPC* dpc,
KDEFERRED_ROUTINE routine,
void* context) {
if (!dpc)
return;
LIST_INIT(&dpc->Link);
dpc->Routine = routine;
dpc->Context = context;
dpc->SystemArgument1 = (void*)0;
dpc->SystemArgument2 = (void*)0;
dpc->Inserted = 0;
}
__declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
void* systemArgument1,
void* systemArgument2) {
if (!dpc || !dpc->Routine)
return false;
KiEnsureDpcInitialized();
// Coalesce repeated inserts of the same KDPC. The first insert owns the
// argument pair until the DPC is popped by the dispatcher.
if (_InterlockedCompareExchange(&dpc->Inserted, 1, 0) != 0)
return false;
dpc->SystemArgument1 = systemArgument1;
dpc->SystemArgument2 = systemArgument2;
uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
ListInsertTail(&gDpcQueue, &dpc->Link);
KeReleaseSpinlock(&gDpcLock, flags);
return true;
}
void KiDrainDpcQueue(void) {
if (!gDpcInitialized)
return;
// Never run DPC routines above DISPATCH_LEVEL. If a future nested IRQ path
// reaches here at DIRQL, leave the work pending for the outermost exit.
if (KeGetCurrentIrql() > DISPATCH_LEVEL)
return;
if (_InterlockedCompareExchange(&gDpcActive, 1, 0) != 0)
return;
KIRQL oldIrql = KeGetCurrentIrql();
if (oldIrql < DISPATCH_LEVEL)
KeRaiseIrql(DISPATCH_LEVEL);
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
LIST_ENTRY* node = ListPopHead(&gDpcQueue);
if (!node) {
KeReleaseSpinlock(&gDpcLock, flags);
break;
}
KDPC* dpc = CONTAINING_RECORD(node, KDPC, Link);
void* arg1 = dpc->SystemArgument1;
void* arg2 = dpc->SystemArgument2;
dpc->SystemArgument1 = (void*)0;
dpc->SystemArgument2 = (void*)0;
dpc->Inserted = 0;
KeReleaseSpinlock(&gDpcLock, flags);
KeAssertIrql(DISPATCH_LEVEL);
dpc->Routine(dpc, dpc->Context, arg1, arg2);
KeAssertIrql(DISPATCH_LEVEL);
}
gDpcActive = 0;
if (oldIrql < DISPATCH_LEVEL)
KeLowerIrql(oldIrql);
}
code/kernel/ke_dpc.h
+44
View File
@@ -0,0 +1,44 @@
#pragma once
// ke_dpc.h - deferred procedure calls
//
// DPCs are the kernel's bottom-half mechanism. Hardware ISRs can queue a DPC
// after acknowledging/capturing device state; the DPC then runs later at
// DISPATCH_LEVEL from the common interrupt-exit path. DPC routines must not
// wait, block, or call PASSIVE_LEVEL-only code.
#include <stdint.h>
#include <stdbool.h>
#include <ke_list.h>
#include <ke_spinlock.h>
typedef struct KDPC KDPC;
typedef void (*KDEFERRED_ROUTINE)(KDPC* dpc,
void* context,
void* systemArgument1,
void* systemArgument2);
struct KDPC {
LIST_ENTRY Link;
KDEFERRED_ROUTINE Routine;
void* Context;
void* SystemArgument1;
void* SystemArgument2;
volatile long Inserted;
};
// Called during kernel bring-up before interrupts are enabled.
void KeInitializeDpcSystem(void);
__declspec(dllexport) void KeInitializeDpc(KDPC* dpc,
KDEFERRED_ROUTINE routine,
void* context);
// Queue a DPC if it is not already queued. Returns false if the DPC was
// already pending or the arguments are invalid.
__declspec(dllexport) bool KeInsertQueueDpc(KDPC* dpc,
void* systemArgument1,
void* systemArgument2);
// Internal: drain queued DPCs at DISPATCH_LEVEL. Called from interrupt exit.
void KiDrainDpcQueue(void);
code/kernel/ke_event.c
+81 -71
View File
@@ -1,71 +1,81 @@
// ke_event.c - synchronisation events
#include <stdbool.h>
#include <ke_event.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ob_object.h>
static void EventDestroy(OBJECT* obj) { (void)obj; }
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
if (!out) return STATUS_INVALID_PARAMETER;
EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
if (!e) return STATUS_NO_MEMORY;
e->Header.Destroy = EventDestroy;
e->Lock = (SPINLOCK)SPINLOCK_INIT;
e->Type = type;
e->Signaled = initialState ? 1 : 0;
LIST_INIT(&e->WaitList);
*out = e;
return STATUS_SUCCESS;
}
__declspec(dllexport) void KeSignalEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (event->Type == EventManualReset) {
event->Signaled = 1;
while (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThread(t);
flags = KeAcquireSpinlock(&event->Lock);
}
} else {
event->Signaled = 1;
if (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThread(t);
return;
}
}
KeReleaseSpinlock(&event->Lock, flags);
}
__declspec(dllexport) void KeClearEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
event->Signaled = 0;
KeReleaseSpinlock(&event->Lock, flags);
}
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event) {
if (!event) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
while (true) {
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (event->Signaled) {
if (event->Type == EventAutoReset) event->Signaled = 0;
KeReleaseSpinlock(&event->Lock, flags);
return STATUS_SUCCESS;
}
ListInsertTail(&event->WaitList, &self->SchedulerLink);
KeReleaseSpinlock(&event->Lock, flags);
KeBlockThread(self, &event->Header);
}
}
// ke_event.c - synchronisation events
#include <stdbool.h>
#include <ke_event.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ke_irql.h>
#include <ke_apc.h>
#include <ob_object.h>
static void KiEventDestroy(OBJECT* obj) { (void)obj; }
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
if (!out) return STATUS_INVALID_PARAMETER;
EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
if (!e) return STATUS_NO_MEMORY;
e->Header.Destroy = KiEventDestroy;
e->Lock = (SPINLOCK)SPINLOCK_INIT;
e->Type = type;
e->Signaled = initialState ? 1 : 0;
LIST_INIT(&e->WaitList);
*out = e;
return STATUS_SUCCESS;
}
__declspec(dllexport) void KeSignalEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (event->Type == EventManualReset) {
event->Signaled = 1;
while (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
flags = KeAcquireSpinlock(&event->Lock);
}
} else {
if (!LIST_IS_EMPTY(&event->WaitList)) {
// Auto-reset handoff: transfer the signal directly to exactly one
// waiter. Do not leave Signaled=1, because a later waiter can steal
// the disk/APC completion before the selected thread resumes.
event->Signaled = 0;
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
return;
}
event->Signaled = 1;
}
KeReleaseSpinlock(&event->Lock, flags);
}
__declspec(dllexport) void KeClearEvent(EVENT* event) {
if (!event) return;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
event->Signaled = 0;
KeReleaseSpinlock(&event->Lock, flags);
}
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
if (!event) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (alertable && KeAreKernelApcsDisabled()) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (event->Signaled) {
if (event->Type == EventAutoReset) event->Signaled = 0;
KeReleaseSpinlock(&event->Lock, flags);
return STATUS_SUCCESS;
}
KePrepareBlockThread(self, &event->Header, alertable);
ListInsertTail(&event->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &event->WaitList);
KeReleaseSpinlock(&event->Lock, flags);
return KeCommitBlockThread(self, alertable);
}
code/kernel/ke_event.h
+23 -22
View File
@@ -1,22 +1,23 @@
#pragma once
// ke_event.h - auto-reset and manual-reset events
#include <stdint.h>
#include <ob_object.h>
#include <ke_spinlock.h>
#include <ke_list.h>
#include <ke_status.h>
typedef enum EVENT_TYPE { EventAutoReset, EventManualReset } EVENT_TYPE;
typedef struct EVENT {
OBJECT Header;
SPINLOCK Lock;
EVENT_TYPE Type;
uint32_t Signaled;
LIST_ENTRY WaitList;
} EVENT;
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out);
__declspec(dllexport) void KeSignalEvent(EVENT* event);
__declspec(dllexport) void KeClearEvent(EVENT* event);
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event);
#pragma once
// ke_event.h - auto-reset and manual-reset events
#include <stdint.h>
#include <ob_object.h>
#include <ke_spinlock.h>
#include <ke_list.h>
#include <ke_status.h>
typedef enum EVENT_TYPE { EventAutoReset, EventManualReset } EVENT_TYPE;
typedef struct EVENT {
OBJECT Header;
SPINLOCK Lock;
EVENT_TYPE Type;
uint32_t Signaled;
LIST_ENTRY WaitList;
} EVENT;
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out);
__declspec(dllexport) void KeSignalEvent(EVENT* event);
__declspec(dllexport) void KeClearEvent(EVENT* event);
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable);
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable);
code/kernel/ke_idt.c
+175 -175
View File
@@ -1,175 +1,175 @@
// ke_idt.c - GDT, IDT, and TSS initialisation
//
// GDT layout (matches segment selectors in ke_switch.asm / ke_syscall.asm):
// 0x00 null
// 0x08 ring-0 code (64-bit)
// 0x10 ring-0 data
// 0x18 ring-3 code (64-bit, DPL=3)
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x28 TSS low
// 0x30 TSS high
#include <stdint.h>
#include <string.h>
#include <ke_idt.h>
// External assembly procedures
extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
extern void KeFlushIdt(void* pIdtr);
extern void KeIrqStub0(void);
extern void KeIrqStub1(void);
extern void KeIrqStub2(void);
extern void KeIrqStub3(void);
extern void KeIrqStub4(void);
extern void KeIrqStub5(void);
extern void KeIrqStub6(void);
extern void KeIrqStub7(void);
extern void KeIrqStub8(void);
extern void KeIrqStub9(void);
extern void KeIrqStub10(void);
extern void KeIrqStub11(void);
extern void KeIrqStub12(void);
extern void KeIrqStub13(void);
extern void KeIrqStub14(void);
extern void KeIrqStub15(void);
extern void KeIrqStub16(void);
extern void KeIrqStub17(void);
extern void KeIrqStub18(void);
extern void KeIrqStub19(void);
extern void KeIrqStub20(void);
extern void KeIrqStub21(void);
extern void KeIrqStub22(void);
extern void KeIrqStub23(void);
#define NUM_IRQS 24
// ---------------------------------------------------------------------------
// GDT
// ---------------------------------------------------------------------------
#define GDT_ENTRIES 8
#define KERNEL_IST_STACK_SIZE 0x4000
static uint64_t gGdt[GDT_ENTRIES];
static uint8_t gKernelStack[KERNEL_IST_STACK_SIZE];
#pragma pack(push, 1)
typedef struct { uint16_t Limit; uint64_t Base; } GDTR;
#pragma pack(pop)
static void GdtSetEntry(int idx, uint32_t base, uint32_t limit,
uint8_t access, uint8_t flags) {
gGdt[idx] =
((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
((uint64_t)access << 40) |
((uint64_t)(limit >> 16 & 0xF)<< 48) |
((uint64_t)(flags & 0xF) << 52) |
((uint64_t)(base >> 24 & 0xFF)<< 56);
}
static void GdtSetTss(int idx, uintptr_t base, uint32_t limit) {
// TSS descriptor is 16 bytes (two GDT slots).
gGdt[idx] = 0;
gGdt[idx+1] = 0;
gGdt[idx] =
((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
(0x89ULL << 40) | // P=1,DPL=0,type=TSS64 avail
((uint64_t)((limit>>16)&0xF) << 48) |
((uint64_t)(base>>24 & 0xFF) << 56);
gGdt[idx+1] = (base >> 32);
}
TSS64 gTssStruct;
TSS64* gTss = &gTssStruct;
void KeInitializeGdt(void) {
memset(gGdt, 0, sizeof(gGdt));
memset(&gTssStruct, 0, sizeof(gTssStruct));
// 0x00: Null
GdtSetEntry(0, 0, 0, 0, 0);
// 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit)
GdtSetEntry(1, 0, 0xFFFFF, 0x9A, 0xA);
// 0x10: Ring-0 Data
GdtSetEntry(2, 0, 0xFFFFF, 0x92, 0xC);
// 0x18: Ring-3 Code
GdtSetEntry(3, 0, 0xFFFFF, 0xFA, 0xA);
// 0x20: Ring-3 Data
GdtSetEntry(4, 0, 0xFFFFF, 0xF2, 0xC);
// 0x28: TSS (Occupies two slots: 0x28 and 0x30)
gTssStruct.IoMapBase = sizeof(TSS64);
gTssStruct.RSP0 = (uint64_t)(gKernelStack + KERNEL_IST_STACK_SIZE); // top of stack
GdtSetTss(5, (uintptr_t)&gTssStruct, sizeof(TSS64) - 1);
GDTR gdtr = { sizeof(gGdt) - 1, (uint64_t)gGdt };
// Call the MASM64 proc to reload everything
// Selectors: CS=0x08, Data=0x10, TSS=0x28
KeFlushGdt(&gdtr, 0x08, 0x10, 0x28);
}
// ---------------------------------------------------------------------------
// IDT
// ---------------------------------------------------------------------------
#define IDT_ENTRIES 256
#pragma pack(push, 1)
typedef struct {
uint16_t OffsetLow;
uint16_t Selector;
uint8_t Ist;
uint8_t TypeAttr;
uint16_t OffsetMid;
uint32_t OffsetHigh;
uint32_t Reserved;
} IDT_ENTRY;
#pragma pack(pop)
static IDT_ENTRY gIdt[IDT_ENTRIES];
#pragma pack(push, 1)
typedef struct { uint16_t Limit; uint64_t Base; } IDTR;
#pragma pack(pop)
static void IdtSetGate(int vec, uintptr_t handler,
uint8_t typeAttr, uint8_t ist) {
gIdt[vec].OffsetLow = (uint16_t)(handler & 0xFFFF);
gIdt[vec].Selector = 0x08; // ring-0 code segment
gIdt[vec].Ist = ist;
gIdt[vec].TypeAttr = typeAttr;
gIdt[vec].OffsetMid = (uint16_t)((handler >> 16) & 0xFFFF);
gIdt[vec].OffsetHigh = (uint32_t)((handler >> 32) & 0xFFFFFFFF);
gIdt[vec].Reserved = 0;
}
void KeInitializeIdt(void) {
memset(gIdt, 0, sizeof(gIdt));
IdtSetGate(0, (uintptr_t)KeFaultDE, 0x8E, 0);
IdtSetGate(13, (uintptr_t)KeFaultGP, 0x8E, 0);
IdtSetGate(14, (uintptr_t)KeFaultPF, 0x8E, 0);
IdtSetGate(0x40, (uintptr_t)KeSyscallEntry, 0xEE, 0);
static void *irqStubs[NUM_IRQS] = {
(void*)KeIrqStub0, (void*)KeIrqStub1, (void*)KeIrqStub2,
(void*)KeIrqStub3, (void*)KeIrqStub4, (void*)KeIrqStub5,
(void*)KeIrqStub6, (void*)KeIrqStub7, (void*)KeIrqStub8,
(void*)KeIrqStub9, (void*)KeIrqStub10, (void*)KeIrqStub11,
(void*)KeIrqStub12, (void*)KeIrqStub13, (void*)KeIrqStub14,
(void*)KeIrqStub15, (void*)KeIrqStub16, (void*)KeIrqStub17,
(void*)KeIrqStub18, (void*)KeIrqStub19, (void*)KeIrqStub20,
(void*)KeIrqStub21, (void*)KeIrqStub22, (void*)KeIrqStub23,
};
for (int i = 0; i < NUM_IRQS; i++)
IdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], 0x8E, 0);
IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt };
// Call the MASM64 proc to load IDT
KeFlushIdt(&idtr);
}
// ke_idt.c - GDT, IDT, and TSS initialisation
//
// GDT layout (matches segment selectors in ke_switch.asm / ke_syscall.asm):
// 0x00 null
// 0x08 ring-0 code (64-bit)
// 0x10 ring-0 data
// 0x18 ring-3 code (64-bit, DPL=3)
// 0x20 ring-3 data (DPL=3) - note: 0x1B/0x23 = 0x18|3 / 0x20|3
// 0x28 TSS low
// 0x30 TSS high
#include <stdint.h>
#include <string.h>
#include <ke_idt.h>
// External assembly procedures
extern void KeFlushGdt(void* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel);
extern void KeFlushIdt(void* pIdtr);
extern void KeIrqStub0(void);
extern void KeIrqStub1(void);
extern void KeIrqStub2(void);
extern void KeIrqStub3(void);
extern void KeIrqStub4(void);
extern void KeIrqStub5(void);
extern void KeIrqStub6(void);
extern void KeIrqStub7(void);
extern void KeIrqStub8(void);
extern void KeIrqStub9(void);
extern void KeIrqStub10(void);
extern void KeIrqStub11(void);
extern void KeIrqStub12(void);
extern void KeIrqStub13(void);
extern void KeIrqStub14(void);
extern void KeIrqStub15(void);
extern void KeIrqStub16(void);
extern void KeIrqStub17(void);
extern void KeIrqStub18(void);
extern void KeIrqStub19(void);
extern void KeIrqStub20(void);
extern void KeIrqStub21(void);
extern void KeIrqStub22(void);
extern void KeIrqStub23(void);
#define NUM_IRQS 24
// ---------------------------------------------------------------------------
// GDT
// ---------------------------------------------------------------------------
#define GDT_ENTRIES 8
#define KERNEL_IST_STACK_SIZE 0x4000
static uint64_t gGdt[GDT_ENTRIES];
static uint8_t gKernelStack[KERNEL_IST_STACK_SIZE];
#pragma pack(push, 1)
typedef struct { uint16_t Limit; uint64_t Base; } GDTR;
#pragma pack(pop)
static void KiGdtSetEntry(int idx, uint32_t base, uint32_t limit,
uint8_t access, uint8_t flags) {
gGdt[idx] =
((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
((uint64_t)access << 40) |
((uint64_t)(limit >> 16 & 0xF)<< 48) |
((uint64_t)(flags & 0xF) << 52) |
((uint64_t)(base >> 24 & 0xFF)<< 56);
}
static void KiGdtSetTss(int idx, uintptr_t base, uint32_t limit) {
// TSS descriptor is 16 bytes (two GDT slots).
gGdt[idx] = 0;
gGdt[idx+1] = 0;
gGdt[idx] =
((uint64_t)(limit & 0xFFFF)) |
((uint64_t)(base & 0xFFFFFF) << 16) |
(0x89ULL << 40) | // P=1,DPL=0,type=TSS64 avail
((uint64_t)((limit>>16)&0xF) << 48) |
((uint64_t)(base>>24 & 0xFF) << 56);
gGdt[idx+1] = (base >> 32);
}
TSS64 gTssStruct;
TSS64* gTss = &gTssStruct;
void KeInitializeGdt(void) {
memset(gGdt, 0, sizeof(gGdt));
memset(&gTssStruct, 0, sizeof(gTssStruct));
// 0x00: Null
KiGdtSetEntry(0, 0, 0, 0, 0);
// 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit)
KiGdtSetEntry(1, 0, 0xFFFFF, 0x9A, 0xA);
// 0x10: Ring-0 Data
KiGdtSetEntry(2, 0, 0xFFFFF, 0x92, 0xC);
// 0x18: Ring-3 Code
KiGdtSetEntry(3, 0, 0xFFFFF, 0xFA, 0xA);
// 0x20: Ring-3 Data
KiGdtSetEntry(4, 0, 0xFFFFF, 0xF2, 0xC);
// 0x28: TSS (Occupies two slots: 0x28 and 0x30)
gTssStruct.IoMapBase = sizeof(TSS64);
gTssStruct.RSP0 = (uint64_t)(gKernelStack + KERNEL_IST_STACK_SIZE); // top of stack
KiGdtSetTss(5, (uintptr_t)&gTssStruct, sizeof(TSS64) - 1);
GDTR gdtr = { sizeof(gGdt) - 1, (uint64_t)gGdt };
// Call the MASM64 proc to reload everything
// Selectors: CS=0x08, Data=0x10, TSS=0x28
KeFlushGdt(&gdtr, 0x08, 0x10, 0x28);
}
// ---------------------------------------------------------------------------
// IDT
// ---------------------------------------------------------------------------
#define IDT_ENTRIES 256
#pragma pack(push, 1)
typedef struct {
uint16_t OffsetLow;
uint16_t Selector;
uint8_t Ist;
uint8_t TypeAttr;
uint16_t OffsetMid;
uint32_t OffsetHigh;
uint32_t Reserved;
} IDT_ENTRY;
#pragma pack(pop)
static IDT_ENTRY gIdt[IDT_ENTRIES];
#pragma pack(push, 1)
typedef struct { uint16_t Limit; uint64_t Base; } IDTR;
#pragma pack(pop)
static void KiIdtSetGate(int vec, uintptr_t handler,
uint8_t typeAttr, uint8_t ist) {
gIdt[vec].OffsetLow = (uint16_t)(handler & 0xFFFF);
gIdt[vec].Selector = 0x08; // ring-0 code segment
gIdt[vec].Ist = ist;
gIdt[vec].TypeAttr = typeAttr;
gIdt[vec].OffsetMid = (uint16_t)((handler >> 16) & 0xFFFF);
gIdt[vec].OffsetHigh = (uint32_t)((handler >> 32) & 0xFFFFFFFF);
gIdt[vec].Reserved = 0;
}
void KeInitializeIdt(void) {
memset(gIdt, 0, sizeof(gIdt));
KiIdtSetGate(0, (uintptr_t)KeFaultDE, 0x8E, 0);
KiIdtSetGate(13, (uintptr_t)KeFaultGP, 0x8E, 0);
KiIdtSetGate(14, (uintptr_t)KeFaultPF, 0x8E, 0);
KiIdtSetGate(0x40, (uintptr_t)KeSyscallEntry, 0xEE, 0);
static void *irqStubs[NUM_IRQS] = {
(void*)KeIrqStub0, (void*)KeIrqStub1, (void*)KeIrqStub2,
(void*)KeIrqStub3, (void*)KeIrqStub4, (void*)KeIrqStub5,
(void*)KeIrqStub6, (void*)KeIrqStub7, (void*)KeIrqStub8,
(void*)KeIrqStub9, (void*)KeIrqStub10, (void*)KeIrqStub11,
(void*)KeIrqStub12, (void*)KeIrqStub13, (void*)KeIrqStub14,
(void*)KeIrqStub15, (void*)KeIrqStub16, (void*)KeIrqStub17,
(void*)KeIrqStub18, (void*)KeIrqStub19, (void*)KeIrqStub20,
(void*)KeIrqStub21, (void*)KeIrqStub22, (void*)KeIrqStub23,
};
for (int i = 0; i < NUM_IRQS; i++)
KiIdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], 0x8E, 0);
IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt };
// Call the MASM64 proc to load IDT
KeFlushIdt(&idtr);
}
code/kernel/ke_irp.c
+45 -27
View File
@@ -10,12 +10,12 @@
// → ke_event.h (KeEventCreate, KeSignalEvent, KeWaitForEvent)
// → ke_semaphore.h (KeSignalSemaphore)
// → ob_object.h (ObDereferenceObject)
// → io_driver.h (HalDiskDispatch)
// hal_disk.c → io_driver.h only (no ke_event.h, no ke_semaphore.h)
// → io_driver.h (DEVICE_OBJECT / IoCallDriver)
#include <ke_irp.h>
#include <ke_event.h>
#include <ke_semaphore.h>
#include <ke_apc.h>
#include <ob_object.h>
#include <io_driver.h>
@@ -51,33 +51,53 @@ __declspec(dllexport) IRP *IrpQueueDequeue(IRP_QUEUE *q) {
return CONTAINING_RECORD(node, IRP, Link);
}
// -------------------------------------------------------------------------
// IoCompleteRequest
// -------------------------------------------------------------------------
static size_t IoIrpTransferBytes(const IRP *irp, KSTATUS status) {
if (!irp || !KSUCCESS(status)) return 0;
return (size_t)irp->SectorCount * 512u;
}
__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information) {
if (!irp) return;
irp->Status = status;
irp->Information = information;
// Queue the APC before waking event waiters so a thread that waits on the
// completion event will see and deliver the APC on its wait-return path.
if (irp->CompletionApc)
KeInsertQueueApc((KAPC *)irp->CompletionApc, irp, (void *)(uintptr_t)status);
if (irp->CompletionEvent)
KeSignalEvent((EVENT *)irp->CompletionEvent);
if (irp->CompletionFn)
irp->CompletionFn(irp, irp->CompletionCtx);
}
// -------------------------------------------------------------------------
// IoSubmitIrp
// -------------------------------------------------------------------------
// Calls HalDiskDispatch() which is HAL-layer safe (no event references).
// If the dispatch ran inline (sync fallback), we fire the completion hooks
// here in the kernel layer where we can legally call KeSignalEvent.
// Dispatches through the registered DEVICE_OBJECT.
// Accepted IRPs are always completed by the target driver via IoCompleteRequest().
__declspec(dllexport) KSTATUS IoSubmitIrp(IRP *irp) {
if (!irp) return STATUS_INVALID_PARAMETER;
KSTATUS st = HalDiskDispatch(irp->DriveIndex, irp);
irp->Status = STATUS_FAILURE;
irp->Information = 0;
if (st == STATUS_NOT_FOUND)
return STATUS_NOT_FOUND;
DEVICE_OBJECT *device = IoGetDiskDevice(irp->DriveIndex);
if (!device) return STATUS_NOT_FOUND;
if (st == STATUS_SUCCESS) {
// Sync fallback ran; irp->Status is already written.
// Fire completion hooks now - we are in kernel context so this is safe.
if (irp->CompletionEvent)
KeSignalEvent((EVENT *)irp->CompletionEvent);
if (irp->CompletionFn)
irp->CompletionFn(irp, irp->CompletionCtx);
}
KSTATUS st = IoCallDriver(device, irp);
// Return STATUS_PENDING in both cases so callers have a uniform contract:
// "completion is signalled via the event/callback, not this return value."
return STATUS_PENDING;
// Definitive driver contract:
// STATUS_PENDING means the IRP was accepted and will be completed via
// IoCompleteRequest(), possibly inline before IoCallDriver returns.
// Any other status means the IRP was rejected and no completion occurs.
return st;
}
// -------------------------------------------------------------------------
@@ -94,19 +114,17 @@ __declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp) {
KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev);
if (!KSUCCESS(st)) return st;
// Attach our event; preserve any callback the caller may have set.
// Attach our event; preserve any APC/callback the caller may have set.
irp->CompletionEvent = ev;
st = IoSubmitIrp(irp);
// IoSubmitIrp always returns STATUS_PENDING on the non-error paths.
// The event is signalled either by the driver worker (async) or by
// IoSubmitIrp itself (sync fallback), so KeWaitForEvent will always
// return promptly in the sync-fallback case.
if (st == STATUS_PENDING)
KeWaitForEvent(ev);
if (st == STATUS_PENDING) {
KeWaitForEvent(ev, 0);
st = irp->Status;
}
// Release our reference to the temporary event object.
ObDereferenceObject(&ev->Header);
return irp->Status;
return st;
}
code/kernel/ke_irp.h
+14 -1
View File
@@ -12,6 +12,7 @@
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <ke_list.h>
#include <ke_spinlock.h>
#include <ke_status.h>
@@ -20,6 +21,7 @@
// We only ever hold pointers in IRP and IRP_QUEUE, so a forward decl suffices.
struct EVENT;
struct SEMAPHORE;
struct KAPC;
// -------------------------------------------------------------------------
// IRP flags
@@ -34,7 +36,9 @@ struct SEMAPHORE;
typedef struct IRP IRP;
// Completion callback: invoked in driver worker context; must not block.
// Completion callback: invoked by IoCompleteRequest() in the completing
// context. Today that is the driver worker or sync fallback path; future
// DPC/APC work will keep all completion policy centralized here.
typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx);
// -------------------------------------------------------------------------
@@ -56,10 +60,15 @@ typedef struct IRP {
// ---- Completion (caller fills before submit; driver writes Status) ----
volatile KSTATUS Status;
volatile size_t Information; // bytes transferred on success
// Forward-declared pointer: callers that create events include ke_event.h.
struct EVENT *CompletionEvent;
// Optional kernel APC. If set, IoCompleteRequest queues it to its
// target thread with arg1=irp and arg2=(void*)(uintptr_t)status.
struct KAPC *CompletionApc;
IRP_COMPLETION_FN CompletionFn;
void *CompletionCtx;
@@ -88,6 +97,10 @@ __declspec(dllexport) IRP *IrpQueueDequeue (IRP_QUEUE *q);
// Public dispatch API - implemented in kernel/ke_irp.c
// -------------------------------------------------------------------------
// Complete an IRP exactly once: publish status/information, signal the
// optional completion event, then invoke the optional completion callback.
__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information);
// Submit asynchronously. Returns STATUS_PENDING on success.
__declspec(dllexport) KSTATUS IoSubmitIrp (IRP *irp);
code/kernel/ke_irq.c
+72 -30
View File
@@ -1,30 +1,72 @@
// ke_irq.c - IRQ dispatch, kernel-side
#include <ke_irq.h>
#include <hal_api.h>
#define MAX_IRQS 24
static void (*gIrqHandlers[MAX_IRQS])(void);
static const HAL_API *gHal;
void KeIrqInitialize(const HAL_API *hal) {
gHal = hal;
}
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
gIrqHandlers[irq] = handler;
gHal->Intc.RouteIrq(irq);
}
void KeIrqDispatch(uint8_t irq) {
// KeIrqDispatch sends EOI after the handler returns, but the handler switched context before returning.
// The EOI call is still sitting on the idle thread's stack, waiting for that thread to be resumed.
// The LAPIC sees ISR[0x20] still set. Its rule is: don't deliver another interrupt at the same or lower priority until EOI is written.
// So the timer never fires again. The user process runs completely unpreempted until it calls SYS_EXIT_PROCESS, which calls KeYield, which cooperatively switches back
// and only then does the idle thread resume, KeIrqDispatch finally sends its EOI, and the timer starts firing again.
// To fix this we send EOI before calling the handler, not after.
gHal->Intc.SendEoi();
if (irq < MAX_IRQS && gIrqHandlers[irq])
gIrqHandlers[irq]();
}
// ke_irq.c - IRQ dispatch, kernel-side
#include <ke_irq.h>
#include <ke_sched.h>
#include <ke_irql.h>
#include <ke_dpc.h>
#include <hal_api.h>
#define MAX_IRQS 24
static void (*gIrqHandlers[MAX_IRQS])(void);
static const HAL_API *gHal;
static volatile uint32_t gInterruptDepth = 0;
static KIRQL KiIrqToIrql(uint8_t irq) {
KIRQL irql = (KIRQL)(DIRQL_BASE + irq);
if (irql >= HIGH_LEVEL)
irql = (KIRQL)(HIGH_LEVEL - 1);
return irql;
}
void KeIrqInitialize(const HAL_API *hal) {
gHal = hal;
}
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
gIrqHandlers[irq] = handler;
gHal->Intc.RouteIrq(irq);
}
void KeIrqDispatch(uint8_t irq) {
KIRQL targetIrql = KiIrqToIrql(irq);
KIRQL currentIrql = KeGetCurrentIrql();
// Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected
// while IF is clear, but this keeps the software state sane if that changes
// later.
if (targetIrql < currentIrql)
targetIrql = currentIrql;
KIRQL oldIrql = KeRaiseIrql(targetIrql);
if (irq < MAX_IRQS && gIrqHandlers[irq])
gIrqHandlers[irq]();
// Handlers are no longer allowed to context-switch directly. Timer IRQs
// only request rescheduling; KiExitInterrupt performs the switch after this
// function returns, so the EOI is guaranteed to be sent first. Keep the
// software level at DIRQL until after the controller has been acknowledged.
gHal->Intc.SendEoi();
KeLowerIrql(oldIrql);
}
void KiEnterInterrupt(void) {
gInterruptDepth++;
}
void KiExitInterrupt(void) {
if (gInterruptDepth)
gInterruptDepth--;
// A reschedule is only legal after the interrupt unwound back to the
// interrupted thread's original PASSIVE context. If a future path ever
// permits nested interrupting of DISPATCH/DIRQL code, leave the request
// pending instead of switching while a spinlock/high-IRQL region is active.
if (gInterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) {
KiDrainDpcQueue();
if (KeGetCurrentIrql() == PASSIVE_LEVEL)
KiScheduleFromInterrupt();
}
}
code/kernel/ke_irq.h
+7
View File
@@ -5,3 +5,10 @@
void KeIrqInitialize(const HAL_API *hal);
__declspec(dllexport) void KeIrqRegister(uint8_t irq, void (*handler)(void));
void KeIrqDispatch(uint8_t irq); // called from ke_syscall.asm
// Low-level interrupt entry/exit hooks. These are called by the common IRQ
// assembly stub around KeIrqDispatch. The exit hook is where deferred
// interrupt-time scheduling now happens, so IRQ handlers never switch away
// before KeIrqDispatch has returned and sent EOI.
void KiEnterInterrupt(void);
void KiExitInterrupt(void);
code/kernel/ke_irql.c
+82
View File
@@ -0,0 +1,82 @@
// ke_irql.c - software IRQL skeleton
#include <ke_irql.h>
#include <ke_debug.h>
#include <ke_apc.h>
// Single-CPU bring-up state. When SMP arrives this becomes per-CPU state.
static volatile KIRQL gCurrentIrql = PASSIVE_LEVEL;
__declspec(dllexport) const char* KeIrqlName(KIRQL irql) {
switch (irql) {
case PASSIVE_LEVEL: return "PASSIVE_LEVEL";
case APC_LEVEL: return "APC_LEVEL";
case DISPATCH_LEVEL: return "DISPATCH_LEVEL";
case HIGH_LEVEL: return "HIGH_LEVEL";
default:
if (irql >= DIRQL_BASE && irql < HIGH_LEVEL)
return "DIRQL";
return "INVALID_IRQL";
}
}
__declspec(dllexport) KIRQL KeGetCurrentIrql(void) {
return gCurrentIrql;
}
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql) {
KIRQL oldIrql = gCurrentIrql;
if (newIrql > HIGH_LEVEL) {
KePanic("KeRaiseIrql: invalid target IRQL %u\n", (unsigned)newIrql);
}
if (newIrql < oldIrql) {
KePanic("KeRaiseIrql: attempted raise from %s(%u) down to %s(%u)\n",
KeIrqlName(oldIrql), (unsigned)oldIrql,
KeIrqlName(newIrql), (unsigned)newIrql);
}
gCurrentIrql = newIrql;
return oldIrql;
}
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql) {
KIRQL currentIrql = gCurrentIrql;
if (oldIrql > HIGH_LEVEL) {
KePanic("KeLowerIrql: invalid target IRQL %u\n", (unsigned)oldIrql);
}
if (oldIrql > currentIrql) {
KePanic("KeLowerIrql: attempted lower from %s(%u) up to %s(%u)\n",
KeIrqlName(currentIrql), (unsigned)currentIrql,
KeIrqlName(oldIrql), (unsigned)oldIrql);
}
gCurrentIrql = oldIrql;
if (oldIrql == PASSIVE_LEVEL && currentIrql != PASSIVE_LEVEL)
KiDeliverKernelApcs();
}
__declspec(dllexport) void KeAssertIrql(KIRQL expected) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql != expected) {
KePanic("IRQL assertion failed: expected %s(%u), current %s(%u)\n",
KeIrqlName(expected), (unsigned)expected,
KeIrqlName(currentIrql), (unsigned)currentIrql);
}
}
__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql) {
KIRQL currentIrql = KeGetCurrentIrql();
if (currentIrql > maxIrql) {
KePanic("IRQL assertion failed: expected <= %s(%u), current %s(%u)\n",
KeIrqlName(maxIrql), (unsigned)maxIrql,
KeIrqlName(currentIrql), (unsigned)currentIrql);
}
}
__declspec(dllexport) int KeCanWait(void) {
return KeGetCurrentIrql() == PASSIVE_LEVEL;
}
code/kernel/ke_irql.h
+34
View File
@@ -0,0 +1,34 @@
#pragma once
// ke_irql.h - software IRQL skeleton
//
// This is intentionally small for the first IRQL/APC bring-up step. It does
// not yet program interrupt hardware or replace the spinlock interrupt-disable
// path. It gives the kernel one authoritative execution-level state plus
// assertions so blocking code can reject illegal callers early.
#include <stdint.h>
typedef uint8_t KIRQL;
#define PASSIVE_LEVEL ((KIRQL)0)
#define APC_LEVEL ((KIRQL)1)
#define DISPATCH_LEVEL ((KIRQL)2)
#define DIRQL_BASE ((KIRQL)16)
#define HIGH_LEVEL ((KIRQL)31)
__declspec(dllexport) KIRQL KeGetCurrentIrql(void);
// Raise to newIrql and return the previous level. Raising to a lower level is
// a kernel bug and will panic in checked/debug-style builds.
__declspec(dllexport) KIRQL KeRaiseIrql(KIRQL newIrql);
// Restore a previously returned level. Lowering to a higher level is a kernel
// bug and will panic.
__declspec(dllexport) void KeLowerIrql(KIRQL oldIrql);
__declspec(dllexport) void KeAssertIrql(KIRQL expected);
__declspec(dllexport) void KeAssertIrqlAtOrBelow(KIRQL maxIrql);
__declspec(dllexport) int KeCanWait(void);
// Internal/debug helper used by diagnostics and panic messages.
__declspec(dllexport) const char* KeIrqlName(KIRQL irql);
code/kernel/ke_mutex.c
+73 -66
View File
@@ -1,66 +1,73 @@
// ke_mutex.c - binary ownership mutex
#include <stdbool.h>
#include <ke_mutex.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <mm_virt.h>
#include <ob_object.h>
static void MutexDestroy(OBJECT* obj) { (void)obj; }
KSTATUS MutexCreate(MUTEX **out) {
if (!out) return STATUS_INVALID_PARAMETER;
MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
if (!m) return STATUS_NO_MEMORY;
m->Header.Destroy = MutexDestroy;
m->Lock = (SPINLOCK)SPINLOCK_INIT;
m->Owner = (struct THREAD *)0;
LIST_INIT(&m->WaitList);
*out = m;
return STATUS_SUCCESS;
}
KSTATUS KeAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
while (true) {
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_SUCCESS;
}
ListInsertTail(&mutex->WaitList, &self->SchedulerLink);
KeReleaseSpinlock(&mutex->Lock, flags);
KeBlockThread(self, &mutex->Header);
}
}
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_FAILURE;
}
void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
mutex->Owner = (struct THREAD *)0;
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&mutex->Lock, flags);
KeUnblockThread(waiter);
return;
}
KeReleaseSpinlock(&mutex->Lock, flags);
}
// ke_mutex.c - binary ownership mutex
#include <stdbool.h>
#include <ke_mutex.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ke_irql.h>
#include <mm_virt.h>
#include <ob_object.h>
static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
KSTATUS KeMutexCreate(MUTEX **out) {
if (!out) return STATUS_INVALID_PARAMETER;
MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
if (!m) return STATUS_NO_MEMORY;
m->Header.Destroy = KiMutexDestroy;
m->Lock = (SPINLOCK)SPINLOCK_INIT;
m->Owner = (struct THREAD *)0;
LIST_INIT(&m->WaitList);
*out = m;
return STATUS_SUCCESS;
}
KSTATUS KeAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_SUCCESS;
}
KePrepareBlockThread(self, &mutex->Header, 0);
ListInsertTail(&mutex->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &mutex->WaitList);
KeReleaseSpinlock(&mutex->Lock, flags);
return KeCommitBlockThread(self, 0);
}
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_FAILURE;
}
void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
// Direct handoff: the selected waiter owns the mutex when its wait
// returns. Do not drop Owner to NULL, or another thread can steal it.
mutex->Owner = waiter;
KeReleaseSpinlock(&mutex->Lock, flags);
KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS);
return;
}
mutex->Owner = (struct THREAD *)0;
KeReleaseSpinlock(&mutex->Lock, flags);
}
code/kernel/ke_mutex.h
+18 -18
View File
@@ -1,18 +1,18 @@
#pragma once
// ke_mutex.h - binary ownership mutex
#include <ob_object.h>
#include <ke_spinlock.h>
#include <ke_list.h>
#include <ke_status.h>
typedef struct MUTEX {
OBJECT Header;
SPINLOCK Lock;
struct THREAD* Owner; /* NULL if unowned */
LIST_ENTRY WaitList;
} MUTEX;
KSTATUS MutexCreate(MUTEX **out);
KSTATUS KeAcquireMutex(MUTEX* mutex);
KSTATUS KeTryAcquireMutex(MUTEX* mutex);
void KeReleaseMutex(MUTEX* mutex);
#pragma once
// ke_mutex.h - binary ownership mutex
#include <ob_object.h>
#include <ke_spinlock.h>
#include <ke_list.h>
#include <ke_status.h>
typedef struct MUTEX {
OBJECT Header;
SPINLOCK Lock;
struct THREAD* Owner; /* NULL if unowned */
LIST_ENTRY WaitList;
} MUTEX;
KSTATUS KeMutexCreate(MUTEX **out);
KSTATUS KeAcquireMutex(MUTEX* mutex);
KSTATUS KeTryAcquireMutex(MUTEX* mutex);
void KeReleaseMutex(MUTEX* mutex);
code/kernel/ke_process.c
+47 -27
View File
@@ -16,7 +16,7 @@ static LIST_ENTRY gProcessList;
static SPINLOCK gProcessListLock;
static uint32_t gNextProcessId = 1;
static void ProcessInit(void) {
static void KiProcessInit(void) {
static int inited = 0;
if (!inited) {
LIST_INIT(&gProcessList);
@@ -26,7 +26,7 @@ static void ProcessInit(void) {
}
LIST_ENTRY* KeGetProcessList(void) {
ProcessInit();
KiProcessInit();
return &gProcessList;
}
@@ -39,7 +39,7 @@ PROCESS* KeGetCurrentProcess(void) {
}
size_t KeCopyProcessIds(uint32_t* pidsOut, size_t maxCount) {
ProcessInit();
KiProcessInit();
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
size_t count = 0;
@@ -56,7 +56,7 @@ size_t KeCopyProcessIds(uint32_t* pidsOut, size_t maxCount) {
// ---------------------------------------------------------------------------
// Destructor (called by ObDereferenceObject when RefCount hits zero)
// ---------------------------------------------------------------------------
static void ProcessDestroy(OBJECT* obj) {
static void KiProcessDestroy(OBJECT* obj) {
PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
// Release all remaining handles.
@@ -69,7 +69,7 @@ static void ProcessDestroy(OBJECT* obj) {
// Release the main executable image.
if (proc->ImageBase && proc->ImageSize) {
MmFreeUserMemory((void *)proc->ImageBase, proc->ImageSize);
MmFreeUserMemory((void *)proc->ImageBase);
proc->ImageBase = 0;
proc->ImageSize = 0;
}
@@ -77,7 +77,7 @@ static void ProcessDestroy(OBJECT* obj) {
// Release per-process modules.
for (uint32_t i = 0; i < proc->ModuleCount; i++) {
if (proc->Modules[i].Base && proc->Modules[i].Size) {
MmFreeUserMemory((void *)proc->Modules[i].Base, proc->Modules[i].Size);
MmFreeUserMemory((void *)proc->Modules[i].Base);
}
proc->Modules[i].Base = 0;
proc->Modules[i].Size = 0;
@@ -87,22 +87,22 @@ static void ProcessDestroy(OBJECT* obj) {
// Release the user-mode process environment.
if (proc->UserPebBase && proc->UserPebSize) {
MmFreeUserMemory((void *)proc->UserPebBase, proc->UserPebSize);
MmFreeUserMemory((void *)proc->UserPebBase);
proc->UserPebBase = 0;
proc->UserPebSize = 0;
}
if (proc->UserImagePathBase && proc->UserImagePathSize) {
MmFreeUserMemory((void *)proc->UserImagePathBase, proc->UserImagePathSize);
MmFreeUserMemory((void *)proc->UserImagePathBase);
proc->UserImagePathBase = 0;
proc->UserImagePathSize = 0;
}
if (proc->UserCmdlineBase && proc->UserCmdlineSize) {
MmFreeUserMemory((void *)proc->UserCmdlineBase, proc->UserCmdlineSize);
MmFreeUserMemory((void *)proc->UserCmdlineBase);
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
}
if (proc->UserCwdBase && proc->UserCwdSize) {
MmFreeUserMemory((void *)proc->UserCwdBase, proc->UserCwdSize);
MmFreeUserMemory((void *)proc->UserCwdBase);
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
}
@@ -126,14 +126,14 @@ static void ProcessDestroy(OBJECT* obj) {
// KeCreateProcess
// ---------------------------------------------------------------------------
KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
ProcessInit();
KiProcessInit();
if (!name || !out) return STATUS_INVALID_PARAMETER;
PROCESS* proc = (PROCESS *)ObAllocateObject(ObjectTypeProcess, sizeof(PROCESS));
if (!proc) return STATUS_NO_MEMORY;
proc->Header.Destroy = ProcessDestroy;
proc->Header.Destroy = KiProcessDestroy;
proc->ProcessId = _InterlockedExchangeAdd((volatile long *)&gNextProcessId, 1);
strncpy(proc->Name, name, PROCESS_NAME_MAX - 1);
@@ -155,6 +155,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->UserCwdSize = 0;
proc->ModuleCount = 0;
proc->CurrentDirectory[0] = '\0';
proc->CurrentDrive = 0;
proc->CwdLock = (SPINLOCK)SPINLOCK_INIT;
// Slot 0 is the invalid handle - leave it NULL.
@@ -180,7 +181,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
// ---------------------------------------------------------------------------
// KeSetupUserProcessEnvironment
// ---------------------------------------------------------------------------
static KSTATUS CopyEnvironmentString(const char* src, uintptr_t* baseOut, size_t* sizeOut) {
static KSTATUS KiCopyEnvironmentString(const char* src, uintptr_t* baseOut, size_t* sizeOut) {
if (!baseOut || !sizeOut) return STATUS_INVALID_PARAMETER;
if (!src) src = "";
@@ -194,7 +195,7 @@ static KSTATUS CopyEnvironmentString(const char* src, uintptr_t* baseOut, size_t
return STATUS_SUCCESS;
}
static KSTATUS CopyEnvironmentStringFixed(const char* src, size_t capacity,
static KSTATUS KiCopyEnvironmentStringFixed(const char* src, size_t capacity,
uintptr_t* baseOut, size_t* sizeOut) {
if (!baseOut || !sizeOut || capacity == 0) return STATUS_INVALID_PARAMETER;
if (!src) src = "";
@@ -213,31 +214,48 @@ static KSTATUS CopyEnvironmentStringFixed(const char* src, size_t capacity,
return STATUS_SUCCESS;
}
static void FreePartialProcessEnvironment(PROCESS* proc) {
static void KiFreePartialProcessEnvironment(PROCESS* proc) {
if (!proc) return;
if (proc->UserPebBase && proc->UserPebSize) {
MmFreeUserMemory((void*)proc->UserPebBase, proc->UserPebSize);
MmFreeUserMemory((void*)proc->UserPebBase);
proc->UserPebBase = 0;
proc->UserPebSize = 0;
}
if (proc->UserImagePathBase && proc->UserImagePathSize) {
MmFreeUserMemory((void*)proc->UserImagePathBase, proc->UserImagePathSize);
MmFreeUserMemory((void*)proc->UserImagePathBase);
proc->UserImagePathBase = 0;
proc->UserImagePathSize = 0;
}
if (proc->UserCmdlineBase && proc->UserCmdlineSize) {
MmFreeUserMemory((void*)proc->UserCmdlineBase, proc->UserCmdlineSize);
MmFreeUserMemory((void*)proc->UserCmdlineBase);
proc->UserCmdlineBase = 0;
proc->UserCmdlineSize = 0;
}
if (proc->UserCwdBase && proc->UserCwdSize) {
MmFreeUserMemory((void*)proc->UserCwdBase, proc->UserCwdSize);
MmFreeUserMemory((void*)proc->UserCwdBase);
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
}
}
static void KiFormatNativeCwd(const PROCESS* proc, const char* currentDirectory,
char* out, size_t outSize) {
if (!out || outSize == 0) return;
const char* rel = currentDirectory ? currentDirectory : "";
uint8_t drive = proc ? proc->CurrentDrive : 0;
if (drive > 7) drive = 0;
if (outSize < 4) { out[0] = '\0'; return; }
out[0] = (char)('0' + drive);
out[1] = ':';
out[2] = '/';
size_t len = strlen(rel);
if (len + 4 > outSize) len = outSize - 4;
memcpy(out + 3, rel, len);
out[3 + len] = '\0';
}
KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
const char* imagePath,
const char* commandLine,
@@ -245,17 +263,19 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
if (!proc || !imagePath) return STATUS_INVALID_PARAMETER;
const char* finalCommandLine = (commandLine && commandLine[0]) ? commandLine : imagePath;
const char* finalCwd = currentDirectory ? currentDirectory : proc->CurrentDirectory;
const char* finalCwdRel = currentDirectory ? currentDirectory : proc->CurrentDirectory;
char finalCwd[SYS_CWD_MAX];
KiFormatNativeCwd(proc, finalCwdRel, finalCwd, sizeof(finalCwd));
FreePartialProcessEnvironment(proc);
KiFreePartialProcessEnvironment(proc);
KSTATUS s = CopyEnvironmentString(imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
KSTATUS s = KiCopyEnvironmentString(imagePath, &proc->UserImagePathBase, &proc->UserImagePathSize);
if (!KSUCCESS(s)) goto fail;
s = CopyEnvironmentString(finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize);
s = KiCopyEnvironmentString(finalCommandLine, &proc->UserCmdlineBase, &proc->UserCmdlineSize);
if (!KSUCCESS(s)) goto fail;
s = CopyEnvironmentStringFixed(finalCwd, SYS_CWD_MAX, &proc->UserCwdBase, &proc->UserCwdSize);
s = KiCopyEnvironmentStringFixed(finalCwd, SYS_CWD_MAX, &proc->UserCwdBase, &proc->UserCwdSize);
if (!KSUCCESS(s)) goto fail;
SYS_PEB* peb = (SYS_PEB*)MmAllocateUserMemory(sizeof(SYS_PEB));
@@ -277,7 +297,7 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* proc,
return STATUS_SUCCESS;
fail:
FreePartialProcessEnvironment(proc);
KiFreePartialProcessEnvironment(proc);
return s;
}
@@ -290,7 +310,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
// Wake any threads waiting for this process to finish.
if (process->ExitEvent) KeSignalEvent(process->ExitEvent);
// The last thread exiting via KeExitThread will drop the process refcount.
// This function is a policy hook; actual cleanup is in ProcessDestroy.
// This function is a policy hook; actual cleanup is in KiProcessDestroy.
ObDereferenceObject(&process->Header);
}
@@ -351,7 +371,7 @@ OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) {
}
PROCESS* KeGetProcessById(uint32_t pid) {
ProcessInit();
KiProcessInit();
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
LIST_ENTRY* head = &gProcessList;
LIST_ENTRY* node = head->Fwd;
code/kernel/ke_process.h
+1
View File
@@ -28,6 +28,7 @@ typedef struct PROCESS {
char Name[PROCESS_NAME_MAX]; /* UTF-8 */
char CurrentDirectory[512];
uint8_t CurrentDrive;
SPINLOCK CwdLock;
/* Thread list. */
code/kernel/ke_queue.c
+92 -85
View File
@@ -1,85 +1,92 @@
// ke_queue.c - ring-buffer message queue
#include <string.h>
#include <ke_queue.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ob_object.h>
static void QueueDestroy(OBJECT* obj) { (void)obj; }
KSTATUS QueueCreate(QUEUE **out) {
if (!out) return STATUS_INVALID_PARAMETER;
QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
if (!q) return STATUS_NO_MEMORY;
q->Header.Destroy = QueueDestroy;
q->Lock = (SPINLOCK)SPINLOCK_INIT;
q->Head = q->Tail = q->Count = 0;
LIST_INIT(&q->SendWaiters);
LIST_INIT(&q->RecvWaiters);
*out = q;
return STATUS_SUCCESS;
}
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count < QUEUE_DEPTH) {
queue->Ring[queue->Tail] = *msg;
queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH;
queue->Count++;
// Wake a receiver if one is waiting.
if (!LIST_IS_EMPTY(&queue->RecvWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_FULL;
}
ListInsertTail(&queue->SendWaiters, &self->SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeBlockThread(self, &queue->Header);
}
}
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count > 0) {
*msg = queue->Ring[queue->Head];
queue->Head = (queue->Head + 1) % QUEUE_DEPTH;
queue->Count--;
// Wake a sender if one is waiting.
if (!LIST_IS_EMPTY(&queue->SendWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->SendWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_EMPTY;
}
ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeBlockThread(self, &queue->Header);
}
}
// ke_queue.c - ring-buffer message queue
#include <string.h>
#include <ke_queue.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ke_irql.h>
#include <ob_object.h>
static void KiQueueDestroy(OBJECT* obj) { (void)obj; }
KSTATUS KeQueueCreate(QUEUE **out) {
if (!out) return STATUS_INVALID_PARAMETER;
QUEUE* q = (QUEUE *)ObAllocateObject(ObjectTypeQueue, sizeof(QUEUE));
if (!q) return STATUS_NO_MEMORY;
q->Header.Destroy = KiQueueDestroy;
q->Lock = (SPINLOCK)SPINLOCK_INIT;
q->Head = q->Tail = q->Count = 0;
LIST_INIT(&q->SendWaiters);
LIST_INIT(&q->RecvWaiters);
*out = q;
return STATUS_SUCCESS;
}
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count < QUEUE_DEPTH) {
queue->Ring[queue->Tail] = *msg;
queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH;
queue->Count++;
// Wake a receiver if one is waiting.
if (!LIST_IS_EMPTY(&queue->RecvWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_FULL;
}
KePrepareBlockThread(self, &queue->Header, 0);
ListInsertTail(&queue->SendWaiters, &self->SchedulerLink);
KeSetThreadWaitList(self, &queue->SendWaiters);
KeReleaseSpinlock(&queue->Lock, flags);
KeCommitBlockThread(self, 0);
}
}
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
if (!queue || !msg) return STATUS_INVALID_PARAMETER;
if (blocking && !KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
while (true) {
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count > 0) {
*msg = queue->Ring[queue->Head];
queue->Head = (queue->Head + 1) % QUEUE_DEPTH;
queue->Count--;
// Wake a sender if one is waiting.
if (!LIST_IS_EMPTY(&queue->SendWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->SendWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock, flags);
}
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_EMPTY;
}
KePrepareBlockThread(self, &queue->Header, 0);
ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink);
KeSetThreadWaitList(self, &queue->RecvWaiters);
KeReleaseSpinlock(&queue->Lock, flags);
KeCommitBlockThread(self, 0);
}
}
code/kernel/ke_queue.h
+32 -32
View File
@@ -1,32 +1,32 @@
#pragma once
// ke_queue.h - fixed-depth message queue
// Processes share an address space, so a pointer in Param[] is valid
// on both sides. The queue is only a synchronisation and ordering tool.
#include <stdint.h>
#include <stdbool.h>
#include <ob_object.h>
#include <ke_spinlock.h>
#include <ke_list.h>
#include <ke_status.h>
#define QUEUE_MESSAGE_SIZE 64 /* bytes, fixed */
#define QUEUE_DEPTH 256
typedef struct QUEUE_MESSAGE {
uint32_t Type;
uint32_t Flags;
uint64_t Param[6]; /* 48 bytes of payload */
} QUEUE_MESSAGE; /* total: 64 bytes */
typedef struct QUEUE {
OBJECT Header;
SPINLOCK Lock;
QUEUE_MESSAGE Ring[QUEUE_DEPTH];
uint32_t Head, Tail, Count;
LIST_ENTRY SendWaiters;
LIST_ENTRY RecvWaiters;
} QUEUE;
KSTATUS QueueCreate(QUEUE **out);
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking);
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking);
#pragma once
// ke_queue.h - fixed-depth message queue
// Processes share an address space, so a pointer in Param[] is valid
// on both sides. The queue is only a synchronisation and ordering tool.
#include <stdint.h>
#include <stdbool.h>
#include <ob_object.h>
#include <ke_spinlock.h>
#include <ke_list.h>
#include <ke_status.h>
#define QUEUE_MESSAGE_SIZE 64 /* bytes, fixed */
#define QUEUE_DEPTH 256
typedef struct QUEUE_MESSAGE {
uint32_t Type;
uint32_t Flags;
uint64_t Param[6]; /* 48 bytes of payload */
} QUEUE_MESSAGE; /* total: 64 bytes */
typedef struct QUEUE {
OBJECT Header;
SPINLOCK Lock;
QUEUE_MESSAGE Ring[QUEUE_DEPTH];
uint32_t Head, Tail, Count;
LIST_ENTRY SendWaiters;
LIST_ENTRY RecvWaiters;
} QUEUE;
KSTATUS KeQueueCreate(QUEUE **out);
KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking);
KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking);
code/kernel/ke_sched.c
+201 -189
View File
@@ -1,189 +1,201 @@
// ke_sched.c - MLFQ scheduler
#include <stddef.h>
#include <stdbool.h>
#include <ke_sched.h>
#include <ke_process.h>
#include <ke_thread.h>
#include <ke_spinlock.h>
#include <ke_list.h>
#include <mm_virt.h>
#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */
static LIST_ENTRY gRunQueue[PriorityCount];
static SPINLOCK gSchedLock;
static THREAD* gCurrentThread;
static THREAD* gIdleThread;
static THREAD* gOutgoingThread;
static uint32_t gAgingCounter = 0;
THREAD* KeGetCurrentThread(void) { return gCurrentThread; }
void KeSetCurrentThread(THREAD* t) { gCurrentThread = t; }
static void IdleLoop(void) {
while (true) { __halt(); }
}
void KeInitializeScheduler(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
PROCESS* idleProc;
KeCreateProcess("idle", &idleProc);
THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
idle->Header.Destroy = (void *)0;
idle->ThreadId = 0;
idle->State = ThreadReady;
idle->Priority = PriorityLow;
idle->QuantaConsumed = 0;
idle->Owner = idleProc;
idle->WaitObject = (OBJECT *)0;
idle->SchedLockFlags = 0x202ULL; // IF=1
void* kstack = MmAllocatePool(KSTACK_SIZE);
idle->KernelStackBase = kstack;
idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
idle->TssRsp0 = idle->KernelStackTop;
uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop;
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
__stosq((unsigned __int64 *)ctx, 0, sizeof(KCONTEXT) / 8);
ctx->Rip = (uint64_t)IdleLoop;
idle->SavedContext = ctx;
KeInitThreadFpuContext(idle);
gIdleThread = idle;
gCurrentThread = idle;
}
// Safely release the lock and clean up any dead threads natively post-switch
void KeSchedulerPostSwitch(void) {
THREAD* cur = KeGetCurrentThread();
KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags);
if (gOutgoingThread && gOutgoingThread->State == ThreadDead) {
THREAD* dead = gOutgoingThread;
gOutgoingThread = (THREAD*)0;
ObDereferenceObject(&dead->Header);
}
}
void KeReadyThread(THREAD* thread) {
if (!thread) return;
thread->State = ThreadReady;
int p = (int)thread->Priority;
if (p < 0) p = 0;
if (p >= PriorityCount) p = PriorityCount - 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
}
void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListRemove(&thread->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
}
static THREAD* PickNext(void) {
for (int p = PriorityCount - 1; p >= 0; p--) {
if (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = gRunQueue[p].Fwd;
ListRemove(node);
return CONTAINING_RECORD(node, THREAD, SchedulerLink);
}
}
return gIdleThread;
}
void KeSchedulerTick(void) {
THREAD* cur = gCurrentThread;
if (!cur) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags; // Issue 5 fix
cur->QuantaConsumed++;
if (cur != gIdleThread && cur->QuantaConsumed >= DEMOTION_QUANTA) {
cur->QuantaConsumed = 0;
if (cur->Priority > PriorityLow) {
cur->Priority = (THREAD_PRIORITY)(cur->Priority - 1);
}
}
// Periodic aging pass to prevent indefinite starvation of PriorityLow threads
gAgingCounter++;
if (gAgingCounter >= 100) {
gAgingCounter = 0;
for (int p = 0; p < PriorityHigh; p++) {
while (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = ListPopHead(&gRunQueue[p]);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
t->Priority = PriorityHigh;
ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink);
}
}
}
THREAD* next = PickNext();
if (next == cur) {
if (cur != gIdleThread) {
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
}
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
cur->State = ThreadReady;
next->State = ThreadRunning;
gCurrentThread = next;
if (cur != gIdleThread && cur->State == ThreadReady) {
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
}
gOutgoingThread = cur;
KeSwitchContext(cur, next);
KeSchedulerPostSwitch();
}
__declspec(dllexport) void KeYield(void) {
THREAD* cur = gCurrentThread;
if (!cur) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags;
THREAD* next = PickNext();
if (next == cur) {
// Only re-queue if still running normally; a ThreadWaiting/ThreadDead caller
// has already placed SchedulerLink elsewhere (or is discarding the thread).
if (cur != gIdleThread && cur->State == ThreadRunning)
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
next->State = ThreadRunning;
gCurrentThread = next;
// Only re-queue the outgoing thread if it is still in the running state.
// If KeBlockThread set State = ThreadWaiting, SchedulerLink was already
// inserted into the sync object's wait list - inserting it here too would
// corrupt both lists simultaneously (same intrusive node in two lists).
// If KeExitThread set State = ThreadDead, the thread must not be re-queued.
if (cur != gIdleThread && cur->State == ThreadRunning) {
cur->State = ThreadReady;
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
}
gOutgoingThread = cur;
KeSwitchContext(cur, next);
KeSchedulerPostSwitch();
}
// ke_sched.c - MLFQ scheduler
#include <stddef.h>
#include <stdbool.h>
#include <ke_sched.h>
#include <ke_process.h>
#include <ke_thread.h>
#include <ke_spinlock.h>
#include <ke_irql.h>
#include <ke_apc.h>
#include <ke_list.h>
#include <mm_virt.h>
#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */
static LIST_ENTRY gRunQueue[PriorityCount];
static SPINLOCK gSchedLock;
static THREAD* gCurrentThread;
static THREAD* gIdleThread;
static THREAD* gOutgoingThread;
static uint32_t gAgingCounter = 0;
static volatile long gReschedulePending = 0;
THREAD* KeGetCurrentThread(void) { return gCurrentThread; }
void KeSetCurrentThread(THREAD* t) { gCurrentThread = t; }
static void KiIdleLoop(void) {
while (true) { __halt(); }
}
void KeInitializeScheduler(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
PROCESS* idleProc;
KeCreateProcess("idle", &idleProc);
THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
idle->Header.Destroy = (void *)0;
idle->ThreadId = 0;
idle->State = ThreadReady;
idle->Priority = PriorityLow;
idle->QuantaConsumed = 0;
idle->Owner = idleProc;
idle->WaitObject = (OBJECT *)0;
idle->WaitStatus = STATUS_SUCCESS;
idle->SchedLockFlags = 0x202ULL; // IF=1
KiInitializeApcState(idle);
idle->TrapFrame = (KTRAP_FRAME*)0;
void* kstack = MmAllocatePool(KSTACK_SIZE);
idle->KernelStackBase = kstack;
idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
idle->TssRsp0 = idle->KernelStackTop;
uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop;
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
__stosq((unsigned __int64 *)ctx, 0, sizeof(KCONTEXT) / 8);
ctx->Rip = (uint64_t)KiIdleLoop;
idle->SavedContext = ctx;
KeInitThreadFpuContext(idle);
gIdleThread = idle;
gCurrentThread = idle;
}
// Safely release the lock and clean up any dead threads natively post-switch
void KeSchedulerPostSwitch(void) {
THREAD* cur = KeGetCurrentThread();
KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags);
KiDeliverKernelApcs();
if (gOutgoingThread && gOutgoingThread->State == ThreadDead) {
THREAD* dead = gOutgoingThread;
gOutgoingThread = (THREAD*)0;
ObDereferenceObject(&dead->Header);
}
}
void KeReadyThread(THREAD* thread) {
if (!thread) return;
thread->State = ThreadReady;
int p = (int)thread->Priority;
if (p < 0) p = 0;
if (p >= PriorityCount) p = PriorityCount - 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
}
void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListRemove(&thread->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
}
static THREAD* KiPickNext(void) {
for (int p = PriorityCount - 1; p >= 0; p--) {
if (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = gRunQueue[p].Fwd;
ListRemove(node);
return CONTAINING_RECORD(node, THREAD, SchedulerLink);
}
}
return gIdleThread;
}
void KiRequestReschedule(void) {
gReschedulePending = 1;
}
int KiIsReschedulePending(void) {
return gReschedulePending != 0;
}
static void KiSchedule(void) {
THREAD* cur = gCurrentThread;
if (!cur) return;
gReschedulePending = 0;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags;
THREAD* next = KiPickNext();
if (next == cur) {
// Only re-queue if still running normally; a ThreadWaiting/ThreadDead
// caller has already placed SchedulerLink elsewhere (or is discarding
// the thread).
if (cur != gIdleThread && cur->State == ThreadRunning)
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
next->State = ThreadRunning;
gCurrentThread = next;
// Only re-queue the outgoing thread if it is still in the running state.
// If KeBlockThread set State = ThreadWaiting, SchedulerLink was already
// inserted into the sync object's wait list - inserting it here too would
// corrupt both lists simultaneously (same intrusive node in two lists).
// If KeExitThread set State = ThreadDead, the thread must not be re-queued.
if (cur != gIdleThread && cur->State == ThreadRunning) {
cur->State = ThreadReady;
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
}
gOutgoingThread = cur;
KeSwitchContext(cur, next);
KeSchedulerPostSwitch();
}
void KiScheduleFromInterrupt(void) {
if (!KiIsReschedulePending()) return;
KiSchedule();
}
void KeSchedulerTick(void) {
THREAD* cur = gCurrentThread;
if (!cur) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags; // Issue 5 fix
cur->QuantaConsumed++;
if (cur != gIdleThread && cur->QuantaConsumed >= DEMOTION_QUANTA) {
cur->QuantaConsumed = 0;
if (cur->Priority > PriorityLow) {
cur->Priority = (THREAD_PRIORITY)(cur->Priority - 1);
}
}
// Periodic aging pass to prevent indefinite starvation of PriorityLow threads
gAgingCounter++;
if (gAgingCounter >= 100) {
gAgingCounter = 0;
for (int p = 0; p < PriorityHigh; p++) {
while (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = ListPopHead(&gRunQueue[p]);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
t->Priority = PriorityHigh;
ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink);
}
}
}
KeReleaseSpinlock(&gSchedLock, flags);
KiRequestReschedule();
}
__declspec(dllexport) void KeYield(void) {
KeAssertIrql(PASSIVE_LEVEL);
KiSchedule();
}
code/kernel/ke_sched.h
+10 -1
View File
@@ -12,9 +12,18 @@ void KeInitializeScheduler(void);
void KeReadyThread(THREAD* thread);
void KeUnreadyThread(THREAD* thread);
// Called from the timer ISR every QUANTUM_MS.
// Called from the timer ISR every QUANTUM_MS. This only performs timer
// accounting and requests a reschedule; the actual context switch happens from
// KiExitInterrupt after the IRQ handler has returned and EOI has been sent.
void KeSchedulerTick(void);
// Internal scheduler hooks used by the interrupt-exit path. KiRequestReschedule
// is cheap and can be called from interrupt context. KiScheduleFromInterrupt
// consumes the pending request and performs the context switch, if needed.
void KiRequestReschedule(void);
int KiIsReschedulePending(void);
void KiScheduleFromInterrupt(void);
// Voluntarily yield the CPU. Picks the next runnable thread immediately.
__declspec(dllexport) void KeYield(void);
code/kernel/ke_selftest.c
+702
View File
@@ -0,0 +1,702 @@
// ke_selftest.c - temporary kernel bring-up self-tests for execution/I/O model.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <hal_api.h>
#include <ke_apc.h>
#include <ke_debug.h>
#include <ke_event.h>
#include <ke_irp.h>
#include <ke_irql.h>
#include <ke_sched.h>
#include <ke_status.h>
#include <ke_thread.h>
#include <io_file.h>
#include <io_driver.h>
#include <ke_trap.h>
#include <mm_virt.h>
#include <ob_object.h>
// -------------------------------------------------------------------------
// Kernel APC self-test
// -------------------------------------------------------------------------
typedef struct KI_APC_SELFTEST_STATE {
EVENT* ReadyEvent;
EVENT* GoEvent;
EVENT* DoneEvent;
EVENT* FinishedEvent;
THREAD* WorkerThread;
volatile int SelfDelivered;
volatile int DeferredDelivered;
volatile int CrossDelivered;
} KI_APC_SELFTEST_STATE;
static void KiApcSelfTestKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("APC selftest: kernel routine apc=0x%llX normal=0x%llX ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n",
(uint64_t)apc,
(uint64_t)*normalRoutine,
(uint64_t)*normalContext,
(uint64_t)*systemArgument1,
(uint64_t)*systemArgument2,
KeGetCurrentIrql());
}
static void KiApcSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
volatile int* counter = (volatile int*)normalContext;
DbgPrint("APC selftest: normal routine ctx=0x%llX arg1=0x%llX arg2=0x%llX irql=%u\n",
(uint64_t)normalContext,
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic("APC selftest: normal routine ran above PASSIVE_LEVEL\n");
(*counter)++;
}
static void KiApcSelfTestCrossThreadRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext;
DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n",
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic("APC selftest: cross-thread APC ran above PASSIVE_LEVEL\n");
st->CrossDelivered++;
KeSignalEvent(st->DoneEvent);
}
static void KiApcSelfTestWorker(void* arg) {
KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)arg;
DbgPrint("APC selftest: worker started, signaling ready\n");
KeSignalEvent(st->ReadyEvent);
DbgPrint("APC selftest: worker waiting for go event\n");
KSTATUS s = KeWaitForEvent(st->GoEvent, 0);
if (!KSUCCESS(s))
KePanic("APC selftest: worker wait failed status=%d\n", s);
// KeCommitBlockThread(..., alertable) should deliver pending kernel APCs before this
// wait returns. If that hook is broken, DoneEvent will never be signaled.
DbgPrint("APC selftest: worker resumed after go event, cross=%d\n",
st->CrossDelivered);
if (st->CrossDelivered != 1)
KePanic("APC selftest: worker resumed but APC was not delivered\n");
KeSignalEvent(st->FinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
typedef struct KI_APC_ALERTABLE_SELFTEST_STATE {
EVENT* ReadyEvent;
EVENT* WaitEvent;
EVENT* FinishedEvent;
THREAD* WorkerThread;
volatile int ApcDelivered;
volatile KSTATUS WaitStatus;
} KI_APC_ALERTABLE_SELFTEST_STATE;
static void KiApcAlertableSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_APC_ALERTABLE_SELFTEST_STATE* st =
(KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext;
DbgPrint("APC selftest: alertable APC delivered arg1=0x%llX arg2=0x%llX irql=%u\n",
(uint64_t)systemArgument1,
(uint64_t)systemArgument2,
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic("APC selftest: alertable APC ran above PASSIVE_LEVEL\n");
st->ApcDelivered++;
}
static void KiApcAlertableSelfTestWorker(void* arg) {
KI_APC_ALERTABLE_SELFTEST_STATE* st =
(KI_APC_ALERTABLE_SELFTEST_STATE*)arg;
DbgPrint("APC selftest: alertable worker ready, entering alertable wait\n");
KeSignalEvent(st->ReadyEvent);
KSTATUS s = KeWaitForEvent(st->WaitEvent, 1);
st->WaitStatus = s;
DbgPrint("APC selftest: alertable wait returned status=%u delivered=%d\n",
s, st->ApcDelivered);
if (s != STATUS_KERNEL_APC)
KePanic("APC selftest: alertable wait returned wrong status=%u\n", s);
if (st->ApcDelivered != 1)
KePanic("APC selftest: alertable APC delivery count wrong=%d\n",
st->ApcDelivered);
KeSignalEvent(st->FinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
static void KiRunKernelApcSelfTest(void) {
DbgPrint("APC selftest: begin\n");
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic("APC selftest: called above PASSIVE_LEVEL\n");
THREAD* self = KeGetCurrentThread();
if (!self)
KePanic("APC selftest: no current thread\n");
// Test 1: queue APC to current thread at PASSIVE_LEVEL.
{
KAPC apc;
volatile int count = 0;
KeInitializeApc(&apc,
self,
KiApcSelfTestKernelRoutine,
KiApcSelfTestNormalRoutine,
(void*)&count);
if (!KeInsertQueueApc(&apc, (void*)0x11111111, (void*)0x22222222))
KePanic("APC selftest: self queue failed\n");
if (count != 1)
KePanic("APC selftest: self APC was not delivered immediately, count=%d\n", count);
DbgPrint("APC selftest: self APC passed\n");
}
// Test 2: disable kernel APCs, queue, then re-enable.
{
KAPC apc;
volatile int count = 0;
KeInitializeApc(&apc,
self,
KiApcSelfTestKernelRoutine,
KiApcSelfTestNormalRoutine,
(void*)&count);
KeEnterCriticalRegion();
if (!KeInsertQueueApc(&apc, (void*)0x33333333, (void*)0x44444444))
KePanic("APC selftest: deferred queue failed\n");
if (count != 0)
KePanic("APC selftest: APC delivered while critical region held, count=%d\n", count);
KeLeaveCriticalRegion();
if (count != 1)
KePanic("APC selftest: deferred APC was not delivered after leave, count=%d\n", count);
DbgPrint("APC selftest: critical-region deferral passed\n");
}
// Test 3: queue APC to another blocked kernel thread, then wake it.
{
KI_APC_SELFTEST_STATE st;
KAPC apc;
memset(&st, 0, sizeof(st));
KSTATUS s;
s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
if (!KSUCCESS(s)) KePanic("APC selftest: ReadyEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.GoEvent);
if (!KSUCCESS(s)) KePanic("APC selftest: GoEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.DoneEvent);
if (!KSUCCESS(s)) KePanic("APC selftest: DoneEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
if (!KSUCCESS(s)) KePanic("APC selftest: FinishedEvent create failed %d\n", s);
s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker,
(uintptr_t)&st,
PriorityNormal,
&st.WorkerThread);
if (!KSUCCESS(s))
KePanic("APC selftest: worker create failed %d\n", s);
s = KeWaitForEvent(st.ReadyEvent, 0);
if (!KSUCCESS(s))
KePanic("APC selftest: ready wait failed %d\n", s);
DbgPrint("APC selftest: worker ready, queueing cross-thread APC\n");
KeInitializeApc(&apc,
st.WorkerThread,
KiApcSelfTestKernelRoutine,
KiApcSelfTestCrossThreadRoutine,
(void*)&st);
if (!KeInsertQueueApc(&apc, (void*)0x55555555, (void*)0x66666666))
KePanic("APC selftest: cross-thread queue failed\n");
if (st.CrossDelivered != 0)
KePanic("APC selftest: cross-thread APC delivered before target resumed\n");
KeSignalEvent(st.GoEvent);
s = KeWaitForEvent(st.DoneEvent, 0);
if (!KSUCCESS(s))
KePanic("APC selftest: done wait failed %d\n", s);
if (st.CrossDelivered != 1)
KePanic("APC selftest: cross-thread delivery count wrong, count=%d\n",
st.CrossDelivered);
// The APC signals DoneEvent before the worker returns from the wait.
// Wait for an explicit worker-owned event before dropping stack-backed
// state and event references. THREAD.ExitEvent is not part of this
// kernel-thread contract on all builds.
s = KeWaitForEvent(st.FinishedEvent, 0);
if (!KSUCCESS(s))
KePanic("APC selftest: worker finished wait failed %d\n", s);
DbgPrint("APC selftest: cross-thread APC passed\n");
ObDereferenceObject(&st.ReadyEvent->Header);
ObDereferenceObject(&st.GoEvent->Header);
ObDereferenceObject(&st.DoneEvent->Header);
ObDereferenceObject(&st.FinishedEvent->Header);
ObDereferenceObject(&st.WorkerThread->Header);
}
// Test 4: queue APC to a thread blocked in an alertable wait.
// The APC itself must wake the waiter; no event signal is sent.
{
KI_APC_ALERTABLE_SELFTEST_STATE st;
KAPC apc;
memset(&st, 0, sizeof(st));
st.WaitStatus = STATUS_PENDING;
KSTATUS s;
s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
if (!KSUCCESS(s)) KePanic("APC selftest: alert ReadyEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.WaitEvent);
if (!KSUCCESS(s)) KePanic("APC selftest: alert WaitEvent create failed %d\n", s);
s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
if (!KSUCCESS(s)) KePanic("APC selftest: alert FinishedEvent create failed %d\n", s);
s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker,
(uintptr_t)&st,
PriorityNormal,
&st.WorkerThread);
if (!KSUCCESS(s))
KePanic("APC selftest: alertable worker create failed %d\n", s);
s = KeWaitForEvent(st.ReadyEvent, 0);
if (!KSUCCESS(s))
KePanic("APC selftest: alert ready wait failed %d\n", s);
DbgPrint("APC selftest: queueing APC to alertable waiter\n");
KeInitializeApc(&apc,
st.WorkerThread,
KiApcSelfTestKernelRoutine,
KiApcAlertableSelfTestNormalRoutine,
(void*)&st);
if (!KeInsertQueueApc(&apc, (void*)(unsigned long long)0x77777777, (void*)(unsigned long long)0x88888888))
KePanic("APC selftest: alertable APC queue failed\n");
s = KeWaitForEvent(st.FinishedEvent, 0);
if (!KSUCCESS(s))
KePanic("APC selftest: alert finished wait failed %d\n", s);
if (st.WaitStatus != STATUS_KERNEL_APC || st.ApcDelivered != 1)
KePanic("APC selftest: alertable result bad status=%u delivered=%d\n",
st.WaitStatus, st.ApcDelivered);
DbgPrint("APC selftest: alertable wait passed\n");
ObDereferenceObject(&st.ReadyEvent->Header);
ObDereferenceObject(&st.WaitEvent->Header);
ObDereferenceObject(&st.FinishedEvent->Header);
ObDereferenceObject(&st.WorkerThread->Header);
}
DbgPrint("APC selftest: PASS\n");
}
// -------------------------------------------------------------------------
// IRP + kernel APC self-test
// -------------------------------------------------------------------------
typedef struct KI_IRP_APC_SELFTEST_STATE {
EVENT* CompletionEvent;
IRP Irp;
KAPC Apc;
uint8_t* Buffer;
volatile int ApcDelivered;
volatile KSTATUS ApcStatus;
volatile size_t ApcInformation;
} KI_IRP_APC_SELFTEST_STATE;
static void KiIrpApcSelfTestKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("IRP/APC selftest: kernel routine apc=0x%llX irp=0x%llX status=%u irql=%u\n",
(uint64_t)apc,
(uint64_t)*systemArgument1,
(uint32_t)(uintptr_t)*systemArgument2,
KeGetCurrentIrql());
(void)normalRoutine;
(void)normalContext;
}
static void KiIrpApcSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_IRP_APC_SELFTEST_STATE* st = (KI_IRP_APC_SELFTEST_STATE*)normalContext;
IRP* irp = (IRP*)systemArgument1;
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
DbgPrint("IRP/APC selftest: APC delivered irp=0x%llX status=%u info=%llu irql=%u\n",
(uint64_t)irp,
status,
(uint64_t)(irp ? irp->Information : 0),
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic("IRP/APC selftest: normal routine ran above PASSIVE_LEVEL\n");
if (irp != &st->Irp)
KePanic("IRP/APC selftest: APC arg1 was not the submitted IRP\n");
st->ApcDelivered++;
st->ApcStatus = status;
st->ApcInformation = irp->Information;
}
static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
DbgPrint("IRP/APC selftest: begin\n");
(void)hal;
if (!IoDiskDevicePresent(0)) {
DbgPrint("IRP/APC selftest: SKIP, drive 0 not present\n");
return;
}
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic("IRP/APC selftest: called above PASSIVE_LEVEL\n");
KI_IRP_APC_SELFTEST_STATE st;
memset(&st, 0, sizeof(st));
KSTATUS s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
if (!KSUCCESS(s))
KePanic("IRP/APC selftest: completion event create failed %u\n", s);
st.Buffer = (uint8_t*)MmAllocatePool(512);
if (!st.Buffer)
KePanic("IRP/APC selftest: no buffer\n");
memset(st.Buffer, 0xCC, 512);
KeInitializeApc(&st.Apc,
KeGetCurrentThread(),
KiIrpApcSelfTestKernelRoutine,
KiIrpApcSelfTestNormalRoutine,
(void*)&st);
LIST_INIT(&st.Irp.Link);
st.Irp.DriveIndex = 0;
st.Irp.Flags = 0;
st.Irp.Lba = 0;
st.Irp.SectorCount = 1;
st.Irp.Buffer = st.Buffer;
st.Irp.Status = STATUS_FAILURE;
st.Irp.Information = 0;
st.Irp.CompletionEvent = st.CompletionEvent;
st.Irp.CompletionApc = &st.Apc;
st.Irp.CompletionFn = (IRP_COMPLETION_FN)0;
st.Irp.CompletionCtx = (void*)0;
st.Irp.HwSlot = 0;
st.Irp.DriverCtx = (void*)0;
DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n");
s = IoSubmitIrp(&st.Irp);
if (s != STATUS_PENDING)
KePanic("IRP/APC selftest: IoSubmitIrp returned %u\n", s);
s = KeWaitForEvent(st.CompletionEvent, 0);
if (!KSUCCESS(s))
KePanic("IRP/APC selftest: completion wait failed %u\n", s);
// The wait return path should have delivered the pending kernel APC.
if (st.ApcDelivered != 1)
KePanic("IRP/APC selftest: APC was not delivered after I/O wait, count=%d\n",
st.ApcDelivered);
if (!KSUCCESS(st.Irp.Status) || !KSUCCESS(st.ApcStatus))
KePanic("IRP/APC selftest: I/O failed irp=%u apc=%u info=%llu\n",
st.Irp.Status, st.ApcStatus, (uint64_t)st.Irp.Information);
if (st.Irp.Information != 512 || st.ApcInformation != 512)
KePanic("IRP/APC selftest: wrong byte count irp=%llu apc=%llu\n",
(uint64_t)st.Irp.Information, (uint64_t)st.ApcInformation);
DbgPrint("IRP/APC selftest: first bytes %02X %02X %02X %02X, tail %02X %02X\n",
st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
st.Buffer[510], st.Buffer[511]);
MmFreePool(st.Buffer);
ObDereferenceObject(&st.CompletionEvent->Header);
DbgPrint("IRP/APC selftest: PASS\n");
}
// -------------------------------------------------------------------------
// Async file I/O self-test
// -------------------------------------------------------------------------
typedef struct KI_FILE_ASYNC_SELFTEST_STATE {
EVENT* CompletionEvent;
FILE_OBJECT* File;
IO_STATUS_BLOCK Iosb;
KAPC Apc;
uint8_t* Buffer;
volatile int ApcDelivered;
volatile KSTATUS ApcStatus;
} KI_FILE_ASYNC_SELFTEST_STATE;
static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%llX iosb=0x%llX status=%u irql=%u\n",
(uint64_t)apc,
(uint64_t)*systemArgument1,
(uint32_t)(uintptr_t)*systemArgument2,
KeGetCurrentIrql());
(void)normalRoutine;
(void)normalContext;
}
static void KiFileAsyncSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_FILE_ASYNC_SELFTEST_STATE* st =
(KI_FILE_ASYNC_SELFTEST_STATE*)normalContext;
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%llX status=%u iosb=%u/%llu irql=%u\n",
(uint64_t)systemArgument1,
status,
st->Iosb.Status,
(uint64_t)st->Iosb.Information,
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic("FILE/ASYNC selftest: APC ran above PASSIVE_LEVEL\n");
st->ApcDelivered++;
st->ApcStatus = status;
}
static void KiRunAsyncFileIoSelfTest(void) {
DbgPrint("FILE/ASYNC selftest: begin\n");
KI_FILE_ASYNC_SELFTEST_STATE st;
memset(&st, 0, sizeof(st));
st.Iosb.Status = STATUS_FAILURE;
KSTATUS s = IoOpenFile("system/init.exe", &st.File);
if (!KSUCCESS(s)) {
DbgPrint("FILE/ASYNC selftest: SKIP, system/init.exe open failed %u\n", s);
return;
}
s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
if (!KSUCCESS(s))
KePanic("FILE/ASYNC selftest: event create failed %u\n", s);
st.Buffer = (uint8_t*)MmAllocatePool(512);
if (!st.Buffer)
KePanic("FILE/ASYNC selftest: no buffer\n");
memset(st.Buffer, 0xCC, 512);
KeInitializeApc(&st.Apc,
KeGetCurrentThread(),
KiFileAsyncSelfTestKernelRoutine,
KiFileAsyncSelfTestNormalRoutine,
(void*)&st);
DbgPrint("FILE/ASYNC selftest: submit read system/init.exe offset=0 size=512\n");
IO_ASYNC_FILE_REQUEST_OBJECT* readReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
s = IoReadFileAsyncKernel(st.File,
st.Buffer,
512,
0,
&st.Iosb,
st.CompletionEvent,
&st.Apc,
&readReq);
if (s != STATUS_PENDING)
KePanic("FILE/ASYNC selftest: submit returned %u\n", s);
s = KeWaitForEvent(st.CompletionEvent, 1);
if (s != STATUS_SUCCESS && s != STATUS_KERNEL_APC)
KePanic("FILE/ASYNC selftest: wait returned %u\n", s);
if (st.Iosb.Status == STATUS_PENDING) {
s = KeWaitForEvent(st.CompletionEvent, 0);
if (!KSUCCESS(s))
KePanic("FILE/ASYNC selftest: final event wait returned %u\n", s);
}
if (st.ApcDelivered != 1)
KePanic("FILE/ASYNC selftest: APC not delivered count=%d\n", st.ApcDelivered);
if (!KSUCCESS(st.Iosb.Status) || !KSUCCESS(st.ApcStatus))
KePanic("FILE/ASYNC selftest: I/O failed iosb=%u apc=%u info=%llu\n",
st.Iosb.Status, st.ApcStatus, (uint64_t)st.Iosb.Information);
if (st.Iosb.Information < 2 || st.Buffer[0] != 'M' || st.Buffer[1] != 'Z')
KePanic("FILE/ASYNC selftest: bad file header info=%llu bytes=%02X %02X\n",
(uint64_t)st.Iosb.Information, st.Buffer[0], st.Buffer[1]);
DbgPrint("FILE/ASYNC selftest: first bytes %02X %02X %02X %02X info=%llu wait=%u\n",
st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
(uint64_t)st.Iosb.Information, s);
if (readReq)
ObDereferenceObject(IoAsyncFileRequestObject(readReq));
MmFreePool(st.Buffer);
ObDereferenceObject(&st.CompletionEvent->Header);
IoCloseFile(st.File);
// Exercise the async write path with an explicit-offset temporary file.
{
static const char payload[] = "async-file-write-selftest\n";
char verify[sizeof(payload)];
FILE_OBJECT* wf = (FILE_OBJECT*)0;
FILE_OBJECT* rf = (FILE_OBJECT*)0;
EVENT* wev = (EVENT*)0;
IO_STATUS_BLOCK wiosb;
size_t got = 0;
memset(verify, 0, sizeof(verify));
wiosb.Status = STATUS_FAILURE;
wiosb.Information = 0;
s = IoCreateFile("asyncio.tmp", &wf);
if (!KSUCCESS(s))
KePanic("FILE/ASYNC selftest: create write temp failed %u\n", s);
s = KeEventCreate(EventAutoReset, 0, &wev);
if (!KSUCCESS(s))
KePanic("FILE/ASYNC selftest: write event create failed %u\n", s);
DbgPrint("FILE/ASYNC selftest: submit write asyncio.tmp size=%llu\n",
(uint64_t)(sizeof(payload) - 1));
IO_ASYNC_FILE_REQUEST_OBJECT* writeReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
s = IoWriteFileAsyncKernel(wf,
payload,
sizeof(payload) - 1,
0,
&wiosb,
wev,
(KAPC*)0,
&writeReq);
if (s != STATUS_PENDING)
KePanic("FILE/ASYNC selftest: write submit returned %u\n", s);
s = KeWaitForEvent(wev, 1);
if (!KSUCCESS(s))
KePanic("FILE/ASYNC selftest: write wait returned %u\n", s);
if (!KSUCCESS(wiosb.Status) || wiosb.Information != sizeof(payload) - 1)
KePanic("FILE/ASYNC selftest: write failed status=%u info=%llu\n",
wiosb.Status, (uint64_t)wiosb.Information);
if (writeReq)
ObDereferenceObject(IoAsyncFileRequestObject(writeReq));
IoCloseFile(wf);
wf = (FILE_OBJECT*)0;
s = IoOpenFile("asyncio.tmp", &rf);
if (!KSUCCESS(s))
KePanic("FILE/ASYNC selftest: reopen write temp failed %u\n", s);
s = IoReadFileAt(rf, 0, verify, sizeof(payload) - 1, &got);
if (!KSUCCESS(s) || got != sizeof(payload) - 1 ||
memcmp(verify, payload, sizeof(payload) - 1) != 0) {
KePanic("FILE/ASYNC selftest: verify failed status=%u got=%llu\n",
s, (uint64_t)got);
}
IoCloseFile(rf);
ObDereferenceObject(&wev->Header);
s = IoDeleteFile("asyncio.tmp");
if (!KSUCCESS(s))
KePanic("FILE/ASYNC selftest: delete temp failed %u\n", s);
DbgPrint("FILE/ASYNC selftest: async write verified\n");
}
DbgPrint("FILE/ASYNC selftest: PASS\n");
}
static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: begin\n");
if (KeGetCurrentTrapFrame())
KePanic("TRAP selftest: kernel selftest unexpectedly has active trap frame\n");
DbgPrint("TRAP selftest: frame size=%llu query size=%llu\n",
(uint64_t)sizeof(KTRAP_FRAME),
(uint64_t)sizeof(SYS_TRAP_FRAME_INFO_K));
if (sizeof(KTRAP_FRAME) != 240)
KePanic("TRAP selftest: KTRAP_FRAME size mismatch %llu\n",
(uint64_t)sizeof(KTRAP_FRAME));
DbgPrint("TRAP selftest: PASS\n");
}
void KiRunKernelSelfTests(const HAL_API* hal) {
DbgPrint("Kernel selftests: begin\n");
KiRunKernelApcSelfTest();
KiRunIrpKernelApcSelfTest(hal);
KiRunAsyncFileIoSelfTest();
KiRunTrapFrameSelfTest();
DbgPrint("Kernel selftests: PASS\n");
}
code/kernel/ke_selftest.h
+9
View File
@@ -0,0 +1,9 @@
#pragma once
// ke_selftest.h - temporary kernel bring-up self-tests.
//
// These tests intentionally live behind one call site so they can be walled
// off or compiled out later without touching kmain.c again.
#include <hal_api.h>
void KiRunKernelSelfTests(const HAL_API* hal);
code/kernel/ke_semaphore.c
+71 -65
View File
@@ -1,65 +1,71 @@
// ke_semaphore.c - counting semaphore
#include <stdbool.h>
#include <ke_semaphore.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ob_object.h>
static void SemDestroy(OBJECT* obj) { (void)obj; }
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out) {
if (!out || initial < 0 || max < 1 || initial > max)
return STATUS_INVALID_PARAMETER;
SEMAPHORE* s = (SEMAPHORE *)ObAllocateObject(ObjectTypeSemaphore, sizeof(SEMAPHORE));
if (!s) return STATUS_NO_MEMORY;
s->Header.Destroy = SemDestroy;
s->Lock = (SPINLOCK)SPINLOCK_INIT;
s->Count = initial;
s->MaxCount = max;
LIST_INIT(&s->WaitList);
*out = s;
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
if (!sem) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
while (true) {
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (sem->Count > 0) {
sem->Count--;
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_SUCCESS;
}
ListInsertTail(&sem->WaitList, &self->SchedulerLink);
KeReleaseSpinlock(&sem->Lock, flags);
KeBlockThread(self, &sem->Header);
}
}
__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
if (!sem) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (sem->Count > 0) {
sem->Count--;
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_FAILURE;
}
__declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
if (!sem) return;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (!LIST_IS_EMPTY(&sem->WaitList)) {
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&sem->Lock, flags);
KeUnblockThread(waiter);
return;
}
if (sem->Count < sem->MaxCount) sem->Count++;
KeReleaseSpinlock(&sem->Lock, flags);
}
// ke_semaphore.c - counting semaphore
#include <stdbool.h>
#include <ke_semaphore.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ke_irql.h>
#include <ob_object.h>
static void KiSemaphoreDestroy(OBJECT* obj) { (void)obj; }
__declspec(dllexport) KSTATUS KeSemaphoreCreate(int32_t initial, int32_t max, SEMAPHORE **out) {
if (!out || initial < 0 || max < 1 || initial > max)
return STATUS_INVALID_PARAMETER;
SEMAPHORE* s = (SEMAPHORE *)ObAllocateObject(ObjectTypeSemaphore, sizeof(SEMAPHORE));
if (!s) return STATUS_NO_MEMORY;
s->Header.Destroy = KiSemaphoreDestroy;
s->Lock = (SPINLOCK)SPINLOCK_INIT;
s->Count = initial;
s->MaxCount = max;
LIST_INIT(&s->WaitList);
*out = s;
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
if (!sem) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (sem->Count > 0) {
sem->Count--;
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_SUCCESS;
}
KePrepareBlockThread(self, &sem->Header, 0);
ListInsertTail(&sem->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &sem->WaitList);
KeReleaseSpinlock(&sem->Lock, flags);
return KeCommitBlockThread(self, 0);
}
__declspec(dllexport) KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
if (!sem) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (sem->Count > 0) {
sem->Count--;
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_FAILURE;
}
__declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
if (!sem) return;
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (!LIST_IS_EMPTY(&sem->WaitList)) {
// Transfer this semaphore release directly to the selected waiter. The
// count stays unchanged; the waiter returns success without looping and
// consuming a count later.
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&sem->Lock, flags);
KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS);
return;
}
if (sem->Count < sem->MaxCount) sem->Count++;
KeReleaseSpinlock(&sem->Lock, flags);
}
code/kernel/ke_spinlock.h
+89 -37
View File
@@ -1,37 +1,89 @@
#pragma once
// ke_spinlock.h - spinlock for short, non-sleeping critical sections
// Uses MSVC interlocked intrinsics. Never hold across a scheduling boundary.
#include <stdint.h>
#include <intrin.h>
typedef struct SPINLOCK {
volatile long Locked;
} SPINLOCK;
#define SPINLOCK_INIT { 0 }
// Issue 1 Fix: Save EFLAGS, disable interrupts, and return old flags.
static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) {
uintptr_t flags = __readeflags();
_disable();
while (_InterlockedCompareExchange(&sl->Locked, 1, 0) != 0)
_mm_pause();
return flags;
}
// Issue 1 Fix: Restore EFLAGS upon release.
static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t flags) {
_InterlockedExchange(&sl->Locked, 0);
__writeeflags(flags);
}
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outFlags) {
uintptr_t flags = __readeflags();
_disable();
if (_InterlockedCompareExchange(&sl->Locked, 1, 0) == 0) {
*outFlags = flags;
return 1;
}
__writeeflags(flags);
return 0;
}
#pragma once
// ke_spinlock.h - spinlock for short, non-sleeping critical sections
//
// Step 3 IRQL bring-up:
// * Acquiring a spinlock disables maskable interrupts and raises the software
// IRQL to at least DISPATCH_LEVEL.
// * Releasing restores both the previous IRQL and the previous RFLAGS.IF
// state through the returned cookie.
//
// The public type remains uintptr_t so existing callers do not need to change.
// Never hold a spinlock across a wait/blocking operation.
#include <stdint.h>
#include <intrin.h>
#include <ke_irql.h>
typedef struct SPINLOCK {
volatile long Locked;
} SPINLOCK;
#define SPINLOCK_INIT { 0 }
#define KI_SPINLOCK_IRQL_SHIFT ((unsigned)(sizeof(uintptr_t) * 8u - 8u))
#define KI_SPINLOCK_FLAGS_MASK (~(((uintptr_t)0xFFu) << KI_SPINLOCK_IRQL_SHIFT))
static __forceinline uintptr_t KiPackSpinlockCookie(uintptr_t flags, KIRQL oldIrql) {
return (flags & KI_SPINLOCK_FLAGS_MASK) |
(((uintptr_t)oldIrql) << KI_SPINLOCK_IRQL_SHIFT);
}
static __forceinline uintptr_t KiSpinlockCookieFlags(uintptr_t cookie) {
return cookie & KI_SPINLOCK_FLAGS_MASK;
}
static __forceinline KIRQL KiSpinlockCookieIrql(uintptr_t cookie) {
return (KIRQL)(cookie >> KI_SPINLOCK_IRQL_SHIFT);
}
// Save RFLAGS, mask interrupts, raise to DISPATCH_LEVEL if needed, acquire.
// If already running at DIRQL, the higher IRQL is preserved.
static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) {
uintptr_t flags = __readeflags();
_disable();
KIRQL oldIrql = KeGetCurrentIrql();
if (oldIrql < DISPATCH_LEVEL)
KeRaiseIrql(DISPATCH_LEVEL);
while (_InterlockedCompareExchange(&sl->Locked, 1, 0) != 0)
_mm_pause();
return KiPackSpinlockCookie(flags, oldIrql);
}
// Release and restore the caller's previous software IRQL/RFLAGS state.
static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t cookie) {
uintptr_t flags = KiSpinlockCookieFlags(cookie);
KIRQL oldIrql = KiSpinlockCookieIrql(cookie);
_InterlockedExchange(&sl->Locked, 0);
KeLowerIrql(oldIrql);
__writeeflags(flags);
}
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outCookie) {
uintptr_t flags = __readeflags();
_disable();
KIRQL oldIrql = KeGetCurrentIrql();
if (oldIrql < DISPATCH_LEVEL)
KeRaiseIrql(DISPATCH_LEVEL);
if (_InterlockedCompareExchange(&sl->Locked, 1, 0) == 0) {
*outCookie = KiPackSpinlockCookie(flags, oldIrql);
return 1;
}
KeLowerIrql(oldIrql);
__writeeflags(flags);
return 0;
}
// Transitional names for the IRQL-aware driver model work that follows.
static __forceinline uintptr_t KeAcquireSpinlockRaiseToDpc(SPINLOCK* sl) {
return KeAcquireSpinlock(sl);
}
static __forceinline void KeReleaseSpinlockFromDpc(SPINLOCK* sl, uintptr_t cookie) {
KeReleaseSpinlock(sl, cookie);
}
code/kernel/ke_status.h
+46 -39
View File
@@ -1,39 +1,46 @@
#pragma once
// ke_status.h - kernel status codes
// All fallible kernel functions return KSTATUS.
// Success is always zero; failures are positive named constants.
#include <stdint.h>
typedef uint32_t KSTATUS;
// --- General & Control (0x0000) ---
#define STATUS_SUCCESS ((KSTATUS)0x0000)
#define STATUS_FAILURE ((KSTATUS)0x0001)
#define STATUS_NOT_SUPPORTED ((KSTATUS)0x0002)
#define STATUS_TIMEOUT ((KSTATUS)0x0003)
#define STATUS_PENDING ((KSTATUS)0x0004)
// --- Validation & Parameters (0x0010) ---
#define STATUS_INVALID_PARAMETER ((KSTATUS)0x0010)
#define STATUS_INVALID_HANDLE ((KSTATUS)0x0011)
#define STATUS_BUFFER_TOO_SMALL ((KSTATUS)0x0012)
#define STATUS_CHECKSUM_MISMATCH ((KSTATUS)0x0013)
// --- Resource & Access (0x0020) ---
#define STATUS_NO_MEMORY ((KSTATUS)0x0020)
#define STATUS_ACCESS_DENIED ((KSTATUS)0x0021)
#define STATUS_NOT_FOUND ((KSTATUS)0x0022)
#define STATUS_ALREADY_EXISTS ((KSTATUS)0x0023)
#define STATUS_OBJECT_NAME_COLLISION ((KSTATUS)0x0024)
// --- State & Lifecycle (0x0030) ---
#define STATUS_OBJECT_DEAD ((KSTATUS)0x0030)
#define STATUS_QUEUE_FULL ((KSTATUS)0x0031)
#define STATUS_QUEUE_EMPTY ((KSTATUS)0x0032)
// --- Hardware & External (0x0040) ---
#define STATUS_DEVICE_ERROR ((KSTATUS)0x0040)
#define STATUS_NO_MORE_FILES ((KSTATUS)0x0050)
#define KSUCCESS(s) ((s) == STATUS_SUCCESS)
#pragma once
// ke_status.h - kernel status codes
// All fallible kernel functions return KSTATUS.
// Success is always zero; failures are positive named constants.
#include <stdint.h>
typedef uint32_t KSTATUS;
// --- General & Control (0x0000) ---
#define STATUS_SUCCESS ((KSTATUS)0x0000)
#define STATUS_FAILURE ((KSTATUS)0x0001)
#define STATUS_NOT_SUPPORTED ((KSTATUS)0x0002)
#define STATUS_TIMEOUT ((KSTATUS)0x0003)
#define STATUS_PENDING ((KSTATUS)0x0004)
#define STATUS_ALERTED ((KSTATUS)0x0005)
#define STATUS_KERNEL_APC ((KSTATUS)0x0006)
#define STATUS_USER_APC ((KSTATUS)0x0007)
// --- Validation & Parameters (0x0010) ---
#define STATUS_INVALID_PARAMETER ((KSTATUS)0x0010)
#define STATUS_INVALID_HANDLE ((KSTATUS)0x0011)
#define STATUS_BUFFER_TOO_SMALL ((KSTATUS)0x0012)
#define STATUS_CHECKSUM_MISMATCH ((KSTATUS)0x0013)
// --- Resource & Access (0x0020) ---
#define STATUS_NO_MEMORY ((KSTATUS)0x0020)
#define STATUS_ACCESS_DENIED ((KSTATUS)0x0021)
#define STATUS_NOT_FOUND ((KSTATUS)0x0022)
#define STATUS_ALREADY_EXISTS ((KSTATUS)0x0023)
#define STATUS_OBJECT_NAME_COLLISION ((KSTATUS)0x0024)
// --- State & Lifecycle (0x0030) ---
#define STATUS_OBJECT_DEAD ((KSTATUS)0x0030)
#define STATUS_QUEUE_FULL ((KSTATUS)0x0031)
#define STATUS_QUEUE_EMPTY ((KSTATUS)0x0032)
#define STATUS_CANCELLED ((KSTATUS)0x0033)
// --- Hardware & External (0x0040) ---
#define STATUS_DEVICE_ERROR ((KSTATUS)0x0040)
#define STATUS_NO_MORE_FILES ((KSTATUS)0x0050)
// --- Execution level (0x0060) ---
#define STATUS_INVALID_IRQL ((KSTATUS)0x0060)
#define KSUCCESS(s) ((s) == STATUS_SUCCESS)
code/kernel/ke_syscall.asm
+364 -171
View File
@@ -1,171 +1,364 @@
; ke_syscall.asm - INT 0x40 kernel entry (MASM64)
;
; On entry the CPU has pushed (ring-3 -> ring-0 transition):
; SS, RSP, RFLAGS, CS, RIP (on the kernel stack via TSS.RSP0)
; RAX = syscall number
; RCX, RDX, R8, R9 = args 0-3 (MS x64)
; R10, R11 = args 4-5 (our extension)
EXTERN KeSyscallDispatch:PROC
.CODE
KeSyscallEntry PROC
sub rsp, 8 ; 16-byte align the stack frame
; We need to allocate 32 bytes for shadow space, plus 24 bytes for
; the 3 stack arguments (a3, a4, a5), plus 8 bytes of padding to keep
; the stack 16-byte aligned. Total = 64.
sub rsp, 64
; Place the 5th, 6th, and 7th arguments onto the stack for KeSyscallDispatch
mov [rsp + 32], r9 ; a3
mov [rsp + 40], r10 ; a4
mov [rsp + 48], r11 ; a5
; Shift the first 4 registers down one slot
mov r9, r8 ; a2
mov r8, rdx ; a1
mov rdx, rcx ; a0
mov rcx, rax ; nr (syscall number)
; Call the C dispatcher
call KeSyscallDispatch
add rsp, 72 ; Clean up (64 + 8)
iretq
KeSyscallEntry ENDP
; ---------------------------------------------------------------------------
; Fault stubs - thin wrappers that collect the CPU exception frame and
; forward to KeFaultDispatch (C), which handles both kernel panics and
; user-process bugchecks.
;
; KeFaultDispatch signature (MS x64):
; void KeFaultDispatch(int vector, // rcx
; UINT64 rip, // rdx
; UINT64 errCode, // r8
; UINT64 cr2, // r9
; UINT64 cs) // [rsp+20h] (5th arg, on stack)
; ---------------------------------------------------------------------------
EXTERN KeFaultDispatch:PROC
; Allocation for a 5-argument call (MS x64):
; 32 bytes shadow space (0x00..0x1F)
; 8 bytes for 5th arg (0x20..0x27)
; Total = 0x28 bytes.
; After "and rsp,-16" rsp%16==0; sub 0x28 (40) → rsp%16==8,
; so the subsequent "call" push makes rsp 16-byte aligned at callee entry. ✓
FAULT_FRAME_SIZE EQU 28h
; ---------------------------------------------------------
; Divide Error (#0) - NO error code pushed by CPU
; Exception frame at entry: [rsp]=RIP [rsp+8]=CS [rsp+16]=RFLAGS
; ---------------------------------------------------------
KeFaultDE PROC
mov rdx, [rsp] ; Arg2: rip
xor r8, r8 ; Arg3: errCode = 0 (no error code for #DE)
xor r9, r9 ; Arg4: cr2 = 0 (not a page fault)
mov r10, [rsp+8] ; cs (save before we clobber rsp)
mov ecx, 0 ; Arg1: vector = 0
and rsp, -16
sub rsp, FAULT_FRAME_SIZE
mov [rsp+20h], r10 ; Arg5: cs
call KeFaultDispatch ; _Noreturn - never returns
KeFaultDE ENDP
; ---------------------------------------------------------
; General Protection Fault (#13) - error code pushed by CPU
; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS
; ---------------------------------------------------------
KeFaultGP PROC
mov r8, [rsp] ; Arg3: errCode
mov rdx, [rsp+8] ; Arg2: rip
xor r9, r9 ; Arg4: cr2 = 0 (not a page fault)
mov r10, [rsp+16] ; cs
mov ecx, 13 ; Arg1: vector = 13
and rsp, -16
sub rsp, FAULT_FRAME_SIZE
mov [rsp+20h], r10 ; Arg5: cs
call KeFaultDispatch
KeFaultGP ENDP
; ---------------------------------------------------------
; Page Fault (#14) - error code pushed by CPU
; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS
; ---------------------------------------------------------
KeFaultPF PROC
mov r8, [rsp] ; Arg3: errCode
mov rdx, [rsp+8] ; Arg2: rip
mov r9, cr2 ; Arg4: cr2 (faulting virtual address)
mov r10, [rsp+16] ; cs
mov ecx, 14 ; Arg1: vector = 14
and rsp, -16
sub rsp, FAULT_FRAME_SIZE
mov [rsp+20h], r10 ; Arg5: cs
call KeFaultDispatch
KeFaultPF ENDP
EXTERN KeIrqDispatch:PROC
IrqStub MACRO num
PUBLIC KeIrqStub&num
KeIrqStub&num PROC
push rax
mov al, num
jmp KeIrqCommon
KeIrqStub&num ENDP
ENDM
IrqStub 0
IrqStub 1
IrqStub 2
IrqStub 3
IrqStub 4
IrqStub 5
IrqStub 6
IrqStub 7
IrqStub 8
IrqStub 9
IrqStub 10
IrqStub 11
IrqStub 12
IrqStub 13
IrqStub 14
IrqStub 15
IrqStub 16
IrqStub 17
IrqStub 18
IrqStub 19
IrqStub 20
IrqStub 21
IrqStub 22
IrqStub 23
KeIrqCommon PROC
; al = irq number, rax saved on stack
; Build a minimal calling-convention-safe frame
push rcx
push rdx
push r8
push r9
push r10
push r11
movzx ecx, al ; first arg = irq number (rcx, Win64 ABI)
sub rsp, 32 ; shadow space
call KeIrqDispatch
add rsp, 32
pop r11
pop r10
pop r9
pop r8
pop rdx
pop rcx
pop rax
iretq
KeIrqCommon ENDP
END
; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64)
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
; slots, which is the foundation for future user APC return redirection.
EXTERN KeSyscallDispatch:PROC
EXTERN KiEnterTrapFrame:PROC
EXTERN KiExitTrapFrame:PROC
KTRAP_FRAME_SIZE EQU 240
KTRAP_FRAME_MAGIC EQU 4650545241544B31h
KTRAP_KIND_SYSCALL EQU 1
KTRAP_KIND_INTERRUPT EQU 2
KTRAP_MODE_KERNEL EQU 0
KTRAP_MODE_USER EQU 1
TF_Magic EQU 0
TF_Kind EQU 8
TF_Vector EQU 12
TF_PreviousMode EQU 16
TF_Reserved EQU 20
TF_Rax EQU 24
TF_Rcx EQU 32
TF_Rdx EQU 40
TF_Rbx EQU 48
TF_Rbp EQU 56
TF_Rsi EQU 64
TF_Rdi EQU 72
TF_R8 EQU 80
TF_R9 EQU 88
TF_R10 EQU 96
TF_R11 EQU 104
TF_R12 EQU 112
TF_R13 EQU 120
TF_R14 EQU 128
TF_R15 EQU 136
TF_Rip EQU 144
TF_Cs EQU 152
TF_Rflags EQU 160
TF_Rsp EQU 168
TF_Ss EQU 176
TF_RipSlot EQU 184
TF_CsSlot EQU 192
TF_RflagsSlot EQU 200
TF_RspSlot EQU 208
TF_SsSlot EQU 216
.CODE
KeSyscallEntry PROC
; CPU frame at entry, ring 3 -> ring 0:
; [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS,
; [rsp+18] user RSP, [rsp+20] SS
; Save RBX because we use it as the trap-frame pointer and user-mode code
; sees the syscall stub as a normal function call.
push rbx
sub rsp, KTRAP_FRAME_SIZE
mov rbx, rsp
mov [rbx + TF_Rax], rax
mov rax, KTRAP_FRAME_MAGIC
mov [rbx + TF_Magic], rax
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
mov dword ptr [rbx + TF_Vector], 40h
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
mov dword ptr [rbx + TF_Reserved], 0
mov [rbx + TF_Rcx], rcx
mov [rbx + TF_Rdx], rdx
mov rax, [rbx + KTRAP_FRAME_SIZE]
mov [rbx + TF_Rbx], rax
mov [rbx + TF_Rbp], rbp
mov [rbx + TF_Rsi], rsi
mov [rbx + TF_Rdi], rdi
mov [rbx + TF_R8], r8
mov [rbx + TF_R9], r9
mov [rbx + TF_R10], r10
mov [rbx + TF_R11], r11
mov [rbx + TF_R12], r12
mov [rbx + TF_R13], r13
mov [rbx + TF_R14], r14
mov [rbx + TF_R15], r15
lea rax, [rbx + KTRAP_FRAME_SIZE + 8]
mov [rbx + TF_RipSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rip], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 16]
mov [rbx + TF_CsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Cs], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 24]
mov [rbx + TF_RflagsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rflags], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 32]
mov [rbx + TF_RspSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rsp], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 40]
mov [rbx + TF_SsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Ss], rdx
sub rsp, 32
mov rcx, rbx
call KiEnterTrapFrame
add rsp, 32
; Re-order original user arguments into KeSyscallDispatch(nr,a0..a5).
; Original registers were saved in the trap frame before any C calls.
sub rsp, 64
mov rax, [rbx + TF_R9]
mov [rsp + 32], rax ; a3
mov rax, [rbx + TF_R10]
mov [rsp + 40], rax ; a4
mov rax, [rbx + TF_R11]
mov [rsp + 48], rax ; a5
mov r9, [rbx + TF_R8] ; a2
mov r8, [rbx + TF_Rdx] ; a1
mov rdx, [rbx + TF_Rcx] ; a0
mov rcx, [rbx + TF_Rax] ; nr
call KeSyscallDispatch
add rsp, 64
mov [rbx + TF_Rax], rax
sub rsp, 32
mov rcx, rbx
call KiExitTrapFrame
add rsp, 32
mov rax, [rbx + TF_Rax]
add rsp, KTRAP_FRAME_SIZE
pop rbx
iretq
KeSyscallEntry ENDP
; ---------------------------------------------------------------------------
; Fault stubs - thin wrappers that collect the CPU exception frame and
; forward to KeFaultDispatch (C), which handles both kernel panics and
; user-process bugchecks.
;
; KeFaultDispatch signature (MS x64):
; void KeFaultDispatch(int vector, // rcx
; UINT64 rip, // rdx
; UINT64 errCode, // r8
; UINT64 cr2, // r9
; UINT64 cs) // [rsp+20h] (5th arg, on stack)
; ---------------------------------------------------------------------------
EXTERN KeFaultDispatch:PROC
; Allocation for a 5-argument call (MS x64):
; 32 bytes shadow space (0x00..0x1F)
; 8 bytes for 5th arg (0x20..0x27)
; Total = 0x28 bytes.
; After "and rsp,-16" rsp%16==0; sub 0x28 (40) → rsp%16==8,
; so the subsequent "call" push makes rsp 16-byte aligned at callee entry. ✓
FAULT_FRAME_SIZE EQU 28h
; ---------------------------------------------------------
; Divide Error (#0) - NO error code pushed by CPU
; Exception frame at entry: [rsp]=RIP [rsp+8]=CS [rsp+16]=RFLAGS
; ---------------------------------------------------------
KeFaultDE PROC
mov rdx, [rsp] ; Arg2: rip
xor r8, r8 ; Arg3: errCode = 0 (no error code for #DE)
xor r9, r9 ; Arg4: cr2 = 0 (not a page fault)
mov r10, [rsp+8] ; cs (save before we clobber rsp)
mov ecx, 0 ; Arg1: vector = 0
and rsp, -16
sub rsp, FAULT_FRAME_SIZE
mov [rsp+20h], r10 ; Arg5: cs
call KeFaultDispatch ; _Noreturn - never returns
KeFaultDE ENDP
; ---------------------------------------------------------
; General Protection Fault (#13) - error code pushed by CPU
; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS
; ---------------------------------------------------------
KeFaultGP PROC
mov r8, [rsp] ; Arg3: errCode
mov rdx, [rsp+8] ; Arg2: rip
xor r9, r9 ; Arg4: cr2 = 0 (not a page fault)
mov r10, [rsp+16] ; cs
mov ecx, 13 ; Arg1: vector = 13
and rsp, -16
sub rsp, FAULT_FRAME_SIZE
mov [rsp+20h], r10 ; Arg5: cs
call KeFaultDispatch
KeFaultGP ENDP
; ---------------------------------------------------------
; Page Fault (#14) - error code pushed by CPU
; Exception frame at entry: [rsp]=ErrCode [rsp+8]=RIP [rsp+16]=CS
; ---------------------------------------------------------
KeFaultPF PROC
mov r8, [rsp] ; Arg3: errCode
mov rdx, [rsp+8] ; Arg2: rip
mov r9, cr2 ; Arg4: cr2 (faulting virtual address)
mov r10, [rsp+16] ; cs
mov ecx, 14 ; Arg1: vector = 14
and rsp, -16
sub rsp, FAULT_FRAME_SIZE
mov [rsp+20h], r10 ; Arg5: cs
call KeFaultDispatch
KeFaultPF ENDP
EXTERN KeIrqDispatch:PROC
EXTERN KiEnterInterrupt:PROC
EXTERN KiExitInterrupt:PROC
IrqStub MACRO num
PUBLIC KeIrqStub&num
KeIrqStub&num PROC
push rax
mov al, num
jmp KeIrqCommon
KeIrqStub&num ENDP
ENDM
IrqStub 0
IrqStub 1
IrqStub 2
IrqStub 3
IrqStub 4
IrqStub 5
IrqStub 6
IrqStub 7
IrqStub 8
IrqStub 9
IrqStub 10
IrqStub 11
IrqStub 12
IrqStub 13
IrqStub 14
IrqStub 15
IrqStub 16
IrqStub 17
IrqStub 18
IrqStub 19
IrqStub 20
IrqStub 21
IrqStub 22
IrqStub 23
KeIrqCommon PROC
; al = irq number, original rax saved by the per-IRQ stub.
push rcx
push rdx
push r8
push r9
push r10
push r11
push rbx
sub rsp, KTRAP_FRAME_SIZE
mov rbx, rsp
movzx edx, al
mov rax, KTRAP_FRAME_MAGIC
mov [rbx + TF_Magic], rax
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT
mov [rbx + TF_Vector], edx
mov dword ptr [rbx + TF_Reserved], 0
; Saved volatile registers live immediately above the trap frame.
mov rax, [rbx + KTRAP_FRAME_SIZE + 56]
mov [rbx + TF_Rax], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 48]
mov [rbx + TF_Rcx], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 40]
mov [rbx + TF_Rdx], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 0]
mov [rbx + TF_Rbx], rax
mov [rbx + TF_Rbp], rbp
mov [rbx + TF_Rsi], rsi
mov [rbx + TF_Rdi], rdi
mov rax, [rbx + KTRAP_FRAME_SIZE + 32]
mov [rbx + TF_R8], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 24]
mov [rbx + TF_R9], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 16]
mov [rbx + TF_R10], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 8]
mov [rbx + TF_R11], rax
mov [rbx + TF_R12], r12
mov [rbx + TF_R13], r13
mov [rbx + TF_R14], r14
mov [rbx + TF_R15], r15
; CPU interrupt frame starts after the 8 saved registers.
lea rax, [rbx + KTRAP_FRAME_SIZE + 64]
mov [rbx + TF_RipSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rip], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 72]
mov [rbx + TF_CsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Cs], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 80]
mov [rbx + TF_RflagsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rflags], rdx
mov rax, [rbx + TF_Cs]
and eax, 3
cmp eax, 3
jne short IrqKernelModeFrame
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
lea rax, [rbx + KTRAP_FRAME_SIZE + 88]
mov [rbx + TF_RspSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rsp], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 96]
mov [rbx + TF_SsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Ss], rdx
jmp short IrqFrameReady
IrqKernelModeFrame:
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_KERNEL
mov qword ptr [rbx + TF_Rsp], 0
mov qword ptr [rbx + TF_Ss], 0
mov qword ptr [rbx + TF_RspSlot], 0
mov qword ptr [rbx + TF_SsSlot], 0
IrqFrameReady:
sub rsp, 40 ; 32-byte shadow space + alignment/local slack
mov rcx, rbx
call KiEnterTrapFrame
call KiEnterInterrupt
mov ecx, [rbx + TF_Vector]
call KeIrqDispatch
call KiExitInterrupt
mov rcx, rbx
call KiExitTrapFrame
add rsp, 40
add rsp, KTRAP_FRAME_SIZE
pop rbx
pop r11
pop r10
pop r9
pop r8
pop rdx
pop rcx
pop rax
iretq
KeIrqCommon ENDP
END
code/kernel/ke_syscall.c
+383 -125
View File
@@ -23,7 +23,7 @@
//
// KeCloseHandle(proc, handle)
// -> ObDereferenceObject(obj)
// -> obj->Destroy(obj) [type-specific cleanup, e.g. FileDestroy]
// -> obj->Destroy(obj) [type-specific cleanup, e.g. KiFileDestroy]
// -> MmFreePool(obj)
//
// There is no separate SyscallCloseFile. Removing that stub was the right
@@ -39,6 +39,7 @@
#include <ke_mutex.h>
#include <ke_semaphore.h>
#include <ke_event.h>
#include <ke_apc.h>
#include <ke_queue.h>
#include <ke_debug.h>
#include <mm_virt.h>
@@ -46,12 +47,43 @@
#include <syscall_nr.h>
#include <hal_api.h>
#include <io_file.h>
#include <ke_trap.h>
// ---------------------------------------------------------------------------
// HAL pointer - set once at boot by KeSyscallSetHal()
// ---------------------------------------------------------------------------
static const HAL_API* gHal;
typedef struct SYS_IO_STATUS_BLOCK_K {
volatile uint32_t Status;
volatile uint64_t Information;
} SYS_IO_STATUS_BLOCK_K;
typedef void (*SYS_USER_APC_ROUTINE_K)(void* normalContext,
SYS_IO_STATUS_BLOCK_K* iosb,
uint64_t information);
typedef struct SYS_ASYNC_FILE_REQUEST_K {
uint32_t FileHandle;
uint32_t EventHandle;
uint32_t Flags;
uint32_t Reserved;
uint64_t Offset;
void* Buffer;
size_t Size;
SYS_IO_STATUS_BLOCK_K* Iosb;
SYS_USER_APC_ROUTINE_K ApcRoutine;
void* ApcContext;
uint32_t* RequestHandleOut; // optional cancellation/query handle output
} SYS_ASYNC_FILE_REQUEST_K;
typedef struct SYS_USER_ASYNC_APC_CONTEXT {
KAPC KernelApc;
THREAD* TargetThread;
SYS_USER_APC_ROUTINE_K UserRoutine;
void* UserContext;
} SYS_USER_ASYNC_APC_CONTEXT;
void KeSyscallSetHal(const HAL_API* hal) { gHal = hal; }
// ---------------------------------------------------------------------------
@@ -59,7 +91,7 @@ void KeSyscallSetHal(const HAL_API* hal) { gHal = hal; }
// Bumps the refcount on success; caller must ObDereferenceObject when done.
// Returns NULL if the handle is invalid or the type does not match.
// ---------------------------------------------------------------------------
static OBJECT* HandleToObject(uint32_t handle, OBJECT_TYPE expected) {
static OBJECT* KiHandleToObject(uint32_t handle, OBJECT_TYPE expected) {
PROCESS* proc = KeGetCurrentProcess();
if (!proc) return NULL;
OBJECT* obj = KeLookupHandle(proc, handle); // bumps refcount
@@ -76,7 +108,7 @@ static OBJECT* HandleToObject(uint32_t handle, OBJECT_TYPE expected) {
// Takes one reference on behalf of the handle slot.
// On failure, drops the reference it would have stored.
// ---------------------------------------------------------------------------
static KSTATUS AllocHandleForCurrent(OBJECT* obj, uint32_t* handleOut) {
static KSTATUS KiAllocHandleForCurrent(OBJECT* obj, uint32_t* handleOut) {
PROCESS* proc = KeGetCurrentProcess();
if (!proc) return STATUS_FAILURE;
KSTATUS s = KeAllocateHandle(proc, obj, handleOut);
@@ -173,6 +205,24 @@ static uint64_t SyscallSleep(uint64_t a0, uint64_t a1, uint64_t a2,
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// SYS_SET_USER_APC_DISPATCHER a0 = dispatcher entry in syscall.dll/user image
// ---------------------------------------------------------------------------
static uint64_t SyscallSetUserApcDispatcher(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KeSetUserApcDispatcher((void*)(uintptr_t)a0);
}
// ---------------------------------------------------------------------------
// SYS_APC_RETURN a0 = SYS_USER_APC_FRAME* on the user stack
// ---------------------------------------------------------------------------
static uint64_t SyscallApcReturn(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KeUserApcReturn((const SYS_USER_APC_FRAME_K*)(uintptr_t)a0);
}
// ---------------------------------------------------------------------------
// SYS_SET_THREAD_PRIORITY (7) a0 = priority
// ---------------------------------------------------------------------------
@@ -197,12 +247,12 @@ static uint64_t SyscallAlloc(uint64_t a0, uint64_t a1, uint64_t a2,
}
// ---------------------------------------------------------------------------
// SYS_FREE (9) a0 = ptr, a1 = size
// SYS_FREE a0 = ptr
// ---------------------------------------------------------------------------
static uint64_t SyscallFree(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
if (a0) MmFreeUserMemory((void*)(uintptr_t)a0, (size_t)a1);
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (a0) MmFreeUserMemory((void*)(uintptr_t)a0);
return STATUS_SUCCESS;
}
@@ -224,10 +274,10 @@ static uint64_t SyscallMutexCreate(uint64_t a0, uint64_t a1, uint64_t a2,
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!a0) return STATUS_INVALID_PARAMETER;
MUTEX* mx = NULL;
KSTATUS s = MutexCreate(&mx);
KSTATUS s = KeMutexCreate(&mx);
if (!KSUCCESS(s)) return s;
uint32_t h = 0;
s = AllocHandleForCurrent(&mx->Header, &h); // takes ownership of the ref
s = KiAllocHandleForCurrent(&mx->Header, &h); // takes ownership of the ref
if (!KSUCCESS(s)) return s;
*(uint32_t*)(uintptr_t)a0 = h;
return STATUS_SUCCESS;
@@ -239,7 +289,7 @@ static uint64_t SyscallMutexCreate(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallMutexAcquire(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
MUTEX* mx = (MUTEX*)HandleToObject((uint32_t)a0, ObjectTypeMutex);
MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex);
if (!mx) return STATUS_NOT_FOUND;
KSTATUS s = KeAcquireMutex(mx);
ObDereferenceObject(&mx->Header);
@@ -252,7 +302,7 @@ static uint64_t SyscallMutexAcquire(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
MUTEX* mx = (MUTEX*)HandleToObject((uint32_t)a0, ObjectTypeMutex);
MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex);
if (!mx) return STATUS_NOT_FOUND;
KSTATUS s = KeTryAcquireMutex(mx);
ObDereferenceObject(&mx->Header);
@@ -265,7 +315,7 @@ static uint64_t SyscallMutexTryAcquire(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallMutexRelease(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
MUTEX* mx = (MUTEX*)HandleToObject((uint32_t)a0, ObjectTypeMutex);
MUTEX* mx = (MUTEX*)KiHandleToObject((uint32_t)a0, ObjectTypeMutex);
if (!mx) return STATUS_NOT_FOUND;
KeReleaseMutex(mx);
ObDereferenceObject(&mx->Header);
@@ -283,7 +333,7 @@ static uint64_t SyscallSemCreate(uint64_t a0, uint64_t a1, uint64_t a2,
KSTATUS s = KeSemaphoreCreate((int32_t)a0, (int32_t)a1, &sem);
if (!KSUCCESS(s)) return s;
uint32_t h = 0;
s = AllocHandleForCurrent(&sem->Header, &h);
s = KiAllocHandleForCurrent(&sem->Header, &h);
if (!KSUCCESS(s)) return s;
*(uint32_t*)(uintptr_t)a2 = h;
return STATUS_SUCCESS;
@@ -295,7 +345,7 @@ static uint64_t SyscallSemCreate(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallSemWait(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SEMAPHORE* sem = (SEMAPHORE*)HandleToObject((uint32_t)a0, ObjectTypeSemaphore);
SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore);
if (!sem) return STATUS_NOT_FOUND;
KSTATUS s = KeWaitSemaphore(sem);
ObDereferenceObject(&sem->Header);
@@ -308,7 +358,7 @@ static uint64_t SyscallSemWait(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SEMAPHORE* sem = (SEMAPHORE*)HandleToObject((uint32_t)a0, ObjectTypeSemaphore);
SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore);
if (!sem) return STATUS_NOT_FOUND;
KSTATUS s = KeTrySemaphore(sem);
ObDereferenceObject(&sem->Header);
@@ -321,7 +371,7 @@ static uint64_t SyscallSemTryWait(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallSemSignal(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
SEMAPHORE* sem = (SEMAPHORE*)HandleToObject((uint32_t)a0, ObjectTypeSemaphore);
SEMAPHORE* sem = (SEMAPHORE*)KiHandleToObject((uint32_t)a0, ObjectTypeSemaphore);
if (!sem) return STATUS_NOT_FOUND;
KeSignalSemaphore(sem);
ObDereferenceObject(&sem->Header);
@@ -339,7 +389,7 @@ static uint64_t SyscallEventCreate(uint64_t a0, uint64_t a1, uint64_t a2,
KSTATUS s = KeEventCreate((EVENT_TYPE)a0, (uint32_t)a1, &ev);
if (!KSUCCESS(s)) return s;
uint32_t h = 0;
s = AllocHandleForCurrent(&ev->Header, &h);
s = KiAllocHandleForCurrent(&ev->Header, &h);
if (!KSUCCESS(s)) return s;
*(uint32_t*)(uintptr_t)a2 = h;
return STATUS_SUCCESS;
@@ -351,7 +401,7 @@ static uint64_t SyscallEventCreate(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallEventSignal(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
EVENT* ev = (EVENT*)HandleToObject((uint32_t)a0, ObjectTypeEvent);
EVENT* ev = (EVENT*)KiHandleToObject((uint32_t)a0, ObjectTypeEvent);
if (!ev) return STATUS_NOT_FOUND;
KeSignalEvent(ev);
ObDereferenceObject(&ev->Header);
@@ -364,7 +414,7 @@ static uint64_t SyscallEventSignal(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallEventClear(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
EVENT* ev = (EVENT*)HandleToObject((uint32_t)a0, ObjectTypeEvent);
EVENT* ev = (EVENT*)KiHandleToObject((uint32_t)a0, ObjectTypeEvent);
if (!ev) return STATUS_NOT_FOUND;
KeClearEvent(ev);
ObDereferenceObject(&ev->Header);
@@ -372,14 +422,20 @@ static uint64_t SyscallEventClear(uint64_t a0, uint64_t a1, uint64_t a2,
}
// ---------------------------------------------------------------------------
// SYS_EVENT_WAIT (22) a0 = handle
// SYS_EVENT_WAIT a0 = handle, a1 = alertable
// ---------------------------------------------------------------------------
static uint64_t SyscallEventWait(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
EVENT* ev = (EVENT*)HandleToObject((uint32_t)a0, ObjectTypeEvent);
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
EVENT* ev = (EVENT*)KiHandleToObject((uint32_t)a0, ObjectTypeEvent);
if (!ev) return STATUS_NOT_FOUND;
KSTATUS s = KeWaitForEvent(ev);
uint8_t alertable = a1 ? 1 : 0;
KSTATUS s = KeWaitForEvent(ev, alertable);
if (alertable) {
THREAD* thread = KeGetCurrentThread();
if (thread && thread->UserApcPending)
thread->UserApcDeliverable = 1;
}
ObDereferenceObject(&ev->Header);
return s;
}
@@ -392,10 +448,10 @@ static uint64_t SyscallQueueCreate(uint64_t a0, uint64_t a1, uint64_t a2,
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
if (!a0) return STATUS_INVALID_PARAMETER;
QUEUE* q = NULL;
KSTATUS s = QueueCreate(&q);
KSTATUS s = KeQueueCreate(&q);
if (!KSUCCESS(s)) return s;
uint32_t h = 0;
s = AllocHandleForCurrent(&q->Header, &h);
s = KiAllocHandleForCurrent(&q->Header, &h);
if (!KSUCCESS(s)) return s;
*(uint32_t*)(uintptr_t)a0 = h;
return STATUS_SUCCESS;
@@ -408,7 +464,7 @@ static uint64_t SyscallQueueSend(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a3; (void)a4; (void)a5;
if (!a1) return STATUS_INVALID_PARAMETER;
QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a0, ObjectTypeQueue);
QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a0, ObjectTypeQueue);
if (!q) return STATUS_NOT_FOUND;
KSTATUS s = KeSendMessage(q, (const QUEUE_MESSAGE*)(uintptr_t)a1, (bool)a2);
ObDereferenceObject(&q->Header);
@@ -422,7 +478,7 @@ static uint64_t SyscallQueueRecv(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a3; (void)a4; (void)a5;
if (!a1) return STATUS_INVALID_PARAMETER;
QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a0, ObjectTypeQueue);
QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a0, ObjectTypeQueue);
if (!q) return STATUS_NOT_FOUND;
KSTATUS s = KeReceiveMessage(q, (QUEUE_MESSAGE*)(uintptr_t)a1, (bool)a2);
ObDereferenceObject(&q->Header);
@@ -434,7 +490,7 @@ static uint64_t SyscallQueueRecv(uint64_t a0, uint64_t a1, uint64_t a2,
//
// The one and only close path. KeCloseHandle drops the handle-table reference
// on 'obj'. If that was the last reference, ObDereferenceObject runs the
// type-specific Destroy callback (FileDestroy, etc.) and frees the pool memory.
// type-specific Destroy callback (KiFileDestroy, etc.) and frees the pool memory.
// No per-type close syscall needed or wanted.
// ---------------------------------------------------------------------------
static uint64_t SyscallCloseHandle(uint64_t a0, uint64_t a1, uint64_t a2,
@@ -457,10 +513,10 @@ static uint64_t SyscallOpenFile(uint64_t a0, uint64_t a1, uint64_t a2,
KSTATUS s = IoOpenFile((const char*)(uintptr_t)a0, &file);
if (!KSUCCESS(s)) return s;
uint32_t h = 0;
s = AllocHandleForCurrent(&file->Header, &h);
s = KiAllocHandleForCurrent(&file->Header, &h);
if (!KSUCCESS(s)) {
// AllocHandleForCurrent already dropped the ref on failure;
// the refcount is now 0, so ObDereferenceObject will run FileDestroy.
// KiAllocHandleForCurrent already dropped the ref on failure;
// the refcount is now 0, so ObDereferenceObject will run KiFileDestroy.
ObDereferenceObject(&file->Header);
return s;
}
@@ -475,7 +531,7 @@ static uint64_t SyscallReadFile(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a4; (void)a5;
if (!a1) return STATUS_INVALID_PARAMETER;
FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile);
FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile);
if (!file) return STATUS_NOT_FOUND;
size_t got = 0;
KSTATUS s = IoReadFile(file, (void*)(uintptr_t)a1, (size_t)a2, &got);
@@ -491,7 +547,7 @@ static uint64_t SyscallWriteFile(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a4; (void)a5;
if (!a1) return STATUS_INVALID_PARAMETER;
FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile);
FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile);
if (!file) return STATUS_NOT_FOUND;
size_t written = 0;
KSTATUS s = IoWriteFile(file, (const void*)(uintptr_t)a1, (size_t)a2, &written);
@@ -500,6 +556,189 @@ static uint64_t SyscallWriteFile(uint64_t a0, uint64_t a1, uint64_t a2,
return s;
}
static void SyscallUserAsyncKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
(void)normalRoutine;
(void)normalContext;
(void)systemArgument1;
(void)systemArgument2;
SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc);
DbgPrint("USER/APC async: kernel routine ctx=0x%llX iosb=0x%llX status=%u info=%llu irql=%u\n",
(uint64_t)ctx,
(uint64_t)(uintptr_t)apc->SystemArgument1,
(uint32_t)(uintptr_t)apc->SystemArgument2,
apc->SystemArgument1 ? ((SYS_IO_STATUS_BLOCK_K*)apc->SystemArgument1)->Information : 0,
KeGetCurrentIrql());
}
static void SyscallUserAsyncApcContextFree(SYS_USER_ASYNC_APC_CONTEXT* ctx) {
if (!ctx) return;
if (ctx->TargetThread) {
ObDereferenceObject(&ctx->TargetThread->Header);
ctx->TargetThread = (THREAD*)0;
}
MmFreePool(ctx);
}
static void SyscallUserAsyncApcRundown(KAPC* apc) {
if (!apc) return;
SYS_USER_ASYNC_APC_CONTEXT* ctx = CONTAINING_RECORD(apc, SYS_USER_ASYNC_APC_CONTEXT, KernelApc);
SyscallUserAsyncApcContextFree(ctx);
}
static void SyscallUserAsyncNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
SYS_USER_ASYNC_APC_CONTEXT* ctx = (SYS_USER_ASYNC_APC_CONTEXT*)normalContext;
SYS_IO_STATUS_BLOCK_K* iosb = (SYS_IO_STATUS_BLOCK_K*)systemArgument1;
uint64_t information = iosb ? iosb->Information : 0;
(void)systemArgument2;
if (ctx && ctx->UserRoutine) {
KSTATUS s = KeQueueUserApc(ctx->TargetThread,
(void*)ctx->UserRoutine,
ctx->UserContext,
iosb,
(void*)(uintptr_t)information);
if (!KSUCCESS(s)) {
DbgPrint("USER/APC async: KeQueueUserApc failed status=%u\n", s);
}
}
SyscallUserAsyncApcContextFree(ctx);
}
static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) {
if (!a0) return STATUS_INVALID_PARAMETER;
const SYS_ASYNC_FILE_REQUEST_K* req =
(const SYS_ASYNC_FILE_REQUEST_K*)(uintptr_t)a0;
if (req->Flags || req->Reserved) return STATUS_INVALID_PARAMETER;
if (!req->Iosb) return STATUS_INVALID_PARAMETER;
if (!req->Buffer && req->Size) return STATUS_INVALID_PARAMETER;
FILE_OBJECT* file =
(FILE_OBJECT*)KiHandleToObject(req->FileHandle, ObjectTypeFile);
if (!file) return STATUS_NOT_FOUND;
EVENT* event = (EVENT*)0;
if (req->EventHandle) {
event = (EVENT*)KiHandleToObject(req->EventHandle, ObjectTypeEvent);
if (!event) {
ObDereferenceObject(&file->Header);
return STATUS_NOT_FOUND;
}
}
KAPC* completionApc = (KAPC*)0;
SYS_USER_ASYNC_APC_CONTEXT* apcCtx = (SYS_USER_ASYNC_APC_CONTEXT*)0;
if (req->ApcRoutine) {
THREAD* thread = KeGetCurrentThread();
if (!thread || !thread->UserApcDispatcher) {
if (event) ObDereferenceObject(&event->Header);
ObDereferenceObject(&file->Header);
return STATUS_INVALID_PARAMETER;
}
apcCtx = (SYS_USER_ASYNC_APC_CONTEXT*)MmAllocatePool(sizeof(SYS_USER_ASYNC_APC_CONTEXT));
if (!apcCtx) {
if (event) ObDereferenceObject(&event->Header);
ObDereferenceObject(&file->Header);
return STATUS_NO_MEMORY;
}
memset(apcCtx, 0, sizeof(*apcCtx));
ObReferenceObject(&thread->Header);
apcCtx->TargetThread = thread;
apcCtx->UserRoutine = req->ApcRoutine;
apcCtx->UserContext = req->ApcContext;
KeInitializeApc(&apcCtx->KernelApc,
thread,
SyscallUserAsyncKernelRoutine,
SyscallUserAsyncNormalRoutine,
apcCtx);
KeSetApcRundownRoutine(&apcCtx->KernelApc, SyscallUserAsyncApcRundown);
completionApc = &apcCtx->KernelApc;
}
IO_ASYNC_FILE_REQUEST_OBJECT* asyncReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
KSTATUS s;
if (write) {
s = IoWriteFileAsyncKernel(file, req->Buffer, req->Size, req->Offset,
(IO_STATUS_BLOCK*)req->Iosb, event,
completionApc,
&asyncReq);
} else {
s = IoReadFileAsyncKernel(file, req->Buffer, req->Size, req->Offset,
(IO_STATUS_BLOCK*)req->Iosb, event,
completionApc,
&asyncReq);
}
if (s == STATUS_PENDING && req->RequestHandleOut && asyncReq) {
uint32_t requestHandle = 0;
KSTATUS hs = KiAllocHandleForCurrent(IoAsyncFileRequestObject(asyncReq), &requestHandle);
if (!KSUCCESS(hs)) {
(void)IoCancelAsyncFileRequest(asyncReq);
ObDereferenceObject(IoAsyncFileRequestObject(asyncReq));
if (event) ObDereferenceObject(&event->Header);
ObDereferenceObject(&file->Header);
return hs;
}
*req->RequestHandleOut = requestHandle;
} else if (req->RequestHandleOut) {
*req->RequestHandleOut = 0;
}
if (asyncReq)
ObDereferenceObject(IoAsyncFileRequestObject(asyncReq)); // drop creator reference
if (!KSUCCESS(s) && s != STATUS_PENDING && apcCtx)
SyscallUserAsyncApcContextFree(apcCtx);
if (event) ObDereferenceObject(&event->Header);
ObDereferenceObject(&file->Header);
return s;
}
// ---------------------------------------------------------------------------
// SYS_READ_FILE_ASYNC (43) a0=*SYS_ASYNC_FILE_REQUEST
// ---------------------------------------------------------------------------
static uint64_t SyscallReadFileAsync(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return SyscallAsyncFileCommon(a0, false);
}
// ---------------------------------------------------------------------------
// SYS_WRITE_FILE_ASYNC (44) a0=*SYS_ASYNC_FILE_REQUEST
// ---------------------------------------------------------------------------
static uint64_t SyscallWriteFileAsync(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return SyscallAsyncFileCommon(a0, true);
}
// ---------------------------------------------------------------------------
// SYS_CANCEL_IO a0 = async request handle
// ---------------------------------------------------------------------------
static uint64_t SyscallCancelIo(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
IO_ASYNC_FILE_REQUEST_OBJECT* req =
(IO_ASYNC_FILE_REQUEST_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeAsyncIo);
if (!req) return STATUS_INVALID_HANDLE;
KSTATUS s = IoCancelAsyncFileRequest(req);
ObDereferenceObject(IoAsyncFileRequestObject(req));
return s;
}
// ---------------------------------------------------------------------------
// SYS_CREATE_FILE (30) a0=*path a1=*handleOut
// ---------------------------------------------------------------------------
@@ -511,7 +750,7 @@ static uint64_t SyscallCreateFile(uint64_t a0, uint64_t a1, uint64_t a2,
KSTATUS s = IoCreateFile((const char*)(uintptr_t)a0, &file);
if (!KSUCCESS(s)) return s;
uint32_t h = 0;
s = AllocHandleForCurrent(&file->Header, &h);
s = KiAllocHandleForCurrent(&file->Header, &h);
if (!KSUCCESS(s)) {
ObDereferenceObject(&file->Header);
return s;
@@ -536,7 +775,7 @@ static uint64_t SyscallDeleteFile(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallTruncateFile(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile);
FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile);
if (!file) return STATUS_NOT_FOUND;
KSTATUS s = IoTruncateFile(file, (uint64_t)a1);
ObDereferenceObject(&file->Header);
@@ -549,9 +788,11 @@ static uint64_t SyscallTruncateFile(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallGetFilePos(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile);
FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile);
if (!file) return (uint64_t)-1;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t pos = file->Offset;
KeReleaseSpinlock(&file->Lock, flags);
ObDereferenceObject(&file->Header);
return pos;
}
@@ -562,7 +803,7 @@ static uint64_t SyscallGetFilePos(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallSeekFile(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile);
FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile);
if (!file) return STATUS_NOT_FOUND;
KSTATUS s = IoSeekFile(file, (uint64_t)a1);
ObDereferenceObject(&file->Header);
@@ -575,7 +816,7 @@ static uint64_t SyscallSeekFile(uint64_t a0, uint64_t a1, uint64_t a2,
static uint64_t SyscallGetFileSize(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
FILE_OBJECT* file = (FILE_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeFile);
FILE_OBJECT* file = (FILE_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeFile);
if (!file) return 0;
uint64_t sz = IoGetFileSize(file);
ObDereferenceObject(&file->Header);
@@ -738,6 +979,15 @@ static uint64_t SyscallDebugPrint(uint64_t a0, uint64_t a1, uint64_t a2,
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// SYS_QUERY_TRAP_FRAME a0=*SYS_TRAP_FRAME_INFO
// ---------------------------------------------------------------------------
static uint64_t SyscallQueryTrapFrame(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
return KeQueryCurrentTrapFrame((SYS_TRAP_FRAME_INFO_K*)(uintptr_t)a0);
}
// ---------------------------------------------------------------------------
// SYS_CREATE_PROCESS (50) a0=*path a1=*handleOut a2=*cmdline (optional)
// ---------------------------------------------------------------------------
@@ -775,6 +1025,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
uintptr_t flags = KeAcquireSpinlock(&parent->CwdLock);
strncpy(proc->CurrentDirectory, parent->CurrentDirectory, sizeof(proc->CurrentDirectory) - 1);
proc->CurrentDirectory[sizeof(proc->CurrentDirectory) - 1] = '\0';
proc->CurrentDrive = parent->CurrentDrive;
KeReleaseSpinlock(&parent->CwdLock, flags);
}
@@ -789,7 +1040,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
s = LdrResolveImportsForProcess(proc, imageBase);
if (!KSUCCESS(s)) {
MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase));
MmFreeUserMemory((void*)imageBase);
ObDereferenceObject(&proc->Header);
return s;
}
@@ -802,7 +1053,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
// to finish wiring it up.
PROCESS* caller = KeGetCurrentProcess();
if (!caller) {
MmFreeUserMemory((void*)imageBase, proc->ImageSize);
MmFreeUserMemory((void*)imageBase);
ObDereferenceObject(&proc->Header);
return STATUS_FAILURE;
}
@@ -813,7 +1064,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
KeCloseHandle(caller, handle);
proc->ImageBase = 0;
proc->ImageSize = 0;
MmFreeUserMemory((void*)imageBase, proc->ImageSize);
MmFreeUserMemory((void*)imageBase);
ObDereferenceObject(&proc->Header);
return s;
}
@@ -825,7 +1076,7 @@ static uint64_t SyscallCreateProcess(uint64_t a0, uint64_t a1, uint64_t a2,
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
s = KeCreateThread(proc, entryPoint, 0, PriorityNormal, &thread);
if (!KSUCCESS(s)) {
MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase));
MmFreeUserMemory((void*)imageBase);
ObDereferenceObject(&proc->Header);
return s;
}
@@ -841,10 +1092,10 @@ static uint64_t SyscallWaitForProcess(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a1; (void)a2; (void)a3; (void)a4; (void)a5;
PROCESS* proc = (PROCESS*)HandleToObject((uint32_t)a0, ObjectTypeProcess);
PROCESS* proc = (PROCESS*)KiHandleToObject((uint32_t)a0, ObjectTypeProcess);
if (!proc) return STATUS_INVALID_HANDLE;
KSTATUS s = KeWaitForEvent(proc->ExitEvent);
KSTATUS s = KeWaitForEvent(proc->ExitEvent, 0);
ObDereferenceObject(&proc->Header);
return s;
}
@@ -856,7 +1107,7 @@ static uint64_t SyscallGetProcessExitCode(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
PROCESS* proc = (PROCESS*)HandleToObject((uint32_t)a0, ObjectTypeProcess);
PROCESS* proc = (PROCESS*)KiHandleToObject((uint32_t)a0, ObjectTypeProcess);
if (!proc) return STATUS_INVALID_HANDLE;
int* exitCodeOut = (int*)(uintptr_t)a1;
@@ -915,12 +1166,12 @@ static uint64_t SyscallReadDir(uint64_t a0, uint64_t a1, uint64_t a2,
// a1 = KDIR_ENTRY* buffer
// Look up a DIR_OBJECT using ObjectTypeDir (not ObjectTypeFile)
DIR_OBJECT* dir = (DIR_OBJECT*)HandleToObject((uint32_t)a0, ObjectTypeDir);
DIR_OBJECT* dir = (DIR_OBJECT*)KiHandleToObject((uint32_t)a0, ObjectTypeDir);
if (!dir) return STATUS_INVALID_PARAMETER;
KSTATUS s = IoReadDir(dir, (void*)(uintptr_t)a1);
ObDereferenceObject(&dir->Header); // Drop the ref taken by HandleToObject
ObDereferenceObject(&dir->Header); // Drop the ref taken by KiHandleToObject
return s;
}
@@ -986,7 +1237,7 @@ static uint64_t SyscallWaitThread(uint64_t a0, uint64_t a1, uint64_t a2,
return STATUS_NOT_FOUND;
}
KSTATUS s = KeWaitForEvent(exitEvent);
KSTATUS s = KeWaitForEvent(exitEvent, 0);
ObDereferenceObject(&exitEvent->Header);
return s;
}
@@ -1000,7 +1251,7 @@ static uint64_t SyscallGetProcessPid(uint64_t a0, uint64_t a1, uint64_t a2,
uint64_t a3, uint64_t a4, uint64_t a5) {
(void)a2; (void)a3; (void)a4; (void)a5;
PROCESS* proc = (PROCESS*)HandleToObject((uint32_t)a0, ObjectTypeProcess);
PROCESS* proc = (PROCESS*)KiHandleToObject((uint32_t)a0, ObjectTypeProcess);
uint32_t* pidOut = (uint32_t*)(uintptr_t)a1;
if (!proc) return STATUS_INVALID_HANDLE;
if (!pidOut) { ObDereferenceObject(&proc->Header); return STATUS_INVALID_PARAMETER; }
@@ -1048,9 +1299,9 @@ static uint64_t SyscallPublishQueue(uint64_t a0, uint64_t a1, uint64_t a2,
const char* name = (const char*)(uintptr_t)a0;
if (!name || !name[0]) return STATUS_INVALID_PARAMETER;
// Resolve the handle to a QUEUE object. HandleToObject bumps the
// Resolve the handle to a QUEUE object. KiHandleToObject bumps the
// refcount by one; we keep that ref in the registry slot.
QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a1, ObjectTypeQueue);
QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a1, ObjectTypeQueue);
if (!q) return STATUS_INVALID_HANDLE;
uintptr_t flags = KeAcquireSpinlock(&gNamedQueueLock);
@@ -1113,9 +1364,9 @@ static uint64_t SyscallOpenNamedQueue(uint64_t a0, uint64_t a1, uint64_t a2,
if (!q) return STATUS_NOT_FOUND;
// AllocHandleForCurrent consumes the ref we just bumped.
// KiAllocHandleForCurrent consumes the ref we just bumped.
// On failure it calls ObDereferenceObject internally.
return AllocHandleForCurrent(&q->Header, handleOut);
return KiAllocHandleForCurrent(&q->Header, handleOut);
}
// ---------------------------------------------------------------------------
@@ -1131,7 +1382,7 @@ static uint64_t SyscallQueueDuplicate(uint64_t a0, uint64_t a1, uint64_t a2,
(void)a3; (void)a4; (void)a5;
if (!a2) return STATUS_INVALID_PARAMETER;
QUEUE* q = (QUEUE*)HandleToObject((uint32_t)a0, ObjectTypeQueue);
QUEUE* q = (QUEUE*)KiHandleToObject((uint32_t)a0, ObjectTypeQueue);
if (!q) return STATUS_NOT_FOUND;
PROCESS* target = KeGetProcessById((uint32_t)a1);
@@ -1250,82 +1501,89 @@ typedef uint64_t (*SYSCALL_HANDLER)(uint64_t, uint64_t, uint64_t,
uint64_t, uint64_t, uint64_t);
static const SYSCALL_HANDLER gSyscallTable[SYS_MAX] = {
/* 00 SYS_EXIT_PROCESS */ SyscallExitProcess,
/* 01 SYS_EXIT_THREAD */ SyscallExitThread,
/* 02 SYS_CREATE_THREAD */ SyscallCreateThread,
/* 03 SYS_GET_CURRENT_TID */ SyscallGetCurrentTid,
/* 04 SYS_GET_CURRENT_PID */ SyscallGetCurrentPid,
/* 05 SYS_YIELD */ SyscallYield,
/* 06 SYS_SLEEP */ SyscallSleep,
/* 07 SYS_SET_THREAD_PRIORITY */ SyscallSetThreadPriority,
/* 08 SYS_ALLOC */ SyscallAlloc,
/* 09 SYS_FREE */ SyscallFree,
/* 10 SYS_REALLOC */ SyscallRealloc,
/* 11 SYS_MUTEX_CREATE */ SyscallMutexCreate,
/* 12 SYS_MUTEX_ACQUIRE */ SyscallMutexAcquire,
/* 13 SYS_MUTEX_TRY_ACQUIRE */ SyscallMutexTryAcquire,
/* 14 SYS_MUTEX_RELEASE */ SyscallMutexRelease,
/* 15 SYS_SEM_CREATE */ SyscallSemCreate,
/* 16 SYS_SEM_WAIT */ SyscallSemWait,
/* 17 SYS_SEM_TRY_WAIT */ SyscallSemTryWait,
/* 18 SYS_SEM_SIGNAL */ SyscallSemSignal,
/* 19 SYS_EVENT_CREATE */ SyscallEventCreate,
/* 20 SYS_EVENT_SIGNAL */ SyscallEventSignal,
/* 21 SYS_EVENT_CLEAR */ SyscallEventClear,
/* 22 SYS_EVENT_WAIT */ SyscallEventWait,
/* 23 SYS_QUEUE_CREATE */ SyscallQueueCreate,
/* 24 SYS_QUEUE_SEND */ SyscallQueueSend,
/* 25 SYS_QUEUE_RECV */ SyscallQueueRecv,
/* 26 SYS_CLOSE_HANDLE */ SyscallCloseHandle,
/* 27 SYS_OPEN_FILE */ SyscallOpenFile,
/* 28 SYS_READ_FILE */ SyscallReadFile,
/* 29 SYS_WRITE_FILE */ SyscallWriteFile,
/* 30 SYS_CREATE_FILE */ SyscallCreateFile,
/* 31 SYS_DELETE_FILE */ SyscallDeleteFile,
/* 32 SYS_TRUNCATE_FILE */ SyscallTruncateFile,
/* 33 SYS_GET_FILE_POS */ SyscallGetFilePos,
/* 34 SYS_SEEK_FILE */ SyscallSeekFile,
/* 35 SYS_GET_FILE_SIZE */ SyscallGetFileSize,
/* 36 SYS_SCREEN_BLIT */ SyscallScreenBlit,
/* 37 SYS_SCREEN_CLEAR */ SyscallScreenClear,
/* 38 SYS_SCREEN_GET_SIZE */ SyscallScreenGetSize,
/* 39 SYS_SCREEN_PUT_CHAR */ SyscallScreenPutChar,
/* 40 SYS_KB_POLL */ SyscallKbPoll,
/* 41 SYS_KB_IS_KEY_DOWN */ SyscallKbIsKeyDown,
/* 42 SYS_KB_READ_CHAR */ SyscallKbReadChar,
/* 43 SYS_KB_READ_SCANCODE */ SyscallKbReadScancode,
/* 44 SYS_KB_GET_MODIFIERS */ SyscallKbGetModifiers,
/* 45 SYS_MOUSE_POLL */ SyscallMousePoll,
/* 46 SYS_MOUSE_READ */ SyscallMouseRead,
/* 47 SYS_MOUSE_GET_POS */ SyscallMouseGetPos,
/* 48 SYS_GET_TICK */ SyscallGetTick,
/* 49 SYS_DEBUG_PRINT */ SyscallDebugPrint,
/* 50 SYS_CREATE_PROCESS */ SyscallCreateProcess,
/* 51 SYS_WAIT_FOR_PROCESS */ SyscallWaitForProcess,
/* 52 SYS_GET_PROCESS_EXIT_CODE */ SyscallGetProcessExitCode,
/* 53 SYS_WRITE_CONSOLE */ SyscallWriteConsole,
/* 54 SYS_OPEN_DIR */ SyscallOpenDir,
/* 55 SYS_READ_DIR */ SyscallReadDir,
/* 56 SYS_SET_CWD */ SyscallSetCwd,
/* 57 SYS_GET_CWD */ SyscallGetCwd,
/* 58 SYS_WAIT_THREAD */ SyscallWaitThread,
/* 59 SYS_GET_PROCESS_PID */ SyscallGetProcessPid,
/* 60 SYS_PUBLISH_QUEUE */ SyscallPublishQueue,
/* 61 SYS_OPEN_NAMED_QUEUE */ SyscallOpenNamedQueue,
/* 62 SYS_QUEUE_DUPLICATE */ SyscallQueueDuplicate,
/* 63 SYS_NAMED_QUEUE_OWNER_PID */ SyscallNamedQueueOwnerPid,
/* 64 SYS_KILL_PROCESS */ SyscallKillProcess,
/* 65 SYS_GET_PROCESS_LIST */ SyscallGetProcessList,
/* 66 SYS_RENAME_FILE */ SyscallRenameFile,
/* 67 SYS_CREATE_DIR */ SyscallCreateDir,
/* 68 SYS_REMOVE_DIR */ SyscallRemoveDir,
/* 00 SYS_EXIT_PROCESS */ SyscallExitProcess,
/* 01 SYS_EXIT_THREAD */ SyscallExitThread,
/* 02 SYS_CREATE_PROCESS */ SyscallCreateProcess,
/* 03 SYS_WAIT_FOR_PROCESS */ SyscallWaitForProcess,
/* 04 SYS_GET_PROCESS_EXIT_CODE */ SyscallGetProcessExitCode,
/* 05 SYS_KILL_PROCESS */ SyscallKillProcess,
/* 06 SYS_GET_PROCESS_LIST */ SyscallGetProcessList,
/* 07 SYS_GET_PROCESS_PID */ SyscallGetProcessPid,
/* 08 SYS_CREATE_THREAD */ SyscallCreateThread,
/* 09 SYS_WAIT_THREAD */ SyscallWaitThread,
/* 10 SYS_GET_CURRENT_TID */ SyscallGetCurrentTid,
/* 11 SYS_GET_CURRENT_PID */ SyscallGetCurrentPid,
/* 12 SYS_SET_THREAD_PRIORITY */ SyscallSetThreadPriority,
/* 13 SYS_YIELD */ SyscallYield,
/* 14 SYS_SLEEP */ SyscallSleep,
/* 15 SYS_SET_USER_APC_DISPATCHER */ SyscallSetUserApcDispatcher,
/* 16 SYS_APC_RETURN */ SyscallApcReturn,
/* 17 SYS_ALLOC */ SyscallAlloc,
/* 18 SYS_FREE */ SyscallFree,
/* 19 SYS_REALLOC */ SyscallRealloc,
/* 20 SYS_CLOSE_HANDLE */ SyscallCloseHandle,
/* 21 SYS_MUTEX_CREATE */ SyscallMutexCreate,
/* 22 SYS_MUTEX_ACQUIRE */ SyscallMutexAcquire,
/* 23 SYS_MUTEX_TRY_ACQUIRE */ SyscallMutexTryAcquire,
/* 24 SYS_MUTEX_RELEASE */ SyscallMutexRelease,
/* 25 SYS_SEM_CREATE */ SyscallSemCreate,
/* 26 SYS_SEM_WAIT */ SyscallSemWait,
/* 27 SYS_SEM_TRY_WAIT */ SyscallSemTryWait,
/* 28 SYS_SEM_SIGNAL */ SyscallSemSignal,
/* 29 SYS_EVENT_CREATE */ SyscallEventCreate,
/* 30 SYS_EVENT_SIGNAL */ SyscallEventSignal,
/* 31 SYS_EVENT_CLEAR */ SyscallEventClear,
/* 32 SYS_EVENT_WAIT */ SyscallEventWait,
/* 34 SYS_QUEUE_CREATE */ SyscallQueueCreate,
/* 35 SYS_QUEUE_SEND */ SyscallQueueSend,
/* 36 SYS_QUEUE_RECV */ SyscallQueueRecv,
/* 37 SYS_PUBLISH_QUEUE */ SyscallPublishQueue,
/* 38 SYS_OPEN_NAMED_QUEUE */ SyscallOpenNamedQueue,
/* 39 SYS_QUEUE_DUPLICATE */ SyscallQueueDuplicate,
/* 40 SYS_NAMED_QUEUE_OWNER_PID */ SyscallNamedQueueOwnerPid,
/* 41 SYS_OPEN_FILE */ SyscallOpenFile,
/* 42 SYS_CREATE_FILE */ SyscallCreateFile,
/* 43 SYS_READ_FILE */ SyscallReadFile,
/* 44 SYS_WRITE_FILE */ SyscallWriteFile,
/* 45 SYS_READ_FILE_ASYNC */ SyscallReadFileAsync,
/* 46 SYS_WRITE_FILE_ASYNC */ SyscallWriteFileAsync,
/* 47 SYS_CANCEL_IO */ SyscallCancelIo,
/* 48 SYS_SEEK_FILE */ SyscallSeekFile,
/* 49 SYS_GET_FILE_POS */ SyscallGetFilePos,
/* 50 SYS_GET_FILE_SIZE */ SyscallGetFileSize,
/* 51 SYS_TRUNCATE_FILE */ SyscallTruncateFile,
/* 52 SYS_DELETE_FILE */ SyscallDeleteFile,
/* 53 SYS_RENAME_FILE */ SyscallRenameFile,
/* 54 SYS_OPEN_DIR */ SyscallOpenDir,
/* 55 SYS_READ_DIR */ SyscallReadDir,
/* 56 SYS_CREATE_DIR */ SyscallCreateDir,
/* 57 SYS_REMOVE_DIR */ SyscallRemoveDir,
/* 58 SYS_SET_CWD */ SyscallSetCwd,
/* 59 SYS_GET_CWD */ SyscallGetCwd,
/* 60 SYS_WRITE_CONSOLE */ SyscallWriteConsole,
/* 61 SYS_SCREEN_BLIT */ SyscallScreenBlit,
/* 62 SYS_SCREEN_CLEAR */ SyscallScreenClear,
/* 63 SYS_SCREEN_GET_SIZE */ SyscallScreenGetSize,
/* 64 SYS_SCREEN_PUT_CHAR */ SyscallScreenPutChar,
/* 65 SYS_KB_POLL */ SyscallKbPoll,
/* 66 SYS_KB_IS_KEY_DOWN */ SyscallKbIsKeyDown,
/* 67 SYS_KB_READ_CHAR */ SyscallKbReadChar,
/* 68 SYS_KB_READ_SCANCODE */ SyscallKbReadScancode,
/* 69 SYS_KB_GET_MODIFIERS */ SyscallKbGetModifiers,
/* 70 SYS_MOUSE_POLL */ SyscallMousePoll,
/* 71 SYS_MOUSE_READ */ SyscallMouseRead,
/* 72 SYS_MOUSE_GET_POS */ SyscallMouseGetPos,
/* 73 SYS_GET_TICK */ SyscallGetTick,
/* 74 SYS_DEBUG_PRINT */ SyscallDebugPrint,
/* 75 SYS_QUERY_TRAP_FRAME */ SyscallQueryTrapFrame,
};
// ---------------------------------------------------------------------------
// Entry point - called from ke_syscall.asm after argument re-ordering
// ---------------------------------------------------------------------------
KSTATUS KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t KeSyscallDispatch(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) {
KiValidateSyscallTrapFrame(nr, a0, a1, a2, a3, a4, a5);
if (nr >= SYS_MAX || !gSyscallTable[nr]) return STATUS_NOT_SUPPORTED;
return (KSTATUS)gSyscallTable[nr](a0, a1, a2, a3, a4, a5);
return gSyscallTable[nr](a0, a1, a2, a3, a4, a5);
}
code/kernel/ke_thread.c
+158 -15
View File
@@ -1,9 +1,12 @@
// ke_thread.c - thread model
#include <string.h>
#include <stddef.h>
#include <intrin.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ke_irql.h>
#include <ke_event.h>
#include <ke_apc.h>
#include <ke_userenv.h>
#include <mm_virt.h>
#include <ob_object.h>
@@ -11,12 +14,26 @@
#define FPU_FCW_DEFAULT 0x037Fu
#define FPU_MXCSR_DEFAULT 0x1F80u
// These offsets are consumed directly by ke_switch.asm. Keep the asserts
// here so future THREAD layout edits fail at build time instead of corrupting
// context switches at runtime.
#define STATIC_ASSERT_OFFSET(type, field, expected) \
_Static_assert(offsetof(type, field) == (expected), \
#type "." #field " offset must match assembly")
STATIC_ASSERT_OFFSET(THREAD, SavedContext, 88);
STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 104);
STATIC_ASSERT_OFFSET(THREAD, Teb, 112);
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 168);
static uint32_t gNextThreadId = 1;
extern void KeThreadEntry(void);
static void ThreadDestroy(OBJECT* obj) {
static void KiThreadDestroy(OBJECT* obj) {
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
KiRundownApcState(thread);
if (thread->ExitEvent) {
ObDereferenceObject(&thread->ExitEvent->Header);
thread->ExitEvent = (EVENT*)0;
@@ -37,13 +54,13 @@ static void ThreadDestroy(OBJECT* obj) {
}
if (thread->Teb && thread->TebSize) {
MmFreeUserMemory(thread->Teb, thread->TebSize);
MmFreeUserMemory(thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
}
if (thread->UserStackBase) {
MmFreeUserMemory(thread->UserStackBase, USTACK_SIZE);
MmFreeUserMemory(thread->UserStackBase);
thread->UserStackBase = (void *)0;
}
}
@@ -56,13 +73,18 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!thread) return STATUS_NO_MEMORY;
thread->Header.Destroy = ThreadDestroy;
thread->Header.Destroy = KiThreadDestroy;
thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
thread->State = ThreadReady;
thread->Priority = priority;
thread->QuantaConsumed = 0;
thread->WaitObject = (OBJECT *)0;
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SchedLockFlags = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
void* kstack = MmAllocatePool(KSTACK_SIZE);
if (!kstack) {
@@ -88,7 +110,7 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemory(sizeof(SYS_TEB));
if (!teb) {
MmFreeUserMemory(ustack, USTACK_SIZE);
MmFreeUserMemory(ustack);
thread->UserStackBase = (void*)0;
MmFreePool(kstack);
thread->KernelStackBase = (void*)0;
@@ -132,10 +154,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
EVENT* exitEvent;
KSTATUS s = KeEventCreate(EventManualReset, 0, &exitEvent);
if (!KSUCCESS(s)) {
if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb, thread->TebSize);
if (thread->Teb && thread->TebSize) MmFreeUserMemory(thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
MmFreeUserMemory(ustack, USTACK_SIZE);
MmFreeUserMemory(ustack);
thread->UserStackBase = (void*)0;
MmFreePool(kstack);
thread->KernelStackBase = (void*)0;
@@ -161,6 +183,8 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
(void)code;
if (!thread) return;
KiRundownApcState(thread);
// Wake any pthread_join waiters before the thread is marked dead and
// potentially freed. Manual-reset means all concurrent waiters unblock.
if (thread->ExitEvent)
@@ -197,13 +221,18 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!thread) return STATUS_NO_MEMORY;
thread->Header.Destroy = ThreadDestroy;
thread->Header.Destroy = KiThreadDestroy;
thread->ThreadId = _InterlockedExchangeAdd((volatile long*)&gNextThreadId, 1);
thread->State = ThreadReady;
thread->Priority = priority;
thread->QuantaConsumed = 0;
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SchedLockFlags = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
thread->UserStackBase = (void*)0; // no user stack
thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx
thread->TebSize = 0;
@@ -245,16 +274,101 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
return STATUS_SUCCESS;
}
void KeBlockThread(THREAD* thread, OBJECT* waitObject) {
thread->State = ThreadWaiting;
thread->WaitObject = waitObject;
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
// Called with the synchronization object's spinlock held, so the caller is
// normally at DISPATCH_LEVEL here. This function only marks intent to
// block; the actual scheduling point is KeCommitBlockThread at PASSIVE.
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
KeYield();
thread->State = ThreadWaiting;
thread->WaitObject = waitObject;
thread->WaitStatus = STATUS_PENDING;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = alertable ? 1 : 0;
}
void KeUnblockThread(THREAD* thread) {
thread->WaitObject = (OBJECT *)0;
thread->State = ThreadReady;
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
thread->WaitListHead = waitListHead;
}
static void KiFinishWait(THREAD* thread, KSTATUS status) {
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = status;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
}
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
KeAssertIrql(PASSIVE_LEVEL);
if (!thread) return STATUS_INVALID_PARAMETER;
if (alertable)
thread->AlertableWait = 1;
// A signal/APC may have arrived after the waiter was placed on the
// object's wait list but before this thread reached the scheduling point.
// In that case KeUnblockThreadWithStatus completed the wait in-place and
// there is nothing left to block on.
if (thread->State != ThreadWaiting) {
KSTATUS status = thread->WaitStatus;
thread->AlertableWait = 0;
return status;
}
// If a user APC was already pending before the thread fully blocked,
// consume the alertable wait immediately instead of going to sleep.
if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
if (thread->WaitListHead)
ListRemove(&thread->SchedulerLink);
KiFinishWait(thread, STATUS_USER_APC);
thread->State = ThreadRunning;
return STATUS_USER_APC;
}
KeYield();
// Kernel APCs are delivered before an alertable wait returns
// STATUS_KERNEL_APC to its caller.
KiDeliverKernelApcs();
thread->AlertableWait = 0;
return thread->WaitStatus;
}
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
KePrepareBlockThread(thread, waitObject, alertable);
return KeCommitBlockThread(thread, alertable);
}
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
// If the selected waiter is still the currently executing thread, the
// signal landed in the prepare->commit window. Complete the wait in-place
// instead of inserting the running thread into the ready queue.
if (thread == KeGetCurrentThread()) {
KiFinishWait(thread, status);
if (thread->State == ThreadWaiting)
thread->State = ThreadRunning;
return;
}
// The same object can be signaled after a timeout/cancel path exists, or a
// buggy caller can try to wake a non-waiting thread. Do not enqueue such a
// thread twice.
if (thread->State != ThreadWaiting) {
thread->WaitStatus = status;
return;
}
KiFinishWait(thread, status);
thread->State = ThreadReady;
if (thread->Priority < PriorityHigh) {
thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1);
@@ -264,6 +378,35 @@ void KeUnblockThread(THREAD* thread) {
KeReadyThread(thread);
}
void KeUnblockThread(THREAD* thread) {
KeUnblockThreadWithStatus(thread, STATUS_SUCCESS);
}
static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return STATUS_INVALID_PARAMETER;
if (thread->State != ThreadWaiting || !thread->AlertableWait)
return STATUS_PENDING;
if (thread->WaitListHead)
ListRemove(&thread->SchedulerLink);
KeUnblockThreadWithStatus(thread, status);
return STATUS_SUCCESS;
}
KSTATUS KeAlertThread(THREAD* thread) {
return KiAlertThreadWithStatus(thread, STATUS_ALERTED);
}
void KiWakeThreadForKernelApc(THREAD* thread) {
(void)KiAlertThreadWithStatus(thread, STATUS_KERNEL_APC);
}
void KiWakeThreadForUserApc(THREAD* thread) {
(void)KiAlertThreadWithStatus(thread, STATUS_USER_APC);
}
void KeInitThreadFpuContext(THREAD* thread)
{
uintptr_t raw =
code/kernel/ke_thread.h
+51 -1
View File
@@ -6,6 +6,8 @@
#include <ke_list.h>
#include <ke_process.h>
#include <ke_event.h>
#include <ke_spinlock.h>
#include <ke_trap.h>
#define KSTACK_SIZE (64 * 1024) /* 64 KB kernel stack */
#define USTACK_SIZE (512 * 1024) /* 512 KB user stack */
@@ -77,6 +79,36 @@ typedef struct THREAD {
void* FpuContext;
uint8_t FpuStorage[528];
// Set by wait primitives before blocking and completed by the object
// that wakes the thread. Keep this field after the assembly-visible
// context-switch fields: ke_switch.asm hard-codes offsets through
// FpuContext.
KSTATUS WaitStatus;
LIST_ENTRY* WaitListHead;
uint8_t AlertableWait;
// Kernel APC state. APCs are queued to a target thread and delivered
// when that thread reaches PASSIVE_LEVEL with APC delivery enabled.
LIST_ENTRY ApcQueue;
SPINLOCK ApcLock;
volatile long KernelApcPending;
int32_t KernelApcDisable;
uint8_t ApcDeliveryActive;
// Non-NULL while executing on a syscall/interrupt/fault entry frame.
// The pointed-to frame lives on this thread's kernel stack.
KTRAP_FRAME* TrapFrame;
// User APC state. User APCs are queued by kernel completions, but are
// delivered only on an alertable return to user mode through the registered
// user-mode dispatcher stub.
LIST_ENTRY UserApcQueue;
SPINLOCK UserApcLock;
volatile long UserApcPending;
uintptr_t UserApcDispatcher;
uint8_t UserApcDeliverable;
uint8_t UserApcDeliveryActive;
} THREAD;
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
@@ -93,8 +125,26 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
__declspec(dllexport) void KeExitThread(THREAD* thread, int code);
// Block / unblock primitives used by sync objects.
void KeBlockThread(THREAD* thread, OBJECT* waitObject);
//
// Correct wait pattern:
// object lock held
// KePrepareBlockThread(self, object, alertable);
// insert self into object's wait list
// release object lock
// KeCommitBlockThread(self, alertable);
//
// The split form prevents the classic lost wakeup where an ISR/DPC signals
// the object after the waiter is queued but before the waiter marks itself
// waiting. Alertable waits are explicit at every call site.
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead);
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status);
void KeUnblockThread(THREAD* thread);
KSTATUS KeAlertThread(THREAD* thread);
void KiWakeThreadForKernelApc(THREAD* thread);
void KiWakeThreadForUserApc(THREAD* thread);
// Assembly stub - implemented in ke_switch.asm.
void KeSwitchContext(THREAD* from, THREAD* to);
code/kernel/ke_trap.c
+148
View File
@@ -0,0 +1,148 @@
// ke_trap.c - current-thread trap-frame tracking and validation
#include <stddef.h>
#include <string.h>
#include <ke_trap.h>
#include <ke_thread.h>
#include <ke_sched.h>
#include <ke_debug.h>
#include <ke_apc.h>
#define STATIC_ASSERT_OFFSET(type, field, expected) \
_Static_assert(offsetof(type, field) == (expected), \
#type "." #field " offset must match assembly")
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Magic, 0);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Kind, 8);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Vector, 12);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousMode, 16);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rax, 24);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rcx, 32);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rdx, 40);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rbx, 48);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rbp, 56);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rsi, 64);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rdi, 72);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R8, 80);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R9, 88);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R10, 96);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R11, 104);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R12, 112);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R13, 120);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R14, 128);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, R15, 136);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rip, 144);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Cs, 152);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rflags, 160);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Rsp, 168);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Ss, 176);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, RipSlot, 184);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, CsSlot, 192);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, RflagsSlot, 200);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, RspSlot, 208);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, SsSlot, 216);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousFrame, 224);
STATIC_ASSERT_OFFSET(KTRAP_FRAME, Reserved2, 232);
_Static_assert(sizeof(KTRAP_FRAME) == 240, "KTRAP_FRAME size must match ke_syscall.asm");
int KeTrapFrameFromUser(const KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC) return 0;
return (frame->Cs & 3u) == 3u;
}
KTRAP_FRAME* KeGetCurrentTrapFrame(void) {
THREAD* t = KeGetCurrentThread();
if (!t) return (KTRAP_FRAME*)0;
return t->TrapFrame;
}
void KiEnterTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic("KiEnterTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame);
THREAD* t = KeGetCurrentThread();
if (!t) return;
frame->PreviousMode = KeTrapFrameFromUser(frame) ? KTRAP_MODE_USER : KTRAP_MODE_KERNEL;
frame->PreviousFrame = t->TrapFrame;
t->TrapFrame = frame;
}
void KiExitTrapFrame(KTRAP_FRAME* frame) {
if (!frame || frame->Magic != KTRAP_FRAME_MAGIC)
KePanic("KiExitTrapFrame: invalid trap frame 0x%llX\n", (uint64_t)frame);
// Give alertable user-mode returns a chance to redirect through the
// registered user APC dispatcher before committing IRET slots.
KiCheckForUserApcDelivery(frame);
// Commit editable return state. Kernel-mode interrupts have no RSP/SS
// hardware slots, so those pointers are NULL in that case.
if (frame->RipSlot) *frame->RipSlot = frame->Rip;
if (frame->CsSlot) *frame->CsSlot = frame->Cs;
if (frame->RflagsSlot) *frame->RflagsSlot = frame->Rflags;
if (frame->RspSlot) *frame->RspSlot = frame->Rsp;
if (frame->SsSlot) *frame->SsSlot = frame->Ss;
THREAD* t = KeGetCurrentThread();
if (t) {
if (t->TrapFrame != frame)
KePanic("KiExitTrapFrame: trap-frame stack mismatch top=0x%llX frame=0x%llX\n",
(uint64_t)t->TrapFrame, (uint64_t)frame);
t->TrapFrame = frame->PreviousFrame;
}
}
void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4,
uint64_t a5) {
KTRAP_FRAME* f = KeGetCurrentTrapFrame();
if (!f)
KePanic("syscall trap validation: no current trap frame for nr=%llu\n", nr);
if (f->Magic != KTRAP_FRAME_MAGIC)
KePanic("syscall trap validation: bad magic 0x%llX\n", f->Magic);
if (f->Kind != KTRAP_KIND_SYSCALL)
KePanic("syscall trap validation: kind=%u, expected syscall\n", f->Kind);
if (!KeTrapFrameFromUser(f) || f->PreviousMode != KTRAP_MODE_USER)
KePanic("syscall trap validation: syscall did not come from user cs=0x%llX mode=%u\n",
f->Cs, f->PreviousMode);
if (f->Rax != nr || f->Rcx != a0 || f->Rdx != a1 || f->R8 != a2 ||
f->R9 != a3 || f->R10 != a4 || f->R11 != a5) {
KePanic("syscall trap validation: arg mismatch nr=%llu tf=%llu/%llX/%llX/%llX/%llX/%llX/%llX call=%llu/%llX/%llX/%llX/%llX/%llX/%llX\n",
nr,
f->Rax, f->Rcx, f->Rdx, f->R8, f->R9, f->R10, f->R11,
nr, a0, a1, a2, a3, a4, a5);
}
}
KSTATUS KeQueryCurrentTrapFrame(SYS_TRAP_FRAME_INFO_K* out) {
if (!out) return STATUS_INVALID_PARAMETER;
KTRAP_FRAME* f = KeGetCurrentTrapFrame();
if (!f || f->Magic != KTRAP_FRAME_MAGIC)
return STATUS_FAILURE;
uint32_t size = out->Size;
if (size < sizeof(SYS_TRAP_FRAME_INFO_K)) {
out->Size = sizeof(SYS_TRAP_FRAME_INFO_K);
return STATUS_BUFFER_TOO_SMALL;
}
memset(out, 0, sizeof(*out));
out->Size = sizeof(*out);
out->Kind = f->Kind;
out->Vector = f->Vector;
out->PreviousMode = f->PreviousMode;
out->Rip = f->Rip;
out->Cs = f->Cs;
out->Rflags = f->Rflags;
out->Rsp = f->Rsp;
out->Ss = f->Ss;
out->Rax = f->Rax;
out->Rcx = f->Rcx;
out->Rdx = f->Rdx;
out->R8 = f->R8;
out->R9 = f->R9;
out->R10 = f->R10;
out->R11 = f->R11;
return STATUS_SUCCESS;
}
code/kernel/ke_trap.h
+88
View File
@@ -0,0 +1,88 @@
#pragma once
// ke_trap.h - trap-frame metadata for syscall/interrupt return paths
//
// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally
// an editable return frame: future user APC delivery can rewrite Rip/Rsp in the
// frame, and KiExitTrapFrame will commit those values back to the hardware IRET
// slots before returning to user mode.
#include <stdint.h>
#include <ke_status.h>
#define KTRAP_FRAME_MAGIC 0x4650545241544B31ULL
#define KTRAP_KIND_NONE 0u
#define KTRAP_KIND_SYSCALL 1u
#define KTRAP_KIND_INTERRUPT 2u
#define KTRAP_MODE_KERNEL 0u
#define KTRAP_MODE_USER 1u
typedef struct KTRAP_FRAME {
uint64_t Magic;
uint32_t Kind;
uint32_t Vector;
uint32_t PreviousMode;
uint32_t Reserved;
uint64_t Rax;
uint64_t Rcx;
uint64_t Rdx;
uint64_t Rbx;
uint64_t Rbp;
uint64_t Rsi;
uint64_t Rdi;
uint64_t R8;
uint64_t R9;
uint64_t R10;
uint64_t R11;
uint64_t R12;
uint64_t R13;
uint64_t R14;
uint64_t R15;
uint64_t Rip;
uint64_t Cs;
uint64_t Rflags;
uint64_t Rsp;
uint64_t Ss;
uint64_t* RipSlot;
uint64_t* CsSlot;
uint64_t* RflagsSlot;
uint64_t* RspSlot;
uint64_t* SsSlot;
struct KTRAP_FRAME* PreviousFrame;
uint64_t Reserved2;
} KTRAP_FRAME;
typedef struct SYS_TRAP_FRAME_INFO_K {
uint32_t Size;
uint32_t Kind;
uint32_t Vector;
uint32_t PreviousMode;
uint64_t Rip;
uint64_t Cs;
uint64_t Rflags;
uint64_t Rsp;
uint64_t Ss;
uint64_t Rax;
uint64_t Rcx;
uint64_t Rdx;
uint64_t R8;
uint64_t R9;
uint64_t R10;
uint64_t R11;
} SYS_TRAP_FRAME_INFO_K;
void KiEnterTrapFrame(KTRAP_FRAME* frame);
void KiExitTrapFrame(KTRAP_FRAME* frame);
KTRAP_FRAME* KeGetCurrentTrapFrame(void);
int KeTrapFrameFromUser(const KTRAP_FRAME* frame);
void KiValidateSyscallTrapFrame(uint64_t nr, uint64_t a0, uint64_t a1,
uint64_t a2, uint64_t a3, uint64_t a4,
uint64_t a5);
KSTATUS KeQueryCurrentTrapFrame(SYS_TRAP_FRAME_INFO_K* out);
code/kernel/ke_userenv.h
+1 -1
View File
@@ -8,7 +8,7 @@
#define SYS_PEB_MAGIC 0x30424550u /* "PEB0" */
#define SYS_TEB_MAGIC 0x30424554u /* "TEB0" */
#define SYS_ENV_VERSION 1u
#define SYS_CWD_MAX 512u
#define SYS_CWD_MAX 520u
typedef struct SYS_PEB {
uint32_t Magic;
code/kernel/kmain.c
+95 -42
View File
@@ -11,12 +11,14 @@
#include <ke_sched.h>
#include <ke_idt.h>
#include <ke_irq.h>
#include <ke_dpc.h>
#include <ke_syscall.h>
#include <ke_debug.h>
#include <ke_util.h>
#include <ldr_image.h>
#include <hal_api.h>
#include <io_file.h>
#include <ke_selftest.h>
#include <ini_parser.h>
typedef struct {
@@ -30,7 +32,49 @@ typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *);
// threading a parameter through every call chain.
static const HAL_API* gKmainHal;
static int PageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
// Kernel-owned DPC wrappers for HAL input devices. The HAL keeps the tiny
// hardware-specific ISR fast and exposes a bottom-half entry point; the kernel
// owns the actual DPC objects and scheduling policy.
static KDPC gKeyboardDpc;
static KDPC gMouseDpc;
static void KeyboardDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
if (gKmainHal && gKmainHal->Keyboard.Dpc)
gKmainHal->Keyboard.Dpc();
}
static void KiMouseDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
(void)dpc;
(void)context;
(void)systemArgument1;
(void)systemArgument2;
if (gKmainHal && gKmainHal->Mouse.Dpc)
gKmainHal->Mouse.Dpc();
}
static void KeyboardIrqWrapper(void) {
if (gKmainHal && gKmainHal->Keyboard.Isr)
gKmainHal->Keyboard.Isr();
if (gKmainHal && gKmainHal->Keyboard.Dpc)
KeInsertQueueDpc(&gKeyboardDpc, (void*)0, (void*)0);
}
static void KiMouseIrqWrapper(void) {
if (gKmainHal && gKmainHal->Mouse.Isr)
gKmainHal->Mouse.Isr();
if (gKmainHal && gKmainHal->Mouse.Dpc)
KeInsertQueueDpc(&gMouseDpc, (void*)0, (void*)0);
}
static int KiPageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS page) {
for (uint32_t i = 0; i < bootInfo->MemoryMap.RegionCount; i++) {
const MEMORY_REGION* r = &bootInfo->MemoryMap.Regions[i];
if (r->Type != MemoryBootloader) continue;
@@ -44,7 +88,7 @@ static int PageInBootloaderRegion(const BOOT_INFO* bootInfo, PHYSICAL_ADDRESS pa
return 0;
}
static void ReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
// Reclaim only the raw buffers backing boot-loaded .sys modules.
// The kernel/HAL images, page tables, and bootloader code remain live.
for (uint32_t i = 2; i < bootInfo->ModuleCount; i++) {
@@ -55,7 +99,7 @@ static void ReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
uintptr_t end = (uintptr_t)((mod->PhysBase + mod->ImageSize + PAGE_SIZE - 1) & PAGE_MASK);
for (uintptr_t page = base; page < end; page += PAGE_SIZE) {
if (!PageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue;
if (!KiPageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue;
MmFreeFrame((PHYSICAL_ADDRESS)page);
}
}
@@ -64,7 +108,7 @@ static void ReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
// ---------------------------------------------------------------------------
// Phase Helper: Resolve Imports for core modules
// ---------------------------------------------------------------------------
static void ResolveSystemImports(BOOT_INFO* bootInfo) {
static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
// Only resolve imports for kernel.exe and hal.dll (typically indices 0 and 1).
// The bootloader maps these, but drivers are now passed as raw physical files.
uint32_t coreModuleCount = (bootInfo->ModuleCount > 2) ? 2 : bootInfo->ModuleCount;
@@ -79,7 +123,7 @@ static void ResolveSystemImports(BOOT_INFO* bootInfo) {
// ---------------------------------------------------------------------------
// Phase Helper: Initialize dynamically loaded .sys drivers
// ---------------------------------------------------------------------------
static void DriversIniCb(const char* section, const char* key,
static void KiDriversIniCb(const char* section, const char* key,
const char* val, void* ud) {
(void)val;
if (!strcmp(section, "runtime")) {
@@ -93,8 +137,8 @@ static void DriversIniCb(const char* section, const char* key,
}
}
static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
DbgPrint("InitializeDrivers: Initializing drivers...\n");
static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
DbgPrint("KiInitializeDrivers: Initializing drivers...\n");
// boot drivers (provided by the bootloader)
for (uint32_t i = 0; i < bootInfo->ModuleCount; i++) {
@@ -104,7 +148,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
// or have no physical data (invalid)
if (mod->VirtBase != 0 || mod->PhysBase == 0) continue;
DbgPrint("InitializeDrivers: Loading boot driver %s\n", mod->Name);
DbgPrint("KiInitializeDrivers: Loading boot driver %s\n", mod->Name);
// Calculate virtual address of the raw file in the physical window
void* rawPeBuffer = (void*)(bootInfo->PhysicalMapBase + mod->PhysBase);
@@ -145,7 +189,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
s = IoOpenFile("drivers\\drivers.ini", &f);
if (s != STATUS_SUCCESS) {
KePanic("InitializeDrivers: Failed to open drivers.ini (status %d)\n", s);
KePanic("KiInitializeDrivers: Failed to open drivers.ini (status %d)\n", s);
}
uint64_t size = IoGetFileSize(f) + 1;
@@ -153,14 +197,14 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
size_t got;
if (!KSUCCESS(IoReadFile(f, mem, size, &got))) {
IoCloseFile(f);
KePanic("InitializeDrivers: Failed to read drivers.ini\n");
KePanic("KiInitializeDrivers: Failed to read drivers.ini\n");
}
mem[got] = '\0';
IoCloseFile(f);
char** runtimeKeys = NULL;
IniParse(mem, DriversIniCb, &runtimeKeys);
IniParse(mem, KiDriversIniCb, &runtimeKeys);
for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]);
@@ -179,7 +223,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
s = IoOpenFile(path, &f);
if (s != STATUS_SUCCESS) {
DbgPrint("InitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path);
DbgPrint("KiInitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path);
continue; // Skip this driver but keep going with others
}
@@ -188,7 +232,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
uint64_t drvSize = IoGetFileSize(f);
void* drvMem = MmAllocatePool(drvSize);
if (!KSUCCESS(IoReadFile(f, drvMem, drvSize, &got))) {
DbgPrint("InitializeDrivers: Failed to read runtime driver: %s\n", runtimeKeys[i]);
DbgPrint("KiInitializeDrivers: Failed to read runtime driver: %s\n", runtimeKeys[i]);
IoCloseFile(f);
continue;
}
@@ -196,14 +240,14 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
uintptr_t mappedBase = 0;
if (!KSUCCESS(LdrLoadImage(drvMem, drvSize, &mappedBase, false))) { // Load into kernel space
DbgPrint("InitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]);
DbgPrint("KiInitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]);
MmFreePool(drvMem);
continue;
}
MmFreePool(drvMem);
if (!KSUCCESS(LdrResolveImports(mappedBase))) {
DbgPrint("InitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
continue;
}
@@ -211,7 +255,7 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
KSTATUS ds = entry(hal);
if (!KSUCCESS(ds)) {
DbgPrint("InitializeDrivers: Driver %s failed to init!\n", runtimeKeys[i]);
DbgPrint("KiInitializeDrivers: Driver %s failed to init!\n", runtimeKeys[i]);
}
}
@@ -235,23 +279,23 @@ static void InitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
// Returns the new PROCESS* with one reference owned by the caller.
// Returns NULL on failure.
// ---------------------------------------------------------------------------
static PROCESS* LaunchUserProcess(const char* filePath, const char* args) {
static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
KSTATUS s;
FILE_OBJECT* f;
// Read the executable file into a raw buffer.
s = IoOpenFile(filePath, &f);
DbgPrint("LaunchUserProcess: IoOpenFile: %d\n", s);
DbgPrint("KiLaunchUserProcess: IoOpenFile: %d\n", s);
if (s != STATUS_SUCCESS) {
DbgPrint("Failed to load file: \"%s\"!\n", filePath);
return 0;
}
uint64_t size = IoGetFileSize(f);
DbgPrint("LaunchUserProcess: IoGetFileSize: %llu\n", size);
DbgPrint("KiLaunchUserProcess: IoGetFileSize: %llu\n", size);
void* mem = MmAllocatePool(size);
DbgPrint("LaunchUserProcess: MmAllocatePool: 0x%llX\n", mem);
DbgPrint("KiLaunchUserProcess: MmAllocatePool: 0x%llX\n", mem);
if (!mem) {
IoCloseFile(f);
return 0;
@@ -259,7 +303,7 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) {
size_t got = 0;
s = IoReadFile(f, mem, size, &got);
DbgPrint("LaunchUserProcess: IoReadFile: %d\n", s);
DbgPrint("KiLaunchUserProcess: IoReadFile: %d\n", s);
IoCloseFile(f);
if (!KSUCCESS(s)) {
MmFreePool(mem);
@@ -280,7 +324,7 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) {
s = KeSetupUserProcessEnvironment(proc, filePath, args, proc->CurrentDirectory);
if (!KSUCCESS(s)) {
DbgPrint("LaunchUserProcess: Failed to build user process environment. Status: %d\n", s);
DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s);
MmFreePool(mem);
ObDereferenceObject(&proc->Header);
return 0;
@@ -291,15 +335,15 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) {
s = LdrLoadImage(mem, size, &imageBase, true);
MmFreePool(mem);
if (!KSUCCESS(s)) {
DbgPrint("LaunchUserProcess: LdrLoadImage failed with status: %d\n", s);
DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s);
ObDereferenceObject(&proc->Header);
return 0;
}
s = LdrResolveImportsForProcess(proc, imageBase);
if (!KSUCCESS(s)) {
DbgPrint("LaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s);
MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase));
DbgPrint("KiLaunchUserProcess: Failed to resolve EXE imports for %s (status %d)\n", filePath, s);
MmFreeUserMemory((void*)imageBase);
ObDereferenceObject(&proc->Header);
return 0;
}
@@ -308,24 +352,24 @@ static PROCESS* LaunchUserProcess(const char* filePath, const char* args) {
proc->ImageSize = LdrGetImageSize(imageBase);
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
DbgPrint("LaunchUserProcess: PE Image mapped! Base 0x%llx, Entry 0x%llX\n", imageBase, entryPoint);
DbgPrint("KiLaunchUserProcess: PE Image mapped! Base 0x%llx, Entry 0x%llX\n", imageBase, entryPoint);
// Create the main thread last. KeCreateThread queues it before returning.
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, PriorityNormal, &thread);
if (!KSUCCESS(s)) {
DbgPrint("LaunchUserProcess: Failed to create thread. Status: %d\n", s);
DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
proc->ImageBase = 0;
proc->ImageSize = 0;
MmFreeUserMemory((void*)imageBase, LdrGetImageSize(imageBase));
MmFreeUserMemory((void*)imageBase);
ObDereferenceObject(&proc->Header);
return 0;
}
DbgPrint("LaunchUserProcess: Thread created and queued!\n");
DbgPrint("KiLaunchUserProcess: Thread created and queued!\n");
// Caller owns the reference returned by KeCreateProcess.
return proc;
}
static void PrintProcessList() {
static void KiPrintProcessList() {
LIST_ENTRY* listHead = KeGetProcessList();
LIST_ENTRY* current;
@@ -349,10 +393,10 @@ void SystemInitThread(void* arg) {
DbgPrint("SystemInitThread: Thread started. Igniting the system...\n");
InitializeDrivers(args->bootInfo, args->hal);
KiInitializeDrivers(args->bootInfo, args->hal);
ResolveSystemImports(args->bootInfo);
ReclaimBootloaderMemory(args->bootInfo);
KiResolveSystemImports(args->bootInfo);
KiReclaimBootloaderMemory(args->bootInfo);
args->hal->Screen.Clear(0x000000);
@@ -362,16 +406,19 @@ void SystemInitThread(void* arg) {
DbgPrint(" -> Kernel Pool: %.2f MB\n", (float)MmGetKernelPoolUsage() / (1024.0f * 1024.0f));
DbgPrint(" -> User Heap: %.2f MB\n", (float)MmGetUserHeapUsage() / (1024.0f * 1024.0f));
if (args->bootInfo->Debug & 2)
KiRunKernelSelfTests(args->hal);
// Launch User-Mode init.exe and restart it until it exits cleanly.
while (true) {
PROCESS* shellProc = LaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
if (!shellProc) {
DbgPrint("SystemInitThread: Failed to start system/init.exe; retrying.\n");
KeYield();
continue;
}
KeWaitForEvent(shellProc->ExitEvent);
KeWaitForEvent(shellProc->ExitEvent, 0);
int exitCode = shellProc->ExitCode;
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
ObDereferenceObject(&shellProc->Header);
@@ -421,7 +468,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
.FramebufferHeight = bootInfo->FramebufferHeight,
.FramebufferStride = bootInfo->FramebufferStride,
.RsdpPhysical = bootInfo->RsdpPhysical,
.DebugEnabled = bootInfo->DebugEnabled
.Debug = bootInfo->Debug
};
KSTATUS s = HalInitialize(&halParams);
@@ -432,13 +479,19 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
const HAL_API* hal = HalGetApi();
gKmainHal = hal;
KeInitDebug(hal->Screen.PrintChar, bootInfo->DebugEnabled); // Enable debug
KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1); // Enable debug
KeIrqInitialize(hal);
KeInitializeDpcSystem();
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr);
KeIrqRegister(hal->Keyboard.IrqNumber, hal->Keyboard.Isr);
KeIrqRegister(hal->Mouse.IrqNumber, hal->Mouse.Isr);
KeInitializeDpc(&gKeyboardDpc, KeyboardDpcRoutine, (void*)0);
KeInitializeDpc(&gMouseDpc, KiMouseDpcRoutine, (void*)0);
KeIrqRegister(hal->Timer.IrqNumber, hal->Timer.Isr);
KeIrqRegister(hal->Keyboard.IrqNumber,
hal->Keyboard.Dpc ? KeyboardIrqWrapper : hal->Keyboard.Isr);
KeIrqRegister(hal->Mouse.IrqNumber,
hal->Mouse.Dpc ? KiMouseIrqWrapper : hal->Mouse.Isr);
HalRegisterScheduler(KeSchedulerTick);
@@ -447,8 +500,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
//
// Dynamic Linking & Driver Execution
//
ResolveSystemImports(bootInfo);
//InitializeDrivers(bootInfo, hal);
KiResolveSystemImports(bootInfo);
//KiInitializeDrivers(bootInfo, hal);
//
// Scheduler Prep & Status Update
code/kernel/ldr_image.c
+516 -516
View File
File diff suppressed because it is too large. Load diff
code/kernel/mm_phys.c
+4 -4
View File
@@ -14,7 +14,7 @@
// ---------------------------------------------------------------------------
static uint64_t gPhysBase = 0; // set by MmInitializePhysical
static inline void* PhysToVirt(PHYSICAL_ADDRESS phys) {
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS phys) {
return (void *)(phys + gPhysBase);
}
@@ -33,7 +33,7 @@ typedef struct {
static FRAME_STACK gFrameStack;
// Push a physical frame address onto the stack.
static void PushFrame(PHYSICAL_ADDRESS frame) {
static void KiPushFrame(PHYSICAL_ADDRESS frame) {
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
if (gFrameStack.Top < gFrameStack.Capacity) {
gFrameStack.Stack[gFrameStack.Top++] = frame;
@@ -72,7 +72,7 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
return false;
}
gFrameStack.Stack = (PHYSICAL_ADDRESS *)PhysToVirt(stackPhys);
gFrameStack.Stack = (PHYSICAL_ADDRESS *)KiPhysToVirt(stackPhys);
gFrameStack.Top = 0;
gFrameStack.Capacity = maxFrames;
gFrameStack.Total = 0;
@@ -91,7 +91,7 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
for (PHYSICAL_ADDRESS frame = base; frame < end; frame += PAGE_SIZE) {
if (frame >= stackPhys && frame < stackEnd) continue;
PushFrame(frame);
KiPushFrame(frame);
}
}
code/kernel/mm_virt.c
+86 -74
View File
@@ -16,17 +16,18 @@
#include <string.h>
#include <mm_virt.h>
#include <ke_spinlock.h>
#include <ke_irql.h>
// ---------------------------------------------------------------------------
// Helpers - physical window access
// ---------------------------------------------------------------------------
static uint64_t gPhysBase = PHYS_MAP_BASE;
static inline void* PhysToVirt(PHYSICAL_ADDRESS p) {
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
return (void *)(p + gPhysBase);
}
static inline uint64_t* PtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)PhysToVirt(tablePhys) + idx;
static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
}
// ---------------------------------------------------------------------------
@@ -43,16 +44,16 @@ static PHYSICAL_ADDRESS gPml4Phys = 0;
// Ensure a page-table entry at 'parentPhys[index]' points to a child table.
// Allocates and zeros a new frame if the entry is not present.
// Returns the physical address of the child table, or 0 on OOM.
static PHYSICAL_ADDRESS EnsureTable(PHYSICAL_ADDRESS parentPhys,
static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
unsigned index, uint64_t extraFlags) {
uint64_t* entry = PtePtr(parentPhys, index);
uint64_t* entry = KiPtePtr(parentPhys, index);
if (*entry & PAGE_PRESENT) {
*entry |= extraFlags;
return* entry & ~0xFFFULL;
}
PHYSICAL_ADDRESS childPhys = MmAllocateFrame();
if (!childPhys) return 0;
memset(PhysToVirt(childPhys), 0, PAGE_SIZE);
memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE);
*entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags;
return childPhys;
}
@@ -63,27 +64,27 @@ static PHYSICAL_ADDRESS EnsureTable(PHYSICAL_ADDRESS parentPhys,
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags) {
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
PHYSICAL_ADDRESS pdpt = EnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
if (!pdpt) return STATUS_NO_MEMORY;
PHYSICAL_ADDRESS pdt = EnsureTable(pdpt, PDPT_IDX(virt), userFlag);
PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
if (!pdt) return STATUS_NO_MEMORY;
PHYSICAL_ADDRESS pt = EnsureTable(pdt, PDT_IDX(virt), userFlag);
PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag);
if (!pt) return STATUS_NO_MEMORY;
uint64_t* pte = PtePtr(pt, PT_IDX(virt));
uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
*pte = phys | flags;
__invlpg((void *)virt);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) {
uint64_t* e4 = PtePtr(gPml4Phys, PML4_IDX(virt));
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e3 = PtePtr(*e4 & ~0xFFFULL, PDPT_IDX(virt));
uint64_t* e3 = KiPtePtr(*e4 & ~0xFFFULL, PDPT_IDX(virt));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e2 = PtePtr(*e3 & ~0xFFFULL, PDT_IDX(virt));
uint64_t* e2 = KiPtePtr(*e3 & ~0xFFFULL, PDT_IDX(virt));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* pte = PtePtr(*e2 & ~0xFFFULL, PT_IDX(virt));
uint64_t* pte = KiPtePtr(*e2 & ~0xFFFULL, PT_IDX(virt));
*pte = 0;
__invlpg((void *)virt);
return STATUS_SUCCESS;
@@ -97,13 +98,13 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint
uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
uint64_t* e4 = PtePtr(gPml4Phys, PML4_IDX(va));
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e3 = PtePtr(*e4 & ~0xFFFULL, PDPT_IDX(va));
uint64_t* e3 = KiPtePtr(*e4 & ~0xFFFULL, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* e2 = PtePtr(*e3 & ~0xFFFULL, PDT_IDX(va));
uint64_t* e2 = KiPtePtr(*e3 & ~0xFFFULL, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
uint64_t* pte = PtePtr(*e2 & ~0xFFFULL, PT_IDX(va));
uint64_t* pte = KiPtePtr(*e2 & ~0xFFFULL, PT_IDX(va));
if (!(*pte & PAGE_PRESENT)) return STATUS_NOT_FOUND;
// Preserve the physical address, replace everything else.
@@ -123,10 +124,10 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
uint64_t pd_idx = (virt >> 21) & 0x1FF;
uint64_t pt_idx = (virt >> 12) & 0x1FF;
uint64_t* pml4 = PhysToVirt(gPml4Phys);
uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
if (!(pml4[pml4_idx] & PAGE_PRESENT)) return 0;
uint64_t* pdpt = PhysToVirt(pml4[pml4_idx] & ~0xFFFULL);
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & ~0xFFFULL);
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) return 0;
// Check for 1GB huge pages (bit 7 - PS)
@@ -134,7 +135,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
return (pdpt[pdpt_idx] & 0x000FFFFFC0000000ULL) + (virt & 0x3FFFFFFF);
}
uint64_t* pd = PhysToVirt(pdpt[pdpt_idx] & ~0xFFFULL);
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & ~0xFFFULL);
if (!(pd[pd_idx] & PAGE_PRESENT)) return 0;
// Check for 2MB huge pages (bit 7 - PS)
@@ -142,7 +143,7 @@ __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
return (pd[pd_idx] & 0x000FFFFFFFE00000ULL) + (virt & 0x1FFFFF);
}
uint64_t* pt = PhysToVirt(pd[pd_idx] & ~0xFFFULL);
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & ~0xFFFULL);
if (!(pt[pt_idx] & PAGE_PRESENT)) return 0;
// Extract the physical frame and add the offset within the page
@@ -177,16 +178,16 @@ typedef struct BLOCK_FTR {
#define OVERHEAD (HDR_SIZE + FTR_SIZE)
#define MIN_BLOCK (OVERHEAD + sizeof(void*))
static inline BLOCK_FTR* HdrToFtr(BLOCK_HDR* h) {
static inline BLOCK_FTR* KiHdrToFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h + HDR_SIZE + h->Size);
}
static inline BLOCK_HDR* FtrToHdr(BLOCK_FTR* f) {
static inline BLOCK_HDR* KiFtrToHdr(BLOCK_FTR* f) {
return (BLOCK_HDR *)((char *)f - HDR_SIZE - f->Size);
}
static inline BLOCK_HDR* NextBlock(BLOCK_HDR* h) {
static inline BLOCK_HDR* KiNextBlock(BLOCK_HDR* h) {
return (BLOCK_HDR *)((char *)h + HDR_SIZE + h->Size + FTR_SIZE);
}
static inline BLOCK_FTR* PrevFtr(BLOCK_HDR* h) {
static inline BLOCK_FTR* KiPrevFtr(BLOCK_HDR* h) {
return (BLOCK_FTR *)((char *)h - FTR_SIZE);
}
@@ -200,7 +201,7 @@ typedef struct HEAP {
} HEAP;
// Extend heap by at least 'need' bytes (rounded up to page boundary).
static int HeapGrow(HEAP* h, size_t need) {
static int KiHeapGrow(HEAP* h, size_t need) {
size_t grow = (need + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
if (h->Brk + grow > h->Limit) return 0;
@@ -223,23 +224,23 @@ static int HeapGrow(HEAP* h, size_t need) {
blk->NextFree = h->FreeList;
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
HdrToFtr(blk)->Free = 1;
HdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Size = blk->Size;
h->Brk += grow;
return 1;
}
// Unlink a free block from the free list.
static void FreeListRemove(HEAP* h, BLOCK_HDR* blk) {
static void KiFreeListRemove(HEAP* h, BLOCK_HDR* blk) {
if (blk->PrevFree) blk->PrevFree->NextFree = blk->NextFree;
else h->FreeList = blk->NextFree;
if (blk->NextFree) blk->NextFree->PrevFree = blk->PrevFree;
blk->PrevFree = blk->NextFree = (BLOCK_HDR *)0;
}
static void HeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
static void KiHeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
if (!blk || blk->Free == 0) return;
while (h->Brk > h->Start) {
@@ -264,13 +265,13 @@ static void HeapTryDecommitTail(HEAP* h, BLOCK_HDR* blk) {
}
if (blk->Free) {
HdrToFtr(blk)->Size = blk->Size;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
HdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
}
static void* HeapAlloc(HEAP* h, size_t size) {
static void* KiHeapAlloc(HEAP* h, size_t size) {
if (!size) return (void *)0;
size = (size + 7) & ~7ULL; // 8-byte align
@@ -286,11 +287,11 @@ static void* HeapAlloc(HEAP* h, size_t size) {
}
if (!best) {
if (!HeapGrow(h, size + OVERHEAD)) {
if (!KiHeapGrow(h, size + OVERHEAD)) {
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
// Re-search; HeapGrow inserted at the front.
// Re-search; KiHeapGrow inserted at the front.
best = h->FreeList;
while (best && best->Size < size) best = best->NextFree;
if (!best) {
@@ -299,7 +300,7 @@ static void* HeapAlloc(HEAP* h, size_t size) {
}
}
FreeListRemove(h, best);
KiFreeListRemove(h, best);
// Split the block if the remainder is big enough.
if (best->Size >= size + OVERHEAD + sizeof(void*)) {
@@ -311,24 +312,24 @@ static void* HeapAlloc(HEAP* h, size_t size) {
rem->NextFree = h->FreeList;
if (h->FreeList) h->FreeList->PrevFree = rem;
h->FreeList = rem;
HdrToFtr(rem)->Magic = BLOCK_MAGIC;
HdrToFtr(rem)->Free = 1;
HdrToFtr(rem)->Size = rem->Size;
KiHdrToFtr(rem)->Magic = BLOCK_MAGIC;
KiHdrToFtr(rem)->Free = 1;
KiHdrToFtr(rem)->Size = rem->Size;
best->Size = size;
}
best->Free = 0;
HdrToFtr(best)->Free = 0;
HdrToFtr(best)->Magic = BLOCK_MAGIC;
HdrToFtr(best)->Size = best->Size;
KiHdrToFtr(best)->Free = 0;
KiHdrToFtr(best)->Magic = BLOCK_MAGIC;
KiHdrToFtr(best)->Size = best->Size;
memset((char *)best + HDR_SIZE, 0, size);
KeReleaseSpinlock(&h->Lock, flags);
return (char *)best + HDR_SIZE;
}
static void HeapFree(HEAP* h, void* ptr) {
static void KiHeapFree(HEAP* h, void* ptr) {
if (!ptr) return;
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt
@@ -338,25 +339,25 @@ static void HeapFree(HEAP* h, void* ptr) {
blk->Free = 1;
// Coalesce with next block if it is free.
BLOCK_HDR* next = NextBlock(blk);
BLOCK_HDR* next = KiNextBlock(blk);
if ((uintptr_t)next < h->Brk && next->Magic == BLOCK_MAGIC && next->Free) {
FreeListRemove(h, next);
KiFreeListRemove(h, next);
blk->Size += OVERHEAD + next->Size;
HdrToFtr(blk)->Size = blk->Size;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
HdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Size = blk->Size;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
}
// Coalesce with previous block if it is free.
if ((uintptr_t)blk > h->Start) {
BLOCK_FTR* prevFtr = PrevFtr(blk);
BLOCK_FTR* prevFtr = KiPrevFtr(blk);
if (prevFtr->Magic == BLOCK_MAGIC && prevFtr->Free) {
BLOCK_HDR* prev = FtrToHdr(prevFtr);
FreeListRemove(h, prev);
BLOCK_HDR* prev = KiFtrToHdr(prevFtr);
KiFreeListRemove(h, prev);
prev->Size += OVERHEAD + blk->Size;
HdrToFtr(prev)->Size = prev->Size;
HdrToFtr(prev)->Magic = BLOCK_MAGIC;
HdrToFtr(prev)->Free = 1;
KiHdrToFtr(prev)->Size = prev->Size;
KiHdrToFtr(prev)->Magic = BLOCK_MAGIC;
KiHdrToFtr(prev)->Free = 1;
blk = prev;
}
}
@@ -367,23 +368,23 @@ static void HeapFree(HEAP* h, void* ptr) {
if (h->FreeList) h->FreeList->PrevFree = blk;
h->FreeList = blk;
HdrToFtr(blk)->Free = 1;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
KiHdrToFtr(blk)->Free = 1;
KiHdrToFtr(blk)->Magic = BLOCK_MAGIC;
HeapTryDecommitTail(h, blk);
KiHeapTryDecommitTail(h, blk);
KeReleaseSpinlock(&h->Lock, flags);
}
static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
static void* KiHeapRealloc(HEAP* h, void* ptr, size_t new_size) {
// If ptr is NULL, realloc acts like malloc
if (!ptr) {
return HeapAlloc(h, new_size);
return KiHeapAlloc(h, new_size);
}
// If size is 0, realloc acts like free
if (new_size == 0) {
HeapFree(h, ptr);
KiHeapFree(h, ptr);
return (void*)0;
}
@@ -395,18 +396,18 @@ static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
return (void*)0;
}
// Align the requested size (matches HeapAlloc 8-byte alignment)
// Align the requested size (matches KiHeapAlloc 8-byte alignment)
size_t aligned_size = (new_size + 7) & ~7ULL;
// If the current block is already big enough, we can reuse it in-place.
// (Note: To save space, you could optionally split the block here if
// the new size is significantly smaller, similar to HeapAlloc's logic).
// the new size is significantly smaller, similar to KiHeapAlloc's logic).
if (blk->Size >= aligned_size) {
return ptr;
}
// The block is too small. Allocate a new, larger block.
void* new_ptr = HeapAlloc(h, aligned_size);
void* new_ptr = KiHeapAlloc(h, aligned_size);
if (!new_ptr) {
return (void*)0; // Out of memory, original block is left untouched
}
@@ -417,7 +418,7 @@ static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
memcpy(new_ptr, ptr, blk->Size);
// Free the old block
HeapFree(h, ptr);
KiHeapFree(h, ptr);
return new_ptr;
}
@@ -449,7 +450,7 @@ void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
// Pre-commit the requested pool size.
if (poolSize) {
poolSize = (poolSize + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
HeapGrow(&gPoolHeap, poolSize);
KiHeapGrow(&gPoolHeap, poolSize);
}
// User heap.
@@ -468,16 +469,27 @@ void MmActivate(void) {
// ---------------------------------------------------------------------------
// Public pool API
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocatePool(size_t size) { return HeapAlloc(&gPoolHeap, size); }
__declspec(dllexport) void MmFreePool(void* ptr) { HeapFree(&gPoolHeap, ptr); }
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return HeapRealloc(&gPoolHeap, ptr, newSize); }
__declspec(dllexport) void* MmAllocatePool(size_t size) { return KiHeapAlloc(&gPoolHeap, size); }
__declspec(dllexport) void MmFreePool(void* ptr) { KiHeapFree(&gPoolHeap, ptr); }
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize) { return KiHeapRealloc(&gPoolHeap, ptr, newSize); }
// ---------------------------------------------------------------------------
// Public user heap API
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) { return HeapAlloc(&gUserHeap, size); }
__declspec(dllexport) void MmFreeUserMemory(void* ptr, size_t size) { (void)size; HeapFree(&gUserHeap, ptr); }
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) { return HeapRealloc(&gUserHeap, ptr, newSize); }
__declspec(dllexport) void* MmAllocateUserMemory(size_t size) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapAlloc(&gUserHeap, size);
}
__declspec(dllexport) void MmFreeUserMemory(void* ptr) {
KeAssertIrql(PASSIVE_LEVEL);
KiHeapFree(&gUserHeap, ptr);
}
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize) {
KeAssertIrql(PASSIVE_LEVEL);
return KiHeapRealloc(&gUserHeap, ptr, newSize);
}
size_t MmGetKernelPoolUsage(void) {
// Returns bytes committed to the kernel pool
code/kernel/mm_virt.h
+73 -73
View File
@@ -1,73 +1,73 @@
#pragma once
// mm_virt.h - virtual memory manager (VMM)
// Manages the single shared PML4, kernel pool, and user heap.
#include <stdint.h>
#include <stddef.h>
#include <mm_phys.h>
#include <ke_status.h>
// ---------------------------------------------------------------------------
// Page flags (x86-64 PTE bits)
// ---------------------------------------------------------------------------
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_NX (1ULL << 63)
// ---------------------------------------------------------------------------
// Virtual address space constants
// ---------------------------------------------------------------------------
#define USER_BASE 0x0000000000001000ULL
#define USER_LIMIT 0x00007FFFFFFFF000ULL
#define KERNEL_BASE 0xFFFF800000000000ULL
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
// ---------------------------------------------------------------------------
// Initialisation
// ---------------------------------------------------------------------------
// Called by KernelMain after MmInitializePhysical.
// bootPml4Phys: the PML4 set up by the bootloader.
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
void MmActivate(void);
// ---------------------------------------------------------------------------
// Page mapping
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags);
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt);
// Modify protection flags on an already-mapped virtual range.
// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags
// entirely (except the physical address, which is preserved).
// Returns STATUS_NOT_FOUND if any page in the range is not mapped.
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags);
// Walk the page tables to translate a virtual address to physical.
// Returns 0 if the address is not mapped.
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt);
// ---------------------------------------------------------------------------
// Kernel pool - ring 0 only, never PAGE_USER.
// Best-fit free-list allocator, thread-safe via spinlock.
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocatePool(size_t size);
__declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// ---------------------------------------------------------------------------
// User heap - shared flat space, ring 3 accessible.
// All allocations are PAGE_PRESENT | PAGE_WRITE | PAGE_USER.
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocateUserMemory(size_t size);
__declspec(dllexport) void MmFreeUserMemory(void* ptr, size_t size);
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize);
size_t MmGetKernelPoolUsage(void);
size_t MmGetUserHeapUsage(void);
#pragma once
// mm_virt.h - virtual memory manager (VMM)
// Manages the single shared PML4, kernel pool, and user heap.
#include <stdint.h>
#include <stddef.h>
#include <mm_phys.h>
#include <ke_status.h>
// ---------------------------------------------------------------------------
// Page flags (x86-64 PTE bits)
// ---------------------------------------------------------------------------
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_NX (1ULL << 63)
// ---------------------------------------------------------------------------
// Virtual address space constants
// ---------------------------------------------------------------------------
#define USER_BASE 0x0000000000001000ULL
#define USER_LIMIT 0x00007FFFFFFFF000ULL
#define KERNEL_BASE 0xFFFF800000000000ULL
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
// ---------------------------------------------------------------------------
// Initialisation
// ---------------------------------------------------------------------------
// Called by KernelMain after MmInitializePhysical.
// bootPml4Phys: the PML4 set up by the bootloader.
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
void MmActivate(void);
// ---------------------------------------------------------------------------
// Page mapping
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags);
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt);
// Modify protection flags on an already-mapped virtual range.
// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags
// entirely (except the physical address, which is preserved).
// Returns STATUS_NOT_FOUND if any page in the range is not mapped.
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags);
// Walk the page tables to translate a virtual address to physical.
// Returns 0 if the address is not mapped.
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt);
// ---------------------------------------------------------------------------
// Kernel pool - ring 0 only, never PAGE_USER.
// Best-fit free-list allocator, thread-safe via spinlock.
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocatePool(size_t size);
__declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// ---------------------------------------------------------------------------
// User heap - shared flat space, ring 3 accessible.
// All allocations are PAGE_PRESENT | PAGE_WRITE | PAGE_USER.
// ---------------------------------------------------------------------------
__declspec(dllexport) void* MmAllocateUserMemory(size_t size);
__declspec(dllexport) void MmFreeUserMemory(void* ptr);
__declspec(dllexport) void* MmReallocateUserMemory(void* ptr, size_t newSize);
size_t MmGetKernelPoolUsage(void);
size_t MmGetUserHeapUsage(void);
code/kernel/ob_object.h
+35 -34
View File
@@ -1,34 +1,35 @@
#pragma once
// ob_object.h - executive object model
// Every resource the kernel tracks is a typed, reference-counted OBJECT.
// The OBJECT header must be the first member of any derived struct.
#include <stdint.h>
#include <stddef.h>
#include <ke_status.h>
typedef enum OBJECT_TYPE {
ObjectTypeProcess = 1,
ObjectTypeThread = 2,
ObjectTypeMutex = 3,
ObjectTypeSemaphore = 4,
ObjectTypeEvent = 5,
ObjectTypeQueue = 6,
ObjectTypeFile = 7,
ObjectTypeDir = 8,
} OBJECT_TYPE;
typedef struct OBJECT {
OBJECT_TYPE Type;
volatile long RefCount; // interlocked
void (*Destroy)(struct OBJECT* self);
} OBJECT;
// Increment the reference count. Call whenever ownership is shared.
void ObReferenceObject(OBJECT* object);
// Decrement. Calls Destroy() and frees the object when count reaches zero.
void ObDereferenceObject(OBJECT* object);
// Allocate a zeroed object of the given size from kernel pool.
// Caller must fill Type and Destroy before publishing the pointer.
OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size);
#pragma once
// ob_object.h - executive object model
// Every resource the kernel tracks is a typed, reference-counted OBJECT.
// The OBJECT header must be the first member of any derived struct.
#include <stdint.h>
#include <stddef.h>
#include <ke_status.h>
typedef enum OBJECT_TYPE {
ObjectTypeProcess = 1,
ObjectTypeThread = 2,
ObjectTypeMutex = 3,
ObjectTypeSemaphore = 4,
ObjectTypeEvent = 5,
ObjectTypeQueue = 6,
ObjectTypeFile = 7,
ObjectTypeDir = 8,
ObjectTypeAsyncIo = 9,
} OBJECT_TYPE;
typedef struct OBJECT {
OBJECT_TYPE Type;
volatile long RefCount; // interlocked
void (*Destroy)(struct OBJECT* self);
} OBJECT;
// Increment the reference count. Call whenever ownership is shared.
void ObReferenceObject(OBJECT* object);
// Decrement. Calls Destroy() and frees the object when count reaches zero.
void ObDereferenceObject(OBJECT* object);
// Allocate a zeroed object of the given size from kernel pool.
// Caller must fill Type and Destroy before publishing the pointer.
OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size);
code/kernel/syscall_nr.h
+81 -174
View File
@@ -1,178 +1,85 @@
#pragma once
// syscall_nr.h - system call numbers
// Shared between kernel (ke_syscall.c) and user mode (syscall.dll).
// Keep in sync with gSyscallTable in ke_syscall.c and SYS_MAX below.
//
// Calling convention (INT 0x40, MS x64):
// syscall_nr.h
// INT 0x40 calling convention, MS x64:
// RAX = syscall number
// RCX, RDX, R8, R9 = args 0-3
// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6 in syscall_stubs.asm)
// Return value in RAX (KSTATUS or data).
//
// Handle semantics:
// Every kernel object (file, mutex, semaphore, event, queue) is reference-
// counted. SYS_CLOSE_HANDLE is the ONE way to close any handle regardless
// of the object type behind it. KeCloseHandle calls ObDereferenceObject,
// which fires the type-specific Destroy callback (e.g. FileDestroy)
// automatically when the refcount reaches zero. There is no separate
// SYS_CLOSE_FILE - that was redundant.
// RCX, RDX, R8, R9 = args 0-3
// R10, R11 = args 4-5 (loaded by SYSCALL_STUB6)
// RAX = return value
// ---------------------------------------------------------------------------
// Process & Thread
// ---------------------------------------------------------------------------
#define SYS_EXIT_PROCESS 0 // (int exitCode) -> void
#define SYS_EXIT_THREAD 1 // (int exitCode) -> void
#define SYS_CREATE_THREAD 2 // (uint64_t entry, uint64_t arg, uint32_t priority, uint32_t* tidOut) -> KSTATUS
#define SYS_GET_CURRENT_TID 3 // () -> uint32_t
#define SYS_GET_CURRENT_PID 4 // () -> uint32_t
#define SYS_YIELD 5 // () -> void
#define SYS_SLEEP 6 // (uint32_t ms) -> void
#define SYS_SET_THREAD_PRIORITY 7 // (uint32_t priority) -> KSTATUS
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_WAIT_FOR_PROCESS 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_WAIT_THREAD 9
#define SYS_GET_CURRENT_TID 10
#define SYS_GET_CURRENT_PID 11
#define SYS_SET_THREAD_PRIORITY 12
#define SYS_YIELD 13
#define SYS_SLEEP 14
#define SYS_SET_USER_APC_DISPATCHER 15
#define SYS_APC_RETURN 16
#define SYS_ALLOC 17
#define SYS_FREE 18
#define SYS_REALLOC 19
#define SYS_CLOSE_HANDLE 20
#define SYS_MUTEX_CREATE 21
#define SYS_MUTEX_ACQUIRE 22
#define SYS_MUTEX_TRY_ACQUIRE 23
#define SYS_MUTEX_RELEASE 24
#define SYS_SEM_CREATE 25
#define SYS_SEM_WAIT 26
#define SYS_SEM_TRY_WAIT 27
#define SYS_SEM_SIGNAL 28
#define SYS_EVENT_CREATE 29
#define SYS_EVENT_SIGNAL 30
#define SYS_EVENT_CLEAR 31
#define SYS_EVENT_WAIT 32
#define SYS_QUEUE_CREATE 33
#define SYS_QUEUE_SEND 34
#define SYS_QUEUE_RECV 35
#define SYS_PUBLISH_QUEUE 36
#define SYS_OPEN_NAMED_QUEUE 37
#define SYS_QUEUE_DUPLICATE 38
#define SYS_NAMED_QUEUE_OWNER_PID 39
#define SYS_OPEN_FILE 40
#define SYS_CREATE_FILE 41
#define SYS_READ_FILE 42
#define SYS_WRITE_FILE 43
#define SYS_READ_FILE_ASYNC 44
#define SYS_WRITE_FILE_ASYNC 45
#define SYS_CANCEL_IO 46
#define SYS_SEEK_FILE 47
#define SYS_GET_FILE_POS 48
#define SYS_GET_FILE_SIZE 49
#define SYS_TRUNCATE_FILE 50
#define SYS_DELETE_FILE 51
#define SYS_RENAME_FILE 52
#define SYS_OPEN_DIR 53
#define SYS_READ_DIR 54
#define SYS_CREATE_DIR 55
#define SYS_REMOVE_DIR 56
#define SYS_SET_CWD 57
#define SYS_GET_CWD 58
#define SYS_WRITE_CONSOLE 59
#define SYS_SCREEN_BLIT 60
#define SYS_SCREEN_CLEAR 61
#define SYS_SCREEN_GET_SIZE 62
#define SYS_SCREEN_PUT_CHAR 63
#define SYS_KB_POLL 64
#define SYS_KB_IS_KEY_DOWN 65
#define SYS_KB_READ_CHAR 66
#define SYS_KB_READ_SCANCODE 67
#define SYS_KB_GET_MODIFIERS 68
#define SYS_MOUSE_POLL 69
#define SYS_MOUSE_READ 70
#define SYS_MOUSE_GET_POS 71
#define SYS_GET_TICK 72
#define SYS_DEBUG_PRINT 73
#define SYS_QUERY_TRAP_FRAME 74
// ---------------------------------------------------------------------------
// Memory
// ---------------------------------------------------------------------------
#define SYS_ALLOC 8 // (size_t size) -> void* (NULL on failure)
#define SYS_FREE 9 // (void* ptr, size_t size) -> void
#define SYS_REALLOC 10 // (void* ptr, size_t newSize) -> void*
// ---------------------------------------------------------------------------
// Synchronisation Objects
// ---------------------------------------------------------------------------
#define SYS_MUTEX_CREATE 11 // (uint32_t* handleOut) -> KSTATUS
#define SYS_MUTEX_ACQUIRE 12 // (uint32_t handle) -> KSTATUS
#define SYS_MUTEX_TRY_ACQUIRE 13 // (uint32_t handle) -> KSTATUS
#define SYS_MUTEX_RELEASE 14 // (uint32_t handle) -> KSTATUS
#define SYS_SEM_CREATE 15 // (int32_t initial, int32_t max, uint32_t* handleOut) -> KSTATUS
#define SYS_SEM_WAIT 16 // (uint32_t handle) -> KSTATUS
#define SYS_SEM_TRY_WAIT 17 // (uint32_t handle) -> KSTATUS
#define SYS_SEM_SIGNAL 18 // (uint32_t handle) -> KSTATUS
#define SYS_EVENT_CREATE 19 // (uint32_t* handleOut) -> KSTATUS
#define SYS_EVENT_SIGNAL 20 // (uint32_t handle) -> KSTATUS
#define SYS_EVENT_CLEAR 21 // (uint32_t handle) -> KSTATUS
#define SYS_EVENT_WAIT 22 // (uint32_t handle) -> KSTATUS
// ---------------------------------------------------------------------------
// Queue
// ---------------------------------------------------------------------------
#define SYS_QUEUE_CREATE 23 // (uint32_t* handleOut) -> KSTATUS
#define SYS_QUEUE_SEND 24 // (uint32_t handle, void* msg, int blocking) -> KSTATUS
#define SYS_QUEUE_RECV 25 // (uint32_t handle, void* msg, int blocking) -> KSTATUS
// ---------------------------------------------------------------------------
// Handle (unified close for all object types)
// ---------------------------------------------------------------------------
#define SYS_CLOSE_HANDLE 26 // (uint32_t handle) -> KSTATUS
// ---------------------------------------------------------------------------
// File I/O
// (Close a file handle via SYS_CLOSE_HANDLE, same as any other object.)
// ---------------------------------------------------------------------------
#define SYS_OPEN_FILE 27 // (const char* path, uint32_t* handleOut) -> KSTATUS
#define SYS_READ_FILE 28 // (uint32_t handle, void* buf, size_t size, size_t* bytesReadOut) -> KSTATUS
#define SYS_WRITE_FILE 29 // (uint32_t handle, const void* buf, size_t size, size_t* bytesWrittenOut) -> KSTATUS
#define SYS_CREATE_FILE 30 // (const char* path, uint32_t* handleOut) -> KSTATUS
#define SYS_DELETE_FILE 31 // (const char* path) -> KSTATUS
#define SYS_TRUNCATE_FILE 32 // (uint32_t handle, uint64_t newSize) -> KSTATUS
#define SYS_GET_FILE_POS 33 // (uint32_t handle) -> uint64_t
#define SYS_SEEK_FILE 34 // (uint32_t handle, uint64_t offset) -> KSTATUS
#define SYS_GET_FILE_SIZE 35 // (uint32_t handle) -> uint64_t
// ---------------------------------------------------------------------------
// Screen / Graphics
// ---------------------------------------------------------------------------
#define SYS_SCREEN_BLIT 36 // (int x, int y, int w, int h, const uint32_t* pixels) -> void
#define SYS_SCREEN_CLEAR 37 // (uint32_t color) -> void
#define SYS_SCREEN_GET_SIZE 38 // (int* widthOut, int* heightOut) -> void
#define SYS_SCREEN_PUT_CHAR 39 // (char c, int x, int y, uint32_t fg, uint32_t bg) -> void
// ---------------------------------------------------------------------------
// Keyboard
// ---------------------------------------------------------------------------
#define SYS_KB_POLL 40 // () -> uint32_t (1 if key available, 0 otherwise)
#define SYS_KB_IS_KEY_DOWN 41 // (int scancode) -> uint32_t (1 if down, 0 otherwise)
#define SYS_KB_READ_CHAR 42 // () -> char (blocks, yields while waiting)
#define SYS_KB_READ_SCANCODE 43 // () -> uint8_t (blocks, yields while waiting)
#define SYS_KB_GET_MODIFIERS 44 // () -> uint32_t (current modifier bitmask)
// ---------------------------------------------------------------------------
// Mouse
// ---------------------------------------------------------------------------
#define SYS_MOUSE_POLL 45 // () -> uint32_t (1 if packet waiting, 0 otherwise)
#define SYS_MOUSE_READ 46 // (int* dx, int* dy, uint8_t* buttons) -> uint32_t (1=ok, 0=empty)
#define SYS_MOUSE_GET_POS 47 // (int* x, int* y) -> void
// ---------------------------------------------------------------------------
// Timer
// ---------------------------------------------------------------------------
#define SYS_GET_TICK 48 // () -> uint64_t (milliseconds since boot)
// ---------------------------------------------------------------------------
// Debug
// ---------------------------------------------------------------------------
#define SYS_DEBUG_PRINT 49 // (const char* str) -> void
// ---------------------------------------------------------------------------
// Process launch & lifecycle
// ---------------------------------------------------------------------------
#define SYS_CREATE_PROCESS 50 // (const char* path, const char* args, uint32_t* pidOut) -> KSTATUS
#define SYS_WAIT_FOR_PROCESS 51 // (uint32_t pid) -> KSTATUS
#define SYS_GET_PROCESS_EXIT_CODE 52 // (uint32_t pid, int* exitCodeOut) -> KSTATUS
// ---------------------------------------------------------------------------
// Console output (white text, for user programs)
// ---------------------------------------------------------------------------
#define SYS_WRITE_CONSOLE 53 // (const char* str, size_t len) -> void
// ---------------------------------------------------------------------------
// Directory enumeration
// ---------------------------------------------------------------------------
#define SYS_OPEN_DIR 54 // (const char* path, uint32_t* handleOut) -> KSTATUS
#define SYS_READ_DIR 55 // (uint32_t handle, DirEntry* entryOut) -> KSTATUS
// ---------------------------------------------------------------------------
// Current Working Directory
// ---------------------------------------------------------------------------
#define SYS_SET_CWD 56 // (const char* path) -> KSTATUS
#define SYS_GET_CWD 57 // (char* buf, size_t buflen) -> KSTATUS
// ---------------------------------------------------------------------------
// Thread join (used by pthread_join in posix.dll)
// ---------------------------------------------------------------------------
#define SYS_WAIT_THREAD 58 // (uint32_t tid) -> KSTATUS
// ---------------------------------------------------------------------------
// Process PID query (used by posix_spawn -> waitpid in posix.dll)
// ---------------------------------------------------------------------------
#define SYS_GET_PROCESS_PID 59 // (uint32_t pid) -> KSTATUS
// ---------------------------------------------------------------------------
// Named Queue Registry
// SysPublishQueue - register a queue under a name (call once at startup)
// SysOpenNamedQueue - resolve a name to a handle (retry until SUCCESS)
// ---------------------------------------------------------------------------
#define SYS_PUBLISH_QUEUE 60 // (const char* name, uint32_t queueHandle) -> KSTATUS
#define SYS_OPEN_NAMED_QUEUE 61 // (const char* name, uint32_t* handleOut) -> KSTATUS
// ---------------------------------------------------------------------------
// Handle duplication
// ---------------------------------------------------------------------------
#define SYS_QUEUE_DUPLICATE 62 // (uint32_t handle, uint32_t targetPid, uint32_t* handleOut) -> KSTATUS
#define SYS_NAMED_QUEUE_OWNER_PID 63 // (const char* name, uint32_t* pidOut) -> KSTATUS
// ---------------------------------------------------------------------------
// Process utilities
// ---------------------------------------------------------------------------
#define SYS_KILL_PROCESS 64 // (uint32_t pid, int exitCode) -> KSTATUS
#define SYS_GET_PROCESS_LIST 65 // (uint32_t* pidsOut, size_t maxCount, size_t* countOut) -> KSTATUS
// ---------------------------------------------------------------------------
// Filesystem mutations
// ---------------------------------------------------------------------------
#define SYS_RENAME_FILE 66 // (const char* oldPath, const char* newPath) -> KSTATUS
#define SYS_CREATE_DIR 67 // (const char* path) -> KSTATUS
#define SYS_REMOVE_DIR 68 // (const char* path) -> KSTATUS
#define SYS_MAX 69
#define SYS_MAX 75