fix a few bugs in scheduling and synchronization, separate OVMF_VARS for debug

This commit is contained in:
maksw committed 2026-05-02 11:34:30 +02:00
1 parent 5cf7a20ea2
commit 129be91e15
17 files changed
+174 -211

No files matched your search

+1 -1
View File
@@ -23,7 +23,7 @@ static volatile uint32_t* gLapic = NULL;
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
static volatile uint64_t gTicks = 0;
void (*gTimerCallback)(void) = NULL;
void (* volatile gTimerCallback)(void) = NULL;
static inline void outb(uint16_t port, uint8_t val) { __outbyte(port, val); }
static inline uint8_t inb(uint16_t port) { return __inbyte(port); }
+10 -12
View File
@@ -21,37 +21,35 @@ KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
void KeSignalEvent(EVENT* event) {
if (!event) return;
KeAcquireSpinlock(&event->Lock);
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (event->Type == EventManualReset) {
// Wake all waiters.
event->Signaled = 1;
while (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&event->Lock);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThread(t);
KeAcquireSpinlock(&event->Lock);
flags = KeAcquireSpinlock(&event->Lock);
}
} else {
// Auto-reset: wake one waiter or set the flag for the next waiter.
if (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&event->Lock);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThread(t);
return;
}
event->Signaled = 1;
}
KeReleaseSpinlock(&event->Lock);
KeReleaseSpinlock(&event->Lock, flags);
}
void KeClearEvent(EVENT* event) {
if (!event) return;
KeAcquireSpinlock(&event->Lock);
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
event->Signaled = 0;
KeReleaseSpinlock(&event->Lock);
KeReleaseSpinlock(&event->Lock, flags);
}
KSTATUS KeWaitForEvent(EVENT* event) {
@@ -59,14 +57,14 @@ KSTATUS KeWaitForEvent(EVENT* event) {
THREAD* self = KeCurrentThread();
for (;;) {
KeAcquireSpinlock(&event->Lock);
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (event->Signaled) {
if (event->Type == EventAutoReset) event->Signaled = 0;
KeReleaseSpinlock(&event->Lock);
KeReleaseSpinlock(&event->Lock, flags);
return STATUS_SUCCESS;
}
ListInsertTail(&event->WaitList, &self->SchedulerLink);
KeReleaseSpinlock(&event->Lock);
KeReleaseSpinlock(&event->Lock, flags);
KeBlockThread(self, &event->Header);
}
}
+2 -3
View File
@@ -14,12 +14,11 @@ void KeIrqInitialize(const HAL_API *hal) {
void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
if (irq >= MAX_IRQS || !handler) return;
gIrqHandlers[irq] = handler;
gHal->Intc.RouteIrq(irq); // programs IOAPIC, clears mask bit
gHal->Intc.RouteIrq(irq);
}
// Called from KeIrqCommon in assembly - this is the only symbol
// the assembly needs to know.
void KeIrqDispatch(uint8_t irq) {
// eoi first, handler second because it doesn't work otherwise.
gHal->Intc.SendEoi();
if (irq < MAX_IRQS && gIrqHandlers[irq])
gIrqHandlers[irq]();
+9 -13
View File
@@ -24,46 +24,42 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) {
THREAD* self = KeCurrentThread();
for (;;) {
KeAcquireSpinlock(&mutex->Lock);
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KeReleaseSpinlock(&mutex->Lock);
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_SUCCESS;
}
// Already owned - add self to wait list and block.
ListInsertTail(&mutex->WaitList, &self->SchedulerLink);
KeReleaseSpinlock(&mutex->Lock);
KeReleaseSpinlock(&mutex->Lock, flags);
KeBlockThread(self, &mutex->Header);
// KeBlockThread calls KeYield; when we return here we've been
// unblocked by KeReleaseMutex - loop back and try to own it.
}
}
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeCurrentThread();
KeAcquireSpinlock(&mutex->Lock);
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!mutex->Owner) {
mutex->Owner = self;
KeReleaseSpinlock(&mutex->Lock);
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&mutex->Lock);
KeReleaseSpinlock(&mutex->Lock, flags);
return STATUS_FAILURE;
}
void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
KeAcquireSpinlock(&mutex->Lock);
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
mutex->Owner = (struct THREAD *)0;
// Wake the first waiter, if any.
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&mutex->Lock);
KeReleaseSpinlock(&mutex->Lock, flags);
KeUnblockThread(waiter);
return;
}
KeReleaseSpinlock(&mutex->Lock);
KeReleaseSpinlock(&mutex->Lock, flags);
}
+13 -13
View File
@@ -1,6 +1,6 @@
// ke_process.c - process model
#include "ke_process.h"
#include "ke_thread.h" // KeExitThread (forward declared below)
#include "ke_thread.h"
#include "mm_virt.h"
#include "ob_object.h"
#include <string.h>
@@ -41,9 +41,9 @@ static void ProcessDestroy(OBJECT* obj) {
}
// Remove from global list.
KeAcquireSpinlock(&gProcessListLock);
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
ListRemove(&proc->GlobalLink);
KeReleaseSpinlock(&gProcessListLock);
KeReleaseSpinlock(&gProcessListLock, flags);
// Memory for the struct itself is freed by ObDereferenceObject after
// Destroy returns.
@@ -76,9 +76,9 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
LIST_INIT(&proc->GlobalLink);
KeAcquireSpinlock(&gProcessListLock);
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
ListInsertTail(&gProcessList, &proc->GlobalLink);
KeReleaseSpinlock(&gProcessListLock);
KeReleaseSpinlock(&gProcessListLock, flags);
*out = proc;
return STATUS_SUCCESS;
@@ -101,43 +101,43 @@ void KeExitProcess(PROCESS* process, int exitCode) {
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut) {
if (!process || !object || !handleOut) return STATUS_INVALID_PARAMETER;
KeAcquireSpinlock(&process->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
if (!process->Handles[i]) {
ObReferenceObject(object);
process->Handles[i] = object;
*handleOut = (uint32_t)i;
KeReleaseSpinlock(&process->HandleLock);
KeReleaseSpinlock(&process->HandleLock, flags);
return STATUS_SUCCESS;
}
}
KeReleaseSpinlock(&process->HandleLock);
KeReleaseSpinlock(&process->HandleLock, flags);
return STATUS_NO_MEMORY;
}
void KeCloseHandle(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return;
KeAcquireSpinlock(&process->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
OBJECT* obj = process->Handles[handle];
if (obj) {
process->Handles[handle] = (OBJECT *)0;
KeReleaseSpinlock(&process->HandleLock);
KeReleaseSpinlock(&process->HandleLock, flags);
ObDereferenceObject(obj);
return;
}
KeReleaseSpinlock(&process->HandleLock);
KeReleaseSpinlock(&process->HandleLock, flags);
}
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) {
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE)
return (OBJECT *)0;
KeAcquireSpinlock(&process->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
OBJECT* obj = process->Handles[handle];
if (obj) ObReferenceObject(obj);
KeReleaseSpinlock(&process->HandleLock);
KeReleaseSpinlock(&process->HandleLock, flags);
return obj;
}
+10 -10
View File
@@ -25,7 +25,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
THREAD* self = KeCurrentThread();
for (;;) {
KeAcquireSpinlock(&queue->Lock);
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count < QUEUE_DEPTH) {
queue->Ring[queue->Tail] = *msg;
queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH;
@@ -35,19 +35,19 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
if (!LIST_IS_EMPTY(&queue->RecvWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
}
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_FULL;
}
ListInsertTail(&queue->SendWaiters, &self->SchedulerLink);
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
KeBlockThread(self, &queue->Header);
}
}
@@ -57,7 +57,7 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
THREAD* self = KeCurrentThread();
for (;;) {
KeAcquireSpinlock(&queue->Lock);
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
if (queue->Count > 0) {
*msg = queue->Ring[queue->Head];
queue->Head = (queue->Head + 1) % QUEUE_DEPTH;
@@ -67,19 +67,19 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
if (!LIST_IS_EMPTY(&queue->SendWaiters)) {
LIST_ENTRY* node = ListPopHead(&queue->SendWaiters);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
KeUnblockThread(t);
} else {
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
}
return STATUS_SUCCESS;
}
if (!blocking) {
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
return STATUS_QUEUE_EMPTY;
}
ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink);
KeReleaseSpinlock(&queue->Lock);
KeReleaseSpinlock(&queue->Lock, flags);
KeBlockThread(self, &queue->Header);
}
}
+48 -70
View File
@@ -1,15 +1,4 @@
// ke_sched.c - MLFQ scheduler
//
// Three queues: PriorityHigh(2), PriorityNormal(1), PriorityLow(0).
// Dispatch always picks the highest non-empty queue.
// Within a queue: strict round-robin.
//
// Demotion threshold: a thread that consumes DEMOTION_QUANTA consecutive
// ticks without blocking drops one level (floor PriorityLow).
// Promotion: a thread that unblocks is bumped one level (ceiling PriorityHigh).
//
// The idle thread (created by KeInitializeScheduler) runs at PriorityLow
// when nothing else is runnable.
#include "ke_sched.h"
#include "ke_thread.h"
#include "ke_spinlock.h"
@@ -17,44 +6,27 @@
#include "mm_virt.h"
#include <stddef.h>
#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */
#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */
// ---------------------------------------------------------------------------
// Run queues
// ---------------------------------------------------------------------------
static LIST_ENTRY gRunQueue[PriorityCount]; // [0]=Low [1]=Normal [2]=High
static LIST_ENTRY gRunQueue[PriorityCount];
static SPINLOCK gSchedLock;
// The currently running thread (also in GS:[0] for fast access).
// On a single-core system this is the only "per-CPU" slot we need.
static THREAD* gCurrentThread;
static THREAD* gIdleThread;
static THREAD* gOutgoingThread;
static uint32_t gAgingCounter = 0;
// ---------------------------------------------------------------------------
// GS:[0] helpers - implemented in ke_switch.asm
// (On single-core we just use a global; SMP would use per-core GS.)
// ---------------------------------------------------------------------------
THREAD* KeCurrentThread(void) { return gCurrentThread; }
void KeSetCurrentThread(THREAD* t){ gCurrentThread = t; }
// ---------------------------------------------------------------------------
// Idle thread body - runs in ring 0 when every other thread is blocked.
// ---------------------------------------------------------------------------
static void IdleLoop(void) {
for (;;) { __halt(); }
}
// ---------------------------------------------------------------------------
// Init
// ---------------------------------------------------------------------------
void KeInitializeScheduler(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
// Create the idle process / thread.
// We call the internal alloc directly to avoid the full KeCreateThread
// path (no user stack, no ring-3 entry - the idle thread never leaves
// ring 0 and never calls KeExitThread).
extern KSTATUS KeCreateProcess(const char *, PROCESS **);
PROCESS* idleProc;
KeCreateProcess("idle", &idleProc);
@@ -67,13 +39,13 @@ void KeInitializeScheduler(void) {
idle->QuantaConsumed = 0;
idle->Owner = idleProc;
idle->WaitObject = (OBJECT *)0;
idle->SchedLockFlags = 0x202ULL; // IF=1
void* kstack = MmAllocatePool(KSTACK_SIZE);
idle->KernelStackBase = kstack;
idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
idle->TssRsp0 = idle->KernelStackTop;
// Initial context: RSP at top of kstack, Rip = IdleLoop.
uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop;
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
@@ -83,13 +55,20 @@ void KeInitializeScheduler(void) {
gIdleThread = idle;
gCurrentThread = idle;
// Don't add idle to the run queue here; KeSchedulerTick falls back to it.
}
// ---------------------------------------------------------------------------
// Run-queue management
// ---------------------------------------------------------------------------
// Safely release the lock and clean up any dead threads natively post-switch
void KeSchedulerPostSwitch(void) {
THREAD* cur = KeCurrentThread();
KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags);
if (gOutgoingThread && gOutgoingThread->State == ThreadDead) {
THREAD* dead = gOutgoingThread;
gOutgoingThread = (THREAD*)0;
ObDereferenceObject(&dead->Header);
}
}
void KeReadyThread(THREAD* thread) {
if (!thread) return;
thread->State = ThreadReady;
@@ -97,22 +76,18 @@ void KeReadyThread(THREAD* thread) {
if (p < 0) p = 0;
if (p >= PriorityCount) p = PriorityCount - 1;
KeAcquireSpinlock(&gSchedLock);
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
KeReleaseSpinlock(&gSchedLock);
KeReleaseSpinlock(&gSchedLock, flags);
}
void KeUnreadyThread(THREAD* thread) {
if (!thread) return;
KeAcquireSpinlock(&gSchedLock);
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
ListRemove(&thread->SchedulerLink);
KeReleaseSpinlock(&gSchedLock);
KeReleaseSpinlock(&gSchedLock, flags);
}
// ---------------------------------------------------------------------------
// Pick the highest-priority runnable thread (caller holds gSchedLock).
// Returns gIdleThread if all queues are empty.
// ---------------------------------------------------------------------------
static THREAD* PickNext(void) {
for (int p = PriorityCount - 1; p >= 0; p--) {
if (!LIST_IS_EMPTY(&gRunQueue[p])) {
@@ -124,18 +99,15 @@ static THREAD* PickNext(void) {
return gIdleThread;
}
// ---------------------------------------------------------------------------
// KeSchedulerTick - called from the timer ISR every QUANTUM_MS
// ---------------------------------------------------------------------------
void KeSchedulerTick(void) {
THREAD* cur = gCurrentThread;
if (!cur) return;
KeAcquireSpinlock(&gSchedLock);
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags; // Issue 5 fix
cur->QuantaConsumed++;
// Demotion: CPU-bound thread exhausted its quanta budget.
if (cur != gIdleThread && cur->QuantaConsumed >= DEMOTION_QUANTA) {
cur->QuantaConsumed = 0;
if (cur->Priority > PriorityLow) {
@@ -143,51 +115,56 @@ void KeSchedulerTick(void) {
}
}
// Check if a higher-priority thread is waiting.
// Periodic aging pass to prevent indefinite starvation of PriorityLow threads
gAgingCounter++;
if (gAgingCounter >= 100) {
gAgingCounter = 0;
for (int p = 0; p < PriorityHigh; p++) {
while (!LIST_IS_EMPTY(&gRunQueue[p])) {
LIST_ENTRY* node = ListPopHead(&gRunQueue[p]);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
t->Priority = PriorityHigh;
ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink);
}
}
}
THREAD* next = PickNext();
if (next == cur) {
// Re-insert ourselves and continue.
if (cur != gIdleThread) {
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
}
KeReleaseSpinlock(&gSchedLock);
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
// Switch.
cur->State = (cur == gIdleThread) ? ThreadReady : ThreadReady;
cur->State = ThreadReady;
next->State = ThreadRunning;
gCurrentThread = next;
// Re-queue the outgoing thread if it is still runnable.
if (cur != gIdleThread && cur->State == ThreadReady) {
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
}
KeReleaseSpinlock(&gSchedLock);
gOutgoingThread = cur;
KeSwitchContext(cur, next);
KeSchedulerPostSwitch();
}
// ---------------------------------------------------------------------------
// KeYield
// ---------------------------------------------------------------------------
void KeYield(void) {
THREAD* cur = gCurrentThread;
if (!cur) return;
KeAcquireSpinlock(&gSchedLock);
// Voluntary block → promote one level.
if (cur != gIdleThread && cur->Priority < PriorityHigh) {
cur->Priority = (THREAD_PRIORITY)(cur->Priority + 1);
cur->QuantaConsumed = 0;
}
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
cur->SchedLockFlags = flags;
// Issue 4 Fix: Deleted misleading voluntary promotion from here.
THREAD* next = PickNext();
if (next == cur) {
if (cur != gIdleThread)
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
KeReleaseSpinlock(&gSchedLock);
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
@@ -197,6 +174,7 @@ void KeYield(void) {
if (cur != gIdleThread && cur->State == ThreadReady)
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
KeReleaseSpinlock(&gSchedLock);
gOutgoingThread = cur;
KeSwitchContext(cur, next);
KeSchedulerPostSwitch();
}
+9 -9
View File
@@ -25,40 +25,40 @@ KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
THREAD* self = KeCurrentThread();
for (;;) {
KeAcquireSpinlock(&sem->Lock);
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (sem->Count > 0) {
sem->Count--;
KeReleaseSpinlock(&sem->Lock);
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_SUCCESS;
}
ListInsertTail(&sem->WaitList, &self->SchedulerLink);
KeReleaseSpinlock(&sem->Lock);
KeReleaseSpinlock(&sem->Lock, flags);
KeBlockThread(self, &sem->Header);
}
}
KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
if (!sem) return STATUS_INVALID_PARAMETER;
KeAcquireSpinlock(&sem->Lock);
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (sem->Count > 0) {
sem->Count--;
KeReleaseSpinlock(&sem->Lock);
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&sem->Lock);
KeReleaseSpinlock(&sem->Lock, flags);
return STATUS_FAILURE;
}
void KeSignalSemaphore(SEMAPHORE* sem) {
if (!sem) return;
KeAcquireSpinlock(&sem->Lock);
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
if (!LIST_IS_EMPTY(&sem->WaitList)) {
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeReleaseSpinlock(&sem->Lock);
KeReleaseSpinlock(&sem->Lock, flags);
KeUnblockThread(waiter);
return;
}
if (sem->Count < sem->MaxCount) sem->Count++;
KeReleaseSpinlock(&sem->Lock);
KeReleaseSpinlock(&sem->Lock, flags);
}
+17 -4
View File
@@ -10,15 +10,28 @@ typedef struct SPINLOCK {
#define SPINLOCK_INIT { 0 }
static __forceinline void KeAcquireSpinlock(SPINLOCK* sl) {
// Issue 1 Fix: Save EFLAGS, disable interrupts, and return old flags.
static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) {
uintptr_t flags = __readeflags();
_disable();
while (_InterlockedCompareExchange(&sl->Locked, 1, 0) != 0)
_mm_pause();
return flags;
}
static __forceinline void KeReleaseSpinlock(SPINLOCK* sl) {
// Issue 1 Fix: Restore EFLAGS upon release.
static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t flags) {
_InterlockedExchange(&sl->Locked, 0);
__writeeflags(flags);
}
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl) {
return _InterlockedCompareExchange(&sl->Locked, 1, 0) == 0;
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outFlags) {
uintptr_t flags = __readeflags();
_disable();
if (_InterlockedCompareExchange(&sl->Locked, 1, 0) == 0) {
*outFlags = flags;
return 1;
}
__writeeflags(flags);
return 0;
}
+7 -2
View File
@@ -64,9 +64,14 @@ KeSwitchContext ENDP
;
; We load RCX from the stack (first arg per MS x64), then IRETQ.
; -------------------------------------------------------------------------
EXTERN KeSchedulerPostSwitch : PROC
KeThreadEntry PROC
pop rcx ; user arg
iretq ; pops RIP, CS, RFLAGS, RSP, SS -> ring 3
sub rsp, 32 ; Reserve shadow space
call KeSchedulerPostSwitch
add rsp, 32
pop rcx ; load user arg
iretq ; -> ring 3
KeThreadEntry ENDP
END
+27 -56
View File
@@ -7,44 +7,32 @@
#include <intrin.h>
static uint32_t gNextThreadId = 1;
// ---------------------------------------------------------------------------
// Initial kernel-side entry point for a new user thread
//
// When KeSwitchContext restores a brand-new thread, execution lands here.
// We set up RFLAGS (enable interrupts) and IRETQ into ring 3.
// ---------------------------------------------------------------------------
// Declared in ke_thread_entry.asm - see that file for the implementation.
extern void KeThreadEntry(void);
// ---------------------------------------------------------------------------
// Destructor
// ---------------------------------------------------------------------------
static void ThreadDestroy(OBJECT* obj) {
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
// Detach from owner process.
if (thread->Owner) {
KeAcquireSpinlock(&thread->Owner->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&thread->Owner->HandleLock);
ListRemove(&thread->ProcessLink);
thread->Owner->ThreadCount--;
KeReleaseSpinlock(&thread->Owner->HandleLock);
KeReleaseSpinlock(&thread->Owner->HandleLock, flags);
ObDereferenceObject(&thread->Owner->Header);
thread->Owner = (PROCESS *)0;
}
// Free kernel stack.
if (thread->KernelStackBase) {
MmFreePool(thread->KernelStackBase);
thread->KernelStackBase = (void *)0;
}
// User stack was allocated from user heap - it is up to the process to
// track and free it. For now we leave it (shared heap, no per-process VA).
// Assumes UserStackBase was added to THREAD struct in ke_thread.h.
if (thread->UserStackBase) {
MmFreeUserMemory(thread->UserStackBase, USTACK_SIZE);
thread->UserStackBase = (void *)0;
}
}
// ---------------------------------------------------------------------------
// KeCreateThread
// ---------------------------------------------------------------------------
KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD_PRIORITY priority, THREAD **out) {
if (!owner || !out) return STATUS_INVALID_PARAMETER;
@@ -53,14 +41,14 @@ KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
if (!thread) return STATUS_NO_MEMORY;
thread->Header.Destroy = ThreadDestroy;
thread->Header.Destroy = ThreadDestroy;
thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
thread->State = ThreadReady;
thread->Priority = priority;
thread->QuantaConsumed = 0;
thread->WaitObject = (OBJECT *)0;
thread->SchedLockFlags = 0x202ULL;
// Allocate kernel stack.
void* kstack = MmAllocatePool(KSTACK_SIZE);
if (!kstack) {
ObDereferenceObject(&thread->Header);
@@ -71,87 +59,70 @@ KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
thread->TssRsp0 = thread->KernelStackTop;
// Allocate user stack in the shared user heap.
void* ustack = MmAllocateUserMemory(USTACK_SIZE);
if (!ustack) {
MmFreePool(kstack);
ObDereferenceObject(&thread->Header);
return STATUS_NO_MEMORY;
}
thread->UserStackBase = ustack;
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE;
thread->UserEntry = entry;
thread->UserArg = arg;
// Set up the initial KCONTEXT so that when KeSwitchContext restores
// this thread, execution lands in KeThreadEntry (in ke_switch.asm),
// which will IRETQ into ring 3 at UserEntry.
// The KCONTEXT sits at the top of the kernel stack.
uintptr_t* ksp = (uintptr_t *)thread->KernelStackTop;
// Push user RSP so KeThreadEntry can find it.
*(--ksp) = userStackTop;
*(--ksp) = 0x23ULL;
*(--ksp) = userStackTop;
*(--ksp) = 0x202ULL;
*(--ksp) = 0x1BULL;
*(--ksp) = entry;
*(--ksp) = arg;
// Push the IRETQ frame: RIP, CS (ring3), RFLAGS, RSP, SS (ring3).
// Segment selector values: 0x1B = ring-3 code, 0x23 = ring-3 data.
*(--ksp) = 0x23ULL; // SS
*(--ksp) = userStackTop; // RSP (already set above but IRETQ needs it)
*(--ksp) = 0x202ULL; // RFLAGS: IF=1, reserved bit 1
*(--ksp) = 0x1BULL; // CS
*(--ksp) = entry; // RIP
// RCX = arg (first argument per MS x64 ABI).
*(--ksp) = arg; // RCX placeholder - KeThreadEntry loads it
// Now build the KCONTEXT that KeSwitchContext will restore.
// Rip points to KeThreadEntry; all other regs zero; RSP = ksp.
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
KCONTEXT* ctx = (KCONTEXT *)ksp;
memset(ctx, 0, sizeof(KCONTEXT));
ctx->Rip = (uint64_t)KeThreadEntry;
thread->SavedContext = ctx;
// Link into the owner process.
ObReferenceObject(&owner->Header);
thread->Owner = owner;
KeAcquireSpinlock(&owner->HandleLock);
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
ListInsertTail(&owner->ThreadList, &thread->ProcessLink);
owner->ThreadCount++;
KeReleaseSpinlock(&owner->HandleLock);
KeReleaseSpinlock(&owner->HandleLock, flags);
// Hand the thread to the scheduler.
KeReadyThread(thread);
*out = thread;
return STATUS_SUCCESS;
}
// ---------------------------------------------------------------------------
// KeExitThread
// ---------------------------------------------------------------------------
void KeExitThread(THREAD* thread, int code) {
(void)code;
if (!thread) return;
thread->State = ThreadDead;
KeUnreadyThread(thread);
ObDereferenceObject(&thread->Header);
// Yield - the scheduler will never pick this thread again.
KeYield();
}
// ---------------------------------------------------------------------------
// Block / unblock
// ---------------------------------------------------------------------------
void KeBlockThread(THREAD* thread, OBJECT* waitObject) {
thread->State = ThreadWaiting;
thread->WaitObject = waitObject;
KeUnreadyThread(thread);
KeYield();
}
void KeUnblockThread(THREAD* thread) {
thread->WaitObject = (OBJECT *)0;
thread->State = ThreadReady;
if (thread->Priority < PriorityHigh) {
thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1);
thread->QuantaConsumed = 0;
}
KeReadyThread(thread);
}
+3
View File
@@ -61,6 +61,9 @@ typedef struct THREAD {
// Ring-3 entry point and argument for newly created threads.
uintptr_t UserEntry;
uintptr_t UserArg;
uintptr_t SchedLockFlags; // Stores RFLAGS while thread is context switched out
void* UserStackBase; // Tracks user stack allocation to prevent memory leaks
} THREAD;
KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
+11 -11
View File
@@ -36,11 +36,11 @@ static FRAME_STACK gFrameStack;
// Push a physical frame address onto the stack.
static void PushFrame(PHYSICAL_ADDRESS frame) {
KeAcquireSpinlock(&gFrameStack.Lock);
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
if (gFrameStack.Top < gFrameStack.Capacity) {
gFrameStack.Stack[gFrameStack.Top++] = frame;
}
KeReleaseSpinlock(&gFrameStack.Lock);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
}
// ---------------------------------------------------------------------------
@@ -104,32 +104,32 @@ void MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
// Public API
// ---------------------------------------------------------------------------
PHYSICAL_ADDRESS MmAllocateFrame(void) {
KeAcquireSpinlock(&gFrameStack.Lock);
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
if (gFrameStack.Top == 0) {
KeReleaseSpinlock(&gFrameStack.Lock);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return (PHYSICAL_ADDRESS)0;
}
PHYSICAL_ADDRESS frame = gFrameStack.Stack[--gFrameStack.Top];
KeReleaseSpinlock(&gFrameStack.Lock);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return frame;
}
void MmFreeFrame(PHYSICAL_ADDRESS frame) {
if (!frame || (frame & (PAGE_SIZE - 1))) return;
KeAcquireSpinlock(&gFrameStack.Lock);
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
if (gFrameStack.Top < gFrameStack.Capacity)
gFrameStack.Stack[gFrameStack.Top++] = frame;
// else: stack full — leak rather than corrupt
KeReleaseSpinlock(&gFrameStack.Lock);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
}
PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
if (!count) return (PHYSICAL_ADDRESS)0;
KeAcquireSpinlock(&gFrameStack.Lock);
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
if (gFrameStack.Top < count) {
KeReleaseSpinlock(&gFrameStack.Lock);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return (PHYSICAL_ADDRESS)0;
}
@@ -152,7 +152,7 @@ PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
}
if (run == (size_t)-1) {
KeReleaseSpinlock(&gFrameStack.Lock);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return (PHYSICAL_ADDRESS)0;
}
@@ -164,7 +164,7 @@ PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
gFrameStack.Stack[run + i] = gFrameStack.Stack[run + count + i];
gFrameStack.Top -= count;
KeReleaseSpinlock(&gFrameStack.Lock);
KeReleaseSpinlock(&gFrameStack.Lock, flags);
return base;
}
+6 -6
View File
@@ -243,7 +243,7 @@ static void* HeapAlloc(HEAP* h, size_t size) {
if (!size) return (void *)0;
size = (size + 7) & ~7ULL; // 8-byte align
KeAcquireSpinlock(&h->Lock);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
// Best-fit search.
BLOCK_HDR* best = (BLOCK_HDR *)0;
@@ -256,14 +256,14 @@ static void* HeapAlloc(HEAP* h, size_t size) {
if (!best) {
if (!HeapGrow(h, size + OVERHEAD)) {
KeReleaseSpinlock(&h->Lock);
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
// Re-search; HeapGrow inserted at the front.
best = h->FreeList;
while (best && best->Size < size) best = best->NextFree;
if (!best) {
KeReleaseSpinlock(&h->Lock);
KeReleaseSpinlock(&h->Lock, flags);
return (void *)0;
}
}
@@ -292,7 +292,7 @@ static void* HeapAlloc(HEAP* h, size_t size) {
HdrToFtr(best)->Magic = BLOCK_MAGIC;
HdrToFtr(best)->Size = best->Size;
KeReleaseSpinlock(&h->Lock);
KeReleaseSpinlock(&h->Lock, flags);
return (char *)best + HDR_SIZE;
}
@@ -301,7 +301,7 @@ static void HeapFree(HEAP* h, void* ptr) {
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt
KeAcquireSpinlock(&h->Lock);
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
blk->Free = 1;
@@ -338,7 +338,7 @@ static void HeapFree(HEAP* h, void* ptr) {
HdrToFtr(blk)->Free = 1;
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
KeReleaseSpinlock(&h->Lock);
KeReleaseSpinlock(&h->Lock, flags);
}
static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {