fix a few bugs in scheduling and synchronization, separate OVMF_VARS for debug
This commit is contained in:
1 parent
5cf7a20ea2
commit
129be91e15
17 files changed
+174
-211
No files matched your search
Binary file not shown.
Binary file not shown.
@@ -23,7 +23,7 @@ static volatile uint32_t* gLapic = NULL;
|
|||||||
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
|
#define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off)))
|
||||||
|
|
||||||
static volatile uint64_t gTicks = 0;
|
static volatile uint64_t gTicks = 0;
|
||||||
void (*gTimerCallback)(void) = NULL;
|
void (* volatile gTimerCallback)(void) = NULL;
|
||||||
|
|
||||||
static inline void outb(uint16_t port, uint8_t val) { __outbyte(port, val); }
|
static inline void outb(uint16_t port, uint8_t val) { __outbyte(port, val); }
|
||||||
static inline uint8_t inb(uint16_t port) { return __inbyte(port); }
|
static inline uint8_t inb(uint16_t port) { return __inbyte(port); }
|
||||||
|
|||||||
+10
-12
@@ -21,37 +21,35 @@ KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
|
|||||||
|
|
||||||
void KeSignalEvent(EVENT* event) {
|
void KeSignalEvent(EVENT* event) {
|
||||||
if (!event) return;
|
if (!event) return;
|
||||||
KeAcquireSpinlock(&event->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
|
||||||
|
|
||||||
if (event->Type == EventManualReset) {
|
if (event->Type == EventManualReset) {
|
||||||
// Wake all waiters.
|
|
||||||
event->Signaled = 1;
|
event->Signaled = 1;
|
||||||
while (!LIST_IS_EMPTY(&event->WaitList)) {
|
while (!LIST_IS_EMPTY(&event->WaitList)) {
|
||||||
LIST_ENTRY* node = ListPopHead(&event->WaitList);
|
LIST_ENTRY* node = ListPopHead(&event->WaitList);
|
||||||
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||||
KeReleaseSpinlock(&event->Lock);
|
KeReleaseSpinlock(&event->Lock, flags);
|
||||||
KeUnblockThread(t);
|
KeUnblockThread(t);
|
||||||
KeAcquireSpinlock(&event->Lock);
|
flags = KeAcquireSpinlock(&event->Lock);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// Auto-reset: wake one waiter or set the flag for the next waiter.
|
|
||||||
if (!LIST_IS_EMPTY(&event->WaitList)) {
|
if (!LIST_IS_EMPTY(&event->WaitList)) {
|
||||||
LIST_ENTRY* node = ListPopHead(&event->WaitList);
|
LIST_ENTRY* node = ListPopHead(&event->WaitList);
|
||||||
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||||
KeReleaseSpinlock(&event->Lock);
|
KeReleaseSpinlock(&event->Lock, flags);
|
||||||
KeUnblockThread(t);
|
KeUnblockThread(t);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
event->Signaled = 1;
|
event->Signaled = 1;
|
||||||
}
|
}
|
||||||
KeReleaseSpinlock(&event->Lock);
|
KeReleaseSpinlock(&event->Lock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
void KeClearEvent(EVENT* event) {
|
void KeClearEvent(EVENT* event) {
|
||||||
if (!event) return;
|
if (!event) return;
|
||||||
KeAcquireSpinlock(&event->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
|
||||||
event->Signaled = 0;
|
event->Signaled = 0;
|
||||||
KeReleaseSpinlock(&event->Lock);
|
KeReleaseSpinlock(&event->Lock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
KSTATUS KeWaitForEvent(EVENT* event) {
|
KSTATUS KeWaitForEvent(EVENT* event) {
|
||||||
@@ -59,14 +57,14 @@ KSTATUS KeWaitForEvent(EVENT* event) {
|
|||||||
THREAD* self = KeCurrentThread();
|
THREAD* self = KeCurrentThread();
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
KeAcquireSpinlock(&event->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
|
||||||
if (event->Signaled) {
|
if (event->Signaled) {
|
||||||
if (event->Type == EventAutoReset) event->Signaled = 0;
|
if (event->Type == EventAutoReset) event->Signaled = 0;
|
||||||
KeReleaseSpinlock(&event->Lock);
|
KeReleaseSpinlock(&event->Lock, flags);
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
ListInsertTail(&event->WaitList, &self->SchedulerLink);
|
ListInsertTail(&event->WaitList, &self->SchedulerLink);
|
||||||
KeReleaseSpinlock(&event->Lock);
|
KeReleaseSpinlock(&event->Lock, flags);
|
||||||
KeBlockThread(self, &event->Header);
|
KeBlockThread(self, &event->Header);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -14,12 +14,11 @@ void KeIrqInitialize(const HAL_API *hal) {
|
|||||||
void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
|
void KeIrqRegister(uint8_t irq, void (*handler)(void)) {
|
||||||
if (irq >= MAX_IRQS || !handler) return;
|
if (irq >= MAX_IRQS || !handler) return;
|
||||||
gIrqHandlers[irq] = handler;
|
gIrqHandlers[irq] = handler;
|
||||||
gHal->Intc.RouteIrq(irq); // programs IOAPIC, clears mask bit
|
gHal->Intc.RouteIrq(irq);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Called from KeIrqCommon in assembly - this is the only symbol
|
|
||||||
// the assembly needs to know.
|
|
||||||
void KeIrqDispatch(uint8_t irq) {
|
void KeIrqDispatch(uint8_t irq) {
|
||||||
|
// eoi first, handler second because it doesn't work otherwise.
|
||||||
gHal->Intc.SendEoi();
|
gHal->Intc.SendEoi();
|
||||||
if (irq < MAX_IRQS && gIrqHandlers[irq])
|
if (irq < MAX_IRQS && gIrqHandlers[irq])
|
||||||
gIrqHandlers[irq]();
|
gIrqHandlers[irq]();
|
||||||
|
|||||||
+9
-13
@@ -24,46 +24,42 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) {
|
|||||||
THREAD* self = KeCurrentThread();
|
THREAD* self = KeCurrentThread();
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
KeAcquireSpinlock(&mutex->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
|
||||||
if (!mutex->Owner) {
|
if (!mutex->Owner) {
|
||||||
mutex->Owner = self;
|
mutex->Owner = self;
|
||||||
KeReleaseSpinlock(&mutex->Lock);
|
KeReleaseSpinlock(&mutex->Lock, flags);
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
// Already owned - add self to wait list and block.
|
|
||||||
ListInsertTail(&mutex->WaitList, &self->SchedulerLink);
|
ListInsertTail(&mutex->WaitList, &self->SchedulerLink);
|
||||||
KeReleaseSpinlock(&mutex->Lock);
|
KeReleaseSpinlock(&mutex->Lock, flags);
|
||||||
KeBlockThread(self, &mutex->Header);
|
KeBlockThread(self, &mutex->Header);
|
||||||
// KeBlockThread calls KeYield; when we return here we've been
|
|
||||||
// unblocked by KeReleaseMutex - loop back and try to own it.
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
|
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
|
||||||
if (!mutex) return STATUS_INVALID_PARAMETER;
|
if (!mutex) return STATUS_INVALID_PARAMETER;
|
||||||
THREAD* self = KeCurrentThread();
|
THREAD* self = KeCurrentThread();
|
||||||
KeAcquireSpinlock(&mutex->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
|
||||||
if (!mutex->Owner) {
|
if (!mutex->Owner) {
|
||||||
mutex->Owner = self;
|
mutex->Owner = self;
|
||||||
KeReleaseSpinlock(&mutex->Lock);
|
KeReleaseSpinlock(&mutex->Lock, flags);
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
KeReleaseSpinlock(&mutex->Lock);
|
KeReleaseSpinlock(&mutex->Lock, flags);
|
||||||
return STATUS_FAILURE;
|
return STATUS_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
||||||
void KeReleaseMutex(MUTEX* mutex) {
|
void KeReleaseMutex(MUTEX* mutex) {
|
||||||
if (!mutex) return;
|
if (!mutex) return;
|
||||||
KeAcquireSpinlock(&mutex->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
|
||||||
mutex->Owner = (struct THREAD *)0;
|
mutex->Owner = (struct THREAD *)0;
|
||||||
|
|
||||||
// Wake the first waiter, if any.
|
|
||||||
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
|
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
|
||||||
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
|
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
|
||||||
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||||
KeReleaseSpinlock(&mutex->Lock);
|
KeReleaseSpinlock(&mutex->Lock, flags);
|
||||||
KeUnblockThread(waiter);
|
KeUnblockThread(waiter);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
KeReleaseSpinlock(&mutex->Lock);
|
KeReleaseSpinlock(&mutex->Lock, flags);
|
||||||
}
|
}
|
||||||
+13
-13
@@ -1,6 +1,6 @@
|
|||||||
// ke_process.c - process model
|
// ke_process.c - process model
|
||||||
#include "ke_process.h"
|
#include "ke_process.h"
|
||||||
#include "ke_thread.h" // KeExitThread (forward declared below)
|
#include "ke_thread.h"
|
||||||
#include "mm_virt.h"
|
#include "mm_virt.h"
|
||||||
#include "ob_object.h"
|
#include "ob_object.h"
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
@@ -41,9 +41,9 @@ static void ProcessDestroy(OBJECT* obj) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Remove from global list.
|
// Remove from global list.
|
||||||
KeAcquireSpinlock(&gProcessListLock);
|
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
|
||||||
ListRemove(&proc->GlobalLink);
|
ListRemove(&proc->GlobalLink);
|
||||||
KeReleaseSpinlock(&gProcessListLock);
|
KeReleaseSpinlock(&gProcessListLock, flags);
|
||||||
|
|
||||||
// Memory for the struct itself is freed by ObDereferenceObject after
|
// Memory for the struct itself is freed by ObDereferenceObject after
|
||||||
// Destroy returns.
|
// Destroy returns.
|
||||||
@@ -76,9 +76,9 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
|
|||||||
|
|
||||||
LIST_INIT(&proc->GlobalLink);
|
LIST_INIT(&proc->GlobalLink);
|
||||||
|
|
||||||
KeAcquireSpinlock(&gProcessListLock);
|
uintptr_t flags = KeAcquireSpinlock(&gProcessListLock);
|
||||||
ListInsertTail(&gProcessList, &proc->GlobalLink);
|
ListInsertTail(&gProcessList, &proc->GlobalLink);
|
||||||
KeReleaseSpinlock(&gProcessListLock);
|
KeReleaseSpinlock(&gProcessListLock, flags);
|
||||||
|
|
||||||
*out = proc;
|
*out = proc;
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
@@ -101,43 +101,43 @@ void KeExitProcess(PROCESS* process, int exitCode) {
|
|||||||
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut) {
|
KSTATUS KeAllocateHandle(PROCESS* process, OBJECT* object, uint32_t* handleOut) {
|
||||||
if (!process || !object || !handleOut) return STATUS_INVALID_PARAMETER;
|
if (!process || !object || !handleOut) return STATUS_INVALID_PARAMETER;
|
||||||
|
|
||||||
KeAcquireSpinlock(&process->HandleLock);
|
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
|
||||||
|
|
||||||
for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
|
for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
|
||||||
if (!process->Handles[i]) {
|
if (!process->Handles[i]) {
|
||||||
ObReferenceObject(object);
|
ObReferenceObject(object);
|
||||||
process->Handles[i] = object;
|
process->Handles[i] = object;
|
||||||
*handleOut = (uint32_t)i;
|
*handleOut = (uint32_t)i;
|
||||||
KeReleaseSpinlock(&process->HandleLock);
|
KeReleaseSpinlock(&process->HandleLock, flags);
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
KeReleaseSpinlock(&process->HandleLock);
|
KeReleaseSpinlock(&process->HandleLock, flags);
|
||||||
return STATUS_NO_MEMORY;
|
return STATUS_NO_MEMORY;
|
||||||
}
|
}
|
||||||
|
|
||||||
void KeCloseHandle(PROCESS* process, uint32_t handle) {
|
void KeCloseHandle(PROCESS* process, uint32_t handle) {
|
||||||
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return;
|
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE) return;
|
||||||
|
|
||||||
KeAcquireSpinlock(&process->HandleLock);
|
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
|
||||||
OBJECT* obj = process->Handles[handle];
|
OBJECT* obj = process->Handles[handle];
|
||||||
if (obj) {
|
if (obj) {
|
||||||
process->Handles[handle] = (OBJECT *)0;
|
process->Handles[handle] = (OBJECT *)0;
|
||||||
KeReleaseSpinlock(&process->HandleLock);
|
KeReleaseSpinlock(&process->HandleLock, flags);
|
||||||
ObDereferenceObject(obj);
|
ObDereferenceObject(obj);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
KeReleaseSpinlock(&process->HandleLock);
|
KeReleaseSpinlock(&process->HandleLock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) {
|
OBJECT* KeLookupHandle(PROCESS* process, uint32_t handle) {
|
||||||
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE)
|
if (!process || handle == 0 || handle >= HANDLE_TABLE_SIZE)
|
||||||
return (OBJECT *)0;
|
return (OBJECT *)0;
|
||||||
|
|
||||||
KeAcquireSpinlock(&process->HandleLock);
|
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
|
||||||
OBJECT* obj = process->Handles[handle];
|
OBJECT* obj = process->Handles[handle];
|
||||||
if (obj) ObReferenceObject(obj);
|
if (obj) ObReferenceObject(obj);
|
||||||
KeReleaseSpinlock(&process->HandleLock);
|
KeReleaseSpinlock(&process->HandleLock, flags);
|
||||||
return obj;
|
return obj;
|
||||||
}
|
}
|
||||||
+10
-10
@@ -25,7 +25,7 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
|
|||||||
THREAD* self = KeCurrentThread();
|
THREAD* self = KeCurrentThread();
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
KeAcquireSpinlock(&queue->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
|
||||||
if (queue->Count < QUEUE_DEPTH) {
|
if (queue->Count < QUEUE_DEPTH) {
|
||||||
queue->Ring[queue->Tail] = *msg;
|
queue->Ring[queue->Tail] = *msg;
|
||||||
queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH;
|
queue->Tail = (queue->Tail + 1) % QUEUE_DEPTH;
|
||||||
@@ -35,19 +35,19 @@ KSTATUS KeSendMessage(QUEUE* queue, const QUEUE_MESSAGE* msg, bool blocking) {
|
|||||||
if (!LIST_IS_EMPTY(&queue->RecvWaiters)) {
|
if (!LIST_IS_EMPTY(&queue->RecvWaiters)) {
|
||||||
LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters);
|
LIST_ENTRY* node = ListPopHead(&queue->RecvWaiters);
|
||||||
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
KeUnblockThread(t);
|
KeUnblockThread(t);
|
||||||
} else {
|
} else {
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
}
|
}
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
if (!blocking) {
|
if (!blocking) {
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
return STATUS_QUEUE_FULL;
|
return STATUS_QUEUE_FULL;
|
||||||
}
|
}
|
||||||
ListInsertTail(&queue->SendWaiters, &self->SchedulerLink);
|
ListInsertTail(&queue->SendWaiters, &self->SchedulerLink);
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
KeBlockThread(self, &queue->Header);
|
KeBlockThread(self, &queue->Header);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -57,7 +57,7 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
|
|||||||
THREAD* self = KeCurrentThread();
|
THREAD* self = KeCurrentThread();
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
KeAcquireSpinlock(&queue->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&queue->Lock);
|
||||||
if (queue->Count > 0) {
|
if (queue->Count > 0) {
|
||||||
*msg = queue->Ring[queue->Head];
|
*msg = queue->Ring[queue->Head];
|
||||||
queue->Head = (queue->Head + 1) % QUEUE_DEPTH;
|
queue->Head = (queue->Head + 1) % QUEUE_DEPTH;
|
||||||
@@ -67,19 +67,19 @@ KSTATUS KeReceiveMessage(QUEUE* queue, QUEUE_MESSAGE* msg, bool blocking) {
|
|||||||
if (!LIST_IS_EMPTY(&queue->SendWaiters)) {
|
if (!LIST_IS_EMPTY(&queue->SendWaiters)) {
|
||||||
LIST_ENTRY* node = ListPopHead(&queue->SendWaiters);
|
LIST_ENTRY* node = ListPopHead(&queue->SendWaiters);
|
||||||
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
KeUnblockThread(t);
|
KeUnblockThread(t);
|
||||||
} else {
|
} else {
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
}
|
}
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
if (!blocking) {
|
if (!blocking) {
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
return STATUS_QUEUE_EMPTY;
|
return STATUS_QUEUE_EMPTY;
|
||||||
}
|
}
|
||||||
ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink);
|
ListInsertTail(&queue->RecvWaiters, &self->SchedulerLink);
|
||||||
KeReleaseSpinlock(&queue->Lock);
|
KeReleaseSpinlock(&queue->Lock, flags);
|
||||||
KeBlockThread(self, &queue->Header);
|
KeBlockThread(self, &queue->Header);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+48
-70
@@ -1,15 +1,4 @@
|
|||||||
// ke_sched.c - MLFQ scheduler
|
// ke_sched.c - MLFQ scheduler
|
||||||
//
|
|
||||||
// Three queues: PriorityHigh(2), PriorityNormal(1), PriorityLow(0).
|
|
||||||
// Dispatch always picks the highest non-empty queue.
|
|
||||||
// Within a queue: strict round-robin.
|
|
||||||
//
|
|
||||||
// Demotion threshold: a thread that consumes DEMOTION_QUANTA consecutive
|
|
||||||
// ticks without blocking drops one level (floor PriorityLow).
|
|
||||||
// Promotion: a thread that unblocks is bumped one level (ceiling PriorityHigh).
|
|
||||||
//
|
|
||||||
// The idle thread (created by KeInitializeScheduler) runs at PriorityLow
|
|
||||||
// when nothing else is runnable.
|
|
||||||
#include "ke_sched.h"
|
#include "ke_sched.h"
|
||||||
#include "ke_thread.h"
|
#include "ke_thread.h"
|
||||||
#include "ke_spinlock.h"
|
#include "ke_spinlock.h"
|
||||||
@@ -17,44 +6,27 @@
|
|||||||
#include "mm_virt.h"
|
#include "mm_virt.h"
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
|
|
||||||
#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */
|
#define DEMOTION_QUANTA 4 /* ticks at a level before demotion */
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
static LIST_ENTRY gRunQueue[PriorityCount];
|
||||||
// Run queues
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
static LIST_ENTRY gRunQueue[PriorityCount]; // [0]=Low [1]=Normal [2]=High
|
|
||||||
static SPINLOCK gSchedLock;
|
static SPINLOCK gSchedLock;
|
||||||
|
|
||||||
// The currently running thread (also in GS:[0] for fast access).
|
|
||||||
// On a single-core system this is the only "per-CPU" slot we need.
|
|
||||||
static THREAD* gCurrentThread;
|
static THREAD* gCurrentThread;
|
||||||
static THREAD* gIdleThread;
|
static THREAD* gIdleThread;
|
||||||
|
static THREAD* gOutgoingThread;
|
||||||
|
static uint32_t gAgingCounter = 0;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// GS:[0] helpers - implemented in ke_switch.asm
|
|
||||||
// (On single-core we just use a global; SMP would use per-core GS.)
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
THREAD* KeCurrentThread(void) { return gCurrentThread; }
|
THREAD* KeCurrentThread(void) { return gCurrentThread; }
|
||||||
void KeSetCurrentThread(THREAD* t){ gCurrentThread = t; }
|
void KeSetCurrentThread(THREAD* t){ gCurrentThread = t; }
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Idle thread body - runs in ring 0 when every other thread is blocked.
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
static void IdleLoop(void) {
|
static void IdleLoop(void) {
|
||||||
for (;;) { __halt(); }
|
for (;;) { __halt(); }
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Init
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
void KeInitializeScheduler(void) {
|
void KeInitializeScheduler(void) {
|
||||||
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
|
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
|
||||||
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
|
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
|
||||||
|
|
||||||
// Create the idle process / thread.
|
|
||||||
// We call the internal alloc directly to avoid the full KeCreateThread
|
|
||||||
// path (no user stack, no ring-3 entry - the idle thread never leaves
|
|
||||||
// ring 0 and never calls KeExitThread).
|
|
||||||
extern KSTATUS KeCreateProcess(const char *, PROCESS **);
|
extern KSTATUS KeCreateProcess(const char *, PROCESS **);
|
||||||
PROCESS* idleProc;
|
PROCESS* idleProc;
|
||||||
KeCreateProcess("idle", &idleProc);
|
KeCreateProcess("idle", &idleProc);
|
||||||
@@ -67,13 +39,13 @@ void KeInitializeScheduler(void) {
|
|||||||
idle->QuantaConsumed = 0;
|
idle->QuantaConsumed = 0;
|
||||||
idle->Owner = idleProc;
|
idle->Owner = idleProc;
|
||||||
idle->WaitObject = (OBJECT *)0;
|
idle->WaitObject = (OBJECT *)0;
|
||||||
|
idle->SchedLockFlags = 0x202ULL; // IF=1
|
||||||
|
|
||||||
void* kstack = MmAllocatePool(KSTACK_SIZE);
|
void* kstack = MmAllocatePool(KSTACK_SIZE);
|
||||||
idle->KernelStackBase = kstack;
|
idle->KernelStackBase = kstack;
|
||||||
idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
|
idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
|
||||||
idle->TssRsp0 = idle->KernelStackTop;
|
idle->TssRsp0 = idle->KernelStackTop;
|
||||||
|
|
||||||
// Initial context: RSP at top of kstack, Rip = IdleLoop.
|
|
||||||
uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop;
|
uintptr_t* ksp = (uintptr_t *)idle->KernelStackTop;
|
||||||
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
|
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
|
||||||
KCONTEXT* ctx = (KCONTEXT *)ksp;
|
KCONTEXT* ctx = (KCONTEXT *)ksp;
|
||||||
@@ -83,13 +55,20 @@ void KeInitializeScheduler(void) {
|
|||||||
|
|
||||||
gIdleThread = idle;
|
gIdleThread = idle;
|
||||||
gCurrentThread = idle;
|
gCurrentThread = idle;
|
||||||
|
|
||||||
// Don't add idle to the run queue here; KeSchedulerTick falls back to it.
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// Safely release the lock and clean up any dead threads natively post-switch
|
||||||
// Run-queue management
|
void KeSchedulerPostSwitch(void) {
|
||||||
// ---------------------------------------------------------------------------
|
THREAD* cur = KeCurrentThread();
|
||||||
|
KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags);
|
||||||
|
|
||||||
|
if (gOutgoingThread && gOutgoingThread->State == ThreadDead) {
|
||||||
|
THREAD* dead = gOutgoingThread;
|
||||||
|
gOutgoingThread = (THREAD*)0;
|
||||||
|
ObDereferenceObject(&dead->Header);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
void KeReadyThread(THREAD* thread) {
|
void KeReadyThread(THREAD* thread) {
|
||||||
if (!thread) return;
|
if (!thread) return;
|
||||||
thread->State = ThreadReady;
|
thread->State = ThreadReady;
|
||||||
@@ -97,22 +76,18 @@ void KeReadyThread(THREAD* thread) {
|
|||||||
if (p < 0) p = 0;
|
if (p < 0) p = 0;
|
||||||
if (p >= PriorityCount) p = PriorityCount - 1;
|
if (p >= PriorityCount) p = PriorityCount - 1;
|
||||||
|
|
||||||
KeAcquireSpinlock(&gSchedLock);
|
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
|
||||||
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
|
ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
|
||||||
KeReleaseSpinlock(&gSchedLock);
|
KeReleaseSpinlock(&gSchedLock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
void KeUnreadyThread(THREAD* thread) {
|
void KeUnreadyThread(THREAD* thread) {
|
||||||
if (!thread) return;
|
if (!thread) return;
|
||||||
KeAcquireSpinlock(&gSchedLock);
|
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
|
||||||
ListRemove(&thread->SchedulerLink);
|
ListRemove(&thread->SchedulerLink);
|
||||||
KeReleaseSpinlock(&gSchedLock);
|
KeReleaseSpinlock(&gSchedLock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Pick the highest-priority runnable thread (caller holds gSchedLock).
|
|
||||||
// Returns gIdleThread if all queues are empty.
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
static THREAD* PickNext(void) {
|
static THREAD* PickNext(void) {
|
||||||
for (int p = PriorityCount - 1; p >= 0; p--) {
|
for (int p = PriorityCount - 1; p >= 0; p--) {
|
||||||
if (!LIST_IS_EMPTY(&gRunQueue[p])) {
|
if (!LIST_IS_EMPTY(&gRunQueue[p])) {
|
||||||
@@ -124,18 +99,15 @@ static THREAD* PickNext(void) {
|
|||||||
return gIdleThread;
|
return gIdleThread;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// KeSchedulerTick - called from the timer ISR every QUANTUM_MS
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
void KeSchedulerTick(void) {
|
void KeSchedulerTick(void) {
|
||||||
THREAD* cur = gCurrentThread;
|
THREAD* cur = gCurrentThread;
|
||||||
if (!cur) return;
|
if (!cur) return;
|
||||||
|
|
||||||
KeAcquireSpinlock(&gSchedLock);
|
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
|
||||||
|
cur->SchedLockFlags = flags; // Issue 5 fix
|
||||||
|
|
||||||
cur->QuantaConsumed++;
|
cur->QuantaConsumed++;
|
||||||
|
|
||||||
// Demotion: CPU-bound thread exhausted its quanta budget.
|
|
||||||
if (cur != gIdleThread && cur->QuantaConsumed >= DEMOTION_QUANTA) {
|
if (cur != gIdleThread && cur->QuantaConsumed >= DEMOTION_QUANTA) {
|
||||||
cur->QuantaConsumed = 0;
|
cur->QuantaConsumed = 0;
|
||||||
if (cur->Priority > PriorityLow) {
|
if (cur->Priority > PriorityLow) {
|
||||||
@@ -143,51 +115,56 @@ void KeSchedulerTick(void) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if a higher-priority thread is waiting.
|
// Periodic aging pass to prevent indefinite starvation of PriorityLow threads
|
||||||
|
gAgingCounter++;
|
||||||
|
if (gAgingCounter >= 100) {
|
||||||
|
gAgingCounter = 0;
|
||||||
|
for (int p = 0; p < PriorityHigh; p++) {
|
||||||
|
while (!LIST_IS_EMPTY(&gRunQueue[p])) {
|
||||||
|
LIST_ENTRY* node = ListPopHead(&gRunQueue[p]);
|
||||||
|
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||||
|
t->Priority = PriorityHigh;
|
||||||
|
ListInsertTail(&gRunQueue[PriorityHigh], &t->SchedulerLink);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
THREAD* next = PickNext();
|
THREAD* next = PickNext();
|
||||||
if (next == cur) {
|
if (next == cur) {
|
||||||
// Re-insert ourselves and continue.
|
|
||||||
if (cur != gIdleThread) {
|
if (cur != gIdleThread) {
|
||||||
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
||||||
}
|
}
|
||||||
KeReleaseSpinlock(&gSchedLock);
|
KeReleaseSpinlock(&gSchedLock, flags);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Switch.
|
cur->State = ThreadReady;
|
||||||
cur->State = (cur == gIdleThread) ? ThreadReady : ThreadReady;
|
|
||||||
next->State = ThreadRunning;
|
next->State = ThreadRunning;
|
||||||
gCurrentThread = next;
|
gCurrentThread = next;
|
||||||
|
|
||||||
// Re-queue the outgoing thread if it is still runnable.
|
|
||||||
if (cur != gIdleThread && cur->State == ThreadReady) {
|
if (cur != gIdleThread && cur->State == ThreadReady) {
|
||||||
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
||||||
}
|
}
|
||||||
|
|
||||||
KeReleaseSpinlock(&gSchedLock);
|
gOutgoingThread = cur;
|
||||||
KeSwitchContext(cur, next);
|
KeSwitchContext(cur, next);
|
||||||
|
KeSchedulerPostSwitch();
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// KeYield
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
void KeYield(void) {
|
void KeYield(void) {
|
||||||
THREAD* cur = gCurrentThread;
|
THREAD* cur = gCurrentThread;
|
||||||
if (!cur) return;
|
if (!cur) return;
|
||||||
|
|
||||||
KeAcquireSpinlock(&gSchedLock);
|
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
|
||||||
|
cur->SchedLockFlags = flags;
|
||||||
// Voluntary block → promote one level.
|
|
||||||
if (cur != gIdleThread && cur->Priority < PriorityHigh) {
|
|
||||||
cur->Priority = (THREAD_PRIORITY)(cur->Priority + 1);
|
|
||||||
cur->QuantaConsumed = 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
// Issue 4 Fix: Deleted misleading voluntary promotion from here.
|
||||||
|
|
||||||
THREAD* next = PickNext();
|
THREAD* next = PickNext();
|
||||||
if (next == cur) {
|
if (next == cur) {
|
||||||
if (cur != gIdleThread)
|
if (cur != gIdleThread)
|
||||||
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
||||||
KeReleaseSpinlock(&gSchedLock);
|
KeReleaseSpinlock(&gSchedLock, flags);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,6 +174,7 @@ void KeYield(void) {
|
|||||||
if (cur != gIdleThread && cur->State == ThreadReady)
|
if (cur != gIdleThread && cur->State == ThreadReady)
|
||||||
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
|
||||||
|
|
||||||
KeReleaseSpinlock(&gSchedLock);
|
gOutgoingThread = cur;
|
||||||
KeSwitchContext(cur, next);
|
KeSwitchContext(cur, next);
|
||||||
|
KeSchedulerPostSwitch();
|
||||||
}
|
}
|
||||||
@@ -25,40 +25,40 @@ KSTATUS KeWaitSemaphore(SEMAPHORE* sem) {
|
|||||||
THREAD* self = KeCurrentThread();
|
THREAD* self = KeCurrentThread();
|
||||||
|
|
||||||
for (;;) {
|
for (;;) {
|
||||||
KeAcquireSpinlock(&sem->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
|
||||||
if (sem->Count > 0) {
|
if (sem->Count > 0) {
|
||||||
sem->Count--;
|
sem->Count--;
|
||||||
KeReleaseSpinlock(&sem->Lock);
|
KeReleaseSpinlock(&sem->Lock, flags);
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
ListInsertTail(&sem->WaitList, &self->SchedulerLink);
|
ListInsertTail(&sem->WaitList, &self->SchedulerLink);
|
||||||
KeReleaseSpinlock(&sem->Lock);
|
KeReleaseSpinlock(&sem->Lock, flags);
|
||||||
KeBlockThread(self, &sem->Header);
|
KeBlockThread(self, &sem->Header);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
|
KSTATUS KeTrySemaphore(SEMAPHORE* sem) {
|
||||||
if (!sem) return STATUS_INVALID_PARAMETER;
|
if (!sem) return STATUS_INVALID_PARAMETER;
|
||||||
KeAcquireSpinlock(&sem->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
|
||||||
if (sem->Count > 0) {
|
if (sem->Count > 0) {
|
||||||
sem->Count--;
|
sem->Count--;
|
||||||
KeReleaseSpinlock(&sem->Lock);
|
KeReleaseSpinlock(&sem->Lock, flags);
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
KeReleaseSpinlock(&sem->Lock);
|
KeReleaseSpinlock(&sem->Lock, flags);
|
||||||
return STATUS_FAILURE;
|
return STATUS_FAILURE;
|
||||||
}
|
}
|
||||||
|
|
||||||
void KeSignalSemaphore(SEMAPHORE* sem) {
|
void KeSignalSemaphore(SEMAPHORE* sem) {
|
||||||
if (!sem) return;
|
if (!sem) return;
|
||||||
KeAcquireSpinlock(&sem->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&sem->Lock);
|
||||||
if (!LIST_IS_EMPTY(&sem->WaitList)) {
|
if (!LIST_IS_EMPTY(&sem->WaitList)) {
|
||||||
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
|
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
|
||||||
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
|
||||||
KeReleaseSpinlock(&sem->Lock);
|
KeReleaseSpinlock(&sem->Lock, flags);
|
||||||
KeUnblockThread(waiter);
|
KeUnblockThread(waiter);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (sem->Count < sem->MaxCount) sem->Count++;
|
if (sem->Count < sem->MaxCount) sem->Count++;
|
||||||
KeReleaseSpinlock(&sem->Lock);
|
KeReleaseSpinlock(&sem->Lock, flags);
|
||||||
}
|
}
|
||||||
@@ -10,15 +10,28 @@ typedef struct SPINLOCK {
|
|||||||
|
|
||||||
#define SPINLOCK_INIT { 0 }
|
#define SPINLOCK_INIT { 0 }
|
||||||
|
|
||||||
static __forceinline void KeAcquireSpinlock(SPINLOCK* sl) {
|
// Issue 1 Fix: Save EFLAGS, disable interrupts, and return old flags.
|
||||||
|
static __forceinline uintptr_t KeAcquireSpinlock(SPINLOCK* sl) {
|
||||||
|
uintptr_t flags = __readeflags();
|
||||||
|
_disable();
|
||||||
while (_InterlockedCompareExchange(&sl->Locked, 1, 0) != 0)
|
while (_InterlockedCompareExchange(&sl->Locked, 1, 0) != 0)
|
||||||
_mm_pause();
|
_mm_pause();
|
||||||
|
return flags;
|
||||||
}
|
}
|
||||||
|
|
||||||
static __forceinline void KeReleaseSpinlock(SPINLOCK* sl) {
|
// Issue 1 Fix: Restore EFLAGS upon release.
|
||||||
|
static __forceinline void KeReleaseSpinlock(SPINLOCK* sl, uintptr_t flags) {
|
||||||
_InterlockedExchange(&sl->Locked, 0);
|
_InterlockedExchange(&sl->Locked, 0);
|
||||||
|
__writeeflags(flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl) {
|
static __forceinline int KeTryAcquireSpinlock(SPINLOCK* sl, uintptr_t* outFlags) {
|
||||||
return _InterlockedCompareExchange(&sl->Locked, 1, 0) == 0;
|
uintptr_t flags = __readeflags();
|
||||||
|
_disable();
|
||||||
|
if (_InterlockedCompareExchange(&sl->Locked, 1, 0) == 0) {
|
||||||
|
*outFlags = flags;
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
__writeeflags(flags);
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
@@ -64,9 +64,14 @@ KeSwitchContext ENDP
|
|||||||
;
|
;
|
||||||
; We load RCX from the stack (first arg per MS x64), then IRETQ.
|
; We load RCX from the stack (first arg per MS x64), then IRETQ.
|
||||||
; -------------------------------------------------------------------------
|
; -------------------------------------------------------------------------
|
||||||
|
EXTERN KeSchedulerPostSwitch : PROC
|
||||||
|
|
||||||
KeThreadEntry PROC
|
KeThreadEntry PROC
|
||||||
pop rcx ; user arg
|
sub rsp, 32 ; Reserve shadow space
|
||||||
iretq ; pops RIP, CS, RFLAGS, RSP, SS -> ring 3
|
call KeSchedulerPostSwitch
|
||||||
|
add rsp, 32
|
||||||
|
pop rcx ; load user arg
|
||||||
|
iretq ; -> ring 3
|
||||||
KeThreadEntry ENDP
|
KeThreadEntry ENDP
|
||||||
|
|
||||||
END
|
END
|
||||||
+27
-56
@@ -7,44 +7,32 @@
|
|||||||
#include <intrin.h>
|
#include <intrin.h>
|
||||||
|
|
||||||
static uint32_t gNextThreadId = 1;
|
static uint32_t gNextThreadId = 1;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Initial kernel-side entry point for a new user thread
|
|
||||||
//
|
|
||||||
// When KeSwitchContext restores a brand-new thread, execution lands here.
|
|
||||||
// We set up RFLAGS (enable interrupts) and IRETQ into ring 3.
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Declared in ke_thread_entry.asm - see that file for the implementation.
|
|
||||||
extern void KeThreadEntry(void);
|
extern void KeThreadEntry(void);
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Destructor
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
static void ThreadDestroy(OBJECT* obj) {
|
static void ThreadDestroy(OBJECT* obj) {
|
||||||
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
|
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
|
||||||
|
|
||||||
// Detach from owner process.
|
|
||||||
if (thread->Owner) {
|
if (thread->Owner) {
|
||||||
KeAcquireSpinlock(&thread->Owner->HandleLock);
|
uintptr_t flags = KeAcquireSpinlock(&thread->Owner->HandleLock);
|
||||||
ListRemove(&thread->ProcessLink);
|
ListRemove(&thread->ProcessLink);
|
||||||
thread->Owner->ThreadCount--;
|
thread->Owner->ThreadCount--;
|
||||||
KeReleaseSpinlock(&thread->Owner->HandleLock);
|
KeReleaseSpinlock(&thread->Owner->HandleLock, flags);
|
||||||
ObDereferenceObject(&thread->Owner->Header);
|
ObDereferenceObject(&thread->Owner->Header);
|
||||||
thread->Owner = (PROCESS *)0;
|
thread->Owner = (PROCESS *)0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Free kernel stack.
|
|
||||||
if (thread->KernelStackBase) {
|
if (thread->KernelStackBase) {
|
||||||
MmFreePool(thread->KernelStackBase);
|
MmFreePool(thread->KernelStackBase);
|
||||||
thread->KernelStackBase = (void *)0;
|
thread->KernelStackBase = (void *)0;
|
||||||
}
|
}
|
||||||
// User stack was allocated from user heap - it is up to the process to
|
|
||||||
// track and free it. For now we leave it (shared heap, no per-process VA).
|
// Assumes UserStackBase was added to THREAD struct in ke_thread.h.
|
||||||
|
if (thread->UserStackBase) {
|
||||||
|
MmFreeUserMemory(thread->UserStackBase, USTACK_SIZE);
|
||||||
|
thread->UserStackBase = (void *)0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// KeCreateThread
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
||||||
THREAD_PRIORITY priority, THREAD **out) {
|
THREAD_PRIORITY priority, THREAD **out) {
|
||||||
if (!owner || !out) return STATUS_INVALID_PARAMETER;
|
if (!owner || !out) return STATUS_INVALID_PARAMETER;
|
||||||
@@ -53,14 +41,14 @@ KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
|||||||
THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
|
THREAD* thread = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
|
||||||
if (!thread) return STATUS_NO_MEMORY;
|
if (!thread) return STATUS_NO_MEMORY;
|
||||||
|
|
||||||
thread->Header.Destroy = ThreadDestroy;
|
thread->Header.Destroy = ThreadDestroy;
|
||||||
thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
|
thread->ThreadId = _InterlockedExchangeAdd((volatile long *)&gNextThreadId, 1);
|
||||||
thread->State = ThreadReady;
|
thread->State = ThreadReady;
|
||||||
thread->Priority = priority;
|
thread->Priority = priority;
|
||||||
thread->QuantaConsumed = 0;
|
thread->QuantaConsumed = 0;
|
||||||
thread->WaitObject = (OBJECT *)0;
|
thread->WaitObject = (OBJECT *)0;
|
||||||
|
thread->SchedLockFlags = 0x202ULL;
|
||||||
|
|
||||||
// Allocate kernel stack.
|
|
||||||
void* kstack = MmAllocatePool(KSTACK_SIZE);
|
void* kstack = MmAllocatePool(KSTACK_SIZE);
|
||||||
if (!kstack) {
|
if (!kstack) {
|
||||||
ObDereferenceObject(&thread->Header);
|
ObDereferenceObject(&thread->Header);
|
||||||
@@ -71,87 +59,70 @@ KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
|||||||
thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
|
thread->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE;
|
||||||
thread->TssRsp0 = thread->KernelStackTop;
|
thread->TssRsp0 = thread->KernelStackTop;
|
||||||
|
|
||||||
// Allocate user stack in the shared user heap.
|
|
||||||
void* ustack = MmAllocateUserMemory(USTACK_SIZE);
|
void* ustack = MmAllocateUserMemory(USTACK_SIZE);
|
||||||
if (!ustack) {
|
if (!ustack) {
|
||||||
MmFreePool(kstack);
|
MmFreePool(kstack);
|
||||||
ObDereferenceObject(&thread->Header);
|
ObDereferenceObject(&thread->Header);
|
||||||
return STATUS_NO_MEMORY;
|
return STATUS_NO_MEMORY;
|
||||||
}
|
}
|
||||||
|
thread->UserStackBase = ustack;
|
||||||
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE;
|
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE;
|
||||||
|
|
||||||
thread->UserEntry = entry;
|
thread->UserEntry = entry;
|
||||||
thread->UserArg = arg;
|
thread->UserArg = arg;
|
||||||
|
|
||||||
// Set up the initial KCONTEXT so that when KeSwitchContext restores
|
|
||||||
// this thread, execution lands in KeThreadEntry (in ke_switch.asm),
|
|
||||||
// which will IRETQ into ring 3 at UserEntry.
|
|
||||||
// The KCONTEXT sits at the top of the kernel stack.
|
|
||||||
uintptr_t* ksp = (uintptr_t *)thread->KernelStackTop;
|
uintptr_t* ksp = (uintptr_t *)thread->KernelStackTop;
|
||||||
|
|
||||||
// Push user RSP so KeThreadEntry can find it.
|
|
||||||
*(--ksp) = userStackTop;
|
*(--ksp) = userStackTop;
|
||||||
|
*(--ksp) = 0x23ULL;
|
||||||
|
*(--ksp) = userStackTop;
|
||||||
|
*(--ksp) = 0x202ULL;
|
||||||
|
*(--ksp) = 0x1BULL;
|
||||||
|
*(--ksp) = entry;
|
||||||
|
*(--ksp) = arg;
|
||||||
|
|
||||||
// Push the IRETQ frame: RIP, CS (ring3), RFLAGS, RSP, SS (ring3).
|
|
||||||
// Segment selector values: 0x1B = ring-3 code, 0x23 = ring-3 data.
|
|
||||||
*(--ksp) = 0x23ULL; // SS
|
|
||||||
*(--ksp) = userStackTop; // RSP (already set above but IRETQ needs it)
|
|
||||||
*(--ksp) = 0x202ULL; // RFLAGS: IF=1, reserved bit 1
|
|
||||||
*(--ksp) = 0x1BULL; // CS
|
|
||||||
*(--ksp) = entry; // RIP
|
|
||||||
// RCX = arg (first argument per MS x64 ABI).
|
|
||||||
*(--ksp) = arg; // RCX placeholder - KeThreadEntry loads it
|
|
||||||
|
|
||||||
// Now build the KCONTEXT that KeSwitchContext will restore.
|
|
||||||
// Rip points to KeThreadEntry; all other regs zero; RSP = ksp.
|
|
||||||
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
|
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
|
||||||
KCONTEXT* ctx = (KCONTEXT *)ksp;
|
KCONTEXT* ctx = (KCONTEXT *)ksp;
|
||||||
memset(ctx, 0, sizeof(KCONTEXT));
|
memset(ctx, 0, sizeof(KCONTEXT));
|
||||||
ctx->Rip = (uint64_t)KeThreadEntry;
|
ctx->Rip = (uint64_t)KeThreadEntry;
|
||||||
thread->SavedContext = ctx;
|
thread->SavedContext = ctx;
|
||||||
|
|
||||||
// Link into the owner process.
|
|
||||||
ObReferenceObject(&owner->Header);
|
ObReferenceObject(&owner->Header);
|
||||||
thread->Owner = owner;
|
thread->Owner = owner;
|
||||||
KeAcquireSpinlock(&owner->HandleLock);
|
|
||||||
|
uintptr_t flags = KeAcquireSpinlock(&owner->HandleLock);
|
||||||
ListInsertTail(&owner->ThreadList, &thread->ProcessLink);
|
ListInsertTail(&owner->ThreadList, &thread->ProcessLink);
|
||||||
owner->ThreadCount++;
|
owner->ThreadCount++;
|
||||||
KeReleaseSpinlock(&owner->HandleLock);
|
KeReleaseSpinlock(&owner->HandleLock, flags);
|
||||||
|
|
||||||
// Hand the thread to the scheduler.
|
|
||||||
KeReadyThread(thread);
|
KeReadyThread(thread);
|
||||||
|
|
||||||
*out = thread;
|
*out = thread;
|
||||||
return STATUS_SUCCESS;
|
return STATUS_SUCCESS;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// KeExitThread
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
void KeExitThread(THREAD* thread, int code) {
|
void KeExitThread(THREAD* thread, int code) {
|
||||||
(void)code;
|
(void)code;
|
||||||
if (!thread) return;
|
if (!thread) return;
|
||||||
|
|
||||||
thread->State = ThreadDead;
|
thread->State = ThreadDead;
|
||||||
KeUnreadyThread(thread);
|
|
||||||
ObDereferenceObject(&thread->Header);
|
|
||||||
|
|
||||||
// Yield - the scheduler will never pick this thread again.
|
|
||||||
KeYield();
|
KeYield();
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Block / unblock
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
void KeBlockThread(THREAD* thread, OBJECT* waitObject) {
|
void KeBlockThread(THREAD* thread, OBJECT* waitObject) {
|
||||||
thread->State = ThreadWaiting;
|
thread->State = ThreadWaiting;
|
||||||
thread->WaitObject = waitObject;
|
thread->WaitObject = waitObject;
|
||||||
KeUnreadyThread(thread);
|
|
||||||
KeYield();
|
KeYield();
|
||||||
}
|
}
|
||||||
|
|
||||||
void KeUnblockThread(THREAD* thread) {
|
void KeUnblockThread(THREAD* thread) {
|
||||||
thread->WaitObject = (OBJECT *)0;
|
thread->WaitObject = (OBJECT *)0;
|
||||||
thread->State = ThreadReady;
|
thread->State = ThreadReady;
|
||||||
|
|
||||||
|
if (thread->Priority < PriorityHigh) {
|
||||||
|
thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1);
|
||||||
|
thread->QuantaConsumed = 0;
|
||||||
|
}
|
||||||
|
|
||||||
KeReadyThread(thread);
|
KeReadyThread(thread);
|
||||||
}
|
}
|
||||||
@@ -61,6 +61,9 @@ typedef struct THREAD {
|
|||||||
// Ring-3 entry point and argument for newly created threads.
|
// Ring-3 entry point and argument for newly created threads.
|
||||||
uintptr_t UserEntry;
|
uintptr_t UserEntry;
|
||||||
uintptr_t UserArg;
|
uintptr_t UserArg;
|
||||||
|
|
||||||
|
uintptr_t SchedLockFlags; // Stores RFLAGS while thread is context switched out
|
||||||
|
void* UserStackBase; // Tracks user stack allocation to prevent memory leaks
|
||||||
} THREAD;
|
} THREAD;
|
||||||
|
|
||||||
KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
||||||
|
|||||||
+11
-11
@@ -36,11 +36,11 @@ static FRAME_STACK gFrameStack;
|
|||||||
|
|
||||||
// Push a physical frame address onto the stack.
|
// Push a physical frame address onto the stack.
|
||||||
static void PushFrame(PHYSICAL_ADDRESS frame) {
|
static void PushFrame(PHYSICAL_ADDRESS frame) {
|
||||||
KeAcquireSpinlock(&gFrameStack.Lock);
|
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
|
||||||
if (gFrameStack.Top < gFrameStack.Capacity) {
|
if (gFrameStack.Top < gFrameStack.Capacity) {
|
||||||
gFrameStack.Stack[gFrameStack.Top++] = frame;
|
gFrameStack.Stack[gFrameStack.Top++] = frame;
|
||||||
}
|
}
|
||||||
KeReleaseSpinlock(&gFrameStack.Lock);
|
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -104,32 +104,32 @@ void MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) {
|
|||||||
// Public API
|
// Public API
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
PHYSICAL_ADDRESS MmAllocateFrame(void) {
|
PHYSICAL_ADDRESS MmAllocateFrame(void) {
|
||||||
KeAcquireSpinlock(&gFrameStack.Lock);
|
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
|
||||||
if (gFrameStack.Top == 0) {
|
if (gFrameStack.Top == 0) {
|
||||||
KeReleaseSpinlock(&gFrameStack.Lock);
|
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||||
return (PHYSICAL_ADDRESS)0;
|
return (PHYSICAL_ADDRESS)0;
|
||||||
}
|
}
|
||||||
PHYSICAL_ADDRESS frame = gFrameStack.Stack[--gFrameStack.Top];
|
PHYSICAL_ADDRESS frame = gFrameStack.Stack[--gFrameStack.Top];
|
||||||
KeReleaseSpinlock(&gFrameStack.Lock);
|
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||||
return frame;
|
return frame;
|
||||||
}
|
}
|
||||||
|
|
||||||
void MmFreeFrame(PHYSICAL_ADDRESS frame) {
|
void MmFreeFrame(PHYSICAL_ADDRESS frame) {
|
||||||
if (!frame || (frame & (PAGE_SIZE - 1))) return;
|
if (!frame || (frame & (PAGE_SIZE - 1))) return;
|
||||||
KeAcquireSpinlock(&gFrameStack.Lock);
|
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
|
||||||
if (gFrameStack.Top < gFrameStack.Capacity)
|
if (gFrameStack.Top < gFrameStack.Capacity)
|
||||||
gFrameStack.Stack[gFrameStack.Top++] = frame;
|
gFrameStack.Stack[gFrameStack.Top++] = frame;
|
||||||
// else: stack full — leak rather than corrupt
|
// else: stack full — leak rather than corrupt
|
||||||
KeReleaseSpinlock(&gFrameStack.Lock);
|
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
|
PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
|
||||||
if (!count) return (PHYSICAL_ADDRESS)0;
|
if (!count) return (PHYSICAL_ADDRESS)0;
|
||||||
|
|
||||||
KeAcquireSpinlock(&gFrameStack.Lock);
|
uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock);
|
||||||
|
|
||||||
if (gFrameStack.Top < count) {
|
if (gFrameStack.Top < count) {
|
||||||
KeReleaseSpinlock(&gFrameStack.Lock);
|
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||||
return (PHYSICAL_ADDRESS)0;
|
return (PHYSICAL_ADDRESS)0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,7 +152,7 @@ PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (run == (size_t)-1) {
|
if (run == (size_t)-1) {
|
||||||
KeReleaseSpinlock(&gFrameStack.Lock);
|
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||||
return (PHYSICAL_ADDRESS)0;
|
return (PHYSICAL_ADDRESS)0;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -164,7 +164,7 @@ PHYSICAL_ADDRESS MmAllocateContiguous(size_t count) {
|
|||||||
gFrameStack.Stack[run + i] = gFrameStack.Stack[run + count + i];
|
gFrameStack.Stack[run + i] = gFrameStack.Stack[run + count + i];
|
||||||
gFrameStack.Top -= count;
|
gFrameStack.Top -= count;
|
||||||
|
|
||||||
KeReleaseSpinlock(&gFrameStack.Lock);
|
KeReleaseSpinlock(&gFrameStack.Lock, flags);
|
||||||
return base;
|
return base;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -243,7 +243,7 @@ static void* HeapAlloc(HEAP* h, size_t size) {
|
|||||||
if (!size) return (void *)0;
|
if (!size) return (void *)0;
|
||||||
size = (size + 7) & ~7ULL; // 8-byte align
|
size = (size + 7) & ~7ULL; // 8-byte align
|
||||||
|
|
||||||
KeAcquireSpinlock(&h->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
|
||||||
|
|
||||||
// Best-fit search.
|
// Best-fit search.
|
||||||
BLOCK_HDR* best = (BLOCK_HDR *)0;
|
BLOCK_HDR* best = (BLOCK_HDR *)0;
|
||||||
@@ -256,14 +256,14 @@ static void* HeapAlloc(HEAP* h, size_t size) {
|
|||||||
|
|
||||||
if (!best) {
|
if (!best) {
|
||||||
if (!HeapGrow(h, size + OVERHEAD)) {
|
if (!HeapGrow(h, size + OVERHEAD)) {
|
||||||
KeReleaseSpinlock(&h->Lock);
|
KeReleaseSpinlock(&h->Lock, flags);
|
||||||
return (void *)0;
|
return (void *)0;
|
||||||
}
|
}
|
||||||
// Re-search; HeapGrow inserted at the front.
|
// Re-search; HeapGrow inserted at the front.
|
||||||
best = h->FreeList;
|
best = h->FreeList;
|
||||||
while (best && best->Size < size) best = best->NextFree;
|
while (best && best->Size < size) best = best->NextFree;
|
||||||
if (!best) {
|
if (!best) {
|
||||||
KeReleaseSpinlock(&h->Lock);
|
KeReleaseSpinlock(&h->Lock, flags);
|
||||||
return (void *)0;
|
return (void *)0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -292,7 +292,7 @@ static void* HeapAlloc(HEAP* h, size_t size) {
|
|||||||
HdrToFtr(best)->Magic = BLOCK_MAGIC;
|
HdrToFtr(best)->Magic = BLOCK_MAGIC;
|
||||||
HdrToFtr(best)->Size = best->Size;
|
HdrToFtr(best)->Size = best->Size;
|
||||||
|
|
||||||
KeReleaseSpinlock(&h->Lock);
|
KeReleaseSpinlock(&h->Lock, flags);
|
||||||
return (char *)best + HDR_SIZE;
|
return (char *)best + HDR_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -301,7 +301,7 @@ static void HeapFree(HEAP* h, void* ptr) {
|
|||||||
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
|
BLOCK_HDR* blk = (BLOCK_HDR *)((char *)ptr - HDR_SIZE);
|
||||||
if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt
|
if (blk->Magic != BLOCK_MAGIC || blk->Free) return; // double-free / corrupt
|
||||||
|
|
||||||
KeAcquireSpinlock(&h->Lock);
|
uintptr_t flags = KeAcquireSpinlock(&h->Lock);
|
||||||
|
|
||||||
blk->Free = 1;
|
blk->Free = 1;
|
||||||
|
|
||||||
@@ -338,7 +338,7 @@ static void HeapFree(HEAP* h, void* ptr) {
|
|||||||
HdrToFtr(blk)->Free = 1;
|
HdrToFtr(blk)->Free = 1;
|
||||||
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
HdrToFtr(blk)->Magic = BLOCK_MAGIC;
|
||||||
|
|
||||||
KeReleaseSpinlock(&h->Lock);
|
KeReleaseSpinlock(&h->Lock, flags);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
|
static void* HeapRealloc(HEAP* h, void* ptr, size_t new_size) {
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M.fd -device ahci,id=ahci -drive format=raw,file=\\.\PhysicalDrive7,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown
|
@start qemu-system-x86_64 -drive if=pflash,format=raw,readonly=on,file=OVMF\OVMF_CODE_4M.fd -drive if=pflash,format=raw,file=OVMF\OVMF_VARS_4M_d.fd -device ahci,id=ahci -drive format=raw,file=\\.\PhysicalDrive7,if=none,id=disk0 -device ide-hd,bus=ahci.0,drive=disk0 -m 4G -serial stdio -d in_asm,cpu_reset,int -D qemu.log -no-reboot -no-shutdown
|
||||||
Reference in new issue
Block a user