914 lines
35 KiB
C
914 lines
35 KiB
C
// Thread manager. This file owns thread allocation, kernel/user startup contexts, and lifecycle transitions.
|
|
|
|
#include <ntlite/atomic.h>
|
|
#include <ntlite/string_view.h>
|
|
#include <stdbool.h>
|
|
#include <string.h>
|
|
#include <kernel/ke_list.h>
|
|
#include <stddef.h>
|
|
#include <intrin.h>
|
|
#include <kernel/ke_thread.h>
|
|
#include <kernel/ke_sched.h>
|
|
#include <kernel/ke_cpu.h>
|
|
#include <kernel/ke_irql.h>
|
|
#include <kernel/ke_event.h>
|
|
#include <kernel/ke_mutex.h>
|
|
#include <kernel/ke_semaphore.h>
|
|
#include <kernel/ke_apc.h>
|
|
#include <kernel/ke_debug.h>
|
|
#include <kernel/ke_userenv.h>
|
|
#include <kernel/mm_virt.h>
|
|
#include <kernel/ob_object.h>
|
|
#include <kernel/ke_id.h>
|
|
|
|
#define FPU_FCW_DEFAULT 0x037Fu
|
|
#define FPU_MXCSR_DEFAULT 0x1F80u
|
|
|
|
// These offsets are consumed directly by ke_switch.asm. Keep the asserts
|
|
// here so THREAD layout edits fail at build time instead of corrupting
|
|
// context switches at runtime.
|
|
#define STATIC_ASSERT_OFFSET(type, field, expected) \
|
|
_Static_assert(offsetof(type, field) == (expected), \
|
|
#type "." #field " offset must match assembly")
|
|
|
|
STATIC_ASSERT_OFFSET(THREAD, SavedContext, 112);
|
|
STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 128);
|
|
STATIC_ASSERT_OFFSET(THREAD, Teb, 136);
|
|
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 192);
|
|
|
|
static KE_ID_ALLOCATOR gThreadIds = KE_ID_ALLOCATOR_INIT(1u);
|
|
extern void KeThreadEntry(void);
|
|
|
|
// Allocates one boot-unique thread identifier for normal and hand-built idle threads.
|
|
KSTATUS KeAllocateThreadId(uint32_t* threadIdOut) {
|
|
return KeAllocateId32(&gThreadIds, threadIdOut);
|
|
}
|
|
|
|
// Prevents a newly scheduled user thread from entering a terminating process.
|
|
void KePrepareUserThreadStart(void) {
|
|
THREAD* thread = KeGetCurrentThread();
|
|
PROCESS* owner = thread ? thread->Owner : (PROCESS*)0;
|
|
if (!thread || !owner || !KeIsProcessTerminating(owner))
|
|
return;
|
|
|
|
KeExitThread(thread, owner->ExitCode);
|
|
while (true) {
|
|
KeYield();
|
|
}
|
|
}
|
|
|
|
// Finalizes one dead thread's completion, user memory, and process membership.
|
|
void KeFinalizeThreadExit(THREAD* thread) {
|
|
if (!thread)
|
|
return;
|
|
if (thread->State != ThreadDead) {
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread finalized before death: thread=0x%016llX tid=%u state=%u\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId,
|
|
(unsigned)thread->State);
|
|
}
|
|
|
|
// Thread completion is manual-reset and idempotent. Publish it before
|
|
// process-owned handles can be released by the final live-thread exit.
|
|
if (thread->ExitEvent)
|
|
KeSignalEvent(thread->ExitEvent);
|
|
|
|
PROCESS* owner = thread->Owner;
|
|
if (!owner)
|
|
return;
|
|
|
|
// Every exiting thread releases its own user allocations before leaving
|
|
// the live-thread set. Therefore the final detach can destroy the remaining
|
|
// process VADs without racing a later THREAD destructor.
|
|
if (!owner->ResourcesReleased) {
|
|
if (thread->Teb && thread->TebSize)
|
|
MmFreeUserMemoryForProcess(owner, thread->Teb);
|
|
if (thread->UserStackBase)
|
|
MmFreeUserMemoryForProcess(owner, thread->UserStackBase);
|
|
}
|
|
thread->Teb = (void*)0;
|
|
thread->TebSize = 0;
|
|
thread->UserStackBase = (void*)0;
|
|
|
|
uint32_t remaining = 0;
|
|
int detached = 0;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
|
|
int fwdDetached = thread->ProcessLink.Fwd == &thread->ProcessLink;
|
|
int bwdDetached = thread->ProcessLink.Bwd == &thread->ProcessLink;
|
|
if (fwdDetached != bwdDetached) {
|
|
KeReleaseSpinlock(&owner->ThreadLock, flags);
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread process linkage corrupt: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId,
|
|
(uint64_t)thread->ProcessLink.Fwd,
|
|
(uint64_t)thread->ProcessLink.Bwd);
|
|
}
|
|
|
|
if (!fwdDetached) {
|
|
if (owner->ThreadCount == 0) {
|
|
KeReleaseSpinlock(&owner->ThreadLock, flags);
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread process count underflow: thread=0x%016llX tid=%u pid=%u\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId,
|
|
owner->ProcessId);
|
|
}
|
|
|
|
KeListRemoveChecked(&thread->ProcessLink);
|
|
owner->ThreadCount--;
|
|
detached = 1;
|
|
}
|
|
remaining = owner->ThreadCount;
|
|
KeReleaseSpinlock(&owner->ThreadLock, flags);
|
|
|
|
if (!detached)
|
|
return;
|
|
|
|
if (remaining == 0) {
|
|
// ThreadCount represents live process membership, not retained thread
|
|
// handles. Dead THREAD objects may remain waitable after process exit.
|
|
NtlAtomicExchange32(&owner->Terminating, 1);
|
|
KeStopProcessConstruction(owner);
|
|
KeCompleteProcessExitIfLastThread(owner);
|
|
}
|
|
}
|
|
|
|
// Implements the thread Destroy operation.
|
|
static void KiThreadDestroy(OBJECT* obj) {
|
|
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
|
|
|
|
// A wait that uses auxiliary stack wait blocks must be removed before the
|
|
// kernel stack is released. A terminated thread may never return to the
|
|
// wait syscall to run normal cleanup.
|
|
KeRundownThreadWait(thread, NTLSTATUS_ALERTED);
|
|
|
|
if (thread->WaitingMutex || !LIST_IS_EMPTY(&thread->OwnedMutexList)) {
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread destroyed with mutex state: thread=0x%016llX tid=%u waiting=0x%016llX ownedEmpty=%u\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId,
|
|
(uint64_t)thread->WaitingMutex,
|
|
LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u);
|
|
}
|
|
if (thread->TimeoutArmed ||
|
|
thread->TimeoutLink.Fwd != &thread->TimeoutLink ||
|
|
thread->TimeoutLink.Bwd != &thread->TimeoutLink) {
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread destroyed with timeout state: thread=0x%016llX tid=%u armed=%u link=0x%016llX/0x%016llX\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId,
|
|
(unsigned)thread->TimeoutArmed,
|
|
(uint64_t)thread->TimeoutLink.Fwd,
|
|
(uint64_t)thread->TimeoutLink.Bwd);
|
|
}
|
|
|
|
KiRundownApcState(thread);
|
|
|
|
if (thread->ExitEvent) {
|
|
ObDereferenceObject(&thread->ExitEvent->Header);
|
|
thread->ExitEvent = (EVENT*)0;
|
|
}
|
|
|
|
if (thread->Teb || thread->TebSize || thread->UserStackBase) {
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread destroyed with user memory: thread=0x%016llX tid=%u teb=0x%016llX tebSize=%llu stack=0x%016llX\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId,
|
|
(uint64_t)thread->Teb,
|
|
(uint64_t)thread->TebSize,
|
|
(uint64_t)thread->UserStackBase);
|
|
}
|
|
|
|
if (thread->ProcessLink.Fwd != &thread->ProcessLink ||
|
|
thread->ProcessLink.Bwd != &thread->ProcessLink) {
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread destroyed while still process-linked: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId,
|
|
(uint64_t)thread->ProcessLink.Fwd,
|
|
(uint64_t)thread->ProcessLink.Bwd);
|
|
}
|
|
|
|
if (thread->Owner) {
|
|
ObDereferenceObject(&thread->Owner->Header);
|
|
thread->Owner = (PROCESS*)0;
|
|
}
|
|
|
|
if (thread->KernelStackBase) {
|
|
MmFreePool(thread->KernelStackBase);
|
|
thread->KernelStackBase = (void*)0;
|
|
}
|
|
}
|
|
|
|
|
|
// Validates one optional kernel-supplied thread description and returns its length.
|
|
static KSTATUS KiValidateThreadName(const char* name, size_t* lengthOut) {
|
|
if (!lengthOut) return NTLSTATUS_INVALID_PARAMETER;
|
|
if (!name) {
|
|
*lengthOut = 0;
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
size_t length = 0;
|
|
while (length < NTL_THREAD_NAME_MAX && name[length])
|
|
length++;
|
|
if (length >= NTL_THREAD_NAME_MAX)
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
|
|
if (!SysSvIsUtf8Text(SysSvFromParts(name, length)))
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
*lengthOut = length;
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
|
|
const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) {
|
|
if (!owner || !parameters || !out) return NTLSTATUS_INVALID_PARAMETER;
|
|
if (parameters->SchedulingClass != KeSchedulingNormal ||
|
|
parameters->Priority >= PriorityCount ||
|
|
(parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED))
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
|
|
size_t initialNameLength = 0;
|
|
KSTATUS nameStatus = KiValidateThreadName(parameters->Name, &initialNameLength);
|
|
if (!KSUCCESS(nameStatus)) return nameStatus;
|
|
|
|
uint8_t initiallySuspended =
|
|
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
|
|
|
|
uint32_t threadId = 0;
|
|
KSTATUS idStatus = KeAllocateThreadId(&threadId);
|
|
if (!KSUCCESS(idStatus)) return idStatus;
|
|
|
|
THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
|
|
if (!thread) return NTLSTATUS_NO_MEMORY;
|
|
|
|
thread->Header.Destroy = KiThreadDestroy;
|
|
thread->ThreadId = threadId;
|
|
thread->State = initiallySuspended ? ThreadSuspended : ThreadReady;
|
|
thread->BaseSchedulingClass = parameters->SchedulingClass;
|
|
thread->BasePriority = parameters->Priority;
|
|
thread->EffectiveSchedulingClass = parameters->SchedulingClass;
|
|
thread->EffectivePriority = parameters->Priority;
|
|
thread->QuantaConsumed = 0;
|
|
thread->QuantumTicks = 0;
|
|
thread->QuantumRemaining = 0;
|
|
thread->CpuBurstTicks = 0;
|
|
thread->LastRunTicks = 0;
|
|
thread->RunningCpu = KE_INVALID_CPU;
|
|
thread->SuspendCount = initiallySuspended ? 1u : 0u;
|
|
LIST_INIT(&thread->OwnedMutexList);
|
|
thread->WaitingMutex = (MUTEX*)0;
|
|
thread->Queued = 0;
|
|
thread->IsIdleThread = 0;
|
|
thread->SchedulerFlags = 0;
|
|
thread->LastCpu = KE_INVALID_CPU;
|
|
thread->AffinityMask = parameters->AffinityMask
|
|
? parameters->AffinityMask
|
|
: KeGetDefaultThreadAffinityMask();
|
|
LIST_INIT(&thread->ProcessLink);
|
|
LIST_INIT(&thread->SchedulerLink);
|
|
thread->WaitObject = (OBJECT*)0;
|
|
thread->WaitStatus = NTLSTATUS_SUCCESS;
|
|
thread->WaitListHead = (LIST_ENTRY*)0;
|
|
thread->AlertableWait = 0;
|
|
LIST_INIT(&thread->TimeoutLink);
|
|
thread->TimeoutArmed = 0;
|
|
thread->TimeoutTick = 0;
|
|
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
|
|
thread->SwitchLockCookie = 0x202ULL;
|
|
KiInitializeApcState(thread);
|
|
thread->TrapFrame = (KTRAP_FRAME*)0;
|
|
thread->UserExceptionDepth = 0;
|
|
thread->UserExceptionNoncontinuableMask = 0;
|
|
thread->UserExceptionFrameUserAddress = 0;
|
|
memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
|
|
thread->TerminationDeferralCount = 0;
|
|
thread->ExitInProgress = 0;
|
|
thread->NameLock = (SPINLOCK)SPINLOCK_INIT;
|
|
memset(thread->Name, 0, sizeof(thread->Name));
|
|
if (initialNameLength)
|
|
memcpy(thread->Name, parameters->Name, initialNameLength);
|
|
|
|
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
|
|
if (!kstack) {
|
|
ObDereferenceObject(&thread->Header);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
memset(kstack, 0, KSTACK_SIZE);
|
|
thread->KernelStackBase = kstack;
|
|
thread->KernelStackTop = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
|
|
thread->TssRsp0 = thread->KernelStackTop;
|
|
|
|
KSTATUS s;
|
|
void* ustack = (void*)0;
|
|
s = MmVadCreateCommitted(owner, USTACK_SIZE, PAGE_SIZE,
|
|
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
|
MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
|
|
MmVadStack,
|
|
&ustack);
|
|
if (!KSUCCESS(s) || !ustack) {
|
|
MmFreePool(kstack);
|
|
thread->KernelStackBase = (void*)0;
|
|
ObDereferenceObject(&thread->Header);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
thread->UserStackBase = ustack;
|
|
// Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
|
|
// nevertheless lets callees immediately use the caller-provided 32-byte
|
|
// home/shadow area above RSP, and MSVC commonly emits a first instruction
|
|
// like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
|
|
// entry stack as if a CALL with a normal home area had happened:
|
|
//
|
|
// RSP % 16 == 8 at function entry
|
|
// [RSP + 0] fake return-address slot
|
|
// [RSP + 8..39] home/shadow space
|
|
//
|
|
// Without the 32-byte home area, the first spill lands in the high stack
|
|
// guard page and every freshly-created MSVC thread faults immediately.
|
|
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8 - 32;
|
|
|
|
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
|
|
if (!teb) {
|
|
MmFreeUserMemoryForProcess(owner, ustack);
|
|
thread->UserStackBase = (void*)0;
|
|
MmFreePool(kstack);
|
|
thread->KernelStackBase = (void*)0;
|
|
ObDereferenceObject(&thread->Header);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
memset(teb, 0, sizeof(SYS_TEB));
|
|
teb->Peb = (SYS_PEB*)owner->UserPebBase;
|
|
teb->StackBase = (void*)userStackTop;
|
|
teb->StackLimit = ustack;
|
|
teb->Self = teb;
|
|
teb->ThreadId = thread->ThreadId;
|
|
teb->Magic = SYS_TEB_MAGIC;
|
|
|
|
thread->Teb = teb;
|
|
thread->TebSize = sizeof(SYS_TEB);
|
|
|
|
thread->UserEntry = entry;
|
|
thread->UserArg = arg;
|
|
|
|
uintptr_t* ksp = (uintptr_t*)thread->KernelStackTop;
|
|
*(--ksp) = userStackTop;
|
|
*(--ksp) = 0x23ULL;
|
|
*(--ksp) = userStackTop;
|
|
*(--ksp) = 0x202ULL;
|
|
*(--ksp) = 0x1BULL;
|
|
*(--ksp) = entry;
|
|
*(--ksp) = arg;
|
|
|
|
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
|
|
KCONTEXT* ctx = (KCONTEXT*)ksp;
|
|
memset(ctx, 0, sizeof(KCONTEXT));
|
|
ctx->Rip = (uint64_t)KeThreadEntry;
|
|
thread->SavedContext = ctx;
|
|
|
|
ObReferenceObject(&owner->Header);
|
|
thread->Owner = owner;
|
|
|
|
// Allocate the exit event before the thread becomes visible. Manual-reset
|
|
// so multiple joiners (if any) all wake up when the thread exits.
|
|
EVENT* exitEvent;
|
|
s = KeEventCreate(EventManualReset, 0, &exitEvent);
|
|
if (!KSUCCESS(s)) {
|
|
if (thread->Teb && thread->TebSize) MmFreeUserMemoryForProcess(owner, thread->Teb);
|
|
thread->Teb = (void*)0;
|
|
thread->TebSize = 0;
|
|
MmFreeUserMemoryForProcess(owner, ustack);
|
|
thread->UserStackBase = (void*)0;
|
|
MmFreePool(kstack);
|
|
thread->KernelStackBase = (void*)0;
|
|
ObDereferenceObject(&owner->Header);
|
|
thread->Owner = (PROCESS*)0;
|
|
ObDereferenceObject(&thread->Header);
|
|
return s;
|
|
}
|
|
thread->ExitEvent = exitEvent;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
|
|
KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
|
|
owner->ThreadCount++;
|
|
KeReleaseSpinlock(&owner->ThreadLock, flags);
|
|
|
|
KeInitThreadFpuContext(thread);
|
|
|
|
// Hand the creator a short-lived reference before making the thread
|
|
// runnable. On SMP a new user thread can run and exit before
|
|
// KeCreateThread() returns; without this reference, callers that merely
|
|
// read ThreadId for logging or syscall output can race the scheduler's
|
|
// dead-thread dereference and touch freed pool.
|
|
ObReferenceObject(&thread->Header);
|
|
*out = thread;
|
|
|
|
if (!initiallySuspended)
|
|
KeReadyThread(thread);
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
|
|
// Enters a region that must retain the current thread during process termination.
|
|
void KeEnterTerminationDeferral(void) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
THREAD* thread = KeGetCurrentThread();
|
|
if (!thread || thread->IsIdleThread)
|
|
return;
|
|
NtlAtomicIncrement32(&thread->TerminationDeferralCount);
|
|
}
|
|
|
|
// Leaves a process-termination deferral region.
|
|
void KeLeaveTerminationDeferral(void) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
THREAD* thread = KeGetCurrentThread();
|
|
if (!thread || thread->IsIdleThread)
|
|
return;
|
|
|
|
int32_t remaining = NtlAtomicDecrement32(&thread->TerminationDeferralCount);
|
|
if (remaining < 0) {
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"thread termination deferral underflow: thread=0x%016llX tid=%u\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId);
|
|
}
|
|
|
|
if (remaining == 0 && thread->Owner && KeIsProcessTerminating(thread->Owner))
|
|
KiRequestReschedule();
|
|
}
|
|
|
|
// Returns whether one thread still owns termination-sensitive kernel work.
|
|
bool KeThreadTerminationDeferred(const THREAD* thread) {
|
|
return thread && NtlAtomicLoad32(&thread->TerminationDeferralCount) > 0;
|
|
}
|
|
|
|
// Returns whether terminal teardown owns the current thread stack.
|
|
bool KeThreadExitInProgress(const THREAD* thread) {
|
|
return thread && NtlAtomicLoad32(&thread->ExitInProgress) != 0;
|
|
}
|
|
|
|
// Implements the exit Thread operation.
|
|
__declspec(dllexport) void KeExitThread(THREAD* thread, int32_t code) {
|
|
(void)code;
|
|
if (!thread) return;
|
|
|
|
// Terminal teardown must run exactly once and must retain this kernel
|
|
// stack until thread/process completion is fully published. Hardware
|
|
// interrupts remain enabled, but interrupt-return scheduling observes
|
|
// ExitInProgress and leaves the thread on-CPU until the final KeYield().
|
|
if (NtlAtomicCompareExchange32(&thread->ExitInProgress, 1, 0) != 0)
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"recursive thread exit: thread=0x%016llX tid=%u\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId);
|
|
|
|
// Reject new APCs as soon as teardown owns the thread. The scheduler must
|
|
// not interpret ThreadDead as permission to switch away until the explicit
|
|
// terminal yield below.
|
|
thread->State = ThreadDead;
|
|
|
|
KiRundownApcState(thread);
|
|
KiMutexRundownThread(thread);
|
|
KeFinalizeThreadExit(thread);
|
|
KeYield();
|
|
|
|
KePanic(KE_PANIC_CODE_SCHEDULER,
|
|
"dead thread resumed after terminal yield: thread=0x%016llX tid=%u\n",
|
|
(uint64_t)thread,
|
|
thread->ThreadId);
|
|
}
|
|
|
|
// KeCreateKernelThread
|
|
// Kernel threads never go to ring 3: they have no user stack, no TEB, and
|
|
// their entry stub (KeKernelThreadEntry) calls the function directly then
|
|
// calls KeExitThread instead of IRETQ-ing.
|
|
|
|
// Stack layout built here (from KernelStackTop downward):
|
|
// [ksp - 8 ] arg <-- popped by KeKernelThreadEntry as r12
|
|
// [ksp - 16] fn <-- popped by KeKernelThreadEntry as rbx
|
|
// [ksp - 16 - sizeof(KCONTEXT)] KCONTEXT { .Rip = KeKernelThreadEntry }.
|
|
extern void KeKernelThreadEntry(void);
|
|
|
|
static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
|
|
const KE_THREAD_CREATE_PARAMETERS* parameters,
|
|
THREAD** out) {
|
|
// Kernel worker threads are owned by the system process. The scheduler's
|
|
// hand-built CPU idle thread stays attached to the dedicated idle process, while
|
|
// driver, async I/O, and init worker threads use the system process.
|
|
PROCESS* owner = KeGetSystemProcess();
|
|
if (!owner) {
|
|
THREAD* currentThread = KeGetCurrentThread();
|
|
owner = currentThread ? currentThread->Owner : (PROCESS*)0;
|
|
}
|
|
if (!owner || !parameters) return NTLSTATUS_INVALID_PARAMETER;
|
|
if (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED)
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
if ((parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) && !out)
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
if (parameters->SchedulingClass == KeSchedulingNormal) {
|
|
if (parameters->Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
|
|
} else if (parameters->SchedulingClass == KeSchedulingRealtime) {
|
|
if (parameters->Priority > KE_REALTIME_PRIORITY_MAX) return NTLSTATUS_INVALID_PARAMETER;
|
|
} else {
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
}
|
|
|
|
size_t initialNameLength = 0;
|
|
KSTATUS nameStatus = KiValidateThreadName(parameters->Name, &initialNameLength);
|
|
if (!KSUCCESS(nameStatus)) return nameStatus;
|
|
|
|
uint8_t initiallySuspended =
|
|
(parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;
|
|
|
|
uint32_t threadId = 0;
|
|
KSTATUS idStatus = KeAllocateThreadId(&threadId);
|
|
if (!KSUCCESS(idStatus)) return idStatus;
|
|
|
|
THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
|
|
if (!thread) return NTLSTATUS_NO_MEMORY;
|
|
|
|
thread->Header.Destroy = KiThreadDestroy;
|
|
thread->ThreadId = threadId;
|
|
thread->State = initiallySuspended ? ThreadSuspended : ThreadReady;
|
|
thread->BaseSchedulingClass = parameters->SchedulingClass;
|
|
thread->BasePriority = parameters->Priority;
|
|
thread->EffectiveSchedulingClass = parameters->SchedulingClass;
|
|
thread->EffectivePriority = parameters->Priority;
|
|
thread->QuantaConsumed = 0;
|
|
thread->QuantumTicks = 0;
|
|
thread->QuantumRemaining = 0;
|
|
thread->CpuBurstTicks = 0;
|
|
thread->LastRunTicks = 0;
|
|
thread->RunningCpu = KE_INVALID_CPU;
|
|
thread->SuspendCount = initiallySuspended ? 1u : 0u;
|
|
LIST_INIT(&thread->OwnedMutexList);
|
|
thread->WaitingMutex = (MUTEX*)0;
|
|
thread->Queued = 0;
|
|
thread->IsIdleThread = 0;
|
|
thread->SchedulerFlags = 0;
|
|
thread->LastCpu = KE_INVALID_CPU;
|
|
thread->AffinityMask = parameters->AffinityMask
|
|
? parameters->AffinityMask
|
|
: KeGetDefaultThreadAffinityMask();
|
|
LIST_INIT(&thread->ProcessLink);
|
|
LIST_INIT(&thread->SchedulerLink);
|
|
thread->WaitObject = (OBJECT*)0;
|
|
thread->WaitStatus = NTLSTATUS_SUCCESS;
|
|
thread->WaitListHead = (LIST_ENTRY*)0;
|
|
thread->AlertableWait = 0;
|
|
LIST_INIT(&thread->TimeoutLink);
|
|
thread->TimeoutArmed = 0;
|
|
thread->TimeoutTick = 0;
|
|
thread->TimeoutStatus = NTLSTATUS_SUCCESS;
|
|
thread->SwitchLockCookie = 0x202ULL;
|
|
KiInitializeApcState(thread);
|
|
thread->TrapFrame = (KTRAP_FRAME*)0;
|
|
thread->UserExceptionDepth = 0;
|
|
thread->UserExceptionNoncontinuableMask = 0;
|
|
thread->UserExceptionFrameUserAddress = 0;
|
|
memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
|
|
thread->TerminationDeferralCount = 0;
|
|
thread->ExitInProgress = 0;
|
|
thread->NameLock = (SPINLOCK)SPINLOCK_INIT;
|
|
memset(thread->Name, 0, sizeof(thread->Name));
|
|
if (initialNameLength)
|
|
memcpy(thread->Name, parameters->Name, initialNameLength);
|
|
thread->UserStackBase = (void*)0; // no user stack
|
|
thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx
|
|
thread->TebSize = 0;
|
|
|
|
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
|
|
if (!kstack) {
|
|
ObDereferenceObject(&thread->Header);
|
|
return NTLSTATUS_NO_MEMORY;
|
|
}
|
|
memset(kstack, 0, KSTACK_SIZE);
|
|
thread->KernelStackBase = kstack;
|
|
thread->KernelStackTop = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
|
|
thread->TssRsp0 = thread->KernelStackTop;
|
|
|
|
// Build the initial stack: arg, fn, then KCONTEXT.
|
|
uintptr_t* ksp = (uintptr_t*)thread->KernelStackTop;
|
|
*(--ksp) = arg; // popped second (-> r12)
|
|
*(--ksp) = fn; // popped first (-> rbx)
|
|
|
|
ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
|
|
KCONTEXT* ctx = (KCONTEXT*)ksp;
|
|
memset(ctx, 0, sizeof(KCONTEXT));
|
|
ctx->Rip = (uint64_t)KeKernelThreadEntry;
|
|
thread->SavedContext = ctx;
|
|
|
|
// Register with the owning process.
|
|
ObReferenceObject(&owner->Header);
|
|
thread->Owner = owner;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
|
|
KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
|
|
owner->ThreadCount++;
|
|
KeReleaseSpinlock(&owner->ThreadLock, flags);
|
|
|
|
// Hand the creator a short-lived reference before making the kernel
|
|
// thread runnable. Kernel workers can run, signal their completion event,
|
|
// exit, and be reaped on another CPU before KeCreateKernelThread()'s
|
|
// caller drops its returned pointer. The live scheduler reference remains
|
|
// separate and is released by KeSchedulerPostSwitch() after the dead
|
|
// thread has switched away.
|
|
if (out) {
|
|
ObReferenceObject(&thread->Header);
|
|
*out = thread;
|
|
}
|
|
|
|
KeInitThreadFpuContext(thread);
|
|
if (!initiallySuspended)
|
|
KeReadyThread(thread);
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
|
|
const KE_THREAD_CREATE_PARAMETERS* parameters,
|
|
THREAD** out) {
|
|
return KiCreateKernelThread(fn, arg, parameters, out);
|
|
}
|
|
|
|
// Replaces one thread's diagnostic description.
|
|
KSTATUS KeSetThreadName(THREAD* thread, const char* name, size_t length) {
|
|
if (!thread || length >= NTL_THREAD_NAME_MAX ||
|
|
!SysSvIsUtf8Text(SysSvFromParts(name, length)))
|
|
return NTLSTATUS_INVALID_PARAMETER;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&thread->NameLock);
|
|
if (length)
|
|
memcpy(thread->Name, name, length);
|
|
thread->Name[length] = '\0';
|
|
KeReleaseSpinlock(&thread->NameLock, flags);
|
|
return NTLSTATUS_SUCCESS;
|
|
}
|
|
|
|
// Queries one stable snapshot of a mutable thread description.
|
|
size_t KeQueryThreadName(THREAD* thread, char* buffer, size_t capacity) {
|
|
if (!thread) return 0;
|
|
|
|
uintptr_t flags = KeAcquireSpinlock(&thread->NameLock);
|
|
size_t length = 0;
|
|
while (length < NTL_THREAD_NAME_MAX && thread->Name[length])
|
|
length++;
|
|
if (buffer && capacity) {
|
|
size_t copyLength = length < capacity - 1u ? length : capacity - 1u;
|
|
if (copyLength)
|
|
memcpy(buffer, thread->Name, copyLength);
|
|
buffer[copyLength] = '\0';
|
|
}
|
|
KeReleaseSpinlock(&thread->NameLock, flags);
|
|
return length;
|
|
}
|
|
|
|
// Prepares block Thread.
|
|
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
|
|
// Called with the synchronization object's spinlock held, so the caller is
|
|
// normally at DISPATCH_LEVEL here. This function only marks intent to
|
|
// block; the actual scheduling point is KeCommitBlockThread at PASSIVE.
|
|
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
|
|
if (!thread) return;
|
|
KiSchedulerMarkThreadWaiting(thread);
|
|
|
|
thread->WaitObject = waitObject;
|
|
thread->WaitStatus = NTLSTATUS_PENDING;
|
|
thread->WaitListHead = (LIST_ENTRY*)0;
|
|
thread->AlertableWait = alertable ? 1 : 0;
|
|
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
|
|
thread->WaitRundownContext = (void*)0;
|
|
}
|
|
|
|
// Updates thread Wait List.
|
|
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
|
|
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
|
|
if (!thread) return;
|
|
thread->WaitListHead = waitListHead;
|
|
}
|
|
|
|
// Registers cleanup for waits with auxiliary wait blocks.
|
|
void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context) {
|
|
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
|
|
if (!thread) return;
|
|
thread->WaitRundownRoutine = routine;
|
|
thread->WaitRundownContext = context;
|
|
}
|
|
|
|
// Clears auxiliary wait cleanup after the owning wait has removed its blocks.
|
|
void KeClearThreadWaitRundown(THREAD* thread) {
|
|
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
|
|
if (!thread) return;
|
|
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
|
|
thread->WaitRundownContext = (void*)0;
|
|
}
|
|
|
|
// Runs auxiliary wait cleanup for abort and thread teardown paths.
|
|
void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
|
|
if (!thread || !thread->WaitRundownRoutine) return;
|
|
|
|
KE_WAIT_RUNDOWN_ROUTINE routine = thread->WaitRundownRoutine;
|
|
void* context = thread->WaitRundownContext;
|
|
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
|
|
thread->WaitRundownContext = (void*)0;
|
|
routine(thread, context, status);
|
|
}
|
|
|
|
// Implements the commit Block Thread operation.
|
|
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
|
|
KeAssertIrql(PASSIVE_LEVEL);
|
|
if (!thread) return NTLSTATUS_INVALID_PARAMETER;
|
|
|
|
if (alertable)
|
|
thread->AlertableWait = 1;
|
|
|
|
// A signal/APC may have arrived after the waiter was placed on the
|
|
// object's wait list but before this thread reached the scheduling point.
|
|
// In that case KeUnblockThreadWithStatus completed the wait in-place and
|
|
// there is nothing left to block on.
|
|
if (thread->State != ThreadWaiting) {
|
|
KSTATUS status = thread->WaitStatus;
|
|
thread->AlertableWait = 0;
|
|
return status;
|
|
}
|
|
|
|
// A kernel APC can be queued before the waiter finishes publishing its
|
|
// alertable state. In that race the queueing CPU cannot wake the wait, and
|
|
// APC delivery is deliberately deferred while State is ThreadWaiting.
|
|
// Consume the prepared wait here before the thread can actually sleep.
|
|
if (alertable && thread->KernelApcPending && thread->KernelApcDisable == 0) {
|
|
if (KeAbortThreadWait(thread, NTLSTATUS_KERNEL_APC)) {
|
|
thread->AlertableWait = 0;
|
|
KiDeliverKernelApcs();
|
|
return NTLSTATUS_KERNEL_APC;
|
|
}
|
|
}
|
|
|
|
// If a user APC was already pending before the thread fully blocked,
|
|
// consume the alertable wait immediately instead of going to sleep.
|
|
if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
|
|
if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC)) {
|
|
thread->AlertableWait = 0;
|
|
return NTLSTATUS_USER_APC;
|
|
}
|
|
}
|
|
|
|
KiSchedulerNoteVoluntaryBlock(thread);
|
|
KeYield();
|
|
|
|
// Kernel APCs are delivered before an alertable wait returns
|
|
// NTLSTATUS_KERNEL_APC to its caller.
|
|
KiDeliverKernelApcs();
|
|
|
|
thread->AlertableWait = 0;
|
|
return thread->WaitStatus;
|
|
}
|
|
|
|
// Implements the block Thread operation.
|
|
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
|
|
KePrepareBlockThread(thread, waitObject, alertable);
|
|
return KeCommitBlockThread(thread, alertable);
|
|
}
|
|
|
|
// Implements the unblock Thread With Status operation.
|
|
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
|
|
if (!thread) return;
|
|
KiSchedulerWakeThread(thread, status);
|
|
}
|
|
|
|
// Returns whether the scheduler link still belongs to an object wait list.
|
|
static bool KiThreadWaitLinkIsLinked(const THREAD* thread) {
|
|
if (!thread) return false;
|
|
return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
|
|
thread->SchedulerLink.Bwd != &thread->SchedulerLink;
|
|
}
|
|
|
|
// Removes a thread from one wait list while that wait object's lock is held.
|
|
static bool KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
|
|
if (!thread || !waitListHead) return false;
|
|
if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead)
|
|
return false;
|
|
|
|
// The signaling side pops the wait-list entry before it wakes the thread.
|
|
// In that handoff window the scheduler link is already self-linked; abort
|
|
// must not remove it again or turn a successful mutex/semaphore handoff
|
|
// into an aborted wait for a thread that may already own the object.
|
|
if (!KiThreadWaitLinkIsLinked(thread))
|
|
return false;
|
|
|
|
KeListRemoveChecked(&thread->SchedulerLink);
|
|
thread->WaitListHead = (LIST_ENTRY*)0;
|
|
return true;
|
|
}
|
|
|
|
// Removes a thread from an object-owned wait queue using that object's lock.
|
|
static int KiAbortObjectWait(THREAD* thread) {
|
|
if (!thread || !thread->WaitObject || !thread->WaitListHead)
|
|
return 0;
|
|
|
|
OBJECT* object = thread->WaitObject;
|
|
switch (object->Type) {
|
|
case ObjectTypeEvent: {
|
|
EVENT* event = CONTAINING_RECORD(object, EVENT, Header);
|
|
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
|
|
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &event->WaitList);
|
|
KeReleaseSpinlock(&event->Lock, flags);
|
|
return removed;
|
|
}
|
|
case ObjectTypeMutex: {
|
|
MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
|
|
return KiMutexAbortWait(mutex, thread);
|
|
}
|
|
case ObjectTypeSemaphore: {
|
|
SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
|
|
uintptr_t flags = KeAcquireSpinlock(&semaphore->Lock);
|
|
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &semaphore->WaitList);
|
|
KeReleaseSpinlock(&semaphore->Lock, flags);
|
|
return removed;
|
|
}
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
// Aborts a thread's current wait regardless of whether the wait was marked
|
|
// alertable. Process teardown uses this for crash/kill paths: a background
|
|
// worker blocked forever on a semaphore must not keep the process exit event
|
|
// unsignaled after the foreground thread has faulted. Object wait-list removal
|
|
// must use the object's own lock; otherwise signal/abort races can leave stale
|
|
// scheduler links in an event, mutex, or semaphore wait queue.
|
|
bool KeAbortThreadWait(THREAD* thread, KSTATUS status) {
|
|
if (!thread) return false;
|
|
if (thread->State != ThreadWaiting) return false;
|
|
|
|
if (thread->WaitRundownRoutine) {
|
|
KeRundownThreadWait(thread, status);
|
|
return thread->State != ThreadWaiting;
|
|
}
|
|
|
|
if (KiAbortObjectWait(thread)) {
|
|
KeUnblockThreadWithStatus(thread, status);
|
|
return true;
|
|
}
|
|
|
|
// A signaler that already dequeued an object waiter owns completion. Let
|
|
// that handoff finish rather than returning timeout after ownership or a
|
|
// semaphore count has already transferred to this thread.
|
|
if (thread->WaitObject)
|
|
return false;
|
|
|
|
// Delays and auxiliary scheduler waits have no object-owned queue.
|
|
KeUnblockThreadFromWaitListWithStatus(thread, status);
|
|
return true;
|
|
}
|
|
|
|
// Implements the unblock Thread operation.
|
|
void KeUnblockThread(THREAD* thread) {
|
|
KeUnblockThreadWithStatus(thread, NTLSTATUS_SUCCESS);
|
|
}
|
|
|
|
// Implements the alert Thread With Status operation.
|
|
static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
|
|
if (!thread) return NTLSTATUS_INVALID_PARAMETER;
|
|
|
|
if (thread->State != ThreadWaiting || !thread->AlertableWait)
|
|
return NTLSTATUS_PENDING;
|
|
|
|
return KeAbortThreadWait(thread, status)
|
|
? NTLSTATUS_SUCCESS
|
|
: NTLSTATUS_PENDING;
|
|
}
|
|
|
|
// Implements the alert Thread operation.
|
|
KSTATUS KeAlertThread(THREAD* thread) {
|
|
return KiAlertThreadWithStatus(thread, NTLSTATUS_ALERTED);
|
|
}
|
|
|
|
// Implements the wake Thread For Kernel Apc operation.
|
|
void KiWakeThreadForKernelApc(THREAD* thread) {
|
|
(void)KiAlertThreadWithStatus(thread, NTLSTATUS_KERNEL_APC);
|
|
}
|
|
|
|
// Implements the wake Thread For User Apc operation.
|
|
void KiWakeThreadForUserApc(THREAD* thread) {
|
|
(void)KiAlertThreadWithStatus(thread, NTLSTATUS_USER_APC);
|
|
}
|
|
|
|
void KeInitThreadFpuContext(THREAD* thread)
|
|
{
|
|
uintptr_t raw =
|
|
(uintptr_t)thread->FpuStorage;
|
|
|
|
uintptr_t aligned =
|
|
(raw + 15) & ~(uintptr_t)15;
|
|
|
|
thread->FpuContext = (void*)aligned;
|
|
|
|
memset(thread->FpuContext, 0, 512);
|
|
|
|
*(uint16_t*)((uint8_t*)thread->FpuContext + 0x00) = 0x037F;
|
|
*(uint32_t*)((uint8_t*)thread->FpuContext + 0x18) = 0x1F80;
|
|
}
|