nt-lite/code/kernel/ke_thread.c

// Thread manager. This file owns thread allocation, kernel/user startup contexts, and lifecycle transitions.

#include <string.h>
#include <kernel/ke_list.h>
#include <stddef.h>
#include <intrin.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_event.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_userenv.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_id.h>

#define FPU_FCW_DEFAULT   0x037Fu
#define FPU_MXCSR_DEFAULT 0x1F80u

// These offsets are consumed directly by ke_switch.asm. Keep the asserts
// here so THREAD layout edits fail at build time instead of corrupting
// context switches at runtime.
#define STATIC_ASSERT_OFFSET(type, field, expected) \
    _Static_assert(offsetof(type, field) == (expected), \
                   #type "." #field " offset must match assembly")

STATIC_ASSERT_OFFSET(THREAD, SavedContext, 112);
STATIC_ASSERT_OFFSET(THREAD, TssRsp0, 128);
STATIC_ASSERT_OFFSET(THREAD, Teb, 136);
STATIC_ASSERT_OFFSET(THREAD, FpuContext, 192);

static KE_ID_ALLOCATOR gThreadIds = KE_ID_ALLOCATOR_INIT(1u);
extern void KeThreadEntry(void);

// Allocates one boot-unique thread identifier for normal and hand-built idle threads.
KSTATUS KeAllocateThreadId(uint32_t* threadIdOut) {
    return KeAllocateId32(&gThreadIds, threadIdOut);
}

// Prevents a newly scheduled user thread from entering a terminating process.
void KePrepareUserThreadStart(void) {
    THREAD* thread = KeGetCurrentThread();
    PROCESS* owner = thread ? thread->Owner : (PROCESS*)0;
    if (!thread || !owner || !KeIsProcessTerminating(owner))
        return;

    KeExitThread(thread, owner->ExitCode);
    while (true) {
        KeYield();
    }
}

// Finalizes one dead thread's completion, user memory, and process membership.
void KeFinalizeThreadExit(THREAD* thread) {
    if (!thread)
        return;
    if (thread->State != ThreadDead) {
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "thread finalized before death: thread=0x%016llX tid=%u state=%u\n",
                (uint64_t)thread,
                thread->ThreadId,
                (unsigned)thread->State);
    }

    // Thread completion is manual-reset and idempotent. Publish it before
    // process-owned handles can be released by the final live-thread exit.
    if (thread->ExitEvent)
        KeSignalEvent(thread->ExitEvent);

    PROCESS* owner = thread->Owner;
    if (!owner)
        return;

    // Every exiting thread releases its own user allocations before leaving
    // the live-thread set. Therefore the final detach can destroy the remaining
    // process VADs without racing a later THREAD destructor.
    if (!owner->ResourcesReleased) {
        if (thread->Teb && thread->TebSize)
            MmFreeUserMemoryForProcess(owner, thread->Teb);
        if (thread->UserStackBase)
            MmFreeUserMemoryForProcess(owner, thread->UserStackBase);
    }
    thread->Teb = (void*)0;
    thread->TebSize = 0;
    thread->UserStackBase = (void*)0;

    uint32_t remaining = 0;
    int detached = 0;

    uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
    int fwdDetached = thread->ProcessLink.Fwd == &thread->ProcessLink;
    int bwdDetached = thread->ProcessLink.Bwd == &thread->ProcessLink;
    if (fwdDetached != bwdDetached) {
        KeReleaseSpinlock(&owner->ThreadLock, flags);
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "thread process linkage corrupt: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
                (uint64_t)thread,
                thread->ThreadId,
                (uint64_t)thread->ProcessLink.Fwd,
                (uint64_t)thread->ProcessLink.Bwd);
    }

    if (!fwdDetached) {
        if (owner->ThreadCount == 0) {
            KeReleaseSpinlock(&owner->ThreadLock, flags);
            KePanic(KE_PANIC_CODE_SCHEDULER,
                    "thread process count underflow: thread=0x%016llX tid=%u pid=%u\n",
                    (uint64_t)thread,
                    thread->ThreadId,
                    owner->ProcessId);
        }

        KeListRemoveChecked(&thread->ProcessLink);
        owner->ThreadCount--;
        detached = 1;
    }
    remaining = owner->ThreadCount;
    KeReleaseSpinlock(&owner->ThreadLock, flags);

    if (!detached)
        return;

    if (remaining == 0) {
        // ThreadCount represents live process membership, not retained thread
        // handles. Dead THREAD objects may remain waitable after process exit.
        _InterlockedExchange(&owner->Terminating, 1);
        KeStopProcessConstruction(owner);
        KeCompleteProcessExitIfLastThread(owner);
    }
}

// Implements the thread Destroy operation.
static void KiThreadDestroy(OBJECT* obj) {
    THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);

    // A wait that uses auxiliary stack wait blocks must be removed before the
    // kernel stack is released. A terminated thread may never return to the
    // wait syscall to run normal cleanup.
    KeRundownThreadWait(thread, NTLSTATUS_ALERTED);

    if (thread->WaitingMutex || !LIST_IS_EMPTY(&thread->OwnedMutexList)) {
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "thread destroyed with mutex state: thread=0x%016llX tid=%u waiting=0x%016llX ownedEmpty=%u\n",
                (uint64_t)thread,
                thread->ThreadId,
                (uint64_t)thread->WaitingMutex,
                LIST_IS_EMPTY(&thread->OwnedMutexList) ? 1u : 0u);
    }
    if (thread->TimeoutArmed ||
        thread->TimeoutLink.Fwd != &thread->TimeoutLink ||
        thread->TimeoutLink.Bwd != &thread->TimeoutLink) {
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "thread destroyed with timeout state: thread=0x%016llX tid=%u armed=%u link=0x%016llX/0x%016llX\n",
                (uint64_t)thread,
                thread->ThreadId,
                (unsigned)thread->TimeoutArmed,
                (uint64_t)thread->TimeoutLink.Fwd,
                (uint64_t)thread->TimeoutLink.Bwd);
    }

    KiRundownApcState(thread);

    if (thread->ExitEvent) {
        ObDereferenceObject(&thread->ExitEvent->Header);
        thread->ExitEvent = (EVENT*)0;
    }

    if (thread->Teb || thread->TebSize || thread->UserStackBase) {
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "thread destroyed with user memory: thread=0x%016llX tid=%u teb=0x%016llX tebSize=%llu stack=0x%016llX\n",
                (uint64_t)thread,
                thread->ThreadId,
                (uint64_t)thread->Teb,
                (uint64_t)thread->TebSize,
                (uint64_t)thread->UserStackBase);
    }

    if (thread->ProcessLink.Fwd != &thread->ProcessLink ||
        thread->ProcessLink.Bwd != &thread->ProcessLink) {
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "thread destroyed while still process-linked: thread=0x%016llX tid=%u fwd=0x%016llX bwd=0x%016llX\n",
                (uint64_t)thread,
                thread->ThreadId,
                (uint64_t)thread->ProcessLink.Fwd,
                (uint64_t)thread->ProcessLink.Bwd);
    }

    if (thread->Owner) {
        ObDereferenceObject(&thread->Owner->Header);
        thread->Owner = (PROCESS*)0;
    }

    if (thread->KernelStackBase) {
        MmFreePool(thread->KernelStackBase);
        thread->KernelStackBase = (void*)0;
    }
}


// Validates one optional kernel-supplied thread description and returns its length.
static KSTATUS KiValidateThreadName(const char* name, size_t* lengthOut) {
    if (!lengthOut) return NTLSTATUS_INVALID_PARAMETER;
    if (!name) {
        *lengthOut = 0;
        return NTLSTATUS_SUCCESS;
    }

    size_t length = 0;
    while (length < NTL_THREAD_NAME_MAX && name[length])
        length++;
    if (length >= NTL_THREAD_NAME_MAX)
        return NTLSTATUS_INVALID_PARAMETER;

    *lengthOut = length;
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
                       const KE_THREAD_CREATE_PARAMETERS* parameters, THREAD** out) {
    if (!owner || !parameters || !out) return NTLSTATUS_INVALID_PARAMETER;
    if (parameters->SchedulingClass != KeSchedulingNormal ||
        parameters->Priority >= PriorityCount ||
        (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED))
        return NTLSTATUS_INVALID_PARAMETER;

    size_t initialNameLength = 0;
    KSTATUS nameStatus = KiValidateThreadName(parameters->Name, &initialNameLength);
    if (!KSUCCESS(nameStatus)) return nameStatus;

    uint8_t initiallySuspended =
        (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;

    uint32_t threadId = 0;
    KSTATUS idStatus = KeAllocateThreadId(&threadId);
    if (!KSUCCESS(idStatus)) return idStatus;

    THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
    if (!thread) return NTLSTATUS_NO_MEMORY;

    thread->Header.Destroy   = KiThreadDestroy;
    thread->ThreadId         = threadId;
    thread->State                    = initiallySuspended ? ThreadSuspended : ThreadReady;
    thread->BaseSchedulingClass      = parameters->SchedulingClass;
    thread->BasePriority             = parameters->Priority;
    thread->EffectiveSchedulingClass = parameters->SchedulingClass;
    thread->EffectivePriority        = parameters->Priority;
    thread->QuantaConsumed           = 0;
    thread->QuantumTicks      = 0;
    thread->QuantumRemaining  = 0;
    thread->CpuBurstTicks     = 0;
    thread->LastRunTicks      = 0;
    thread->RunningCpu        = KE_INVALID_CPU;
    thread->SuspendCount      = initiallySuspended ? 1u : 0u;
    LIST_INIT(&thread->OwnedMutexList);
    thread->WaitingMutex      = (MUTEX*)0;
    thread->Queued            = 0;
    thread->IsIdleThread      = 0;
    thread->SchedulerFlags    = 0;
    thread->LastCpu           = KE_INVALID_CPU;
    thread->AffinityMask      = parameters->AffinityMask
        ? parameters->AffinityMask
        : KeGetDefaultThreadAffinityMask();
    LIST_INIT(&thread->ProcessLink);
    LIST_INIT(&thread->SchedulerLink);
    thread->WaitObject       = (OBJECT*)0;
    thread->WaitStatus       = NTLSTATUS_SUCCESS;
    thread->WaitListHead     = (LIST_ENTRY*)0;
    thread->AlertableWait    = 0;
    LIST_INIT(&thread->TimeoutLink);
    thread->TimeoutArmed     = 0;
    thread->TimeoutTick      = 0;
    thread->TimeoutStatus    = NTLSTATUS_SUCCESS;
    thread->SwitchLockCookie = 0x202ULL;
    KiInitializeApcState(thread);
    thread->TrapFrame = (KTRAP_FRAME*)0;
    thread->UserExceptionDepth = 0;
    thread->UserExceptionNoncontinuableMask = 0;
    thread->UserExceptionFrameUserAddress = 0;
    memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
    thread->TerminationDeferralCount = 0;
    thread->ExitInProgress = 0;
    thread->NameLock = (SPINLOCK)SPINLOCK_INIT;
    memset(thread->Name, 0, sizeof(thread->Name));
    if (initialNameLength)
        memcpy(thread->Name, parameters->Name, initialNameLength);

    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
    if (!kstack) {
        ObDereferenceObject(&thread->Header);
        return NTLSTATUS_NO_MEMORY;
    }
    memset(kstack, 0, KSTACK_SIZE);
    thread->KernelStackBase = kstack;
    thread->KernelStackTop  = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
    thread->TssRsp0         = thread->KernelStackTop;

    KSTATUS s;
    void* ustack = (void*)0;
    s = MmVadCreateCommitted(owner, USTACK_SIZE, PAGE_SIZE,
                           PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                           MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
                           MmVadStack,
                           &ustack);
    if (!KSUCCESS(s) || !ustack) {
        MmFreePool(kstack);
        thread->KernelStackBase = (void*)0;
        ObDereferenceObject(&thread->Header);
        return NTLSTATUS_NO_MEMORY;
    }
    thread->UserStackBase = ustack;
    // Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
    // nevertheless lets callees immediately use the caller-provided 32-byte
    // home/shadow area above RSP, and MSVC commonly emits a first instruction
    // like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
    // entry stack as if a CALL with a normal home area had happened:
    //
    //   RSP % 16 == 8 at function entry
    //   [RSP + 0]     fake return-address slot
    //   [RSP + 8..39] home/shadow space
    //
    // Without the 32-byte home area, the first spill lands in the high stack
    // guard page and every freshly-created MSVC thread faults immediately.
    uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8 - 32;

    SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
    if (!teb) {
        MmFreeUserMemoryForProcess(owner, ustack);
        thread->UserStackBase = (void*)0;
        MmFreePool(kstack);
        thread->KernelStackBase = (void*)0;
        ObDereferenceObject(&thread->Header);
        return NTLSTATUS_NO_MEMORY;
    }
    memset(teb, 0, sizeof(SYS_TEB));
    teb->Peb = (SYS_PEB*)owner->UserPebBase;
    teb->StackBase = (void*)userStackTop;
    teb->StackLimit = ustack;
    teb->Self = teb;
    teb->ThreadId = thread->ThreadId;
    teb->Magic = SYS_TEB_MAGIC;

    thread->Teb = teb;
    thread->TebSize = sizeof(SYS_TEB);

    thread->UserEntry = entry;
    thread->UserArg   = arg;

    uintptr_t* ksp = (uintptr_t*)thread->KernelStackTop;
    *(--ksp) = userStackTop;
    *(--ksp) = 0x23ULL;
    *(--ksp) = userStackTop;
    *(--ksp) = 0x202ULL;
    *(--ksp) = 0x1BULL;
    *(--ksp) = entry;
    *(--ksp) = arg;

    ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
    KCONTEXT* ctx = (KCONTEXT*)ksp;
    memset(ctx, 0, sizeof(KCONTEXT));
    ctx->Rip = (uint64_t)KeThreadEntry;
    thread->SavedContext = ctx;

    ObReferenceObject(&owner->Header);
    thread->Owner = owner;
    
    // Allocate the exit event before the thread becomes visible. Manual-reset
    // so multiple joiners (if any) all wake up when the thread exits.
    EVENT* exitEvent;
    s = KeEventCreate(EventManualReset, 0, &exitEvent);
    if (!KSUCCESS(s)) {
        if (thread->Teb && thread->TebSize) MmFreeUserMemoryForProcess(owner, thread->Teb);
        thread->Teb = (void*)0;
        thread->TebSize = 0;
        MmFreeUserMemoryForProcess(owner, ustack);
        thread->UserStackBase = (void*)0;
        MmFreePool(kstack);
        thread->KernelStackBase = (void*)0;
        ObDereferenceObject(&owner->Header);
        thread->Owner = (PROCESS*)0;
        ObDereferenceObject(&thread->Header);
        return s;
    }
    thread->ExitEvent = exitEvent;

    uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
    KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
    owner->ThreadCount++;
    KeReleaseSpinlock(&owner->ThreadLock, flags);

    KeInitThreadFpuContext(thread);

    // Hand the creator a short-lived reference before making the thread
    // runnable. On SMP a new user thread can run and exit before
    // KeCreateThread() returns; without this reference, callers that merely
    // read ThreadId for logging or syscall output can race the scheduler's
    // dead-thread dereference and touch freed pool.
    ObReferenceObject(&thread->Header);
    *out = thread;

    if (!initiallySuspended)
        KeReadyThread(thread);
    return NTLSTATUS_SUCCESS;
}


// Enters a region that must retain the current thread during process termination.
void KeEnterTerminationDeferral(void) {
    KeAssertIrql(PASSIVE_LEVEL);
    THREAD* thread = KeGetCurrentThread();
    if (!thread || thread->IsIdleThread)
        return;
    _InterlockedIncrement(&thread->TerminationDeferralCount);
}

// Leaves a process-termination deferral region.
void KeLeaveTerminationDeferral(void) {
    KeAssertIrql(PASSIVE_LEVEL);
    THREAD* thread = KeGetCurrentThread();
    if (!thread || thread->IsIdleThread)
        return;

    long remaining = _InterlockedDecrement(&thread->TerminationDeferralCount);
    if (remaining < 0) {
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "thread termination deferral underflow: thread=0x%016llX tid=%u\n",
                (uint64_t)thread,
                thread->ThreadId);
    }

    if (remaining == 0 && thread->Owner && KeIsProcessTerminating(thread->Owner))
        KiRequestReschedule();
}

// Returns whether one thread still owns termination-sensitive kernel work.
int KeThreadTerminationDeferred(const THREAD* thread) {
    return thread && _InterlockedCompareExchange((volatile long*)&thread->TerminationDeferralCount,
                                                  0,
                                                  0) > 0;
}

// Returns whether terminal teardown owns the current thread stack.
int KeThreadExitInProgress(const THREAD* thread) {
    return thread && _InterlockedCompareExchange((volatile long*)&thread->ExitInProgress,
                                                  0,
                                                  0) != 0;
}

// Implements the exit Thread operation.
__declspec(dllexport) void KeExitThread(THREAD* thread, int code) {
    (void)code;
    if (!thread) return;

    // Terminal teardown must run exactly once and must retain this kernel
    // stack until thread/process completion is fully published. Hardware
    // interrupts remain enabled, but interrupt-return scheduling observes
    // ExitInProgress and leaves the thread on-CPU until the final KeYield().
    if (_InterlockedCompareExchange(&thread->ExitInProgress, 1, 0) != 0)
        KePanic(KE_PANIC_CODE_SCHEDULER,
                "recursive thread exit: thread=0x%016llX tid=%u\n",
                (uint64_t)thread,
                thread->ThreadId);

    // Reject new APCs as soon as teardown owns the thread. The scheduler must
    // not interpret ThreadDead as permission to switch away until the explicit
    // terminal yield below.
    thread->State = ThreadDead;

    KiRundownApcState(thread);
    KiMutexRundownThread(thread);
    KeFinalizeThreadExit(thread);
    KeYield();

    KePanic(KE_PANIC_CODE_SCHEDULER,
            "dead thread resumed after terminal yield: thread=0x%016llX tid=%u\n",
            (uint64_t)thread,
            thread->ThreadId);
}

// KeCreateKernelThread
// Kernel threads never go to ring 3: they have no user stack, no TEB, and
// their entry stub (KeKernelThreadEntry) calls the function directly then
// calls KeExitThread instead of IRETQ-ing.

// Stack layout built here (from KernelStackTop downward):
//   [ksp - 8 ]  arg          <-- popped by KeKernelThreadEntry as r12
//   [ksp - 16]  fn           <-- popped by KeKernelThreadEntry as rbx
//   [ksp - 16 - sizeof(KCONTEXT)]  KCONTEXT { .Rip = KeKernelThreadEntry }.
extern void KeKernelThreadEntry(void);

static KSTATUS KiCreateKernelThread(uintptr_t fn, uintptr_t arg,
                                      const KE_THREAD_CREATE_PARAMETERS* parameters,
                                      THREAD** out) {
    // Kernel worker threads are owned by the system process. The scheduler's
    // hand-built CPU idle thread stays attached to the dedicated idle process, while
    // driver, async I/O, and init worker threads use the system process.
    PROCESS* owner = KeGetSystemProcess();
    if (!owner) {
        THREAD* currentThread = KeGetCurrentThread();
        owner = currentThread ? currentThread->Owner : (PROCESS*)0;
    }
    if (!owner || !parameters) return NTLSTATUS_INVALID_PARAMETER;
    if (parameters->Flags & ~KE_THREAD_CREATE_FLAG_SUSPENDED)
        return NTLSTATUS_INVALID_PARAMETER;
    if ((parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) && !out)
        return NTLSTATUS_INVALID_PARAMETER;
    if (parameters->SchedulingClass == KeSchedulingNormal) {
        if (parameters->Priority >= PriorityCount) return NTLSTATUS_INVALID_PARAMETER;
    } else if (parameters->SchedulingClass == KeSchedulingRealtime) {
        if (parameters->Priority > KE_REALTIME_PRIORITY_MAX) return NTLSTATUS_INVALID_PARAMETER;
    } else {
        return NTLSTATUS_INVALID_PARAMETER;
    }

    size_t initialNameLength = 0;
    KSTATUS nameStatus = KiValidateThreadName(parameters->Name, &initialNameLength);
    if (!KSUCCESS(nameStatus)) return nameStatus;

    uint8_t initiallySuspended =
        (parameters->Flags & KE_THREAD_CREATE_FLAG_SUSPENDED) ? 1u : 0u;

    uint32_t threadId = 0;
    KSTATUS idStatus = KeAllocateThreadId(&threadId);
    if (!KSUCCESS(idStatus)) return idStatus;

    THREAD* thread = (THREAD*)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
    if (!thread) return NTLSTATUS_NO_MEMORY;

    thread->Header.Destroy  = KiThreadDestroy;
    thread->ThreadId        = threadId;
    thread->State                    = initiallySuspended ? ThreadSuspended : ThreadReady;
    thread->BaseSchedulingClass      = parameters->SchedulingClass;
    thread->BasePriority             = parameters->Priority;
    thread->EffectiveSchedulingClass = parameters->SchedulingClass;
    thread->EffectivePriority        = parameters->Priority;
    thread->QuantaConsumed           = 0;
    thread->QuantumTicks     = 0;
    thread->QuantumRemaining = 0;
    thread->CpuBurstTicks    = 0;
    thread->LastRunTicks     = 0;
    thread->RunningCpu       = KE_INVALID_CPU;
    thread->SuspendCount     = initiallySuspended ? 1u : 0u;
    LIST_INIT(&thread->OwnedMutexList);
    thread->WaitingMutex     = (MUTEX*)0;
    thread->Queued           = 0;
    thread->IsIdleThread     = 0;
    thread->SchedulerFlags   = 0;
    thread->LastCpu          = KE_INVALID_CPU;
    thread->AffinityMask     = parameters->AffinityMask
        ? parameters->AffinityMask
        : KeGetDefaultThreadAffinityMask();
    LIST_INIT(&thread->ProcessLink);
    LIST_INIT(&thread->SchedulerLink);
    thread->WaitObject      = (OBJECT*)0;
    thread->WaitStatus      = NTLSTATUS_SUCCESS;
    thread->WaitListHead    = (LIST_ENTRY*)0;
    thread->AlertableWait   = 0;
    LIST_INIT(&thread->TimeoutLink);
    thread->TimeoutArmed    = 0;
    thread->TimeoutTick     = 0;
    thread->TimeoutStatus   = NTLSTATUS_SUCCESS;
    thread->SwitchLockCookie = 0x202ULL;
    KiInitializeApcState(thread);
    thread->TrapFrame       = (KTRAP_FRAME*)0;
    thread->UserExceptionDepth = 0;
    thread->UserExceptionNoncontinuableMask = 0;
    thread->UserExceptionFrameUserAddress = 0;
    memset(thread->UserExceptionExitStatus, 0, sizeof(thread->UserExceptionExitStatus));
    thread->TerminationDeferralCount = 0;
    thread->ExitInProgress = 0;
    thread->NameLock = (SPINLOCK)SPINLOCK_INIT;
    memset(thread->Name, 0, sizeof(thread->Name));
    if (initialNameLength)
        memcpy(thread->Name, parameters->Name, initialNameLength);
    thread->UserStackBase   = (void*)0;   // no user stack
    thread->Teb             = (void*)0;   // no TEB; GS_BASE will be 0 in kernel ctx
    thread->TebSize         = 0;

    void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
    if (!kstack) {
        ObDereferenceObject(&thread->Header);
        return NTLSTATUS_NO_MEMORY;
    }
    memset(kstack, 0, KSTACK_SIZE);
    thread->KernelStackBase = kstack;
    thread->KernelStackTop  = ((uintptr_t)kstack + KSTACK_SIZE) & ~(uintptr_t)15;
    thread->TssRsp0         = thread->KernelStackTop;

    // Build the initial stack: arg, fn, then KCONTEXT.
    uintptr_t* ksp = (uintptr_t*)thread->KernelStackTop;
    *(--ksp) = arg;   // popped second (-> r12)
    *(--ksp) = fn;    // popped first (-> rbx)

    ksp -= sizeof(KCONTEXT) / sizeof(uintptr_t);
    KCONTEXT* ctx = (KCONTEXT*)ksp;
    memset(ctx, 0, sizeof(KCONTEXT));
    ctx->Rip = (uint64_t)KeKernelThreadEntry;
    thread->SavedContext = ctx;

    // Register with the owning process.
    ObReferenceObject(&owner->Header);
    thread->Owner = owner;

    uintptr_t flags = KeAcquireSpinlock(&owner->ThreadLock);
    KeListInsertTailChecked(&owner->ThreadList, &thread->ProcessLink);
    owner->ThreadCount++;
    KeReleaseSpinlock(&owner->ThreadLock, flags);

    // Hand the creator a short-lived reference before making the kernel
    // thread runnable. Kernel workers can run, signal their completion event,
    // exit, and be reaped on another CPU before KeCreateKernelThread()'s
    // caller drops its returned pointer. The live scheduler reference remains
    // separate and is released by KeSchedulerPostSwitch() after the dead
    // thread has switched away.
    if (out) {
        ObReferenceObject(&thread->Header);
        *out = thread;
    }

    KeInitThreadFpuContext(thread);
    if (!initiallySuspended)
        KeReadyThread(thread);
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
                              const KE_THREAD_CREATE_PARAMETERS* parameters,
                              THREAD** out) {
    return KiCreateKernelThread(fn, arg, parameters, out);
}

// Replaces one thread's diagnostic description.
KSTATUS KeSetThreadName(THREAD* thread, const char* name, size_t length) {
    if (!thread || (length && !name) || length >= NTL_THREAD_NAME_MAX)
        return NTLSTATUS_INVALID_PARAMETER;

    uintptr_t flags = KeAcquireSpinlock(&thread->NameLock);
    if (length)
        memcpy(thread->Name, name, length);
    thread->Name[length] = '\0';
    KeReleaseSpinlock(&thread->NameLock, flags);
    return NTLSTATUS_SUCCESS;
}

// Queries one stable snapshot of a mutable thread description.
size_t KeQueryThreadName(THREAD* thread, char* buffer, size_t capacity) {
    if (!thread) return 0;

    uintptr_t flags = KeAcquireSpinlock(&thread->NameLock);
    size_t length = 0;
    while (length < NTL_THREAD_NAME_MAX && thread->Name[length])
        length++;
    if (buffer && capacity) {
        size_t copyLength = length < capacity - 1u ? length : capacity - 1u;
        if (copyLength)
            memcpy(buffer, thread->Name, copyLength);
        buffer[copyLength] = '\0';
    }
    KeReleaseSpinlock(&thread->NameLock, flags);
    return length;
}

// Prepares block Thread.
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
    // Called with the synchronization object's spinlock held, so the caller is
    // normally at DISPATCH_LEVEL here. This function only marks intent to
    // block; the actual scheduling point is KeCommitBlockThread at PASSIVE.
    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
    if (!thread) return;
    KiSchedulerMarkThreadWaiting(thread);

    thread->WaitObject    = waitObject;
    thread->WaitStatus    = NTLSTATUS_PENDING;
    thread->WaitListHead  = (LIST_ENTRY*)0;
    thread->AlertableWait = alertable ? 1 : 0;
    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
    thread->WaitRundownContext = (void*)0;
}

// Updates thread Wait List.
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
    if (!thread) return;
    thread->WaitListHead = waitListHead;
}

// Registers cleanup for waits with auxiliary wait blocks.
void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context) {
    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
    if (!thread) return;
    thread->WaitRundownRoutine = routine;
    thread->WaitRundownContext = context;
}

// Clears auxiliary wait cleanup after the owning wait has removed its blocks.
void KeClearThreadWaitRundown(THREAD* thread) {
    KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
    if (!thread) return;
    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
    thread->WaitRundownContext = (void*)0;
}

// Runs auxiliary wait cleanup for abort and thread teardown paths.
void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
    if (!thread || !thread->WaitRundownRoutine) return;

    KE_WAIT_RUNDOWN_ROUTINE routine = thread->WaitRundownRoutine;
    void* context = thread->WaitRundownContext;
    thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
    thread->WaitRundownContext = (void*)0;
    routine(thread, context, status);
}

// Implements the commit Block Thread operation.
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
    KeAssertIrql(PASSIVE_LEVEL);
    if (!thread) return NTLSTATUS_INVALID_PARAMETER;

    if (alertable)
        thread->AlertableWait = 1;

    // A signal/APC may have arrived after the waiter was placed on the
    // object's wait list but before this thread reached the scheduling point.
    // In that case KeUnblockThreadWithStatus completed the wait in-place and
    // there is nothing left to block on.
    if (thread->State != ThreadWaiting) {
        KSTATUS status = thread->WaitStatus;
        thread->AlertableWait = 0;
        return status;
    }

    // A kernel APC can be queued before the waiter finishes publishing its
    // alertable state. In that race the queueing CPU cannot wake the wait, and
    // APC delivery is deliberately deferred while State is ThreadWaiting.
    // Consume the prepared wait here before the thread can actually sleep.
    if (alertable && thread->KernelApcPending && thread->KernelApcDisable == 0) {
        if (KeAbortThreadWait(thread, NTLSTATUS_KERNEL_APC)) {
            thread->AlertableWait = 0;
            KiDeliverKernelApcs();
            return NTLSTATUS_KERNEL_APC;
        }
    }

    // If a user APC was already pending before the thread fully blocked,
    // consume the alertable wait immediately instead of going to sleep.
    if (alertable && thread->UserApcPending && thread->UserApcDispatcher) {
        if (KeAbortThreadWait(thread, NTLSTATUS_USER_APC)) {
            thread->AlertableWait = 0;
            return NTLSTATUS_USER_APC;
        }
    }

    KiSchedulerNoteVoluntaryBlock(thread);
    KeYield();

    // Kernel APCs are delivered before an alertable wait returns
    // NTLSTATUS_KERNEL_APC to its caller.
    KiDeliverKernelApcs();

    thread->AlertableWait = 0;
    return thread->WaitStatus;
}

// Implements the block Thread operation.
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) {
    KePrepareBlockThread(thread, waitObject, alertable);
    return KeCommitBlockThread(thread, alertable);
}

// Implements the unblock Thread With Status operation.
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
    if (!thread) return;
    KiSchedulerWakeThread(thread, status);
}

// Returns whether the scheduler link still belongs to an object wait list.
static int KiThreadWaitLinkIsLinked(const THREAD* thread) {
    if (!thread) return 0;
    return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
           thread->SchedulerLink.Bwd != &thread->SchedulerLink;
}

// Removes a thread from one wait list while that wait object's lock is held.
static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
    if (!thread || !waitListHead) return 0;
    if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead)
        return 0;

    // The signaling side pops the wait-list entry before it wakes the thread.
    // In that handoff window the scheduler link is already self-linked; abort
    // must not remove it again or turn a successful mutex/semaphore handoff
    // into an aborted wait for a thread that may already own the object.
    if (!KiThreadWaitLinkIsLinked(thread))
        return 0;

    KeListRemoveChecked(&thread->SchedulerLink);
    thread->WaitListHead = (LIST_ENTRY*)0;
    return 1;
}

// Removes a thread from an object-owned wait queue using that object's lock.
static int KiAbortObjectWait(THREAD* thread) {
    if (!thread || !thread->WaitObject || !thread->WaitListHead)
        return 0;

    OBJECT* object = thread->WaitObject;
    switch (object->Type) {
        case ObjectTypeEvent: {
            EVENT* event = CONTAINING_RECORD(object, EVENT, Header);
            uintptr_t flags = KeAcquireSpinlock(&event->Lock);
            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &event->WaitList);
            KeReleaseSpinlock(&event->Lock, flags);
            return removed;
        }
        case ObjectTypeMutex: {
            MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
            return KiMutexAbortWait(mutex, thread);
        }
        case ObjectTypeSemaphore: {
            SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
            uintptr_t flags = KeAcquireSpinlock(&semaphore->Lock);
            int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &semaphore->WaitList);
            KeReleaseSpinlock(&semaphore->Lock, flags);
            return removed;
        }
        default:
            return 0;
    }
}

// Aborts a thread's current wait regardless of whether the wait was marked
// alertable. Process teardown uses this for crash/kill paths: a background
// worker blocked forever on a semaphore must not keep the process exit event
// unsignaled after the foreground thread has faulted. Object wait-list removal
// must use the object's own lock; otherwise signal/abort races can leave stale
// scheduler links in an event, mutex, or semaphore wait queue.
int KeAbortThreadWait(THREAD* thread, KSTATUS status) {
    if (!thread) return 0;
    if (thread->State != ThreadWaiting) return 0;

    if (thread->WaitRundownRoutine) {
        KeRundownThreadWait(thread, status);
        return thread->State != ThreadWaiting;
    }

    if (KiAbortObjectWait(thread)) {
        KeUnblockThreadWithStatus(thread, status);
        return 1;
    }

    // A signaler that already dequeued an object waiter owns completion. Let
    // that handoff finish rather than returning timeout after ownership or a
    // semaphore count has already transferred to this thread.
    if (thread->WaitObject)
        return 0;

    // Delays and auxiliary scheduler waits have no object-owned queue.
    KeUnblockThreadFromWaitListWithStatus(thread, status);
    return 1;
}

// Implements the unblock Thread operation.
void KeUnblockThread(THREAD* thread) {
    KeUnblockThreadWithStatus(thread, NTLSTATUS_SUCCESS);
}

// Implements the alert Thread With Status operation.
static KSTATUS KiAlertThreadWithStatus(THREAD* thread, KSTATUS status) {
    if (!thread) return NTLSTATUS_INVALID_PARAMETER;

    if (thread->State != ThreadWaiting || !thread->AlertableWait)
        return NTLSTATUS_PENDING;

    return KeAbortThreadWait(thread, status)
        ? NTLSTATUS_SUCCESS
        : NTLSTATUS_PENDING;
}

// Implements the alert Thread operation.
KSTATUS KeAlertThread(THREAD* thread) {
    return KiAlertThreadWithStatus(thread, NTLSTATUS_ALERTED);
}

// Implements the wake Thread For Kernel Apc operation.
void KiWakeThreadForKernelApc(THREAD* thread) {
    (void)KiAlertThreadWithStatus(thread, NTLSTATUS_KERNEL_APC);
}

// Implements the wake Thread For User Apc operation.
void KiWakeThreadForUserApc(THREAD* thread) {
    (void)KiAlertThreadWithStatus(thread, NTLSTATUS_USER_APC);
}

void KeInitThreadFpuContext(THREAD* thread)
{
    uintptr_t raw =
        (uintptr_t)thread->FpuStorage;

    uintptr_t aligned =
        (raw + 15) & ~(uintptr_t)15;

    thread->FpuContext = (void*)aligned;

    memset(thread->FpuContext, 0, 512);

    *(uint16_t*)((uint8_t*)thread->FpuContext + 0x00) = 0x037F;
    *(uint32_t*)((uint8_t*)thread->FpuContext + 0x18) = 0x1F80;
}