nt-lite/code/kernel/ke_work.c

// Kernel passive-level work queue. This file owns the generic mechanism used
// to move ISR/DPC-originated work to a sleepable system thread context.

#include <stdbool.h>
#include <kernel/ke_list.h>
#include <stddef.h>
#include <intrin.h>

#include <kernel/ke_work.h>
#include <kernel/ke_event.h>
#include <kernel/ke_dpc.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_debug.h>

static SPINLOCK gWorkQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gWorkQueueList;
static EVENT* gWorkQueueEvent;
static KDPC gWorkQueueWakeDpc;
static volatile long gWorkQueueListReady;
static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started.

// Signals the passive worker after an enqueue from hardware interrupt context.
static void KiWorkQueueWakeDpc(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) {
    (void)dpc;
    (void)context;
    (void)systemArgument1;
    (void)systemArgument2;
    KeAssertIrql(DISPATCH_LEVEL);

    EVENT* event = gWorkQueueEvent;
    if (event)
        KeSignalEvent(event);
}

static void KiWorkQueueWorker(void* arg) {
    (void)arg;
    KeAssertIrql(PASSIVE_LEVEL);

    while (true) {
        (void)KeWaitForEvent(gWorkQueueEvent, 0);

        while (true) {
            KWORK_ITEM* item = (KWORK_ITEM*)0;
            uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
            if (!LIST_IS_EMPTY(&gWorkQueueList)) {
                LIST_ENTRY* node = KeListPopHeadChecked(&gWorkQueueList);
                item = CONTAINING_RECORD(node, KWORK_ITEM, Link);
            }
            KeReleaseSpinlock(&gWorkQueueLock, flags);

            if (!item)
                break;

            // Allow the item to be queued again while it is running. This keeps
            // level-triggered users such as cursor updates naturally coalesced
            // but does not lose an update that arrives during the callback.
            _InterlockedExchange(&item->Queued, 0);
            if (item->Routine)
                item->Routine(item->Context);
        }
    }
}

void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context) {
    if (!item)
        return;
    LIST_INIT(&item->Link);
    item->Routine = routine;
    item->Context = context;
    item->Queued = 0;
}

bool KeIsWorkQueueStarted(void) {
    return _InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 2;
}

bool KeQueueWorkItem(KWORK_ITEM* item) {
    if (!item || !item->Routine)
        return false;

    if (_InterlockedCompareExchange(&item->Queued, 1, 0) != 0)
        return false;

    if (!_InterlockedCompareExchange(&gWorkQueueListReady, 1, 1)) {
        _InterlockedExchange(&item->Queued, 0);
        return false;
    }

    uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
    KeListInsertTailChecked(&gWorkQueueList, &item->Link);
    EVENT* event = gWorkQueueEvent;
    KeReleaseSpinlock(&gWorkQueueLock, flags);

    if (event) {
        // Event signaling can detach waiters, so callers above passive level
        // defer the wake until no caller-owned spinlock remains held.
        if (KeGetCurrentIrql() == PASSIVE_LEVEL)
            KeSignalEvent(event);
        else
            (void)KeInsertQueueDpc(&gWorkQueueWakeDpc, (void*)0, (void*)0);
    }
    return true;
}

KSTATUS KeStartWorkQueue(void) {
    long state = _InterlockedCompareExchange(&gWorkQueueStartState, 1, 0);
    if (state == 2)
        return NTLSTATUS_SUCCESS;
    if (state == 1) {
        while (_InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 1)
            KeYield();
        return KeIsWorkQueueStarted() ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
    }

    LIST_INIT(&gWorkQueueList);
    KeInitializeDpc(&gWorkQueueWakeDpc, KiWorkQueueWakeDpc, (void*)0);
    _InterlockedExchange(&gWorkQueueListReady, 1);

    KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
    if (KSUCCESS(s)) {
        THREAD* worker = (THREAD*)0;
        const KE_THREAD_CREATE_PARAMETERS parameters =
            KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "work.queue");
        s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker,
                                 0,
                                 &parameters,
                                 &worker);
        // Persistent system worker; keep the creator reference for kernel life.
        (void)worker;
    }

    if (KSUCCESS(s)) {
        _InterlockedExchange(&gWorkQueueStartState, 2);
        KeSignalEvent(gWorkQueueEvent);
    } else {
        _InterlockedExchange(&gWorkQueueStartState, 0);
    }
    return s;
}