nt-lite/code/kernel/ke_selftest.c

// Kernel self-tests. This file runs bring-up regression checks before normal user workloads start.

#include <stdint.h>
#include <stddef.h>
#include <string.h>

#include <kernel/hal/hal_api.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_event.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_sched.h>
#include <ntlite/status.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <kernel/io_dma.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_swap.h>
#include <kernel/mm_pfn.h>
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>

// Kernel APC self-test.

typedef struct KI_APC_SELFTEST_STATE {
    EVENT* ReadyEvent;
    EVENT* GoEvent;
    EVENT* DoneEvent;
    EVENT* FinishedEvent;

    THREAD* WorkerThread;

    volatile int SelfDelivered;
    volatile int DeferredDelivered;
    volatile int CrossDelivered;
} KI_APC_SELFTEST_STATE;

static void KiApcSelfTestKernelRoutine(KAPC* apc,
                                        void** normalRoutine,
                                        void** normalContext,
                                        void** systemArgument1,
                                        void** systemArgument2) {
    DbgPrint("APC selftest: kernel routine apc=0x%016llX normal=0x%016llX ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)apc,
             (uint64_t)*normalRoutine,
             (uint64_t)*normalContext,
             (uint64_t)*systemArgument1,
             (uint64_t)*systemArgument2,
             KeGetCurrentIrql());
}

static void KiApcSelfTestNormalRoutine(void* normalContext,
                                        void* systemArgument1,
                                        void* systemArgument2) {
    volatile int* counter = (volatile int*)normalContext;

    DbgPrint("APC selftest: normal routine ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)normalContext,
             (uint64_t)systemArgument1,
             (uint64_t)systemArgument2,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: normal routine ran above PASSIVE_LEVEL\n");

    (*counter)++;
}

static void KiApcSelfTestCrossThreadRoutine(void* normalContext,
                                             void* systemArgument1,
                                             void* systemArgument2) {
    KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext;

    DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)systemArgument1,
             (uint64_t)systemArgument2,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread APC ran above PASSIVE_LEVEL\n");

    st->CrossDelivered++;
    KeSignalEvent(st->DoneEvent);
}

// Implements the apc Self Test Worker operation.
static void KiApcSelfTestWorker(void* arg) {
    KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)arg;

    DbgPrint("APC selftest: worker started, signaling ready\n");
    KeSignalEvent(st->ReadyEvent);

    DbgPrint("APC selftest: worker waiting for go event\n");
    KSTATUS s = KeWaitForEvent(st->GoEvent, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker wait failed status=%d\n", s);

    // KeCommitBlockThread(..., alertable) should deliver pending kernel APCs before this
    // wait returns. If that hook is broken, DoneEvent will never be signaled.
    DbgPrint("APC selftest: worker resumed after go event, cross=%d\n",
             st->CrossDelivered);

    if (st->CrossDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker resumed but APC was not delivered\n");

    KeSignalEvent(st->FinishedEvent);
    KeExitThread(KeGetCurrentThread(), 0);
}

typedef struct KI_APC_ALERTABLE_SELFTEST_STATE {
    EVENT* ReadyEvent;
    EVENT* WaitEvent;
    EVENT* FinishedEvent;
    THREAD* WorkerThread;

    volatile int ApcDelivered;
    volatile KSTATUS WaitStatus;
} KI_APC_ALERTABLE_SELFTEST_STATE;

static void KiApcAlertableSelfTestNormalRoutine(void* normalContext,
                                                 void* systemArgument1,
                                                 void* systemArgument2) {
    KI_APC_ALERTABLE_SELFTEST_STATE* st =
        (KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext;

    DbgPrint("APC selftest: alertable APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)systemArgument1,
             (uint64_t)systemArgument2,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC ran above PASSIVE_LEVEL\n");

    st->ApcDelivered++;
}

// Implements the apc Alertable Self Test Worker operation.
static void KiApcAlertableSelfTestWorker(void* arg) {
    KI_APC_ALERTABLE_SELFTEST_STATE* st =
        (KI_APC_ALERTABLE_SELFTEST_STATE*)arg;

    DbgPrint("APC selftest: alertable worker ready, entering alertable wait\n");
    KeSignalEvent(st->ReadyEvent);

    KSTATUS s = KeWaitForEvent(st->WaitEvent, 1);
    st->WaitStatus = s;

    DbgPrint("APC selftest: alertable wait returned status=%u delivered=%d\n",
             s, st->ApcDelivered);

    if (s != NTLSTATUS_KERNEL_APC)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable wait returned wrong status=%u\n", s);

    if (st->ApcDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC delivery count wrong=%d\n",
                st->ApcDelivered);

    KeSignalEvent(st->FinishedEvent);
    KeExitThread(KeGetCurrentThread(), 0);
}

// Runs kernel Apc Self Test.
static void KiRunKernelApcSelfTest(void) {
    DbgPrint("APC selftest: begin\n");

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: called above PASSIVE_LEVEL\n");

    THREAD* self = KeGetCurrentThread();
    if (!self)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: no current thread\n");

    // Test 1: queue APC to current thread at PASSIVE_LEVEL.
    {
        KAPC apc;
        volatile int count = 0;

        KeInitializeApc(&apc,
                        self,
                        KiApcSelfTestKernelRoutine,
                        KiApcSelfTestNormalRoutine,
                        (void*)&count);

        if (!KeInsertQueueApc(&apc, (void*)0x11111111, (void*)0x22222222))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: self queue failed\n");

        if (count != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: self APC was not delivered immediately, count=%d\n", count);

        DbgPrint("APC selftest: self APC passed\n");
    }

    // Test 2: disable kernel APCs, queue, then re-enable.
    {
        KAPC apc;
        volatile int count = 0;

        KeInitializeApc(&apc,
                        self,
                        KiApcSelfTestKernelRoutine,
                        KiApcSelfTestNormalRoutine,
                        (void*)&count);

        KeEnterCriticalRegion();

        if (!KeInsertQueueApc(&apc, (void*)0x33333333, (void*)0x44444444))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: deferred queue failed\n");

        if (count != 0)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: APC delivered while critical region held, count=%d\n", count);

        KeLeaveCriticalRegion();

        if (count != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: deferred APC was not delivered after leave, count=%d\n", count);

        DbgPrint("APC selftest: critical-region deferral passed\n");
    }

    // Test 3: queue APC to another blocked kernel thread, then wake it.
    {
        KI_APC_SELFTEST_STATE st;
        KAPC apc;
        memset(&st, 0, sizeof(st));

        KSTATUS s;
        s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: ReadyEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.GoEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: GoEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.DoneEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: DoneEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: FinishedEvent create failed %d\n", s);

        const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
        s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker,
                                 (uintptr_t)&st,
                                 &workerParameters,
                                 &st.WorkerThread);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker create failed %d\n", s);

        s = KeWaitForEvent(st.ReadyEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: ready wait failed %d\n", s);

        DbgPrint("APC selftest: worker ready, queueing cross-thread APC\n");

        KeInitializeApc(&apc,
                        st.WorkerThread,
                        KiApcSelfTestKernelRoutine,
                        KiApcSelfTestCrossThreadRoutine,
                        (void*)&st);

        if (!KeInsertQueueApc(&apc, (void*)0x55555555, (void*)0x66666666))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread queue failed\n");

        if (st.CrossDelivered != 0)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread APC delivered before target resumed\n");

        KeSignalEvent(st.GoEvent);

        s = KeWaitForEvent(st.DoneEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: done wait failed %d\n", s);

        if (st.CrossDelivered != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread delivery count wrong, count=%d\n",
                    st.CrossDelivered);

        // The APC signals DoneEvent before the worker returns from the wait.
        // Wait for an explicit worker-owned event before dropping stack-backed
        // state and event references. THREAD.ExitEvent is not part of this
        // kernel-thread contract on all builds.
        s = KeWaitForEvent(st.FinishedEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker finished wait failed %d\n", s);

        DbgPrint("APC selftest: cross-thread APC passed\n");

        ObDereferenceObject(&st.ReadyEvent->Header);
        ObDereferenceObject(&st.GoEvent->Header);
        ObDereferenceObject(&st.DoneEvent->Header);
        ObDereferenceObject(&st.FinishedEvent->Header);
        ObDereferenceObject(&st.WorkerThread->Header);
    }

    // Test 4: queue APC to a thread blocked in an alertable wait.
    // The APC itself must wake the waiter; no event signal is sent.
    {
        KI_APC_ALERTABLE_SELFTEST_STATE st;
        KAPC apc;
        memset(&st, 0, sizeof(st));
        st.WaitStatus = NTLSTATUS_PENDING;

        KSTATUS s;
        s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert ReadyEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.WaitEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert WaitEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert FinishedEvent create failed %d\n", s);

        const KE_THREAD_CREATE_PARAMETERS workerParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
        s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker,
                                 (uintptr_t)&st,
                                 &workerParameters,
                                 &st.WorkerThread);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable worker create failed %d\n", s);

        s = KeWaitForEvent(st.ReadyEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert ready wait failed %d\n", s);

        DbgPrint("APC selftest: queueing APC to alertable waiter\n");

        KeInitializeApc(&apc,
                        st.WorkerThread,
                        KiApcSelfTestKernelRoutine,
                        KiApcAlertableSelfTestNormalRoutine,
                        (void*)&st);

        if (!KeInsertQueueApc(&apc, (void*)(unsigned long long)0x77777777, (void*)(unsigned long long)0x88888888))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC queue failed\n");

        s = KeWaitForEvent(st.FinishedEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert finished wait failed %d\n", s);

        if (st.WaitStatus != NTLSTATUS_KERNEL_APC || st.ApcDelivered != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable result bad status=%u delivered=%d\n",
                    st.WaitStatus, st.ApcDelivered);

        DbgPrint("APC selftest: alertable wait passed\n");

        ObDereferenceObject(&st.ReadyEvent->Header);
        ObDereferenceObject(&st.WaitEvent->Header);
        ObDereferenceObject(&st.FinishedEvent->Header);
        ObDereferenceObject(&st.WorkerThread->Header);
    }

    DbgPrint("APC selftest: PASS\n");
}

// IRP + kernel APC self-test.

typedef struct KI_IRP_APC_SELFTEST_STATE {
    EVENT* CompletionEvent;
    DEVICE_OBJECT* DiskDevice;
    IRP Irp;
    KAPC Apc;
    uint8_t* Buffer;

    volatile int ApcDelivered;
    volatile KSTATUS ApcStatus;
    volatile size_t ApcInformation;
} KI_IRP_APC_SELFTEST_STATE;

static void KiIrpApcSelfTestKernelRoutine(KAPC* apc,
                                           void** normalRoutine,
                                           void** normalContext,
                                           void** systemArgument1,
                                           void** systemArgument2) {
    DbgPrint("IRP/APC selftest: kernel routine apc=0x%016llX irp=0x%016llX status=%u irql=%u\n",
             (uint64_t)apc,
             (uint64_t)*systemArgument1,
             (uint32_t)(uintptr_t)*systemArgument2,
             KeGetCurrentIrql());

    (void)normalRoutine;
    (void)normalContext;
}

static void KiIrpApcSelfTestNormalRoutine(void* normalContext,
                                           void* systemArgument1,
                                           void* systemArgument2) {
    KI_IRP_APC_SELFTEST_STATE* st = (KI_IRP_APC_SELFTEST_STATE*)normalContext;
    IRP* irp = (IRP*)systemArgument1;
    KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;

    DbgPrint("IRP/APC selftest: APC delivered irp=0x%016llX status=%u info=%llu irql=%u\n",
             (uint64_t)irp,
             status,
             (uint64_t)(irp ? irp->Information : 0),
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: normal routine ran above PASSIVE_LEVEL\n");

    if (irp != &st->Irp)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: APC arg1 was not the submitted IRP\n");

    st->ApcDelivered++;
    st->ApcStatus = status;
    st->ApcInformation = irp->Information;
}

// Runs irp Kernel Apc Self Test.
static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
    DbgPrint("IRP/APC selftest: begin\n");

    (void)hal;
    if (!IoDiskDevicePresent(0)) {
        DbgPrint("IRP/APC selftest: SKIP, drive 0 not present\n");
        return;
    }

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: called above PASSIVE_LEVEL\n");

    KI_IRP_APC_SELFTEST_STATE st;
    memset(&st, 0, sizeof(st));

    KSTATUS s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: completion event create failed %u\n", s);

    st.Buffer = (uint8_t*)MmAllocatePool(512);
    if (!st.Buffer)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: no buffer\n");
    memset(st.Buffer, 0xCC, 512);

    KeInitializeApc(&st.Apc,
                    KeGetCurrentThread(),
                    KiIrpApcSelfTestKernelRoutine,
                    KiIrpApcSelfTestNormalRoutine,
                    (void*)&st);

    st.DiskDevice = IoReferenceDiskDevice(0);
    if (!st.DiskDevice)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: drive 0 disappeared before submit\n");

    IoBuildReadWriteIrp(&st.Irp, IRP_MJ_READ, 0, 512, st.Buffer);
    st.Irp.CompletionEvent = st.CompletionEvent;
    st.Irp.CompletionApc = &st.Apc;
    st.Irp.CompletionFn = (IRP_COMPLETION_FN)0;
    st.Irp.CompletionCtx = (void*)0;

    DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n");
    s = IoSubmitIrp(st.DiskDevice, &st.Irp);
    if (s != NTLSTATUS_PENDING)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: IoSubmitIrp returned %u\n", s);

    s = KeWaitForEvent(st.CompletionEvent, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: completion wait failed %u\n", s);

    // The wait return path should have delivered the pending kernel APC.
    if (st.ApcDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: APC was not delivered after I/O wait, count=%d\n",
                st.ApcDelivered);

    if (!KSUCCESS(st.Irp.Status) || !KSUCCESS(st.ApcStatus))
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: I/O failed irp=%u apc=%u info=%llu\n",
                st.Irp.Status, st.ApcStatus, (uint64_t)st.Irp.Information);

    if (st.Irp.Information != 512 || st.ApcInformation != 512)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: wrong byte count irp=%llu apc=%llu\n",
                (uint64_t)st.Irp.Information, (uint64_t)st.ApcInformation);

    DbgPrint("IRP/APC selftest: first bytes %02X %02X %02X %02X, tail %02X %02X\n",
             st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
             st.Buffer[510], st.Buffer[511]);

    MmFreePool(st.Buffer);
    if (st.DiskDevice) IoDereferenceDeviceObject(st.DiskDevice);
    ObDereferenceObject(&st.CompletionEvent->Header);

    DbgPrint("IRP/APC selftest: PASS\n");
}

// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
static void KiRunPfnDatabaseSelfTest(void) {
    MM_PFN_STATS before;
    MM_PFN_STATS after;
    MmPfnQueryStats(&before);

    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
    if (!frame)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n");

    MM_PFN* pfn = MmPfnGet(frame);
    if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "PFN selftest: allocated frame metadata invalid phys=%016llx\n",
                (unsigned long long)frame);

    if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount);
    if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount);

    if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount);
    if (MmPfnFree(frame) != NTLSTATUS_ACCESS_DENIED)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n");
    if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount);
    if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount);
    if (MmPfnFree(frame) != NTLSTATUS_ACCESS_DENIED)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n");
    if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount);
    if (!KSUCCESS(MmPfnFree(frame)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n");

    PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma);
    if (!run)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n");
    MM_PFN* p0 = MmPfnGet(run);
    MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE);
    if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n");
    MmPfnFree(run);
    MmPfnFree(run + PAGE_SIZE);

    // Accounting buckets are part of the VM contract. A missing increment here
    // usually means a new allocation path is using an anonymous usage class.
    PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache);
    PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage);
    PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile);
    PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous);
    PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow);
    PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface);
    if (!clean || !image || !mapped || !anon || !cow || !surface)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n");

    MM_PFN_STATS bucketStats;
    MmPfnQueryStats(&bucketStats);
    if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 ||
        bucketStats.ImageFrames < before.ImageFrames + 1 ||
        bucketStats.MappedFileFrames < before.MappedFileFrames + 1 ||
        bucketStats.AnonymousFrames < before.AnonymousFrames + 1 ||
        bucketStats.CowFrames < before.CowFrames + 1 ||
        bucketStats.SurfaceFrames < before.SurfaceFrames + 1) {
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n");
    }

    // Standby is the clean-page holding state used by later file-cache and image
    // reclaim. It must not change the free count until the owning cache drops it.
    if (!KSUCCESS(MmPfnMoveToStandby(clean)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n");
    if (!KSUCCESS(MmPfnMoveToStandby(image)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n");

    MM_PFN_STATS standbyStats;
    MmPfnQueryStats(&standbyStats);
    if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 ||
        standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 ||
        standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 ||
        standbyStats.FreeFrames != bucketStats.FreeFrames) {
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n");
    }

    if (!KSUCCESS(MmPfnActivate(clean)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n");
    if (!KSUCCESS(MmPfnActivate(image)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n");

    (void)MmPfnFree(clean);
    (void)MmPfnFree(image);
    (void)MmPfnFree(mapped);
    (void)MmPfnFree(anon);
    (void)MmPfnFree(cow);
    (void)MmPfnFree(surface);

    MmPfnQueryStats(&after);
    if (before.FreeFrames != after.FreeFrames)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "PFN selftest: free count leak before=%llu after=%llu\n",
                (unsigned long long)before.FreeFrames,
                (unsigned long long)after.FreeFrames);

    DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
}

// Verifies common-buffer, direct mapping, bounce mapping, and teardown accounting.
static void KiRunDmaSelfTest(void) {
    const NTL_DDK_DMA* dma = IoGetDmaDdk();
    if (!dma)
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: DDK table missing\n");

    IO_DMA_STATS before;
    IO_DMA_STATS after;
    dma->QueryStats(&before);

    DRIVER_OBJECT driver;
    memset(&driver, 0, sizeof(driver));
    IoInitializeDriverObject(&driver, SYS_SV_LITERAL("dma-selftest"));

    DEVICE_OBJECT* device = (DEVICE_OBJECT*)0;
    KSTATUS status = IoCreateDevice(&driver, 0, &device);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: device creation failed status=%u\n", status);

    IO_DMA_REQUIREMENTS requirements;
    memset(&requirements, 0, sizeof(requirements));
    requirements.MaximumDeviceAddress = UINT64_MAX;
    requirements.Alignment = PAGE_SIZE;
    requirements.MaximumSegmentLength = PAGE_SIZE;
    requirements.MaximumSegments = 4;

    IO_DMA_ADAPTER* adapter = (IO_DMA_ADAPTER*)0;
    IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
    MM_PINNED_MEMORY* pinned = (MM_PINNED_MEMORY*)0;
    IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)0;

    status = dma->GetAdapter(device, &requirements, &adapter);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: adapter creation failed status=%u\n", status);

    status = dma->AllocateCommonBuffer(adapter, PAGE_SIZE, PAGE_SIZE, &buffer);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: common buffer allocation failed status=%u\n", status);

    uint8_t* bytes = (uint8_t*)dma->GetCommonBufferAddress(buffer);
    uint64_t deviceAddress = dma->GetCommonBufferDeviceAddress(buffer);
    if (!bytes || !deviceAddress || (deviceAddress & (PAGE_SIZE - 1)) != 0 ||
        dma->GetCommonBufferSize(buffer) != PAGE_SIZE)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "DMA selftest: common buffer metadata invalid address=0x%016llX\n",
                (unsigned long long)deviceAddress);

    status = dma->PinKernelMemory(bytes, PAGE_SIZE, IoDmaBidirectional, &pinned);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: kernel pin failed status=%u\n", status);
    status = dma->MapPinnedMemory(adapter,
                                  pinned,
                                  0,
                                  PAGE_SIZE,
                                  IoDmaBidirectional,
                                  IO_DMA_MAP_NO_BOUNCE,
                                  &mapping);
    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
        dma->GetSegments(mapping)[0].DeviceAddress != deviceAddress)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "DMA selftest: direct mapping failed status=%u segments=%u\n",
                status,
                dma->GetSegmentCount(mapping));
    dma->UnmapMemory(mapping);
    mapping = (IO_DMA_MAPPING*)0;
    dma->UnpinMemory(pinned);
    pinned = (MM_PINNED_MEMORY*)0;

    for (size_t i = 0; i < 128; i++)
        bytes[i + 1] = (uint8_t)(i ^ 0x5Au);
    status = dma->PinKernelMemory(bytes + 1, 128, IoDmaToDevice, &pinned);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: unaligned kernel pin failed status=%u\n", status);
    status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaToDevice, 0, &mapping);
    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
        (dma->GetSegments(mapping)[0].DeviceAddress & (PAGE_SIZE - 1)) != 0 ||
        dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "DMA selftest: bounce mapping failed status=%u segments=%u\n",
                status,
                dma->GetSegmentCount(mapping));
    status = dma->SynchronizeForDevice(mapping);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: bounce synchronization failed status=%u\n", status);
    dma->UnmapMemory(mapping);
    mapping = (IO_DMA_MAPPING*)0;
    dma->UnpinMemory(pinned);
    pinned = (MM_PINNED_MEMORY*)0;

    memset(bytes + 1, 0, 128);
    status = dma->PinKernelMemory(bytes + 1, 128, IoDmaFromDevice, &pinned);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive pin failed status=%u\n", status);
    status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaFromDevice, 0, &mapping);
    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
        dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "DMA selftest: receive bounce mapping failed status=%u segments=%u\n",
                status,
                dma->GetSegmentCount(mapping));
    status = dma->SynchronizeForDevice(mapping);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive handoff failed status=%u\n", status);

    uint64_t receiveDeviceAddress = dma->GetSegments(mapping)[0].DeviceAddress;
    uint8_t* receiveDeviceAlias = (uint8_t*)MmPfnToVirtual(receiveDeviceAddress & PAGE_MASK);
    if (!receiveDeviceAlias)
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive device alias missing\n");
    receiveDeviceAlias += receiveDeviceAddress & (PAGE_SIZE - 1);
    for (size_t i = 0; i < 128; i++)
        receiveDeviceAlias[i] = (uint8_t)(i ^ 0xA5u);

    status = dma->SynchronizeForCpu(mapping);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive synchronization failed status=%u\n", status);
    for (size_t i = 0; i < 128; i++) {
        if (bytes[i + 1] != (uint8_t)(i ^ 0xA5u))
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "DMA selftest: receive bounce copy mismatch offset=%llu\n",
                    (unsigned long long)i);
    }
    dma->UnmapMemory(mapping);
    mapping = (IO_DMA_MAPPING*)0;
    dma->UnpinMemory(pinned);
    pinned = (MM_PINNED_MEMORY*)0;

    PROCESS* process = KeGetCurrentProcess();
    if (!process)
        process = KeGetSystemProcess();
    if (!process)
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: no process for user pinning\n");

    void* userBase = (void*)0;
    status = MmVadCreateCommitted(process,
                                  PAGE_SIZE,
                                  PAGE_SIZE,
                                  PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                                  MM_VAD_ZERO,
                                  MmVadGeneric,
                                  &userBase);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD creation failed status=%u\n", status);
    status = MmVadEnsureResident(process, userBase, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD materialization failed status=%u\n", status);
    ((uint8_t*)userBase)[0] = 0xD5u;

    status = dma->PinUserMemory(process, userBase, PAGE_SIZE, IoDmaToDevice, &pinned);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user pin failed status=%u\n", status);
    status = MmVadDestroy(process, userBase);
    if (status != NTLSTATUS_ACCESS_DENIED)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "DMA selftest: pinned user VAD was destroyable status=%u\n",
                status);
    status = dma->MapPinnedMemory(adapter,
                                  pinned,
                                  0,
                                  PAGE_SIZE,
                                  IoDmaToDevice,
                                  IO_DMA_MAP_NO_BOUNCE,
                                  &mapping);
    if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "DMA selftest: user mapping failed status=%u segments=%u\n",
                status,
                dma->GetSegmentCount(mapping));
    status = dma->SynchronizeForDevice(mapping);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user synchronization failed status=%u\n", status);
    dma->UnmapMemory(mapping);
    mapping = (IO_DMA_MAPPING*)0;
    dma->UnpinMemory(pinned);
    pinned = (MM_PINNED_MEMORY*)0;
    status = MmVadDestroy(process, userBase);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD destroy failed status=%u\n", status);

    dma->FreeCommonBuffer(buffer);
    dma->PutAdapter(adapter);
    IoDeleteDevice(device);

    dma->QueryStats(&after);
    if (after.AdaptersCreated != before.AdaptersCreated + 1 ||
        after.AdaptersDestroyed != before.AdaptersDestroyed + 1 ||
        after.AdaptersInUse != before.AdaptersInUse ||
        after.CommonBuffersAllocated < before.CommonBuffersAllocated + 3 ||
        after.CommonBuffersFreed < before.CommonBuffersFreed + 3 ||
        after.CommonBuffersInUse != before.CommonBuffersInUse ||
        after.CommonBufferBytesInUse != before.CommonBufferBytesInUse ||
        after.PinnedRangesInUse != before.PinnedRangesInUse ||
        after.PinnedPagesInUse != before.PinnedPagesInUse ||
        after.PinnedKernelRanges != before.PinnedKernelRanges + 3 ||
        after.PinnedUserRanges != before.PinnedUserRanges + 1 ||
        after.PinnedRangesReleased != before.PinnedRangesReleased + 4 ||
        after.StreamingMappings != before.StreamingMappings + 4 ||
        after.StreamingMappingsUnmapped != before.StreamingMappingsUnmapped + 4 ||
        after.StreamingMappingsInUse != before.StreamingMappingsInUse ||
        after.DirectMappings != before.DirectMappings + 2 ||
        after.BounceMappings != before.BounceMappings + 2 ||
        after.BounceBytesCopiedToDevice < before.BounceBytesCopiedToDevice + 128 ||
        after.BounceBytesCopiedToCpu < before.BounceBytesCopiedToCpu + 128)
        KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: accounting mismatch\n");

    DbgPrint("DMA selftest: common/kernel-pin/user-pin/direct/send-bounce/receive-bounce/teardown PASS\n");
}

// Verifies the page-table manager invariants that the higher VMM will depend on.
static void KiRunMemoryMappingSelfTest(void) {
    PHYSICAL_ADDRESS phys = 0;
    uint64_t flags = 0;
    KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags);
    if (s != NTLSTATUS_NOT_FOUND) {
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n",
                (unsigned long long)phys,
                (unsigned long long)flags,
                (unsigned)s);
    }

    uintptr_t testVa = 0xFFFFA00000000000ULL;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (s != NTLSTATUS_NOT_FOUND) {
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n",
                (unsigned long long)testVa,
                (unsigned long long)phys,
                (unsigned long long)flags);
    }

    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
    if (!frame)
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n");

    s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent);
    if (!KSUCCESS(s)) {
        MmPfnFree(frame);
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s);
    }

    volatile uint64_t* probe = (volatile uint64_t*)testVa;
    *probe = 0x1122334455667788ULL;
    if (*probe != 0x1122334455667788ULL)
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n");

    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n",
                s,
                (unsigned long long)phys,
                (unsigned long long)frame,
                (unsigned long long)flags);

    s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (!KSUCCESS(s) || (flags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n",
                s,
                (unsigned long long)flags);

    s = MmPteUnmapPage(testVa);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (s != NTLSTATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s);

    MmPfnFree(frame);
    DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
}

// Verifies sparse VAD commitment and NOACCESS protection metadata.
static void KiRunVadSparseSelfTest(void) {
    PROCESS* process = KeGetCurrentProcess();
    if (!process)
        process = KeGetSystemProcess();
    if (!process)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n");

    size_t beforeVadCount = 0;
    size_t beforeReserved = 0;
    size_t beforeCommitted = 0;
    MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted);

    void* base = (void*)0;
    KSTATUS s = MmVadReserve(process,
                             PAGE_SIZE * 4,
                             PAGE_SIZE,
                             PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                             MM_VAD_ZERO,
                             MmVadGeneric,
                             &base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s);

    uint8_t* p = (uint8_t*)base;
    s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s);

    s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s);

    void* queryBase = (void*)0;
    size_t querySize = 0;
    uint64_t queryFlags = 0;
    s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
    if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=0x%016llX size=%llu flags=%016llx\n",
                s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
    if (s != NTLSTATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s);

    s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s);
    s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s);

    s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
    if (s != NTLSTATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s);

    p[0] = 0x5A;
    p[PAGE_SIZE * 2] = 0xA5;

    s = MmVadProtect(process, p, PAGE_SIZE, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
    if (s != NTLSTATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s);

    s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
    if (!KSUCCESS(s) || queryFlags != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n",
                s, (unsigned long long)queryFlags);

    s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s);
    if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n");

    size_t afterVadCount = 0;
    size_t afterReserved = 0;
    size_t afterCommitted = 0;
    MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
    if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n",
                (unsigned long long)beforeCommitted,
                (unsigned long long)afterCommitted,
                (unsigned long long)beforeVadCount,
                (unsigned long long)afterVadCount);

    s = MmVadDestroy(process, base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s);

    MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
    if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n");

    DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
}

// Verifies that writable copy-on-write views split before either alias can modify shared state.
static void KiRunVadCopyOnWriteSelfTest(void) {
    PROCESS* process = KeGetCurrentProcess();
    if (!process)
        process = KeGetSystemProcess();
    if (!process)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n");

    void* base = (void*)0;
    KSTATUS s = MmVadCreateCommitted(process,
                                     PAGE_SIZE,
                                     PAGE_SIZE,
                                     PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                                     MM_VAD_ZERO,
                                     MmVadGeneric,
                                     &base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s);

    s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s);

    uint8_t* source = (uint8_t*)base;
    source[0] = 0x31;

    void* cowBase = (void*)0;
    s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s);

    PHYSICAL_ADDRESS sourcePhys = 0;
    PHYSICAL_ADDRESS cowPhys = 0;
    uint64_t sourceFlags = 0;
    uint64_t cowFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags);
    if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n",
                s, (unsigned long long)sourceFlags);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags);
    if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n",
                s,
                (unsigned long long)sourcePhys,
                (unsigned long long)cowPhys,
                (unsigned long long)cowFlags);

    s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s);

    uint8_t* copy = (uint8_t*)cowBase;
    copy[0] = 0x62;
    if (source[0] != 0x31 || copy[0] != 0x62)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n",
                source[0], copy[0]);

    PHYSICAL_ADDRESS splitPhys = 0;
    uint64_t splitFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags);
    if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
                s,
                (unsigned long long)sourcePhys,
                (unsigned long long)splitPhys,
                (unsigned long long)splitFlags);

    (void)MmVadDestroy(process, cowBase);
    (void)MmVadDestroy(process, base);

    void* zeroBase = (void*)0;
    s = MmVadCreateCommitted(process,
                             PAGE_SIZE,
                             PAGE_SIZE,
                             PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                             MM_VAD_ZERO,
                             MmVadGeneric,
                             &zeroBase);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s);

    s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s);

    PHYSICAL_ADDRESS zeroPhys = 0;
    uint64_t zeroFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags);
    if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n",
                s, (unsigned long long)zeroFlags);

    s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s);

    PHYSICAL_ADDRESS privatePhys = 0;
    uint64_t privateFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags);
    if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
                s,
                (unsigned long long)zeroPhys,
                (unsigned long long)privatePhys,
                (unsigned long long)privateFlags);

    ((uint8_t*)zeroBase)[0] = 0x9C;
    if (((uint8_t*)zeroBase)[0] != 0x9C)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n");

    (void)MmVadDestroy(process, zeroBase);
    DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
}

// Verifies that a private resident VAD page can move to swap and fault back in.
static void KiRunVadSwapSelfTest(void) {
    if (!MmSwapIsEnabled()) {
        DbgPrint("VAD swap selftest: SKIP, swap is disabled\n");
        return;
    }

    PROCESS* process = KeGetCurrentProcess();
    if (!process)
        process = KeGetSystemProcess();
    if (!process)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n");

    void* base = (void*)0;
    KSTATUS s = MmVadCreateCommitted(process,
                                     PAGE_SIZE,
                                     PAGE_SIZE,
                                     PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                                     MM_VAD_ZERO,
                                     MmVadGeneric,
                                     &base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s);

    s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s);

    uint8_t* p = (uint8_t*)base;
    p[0] = 0xC1;
    p[PAGE_SIZE - 1] = 0x5E;

    void* trimmed = (void*)0;
    s = MmVadTrimOnePageForTest(process, &trimmed);
    if (!KSUCCESS(s) || trimmed != base)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=0x%016llX base=0x%016llX\n", s, trimmed, base);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
    if (s != NTLSTATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s);

    s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s);

    if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]);

    (void)MmVadDestroy(process, base);
    DbgPrint("VAD swap selftest: private pageout/pagein PASS\n");
}

// String-view ABI self-test.
static void KiRunStringViewSelfTest(void) {
    DbgPrint("STRINGVIEW selftest: begin\n");

    SYS_STRING_VIEW hello = SYS_SV_LITERAL("Hello");
    if (hello.Length != 5 || !SysSvEqualLit(hello, "Hello"))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: literal/equality failed\n");

    static const char embedded[] = { 'a', 'b', '\0', 'c', 'd' };
    SYS_STRING_VIEW emb = SysSvFromParts(embedded, sizeof(embedded));
    if (emb.Length != 5 || !SysSvContainsByte(emb, '\0'))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: embedded-NUL detection failed\n");

    char copy[6];
    if (!SysSvCopyZ(copy, sizeof(copy), emb))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: bounded copy failed\n");
    if (memcmp(copy, embedded, sizeof(embedded)) != 0 || copy[5] != '\0')
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: bounded copy contents wrong\n");

    if (SysSvCopyZ(copy, 4, emb))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: undersized copy unexpectedly succeeded\n");

    // Paths are string views at the ABI boundary, but provider path names do not
    // accept embedded NUL bytes. This must fail before reaching any filesystem.
    IO_VOLUME* volume = (IO_VOLUME*)0;
    KSTATUS s = KeReferenceSystemRootVolume(&volume);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: system volume reference status=%u\n", s);
    FILE_OBJECT* f = (FILE_OBJECT*)0;
    s = IoOpenFileAtVolume(volume, SysSvFromParts("system/init.exe\0junk", 20), &f);
    IoReleaseVolume(volume);
    if (s != NTLSTATUS_INVALID_PARAMETER)
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: embedded-NUL path status=%u\n", s);

    DbgPrint("STRINGVIEW selftest: sample view='%v' len=%llu\n",
             hello, (uint64_t)hello.Length);
    DbgPrint("STRINGVIEW selftest: PASS\n");
}

// Capability-relative native-path self-test.
static void KiRunVolumeRelativePathSelfTest(void) {
    DbgPrint("VOLUME/PATH selftest: begin\n");

    IO_VOLUME* volume = (IO_VOLUME*)0;
    KSTATUS status = KeReferenceSystemRootVolume(&volume);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: system volume reference failed %u\n", status);

    FILE_OBJECT* file = (FILE_OBJECT*)0;
    status = IoOpenFileAtVolume(volume, SYS_SV_LITERAL("system/init.exe"), &file);
    if (!KSUCCESS(status) || !file)
        KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: direct open failed %u\n", status);
    IoCloseFile(file);

    const SYS_STRING_VIEW invalidPaths[] = {
        SYS_SV_LITERAL("/system/init.exe"),
        SYS_SV_LITERAL("system\\init.exe"),
        SYS_SV_LITERAL("system//init.exe"),
        SYS_SV_LITERAL("system/./init.exe"),
        SYS_SV_LITERAL("system/../init.exe"),
        SYS_SV_LITERAL("system/init.exe/"),
    };
    for (size_t i = 0; i < sizeof(invalidPaths) / sizeof(invalidPaths[0]); ++i) {
        file = (FILE_OBJECT*)0;
        status = IoOpenFileAtVolume(volume, invalidPaths[i], &file);
        if (status != NTLSTATUS_INVALID_PARAMETER || file)
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "VOLUME/PATH selftest: invalid path %llu status=%u file=0x%016llX\n",
                    (uint64_t)i, status, (uint64_t)(uintptr_t)file);
    }

    DIR_OBJECT* directory = (DIR_OBJECT*)0;
    status = IoOpenDirAtVolume(volume, SYS_SV_LITERAL("system"), &directory);
    if (!KSUCCESS(status) || !directory)
        KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: directory open failed %u\n", status);

    file = (FILE_OBJECT*)0;
    status = IoOpenFileAtDirectory(directory, SYS_SV_LITERAL("init.exe"), &file);
    if (!KSUCCESS(status) || !file)
        KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: directory-relative open failed %u\n", status);
    IoCloseFile(file);

    file = (FILE_OBJECT*)0;
    status = IoOpenFileAtDirectory(directory, SYS_SV_LITERAL("../init.exe"), &file);
    if (status != NTLSTATUS_INVALID_PARAMETER || file)
        KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: parent traversal status=%u\n", status);

    IoCloseDir(directory);
    IoReleaseVolume(volume);
    DbgPrint("VOLUME/PATH selftest: PASS\n");
}

// Async file I/O self-test.

typedef struct KI_FILE_ASYNC_SELFTEST_STATE {
    EVENT* CompletionEvent;
    FILE_OBJECT* File;
    IO_STATUS_BLOCK Iosb;
    KAPC Apc;
    uint8_t* Buffer;

    volatile int ApcDelivered;
    volatile KSTATUS ApcStatus;
} KI_FILE_ASYNC_SELFTEST_STATE;

static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc,
                                             void** normalRoutine,
                                             void** normalContext,
                                             void** systemArgument1,
                                             void** systemArgument2) {
    DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%016llX iosb=0x%016llX status=%u irql=%u\n",
             (uint64_t)apc,
             (uint64_t)*systemArgument1,
             (uint32_t)(uintptr_t)*systemArgument2,
             KeGetCurrentIrql());

    (void)normalRoutine;
    (void)normalContext;
}

static void KiFileAsyncSelfTestNormalRoutine(void* normalContext,
                                             void* systemArgument1,
                                             void* systemArgument2) {
    KI_FILE_ASYNC_SELFTEST_STATE* st =
        (KI_FILE_ASYNC_SELFTEST_STATE*)normalContext;
    KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;

    DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%016llX status=%u iosb=%u/%llu irql=%u\n",
             (uint64_t)systemArgument1,
             status,
             st->Iosb.Status,
             (uint64_t)st->Iosb.Information,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: APC ran above PASSIVE_LEVEL\n");

    st->ApcDelivered++;
    st->ApcStatus = status;
}

// Runs async File Io Self Test.
static void KiRunAsyncFileIoSelfTest(void) {
    DbgPrint("FILE/ASYNC selftest: begin\n");

    KI_FILE_ASYNC_SELFTEST_STATE st;
    memset(&st, 0, sizeof(st));
    st.Iosb.Status = NTLSTATUS_FAILURE;

    KSTATUS s = KeOpenSystemRootFile("system/init.exe", &st.File);
    if (!KSUCCESS(s)) {
        DbgPrint("FILE/ASYNC selftest: SKIP, system/init.exe open failed %u\n", s);
        return;
    }

    s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: event create failed %u\n", s);

    st.Buffer = (uint8_t*)MmAllocatePool(512);
    if (!st.Buffer)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: no buffer\n");
    memset(st.Buffer, 0xCC, 512);

    KeInitializeApc(&st.Apc,
                    KeGetCurrentThread(),
                    KiFileAsyncSelfTestKernelRoutine,
                    KiFileAsyncSelfTestNormalRoutine,
                    (void*)&st);

    DbgPrint("FILE/ASYNC selftest: submit read system/init.exe offset=0 size=512\n");
    IO_ASYNC_FILE_REQUEST_OBJECT* readReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
    s = IoReadFileAsyncKernel(st.File,
                              st.Buffer,
                              512,
                              0,
                              &st.Iosb,
                              st.CompletionEvent,
                              &st.Apc,
                              (MM_LOCKED_USER_BUFFER*)0,
                              (MM_LOCKED_USER_BUFFER*)0,
                              &readReq);
    if (s != NTLSTATUS_PENDING)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s);

    s = KeWaitForEvent(st.CompletionEvent, 1);
    if (s != NTLSTATUS_SUCCESS && s != NTLSTATUS_KERNEL_APC)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: wait returned %u\n", s);

    if (st.Iosb.Status == NTLSTATUS_PENDING) {
        s = KeWaitForEvent(st.CompletionEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: final event wait returned %u\n", s);
    }

    if (st.ApcDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: APC not delivered count=%d\n", st.ApcDelivered);

    if (!KSUCCESS(st.Iosb.Status) || !KSUCCESS(st.ApcStatus))
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: I/O failed iosb=%u apc=%u info=%llu\n",
                st.Iosb.Status, st.ApcStatus, (uint64_t)st.Iosb.Information);

    if (st.Iosb.Information < 2 || st.Buffer[0] != 'M' || st.Buffer[1] != 'Z')
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: bad file header info=%llu bytes=%02X %02X\n",
                (uint64_t)st.Iosb.Information, st.Buffer[0], st.Buffer[1]);

    DbgPrint("FILE/ASYNC selftest: first bytes %02X %02X %02X %02X info=%llu wait=%u\n",
             st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
             (uint64_t)st.Iosb.Information, s);

    if (readReq)
        ObDereferenceObject(IoAsyncFileRequestObject(readReq));

    MmFreePool(st.Buffer);
    ObDereferenceObject(&st.CompletionEvent->Header);
    IoCloseFile(st.File);

    // Exercise the async write path with an explicit-offset temporary file.
    {
        static const char payload[] = "async-file-write-selftest\n";
        char verify[sizeof(payload)];
        FILE_OBJECT* wf = (FILE_OBJECT*)0;
        FILE_OBJECT* rf = (FILE_OBJECT*)0;
        EVENT* wev = (EVENT*)0;
        IO_STATUS_BLOCK wiosb;
        size_t got = 0;

        memset(verify, 0, sizeof(verify));
        wiosb.Status = NTLSTATUS_FAILURE;
        wiosb.Information = 0;

        (void)KeDeleteSystemRootFile("asyncio.tmp");
        s = KeCreateSystemRootFile("asyncio.tmp", &wf);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: create write temp failed %u\n", s);

        s = KeEventCreate(EventAutoReset, 0, &wev);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write event create failed %u\n", s);

        DbgPrint("FILE/ASYNC selftest: submit write asyncio.tmp size=%llu\n",
                 (uint64_t)(sizeof(payload) - 1));

        IO_ASYNC_FILE_REQUEST_OBJECT* writeReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
        s = IoWriteFileAsyncKernel(wf,
                                   payload,
                                   sizeof(payload) - 1,
                                   0,
                                   &wiosb,
                                   wev,
                                   (KAPC*)0,
                                   (MM_LOCKED_USER_BUFFER*)0,
                                   (MM_LOCKED_USER_BUFFER*)0,
                                   &writeReq);
        if (s != NTLSTATUS_PENDING)
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s);

        s = KeWaitForEvent(wev, 1);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write wait returned %u\n", s);

        if (!KSUCCESS(wiosb.Status) || wiosb.Information != sizeof(payload) - 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write failed status=%u info=%llu\n",
                    wiosb.Status, (uint64_t)wiosb.Information);

        if (writeReq)
            ObDereferenceObject(IoAsyncFileRequestObject(writeReq));

        IoCloseFile(wf);
        wf = (FILE_OBJECT*)0;

        s = KeOpenSystemRootFile("asyncio.tmp", &rf);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: reopen write temp failed %u\n", s);

        s = IoReadFileAt(rf, 0, verify, sizeof(payload) - 1, &got);
        if (!KSUCCESS(s) || got != sizeof(payload) - 1 ||
            // Implements the memcmp operation.
            memcmp(verify, payload, sizeof(payload) - 1) != 0) {
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: verify failed status=%u got=%llu\n",
                    s, (uint64_t)got);
        }

        IoCloseFile(rf);
        ObDereferenceObject(&wev->Header);
        s = KeDeleteSystemRootFile("asyncio.tmp");
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: delete temp failed %u\n", s);

        DbgPrint("FILE/ASYNC selftest: async write verified\n");
    }

    DbgPrint("FILE/ASYNC selftest: PASS\n");
}

typedef struct KI_SCHEDULER_SELFTEST_STATE {
    MUTEX* GateMutex;
    MUTEX* LinkMutex;
    EVENT* ChainReadyEvent;
    EVENT* RealtimeReadyEvent;
    EVENT* ChainFinishedEvent;
    EVENT* RealtimeFinishedEvent;
    volatile KSTATUS ChainAcquireStatus;
    volatile KSTATUS RealtimeAcquireStatus;
} KI_SCHEDULER_SELFTEST_STATE;

// Waits until one worker has entered the expected mutex wait state.
static KE_THREAD_SCHEDULER_SNAPSHOT KiSchedulerAwaitMutexWait(THREAD* thread, MUTEX* mutex) {
    KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
    memset(&snapshot, 0, sizeof(snapshot));

    for (uint32_t attempt = 0; attempt < 100; attempt++) {
        KiSchedulerSnapshotThread(thread, &snapshot);
        if (snapshot.State == ThreadWaiting && snapshot.WaitingMutex == mutex)
            return snapshot;

        KSTATUS status = KeDelayThread(SCHED_TICK_MS);
        if (!KSUCCESS(status))
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "SCHED selftest: state wait failed status=%u\n", status);
    }

    KePanic(KE_PANIC_CODE_SELFTEST,
            "SCHED selftest: thread %u did not block on mutex=0x%016llX state=%u waiting=0x%016llX\n",
            thread ? thread->ThreadId : 0,
            (uint64_t)mutex,
            (unsigned)snapshot.State,
            (uint64_t)snapshot.WaitingMutex);
}

typedef struct KI_SYNC_TIMEOUT_SELFTEST_STATE {
    MUTEX* Mutex;
    EVENT* ReadyEvent;
    EVENT* FinishedEvent;
    volatile KSTATUS MutexStatus;
} KI_SYNC_TIMEOUT_SELFTEST_STATE;

// Waits on the held mutex long enough for the timeout engine to remove donation state.
static void KiSynchronizationTimeoutWorker(void* argument) {
    KI_SYNC_TIMEOUT_SELFTEST_STATE* state = (KI_SYNC_TIMEOUT_SELFTEST_STATE*)argument;
    KeSignalEvent(state->ReadyEvent);

    state->MutexStatus = KeAcquireMutex(state->Mutex, SCHED_TICK_MS * 20ULL);
    if (KE_MUTEX_ACQUIRE_SUCCEEDED(state->MutexStatus))
        KeReleaseMutex(state->Mutex);

    KeSignalEvent(state->FinishedEvent);
    KeExitThread(KeGetCurrentThread(), 0);
}

// Verifies finite mutex and semaphore waits consume no object state after timeout.
static void KiRunSynchronizationTimeoutSelfTest(void) {
    DbgPrint("SYNC/TIMEOUT selftest: begin\n");

    KI_SYNC_TIMEOUT_SELFTEST_STATE state;
    memset(&state, 0, sizeof(state));
    state.MutexStatus = NTLSTATUS_PENDING;

    KSTATUS status = KeMutexCreate(&state.Mutex);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: mutex create failed status=%u\n", status);

    status = KeEventCreate(EventAutoReset, 0, &state.ReadyEvent);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: ready event create failed status=%u\n", status);

    status = KeEventCreate(EventAutoReset, 0, &state.FinishedEvent);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: finished event create failed status=%u\n", status);

    status = KeAcquireMutex(state.Mutex, KE_WAIT_INFINITE);
    if (status != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: owner acquire failed status=%u\n", status);

    THREAD* worker = (THREAD*)0;
    const KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_REALTIME(12);
    status = KeCreateKernelThread((uintptr_t)KiSynchronizationTimeoutWorker,
                                  (uintptr_t)&state,
                                  &parameters,
                                  &worker);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: worker create failed status=%u\n", status);

    status = KeWaitForEvent(state.ReadyEvent, 0);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: ready wait failed status=%u\n", status);

    (void)KiSchedulerAwaitMutexWait(worker, state.Mutex);

    THREAD* self = KeGetCurrentThread();
    KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
    KiSchedulerSnapshotThread(self, &snapshot);
    if (snapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
        snapshot.EffectivePriority != 12) {
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: finite waiter donation missing class=%u priority=%u\n",
                snapshot.EffectiveSchedulingClass,
                snapshot.EffectivePriority);
    }

    status = KeWaitForEvent(state.FinishedEvent, 0);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: finished wait failed status=%u\n", status);
    if (state.MutexStatus != NTLSTATUS_TIMEOUT)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: mutex wait returned status=%u\n",
                state.MutexStatus);

    KiSchedulerSnapshotThread(self, &snapshot);
    if (snapshot.EffectiveSchedulingClass != snapshot.BaseSchedulingClass ||
        snapshot.EffectivePriority != snapshot.BasePriority) {
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: timed-out donation remained base=%u/%u effective=%u/%u\n",
                snapshot.BaseSchedulingClass,
                snapshot.BasePriority,
                snapshot.EffectiveSchedulingClass,
                snapshot.EffectivePriority);
    }

    KeReleaseMutex(state.Mutex);
    status = KeAcquireMutex(state.Mutex, 0);
    if (status != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: zero-timeout mutex acquire failed status=%u\n", status);
    KeReleaseMutex(state.Mutex);

    SEMAPHORE* semaphore = (SEMAPHORE*)0;
    status = KeSemaphoreCreate(0, 1, &semaphore);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: semaphore create failed status=%u\n", status);

    status = KeWaitSemaphore(semaphore, 0);
    if (status != NTLSTATUS_TIMEOUT)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: zero-timeout semaphore wait returned status=%u\n", status);

    status = KeWaitSemaphore(semaphore, SCHED_TICK_MS * 3ULL);
    if (status != NTLSTATUS_TIMEOUT)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: finite semaphore wait returned status=%u\n", status);

    KeSignalSemaphore(semaphore);
    status = KeWaitSemaphore(semaphore, 0);
    if (status != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: signaled semaphore probe failed status=%u\n", status);
    status = KeWaitSemaphore(semaphore, 0);
    if (status != NTLSTATUS_TIMEOUT)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SYNC/TIMEOUT selftest: semaphore count was consumed twice status=%u\n", status);

    ObDereferenceObject(&worker->Header);
    ObDereferenceObject(&semaphore->Header);
    ObDereferenceObject(&state.Mutex->Header);
    ObDereferenceObject(&state.ReadyEvent->Header);
    ObDereferenceObject(&state.FinishedEvent->Header);

    DbgPrint("SYNC/TIMEOUT selftest: PASS\n");
}

// Owns the link mutex and then blocks behind the self-test thread.
static void KiSchedulerChainWorker(void* argument) {
    KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;

    KSTATUS status = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
    if (status != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: chain link acquire failed status=%u\n", status);

    KeSignalEvent(state->ChainReadyEvent);
    state->ChainAcquireStatus = KeAcquireMutex(state->GateMutex, KE_WAIT_INFINITE);
    if (state->ChainAcquireStatus != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: chain gate acquire failed status=%u\n",
                state->ChainAcquireStatus);

    KeReleaseMutex(state->GateMutex);
    KeReleaseMutex(state->LinkMutex);
    KeSignalEvent(state->ChainFinishedEvent);
    KeExitThread(KeGetCurrentThread(), 0);
}

// Blocks a real-time thread behind the normal chain worker.
static void KiSchedulerRealtimeWorker(void* argument) {
    KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;
    KeSignalEvent(state->RealtimeReadyEvent);

    state->RealtimeAcquireStatus = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
    if (state->RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: real-time acquire failed status=%u\n",
                state->RealtimeAcquireStatus);

    KeReleaseMutex(state->LinkMutex);
    KeSignalEvent(state->RealtimeFinishedEvent);
    KeExitThread(KeGetCurrentThread(), 0);
}

// Verifies real-time dispatch and transitive mutex priority inheritance.
static void KiRunRealtimeSchedulerSelfTest(void) {
    DbgPrint("SCHED selftest: begin\n");

    THREAD* self = KeGetCurrentThread();
    if (!self || self->BaseSchedulingClass != KeSchedulingNormal)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: caller is not a normal thread\n");

    KE_SCHED_STATS before;
    KE_SCHED_STATS after;
    KeGetSchedulerStats(&before);

    KI_SCHEDULER_SELFTEST_STATE state;
    memset(&state, 0, sizeof(state));
    state.ChainAcquireStatus = NTLSTATUS_PENDING;
    state.RealtimeAcquireStatus = NTLSTATUS_PENDING;

    KSTATUS status = KeMutexCreate(&state.GateMutex);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: gate mutex create failed status=%u\n", status);

    status = KeMutexCreate(&state.LinkMutex);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: link mutex create failed status=%u\n", status);

    status = KeEventCreate(EventAutoReset, 0, &state.ChainReadyEvent);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: chain ready event create failed status=%u\n", status);

    status = KeEventCreate(EventAutoReset, 0, &state.RealtimeReadyEvent);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: real-time ready event create failed status=%u\n", status);

    status = KeEventCreate(EventAutoReset, 0, &state.ChainFinishedEvent);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: chain finished event create failed status=%u\n", status);

    status = KeEventCreate(EventAutoReset, 0, &state.RealtimeFinishedEvent);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: real-time finished event create failed status=%u\n", status);

    status = KeAcquireMutex(state.GateMutex, KE_WAIT_INFINITE);
    if (status != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: owner acquire failed status=%u\n", status);

    THREAD* chainWorker = (THREAD*)0;
    const KE_THREAD_CREATE_PARAMETERS chainParameters = KE_THREAD_CREATE_NORMAL(PriorityNormal);
    status = KeCreateKernelThread((uintptr_t)KiSchedulerChainWorker,
                                  (uintptr_t)&state,
                                  &chainParameters,
                                  &chainWorker);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: chain worker create failed status=%u\n", status);

    status = KeWaitForEvent(state.ChainReadyEvent, 0);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: chain ready wait failed status=%u\n", status);

    (void)KiSchedulerAwaitMutexWait(chainWorker, state.GateMutex);

    status = KeAcquireMutex(state.LinkMutex, KE_WAIT_INFINITE);
    if (status != NTLSTATUS_DEADLOCK)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: mutex cycle was not rejected status=%u\n", status);

    THREAD* realtimeWorker = (THREAD*)0;
    const KE_THREAD_CREATE_PARAMETERS realtimeParameters = KE_THREAD_CREATE_REALTIME(16);
    status = KeCreateKernelThread((uintptr_t)KiSchedulerRealtimeWorker,
                                  (uintptr_t)&state,
                                  &realtimeParameters,
                                  &realtimeWorker);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: real-time worker create failed status=%u\n", status);

    status = KeWaitForEvent(state.RealtimeReadyEvent, 0);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: real-time ready wait failed status=%u\n", status);

    KE_THREAD_SCHEDULER_SNAPSHOT realtimeSnapshot =
        KiSchedulerAwaitMutexWait(realtimeWorker, state.LinkMutex);
    (void)realtimeSnapshot;

    KE_THREAD_SCHEDULER_SNAPSHOT chainSnapshot;
    KE_THREAD_SCHEDULER_SNAPSHOT selfSnapshot;
    KiSchedulerSnapshotThread(chainWorker, &chainSnapshot);
    KiSchedulerSnapshotThread(self, &selfSnapshot);

    if (chainSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
        chainSnapshot.EffectivePriority != 16)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: intermediate donation missing class=%u priority=%u\n",
                chainSnapshot.EffectiveSchedulingClass,
                chainSnapshot.EffectivePriority);

    if (selfSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
        selfSnapshot.EffectivePriority != 16)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: transitive donation missing class=%u priority=%u\n",
                selfSnapshot.EffectiveSchedulingClass,
                selfSnapshot.EffectivePriority);

    KeReleaseMutex(state.GateMutex);

    KiSchedulerSnapshotThread(self, &selfSnapshot);
    if (selfSnapshot.EffectiveSchedulingClass != selfSnapshot.BaseSchedulingClass ||
        selfSnapshot.EffectivePriority != selfSnapshot.BasePriority)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: inherited priority was not restored base=%u/%u effective=%u/%u\n",
                selfSnapshot.BaseSchedulingClass,
                selfSnapshot.BasePriority,
                selfSnapshot.EffectiveSchedulingClass,
                selfSnapshot.EffectivePriority);

    status = KeWaitForEvent(state.ChainFinishedEvent, 0);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: chain finished wait failed status=%u\n", status);

    status = KeWaitForEvent(state.RealtimeFinishedEvent, 0);
    if (!KSUCCESS(status))
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: real-time finished wait failed status=%u\n", status);

    if (state.ChainAcquireStatus != NTLSTATUS_SUCCESS ||
        state.RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: handoff statuses chain=%u realtime=%u\n",
                state.ChainAcquireStatus,
                state.RealtimeAcquireStatus);

    KeGetSchedulerStats(&after);
    if (after.RealtimeDispatches <= before.RealtimeDispatches ||
        after.PriorityInheritanceBoosts < before.PriorityInheritanceBoosts + 2 ||
        after.PriorityInheritanceDrops < before.PriorityInheritanceDrops + 2 ||
        after.PriorityInheritancePropagations < before.PriorityInheritancePropagations + 2 ||
        after.PriorityInheritanceMaxDepth < 2)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SCHED selftest: scheduler counters did not prove transitive inheritance\n");

    ObDereferenceObject(&chainWorker->Header);
    ObDereferenceObject(&realtimeWorker->Header);
    ObDereferenceObject(&state.GateMutex->Header);
    ObDereferenceObject(&state.LinkMutex->Header);
    ObDereferenceObject(&state.ChainReadyEvent->Header);
    ObDereferenceObject(&state.RealtimeReadyEvent->Header);
    ObDereferenceObject(&state.ChainFinishedEvent->Header);
    ObDereferenceObject(&state.RealtimeFinishedEvent->Header);

    DbgPrint("SCHED selftest: PASS\n");
}

// Runs trap Frame Self Test.
static void KiRunTrapFrameSelfTest(void) {
    DbgPrint("TRAP selftest: begin\n");

    if (KeGetCurrentTrapFrame())
        KePanic(KE_PANIC_CODE_SELFTEST, "TRAP selftest: kernel selftest unexpectedly has active trap frame\n");

    DbgPrint("TRAP selftest: frame size=%llu query size=%llu\n",
             (uint64_t)sizeof(KTRAP_FRAME),
             (uint64_t)sizeof(SYS_TRAP_FRAME_INFO_K));

    if (sizeof(KTRAP_FRAME) != 240)
        KePanic(KE_PANIC_CODE_SELFTEST, "TRAP selftest: KTRAP_FRAME size mismatch %llu\n",
                (uint64_t)sizeof(KTRAP_FRAME));

    DbgPrint("TRAP selftest: PASS\n");
}

// Runs core Hardening Self Tests.
void KiRunCoreHardeningSelfTests(void) {
    DbgPrint("CORE selftest: begin\n");

    if (gCpuCount == 0 || gCpuCount > KE_MAX_CPUS)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid CPU count %u\n", gCpuCount);

    KCPU* cpu = KeGetCurrentCpu();
    if (cpu != &gCpus[0])
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%016llX cpu0=0x%016llX\n",
                (uint64_t)cpu, (uint64_t)&gCpus[0]);
    if (cpu->Self != cpu)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%016llX cpu=0x%016llX\n",
                (uint64_t)cpu->Self, (uint64_t)cpu);
    if (KeGetCurrentCpuIndex() != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not boot processor/0\n");
    if (cpu->State != KeCpuOnline || !cpu->IsBootProcessor)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: boot-processor lifecycle invalid state=%ld boot=%u\n",
                cpu->State, cpu->IsBootProcessor);
    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql());
    if (KeGetCurrentTss() != &cpu->Tss)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n");
    if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64))
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid boot-processor TSS rsp0=0x%016llX iomap=%u\n",
                cpu->Tss.RSP0, cpu->Tss.IoMapBase);
    if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%016llX gdt=0x%016llX\n",
                cpu->Gdtr.Base, (uint64_t)cpu->Gdt);

    for (uint32_t i = 1; i < gCpuCount; i++) {
        KCPU* ap = &gCpus[i];
        if (ap->State == KeCpuAbsent)
            continue;
        if (ap->State != KeCpuOnline || ap->IsBootProcessor)
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "CORE selftest: AP%u lifecycle invalid state=%ld boot=%u\n",
                    i, ap->State, ap->IsBootProcessor);
        if (!ap->KernelStack || !ap->KernelStackTop)
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
        if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%016llX iomap=%u\n",
                    i, ap->Tss.RSP0, ap->Tss.IoMapBase);
        if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt)
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i);
        if (!ap->IdleThread || !ap->CurrentThread || ap->CurrentThread != ap->IdleThread)
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u idle/current thread missing\n", i);
        if (!ap->IdleThread->IsIdleThread || ap->IdleThread->Queued || ap->IdleThread->RunningCpu != i)
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "CORE selftest: AP%u idle invariant failed queued=%u idle=%u runningCpu=%u\n",
                    i, ap->IdleThread->Queued, ap->IdleThread->IsIdleThread, ap->IdleThread->RunningCpu);
    }

    THREAD* cur = KeGetCurrentThread();
    if (!cur || cur != cpu->CurrentThread)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current thread mismatch\n");
    if (!cpu->IdleThread || cur != cpu->IdleThread)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: scheduler did not start on CPU0 idle thread\n");
    if (!cur->IsIdleThread || cur->Queued || cur->RunningCpu != cpu->CpuIndex)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: idle invariant failed queued=%u idle=%u runningCpu=%u\n",
                cur->Queued, cur->IsIdleThread, cur->RunningCpu);

    DbgPrint("CORE selftest: PASS\n");
}

// Runs early Kernel Self Tests.
void KiRunEarlyKernelSelfTests(void) {
    KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests();
    if (!KSUCCESS(s)) {
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SMP scheduler selftest failed status=0x%x\n",
                s);
    }
}

// Runs kernel Self Tests.
void KiRunKernelSelfTests(const HAL_API* hal) {
    DbgPrint("Kernel selftests: begin\n");
    KiRunSynchronizationTimeoutSelfTest();
    KiRunRealtimeSchedulerSelfTest();
    KiRunKernelApcSelfTest();
    KiRunPfnDatabaseSelfTest();
    KiRunDmaSelfTest();
    KiRunMemoryMappingSelfTest();
    KiRunVadSparseSelfTest();
    KiRunVadCopyOnWriteSelfTest();
    KiRunVadSwapSelfTest();
    KiRunStringViewSelfTest();
    KiRunVolumeRelativePathSelfTest();
    KiRunIrpKernelApcSelfTest(hal);
    KiRunAsyncFileIoSelfTest();
    KiRunTrapFrameSelfTest();
    DbgPrint("Kernel selftests: PASS\n");
}