nt-lite/code/kernel/ke_selftest.c

// Kernel self-tests. This file runs bring-up regression checks before normal user workloads start.

#include <stdint.h>
#include <stddef.h>
#include <string.h>

#include <ntlite/ddi/hal_api.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_event.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_sched.h>
#include <ntlite/status.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_swap.h>
#include <kernel/mm_pfn.h>
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>

// Kernel APC self-test.

typedef struct KI_APC_SELFTEST_STATE {
    EVENT* ReadyEvent;
    EVENT* GoEvent;
    EVENT* DoneEvent;
    EVENT* FinishedEvent;

    THREAD* WorkerThread;

    volatile int SelfDelivered;
    volatile int DeferredDelivered;
    volatile int CrossDelivered;
} KI_APC_SELFTEST_STATE;

static void KiApcSelfTestKernelRoutine(KAPC* apc,
                                        void** normalRoutine,
                                        void** normalContext,
                                        void** systemArgument1,
                                        void** systemArgument2) {
    DbgPrint("APC selftest: kernel routine apc=0x%016llX normal=0x%016llX ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)apc,
             (uint64_t)*normalRoutine,
             (uint64_t)*normalContext,
             (uint64_t)*systemArgument1,
             (uint64_t)*systemArgument2,
             KeGetCurrentIrql());
}

static void KiApcSelfTestNormalRoutine(void* normalContext,
                                        void* systemArgument1,
                                        void* systemArgument2) {
    volatile int* counter = (volatile int*)normalContext;

    DbgPrint("APC selftest: normal routine ctx=0x%016llX arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)normalContext,
             (uint64_t)systemArgument1,
             (uint64_t)systemArgument2,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: normal routine ran above PASSIVE_LEVEL\n");

    (*counter)++;
}

static void KiApcSelfTestCrossThreadRoutine(void* normalContext,
                                             void* systemArgument1,
                                             void* systemArgument2) {
    KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)normalContext;

    DbgPrint("APC selftest: cross-thread APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)systemArgument1,
             (uint64_t)systemArgument2,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread APC ran above PASSIVE_LEVEL\n");

    st->CrossDelivered++;
    KeSignalEvent(st->DoneEvent);
}

// Implements the apc Self Test Worker operation.
static void KiApcSelfTestWorker(void* arg) {
    KI_APC_SELFTEST_STATE* st = (KI_APC_SELFTEST_STATE*)arg;

    DbgPrint("APC selftest: worker started, signaling ready\n");
    KeSignalEvent(st->ReadyEvent);

    DbgPrint("APC selftest: worker waiting for go event\n");
    KSTATUS s = KeWaitForEvent(st->GoEvent, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker wait failed status=%d\n", s);

    // KeCommitBlockThread(..., alertable) should deliver pending kernel APCs before this
    // wait returns. If that hook is broken, DoneEvent will never be signaled.
    DbgPrint("APC selftest: worker resumed after go event, cross=%d\n",
             st->CrossDelivered);

    if (st->CrossDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker resumed but APC was not delivered\n");

    KeSignalEvent(st->FinishedEvent);
    KeExitThread(KeGetCurrentThread(), 0);
}

typedef struct KI_APC_ALERTABLE_SELFTEST_STATE {
    EVENT* ReadyEvent;
    EVENT* WaitEvent;
    EVENT* FinishedEvent;
    THREAD* WorkerThread;

    volatile int ApcDelivered;
    volatile KSTATUS WaitStatus;
} KI_APC_ALERTABLE_SELFTEST_STATE;

static void KiApcAlertableSelfTestNormalRoutine(void* normalContext,
                                                 void* systemArgument1,
                                                 void* systemArgument2) {
    KI_APC_ALERTABLE_SELFTEST_STATE* st =
        (KI_APC_ALERTABLE_SELFTEST_STATE*)normalContext;

    DbgPrint("APC selftest: alertable APC delivered arg1=0x%016llX arg2=0x%016llX irql=%u\n",
             (uint64_t)systemArgument1,
             (uint64_t)systemArgument2,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC ran above PASSIVE_LEVEL\n");

    st->ApcDelivered++;
}

// Implements the apc Alertable Self Test Worker operation.
static void KiApcAlertableSelfTestWorker(void* arg) {
    KI_APC_ALERTABLE_SELFTEST_STATE* st =
        (KI_APC_ALERTABLE_SELFTEST_STATE*)arg;

    DbgPrint("APC selftest: alertable worker ready, entering alertable wait\n");
    KeSignalEvent(st->ReadyEvent);

    KSTATUS s = KeWaitForEvent(st->WaitEvent, 1);
    st->WaitStatus = s;

    DbgPrint("APC selftest: alertable wait returned status=%u delivered=%d\n",
             s, st->ApcDelivered);

    if (s != STATUS_KERNEL_APC)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable wait returned wrong status=%u\n", s);

    if (st->ApcDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC delivery count wrong=%d\n",
                st->ApcDelivered);

    KeSignalEvent(st->FinishedEvent);
    KeExitThread(KeGetCurrentThread(), 0);
}

// Runs kernel Apc Self Test.
static void KiRunKernelApcSelfTest(void) {
    DbgPrint("APC selftest: begin\n");

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: called above PASSIVE_LEVEL\n");

    THREAD* self = KeGetCurrentThread();
    if (!self)
        KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: no current thread\n");

    // Test 1: queue APC to current thread at PASSIVE_LEVEL.
    {
        KAPC apc;
        volatile int count = 0;

        KeInitializeApc(&apc,
                        self,
                        KiApcSelfTestKernelRoutine,
                        KiApcSelfTestNormalRoutine,
                        (void*)&count);

        if (!KeInsertQueueApc(&apc, (void*)0x11111111, (void*)0x22222222))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: self queue failed\n");

        if (count != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: self APC was not delivered immediately, count=%d\n", count);

        DbgPrint("APC selftest: self APC passed\n");
    }

    // Test 2: disable kernel APCs, queue, then re-enable.
    {
        KAPC apc;
        volatile int count = 0;

        KeInitializeApc(&apc,
                        self,
                        KiApcSelfTestKernelRoutine,
                        KiApcSelfTestNormalRoutine,
                        (void*)&count);

        KeEnterCriticalRegion();

        if (!KeInsertQueueApc(&apc, (void*)0x33333333, (void*)0x44444444))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: deferred queue failed\n");

        if (count != 0)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: APC delivered while critical region held, count=%d\n", count);

        KeLeaveCriticalRegion();

        if (count != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: deferred APC was not delivered after leave, count=%d\n", count);

        DbgPrint("APC selftest: critical-region deferral passed\n");
    }

    // Test 3: queue APC to another blocked kernel thread, then wake it.
    {
        KI_APC_SELFTEST_STATE st;
        KAPC apc;
        memset(&st, 0, sizeof(st));

        KSTATUS s;
        s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: ReadyEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.GoEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: GoEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.DoneEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: DoneEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: FinishedEvent create failed %d\n", s);

        s = KeCreateKernelThread((uintptr_t)KiApcSelfTestWorker,
                                 (uintptr_t)&st,
                                 PriorityNormal,
                                 &st.WorkerThread);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker create failed %d\n", s);

        s = KeWaitForEvent(st.ReadyEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: ready wait failed %d\n", s);

        DbgPrint("APC selftest: worker ready, queueing cross-thread APC\n");

        KeInitializeApc(&apc,
                        st.WorkerThread,
                        KiApcSelfTestKernelRoutine,
                        KiApcSelfTestCrossThreadRoutine,
                        (void*)&st);

        if (!KeInsertQueueApc(&apc, (void*)0x55555555, (void*)0x66666666))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread queue failed\n");

        if (st.CrossDelivered != 0)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread APC delivered before target resumed\n");

        KeSignalEvent(st.GoEvent);

        s = KeWaitForEvent(st.DoneEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: done wait failed %d\n", s);

        if (st.CrossDelivered != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: cross-thread delivery count wrong, count=%d\n",
                    st.CrossDelivered);

        // The APC signals DoneEvent before the worker returns from the wait.
        // Wait for an explicit worker-owned event before dropping stack-backed
        // state and event references. THREAD.ExitEvent is not part of this
        // kernel-thread contract on all builds.
        s = KeWaitForEvent(st.FinishedEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: worker finished wait failed %d\n", s);

        DbgPrint("APC selftest: cross-thread APC passed\n");

        ObDereferenceObject(&st.ReadyEvent->Header);
        ObDereferenceObject(&st.GoEvent->Header);
        ObDereferenceObject(&st.DoneEvent->Header);
        ObDereferenceObject(&st.FinishedEvent->Header);
        ObDereferenceObject(&st.WorkerThread->Header);
    }

    // Test 4: queue APC to a thread blocked in an alertable wait.
    // The APC itself must wake the waiter; no event signal is sent.
    {
        KI_APC_ALERTABLE_SELFTEST_STATE st;
        KAPC apc;
        memset(&st, 0, sizeof(st));
        st.WaitStatus = STATUS_PENDING;

        KSTATUS s;
        s = KeEventCreate(EventAutoReset, 0, &st.ReadyEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert ReadyEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.WaitEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert WaitEvent create failed %d\n", s);

        s = KeEventCreate(EventAutoReset, 0, &st.FinishedEvent);
        if (!KSUCCESS(s)) KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert FinishedEvent create failed %d\n", s);

        s = KeCreateKernelThread((uintptr_t)KiApcAlertableSelfTestWorker,
                                 (uintptr_t)&st,
                                 PriorityNormal,
                                 &st.WorkerThread);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable worker create failed %d\n", s);

        s = KeWaitForEvent(st.ReadyEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert ready wait failed %d\n", s);

        DbgPrint("APC selftest: queueing APC to alertable waiter\n");

        KeInitializeApc(&apc,
                        st.WorkerThread,
                        KiApcSelfTestKernelRoutine,
                        KiApcAlertableSelfTestNormalRoutine,
                        (void*)&st);

        if (!KeInsertQueueApc(&apc, (void*)(unsigned long long)0x77777777, (void*)(unsigned long long)0x88888888))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable APC queue failed\n");

        s = KeWaitForEvent(st.FinishedEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alert finished wait failed %d\n", s);

        if (st.WaitStatus != STATUS_KERNEL_APC || st.ApcDelivered != 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "APC selftest: alertable result bad status=%u delivered=%d\n",
                    st.WaitStatus, st.ApcDelivered);

        DbgPrint("APC selftest: alertable wait passed\n");

        ObDereferenceObject(&st.ReadyEvent->Header);
        ObDereferenceObject(&st.WaitEvent->Header);
        ObDereferenceObject(&st.FinishedEvent->Header);
        ObDereferenceObject(&st.WorkerThread->Header);
    }

    DbgPrint("APC selftest: PASS\n");
}

// IRP + kernel APC self-test.

typedef struct KI_IRP_APC_SELFTEST_STATE {
    EVENT* CompletionEvent;
    DEVICE_OBJECT* DiskDevice;
    IRP Irp;
    KAPC Apc;
    uint8_t* Buffer;

    volatile int ApcDelivered;
    volatile KSTATUS ApcStatus;
    volatile size_t ApcInformation;
} KI_IRP_APC_SELFTEST_STATE;

static void KiIrpApcSelfTestKernelRoutine(KAPC* apc,
                                           void** normalRoutine,
                                           void** normalContext,
                                           void** systemArgument1,
                                           void** systemArgument2) {
    DbgPrint("IRP/APC selftest: kernel routine apc=0x%016llX irp=0x%016llX status=%u irql=%u\n",
             (uint64_t)apc,
             (uint64_t)*systemArgument1,
             (uint32_t)(uintptr_t)*systemArgument2,
             KeGetCurrentIrql());

    (void)normalRoutine;
    (void)normalContext;
}

static void KiIrpApcSelfTestNormalRoutine(void* normalContext,
                                           void* systemArgument1,
                                           void* systemArgument2) {
    KI_IRP_APC_SELFTEST_STATE* st = (KI_IRP_APC_SELFTEST_STATE*)normalContext;
    IRP* irp = (IRP*)systemArgument1;
    KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;

    DbgPrint("IRP/APC selftest: APC delivered irp=0x%016llX status=%u info=%llu irql=%u\n",
             (uint64_t)irp,
             status,
             (uint64_t)(irp ? irp->Information : 0),
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: normal routine ran above PASSIVE_LEVEL\n");

    if (irp != &st->Irp)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: APC arg1 was not the submitted IRP\n");

    st->ApcDelivered++;
    st->ApcStatus = status;
    st->ApcInformation = irp->Information;
}

// Runs irp Kernel Apc Self Test.
static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
    DbgPrint("IRP/APC selftest: begin\n");

    (void)hal;
    if (!IoDiskDevicePresent(0)) {
        DbgPrint("IRP/APC selftest: SKIP, drive 0 not present\n");
        return;
    }

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: called above PASSIVE_LEVEL\n");

    KI_IRP_APC_SELFTEST_STATE st;
    memset(&st, 0, sizeof(st));

    KSTATUS s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: completion event create failed %u\n", s);

    st.Buffer = (uint8_t*)MmAllocatePool(512);
    if (!st.Buffer)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: no buffer\n");
    memset(st.Buffer, 0xCC, 512);

    KeInitializeApc(&st.Apc,
                    KeGetCurrentThread(),
                    KiIrpApcSelfTestKernelRoutine,
                    KiIrpApcSelfTestNormalRoutine,
                    (void*)&st);

    st.DiskDevice = IoReferenceDiskDevice(0);
    if (!st.DiskDevice)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: drive 0 disappeared before submit\n");

    IoBuildReadWriteIrp(&st.Irp, IRP_MJ_READ, 0, 512, st.Buffer);
    st.Irp.CompletionEvent = st.CompletionEvent;
    st.Irp.CompletionApc = &st.Apc;
    st.Irp.CompletionFn = (IRP_COMPLETION_FN)0;
    st.Irp.CompletionCtx = (void*)0;

    DbgPrint("IRP/APC selftest: submitting drive=0 lba=0 sectors=1\n");
    s = IoSubmitIrp(st.DiskDevice, &st.Irp);
    if (s != STATUS_PENDING)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: IoSubmitIrp returned %u\n", s);

    s = KeWaitForEvent(st.CompletionEvent, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: completion wait failed %u\n", s);

    // The wait return path should have delivered the pending kernel APC.
    if (st.ApcDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: APC was not delivered after I/O wait, count=%d\n",
                st.ApcDelivered);

    if (!KSUCCESS(st.Irp.Status) || !KSUCCESS(st.ApcStatus))
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: I/O failed irp=%u apc=%u info=%llu\n",
                st.Irp.Status, st.ApcStatus, (uint64_t)st.Irp.Information);

    if (st.Irp.Information != 512 || st.ApcInformation != 512)
        KePanic(KE_PANIC_CODE_SELFTEST, "IRP/APC selftest: wrong byte count irp=%llu apc=%llu\n",
                (uint64_t)st.Irp.Information, (uint64_t)st.ApcInformation);

    DbgPrint("IRP/APC selftest: first bytes %02X %02X %02X %02X, tail %02X %02X\n",
             st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
             st.Buffer[510], st.Buffer[511]);

    MmFreePool(st.Buffer);
    if (st.DiskDevice) IoDereferenceDeviceObject(st.DiskDevice);
    ObDereferenceObject(&st.CompletionEvent->Header);

    DbgPrint("IRP/APC selftest: PASS\n");
}





// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
static void KiRunPfnDatabaseSelfTest(void) {
    MM_PFN_STATS before;
    MM_PFN_STATS after;
    MmPfnQueryStats(&before);

    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
    if (!frame)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n");

    MM_PFN* pfn = MmPfnGet(frame);
    if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "PFN selftest: allocated frame metadata invalid phys=%016llx\n",
                (unsigned long long)frame);

    if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount);
    if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount);

    if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount);
    if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n");
    if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount);
    if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount);
    if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n");
    if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount);
    if (!KSUCCESS(MmPfnFree(frame)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n");

    PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma);
    if (!run)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n");
    MM_PFN* p0 = MmPfnGet(run);
    MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE);
    if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n");
    MmPfnFree(run);
    MmPfnFree(run + PAGE_SIZE);

    // Accounting buckets are part of the VM contract. A missing increment here
    // usually means a new allocation path is using an anonymous usage class.
    PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache);
    PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage);
    PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile);
    PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous);
    PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow);
    PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface);
    if (!clean || !image || !mapped || !anon || !cow || !surface)
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n");

    MM_PFN_STATS bucketStats;
    MmPfnQueryStats(&bucketStats);
    if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 ||
        bucketStats.ImageFrames < before.ImageFrames + 1 ||
        bucketStats.MappedFileFrames < before.MappedFileFrames + 1 ||
        bucketStats.AnonymousFrames < before.AnonymousFrames + 1 ||
        bucketStats.CowFrames < before.CowFrames + 1 ||
        bucketStats.SurfaceFrames < before.SurfaceFrames + 1) {
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n");
    }

    // Standby is the clean-page holding state used by later file-cache and image
    // reclaim. It must not change the free count until the owning cache drops it.
    if (!KSUCCESS(MmPfnMoveToStandby(clean)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n");
    if (!KSUCCESS(MmPfnMoveToStandby(image)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n");

    MM_PFN_STATS standbyStats;
    MmPfnQueryStats(&standbyStats);
    if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 ||
        standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 ||
        standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 ||
        standbyStats.FreeFrames != bucketStats.FreeFrames) {
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n");
    }

    if (!KSUCCESS(MmPfnActivate(clean)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n");
    if (!KSUCCESS(MmPfnActivate(image)))
        KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n");

    (void)MmPfnFree(clean);
    (void)MmPfnFree(image);
    (void)MmPfnFree(mapped);
    (void)MmPfnFree(anon);
    (void)MmPfnFree(cow);
    (void)MmPfnFree(surface);

    MmPfnQueryStats(&after);
    if (before.FreeFrames != after.FreeFrames)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "PFN selftest: free count leak before=%llu after=%llu\n",
                (unsigned long long)before.FreeFrames,
                (unsigned long long)after.FreeFrames);

    DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
}

// Verifies the page-table manager invariants that the higher VMM will depend on.
static void KiRunMemoryMappingSelfTest(void) {
    PHYSICAL_ADDRESS phys = 0;
    uint64_t flags = 0;
    KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags);
    if (s != STATUS_NOT_FOUND) {
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n",
                (unsigned long long)phys,
                (unsigned long long)flags,
                (unsigned)s);
    }

    uintptr_t testVa = 0xFFFFA00000000000ULL;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (s != STATUS_NOT_FOUND) {
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n",
                (unsigned long long)testVa,
                (unsigned long long)phys,
                (unsigned long long)flags);
    }

    PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
    if (!frame)
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n");

    s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent);
    if (!KSUCCESS(s)) {
        MmPfnFree(frame);
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s);
    }

    volatile uint64_t* probe = (volatile uint64_t*)testVa;
    *probe = 0x1122334455667788ULL;
    if (*probe != 0x1122334455667788ULL)
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n");

    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n",
                s,
                (unsigned long long)phys,
                (unsigned long long)frame,
                (unsigned long long)flags);

    s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (!KSUCCESS(s) || (flags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n",
                s,
                (unsigned long long)flags);

    s = MmPteUnmapPage(testVa);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
    if (s != STATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s);

    MmPfnFree(frame);
    DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
}


// Verifies sparse VAD commitment and NOACCESS protection metadata.
static void KiRunVadSparseSelfTest(void) {
    PROCESS* process = KeGetCurrentProcess();
    if (!process)
        process = KeGetSystemProcess();
    if (!process)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n");

    size_t beforeVadCount = 0;
    size_t beforeReserved = 0;
    size_t beforeCommitted = 0;
    MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted);

    void* base = (void*)0;
    KSTATUS s = MmVadReserve(process,
                             PAGE_SIZE * 4,
                             PAGE_SIZE,
                             PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                             MM_VAD_ZERO,
                             MmVadGeneric,
                             &base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s);

    uint8_t* p = (uint8_t*)base;
    s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s);

    s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s);

    void* queryBase = (void*)0;
    size_t querySize = 0;
    uint64_t queryFlags = 0;
    s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
    if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=%p size=%llu flags=%016llx\n",
                s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
    if (s != STATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s);

    s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s);
    s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s);

    s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
    if (s != STATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s);

    p[0] = 0x5A;
    p[PAGE_SIZE * 2] = 0xA5;

    s = MmVadProtect(process, p, PAGE_SIZE, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
    if (s != STATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s);

    s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
    if (!KSUCCESS(s) || queryFlags != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n",
                s, (unsigned long long)queryFlags);

    s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s);
    if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n");

    size_t afterVadCount = 0;
    size_t afterReserved = 0;
    size_t afterCommitted = 0;
    MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
    if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2)
        KePanic(KE_PANIC_CODE_SELFTEST,
                "VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n",
                (unsigned long long)beforeCommitted,
                (unsigned long long)afterCommitted,
                (unsigned long long)beforeVadCount,
                (unsigned long long)afterVadCount);

    s = MmVadDestroy(process, base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s);

    MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
    if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n");

    DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
}


// Verifies that writable copy-on-write views split before either alias can modify shared state.
static void KiRunVadCopyOnWriteSelfTest(void) {
    PROCESS* process = KeGetCurrentProcess();
    if (!process)
        process = KeGetSystemProcess();
    if (!process)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n");

    void* base = (void*)0;
    KSTATUS s = MmVadCreateCommitted(process,
                                     PAGE_SIZE,
                                     PAGE_SIZE,
                                     PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                                     MM_VAD_ZERO,
                                     MmVadGeneric,
                                     &base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s);

    s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s);

    uint8_t* source = (uint8_t*)base;
    source[0] = 0x31;

    void* cowBase = (void*)0;
    s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s);

    PHYSICAL_ADDRESS sourcePhys = 0;
    PHYSICAL_ADDRESS cowPhys = 0;
    uint64_t sourceFlags = 0;
    uint64_t cowFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags);
    if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n",
                s, (unsigned long long)sourceFlags);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags);
    if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n",
                s,
                (unsigned long long)sourcePhys,
                (unsigned long long)cowPhys,
                (unsigned long long)cowFlags);

    s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s);

    uint8_t* copy = (uint8_t*)cowBase;
    copy[0] = 0x62;
    if (source[0] != 0x31 || copy[0] != 0x62)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n",
                source[0], copy[0]);

    PHYSICAL_ADDRESS splitPhys = 0;
    uint64_t splitFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags);
    if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
                s,
                (unsigned long long)sourcePhys,
                (unsigned long long)splitPhys,
                (unsigned long long)splitFlags);

    (void)MmVadDestroy(process, cowBase);
    (void)MmVadDestroy(process, base);

    void* zeroBase = (void*)0;
    s = MmVadCreateCommitted(process,
                             PAGE_SIZE,
                             PAGE_SIZE,
                             PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                             MM_VAD_ZERO,
                             MmVadGeneric,
                             &zeroBase);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s);

    s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s);

    PHYSICAL_ADDRESS zeroPhys = 0;
    uint64_t zeroFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags);
    if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n",
                s, (unsigned long long)zeroFlags);

    s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s);

    PHYSICAL_ADDRESS privatePhys = 0;
    uint64_t privateFlags = 0;
    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags);
    if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
                s,
                (unsigned long long)zeroPhys,
                (unsigned long long)privatePhys,
                (unsigned long long)privateFlags);

    ((uint8_t*)zeroBase)[0] = 0x9C;
    if (((uint8_t*)zeroBase)[0] != 0x9C)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n");

    (void)MmVadDestroy(process, zeroBase);
    DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
}


// Verifies that a private resident VAD page can move to swap and fault back in.
static void KiRunVadSwapSelfTest(void) {
    if (!MmSwapIsEnabled()) {
        DbgPrint("VAD swap selftest: SKIP, swap is disabled\n");
        return;
    }

    PROCESS* process = KeGetCurrentProcess();
    if (!process)
        process = KeGetSystemProcess();
    if (!process)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n");

    void* base = (void*)0;
    KSTATUS s = MmVadCreateCommitted(process,
                                     PAGE_SIZE,
                                     PAGE_SIZE,
                                     PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
                                     MM_VAD_ZERO,
                                     MmVadGeneric,
                                     &base);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s);

    s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s);

    uint8_t* p = (uint8_t*)base;
    p[0] = 0xC1;
    p[PAGE_SIZE - 1] = 0x5E;

    void* trimmed = (void*)0;
    s = MmVadTrimOnePageForTest(process, &trimmed);
    if (!KSUCCESS(s) || trimmed != base)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=%p base=%p\n", s, trimmed, base);

    s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
    if (s != STATUS_NOT_FOUND)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s);

    s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s);

    if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E)
        KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]);

    (void)MmVadDestroy(process, base);
    DbgPrint("VAD swap selftest: private pageout/pagein PASS\n");
}

// String-view ABI self-test.
static void KiRunStringViewSelfTest(void) {
    DbgPrint("STRINGVIEW selftest: begin\n");

    SYS_STRING_VIEW hello = SYS_SV_LITERAL("Hello");
    if (hello.Length != 5 || !SysSvEqualLit(hello, "Hello"))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: literal/equality failed\n");

    static const char embedded[] = { 'a', 'b', '\0', 'c', 'd' };
    SYS_STRING_VIEW emb = SysSvFromParts(embedded, sizeof(embedded));
    if (emb.Length != 5 || !SysSvContainsByte(emb, '\0'))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: embedded-NUL detection failed\n");

    char copy[6];
    if (!SysSvCopyZ(copy, sizeof(copy), emb))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: bounded copy failed\n");
    if (memcmp(copy, embedded, sizeof(embedded)) != 0 || copy[5] != '\0')
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: bounded copy contents wrong\n");

    if (SysSvCopyZ(copy, 4, emb))
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: undersized copy unexpectedly succeeded\n");

    // Paths are string views at the ABI boundary, but FAT/VFS path names do not
    // accept embedded NUL bytes. This must fail before reaching any filesystem.
    FILE_OBJECT* f = (FILE_OBJECT*)0;
    KSTATUS s = IoOpenFile(SysSvFromParts("system/init.exe\0junk", 20), &f);
    if (s != STATUS_INVALID_PARAMETER)
        KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: embedded-NUL path status=%u\n", s);

    DbgPrint("STRINGVIEW selftest: sample view='%v' len=%llu\n",
             hello, (uint64_t)hello.Length);
    DbgPrint("STRINGVIEW selftest: PASS\n");
}

// Async file I/O self-test.

typedef struct KI_FILE_ASYNC_SELFTEST_STATE {
    EVENT* CompletionEvent;
    FILE_OBJECT* File;
    IO_STATUS_BLOCK Iosb;
    KAPC Apc;
    uint8_t* Buffer;

    volatile int ApcDelivered;
    volatile KSTATUS ApcStatus;
} KI_FILE_ASYNC_SELFTEST_STATE;

static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc,
                                             void** normalRoutine,
                                             void** normalContext,
                                             void** systemArgument1,
                                             void** systemArgument2) {
    DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%016llX iosb=0x%016llX status=%u irql=%u\n",
             (uint64_t)apc,
             (uint64_t)*systemArgument1,
             (uint32_t)(uintptr_t)*systemArgument2,
             KeGetCurrentIrql());

    (void)normalRoutine;
    (void)normalContext;
}

static void KiFileAsyncSelfTestNormalRoutine(void* normalContext,
                                             void* systemArgument1,
                                             void* systemArgument2) {
    KI_FILE_ASYNC_SELFTEST_STATE* st =
        (KI_FILE_ASYNC_SELFTEST_STATE*)normalContext;
    KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;

    DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%016llX status=%u iosb=%u/%llu irql=%u\n",
             (uint64_t)systemArgument1,
             status,
             st->Iosb.Status,
             (uint64_t)st->Iosb.Information,
             KeGetCurrentIrql());

    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: APC ran above PASSIVE_LEVEL\n");

    st->ApcDelivered++;
    st->ApcStatus = status;
}

// Runs async File Io Self Test.
static void KiRunAsyncFileIoSelfTest(void) {
    DbgPrint("FILE/ASYNC selftest: begin\n");

    KI_FILE_ASYNC_SELFTEST_STATE st;
    memset(&st, 0, sizeof(st));
    st.Iosb.Status = STATUS_FAILURE;

    KSTATUS s = IoOpenFile(SYS_SV_LITERAL("system/init.exe"), &st.File);
    if (!KSUCCESS(s)) {
        DbgPrint("FILE/ASYNC selftest: SKIP, system/init.exe open failed %u\n", s);
        return;
    }

    s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
    if (!KSUCCESS(s))
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: event create failed %u\n", s);

    st.Buffer = (uint8_t*)MmAllocatePool(512);
    if (!st.Buffer)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: no buffer\n");
    memset(st.Buffer, 0xCC, 512);

    KeInitializeApc(&st.Apc,
                    KeGetCurrentThread(),
                    KiFileAsyncSelfTestKernelRoutine,
                    KiFileAsyncSelfTestNormalRoutine,
                    (void*)&st);

    DbgPrint("FILE/ASYNC selftest: submit read system/init.exe offset=0 size=512\n");
    IO_ASYNC_FILE_REQUEST_OBJECT* readReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
    s = IoReadFileAsyncKernel(st.File,
                              st.Buffer,
                              512,
                              0,
                              &st.Iosb,
                              st.CompletionEvent,
                              &st.Apc,
                              (MM_LOCKED_USER_BUFFER*)0,
                              (MM_LOCKED_USER_BUFFER*)0,
                              &readReq);
    if (s != STATUS_PENDING)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s);

    s = KeWaitForEvent(st.CompletionEvent, 1);
    if (s != STATUS_SUCCESS && s != STATUS_KERNEL_APC)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: wait returned %u\n", s);

    if (st.Iosb.Status == STATUS_PENDING) {
        s = KeWaitForEvent(st.CompletionEvent, 0);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: final event wait returned %u\n", s);
    }

    if (st.ApcDelivered != 1)
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: APC not delivered count=%d\n", st.ApcDelivered);

    if (!KSUCCESS(st.Iosb.Status) || !KSUCCESS(st.ApcStatus))
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: I/O failed iosb=%u apc=%u info=%llu\n",
                st.Iosb.Status, st.ApcStatus, (uint64_t)st.Iosb.Information);

    if (st.Iosb.Information < 2 || st.Buffer[0] != 'M' || st.Buffer[1] != 'Z')
        KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: bad file header info=%llu bytes=%02X %02X\n",
                (uint64_t)st.Iosb.Information, st.Buffer[0], st.Buffer[1]);

    DbgPrint("FILE/ASYNC selftest: first bytes %02X %02X %02X %02X info=%llu wait=%u\n",
             st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
             (uint64_t)st.Iosb.Information, s);

    if (readReq)
        ObDereferenceObject(IoAsyncFileRequestObject(readReq));

    MmFreePool(st.Buffer);
    ObDereferenceObject(&st.CompletionEvent->Header);
    IoCloseFile(st.File);

    // Exercise the async write path with an explicit-offset temporary file.
    {
        static const char payload[] = "async-file-write-selftest\n";
        char verify[sizeof(payload)];
        FILE_OBJECT* wf = (FILE_OBJECT*)0;
        FILE_OBJECT* rf = (FILE_OBJECT*)0;
        EVENT* wev = (EVENT*)0;
        IO_STATUS_BLOCK wiosb;
        size_t got = 0;

        memset(verify, 0, sizeof(verify));
        wiosb.Status = STATUS_FAILURE;
        wiosb.Information = 0;

        s = IoCreateFile(SYS_SV_LITERAL("asyncio.tmp"), &wf);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: create write temp failed %u\n", s);

        s = KeEventCreate(EventAutoReset, 0, &wev);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write event create failed %u\n", s);

        DbgPrint("FILE/ASYNC selftest: submit write asyncio.tmp size=%llu\n",
                 (uint64_t)(sizeof(payload) - 1));

        IO_ASYNC_FILE_REQUEST_OBJECT* writeReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
        s = IoWriteFileAsyncKernel(wf,
                                   payload,
                                   sizeof(payload) - 1,
                                   0,
                                   &wiosb,
                                   wev,
                                   (KAPC*)0,
                                   (MM_LOCKED_USER_BUFFER*)0,
                                   (MM_LOCKED_USER_BUFFER*)0,
                                   &writeReq);
        if (s != STATUS_PENDING)
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s);

        s = KeWaitForEvent(wev, 1);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write wait returned %u\n", s);

        if (!KSUCCESS(wiosb.Status) || wiosb.Information != sizeof(payload) - 1)
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write failed status=%u info=%llu\n",
                    wiosb.Status, (uint64_t)wiosb.Information);

        if (writeReq)
            ObDereferenceObject(IoAsyncFileRequestObject(writeReq));

        IoCloseFile(wf);
        wf = (FILE_OBJECT*)0;

        s = IoOpenFile(SYS_SV_LITERAL("asyncio.tmp"), &rf);
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: reopen write temp failed %u\n", s);

        s = IoReadFileAt(rf, 0, verify, sizeof(payload) - 1, &got);
        if (!KSUCCESS(s) || got != sizeof(payload) - 1 ||
            // Implements the memcmp operation.
            memcmp(verify, payload, sizeof(payload) - 1) != 0) {
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: verify failed status=%u got=%llu\n",
                    s, (uint64_t)got);
        }

        IoCloseFile(rf);
        ObDereferenceObject(&wev->Header);
        s = IoDeleteFile(SYS_SV_LITERAL("asyncio.tmp"));
        if (!KSUCCESS(s))
            KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: delete temp failed %u\n", s);

        DbgPrint("FILE/ASYNC selftest: async write verified\n");
    }

    DbgPrint("FILE/ASYNC selftest: PASS\n");
}


// Runs trap Frame Self Test.
static void KiRunTrapFrameSelfTest(void) {
    DbgPrint("TRAP selftest: begin\n");

    if (KeGetCurrentTrapFrame())
        KePanic(KE_PANIC_CODE_SELFTEST, "TRAP selftest: kernel selftest unexpectedly has active trap frame\n");

    DbgPrint("TRAP selftest: frame size=%llu query size=%llu\n",
             (uint64_t)sizeof(KTRAP_FRAME),
             (uint64_t)sizeof(SYS_TRAP_FRAME_INFO_K));

    if (sizeof(KTRAP_FRAME) != 240)
        KePanic(KE_PANIC_CODE_SELFTEST, "TRAP selftest: KTRAP_FRAME size mismatch %llu\n",
                (uint64_t)sizeof(KTRAP_FRAME));

    DbgPrint("TRAP selftest: PASS\n");
}



// Runs core Hardening Self Tests.
void KiRunCoreHardeningSelfTests(void) {
    DbgPrint("CORE selftest: begin\n");

    if (gCpuCount == 0 || gCpuCount > KE_MAX_CPUS)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid CPU count %u\n", gCpuCount);

    KCPU* cpu = KeGetCurrentCpu();
    if (cpu != &gCpus[0])
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%016llX cpu0=0x%016llX\n",
                (uint64_t)cpu, (uint64_t)&gCpus[0]);
    if (cpu->Self != cpu)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%016llX cpu=0x%016llX\n",
                (uint64_t)cpu->Self, (uint64_t)cpu);
    if (KeGetCurrentCpuIndex() != 0)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n");
    if (!cpu->Present || !cpu->Started || !cpu->Online || !cpu->IsBsp)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle flags invalid P/S/O/B=%u/%u/%u/%u\n",
                cpu->Present, cpu->Started, cpu->Online, cpu->IsBsp);
    if (cpu->Parked)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP is marked parked\n");
    if (KeGetCurrentIrql() != PASSIVE_LEVEL)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql());
    if (KeGetCurrentTss() != &cpu->Tss)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n");
    if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64))
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%016llX iomap=%u\n",
                cpu->Tss.RSP0, cpu->Tss.IoMapBase);
    if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%016llX gdt=0x%016llX\n",
                cpu->Gdtr.Base, (uint64_t)cpu->Gdt);

    for (uint32_t i = 1; i < gCpuCount; i++) {
        KCPU* ap = &gCpus[i];
        if (!ap->Present)
            continue;
        if (!ap->Started || ap->Parked || !ap->Online || ap->IsBsp)
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n",
                    i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp);
        if (!ap->SchedulerCommand || !ap->SchedulerActive)
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "CORE selftest: AP%u scheduler admission invalid command=%ld active=%ld\n",
                    i, ap->SchedulerCommand, ap->SchedulerActive);
        if (!ap->KernelStack || !ap->KernelStackTop)
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
        if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%016llX iomap=%u\n",
                    i, ap->Tss.RSP0, ap->Tss.IoMapBase);
        if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt)
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i);
        if (!ap->IdleThread || !ap->CurrentThread || ap->CurrentThread != ap->IdleThread)
            KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u idle/current thread missing\n", i);
        if (!ap->IdleThread->IsIdleThread || ap->IdleThread->Queued || ap->IdleThread->RunningCpu != i)
            KePanic(KE_PANIC_CODE_SELFTEST,
                    "CORE selftest: AP%u idle invariant failed queued=%u idle=%u runningCpu=%u\n",
                    i, ap->IdleThread->Queued, ap->IdleThread->IsIdleThread, ap->IdleThread->RunningCpu);
    }

    THREAD* cur = KeGetCurrentThread();
    if (!cur || cur != cpu->CurrentThread)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current thread mismatch\n");
    if (!cpu->IdleThread || cur != cpu->IdleThread)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: scheduler did not start on CPU0 idle thread\n");
    if (!cur->IsIdleThread || cur->Queued || cur->RunningCpu != cpu->CpuIndex)
        KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: idle invariant failed queued=%u idle=%u runningCpu=%u\n",
                cur->Queued, cur->IsIdleThread, cur->RunningCpu);

    DbgPrint("CORE selftest: PASS\n");
}


// Runs early Kernel Self Tests.
void KiRunEarlyKernelSelfTests(void) {
    KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests();
    if (!KSUCCESS(s)) {
        KePanic(KE_PANIC_CODE_SELFTEST,
                "SMP scheduler selftest failed status=0x%x\n",
                s);
    }
}

// Runs kernel Self Tests.
void KiRunKernelSelfTests(const HAL_API* hal) {
    DbgPrint("Kernel selftests: begin\n");
    KiRunKernelApcSelfTest();
    KiRunPfnDatabaseSelfTest();
    KiRunMemoryMappingSelfTest();
    KiRunVadSparseSelfTest();
    KiRunVadCopyOnWriteSelfTest();
    KiRunVadSwapSelfTest();
    KiRunStringViewSelfTest();
    KiRunIrpKernelApcSelfTest(hal);
    KiRunAsyncFileIoSelfTest();
    KiRunTrapFrameSelfTest();
    DbgPrint("Kernel selftests: PASS\n");
}