maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
73abd6f90dde1785e8a6aa91a56880e5673233b1
nt-lite/code/kernel/ke_selftest.c
T
maksw 73abd6f90d isolate POSIX builds from the host UCRT, wire lwext4 and ntfs-3g as ext4.sys and ntfs.sys, split the FAT driver into directory/file/async/driver/mount units, add posixsrv and dwm plus input/console/desktop syscall clients and a pile of new smoke targets, rename serial.sys to cereal.sys, name HDA/USB/lwIP/AHCI workers, harden ATA probing/reset/removal around actual hardware deadlines, absent-channel detection, diagnostics and explicit worker lifetime, redo ACPI S5 shutdown around GAS/MMIO and HW-reduced firmware, scan SSDTs for _S5 and deal with Intel SMI sleep trapping, make serial and SMP timeouts use real monotonic deadlines, add thread names and fix terminal thread teardown, scheduler and APC prepare/commit races, make advisory file locks queryable and preserve split ranges and add mutable file append state, add shared-memory pipes with kernel-assisted wait/wake, introduce capability-scoped input and display managers, split surface map rights into read/write, add explicit unmap and replace present with commit/generation/damage tracking while keeping physical presentation under the display manager, decouple native exception classes from Win32 exception numbers and translate them in kernel32, add opaque process startup data, bump the resource protocol to v3 with namespace directories, symlinks and enumeration and service control to v2 with generic READY handle exports, add the new POSIX errno/status surface and rebuild fds around posixsrv-owned shared open descriptions, descriptor-local flags, exact inheritance snapshots, dup/fcntl/record locks and shared-ring pipes, add waitpid, POSIX credentials/groups/sessions/passwd/group lookups and move sockets onto stable open descriptions, port the shells and apps to the new console/input services and surface commit API, teach init to load services.ini and start/supervise RSSRV, DWM and generic services under restricted capabilities while publishing their endpoints through RSSRV, split RSSRV into namespace, volume, display, session, input, console and resource components and make /SystemRoot a real symlink to the independently published system volume, shift the syscall table again, expand lock/resource/surface/POSIX/service/thread smoke coverage, make docs scream ASK QUESTIONS and DISCUSS at LLMs, replace MIT No Attribution with WTFPL+.
2026-09-16 10:33:20 +02:00

1542 lines
68 KiB
C
Raw Blame History

// Kernel self-tests. This file runs bring-up regression checks before normal user workloads start.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_event.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_sched.h>
#include <ntlite/status.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <kernel/io_dma.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_swap.h>
#include <kernel/mm_pfn.h>
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>
// Focused APC wait-commit race self-test. User-mode APC return and async I/O
// behavior are covered by executable smoke tests; keep only the scheduler race
// that cannot be constructed through the public ABI.
// Verifies that one APC normal routine executes in interruptible PASSIVE thread context.
static void KiAssertApcNormalContext(const char* name) {
KCPU* cpu = KeGetCurrentCpu();
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "%s ran above PASSIVE_LEVEL\n", name);
if ((__readeflags() & 0x200u) == 0)
KePanic(KE_PANIC_CODE_SELFTEST, "%s ran with interrupts disabled\n", name);
if (!cpu || cpu->InterruptDepth != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "%s ran inside interrupt context depth=%u\n",
name, cpu ? cpu->InterruptDepth : 0xFFFFFFFFu);
}
// Leaves the APC's normal routine and arguments unchanged.
static void KiApcWaitRaceKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
(void)apc;
(void)normalRoutine;
(void)normalContext;
(void)systemArgument1;
(void)systemArgument2;
}
// Records delivery for the prepared-wait race probe.
static void KiApcWaitRaceNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
(void)systemArgument1;
(void)systemArgument2;
KiAssertApcNormalContext("APC wait-race normal routine");
(*(volatile int*)normalContext)++;
}
// Reproduces the prepare-to-commit APC race deterministically.
static void KiRunApcWaitRaceSelfTest(void) {
DbgPrint("APC wait-race selftest: begin\n");
THREAD* self = KeGetCurrentThread();
if (!self || KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: invalid execution context\n");
EVENT* waitEvent = (EVENT*)0;
KAPC apc;
volatile int delivered = 0;
KSTATUS status = KeEventCreate(EventAutoReset, 0, &waitEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: event create failed %u\n", status);
KeInitializeApc(&apc, self, KiApcWaitRaceKernelRoutine,
KiApcWaitRaceNormalRoutine, (void*)&delivered);
uintptr_t flags = KeAcquireSpinlock(&waitEvent->Lock);
if (!KeInsertQueueApc(&apc, (void*)(uintptr_t)0x99999999u,
(void*)(uintptr_t)0xAAAAAAAAu))
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: queue failed\n");
if (delivered != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "APC wait-race selftest: delivered above PASSIVE_LEVEL\n");
KePrepareBlockThread(self, &waitEvent->Header, 1);
KeListInsertTailChecked(&waitEvent->WaitList, &self->SchedulerLink);
KeSetThreadWaitList(self, &waitEvent->WaitList);
KeReleaseSpinlock(&waitEvent->Lock, flags);
status = KeCommitBlockThread(self, 1);
if (status != NTLSTATUS_KERNEL_APC || delivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"APC wait-race selftest: bad result status=%u delivered=%d\n",
status, delivered);
ObDereferenceObject(&waitEvent->Header);
DbgPrint("APC wait-race selftest: PASS\n");
}
// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
static void KiRunPfnDatabaseSelfTest(void) {
MM_PFN_STATS before;
MM_PFN_STATS after;
MmPfnQueryStats(&before);
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n");
MM_PFN* pfn = MmPfnGet(frame);
if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"PFN selftest: allocated frame metadata invalid phys=%016llx\n",
(unsigned long long)frame);
if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount);
if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount);
if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount);
if (MmPfnFree(frame) != NTLSTATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n");
if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount);
if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount);
if (MmPfnFree(frame) != NTLSTATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n");
if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount);
if (!KSUCCESS(MmPfnFree(frame)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n");
PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma);
if (!run)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n");
MM_PFN* p0 = MmPfnGet(run);
MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE);
if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n");
MmPfnFree(run);
MmPfnFree(run + PAGE_SIZE);
// Accounting buckets are part of the VM contract. A missing increment here
// usually means a new allocation path is using an anonymous usage class.
PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache);
PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage);
PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile);
PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous);
PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow);
PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface);
if (!clean || !image || !mapped || !anon || !cow || !surface)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n");
MM_PFN_STATS bucketStats;
MmPfnQueryStats(&bucketStats);
if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 ||
bucketStats.ImageFrames < before.ImageFrames + 1 ||
bucketStats.MappedFileFrames < before.MappedFileFrames + 1 ||
bucketStats.AnonymousFrames < before.AnonymousFrames + 1 ||
bucketStats.CowFrames < before.CowFrames + 1 ||
bucketStats.SurfaceFrames < before.SurfaceFrames + 1) {
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n");
}
// Standby is the clean-page holding state used by later file-cache and image
// reclaim. It must not change the free count until the owning cache drops it.
if (!KSUCCESS(MmPfnMoveToStandby(clean)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n");
if (!KSUCCESS(MmPfnMoveToStandby(image)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n");
MM_PFN_STATS standbyStats;
MmPfnQueryStats(&standbyStats);
if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 ||
standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 ||
standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 ||
standbyStats.FreeFrames != bucketStats.FreeFrames) {
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n");
}
if (!KSUCCESS(MmPfnActivate(clean)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n");
if (!KSUCCESS(MmPfnActivate(image)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n");
(void)MmPfnFree(clean);
(void)MmPfnFree(image);
(void)MmPfnFree(mapped);
(void)MmPfnFree(anon);
(void)MmPfnFree(cow);
(void)MmPfnFree(surface);
MmPfnQueryStats(&after);
if (before.FreeFrames != after.FreeFrames)
KePanic(KE_PANIC_CODE_SELFTEST,
"PFN selftest: free count leak before=%llu after=%llu\n",
(unsigned long long)before.FreeFrames,
(unsigned long long)after.FreeFrames);
DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
}
// Verifies common-buffer, direct mapping, bounce mapping, and teardown accounting.
static void KiRunDmaSelfTest(void) {
const NTL_DDK_DMA* dma = IoGetDmaDdk();
if (!dma)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: DDK table missing\n");
IO_DMA_STATS before;
IO_DMA_STATS after;
dma->QueryStats(&before);
DRIVER_OBJECT driver;
memset(&driver, 0, sizeof(driver));
IoInitializeDriverObject(&driver, SYS_SV_LITERAL("dma-selftest"));
DEVICE_OBJECT* device = (DEVICE_OBJECT*)0;
KSTATUS status = IoCreateDevice(&driver, 0, &device);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: device creation failed status=%u\n", status);
IO_DMA_REQUIREMENTS requirements;
memset(&requirements, 0, sizeof(requirements));
requirements.MaximumDeviceAddress = UINT64_MAX;
requirements.Alignment = PAGE_SIZE;
requirements.MaximumSegmentLength = PAGE_SIZE;
requirements.MaximumSegments = 4;
IO_DMA_ADAPTER* adapter = (IO_DMA_ADAPTER*)0;
IO_DMA_BUFFER* buffer = (IO_DMA_BUFFER*)0;
MM_PINNED_MEMORY* pinned = (MM_PINNED_MEMORY*)0;
IO_DMA_MAPPING* mapping = (IO_DMA_MAPPING*)0;
status = dma->GetAdapter(device, &requirements, &adapter);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: adapter creation failed status=%u\n", status);
status = dma->AllocateCommonBuffer(adapter, PAGE_SIZE, PAGE_SIZE, &buffer);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: common buffer allocation failed status=%u\n", status);
uint8_t* bytes = (uint8_t*)dma->GetCommonBufferAddress(buffer);
uint64_t deviceAddress = dma->GetCommonBufferDeviceAddress(buffer);
if (!bytes || !deviceAddress || (deviceAddress & (PAGE_SIZE - 1)) != 0 ||
dma->GetCommonBufferSize(buffer) != PAGE_SIZE)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: common buffer metadata invalid address=0x%016llX\n",
(unsigned long long)deviceAddress);
status = dma->PinKernelMemory(bytes, PAGE_SIZE, IoDmaBidirectional, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: kernel pin failed status=%u\n", status);
status = dma->MapPinnedMemory(adapter,
pinned,
0,
PAGE_SIZE,
IoDmaBidirectional,
IO_DMA_MAP_NO_BOUNCE,
&mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
dma->GetSegments(mapping)[0].DeviceAddress != deviceAddress)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: direct mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
for (size_t i = 0; i < 128; i++)
bytes[i + 1] = (uint8_t)(i ^ 0x5Au);
status = dma->PinKernelMemory(bytes + 1, 128, IoDmaToDevice, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: unaligned kernel pin failed status=%u\n", status);
status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaToDevice, 0, &mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
(dma->GetSegments(mapping)[0].DeviceAddress & (PAGE_SIZE - 1)) != 0 ||
dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: bounce mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
status = dma->SynchronizeForDevice(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: bounce synchronization failed status=%u\n", status);
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
memset(bytes + 1, 0, 128);
status = dma->PinKernelMemory(bytes + 1, 128, IoDmaFromDevice, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive pin failed status=%u\n", status);
status = dma->MapPinnedMemory(adapter, pinned, 0, 128, IoDmaFromDevice, 0, &mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1 ||
dma->GetSegments(mapping)[0].DeviceAddress == deviceAddress + 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: receive bounce mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
status = dma->SynchronizeForDevice(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive handoff failed status=%u\n", status);
uint64_t receiveDeviceAddress = dma->GetSegments(mapping)[0].DeviceAddress;
uint8_t* receiveDeviceAlias = (uint8_t*)MmPfnToVirtual(receiveDeviceAddress & PAGE_MASK);
if (!receiveDeviceAlias)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive device alias missing\n");
receiveDeviceAlias += receiveDeviceAddress & (PAGE_SIZE - 1);
for (size_t i = 0; i < 128; i++)
receiveDeviceAlias[i] = (uint8_t)(i ^ 0xA5u);
status = dma->SynchronizeForCpu(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: receive synchronization failed status=%u\n", status);
for (size_t i = 0; i < 128; i++) {
if (bytes[i + 1] != (uint8_t)(i ^ 0xA5u))
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: receive bounce copy mismatch offset=%llu\n",
(unsigned long long)i);
}
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: no process for user pinning\n");
void* userBase = (void*)0;
status = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&userBase);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD creation failed status=%u\n", status);
status = MmVadEnsureResident(process, userBase, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD materialization failed status=%u\n", status);
((uint8_t*)userBase)[0] = 0xD5u;
status = dma->PinUserMemory(process, userBase, PAGE_SIZE, IoDmaToDevice, &pinned);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user pin failed status=%u\n", status);
status = MmVadDestroy(process, userBase);
if (status != NTLSTATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: pinned user VAD was destroyable status=%u\n",
status);
status = dma->MapPinnedMemory(adapter,
pinned,
0,
PAGE_SIZE,
IoDmaToDevice,
IO_DMA_MAP_NO_BOUNCE,
&mapping);
if (!KSUCCESS(status) || dma->GetSegmentCount(mapping) != 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"DMA selftest: user mapping failed status=%u segments=%u\n",
status,
dma->GetSegmentCount(mapping));
status = dma->SynchronizeForDevice(mapping);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user synchronization failed status=%u\n", status);
dma->UnmapMemory(mapping);
mapping = (IO_DMA_MAPPING*)0;
dma->UnpinMemory(pinned);
pinned = (MM_PINNED_MEMORY*)0;
status = MmVadDestroy(process, userBase);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: user VAD destroy failed status=%u\n", status);
dma->FreeCommonBuffer(buffer);
dma->PutAdapter(adapter);
IoDeleteDevice(device);
dma->QueryStats(&after);
if (after.AdaptersCreated != before.AdaptersCreated + 1 ||
after.AdaptersDestroyed != before.AdaptersDestroyed + 1 ||
after.AdaptersInUse != before.AdaptersInUse ||
after.CommonBuffersAllocated < before.CommonBuffersAllocated + 3 ||
after.CommonBuffersFreed < before.CommonBuffersFreed + 3 ||
after.CommonBuffersInUse != before.CommonBuffersInUse ||
after.CommonBufferBytesInUse != before.CommonBufferBytesInUse ||
after.PinnedRangesInUse != before.PinnedRangesInUse ||
after.PinnedPagesInUse != before.PinnedPagesInUse ||
after.PinnedKernelRanges != before.PinnedKernelRanges + 3 ||
after.PinnedUserRanges != before.PinnedUserRanges + 1 ||
after.PinnedRangesReleased != before.PinnedRangesReleased + 4 ||
after.StreamingMappings != before.StreamingMappings + 4 ||
after.StreamingMappingsUnmapped != before.StreamingMappingsUnmapped + 4 ||
after.StreamingMappingsInUse != before.StreamingMappingsInUse ||
after.DirectMappings != before.DirectMappings + 2 ||
after.BounceMappings != before.BounceMappings + 2 ||
after.BounceBytesCopiedToDevice < before.BounceBytesCopiedToDevice + 128 ||
after.BounceBytesCopiedToCpu < before.BounceBytesCopiedToCpu + 128)
KePanic(KE_PANIC_CODE_SELFTEST, "DMA selftest: accounting mismatch\n");
DbgPrint("DMA selftest: common/kernel-pin/user-pin/direct/send-bounce/receive-bounce/teardown PASS\n");
}
// Verifies the page-table manager invariants that the higher VMM will depend on.
static void KiRunMemoryMappingSelfTest(void) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags);
if (s != NTLSTATUS_NOT_FOUND) {
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n",
(unsigned long long)phys,
(unsigned long long)flags,
(unsigned)s);
}
uintptr_t testVa = 0xFFFFA00000000000ULL;
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (s != NTLSTATUS_NOT_FOUND) {
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n",
(unsigned long long)testVa,
(unsigned long long)phys,
(unsigned long long)flags);
}
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n");
s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent);
if (!KSUCCESS(s)) {
MmPfnFree(frame);
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s);
}
volatile uint64_t* probe = (volatile uint64_t*)testVa;
*probe = 0x1122334455667788ULL;
if (*probe != 0x1122334455667788ULL)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n");
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n",
s,
(unsigned long long)phys,
(unsigned long long)frame,
(unsigned long long)flags);
s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (!KSUCCESS(s) || (flags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n",
s,
(unsigned long long)flags);
s = MmPteUnmapPage(testVa);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (s != NTLSTATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s);
MmPfnFree(frame);
DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
}
// Verifies sparse VAD commitment and NOACCESS protection metadata.
static void KiRunVadSparseSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n");
size_t beforeVadCount = 0;
size_t beforeReserved = 0;
size_t beforeCommitted = 0;
MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted);
void* base = (void*)0;
KSTATUS s = MmVadReserve(process,
PAGE_SIZE * 4,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s);
uint8_t* p = (uint8_t*)base;
s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s);
s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s);
void* queryBase = (void*)0;
size_t querySize = 0;
uint64_t queryFlags = 0;
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=0x%016llX size=%llu flags=%016llx\n",
s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != NTLSTATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s);
s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s);
s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s);
s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (s != NTLSTATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s);
p[0] = 0x5A;
p[PAGE_SIZE * 2] = 0xA5;
s = MmVadProtect(process, p, PAGE_SIZE, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != NTLSTATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s);
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (!KSUCCESS(s) || queryFlags != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n",
s, (unsigned long long)queryFlags);
s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s);
if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n");
size_t afterVadCount = 0;
size_t afterReserved = 0;
size_t afterCommitted = 0;
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2)
KePanic(KE_PANIC_CODE_SELFTEST,
"VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n",
(unsigned long long)beforeCommitted,
(unsigned long long)afterCommitted,
(unsigned long long)beforeVadCount,
(unsigned long long)afterVadCount);
s = MmVadDestroy(process, base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s);
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n");
DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
}
// Verifies that writable copy-on-write views split before either alias can modify shared state.
static void KiRunVadCopyOnWriteSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n");
void* base = (void*)0;
KSTATUS s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s);
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s);
uint8_t* source = (uint8_t*)base;
source[0] = 0x31;
void* cowBase = (void*)0;
s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s);
PHYSICAL_ADDRESS sourcePhys = 0;
PHYSICAL_ADDRESS cowPhys = 0;
uint64_t sourceFlags = 0;
uint64_t cowFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags);
if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n",
s, (unsigned long long)sourceFlags);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags);
if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n",
s,
(unsigned long long)sourcePhys,
(unsigned long long)cowPhys,
(unsigned long long)cowFlags);
s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s);
uint8_t* copy = (uint8_t*)cowBase;
copy[0] = 0x62;
if (source[0] != 0x31 || copy[0] != 0x62)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n",
source[0], copy[0]);
PHYSICAL_ADDRESS splitPhys = 0;
uint64_t splitFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags);
if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
s,
(unsigned long long)sourcePhys,
(unsigned long long)splitPhys,
(unsigned long long)splitFlags);
(void)MmVadDestroy(process, cowBase);
(void)MmVadDestroy(process, base);
void* zeroBase = (void*)0;
s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&zeroBase);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s);
s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s);
PHYSICAL_ADDRESS zeroPhys = 0;
uint64_t zeroFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags);
if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n",
s, (unsigned long long)zeroFlags);
s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s);
PHYSICAL_ADDRESS privatePhys = 0;
uint64_t privateFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags);
if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
s,
(unsigned long long)zeroPhys,
(unsigned long long)privatePhys,
(unsigned long long)privateFlags);
((uint8_t*)zeroBase)[0] = 0x9C;
if (((uint8_t*)zeroBase)[0] != 0x9C)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n");
(void)MmVadDestroy(process, zeroBase);
DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
}
// Verifies that a private resident VAD page can move to swap and fault back in.
static void KiRunVadSwapSelfTest(void) {
if (!MmSwapIsEnabled()) {
DbgPrint("VAD swap selftest: SKIP, swap is disabled\n");
return;
}
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n");
void* base = (void*)0;
KSTATUS s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s);
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s);
uint8_t* p = (uint8_t*)base;
p[0] = 0xC1;
p[PAGE_SIZE - 1] = 0x5E;
void* trimmed = (void*)0;
s = MmVadTrimOnePageForTest(process, &trimmed);
if (!KSUCCESS(s) || trimmed != base)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=0x%016llX base=0x%016llX\n", s, trimmed, base);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != NTLSTATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s);
s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s);
if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]);
(void)MmVadDestroy(process, base);
DbgPrint("VAD swap selftest: private pageout/pagein PASS\n");
}
// String-view ABI self-test.
static void KiRunStringViewSelfTest(void) {
DbgPrint("STRINGVIEW selftest: begin\n");
SYS_STRING_VIEW hello = SYS_SV_LITERAL("Hello");
if (hello.Length != 5 || !SysSvEqualLit(hello, "Hello"))
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: literal/equality failed\n");
static const char embedded[] = { 'a', 'b', '\0', 'c', 'd' };
SYS_STRING_VIEW emb = SysSvFromParts(embedded, sizeof(embedded));
if (emb.Length != 5 || !SysSvContainsByte(emb, '\0'))
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: embedded-NUL detection failed\n");
char copy[6];
if (!SysSvCopyZ(copy, sizeof(copy), emb))
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: bounded copy failed\n");
if (memcmp(copy, embedded, sizeof(embedded)) != 0 || copy[5] != '\0')
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: bounded copy contents wrong\n");
if (SysSvCopyZ(copy, 4, emb))
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: undersized copy unexpectedly succeeded\n");
// Paths are string views at the ABI boundary, but provider path names do not
// accept embedded NUL bytes. This must fail before reaching any filesystem.
IO_VOLUME* volume = (IO_VOLUME*)0;
KSTATUS s = KeReferenceSystemRootVolume(&volume);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: system volume reference status=%u\n", s);
FILE_OBJECT* f = (FILE_OBJECT*)0;
s = IoOpenFileAtVolume(volume, SysSvFromParts("system/init.exe\0junk", 20), &f);
IoReleaseVolume(volume);
if (s != NTLSTATUS_INVALID_PARAMETER)
KePanic(KE_PANIC_CODE_SELFTEST, "STRINGVIEW selftest: embedded-NUL path status=%u\n", s);
DbgPrint("STRINGVIEW selftest: sample view='%v' len=%llu\n",
hello, (uint64_t)hello.Length);
DbgPrint("STRINGVIEW selftest: PASS\n");
}
// Capability-relative native-path self-test.
static void KiRunVolumeRelativePathSelfTest(void) {
DbgPrint("VOLUME/PATH selftest: begin\n");
IO_VOLUME* volume = (IO_VOLUME*)0;
KSTATUS status = KeReferenceSystemRootVolume(&volume);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: system volume reference failed %u\n", status);
FILE_OBJECT* file = (FILE_OBJECT*)0;
status = IoOpenFileAtVolume(volume, SYS_SV_LITERAL("system/init.exe"), &file);
if (!KSUCCESS(status) || !file)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: direct open failed %u\n", status);
IoCloseFile(file);
const SYS_STRING_VIEW invalidPaths[] = {
SYS_SV_LITERAL("/system/init.exe"),
SYS_SV_LITERAL("system\\init.exe"),
SYS_SV_LITERAL("system//init.exe"),
SYS_SV_LITERAL("system/./init.exe"),
SYS_SV_LITERAL("system/../init.exe"),
SYS_SV_LITERAL("system/init.exe/"),
};
for (size_t i = 0; i < sizeof(invalidPaths) / sizeof(invalidPaths[0]); ++i) {
file = (FILE_OBJECT*)0;
status = IoOpenFileAtVolume(volume, invalidPaths[i], &file);
if (status != NTLSTATUS_INVALID_PARAMETER || file)
KePanic(KE_PANIC_CODE_SELFTEST,
"VOLUME/PATH selftest: invalid path %llu status=%u file=0x%016llX\n",
(uint64_t)i, status, (uint64_t)(uintptr_t)file);
}
DIR_OBJECT* directory = (DIR_OBJECT*)0;
status = IoOpenDirAtVolume(volume, SYS_SV_LITERAL("system"), &directory);
if (!KSUCCESS(status) || !directory)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: directory open failed %u\n", status);
file = (FILE_OBJECT*)0;
status = IoOpenFileAtDirectory(directory, SYS_SV_LITERAL("init.exe"), &file);
if (!KSUCCESS(status) || !file)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: directory-relative open failed %u\n", status);
IoCloseFile(file);
file = (FILE_OBJECT*)0;
status = IoOpenFileAtDirectory(directory, SYS_SV_LITERAL("../init.exe"), &file);
if (status != NTLSTATUS_INVALID_PARAMETER || file)
KePanic(KE_PANIC_CODE_SELFTEST, "VOLUME/PATH selftest: parent traversal status=%u\n", status);
IoCloseDir(directory);
IoReleaseVolume(volume);
DbgPrint("VOLUME/PATH selftest: PASS\n");
}
// Async file I/O self-test.
typedef struct KI_FILE_ASYNC_SELFTEST_STATE {
EVENT* CompletionEvent;
FILE_OBJECT* File;
IO_STATUS_BLOCK Iosb;
KAPC Apc;
uint8_t* Buffer;
volatile int ApcDelivered;
volatile KSTATUS ApcStatus;
} KI_FILE_ASYNC_SELFTEST_STATE;
static void KiFileAsyncSelfTestKernelRoutine(KAPC* apc,
void** normalRoutine,
void** normalContext,
void** systemArgument1,
void** systemArgument2) {
DbgPrint("FILE/ASYNC selftest: kernel routine apc=0x%016llX iosb=0x%016llX status=%u irql=%u\n",
(uint64_t)apc,
(uint64_t)*systemArgument1,
(uint32_t)(uintptr_t)*systemArgument2,
KeGetCurrentIrql());
(void)normalRoutine;
(void)normalContext;
}
static void KiFileAsyncSelfTestNormalRoutine(void* normalContext,
void* systemArgument1,
void* systemArgument2) {
KI_FILE_ASYNC_SELFTEST_STATE* st =
(KI_FILE_ASYNC_SELFTEST_STATE*)normalContext;
KSTATUS status = (KSTATUS)(uintptr_t)systemArgument2;
DbgPrint("FILE/ASYNC selftest: APC delivered iosb=0x%016llX status=%u iosb=%u/%llu irql=%u\n",
(uint64_t)systemArgument1,
status,
st->Iosb.Status,
(uint64_t)st->Iosb.Information,
KeGetCurrentIrql());
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: APC ran above PASSIVE_LEVEL\n");
st->ApcDelivered++;
st->ApcStatus = status;
}
// Runs async File Io Self Test.
static void KiRunAsyncFileIoSelfTest(void) {
DbgPrint("FILE/ASYNC selftest: begin\n");
KI_FILE_ASYNC_SELFTEST_STATE st;
memset(&st, 0, sizeof(st));
st.Iosb.Status = NTLSTATUS_FAILURE;
KSTATUS s = KeOpenSystemRootFile("system/init.exe", &st.File);
if (!KSUCCESS(s)) {
DbgPrint("FILE/ASYNC selftest: SKIP, system/init.exe open failed %u\n", s);
return;
}
s = KeEventCreate(EventAutoReset, 0, &st.CompletionEvent);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: event create failed %u\n", s);
st.Buffer = (uint8_t*)MmAllocatePool(512);
if (!st.Buffer)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: no buffer\n");
memset(st.Buffer, 0xCC, 512);
KeInitializeApc(&st.Apc,
KeGetCurrentThread(),
KiFileAsyncSelfTestKernelRoutine,
KiFileAsyncSelfTestNormalRoutine,
(void*)&st);
DbgPrint("FILE/ASYNC selftest: submit read system/init.exe offset=0 size=512\n");
IO_ASYNC_FILE_REQUEST_OBJECT* readReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
s = IoReadFileAsyncKernel(st.File,
st.Buffer,
512,
0,
&st.Iosb,
st.CompletionEvent,
&st.Apc,
(MM_LOCKED_USER_BUFFER*)0,
(MM_LOCKED_USER_BUFFER*)0,
&readReq);
if (s != NTLSTATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s);
s = KeWaitForEvent(st.CompletionEvent, 1);
if (s != NTLSTATUS_SUCCESS && s != NTLSTATUS_KERNEL_APC)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: wait returned %u\n", s);
if (st.Iosb.Status == NTLSTATUS_PENDING) {
s = KeWaitForEvent(st.CompletionEvent, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: final event wait returned %u\n", s);
}
if (st.ApcDelivered != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: APC not delivered count=%d\n", st.ApcDelivered);
if (!KSUCCESS(st.Iosb.Status) || !KSUCCESS(st.ApcStatus))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: I/O failed iosb=%u apc=%u info=%llu\n",
st.Iosb.Status, st.ApcStatus, (uint64_t)st.Iosb.Information);
if (st.Iosb.Information < 2 || st.Buffer[0] != 'M' || st.Buffer[1] != 'Z')
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: bad file header info=%llu bytes=%02X %02X\n",
(uint64_t)st.Iosb.Information, st.Buffer[0], st.Buffer[1]);
DbgPrint("FILE/ASYNC selftest: first bytes %02X %02X %02X %02X info=%llu wait=%u\n",
st.Buffer[0], st.Buffer[1], st.Buffer[2], st.Buffer[3],
(uint64_t)st.Iosb.Information, s);
if (readReq)
ObDereferenceObject(IoAsyncFileRequestObject(readReq));
MmFreePool(st.Buffer);
ObDereferenceObject(&st.CompletionEvent->Header);
IoCloseFile(st.File);
// Exercise the async write path with an explicit-offset temporary file.
{
static const char payload[] = "async-file-write-selftest\n";
char verify[sizeof(payload)];
FILE_OBJECT* wf = (FILE_OBJECT*)0;
FILE_OBJECT* rf = (FILE_OBJECT*)0;
EVENT* wev = (EVENT*)0;
IO_STATUS_BLOCK wiosb;
size_t got = 0;
memset(verify, 0, sizeof(verify));
wiosb.Status = NTLSTATUS_FAILURE;
wiosb.Information = 0;
(void)KeDeleteSystemRootFile("asyncio.tmp");
s = KeCreateSystemRootFile("asyncio.tmp", &wf);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: create write temp failed %u\n", s);
s = KeEventCreate(EventAutoReset, 0, &wev);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write event create failed %u\n", s);
DbgPrint("FILE/ASYNC selftest: submit write asyncio.tmp size=%llu\n",
(uint64_t)(sizeof(payload) - 1));
IO_ASYNC_FILE_REQUEST_OBJECT* writeReq = (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
s = IoWriteFileAsyncKernel(wf,
payload,
sizeof(payload) - 1,
0,
&wiosb,
wev,
(KAPC*)0,
(MM_LOCKED_USER_BUFFER*)0,
(MM_LOCKED_USER_BUFFER*)0,
&writeReq);
if (s != NTLSTATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s);
s = KeWaitForEvent(wev, 1);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write wait returned %u\n", s);
if (!KSUCCESS(wiosb.Status) || wiosb.Information != sizeof(payload) - 1)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write failed status=%u info=%llu\n",
wiosb.Status, (uint64_t)wiosb.Information);
if (writeReq)
ObDereferenceObject(IoAsyncFileRequestObject(writeReq));
IoCloseFile(wf);
wf = (FILE_OBJECT*)0;
s = KeOpenSystemRootFile("asyncio.tmp", &rf);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: reopen write temp failed %u\n", s);
s = IoReadFileAt(rf, 0, verify, sizeof(payload) - 1, &got);
if (!KSUCCESS(s) || got != sizeof(payload) - 1 ||
// Implements the memcmp operation.
memcmp(verify, payload, sizeof(payload) - 1) != 0) {
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: verify failed status=%u got=%llu\n",
s, (uint64_t)got);
}
IoCloseFile(rf);
ObDereferenceObject(&wev->Header);
s = KeDeleteSystemRootFile("asyncio.tmp");
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: delete temp failed %u\n", s);
DbgPrint("FILE/ASYNC selftest: async write verified\n");
}
DbgPrint("FILE/ASYNC selftest: PASS\n");
}
typedef struct KI_SCHEDULER_SELFTEST_STATE {
MUTEX* GateMutex;
MUTEX* LinkMutex;
EVENT* ChainReadyEvent;
EVENT* RealtimeReadyEvent;
EVENT* ChainFinishedEvent;
EVENT* RealtimeFinishedEvent;
volatile KSTATUS ChainAcquireStatus;
volatile KSTATUS RealtimeAcquireStatus;
} KI_SCHEDULER_SELFTEST_STATE;
// Waits until one worker has entered the expected mutex wait state.
static KE_THREAD_SCHEDULER_SNAPSHOT KiSchedulerAwaitMutexWait(THREAD* thread, MUTEX* mutex) {
KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
memset(&snapshot, 0, sizeof(snapshot));
for (uint32_t attempt = 0; attempt < 100; attempt++) {
KiSchedulerSnapshotThread(thread, &snapshot);
if (snapshot.State == ThreadWaiting && snapshot.WaitingMutex == mutex)
return snapshot;
KSTATUS status = KeDelayThread(SCHED_TICK_MS);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: state wait failed status=%u\n", status);
}
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: thread %u did not block on mutex=0x%016llX state=%u waiting=0x%016llX\n",
thread ? thread->ThreadId : 0,
(uint64_t)mutex,
(unsigned)snapshot.State,
(uint64_t)snapshot.WaitingMutex);
}
typedef struct KI_SYNC_TIMEOUT_SELFTEST_STATE {
MUTEX* Mutex;
EVENT* ReadyEvent;
EVENT* FinishedEvent;
volatile KSTATUS MutexStatus;
} KI_SYNC_TIMEOUT_SELFTEST_STATE;
// Waits on the held mutex long enough for the timeout engine to remove donation state.
static void KiSynchronizationTimeoutWorker(void* argument) {
KI_SYNC_TIMEOUT_SELFTEST_STATE* state = (KI_SYNC_TIMEOUT_SELFTEST_STATE*)argument;
KeSignalEvent(state->ReadyEvent);
state->MutexStatus = KeAcquireMutex(state->Mutex, SCHED_TICK_MS * 20ULL);
if (KE_MUTEX_ACQUIRE_SUCCEEDED(state->MutexStatus))
KeReleaseMutex(state->Mutex);
KeSignalEvent(state->FinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
// Verifies finite mutex and semaphore waits consume no object state after timeout.
static void KiRunSynchronizationTimeoutSelfTest(void) {
DbgPrint("SYNC/TIMEOUT selftest: begin\n");
KI_SYNC_TIMEOUT_SELFTEST_STATE state;
memset(&state, 0, sizeof(state));
state.MutexStatus = NTLSTATUS_PENDING;
KSTATUS status = KeMutexCreate(&state.Mutex);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: mutex create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.ReadyEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: ready event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.FinishedEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finished event create failed status=%u\n", status);
status = KeAcquireMutex(state.Mutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: owner acquire failed status=%u\n", status);
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_REALTIME(12, "selftest.sync.timeout");
status = KeCreateKernelThread((uintptr_t)KiSynchronizationTimeoutWorker,
(uintptr_t)&state,
&parameters,
&worker);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: worker create failed status=%u\n", status);
status = KeWaitForEvent(state.ReadyEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: ready wait failed status=%u\n", status);
(void)KiSchedulerAwaitMutexWait(worker, state.Mutex);
THREAD* self = KeGetCurrentThread();
KE_THREAD_SCHEDULER_SNAPSHOT snapshot;
KiSchedulerSnapshotThread(self, &snapshot);
if (snapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
snapshot.EffectivePriority != 12) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finite waiter donation missing class=%u priority=%u\n",
snapshot.EffectiveSchedulingClass,
snapshot.EffectivePriority);
}
status = KeWaitForEvent(state.FinishedEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finished wait failed status=%u\n", status);
if (state.MutexStatus != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: mutex wait returned status=%u\n",
state.MutexStatus);
KiSchedulerSnapshotThread(self, &snapshot);
if (snapshot.EffectiveSchedulingClass != snapshot.BaseSchedulingClass ||
snapshot.EffectivePriority != snapshot.BasePriority) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: timed-out donation remained base=%u/%u effective=%u/%u\n",
snapshot.BaseSchedulingClass,
snapshot.BasePriority,
snapshot.EffectiveSchedulingClass,
snapshot.EffectivePriority);
}
KeReleaseMutex(state.Mutex);
status = KeAcquireMutex(state.Mutex, 0);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: zero-timeout mutex acquire failed status=%u\n", status);
KeReleaseMutex(state.Mutex);
SEMAPHORE* semaphore = (SEMAPHORE*)0;
status = KeSemaphoreCreate(0, 1, &semaphore);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: semaphore create failed status=%u\n", status);
status = KeWaitSemaphore(semaphore, 0);
if (status != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: zero-timeout semaphore wait returned status=%u\n", status);
status = KeWaitSemaphore(semaphore, SCHED_TICK_MS * 3ULL);
if (status != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: finite semaphore wait returned status=%u\n", status);
KeSignalSemaphore(semaphore);
status = KeWaitSemaphore(semaphore, 0);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: signaled semaphore probe failed status=%u\n", status);
status = KeWaitSemaphore(semaphore, 0);
if (status != NTLSTATUS_TIMEOUT)
KePanic(KE_PANIC_CODE_SELFTEST,
"SYNC/TIMEOUT selftest: semaphore count was consumed twice status=%u\n", status);
ObDereferenceObject(&worker->Header);
ObDereferenceObject(&semaphore->Header);
ObDereferenceObject(&state.Mutex->Header);
ObDereferenceObject(&state.ReadyEvent->Header);
ObDereferenceObject(&state.FinishedEvent->Header);
DbgPrint("SYNC/TIMEOUT selftest: PASS\n");
}
// Owns the link mutex and then blocks behind the self-test thread.
static void KiSchedulerChainWorker(void* argument) {
KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;
KSTATUS status = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain link acquire failed status=%u\n", status);
KeSignalEvent(state->ChainReadyEvent);
state->ChainAcquireStatus = KeAcquireMutex(state->GateMutex, KE_WAIT_INFINITE);
if (state->ChainAcquireStatus != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain gate acquire failed status=%u\n",
state->ChainAcquireStatus);
KeReleaseMutex(state->GateMutex);
KeReleaseMutex(state->LinkMutex);
KeSignalEvent(state->ChainFinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
// Blocks a real-time thread behind the normal chain worker.
static void KiSchedulerRealtimeWorker(void* argument) {
KI_SCHEDULER_SELFTEST_STATE* state = (KI_SCHEDULER_SELFTEST_STATE*)argument;
KeSignalEvent(state->RealtimeReadyEvent);
state->RealtimeAcquireStatus = KeAcquireMutex(state->LinkMutex, KE_WAIT_INFINITE);
if (state->RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time acquire failed status=%u\n",
state->RealtimeAcquireStatus);
KeReleaseMutex(state->LinkMutex);
KeSignalEvent(state->RealtimeFinishedEvent);
KeExitThread(KeGetCurrentThread(), 0);
}
// Verifies real-time dispatch and transitive mutex priority inheritance.
static void KiRunRealtimeSchedulerSelfTest(void) {
DbgPrint("SCHED selftest: begin\n");
THREAD* self = KeGetCurrentThread();
if (!self || self->BaseSchedulingClass != KeSchedulingNormal)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: caller is not a normal thread\n");
KE_SCHED_STATS before;
KE_SCHED_STATS after;
KeGetSchedulerStats(&before);
KI_SCHEDULER_SELFTEST_STATE state;
memset(&state, 0, sizeof(state));
state.ChainAcquireStatus = NTLSTATUS_PENDING;
state.RealtimeAcquireStatus = NTLSTATUS_PENDING;
KSTATUS status = KeMutexCreate(&state.GateMutex);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: gate mutex create failed status=%u\n", status);
status = KeMutexCreate(&state.LinkMutex);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: link mutex create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.ChainReadyEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain ready event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.RealtimeReadyEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time ready event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.ChainFinishedEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain finished event create failed status=%u\n", status);
status = KeEventCreate(EventAutoReset, 0, &state.RealtimeFinishedEvent);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time finished event create failed status=%u\n", status);
status = KeAcquireMutex(state.GateMutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: owner acquire failed status=%u\n", status);
THREAD* chainWorker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS chainParameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "selftest.sched.chain");
status = KeCreateKernelThread((uintptr_t)KiSchedulerChainWorker,
(uintptr_t)&state,
&chainParameters,
&chainWorker);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain worker create failed status=%u\n", status);
status = KeWaitForEvent(state.ChainReadyEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain ready wait failed status=%u\n", status);
(void)KiSchedulerAwaitMutexWait(chainWorker, state.GateMutex);
status = KeAcquireMutex(state.LinkMutex, KE_WAIT_INFINITE);
if (status != NTLSTATUS_DEADLOCK)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: mutex cycle was not rejected status=%u\n", status);
THREAD* realtimeWorker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS realtimeParameters =
KE_THREAD_CREATE_NAMED_REALTIME(16, "selftest.sched.rt");
status = KeCreateKernelThread((uintptr_t)KiSchedulerRealtimeWorker,
(uintptr_t)&state,
&realtimeParameters,
&realtimeWorker);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time worker create failed status=%u\n", status);
status = KeWaitForEvent(state.RealtimeReadyEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time ready wait failed status=%u\n", status);
KE_THREAD_SCHEDULER_SNAPSHOT realtimeSnapshot =
KiSchedulerAwaitMutexWait(realtimeWorker, state.LinkMutex);
(void)realtimeSnapshot;
KE_THREAD_SCHEDULER_SNAPSHOT chainSnapshot;
KE_THREAD_SCHEDULER_SNAPSHOT selfSnapshot;
KiSchedulerSnapshotThread(chainWorker, &chainSnapshot);
KiSchedulerSnapshotThread(self, &selfSnapshot);
if (chainSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
chainSnapshot.EffectivePriority != 16)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: intermediate donation missing class=%u priority=%u\n",
chainSnapshot.EffectiveSchedulingClass,
chainSnapshot.EffectivePriority);
if (selfSnapshot.EffectiveSchedulingClass != KeSchedulingRealtime ||
selfSnapshot.EffectivePriority != 16)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: transitive donation missing class=%u priority=%u\n",
selfSnapshot.EffectiveSchedulingClass,
selfSnapshot.EffectivePriority);
KeReleaseMutex(state.GateMutex);
KiSchedulerSnapshotThread(self, &selfSnapshot);
if (selfSnapshot.EffectiveSchedulingClass != selfSnapshot.BaseSchedulingClass ||
selfSnapshot.EffectivePriority != selfSnapshot.BasePriority)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: inherited priority was not restored base=%u/%u effective=%u/%u\n",
selfSnapshot.BaseSchedulingClass,
selfSnapshot.BasePriority,
selfSnapshot.EffectiveSchedulingClass,
selfSnapshot.EffectivePriority);
status = KeWaitForEvent(state.ChainFinishedEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: chain finished wait failed status=%u\n", status);
status = KeWaitForEvent(state.RealtimeFinishedEvent, 0);
if (!KSUCCESS(status))
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: real-time finished wait failed status=%u\n", status);
if (state.ChainAcquireStatus != NTLSTATUS_SUCCESS ||
state.RealtimeAcquireStatus != NTLSTATUS_SUCCESS)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: handoff statuses chain=%u realtime=%u\n",
state.ChainAcquireStatus,
state.RealtimeAcquireStatus);
KeGetSchedulerStats(&after);
if (after.RealtimeDispatches <= before.RealtimeDispatches ||
after.PriorityInheritanceBoosts < before.PriorityInheritanceBoosts + 2 ||
after.PriorityInheritanceDrops < before.PriorityInheritanceDrops + 2 ||
after.PriorityInheritancePropagations < before.PriorityInheritancePropagations + 2 ||
after.PriorityInheritanceMaxDepth < 2)
KePanic(KE_PANIC_CODE_SELFTEST,
"SCHED selftest: scheduler counters did not prove transitive inheritance\n");
ObDereferenceObject(&chainWorker->Header);
ObDereferenceObject(&realtimeWorker->Header);
ObDereferenceObject(&state.GateMutex->Header);
ObDereferenceObject(&state.LinkMutex->Header);
ObDereferenceObject(&state.ChainReadyEvent->Header);
ObDereferenceObject(&state.RealtimeReadyEvent->Header);
ObDereferenceObject(&state.ChainFinishedEvent->Header);
ObDereferenceObject(&state.RealtimeFinishedEvent->Header);
DbgPrint("SCHED selftest: PASS\n");
}
// Runs trap Frame Self Test.
static void KiRunTrapFrameSelfTest(void) {
DbgPrint("TRAP selftest: begin\n");
if (KeGetCurrentTrapFrame())
KePanic(KE_PANIC_CODE_SELFTEST, "TRAP selftest: kernel selftest unexpectedly has active trap frame\n");
DbgPrint("TRAP selftest: frame size=%llu query size=%llu\n",
(uint64_t)sizeof(KTRAP_FRAME),
(uint64_t)sizeof(SYS_TRAP_FRAME_INFO_K));
if (sizeof(KTRAP_FRAME) != 240)
KePanic(KE_PANIC_CODE_SELFTEST, "TRAP selftest: KTRAP_FRAME size mismatch %llu\n",
(uint64_t)sizeof(KTRAP_FRAME));
DbgPrint("TRAP selftest: PASS\n");
}
// Runs core Hardening Self Tests.
void KiRunCoreHardeningSelfTests(void) {
DbgPrint("CORE selftest: begin\n");
if (gCpuCount == 0 || gCpuCount > KE_MAX_CPUS)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid CPU count %u\n", gCpuCount);
KCPU* cpu = KeGetCurrentCpu();
if (cpu != &gCpus[0])
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%016llX cpu0=0x%016llX\n",
(uint64_t)cpu, (uint64_t)&gCpus[0]);
if (cpu->Self != cpu)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%016llX cpu=0x%016llX\n",
(uint64_t)cpu->Self, (uint64_t)cpu);
if (KeGetCurrentCpuIndex() != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not boot processor/0\n");
if (cpu->State != KeCpuOnline || !cpu->IsBootProcessor)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: boot-processor lifecycle invalid state=%ld boot=%u\n",
cpu->State, cpu->IsBootProcessor);
if (KeGetCurrentIrql() != PASSIVE_LEVEL)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql());
if (KeGetCurrentTss() != &cpu->Tss)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n");
if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64))
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid boot-processor TSS rsp0=0x%016llX iomap=%u\n",
cpu->Tss.RSP0, cpu->Tss.IoMapBase);
if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%016llX gdt=0x%016llX\n",
cpu->Gdtr.Base, (uint64_t)cpu->Gdt);
for (uint32_t i = 1; i < gCpuCount; i++) {
KCPU* ap = &gCpus[i];
if (ap->State == KeCpuAbsent)
continue;
if (ap->State != KeCpuOnline || ap->IsBootProcessor)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u lifecycle invalid state=%ld boot=%u\n",
i, ap->State, ap->IsBootProcessor);
if (!ap->KernelStack || !ap->KernelStackTop)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i);
if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64))
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%016llX iomap=%u\n",
i, ap->Tss.RSP0, ap->Tss.IoMapBase);
if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i);
if (!ap->IdleThread || !ap->CurrentThread || ap->CurrentThread != ap->IdleThread)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u idle/current thread missing\n", i);
if (!ap->IdleThread->IsIdleThread || ap->IdleThread->Queued || ap->IdleThread->RunningCpu != i)
KePanic(KE_PANIC_CODE_SELFTEST,
"CORE selftest: AP%u idle invariant failed queued=%u idle=%u runningCpu=%u\n",
i, ap->IdleThread->Queued, ap->IdleThread->IsIdleThread, ap->IdleThread->RunningCpu);
}
THREAD* cur = KeGetCurrentThread();
if (!cur || cur != cpu->CurrentThread)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current thread mismatch\n");
if (!cpu->IdleThread || cur != cpu->IdleThread)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: scheduler did not start on CPU0 idle thread\n");
if (!cur->IsIdleThread || cur->Queued || cur->RunningCpu != cpu->CpuIndex)
KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: idle invariant failed queued=%u idle=%u runningCpu=%u\n",
cur->Queued, cur->IsIdleThread, cur->RunningCpu);
DbgPrint("CORE selftest: PASS\n");
}
// Runs early Kernel Self Tests.
void KiRunEarlyKernelSelfTests(void) {
KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests();
if (!KSUCCESS(s)) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest failed status=0x%x\n",
s);
}
}
// Runs kernel Self Tests.
void KiRunKernelSelfTests(const HAL_API* hal) {
(void)hal;
DbgPrint("Kernel selftests: begin\n");
KiRunSynchronizationTimeoutSelfTest();
KiRunRealtimeSchedulerSelfTest();
KiRunApcWaitRaceSelfTest();
KiRunPfnDatabaseSelfTest();
KiRunDmaSelfTest();
KiRunMemoryMappingSelfTest();
KiRunVadSparseSelfTest();
KiRunVadCopyOnWriteSelfTest();
KiRunVadSwapSelfTest();
KiRunStringViewSelfTest();
KiRunVolumeRelativePathSelfTest();
KiRunAsyncFileIoSelfTest();
KiRunTrapFrameSelfTest();
DbgPrint("Kernel selftests: PASS\n");
}
Reference in new issue View Git Blame Copy Permalink