add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.
This commit is contained in:
maksw committed
1 parent
61ec2442eb
commit
2c4a68a75f
114 files changed
+18069
-2620
No files matched your search
+443
-3
@@ -14,10 +14,13 @@
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_process.h>
|
||||
#include <kernel/io_file.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <kernel/ke_trap.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/mm_swap.h>
|
||||
#include <kernel/mm_pfn.h>
|
||||
#include <kernel/ob_object.h>
|
||||
#include <kernel/ke_smp.h>
|
||||
|
||||
@@ -478,11 +481,118 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
|
||||
|
||||
|
||||
|
||||
// Virtual memory mapping invariants.
|
||||
|
||||
// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
|
||||
static void KiRunPfnDatabaseSelfTest(void) {
|
||||
MM_PFN_STATS before;
|
||||
MM_PFN_STATS after;
|
||||
MmPfnQueryStats(&before);
|
||||
|
||||
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
|
||||
if (!frame)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n");
|
||||
|
||||
MM_PFN* pfn = MmPfnGet(frame);
|
||||
if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"PFN selftest: allocated frame metadata invalid phys=%016llx\n",
|
||||
(unsigned long long)frame);
|
||||
|
||||
if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount);
|
||||
if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount);
|
||||
|
||||
if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount);
|
||||
if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n");
|
||||
if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount);
|
||||
if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount);
|
||||
if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n");
|
||||
if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount);
|
||||
if (!KSUCCESS(MmPfnFree(frame)))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n");
|
||||
|
||||
PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma);
|
||||
if (!run)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n");
|
||||
MM_PFN* p0 = MmPfnGet(run);
|
||||
MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE);
|
||||
if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n");
|
||||
MmPfnFree(run);
|
||||
MmPfnFree(run + PAGE_SIZE);
|
||||
|
||||
// Accounting buckets are part of the VM contract. A missing increment here
|
||||
// usually means a new allocation path is using an anonymous usage class.
|
||||
PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache);
|
||||
PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage);
|
||||
PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile);
|
||||
PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous);
|
||||
PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow);
|
||||
PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface);
|
||||
if (!clean || !image || !mapped || !anon || !cow || !surface)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n");
|
||||
|
||||
MM_PFN_STATS bucketStats;
|
||||
MmPfnQueryStats(&bucketStats);
|
||||
if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 ||
|
||||
bucketStats.ImageFrames < before.ImageFrames + 1 ||
|
||||
bucketStats.MappedFileFrames < before.MappedFileFrames + 1 ||
|
||||
bucketStats.AnonymousFrames < before.AnonymousFrames + 1 ||
|
||||
bucketStats.CowFrames < before.CowFrames + 1 ||
|
||||
bucketStats.SurfaceFrames < before.SurfaceFrames + 1) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n");
|
||||
}
|
||||
|
||||
// Standby is the clean-page holding state used by later file-cache and image
|
||||
// reclaim. It must not change the free count until the owning cache drops it.
|
||||
if (!KSUCCESS(MmPfnMoveToStandby(clean)))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n");
|
||||
if (!KSUCCESS(MmPfnMoveToStandby(image)))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n");
|
||||
|
||||
MM_PFN_STATS standbyStats;
|
||||
MmPfnQueryStats(&standbyStats);
|
||||
if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 ||
|
||||
standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 ||
|
||||
standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 ||
|
||||
standbyStats.FreeFrames != bucketStats.FreeFrames) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n");
|
||||
}
|
||||
|
||||
if (!KSUCCESS(MmPfnActivate(clean)))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n");
|
||||
if (!KSUCCESS(MmPfnActivate(image)))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n");
|
||||
|
||||
(void)MmPfnFree(clean);
|
||||
(void)MmPfnFree(image);
|
||||
(void)MmPfnFree(mapped);
|
||||
(void)MmPfnFree(anon);
|
||||
(void)MmPfnFree(cow);
|
||||
(void)MmPfnFree(surface);
|
||||
|
||||
MmPfnQueryStats(&after);
|
||||
if (before.FreeFrames != after.FreeFrames)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"PFN selftest: free count leak before=%llu after=%llu\n",
|
||||
(unsigned long long)before.FreeFrames,
|
||||
(unsigned long long)after.FreeFrames);
|
||||
|
||||
DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
|
||||
}
|
||||
|
||||
// Verifies the page-table manager invariants that the higher VMM will depend on.
|
||||
static void KiRunMemoryMappingSelfTest(void) {
|
||||
PHYSICAL_ADDRESS phys = 0;
|
||||
uint64_t flags = 0;
|
||||
KSTATUS s = MmQueryPage(0, &phys, &flags);
|
||||
KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags);
|
||||
if (s != STATUS_NOT_FOUND) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n",
|
||||
(unsigned long long)phys,
|
||||
@@ -490,7 +600,329 @@ static void KiRunMemoryMappingSelfTest(void) {
|
||||
(unsigned)s);
|
||||
}
|
||||
|
||||
DbgPrint("MM selftest: page zero unmapped\n");
|
||||
uintptr_t testVa = 0xFFFFA00000000000ULL;
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
|
||||
if (s != STATUS_NOT_FOUND) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n",
|
||||
(unsigned long long)testVa,
|
||||
(unsigned long long)phys,
|
||||
(unsigned long long)flags);
|
||||
}
|
||||
|
||||
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
|
||||
if (!frame)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n");
|
||||
|
||||
s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent);
|
||||
if (!KSUCCESS(s)) {
|
||||
MmPfnFree(frame);
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s);
|
||||
}
|
||||
|
||||
volatile uint64_t* probe = (volatile uint64_t*)testVa;
|
||||
*probe = 0x1122334455667788ULL;
|
||||
if (*probe != 0x1122334455667788ULL)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n");
|
||||
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
|
||||
if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n",
|
||||
s,
|
||||
(unsigned long long)phys,
|
||||
(unsigned long long)frame,
|
||||
(unsigned long long)flags);
|
||||
|
||||
s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s);
|
||||
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
|
||||
if (!KSUCCESS(s) || (flags & PAGE_WRITE))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n",
|
||||
s,
|
||||
(unsigned long long)flags);
|
||||
|
||||
s = MmPteUnmapPage(testVa);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s);
|
||||
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
|
||||
if (s != STATUS_NOT_FOUND)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s);
|
||||
|
||||
MmPfnFree(frame);
|
||||
DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
|
||||
}
|
||||
|
||||
|
||||
// Verifies sparse VAD commitment and NOACCESS protection metadata.
|
||||
static void KiRunVadSparseSelfTest(void) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
if (!process)
|
||||
process = KeGetSystemProcess();
|
||||
if (!process)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n");
|
||||
|
||||
size_t beforeVadCount = 0;
|
||||
size_t beforeReserved = 0;
|
||||
size_t beforeCommitted = 0;
|
||||
MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted);
|
||||
|
||||
void* base = (void*)0;
|
||||
KSTATUS s = MmVadReserve(process,
|
||||
PAGE_SIZE * 4,
|
||||
PAGE_SIZE,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
||||
MM_VAD_ZERO,
|
||||
MmVadGeneric,
|
||||
&base);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s);
|
||||
|
||||
uint8_t* p = (uint8_t*)base;
|
||||
s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s);
|
||||
|
||||
s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s);
|
||||
|
||||
void* queryBase = (void*)0;
|
||||
size_t querySize = 0;
|
||||
uint64_t queryFlags = 0;
|
||||
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
|
||||
if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=%p size=%llu flags=%016llx\n",
|
||||
s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags);
|
||||
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
|
||||
if (s != STATUS_NOT_FOUND)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s);
|
||||
|
||||
s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s);
|
||||
s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s);
|
||||
|
||||
s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
|
||||
if (s != STATUS_NOT_FOUND)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s);
|
||||
|
||||
p[0] = 0x5A;
|
||||
p[PAGE_SIZE * 2] = 0xA5;
|
||||
|
||||
s = MmVadProtect(process, p, PAGE_SIZE, 0);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s);
|
||||
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
|
||||
if (s != STATUS_NOT_FOUND)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s);
|
||||
|
||||
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
|
||||
if (!KSUCCESS(s) || queryFlags != 0)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n",
|
||||
s, (unsigned long long)queryFlags);
|
||||
|
||||
s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s);
|
||||
if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n");
|
||||
|
||||
size_t afterVadCount = 0;
|
||||
size_t afterReserved = 0;
|
||||
size_t afterCommitted = 0;
|
||||
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
|
||||
if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n",
|
||||
(unsigned long long)beforeCommitted,
|
||||
(unsigned long long)afterCommitted,
|
||||
(unsigned long long)beforeVadCount,
|
||||
(unsigned long long)afterVadCount);
|
||||
|
||||
s = MmVadDestroy(process, base);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s);
|
||||
|
||||
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
|
||||
if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n");
|
||||
|
||||
DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
|
||||
}
|
||||
|
||||
|
||||
// Verifies that writable copy-on-write views split before either alias can modify shared state.
|
||||
static void KiRunVadCopyOnWriteSelfTest(void) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
if (!process)
|
||||
process = KeGetSystemProcess();
|
||||
if (!process)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n");
|
||||
|
||||
void* base = (void*)0;
|
||||
KSTATUS s = MmVadCreateCommitted(process,
|
||||
PAGE_SIZE,
|
||||
PAGE_SIZE,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
||||
MM_VAD_ZERO,
|
||||
MmVadGeneric,
|
||||
&base);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s);
|
||||
|
||||
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s);
|
||||
|
||||
uint8_t* source = (uint8_t*)base;
|
||||
source[0] = 0x31;
|
||||
|
||||
void* cowBase = (void*)0;
|
||||
s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s);
|
||||
|
||||
PHYSICAL_ADDRESS sourcePhys = 0;
|
||||
PHYSICAL_ADDRESS cowPhys = 0;
|
||||
uint64_t sourceFlags = 0;
|
||||
uint64_t cowFlags = 0;
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags);
|
||||
if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n",
|
||||
s, (unsigned long long)sourceFlags);
|
||||
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags);
|
||||
if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n",
|
||||
s,
|
||||
(unsigned long long)sourcePhys,
|
||||
(unsigned long long)cowPhys,
|
||||
(unsigned long long)cowFlags);
|
||||
|
||||
s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s);
|
||||
|
||||
uint8_t* copy = (uint8_t*)cowBase;
|
||||
copy[0] = 0x62;
|
||||
if (source[0] != 0x31 || copy[0] != 0x62)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n",
|
||||
source[0], copy[0]);
|
||||
|
||||
PHYSICAL_ADDRESS splitPhys = 0;
|
||||
uint64_t splitFlags = 0;
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags);
|
||||
if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
|
||||
s,
|
||||
(unsigned long long)sourcePhys,
|
||||
(unsigned long long)splitPhys,
|
||||
(unsigned long long)splitFlags);
|
||||
|
||||
(void)MmVadDestroy(process, cowBase);
|
||||
(void)MmVadDestroy(process, base);
|
||||
|
||||
void* zeroBase = (void*)0;
|
||||
s = MmVadCreateCommitted(process,
|
||||
PAGE_SIZE,
|
||||
PAGE_SIZE,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
||||
MM_VAD_ZERO,
|
||||
MmVadGeneric,
|
||||
&zeroBase);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s);
|
||||
|
||||
s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s);
|
||||
|
||||
PHYSICAL_ADDRESS zeroPhys = 0;
|
||||
uint64_t zeroFlags = 0;
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags);
|
||||
if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n",
|
||||
s, (unsigned long long)zeroFlags);
|
||||
|
||||
s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s);
|
||||
|
||||
PHYSICAL_ADDRESS privatePhys = 0;
|
||||
uint64_t privateFlags = 0;
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags);
|
||||
if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
|
||||
s,
|
||||
(unsigned long long)zeroPhys,
|
||||
(unsigned long long)privatePhys,
|
||||
(unsigned long long)privateFlags);
|
||||
|
||||
((uint8_t*)zeroBase)[0] = 0x9C;
|
||||
if (((uint8_t*)zeroBase)[0] != 0x9C)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n");
|
||||
|
||||
(void)MmVadDestroy(process, zeroBase);
|
||||
DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
|
||||
}
|
||||
|
||||
|
||||
// Verifies that a private resident VAD page can move to swap and fault back in.
|
||||
static void KiRunVadSwapSelfTest(void) {
|
||||
if (!MmSwapIsEnabled()) {
|
||||
DbgPrint("VAD swap selftest: SKIP, swap is disabled\n");
|
||||
return;
|
||||
}
|
||||
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
if (!process)
|
||||
process = KeGetSystemProcess();
|
||||
if (!process)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n");
|
||||
|
||||
void* base = (void*)0;
|
||||
KSTATUS s = MmVadCreateCommitted(process,
|
||||
PAGE_SIZE,
|
||||
PAGE_SIZE,
|
||||
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
|
||||
MM_VAD_ZERO,
|
||||
MmVadGeneric,
|
||||
&base);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s);
|
||||
|
||||
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s);
|
||||
|
||||
uint8_t* p = (uint8_t*)base;
|
||||
p[0] = 0xC1;
|
||||
p[PAGE_SIZE - 1] = 0x5E;
|
||||
|
||||
void* trimmed = (void*)0;
|
||||
s = MmVadTrimOnePageForTest(process, &trimmed);
|
||||
if (!KSUCCESS(s) || trimmed != base)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=%p base=%p\n", s, trimmed, base);
|
||||
|
||||
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
|
||||
if (s != STATUS_NOT_FOUND)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s);
|
||||
|
||||
s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0);
|
||||
if (!KSUCCESS(s))
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s);
|
||||
|
||||
if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]);
|
||||
|
||||
(void)MmVadDestroy(process, base);
|
||||
DbgPrint("VAD swap selftest: private pageout/pagein PASS\n");
|
||||
}
|
||||
|
||||
// String-view ABI self-test.
|
||||
@@ -614,6 +1046,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
|
||||
&st.Iosb,
|
||||
st.CompletionEvent,
|
||||
&st.Apc,
|
||||
(MM_LOCKED_USER_BUFFER*)0,
|
||||
(MM_LOCKED_USER_BUFFER*)0,
|
||||
&readReq);
|
||||
if (s != STATUS_PENDING)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s);
|
||||
@@ -683,6 +1117,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
|
||||
&wiosb,
|
||||
wev,
|
||||
(KAPC*)0,
|
||||
(MM_LOCKED_USER_BUFFER*)0,
|
||||
(MM_LOCKED_USER_BUFFER*)0,
|
||||
&writeReq);
|
||||
if (s != STATUS_PENDING)
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s);
|
||||
@@ -832,7 +1268,11 @@ void KiRunEarlyKernelSelfTests(void) {
|
||||
void KiRunKernelSelfTests(const HAL_API* hal) {
|
||||
DbgPrint("Kernel selftests: begin\n");
|
||||
KiRunKernelApcSelfTest();
|
||||
KiRunPfnDatabaseSelfTest();
|
||||
KiRunMemoryMappingSelfTest();
|
||||
KiRunVadSparseSelfTest();
|
||||
KiRunVadCopyOnWriteSelfTest();
|
||||
KiRunVadSwapSelfTest();
|
||||
KiRunStringViewSelfTest();
|
||||
KiRunIrpKernelApcSelfTest(hal);
|
||||
KiRunAsyncFileIoSelfTest();
|
||||
|
||||
Reference in new issue
Block a user