maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

parallel compile, SMP, swapgs, cleanup, remove useless Reserved fields

Browse Source
This commit is contained in:
maksw committed 2026-05-30 11:18:46 +02:00
1 parent b17c9ac854
commit bfe2b55142
42 files changed
+2131 -426
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/mm_virt.c
+197 -30
View File
@@ -14,9 +14,13 @@
// Backed on demand from the PMM. No per-process heaps.
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <kernel/hal/hal_apic.h>
// ---------------------------------------------------------------------------
// Helpers - physical window access
@@ -34,6 +38,14 @@ static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
// PML4 / page-table helpers
// ---------------------------------------------------------------------------
static PHYSICAL_ADDRESS gPml4Phys = 0;
static SPINLOCK gPageTableLock = SPINLOCK_INIT;
static volatile long gTlbShootdownLock = 0;
static volatile long gTlbShootdownActive = 0;
static volatile long gTlbShootdownAcks = 0;
static uintptr_t gTlbShootdownVirt = 0;
#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u
// Extract the page-table index for each level from a virtual address.
#define PML4_IDX(va) (((va) >> 39) & 0x1FF)
@@ -64,38 +76,150 @@ static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
return childPhys;
}
// ---------------------------------------------------------------------------
// TLB/cache coherency hooks
// ---------------------------------------------------------------------------
void MmFlushTlbLocal(uintptr_t virt) {
__invlpg((void*)virt);
}
static void KiAcquireTlbShootdownLock(void) {
// Do not use the normal spinlock here: it masks interrupts while spinning.
// A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
// concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is
// meant to avoid.
while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0)
_mm_pause();
_ReadWriteBarrier();
}
static void KiReleaseTlbShootdownLock(void) {
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownLock, 0);
}
void MmFlushTlbForKernelMapping(uintptr_t virt) {
MmFlushTlbLocal(virt);
if (gCpuCount <= 1)
return;
KCPU* self = KeGetCurrentCpu();
uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU;
KiAcquireTlbShootdownLock();
long targets = 0;
gTlbShootdownVirt = virt;
_InterlockedExchange(&gTlbShootdownAcks, 0);
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownActive, 1);
_ReadWriteBarrier();
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (i == selfIndex)
continue;
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
targets++;
HalApicSendTlbShootdownIpi(cpu->LapicId);
}
for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets)
break;
_mm_pause();
}
long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0);
_InterlockedExchange(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
if (acks < targets) {
KePanic(KE_PANIC_CODE_GENERIC,
"TLB shootdown timeout: va=0x%llX targets=%ld acks=%ld selfCpu=%u\n",
(uint64_t)virt, targets, acks, selfIndex);
}
}
static int KiHandleTlbShootdown(void) {
if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
return 0;
uintptr_t va = gTlbShootdownVirt;
_ReadWriteBarrier();
MmFlushTlbLocal(va);
_ReadWriteBarrier();
_InterlockedIncrement(&gTlbShootdownAcks);
return 1;
}
int MmHandleTlbShootdownNmi(void) {
return KiHandleTlbShootdown();
}
// ---------------------------------------------------------------------------
// Public mapping
// ---------------------------------------------------------------------------
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags) {
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
if (!pdpt) return STATUS_NO_MEMORY;
if (!pdpt) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
if (!pdt) return STATUS_NO_MEMORY;
if (!pdt) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag);
if (!pt) return STATUS_NO_MEMORY;
if (!pt) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
*pte = phys | flags;
__invlpg((void *)virt);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
MmFlushTlbForKernelMapping(virt);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) {
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e4 & PAGE_PRESENT)) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e3 & PAGE_PRESENT)) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e2 & PAGE_PRESENT)) {
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt));
*pte = 0;
__invlpg((void *)virt);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
MmFlushTlbForKernelMapping(virt);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags) {
if (!size) return STATUS_SUCCESS;
@@ -103,60 +227,92 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint
uintptr_t base = virt & PAGE_MASK;
uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
KSTATUS status = STATUS_SUCCESS;
uintptr_t flushEnd = base;
for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e4 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e3 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*e2 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(*pte & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
// Preserve the physical address, replace everything else.
PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK;
*pte = phys | flags;
__invlpg((void *)va);
flushEnd = va + PAGE_SIZE;
}
return STATUS_SUCCESS;
KeReleaseSpinlock(&gPageTableLock, lockFlags);
for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE)
MmFlushTlbForKernelMapping(va);
return status;
}
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) {
if (!gPml4Phys) return 0; // Not initialized
PHYSICAL_ADDRESS result = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t pml4_idx = (virt >> 39) & 0x1FF;
uint64_t pdpt_idx = (virt >> 30) & 0x1FF;
uint64_t pd_idx = (virt >> 21) & 0x1FF;
uint64_t pt_idx = (virt >> 12) & 0x1FF;
uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
if (!(pml4[pml4_idx] & PAGE_PRESENT)) return 0;
if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out;
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK);
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) return 0;
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out;
// Check for 1GB huge pages (bit 7 - PS)
if (pdpt[pdpt_idx] & 0x80) {
return (pdpt[pdpt_idx] & 0x000FFFFFC0000000ULL) + (virt & 0x3FFFFFFF);
if (pdpt[pdpt_idx] & PAGE_PS) {
result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF);
goto out;
}
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK);
if (!(pd[pd_idx] & PAGE_PRESENT)) return 0;
if (!(pd[pd_idx] & PAGE_PRESENT)) goto out;
// Check for 2MB huge pages (bit 7 - PS)
if (pd[pd_idx] & 0x80) {
return (pd[pd_idx] & 0x000FFFFFFFE00000ULL) + (virt & 0x1FFFFF);
if (pd[pd_idx] & PAGE_PS) {
result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF);
goto out;
}
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK);
if (!(pt[pt_idx] & PAGE_PRESENT)) return 0;
if (!(pt[pt_idx] & PAGE_PRESENT)) goto out;
// Extract the physical frame and add the offset within the page
return (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return result;
}
// ---------------------------------------------------------------------------
// Page query and user/kernel copy boundary helpers
// ---------------------------------------------------------------------------
@@ -165,9 +321,12 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
uint64_t* flagsOut) {
if (!gPml4Phys) return STATUS_NOT_FOUND;
KSTATUS status = STATUS_NOT_FOUND;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys);
uint64_t e4 = pml4[PML4_IDX(virt)];
if (!(e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e4 & PAGE_PRESENT)) goto out;
uint64_t effectiveWrite = e4 & PAGE_WRITE;
uint64_t effectiveUser = e4 & PAGE_USER;
@@ -175,7 +334,7 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK);
uint64_t e3 = pdpt[PDPT_IDX(virt)];
if (!(e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e3 & PAGE_PRESENT)) goto out;
effectiveWrite &= e3 & PAGE_WRITE;
effectiveUser &= e3 & PAGE_USER;
effectiveNx |= e3 & PAGE_NX;
@@ -189,12 +348,13 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
return STATUS_SUCCESS;
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK);
uint64_t e2 = pd[PDT_IDX(virt)];
if (!(e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e2 & PAGE_PRESENT)) goto out;
effectiveWrite &= e2 & PAGE_WRITE;
effectiveUser &= e2 & PAGE_USER;
effectiveNx |= e2 & PAGE_NX;
@@ -208,12 +368,13 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
return STATUS_SUCCESS;
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK);
uint64_t e1 = pt[PT_IDX(virt)];
if (!(e1 & PAGE_PRESENT)) return STATUS_NOT_FOUND;
if (!(e1 & PAGE_PRESENT)) goto out;
effectiveWrite &= e1 & PAGE_WRITE;
effectiveUser &= e1 & PAGE_USER;
effectiveNx |= e1 & PAGE_NX;
@@ -226,9 +387,14 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt,
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
return STATUS_SUCCESS;
status = STATUS_SUCCESS;
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return status;
}
static int KiUserRangeValid(uintptr_t addr, size_t size) {
if (size == 0) return 1;
if (addr < USER_BASE) return 0;
@@ -581,6 +747,7 @@ static uintptr_t gUserImageNext = USER_LIMIT;
// ---------------------------------------------------------------------------
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) {
gPml4Phys = bootPml4Phys;
gPageTableLock = (SPINLOCK)SPINLOCK_INIT;
// Kernel pool heap.
gPoolHeap.FreeList = (BLOCK_HDR *)0;