add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

code/kernel/ldr_image.c

@@ -3,11 +3,13 @@
 #include <stdint.h>
 #include <string.h>
 #include <kernel/ldr_image.h>
+#include <kernel/ldr_section.h>
 #include <ntlite/boot_info.h>
 #include <kernel/ke_process.h>
 #include <kernel/io_file.h>
 #include <kernel/mm_virt.h>
 #include <kernel/ke_debug.h>
+#include <kernel/ke_system_root.h>
 
 #ifndef LDR_DEBUG_VERBOSE
 #define LDR_DEBUG_VERBOSE 0
@@ -155,6 +157,8 @@ typedef struct {
 
 #define IMAGE_REL_BASED_DIR64 10
 #define IMAGE_SCN_CNT_CODE               0x00000020u
+#define IMAGE_SCN_CNT_INITIALIZED_DATA   0x00000040u
+#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u
 #define IMAGE_SCN_MEM_EXECUTE            0x20000000u
 #define IMAGE_SCN_MEM_READ               0x40000000u
 #define IMAGE_SCN_MEM_WRITE              0x80000000u
@@ -171,6 +175,11 @@ typedef struct {
 static MODULE_ENTRY gModules[MAX_MODULES];
 static uint32_t     gModuleCount = 0;
 
+// Ensures a user image page is privately writable before the kernel patches it.
+static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t address, size_t size) {
+    if (!user) return STATUS_SUCCESS;
+    return MmVadMakeImageRangePrivate(process, (void*)address, size);
+}
 
 // Implements the canonical Module Name operation.
 static const char* KiCanonicalModuleName(const char* name) {
@@ -310,10 +319,10 @@ size_t LdrGetImageSize(uintptr_t base) {
 size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
     if (!base) return 0;
     if (process) {
-        size_t regionSize = MmGetUserRegionSize(process, (const void*)base);
+        size_t regionSize = MmVadGetSize(process, (const void*)base);
         if (regionSize) return regionSize;
     }
-    size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base);
+    size_t globalRegionSize = MmVadGetSizeByBase((const void*)base);
     if (globalRegionSize) return globalRegionSize;
     return LdrGetImageSize(base);
 }
@@ -327,13 +336,26 @@ static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, b
     else MmFreePool((void*)imageBase);
 }
 
-// Releases loaded Image.
-static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
-    KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user);
+// Returns the final page flags for a PE section.
+static uint64_t KiSectionPageProtection(const SECTION_HEADER* section, bool user) {
+    uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
+
+    // Some old toolchains mark BSS-like sections as uninitialized data without
+    // setting MEM_WRITE. Treating those pages as read-only breaks programs that
+    // legitimately write their zero-fill storage.
+    if ((section->Characteristics & IMAGE_SCN_MEM_WRITE) ||
+        (section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)) {
+        prot |= PAGE_WRITE;
+    }
+
+    if (!((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
+          (section->Characteristics & IMAGE_SCN_CNT_CODE))) {
+        prot |= PAGE_NX;
+    }
+    return prot;
 }
 
-
-// Implements the ldr Protect Image For Process operation.
+// Applies final page protections to a mapped PE image.
 KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
     if (!imageBase) return STATUS_INVALID_PARAMETER;
     if (user && !process) return STATUS_INVALID_PARAMETER;
@@ -353,6 +375,14 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
     if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset)
         return STATUS_FAILURE;
 
+    size_t pageCount = (mappedSize + PAGE_SIZE - 1) / PAGE_SIZE;
+    uint64_t* pageProt = (uint64_t*)MmAllocatePoolTag(pageCount * sizeof(uint64_t), MM_POOL_TAG('i', 'p', 'r', 't'));
+    if (!pageProt) return STATUS_NO_MEMORY;
+
+    uint64_t headerProt = PAGE_PRESENT | (user ? PAGE_USER : 0) | PAGE_NX;
+    for (size_t i = 0; i < pageCount; ++i)
+        pageProt[i] = headerProt;
+
     SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset);
     for (int i = 0; i < pe->NumberOfSections; i++) {
         size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData;
@@ -368,35 +398,56 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
                    (unsigned long long)imageBase, (unsigned long long)mappedSize);
             continue;
         }
-        if (secEnd < secBase) return STATUS_NO_MEMORY;
+        if (secEnd < secBase) {
+            MmFreePool(pageProt);
+            return STATUS_NO_MEMORY;
+        }
         if (secEnd > mapEnd) {
             LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n",
                    i, (unsigned long long)secEnd, (unsigned long long)mapEnd);
-            protectSize = (size_t)(mapEnd - secBase);
+            secEnd = mapEnd;
         }
 
-        uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
-        if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE)
-            prot |= PAGE_WRITE;
-        if (!((sec[i].Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
-              (sec[i].Characteristics & IMAGE_SCN_CNT_CODE)))
-            prot |= PAGE_NX;
-
-        KSTATUS s;
-        if (user) {
-            s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot);
-        } else {
-            s = MmSetPageProtect(secBase, protectSize, prot);
-        }
-        if (!KSUCCESS(s)) {
-            LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
-                   i, (unsigned long long)secBase, (unsigned long long)protectSize,
-                   (unsigned long long)prot, s,
-                   (unsigned long long)imageBase, (unsigned long long)mappedSize);
-            return s;
+        size_t firstPage = (size_t)((secBase - imageBase) / PAGE_SIZE);
+        size_t lastPage = (size_t)((secEnd - 1 - imageBase) / PAGE_SIZE);
+        uint64_t prot = KiSectionPageProtection(&sec[i], user);
+        for (size_t page = firstPage; page <= lastPage && page < pageCount; ++page) {
+            // Protection is page-granular. If sections ever share a page, the
+            // hardware page must allow the union of their access requirements.
+            if (prot & PAGE_WRITE)
+                pageProt[page] |= PAGE_WRITE;
+            if ((prot & PAGE_NX) == 0)
+                pageProt[page] &= ~PAGE_NX;
         }
     }
-    return STATUS_SUCCESS;
+
+    size_t runStart = 0;
+    KSTATUS s = STATUS_SUCCESS;
+    while (runStart < pageCount) {
+        uint64_t prot = pageProt[runStart];
+        size_t runEnd = runStart + 1;
+        while (runEnd < pageCount && pageProt[runEnd] == prot)
+            runEnd++;
+
+        uintptr_t va = imageBase + runStart * PAGE_SIZE;
+        size_t bytes = (runEnd - runStart) * PAGE_SIZE;
+        if (user)
+            s = MmVadProtect(process, (void*)va, bytes, prot);
+        else
+            s = MmPteProtectRange(va, bytes, prot);
+
+        if (!KSUCCESS(s)) {
+            LdrDbg("LDR: protect failed va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
+                   (unsigned long long)va, (unsigned long long)bytes,
+                   (unsigned long long)prot, s,
+                   (unsigned long long)imageBase, (unsigned long long)mappedSize);
+            break;
+        }
+        runStart = runEnd;
+    }
+
+    MmFreePool(pageProt);
+    return s;
 }
 
 
@@ -408,7 +459,7 @@ uintptr_t LdrGetEntryPoint(uintptr_t base) {
 }
 
 // Base Relocation.
-static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
+static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, uintptr_t delta, bool user) {
     uint32_t peOffset = *(uint32_t *)(base + 0x3C);
     PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
     OPT_HEADER64* opt = &pe->OptionalHeader;
@@ -432,6 +483,8 @@ static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
             // For 64-bit PEs, we mostly care about DIR64 (Type 10).
             if (type == IMAGE_REL_BASED_DIR64) {
                 uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset);
+                KSTATUS ws = KiEnsureUserImageWrite(process, user, (uintptr_t)ptr, sizeof(*ptr));
+                if (!KSUCCESS(ws)) return ws;
                 *ptr += delta;
             }
         }
@@ -534,6 +587,8 @@ static KSTATUS LdrResolveImportsCommon(
                 return STATUS_NOT_FOUND;
             }
 
+            KSTATUS ws = KiEnsureUserImageWrite(process, process != (PROCESS*)0, (uintptr_t)thunk, sizeof(*thunk));
+            if (!KSUCCESS(ws)) return ws;
             *thunk = (uint64_t)addr;
 
             thunk++;
@@ -566,6 +621,41 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
     );
 }
 
+// Loads a user image from an open file through the image-section object model.
+KSTATUS LdrLoadImageFileForProcess(PROCESS* process, FILE_OBJECT* file, uintptr_t* baseOut) {
+    if (!process || !file || !baseOut)
+        return STATUS_INVALID_PARAMETER;
+    *baseOut = 0;
+
+    IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)0;
+    KSTATUS s = LdrCreateImageSectionFromFile(file, &section);
+    if (!KSUCCESS(s))
+        return s;
+
+    void* view = (void*)0;
+    s = LdrMapImageSection(process, section, &view);
+    if (!KSUCCESS(s)) {
+        LdrDereferenceImageSection(section);
+        return s;
+    }
+
+    const IMAGE_SECTION_METADATA* metadata = LdrImageSectionMetadata(section);
+    uintptr_t imageBase = (uintptr_t)view;
+    uintptr_t delta = imageBase - metadata->PreferredBase;
+    if (delta != 0) {
+        s = LdrApplyRelocationsForProcess(process, imageBase, delta, true);
+        if (!KSUCCESS(s)) {
+            KiFreeLoadedImageForProcess(process, imageBase, true);
+            LdrDereferenceImageSection(section);
+            return s;
+        }
+    }
+
+    LdrDereferenceImageSection(section);
+    *baseOut = imageBase;
+    return STATUS_SUCCESS;
+}
+
 // Implements the ldr Register Process Module operation.
 static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) {
     if (!process || !name || !base) return STATUS_INVALID_PARAMETER;
@@ -602,36 +692,27 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
     FILE_OBJECT* f;
     KSTATUS s = IoOpenFile(SysSvFromCstr(filename), &f);
     if (!KSUCCESS(s)) {
-        char path[260];
-        memcpy(path, "0:/system/", 10);
-        memcpy(path + 10, filename, strnlen(filename, 248) + 1);
-        path[259] = '\0';
-        s = IoOpenFile(SysSvFromCstr(path), &f);
+        char relative[260];
+        size_t nameLen = strnlen(filename, sizeof(relative) - 8);
+        if (nameLen + 8 >= sizeof(relative))
+            return STATUS_BUFFER_TOO_SMALL;
+        memcpy(relative, "system/", 7);
+        memcpy(relative + 7, filename, nameLen);
+        relative[7 + nameLen] = '\0';
+
+        s = KeOpenSystemRootFile(relative, &f);
         if (!KSUCCESS(s)) {
-            DbgPrint("LdrLoadModuleForProcess: Failed to open %s (%d)\n", filename, s);
+            DbgPrint("LdrLoadModuleForProcess: Failed to open %s on system root (%d)\n", filename, s);
             return s;
         }
     }
 
     uint64_t fileSize = IoGetFileSize(f);
     LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize);
-    void* mem = MmAllocatePool((size_t)fileSize);
-    if (!mem) {
-        IoCloseFile(f);
-        return STATUS_NO_MEMORY;
-    }
-
-    size_t got = 0;
-    s = IoReadFile(f, mem, (size_t)fileSize, &got);
-    IoCloseFile(f);
-    if (!KSUCCESS(s)) {
-        MmFreePool(mem);
-        return s;
-    }
 
     uintptr_t imageBase = 0;
-    s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase);
-    MmFreePool(mem);
+    s = LdrLoadImageFileForProcess(process, f, &imageBase);
+    IoCloseFile(f);
     if (!KSUCCESS(s)) {
         DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s);
         return s;
@@ -665,10 +746,9 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
     return STATUS_SUCCESS;
 }
 
-// LdrLoadImage - load a PE from a raw buffer into virtual space.
-static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
+// Loads a boot/kernel PE image from a raw memory buffer into kernel space.
+static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, uintptr_t* baseOut) {
     if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER;
-    if (user && !process) return STATUS_INVALID_PARAMETER;
     if (size < 0x40) return STATUS_FAILURE;
 
     const uint8_t* bytes = (const uint8_t*)buffer;
@@ -687,13 +767,13 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
 
     uint16_t optionalMagic = *(const uint16_t *)(bytes + peOffset + 24);
     if (machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
-        DbgPrint("LdrLoadImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
+        DbgPrint("LdrLoadKernelImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
                  machine, optionalMagic);
         return STATUS_NOT_SUPPORTED;
     }
 
     if (machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
-        DbgPrint("LdrLoadImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
+        DbgPrint("LdrLoadKernelImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
                  machine, optionalMagic);
         return STATUS_NOT_SUPPORTED;
     }
@@ -704,17 +784,16 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
     KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize);
     if (!KSUCCESS(sizeStatus)) return sizeStatus;
 
-    LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n",
+    LdrDbg("LDR: kernel image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u\n",
            opt->SizeOfImage,
            (unsigned long long)imageSize,
            (unsigned long long)opt->ImageBase,
-           pe->NumberOfSections,
-           user ? 1 : 0,
-           process ? process->ProcessId : 0);
-    void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize);
+           pe->NumberOfSections);
+
+    void* mem = MmAllocatePoolTag(imageSize, MM_POOL_TAG('i', 'm', 'g', ' '));
     if (!mem) {
-        DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n",
-                 (unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0);
+        DbgPrint("LDR: kernel image allocation failed required=%llx\n",
+                 (unsigned long long)imageSize);
         return STATUS_NO_MEMORY;
     }
     memset(mem, 0, imageSize);
@@ -730,27 +809,37 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
             memcpy(dst, src, copySize);
         }
 
-        uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0);
-        MmSetPageProtect((uintptr_t)mem + sec[i].VirtualAddress,
+        MmPteProtectRange((uintptr_t)mem + sec[i].VirtualAddress,
                          sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData,
-                         prot);
+                         PAGE_PRESENT | PAGE_WRITE);
     }
 
     uintptr_t delta = (uintptr_t)mem - opt->ImageBase;
     if (delta != 0) {
-        LdrApplyRelocations((uintptr_t)mem, delta);
+        KSTATUS rs = LdrApplyRelocationsForProcess((PROCESS*)0, (uintptr_t)mem, delta, false);
+        if (!KSUCCESS(rs)) {
+            MmFreePool(mem);
+            return rs;
+        }
+    }
+
+    // Kernel drivers are still pool-backed images. Section protections cannot
+    // safely make pages read-only yet because imports and driver-owned writable
+    // image data are not separated from callback code by a section object. What
+    // must be true is simpler: every page in the loaded image must be executable
+    // enough for driver callbacks, including section-alignment slack near the
+    // end of the image.
+    KSTATUS xs = MmPteProtectRange((uintptr_t)mem, imageSize, PAGE_PRESENT | PAGE_WRITE);
+    if (!KSUCCESS(xs)) {
+        MmFreePool(mem);
+        return xs;
     }
 
     *baseOut = (uintptr_t)mem;
     return STATUS_SUCCESS;
 }
 
-// Implements the ldr Load Image For Process operation.
-KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) {
-    return KiLdrLoadImageCommon(process, buffer, size, baseOut, true);
-}
-
-// Implements the ldr Load Image operation.
-KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
-    return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user);
+// Loads a kernel image from a bootloader or driver raw buffer.
+KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut) {
+    return KiLdrLoadKernelImageFromBuffer(buffer, size, baseOut);
 }