add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

code/kernel/ke_debug.c

@@ -1,20 +1,24 @@
 // Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.
 
 #include <stdarg.h>
+#include <stdbool.h>
 #include <stdint.h>
 #include <string.h>
 #include <ntlite/ddi/hal_api.h>
 #include <kernel/hal/hal_apic.h>
 #include <intrin.h>
 #pragma intrinsic(_disable)
+#pragma intrinsic(_enable)
 #include <kernel/ke_debug.h>
 #include <kernel/ke_cpu.h>
 #include <kernel/ke_spinlock.h>
 #include <kernel/ke_process.h>
 #include <kernel/ke_thread.h>
 #include <kernel/ke_sched.h>
+#include <kernel/ke_work.h>
 #include <kernel/mm_virt.h>
 #include <kernel/io_file.h>
+#include <kernel/ke_system_root.h>
 #include <common/printf.h>
 int _fltused; // MSVC floating-point helper anchor
 
@@ -22,15 +26,25 @@ int gDebugEnabled = 0;
 static uint32_t gDebugFlags = 0;
 void (*PrintChar)(char, uint32_t, uint32_t);
 
-#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
-#define KE_DEBUG_LOG_LINE_SIZE   1024u
+#define KE_DEBUG_LOG_BUFFER_SIZE    (50u * 1024u)
+#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u)
+#define KE_DEBUG_LOG_LINE_SIZE      1024u
+#define KE_DEBUG_SCREEN_DRAIN_SIZE  512u
 
 static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
 static char*    gDebugLogBuffer;
 static size_t   gDebugLogSize;
 static size_t   gDebugLogWrite;
 static size_t   gDebugLogUsed;
+static char*    gDebugScreenBuffer;
+static size_t   gDebugScreenSize;
+static size_t   gDebugScreenRead;
+static size_t   gDebugScreenWrite;
+static size_t   gDebugScreenUsed;
 static volatile long gDebugLogFlushing;
+static volatile long gDebugConsoleOwnsSerial;
+static volatile long gDebugConsoleWorkReady;
+static KWORK_ITEM gDebugConsoleWorkItem;
 
 // Implements the debug Log Append Locked operation.
 static void KiDebugLogAppendLocked(const char* text, size_t len) {
@@ -55,6 +69,95 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
         gDebugLogUsed += len;
 }
 
+// Implements the debug Screen Append Locked operation.
+static void KiDebugScreenAppendLocked(const char* text, size_t len) {
+    if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0)
+        return;
+
+    if (len >= gDebugScreenSize) {
+        text += len - (gDebugScreenSize - 1u);
+        len = gDebugScreenSize - 1u;
+    }
+
+    while (gDebugScreenUsed + len > gDebugScreenSize) {
+        gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize;
+        gDebugScreenUsed--;
+    }
+
+    size_t first = gDebugScreenSize - gDebugScreenWrite;
+    if (first > len) first = len;
+    memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first);
+    if (len > first)
+        memcpy(gDebugScreenBuffer, text + first, len - first);
+
+    gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize;
+    gDebugScreenUsed += len;
+}
+
+// Implements the debug Screen Drain Locked operation.
+static size_t KiDebugScreenDrainLocked(char* out, size_t max) {
+    if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0)
+        return 0;
+
+    size_t len = gDebugScreenUsed;
+    if (len > max) len = max;
+
+    size_t first = gDebugScreenSize - gDebugScreenRead;
+    if (first > len) first = len;
+    memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first);
+    if (len > first)
+        memcpy(out + first, gDebugScreenBuffer, len - first);
+
+    gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize;
+    gDebugScreenUsed -= len;
+    return len;
+}
+
+// Ensures the asynchronous screen ring exists before console output starts using it.
+static bool KiDebugEnsureScreenBuffer(void) {
+    if (gDebugScreenBuffer && gDebugScreenSize != 0)
+        return true;
+
+    char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r'));
+    if (!buffer)
+        return false;
+    memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE);
+
+    bool installed = false;
+    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    if (!gDebugScreenBuffer) {
+        gDebugScreenBuffer = buffer;
+        gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE;
+        gDebugScreenRead = 0;
+        gDebugScreenWrite = 0;
+        gDebugScreenUsed = 0;
+        installed = true;
+    }
+    KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+    if (!installed)
+        MmFreePool(buffer);
+    return true;
+}
+
+// Implements the debug Serial Write Char operation.
+static void KiDebugSerialWriteChar(char c) {
+    while ((__inbyte(0x3FD) & 0x20) == 0) { }
+    if (c == '\n') {
+        __outbyte(0x3F8, '\r');
+        while ((__inbyte(0x3FD) & 0x20) == 0) { }
+    }
+    __outbyte(0x3F8, (uint8_t)c);
+}
+
+// Implements the debug Serial Write Locked operation.
+static void KiDebugSerialWriteLocked(const char* text, size_t len) {
+    if (!text)
+        return;
+    for (size_t i = 0; i < len; i++)
+        KiDebugSerialWriteChar(text[i]);
+}
+
 // Implements the debug Console Write operation.
 static void KiDebugConsoleWrite(const char* text, size_t len) {
     if (!PrintChar || !text)
@@ -63,6 +166,28 @@ static void KiDebugConsoleWrite(const char* text, size_t len) {
         PrintChar(text[i], 0xCCCCCC, 0x012456);
 }
 
+// Implements the debug Console Work operation.
+static void KiDebugConsoleWork(void* context) {
+    (void)context;
+    char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE];
+
+    for (;;) {
+        uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+        size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk));
+        KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+        if (len == 0)
+            break;
+        KiDebugConsoleWrite(chunk, len);
+    }
+}
+
+// Queues the debug Console Work operation if the passive work queue is ready.
+static void KiDebugQueueConsoleWork(void) {
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted())
+        (void)KeQueueWorkItem(&gDebugConsoleWorkItem);
+}
+
 static volatile long gPanicInProgress = 0;
 static volatile uintptr_t gPanicOwnerToken = 0;
 static volatile long gPanicStoppedCpuCount = 0;
@@ -116,7 +241,7 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
     if (KiPanicIsActive())
         KiPanicSecondaryHalt();
 
-    if (MmHandleTlbShootdownNmi())
+    if (MmPteHandleShootdownNmi())
         return 1;
 
     return 0;
@@ -124,6 +249,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
 
 // Implements the kernel Put Char operation.
 void KiKernelPutChar(char c) {
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteChar(c);
     if (PrintChar)
         PrintChar(c, 0xCCCCCC, 0x012456);
 }
@@ -135,8 +262,11 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
     PrintChar = PrintPtr;
     printf_set_putchar(KiKernelPutChar);
 
+    if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0))
+        KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0);
+
     if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
-        gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE);
+        gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
         if (gDebugLogBuffer) {
             gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
             gDebugLogWrite = 0;
@@ -144,6 +274,9 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
             memset(gDebugLogBuffer, 0, gDebugLogSize);
         }
     }
+
+    if (flags & KE_DEBUG_CONSOLE)
+        (void)KiDebugEnsureScreenBuffer();
 }
 
 // Returns debug Flags.
@@ -157,6 +290,13 @@ void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
         return;
     PrintChar = PrintPtr;
     printf_set_putchar(KiKernelPutChar);
+
+    // After the display manager owns debug drawing, DbgPrint and user console
+    // writes mirror to COM1 themselves. The display writer is screen-only so
+    // asynchronous screen flushing does not duplicate serial bytes.
+    (void)KiDebugEnsureScreenBuffer();
+    _InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1);
+    KiDebugQueueConsoleWork();
 }
 
 // Implements the dbg Print operation.
@@ -179,17 +319,37 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
     if (KiPanicIsActive()) {
         if (!KiPanicCurrentCpuOwnsReport())
             KiPanicSecondaryHalt();
-        if (gDebugFlags & KE_DEBUG_CONSOLE)
+        if (gDebugFlags & KE_DEBUG_CONSOLE) {
+            if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+                KiDebugSerialWriteLocked(line, len);
             KiDebugConsoleWrite(line, len);
+        }
         return;
     }
 
+    bool queueConsole = false;
+    bool writeConsoleSync = false;
+
     uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteLocked(line, len);
     if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
         KiDebugLogAppendLocked(line, len);
-    if (gDebugFlags & KE_DEBUG_CONSOLE)
-        KiDebugConsoleWrite(line, len);
+
+    if (gDebugFlags & KE_DEBUG_CONSOLE) {
+        if (KeIsWorkQueueStarted() && gDebugScreenBuffer) {
+            KiDebugScreenAppendLocked(line, len);
+            queueConsole = true;
+        } else {
+            writeConsoleSync = true;
+        }
+    }
     KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+    if (queueConsole)
+        KiDebugQueueConsoleWork();
+    else if (writeConsoleSync)
+        KiDebugConsoleWrite(line, len);
 }
 
 // Flushes debug Log To File.
@@ -242,6 +402,23 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
     return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
 }
 
+
+// Attempts to persist the debug log during a panic when the filesystem is usable.
+static void KiTryFlushPanicDebugLog(void) {
+    if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
+        return;
+    if (!KeCanWait() || !KeSystemRootReady())
+        return;
+
+    char path[1024];
+    if (!KeBuildSystemRootPath("kernel-panic.log", path, sizeof(path)))
+        return;
+
+    KSTATUS st = KeFlushDebugLogToFile(SysSvFromCstr(path));
+    if (!KSUCCESS(st))
+        DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
+}
+
 // KePanic.
 __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
     KiBeginFatalReport();
@@ -279,15 +456,39 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
         va_end(va);
     }
 
+    KiTryFlushPanicDebugLog();
+
     for (;;) __halt();
 }
 
 // KePrintConsole - write user-visible console text using the default console palette.
-// Panic output keeps a distinct color so fatal paths remain visually obvious.
+// User text mirrors to COM1 immediately, then queues expensive screen drawing
+// through the passive debug-console worker when that path is available.
 __declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
-    if (!buf || !PrintChar) return;
-    for (size_t i = 0; i < len; i++)
-        PrintChar(buf[i], 0xCCCCCC, 0x012456);
+    if (!buf || len == 0) return;
+
+    bool queueConsole = false;
+    bool writeConsoleSync = false;
+    bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer();
+
+    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteLocked(buf, len);
+
+    if (PrintChar) {
+        if (!KiPanicIsActive() && canQueueScreen) {
+            KiDebugScreenAppendLocked(buf, len);
+            queueConsole = true;
+        } else {
+            writeConsoleSync = true;
+        }
+    }
+    KeReleaseSpinlock(&gDebugPrintLock, flags);
+
+    if (queueConsole)
+        KiDebugQueueConsoleWork();
+    else if (writeConsoleSync)
+        KiDebugConsoleWrite(buf, len);
 }
 
 // KeBugCheckProcess - user-mode fault handler
@@ -613,8 +814,22 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
 
 // Prints crash Char.
 static void KiPrintCrashChar(char c) {
+    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
+        KiDebugSerialWriteChar(c);
     if (PrintChar)
-        PrintChar(c, 0xFFAA00, 0x000000);   // amber - distinct from green debug / red panic
+        PrintChar(c, 0xFFAA00, 0x000000);
+}
+
+static KSTATUS KiFaultExitStatus(int vector) {
+    switch (vector) {
+        case 0:  return STATUS_PROCESS_DIVIDE_BY_ZERO;
+        case 3:  return STATUS_PROCESS_BREAKPOINT;
+        case 6:  return STATUS_PROCESS_ILLEGAL_OPCODE;
+        case 13: return STATUS_PROCESS_GENERAL_PROTECTION;
+        case 14: return STATUS_PROCESS_PAGE_FAULT;
+        case 17: return STATUS_PROCESS_ALIGNMENT_FAULT;
+        default: return STATUS_PROCESS_UNKNOWN_FAULT;
+    }
 }
 
 static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
@@ -648,13 +863,23 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
 }
 
 // Implements the finish Faulted Process operation.
-static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread) {
+static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
     printf_set_putchar(KiKernelPutChar);
 
-    if (proc)   KeExitProcess(proc, -1);
-    if (thread) KeExitThread(thread, -1);
+    int exitCode = (int)exitStatus;
+    if (proc)   (void)KeTerminateProcess(proc, exitCode);
+    if (thread) KeExitThread(thread, exitCode);
 
-    while (1) __halt();
+    // KeExitThread normally switches away and never resumes this context.  If
+    // a fault path re-enters here during scheduler bring-up or with no runnable
+    // peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
+    // dead thread to the scheduler with interrupts enabled so waiters such as
+    // the foreground shell can observe the process exit event.
+    while (true) {
+        _enable();
+        KeYield();
+        __halt();
+    }
 }
 
 _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
@@ -664,11 +889,11 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
     KiPrintCrashHeader(proc, vector, rip, errCode, cr2);
     KiDumpModules(proc);
 
-    KiFinishFaultedProcess(proc, thread);
+    KiFinishFaultedProcess(proc, thread, KiFaultExitStatus(vector));
 }
 
-// Implements the fault Dispatch Frame operation.
-__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
+// Dispatches a CPU fault frame and returns only when a user #PF was repaired.
+__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
     // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
     // the interrupted context on a secondary CPU once another CPU owns panic.
     if (KiPanicIsActive())
@@ -682,6 +907,12 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
                 (unsigned long long)(uintptr_t)frame);
     }
 
+    if (frame->Vector == 14) {
+        KSTATUS handled = MmVadResolvePageFaultByAddress((uintptr_t)frame->Cr2, frame->ErrorCode);
+        if (KSUCCESS(handled))
+            return STATUS_SUCCESS;
+    }
+
     if ((frame->Cs & 3) == 0) {
         KiBeginFatalReport();
 
@@ -702,7 +933,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
         if (vector == 14) {
             MmDebugDescribeAddress((uintptr_t)frame->Cr2);
             if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
-            MmDebugDumpUserRegions(proc);
+            MmVadDebugDumpProcess(proc);
         }
 
         if (vector == 14) {
@@ -743,7 +974,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
     KiDumpFaultFrame(proc, frame);
     KiDumpModules(proc);
 
-    KiFinishFaultedProcess(proc, thread);
+    KiFinishFaultedProcess(proc, thread, KiFaultExitStatus((int)frame->Vector));
 }
 
 // KeFaultDispatch - single C entry point for all three IDT fault stubs.
@@ -762,5 +993,6 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
     frame.Cr2 = cr2;
     frame.Rip = rip;
     frame.Cs = cs;
-    KeFaultDispatchFrame(&frame);
+    (void)KeFaultDispatchFrame(&frame);
+    while (1) __halt();
 }