nt-lite/code/kernel/ke_debug.c

// Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.

#include <ntlite/atomic.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <kernel/hal/hal_api.h>
#include <kernel/ke_hal.h>
#include <intrin.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_exception.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/mm_virt.h>
#include <kernel/ldr_image.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor.

int gDebugEnabled = 0;
static uint32_t gDebugFlags;
static const HAL_API* gDebugHal;

#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE   1024u
#define KE_FATAL_OUTPUT_LINE_SIZE 2048u
#define KE_STACK_DUMP_SLOT_COUNT 12u
#define KE_RBP_CALL_STACK_DEPTH 16u

static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static volatile int32_t gDebugLogFlushing;

// Appends bytes to the in-memory debug log while its lock is held.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
    if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
        return;

    if (len >= gDebugLogSize) {
        text += len - (gDebugLogSize - 1u);
        len = gDebugLogSize - 1u;
    }

    size_t first = gDebugLogSize - gDebugLogWrite;
    if (first > len)
        first = len;
    memcpy(gDebugLogBuffer + gDebugLogWrite, text, first);
    if (len > first)
        memcpy(gDebugLogBuffer, text + first, len - first);

    gDebugLogWrite = (gDebugLogWrite + len) % gDebugLogSize;
    if (gDebugLogUsed + len >= gDebugLogSize)
        gDebugLogUsed = gDebugLogSize;
    else
        gDebugLogUsed += len;
}

// Writes one record through the HAL output router.
static void KiOutputWrite(uint32_t stream, uint32_t flags,
                          uint32_t foreground, uint32_t background,
                          const char* text, size_t length) {
    if (!gDebugHal || !gDebugHal->Output.Write || !text || length == 0)
        return;

    HAL_OUTPUT_RECORD record;
    record.Stream = stream;
    record.Flags = flags;
    record.Foreground = foreground;
    record.Background = background;
    record.Buffer = text;
    record.Length = length;
    gDebugHal->Output.Write(&record);
}

// Formats one bounded output record and routes it through the HAL.
static void KiOutputVPrintf(uint32_t stream, uint32_t flags,
                            uint32_t foreground, uint32_t background,
                            const char* format, va_list args) {
    if (!format)
        return;

    char line[KE_FATAL_OUTPUT_LINE_SIZE];
    int count = vsnprintf_(line, sizeof(line), format, args);
    size_t length = 0;
    if (count > 0) {
        length = (size_t)count;
        if (length >= sizeof(line))
            length = sizeof(line) - 1u;
    }
    KiOutputWrite(stream, flags, foreground, background, line, length);
}

static volatile int32_t gPanicInProgress;
static volatile uintptr_t gPanicOwnerToken;
static volatile int32_t gPanicStoppedCpuCount;

// Permanently stops a secondary processor that entered an active panic.
static _Noreturn void KiPanicSecondaryHalt(void) {
    NtlAtomicIncrement32(&gPanicStoppedCpuCount);
    KeStopProcessor();
}

// Returns a stable token for the current execution context.
static uintptr_t KiPanicCurrentToken(void) {
    THREAD* thread = KeGetCurrentThread();
    if (thread)
        return (uintptr_t)thread;
    KCPU* cpu = KeGetCurrentCpu();
    if (cpu)
        return (uintptr_t)cpu;
    return 1u;
}

// Returns whether a fatal report is active.
static bool KiPanicIsActive(void) {
    return NtlAtomicLoad32(&gPanicInProgress) != 0;
}

// Returns whether the current processor owns the active fatal report.
static bool KiPanicCurrentCpuOwnsReport(void) {
    uintptr_t token = KiPanicCurrentToken();
    return token != 0 && gPanicOwnerToken == token;
}

// Claims fatal-report ownership and masks interrupts on this processor.
static void KiBeginFatalReport(void) {
    (void)KeDisableInterrupts();
    uintptr_t token = KiPanicCurrentToken();
    if (NtlAtomicCompareExchange32(&gPanicInProgress, 1, 0) == 0) {
        gPanicOwnerToken = token;
        return;
    }
    if (gPanicOwnerToken != token)
        KiPanicSecondaryHalt();
}

// Handles NMI-based panic stops and TLB shootdowns.
__declspec(dllexport) bool KeHandleNmiIpi(void) {
    if (KiPanicIsActive())
        KiPanicSecondaryHalt();

    if (MmPteHandleShootdownNmi())
        return true;

    return false;
}

// Initializes debug logging and initial output subscriptions.
void KeInitDebug(const HAL_API* hal, uint32_t flags) {
    gDebugHal = hal;
    gDebugFlags = flags;
    gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;

    if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
        gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
        if (gDebugLogBuffer) {
            gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
            gDebugLogWrite = 0;
            gDebugLogUsed = 0;
            memset(gDebugLogBuffer, 0, gDebugLogSize);
        }
    }

    if (hal && hal->Output.SetEndpointStreams &&
        hal->Output.BootFramebufferEndpointId != HAL_OUTPUT_ENDPOINT_INVALID) {
        uint32_t streams = HAL_OUTPUT_STREAM_CONSOLE |
                           HAL_OUTPUT_STREAM_CRASH |
                           HAL_OUTPUT_STREAM_PANIC;
        if (flags & KE_DEBUG_CONSOLE)
            streams |= HAL_OUTPUT_STREAM_DEBUG;
        (void)hal->Output.SetEndpointStreams(hal->Output.BootFramebufferEndpointId, streams);
    }
}

// Returns the active debug flags.
uint32_t KeGetDebugFlags(void) {
    return gDebugFlags;
}

// Writes one formatted kernel diagnostic.
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
    if (!gDebugEnabled || !fmt)
        return;

    char line[KE_DEBUG_LOG_LINE_SIZE];
    va_list args;
    va_start(args, fmt);
    int count = vsnprintf_(line, sizeof(line), fmt, args);
    va_end(args);

    size_t length = 0;
    if (count > 0) {
        length = (size_t)count;
        if (length >= sizeof(line))
            length = sizeof(line) - 1u;
    }

    bool panic = KiPanicIsActive();
    if (panic && !KiPanicCurrentCpuOwnsReport())
        KiPanicSecondaryHalt();

    if (!panic) {
        uintptr_t lockFlags = KeAcquireSpinlock(&gDebugPrintLock);
        if (!NtlAtomicLoad32(&gDebugLogFlushing))
            KiDebugLogAppendLocked(line, length);
        KeReleaseSpinlock(&gDebugPrintLock, lockFlags);
    }

    KiOutputWrite(panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_DEBUG,
                  panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0,
                  panic ? 0x00FF2222u : 0x00CCCCCCu,
                  panic ? 0x00000000u : 0x00012456u,
                  line, length);
}

// Writes one debug-log segment and rejects successful zero-progress writes.
static KSTATUS KiDebugLogWriteAll(FILE_OBJECT* file,
                                  const char* data,
                                  size_t length,
                                  size_t* totalWritten) {
    if (!file || (!data && length) || !totalWritten)
        return NTLSTATUS_INVALID_PARAMETER;

    while (length != 0) {
        size_t written = 0;
        KSTATUS st = IoWriteFile(file, data, length, &written);
        if (!KSUCCESS(st))
            return st;
        if (written == 0)
            return NTLSTATUS_DEVICE_ERROR;
        data += written;
        length -= written;
        *totalWritten += written;
    }
    return NTLSTATUS_SUCCESS;
}

// Persists the in-memory debug ring and flushes it through the storage stack.
KSTATUS KeFlushDebugLogToSystemRoot(const char* relativePath) {
    if (!gDebugLogBuffer || gDebugLogSize == 0 || !relativePath || !relativePath[0])
        return NTLSTATUS_INVALID_PARAMETER;
    if (!KeCanWait())
        return NTLSTATUS_INVALID_IRQL;

    if (NtlAtomicCompareExchange32(&gDebugLogFlushing, 1, 0) != 0)
        return NTLSTATUS_PENDING;

    size_t used;
    size_t start;
    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
    used = gDebugLogUsed;
    start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
    KeReleaseSpinlock(&gDebugPrintLock, flags);

    (void)KeDeleteSystemRootFile(relativePath);

    FILE_OBJECT* file = (FILE_OBJECT*)0;
    KSTATUS st = KeCreateSystemRootFile(relativePath, &file);
    if (!KSUCCESS(st)) {
        NtlAtomicExchange32(&gDebugLogFlushing, 0);
        return st;
    }

    // Shutdown logging must not depend on deferred file-cache writeback. FAT
    // still updates the directory entry and cluster chain through its normal
    // write path, while IoFlushFile pushes metadata and the device cache.
    file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;

    size_t totalWritten = 0;
    if (used != 0) {
        size_t first = gDebugLogSize - start;
        if (first > used)
            first = used;

        st = KiDebugLogWriteAll(file, gDebugLogBuffer + start, first, &totalWritten);
        if (KSUCCESS(st) && used > first) {
            st = KiDebugLogWriteAll(file, gDebugLogBuffer,
                                    used - first, &totalWritten);
        }
    }

    if (KSUCCESS(st) && totalWritten != used)
        st = NTLSTATUS_DEVICE_ERROR;
    if (KSUCCESS(st))
        st = IoFlushFile(file);

    IoCloseFile(file);
    NtlAtomicExchange32(&gDebugLogFlushing, 0);
    return st;
}

// Attempts to persist the debug log during a panic when the filesystem is usable.
static void KiTryFlushPanicDebugLog(void) {
    if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
        return;
    if (!KeCanWait() || !KeSystemRootReady())
        return;

    KSTATUS st = KeFlushDebugLogToSystemRoot("kernel-panic.log");
    if (!KSUCCESS(st))
        DbgPrint("debuglog: panic flush failed st=0x%x\n", st);
}

// Writes one formatted panic record through panic-safe endpoints.
static void KiPanicPrintf(const char* format, ...) {
    va_list args;
    va_start(args, format);
    KiOutputVPrintf(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
                    0x00FF2222u, 0x00000000u, format, args);
    va_end(args);
}

// Stops the system after reporting a fatal kernel error.
__declspec(dllexport) _Noreturn void KePanic(int32_t code, const char* fmt, ...) {
    KiBeginFatalReport();

    // Stop every other CPU before writing the final report. Panic-safe endpoints
    // do not depend on scheduler locks held by processors stopped by this IPI.
    KeBroadcastProcessorNmi();

    static const char banner[] = "\n*** KERNEL PANIC ***\n";
    KiOutputWrite(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
                  0x00FF2222u, 0x00000000u, banner, sizeof(banner) - 1u);
    KiPanicPrintf("Panic code: %d\n", code);

    KCPU* cpu = KeGetCurrentCpu();
    THREAD* thread = KeGetCurrentThread();
    if (cpu) {
        KiPanicPrintf("Panic CPU: %u hardware=%u",
                      (unsigned)cpu->CpuIndex,
                      (unsigned)cpu->HardwareId);
        if (thread) {
            KiPanicPrintf(" thread=0x%016llX tid=%llu",
                          thread,
                          (unsigned long long)thread->ThreadId);
        }
        KiPanicPrintf("\n");
    }

    va_list args;
    va_start(args, fmt);
    KiOutputVPrintf(HAL_OUTPUT_STREAM_PANIC, HAL_OUTPUT_WRITE_EMERGENCY,
                    0x00FF2222u, 0x00000000u, fmt, args);
    va_end(args);

    KiTryFlushPanicDebugLog();
    KeStopProcessor();
}

// Writes user-visible console text through subscribed output endpoints.
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
    KiOutputWrite(HAL_OUTPUT_STREAM_CONSOLE, 0,
                  0x00CCCCCCu, 0x00012456u, buf, len);
}

// Writes one formatted process-fault or kernel-fault diagnostic.
static void KiFaultPrintf(const char* format, ...) {
    bool panic = KiPanicIsActive();
    va_list args;
    va_start(args, format);
    KiOutputVPrintf(panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_CRASH,
                    panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0,
                    panic ? 0x00FF2222u : 0x00FFAA00u,
                    0x00000000u,
                    format, args);
    va_end(args);
}

// Resolves the main user image or a registered kernel image that owns an address.
// User-mode DLL bookkeeping belongs to the user-mode loader and is intentionally
// not duplicated in kernel diagnostics.
static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
                                     uintptr_t* baseOut, size_t* sizeOut) {
    if (baseOut) *baseOut = 0;
    if (sizeOut) *sizeOut = 0;

    if (!proc) return "unknown";

    if (proc->ImageBase && proc->ImageSize &&
        addr >= proc->ImageBase && addr < proc->ImageBase + proc->ImageSize) {
        if (baseOut) *baseOut = proc->ImageBase;
        if (sizeOut) *sizeOut = proc->ImageSize;
        return proc->Name;
    }

    uintptr_t kernelBase = 0;
    size_t kernelSize = 0;
    const char* kernelModule = LdrFindKernelModuleByAddress(addr, &kernelBase, &kernelSize);
    if (kernelBase) {
        if (baseOut) *baseOut = kernelBase;
        if (sizeOut) *sizeOut = kernelSize;
        return kernelModule;
    }

    return "unknown";
}

// Implements the resolve Module operation.
static const char* KiResolveModule(PROCESS* proc, uintptr_t addr) {
    return KiResolveModuleEx(proc, addr, (uintptr_t*)0, (size_t*)0);
}

// Prints address.
static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
    uintptr_t base = 0;
    size_t size = 0;
    const char* mod = KiResolveModuleEx(proc, addr, &base, &size);

    if (base) {
        KiFaultPrintf("%s%016llx <%s+0x%016llX>\n",
               label,
               (unsigned long long)addr,
               mod,
               (unsigned long long)(addr - base));
    } else {
        KiFaultPrintf("%s%016llx <unknown>\n",
               label,
               (unsigned long long)addr);
    }
}

// Prints a resolved address without ending the current line.
static void KiPrintAddressValue(PROCESS* proc, uintptr_t addr) {
    uintptr_t base = 0;
    size_t size = 0;
    const char* mod = KiResolveModuleEx(proc, addr, &base, &size);

    if (base) {
        KiFaultPrintf("%016llx <%s+0x%016llX>",
                      (unsigned long long)addr,
                      mod,
                      (unsigned long long)(addr - base));
    } else {
        KiFaultPrintf("%016llx", (unsigned long long)addr);
    }
}

// Checks whether a kernel virtual range is mapped before diagnostic reads.
static bool KiKernelRangeReadable(uintptr_t addr, size_t size) {
    if (!addr || size == 0) return false;
    if (size - 1u > (size_t)(UINTPTR_MAX - addr)) return false;

    uintptr_t start = addr & PAGE_MASK;
    uintptr_t last = (addr + size - 1u) & PAGE_MASK;

    for (uintptr_t page = start;; page += PAGE_SIZE) {
        PHYSICAL_ADDRESS phys = 0;
        uint64_t flags = 0;
        if (!KSUCCESS(MmPteQueryPageFor(MmPteQueryDebug, page, &phys, &flags)))
            return false;
        if ((flags & PAGE_PRESENT) == 0)
            return false;
        if (page == last)
            break;
        if (page > UINTPTR_MAX - PAGE_SIZE)
            return false;
    }
    return true;
}

// Reads diagnostic memory from either user space or a mapped kernel range.
static bool KiReadDiagnosticMemory(uintptr_t addr, void* out, size_t size) {
    if (!out || !addr || size == 0) return false;

    if (addr >= KERNEL_BASE) {
        if (!KiKernelRangeReadable(addr, size))
            return false;
        memcpy(out, (const void*)addr, size);
        return true;
    }

    if (!KSUCCESS(MmProbeUserRead((const void*)addr, size)))
        return false;
    memcpy(out, (const void*)addr, size);
    return true;
}

// Reads one diagnostic uint64_t.
static bool KiUserReadU64(uintptr_t addr, uint64_t* out) {
    return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}

// Reads one diagnostic byte.
static bool KiUserReadU8(uintptr_t addr, uint8_t* out) {
    return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}

// Dumps bytes.
static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
    if (!center) {
        KiFaultPrintf("%s: <null>\n", label);
        return;
    }

    KiFaultPrintf("%s around ", label);
    KiPrintAddress(proc, "", center);

    uintptr_t start = center >= 16 ? center - 16 : 0;
    for (int row = 0; row < 2; row++) {
        uintptr_t addr = start + (uintptr_t)row * 16u;
        KiFaultPrintf("  %016llx:", (unsigned long long)addr);
        for (int i = 0; i < 16; i++) {
            uint8_t b = 0;
            if (KiUserReadU8(addr + (uintptr_t)i, &b))
                KiFaultPrintf(" %02x", b);
            else
                KiFaultPrintf(" ??");
        }
        KiFaultPrintf("\n");
    }
}

// Reads one diagnostic int32_t.
static bool KiUserReadI32(uintptr_t addr, int32_t* out) {
    return KiReadDiagnosticMemory(addr, out, sizeof(*out));
}

// Dumps indirect Call Slot.
static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
    if (!ret) return;

    // Most x64 import/function-pointer calls are one of:
    //   FF 15 disp32       call qword ptr [rip+disp32]
    //   48 FF 15 disp32    rex.w call qword ptr [rip+disp32]
    // The saved return address points immediately after the instruction.
    uintptr_t insn = 0;
    int32_t disp = 0;
    uint8_t b0 = 0, b1 = 0, b2 = 0;

    if (KiUserReadU8(ret - 6, &b0) &&
        KiUserReadU8(ret - 5, &b1) &&
        b0 == 0xff && b1 == 0x15 &&
        // Implements the user Read I32 operation.
        KiUserReadI32(ret - 4, &disp)) {
        insn = ret - 6;
    } else if (KiUserReadU8(ret - 7, &b0) &&
               KiUserReadU8(ret - 6, &b1) &&
               KiUserReadU8(ret - 5, &b2) &&
               b0 == 0x48 && b1 == 0xff && b2 == 0x15 &&
               // Implements the user Read I32 operation.
               KiUserReadI32(ret - 4, &disp)) {
        insn = ret - 7;
    } else {
        KiFaultPrintf("No RIP-relative indirect call pattern found before return address.\n");
        return;
    }

    uintptr_t slot = ret + (intptr_t)disp;
    uint64_t target = 0;

    KiFaultPrintf("Decoded null indirect call:\n");
    KiPrintAddress(proc, "  insn=", insn);
    KiPrintAddress(proc, "  slot=", slot);

    if (KiUserReadU64(slot, &target)) {
        KiPrintAddress(proc, "  *slot=", (uintptr_t)target);
    } else {
        KiFaultPrintf("  *slot=<unreadable>\n");
    }

    KiDumpBytes(proc, "Bytes near indirect slot", slot);
}

// Dumps modules.
static void KiDumpModules(PROCESS* proc) {
    if (!proc) return;

    KiFaultPrintf("Images:\n");
    if (proc->ImageBase && proc->ImageSize) {
        KiFaultPrintf("  %-31s %016llx-%016llx size=%llx\n",
               proc->Name,
               (unsigned long long)proc->ImageBase,
               (unsigned long long)(proc->ImageBase + proc->ImageSize),
               (unsigned long long)proc->ImageSize);
    }

}

#if 0
// Returns whether a range is inside the current kernel stack.
static bool KiKernelStackContains(const THREAD* thread, uintptr_t addr, size_t size) {
    if (!thread || !thread->KernelStackBase || !thread->KernelStackTop || !addr || size == 0)
        return false;
    if (size - 1u > (size_t)(UINTPTR_MAX - addr))
        return false;

    uintptr_t base = (uintptr_t)thread->KernelStackBase;
    uintptr_t top = thread->KernelStackTop;
    uintptr_t end = addr + size;
    return addr >= base && end <= top;
}

// Returns whether the fault frame is using the current kernel stack.
static bool KiFaultFrameOnKernelStack(const THREAD* thread, const KFAULT_FRAME* f) {
    if (!f || !thread)
        return false;
    return KiKernelStackContains(thread, (uintptr_t)f->Rsp, sizeof(uint64_t));
}

#endif
// Dumps stack words near RSP.
static void KiDumpStack(PROCESS* proc, uintptr_t rsp) {
    KiFaultPrintf("Stack @ %016llx:\n", (unsigned long long)rsp);
    if (!rsp) {
        KiFaultPrintf("  <no RSP captured>\n");
        return;
    }

    for (uint32_t i = 0; i < KE_STACK_DUMP_SLOT_COUNT; i++) {
        uintptr_t slot = rsp + (uintptr_t)i * sizeof(uint64_t);
        uint64_t value = 0;
        KiFaultPrintf("  [%02u] %016llx : ",
                      (unsigned)i,
                      (unsigned long long)slot);
        if (KiUserReadU64(slot, &value)) {
            KiPrintAddressValue(proc, (uintptr_t)value);
            KiFaultPrintf("\n");
        } else {
            KiFaultPrintf("<unreadable>\n");
        }
    }
}

#if 0
// Dumps a best-effort frame-pointer call stack.
static void KiDumpRbpCallStack(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
    uintptr_t rbp = f ? (uintptr_t)f->Rbp : 0;
    KiFaultPrintf("RBP call stack @ %016llx:\n", (unsigned long long)rbp);
    if (!rbp) {
        KiFaultPrintf("  <no RBP captured>\n");
        return;
    }

    int kernelStack = KiFaultFrameOnKernelStack(thread, f);
    for (uint32_t depth = 0; depth < KE_RBP_CALL_STACK_DEPTH; depth++) {
        if (rbp & (sizeof(uint64_t) - 1u)) {
            KiFaultPrintf("  stop: unaligned RBP %016llx\n",
                          (unsigned long long)rbp);
            return;
        }
        if (kernelStack && !KiKernelStackContains(thread, rbp, sizeof(uint64_t) * 2u)) {
            KiFaultPrintf("  stop: RBP outside kernel stack %016llx\n",
                          (unsigned long long)rbp);
            return;
        }

        uint64_t next = 0;
        uint64_t ret = 0;
        if (!KiUserReadU64(rbp, &next) || !KiUserReadU64(rbp + sizeof(uint64_t), &ret)) {
            KiFaultPrintf("  stop: frame unreadable at %016llx\n",
                          (unsigned long long)rbp);
            return;
        }

        KiFaultPrintf("  #%02u rbp=%016llx ret=",
                      (unsigned)depth,
                      (unsigned long long)rbp);
        KiPrintAddressValue(proc, (uintptr_t)ret);
        KiFaultPrintf(" next=%016llx\n", (unsigned long long)next);

        if (!next)
            return;
        if ((uintptr_t)next <= rbp) {
            KiFaultPrintf("  stop: non-forward RBP chain %016llx -> %016llx\n",
                          (unsigned long long)rbp,
                          (unsigned long long)next);
            return;
        }
        rbp = (uintptr_t)next;
    }
}

#endif
// Dumps a fault frame.
static void KiDumpFaultFrame(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* f) {
    if (!f) return;

    KiFaultPrintf("Regs:\n");
    KiFaultPrintf("  RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n",
           (unsigned long long)f->Rax, (unsigned long long)f->Rbx,
           (unsigned long long)f->Rcx, (unsigned long long)f->Rdx);
    KiFaultPrintf("  RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n",
           (unsigned long long)f->Rsi, (unsigned long long)f->Rdi,
           (unsigned long long)f->Rbp, (unsigned long long)f->Rsp);
    KiFaultPrintf("  R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n",
           (unsigned long long)f->R8,  (unsigned long long)f->R9,
           (unsigned long long)f->R10, (unsigned long long)f->R11);
    KiFaultPrintf("  R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n",
           (unsigned long long)f->R12, (unsigned long long)f->R13,
           (unsigned long long)f->R14, (unsigned long long)f->R15);
    KiFaultPrintf("  CS=%04llx SS=%04llx RFLAGS=%016llx\n",
           (unsigned long long)f->Cs,
           (unsigned long long)f->Ss,
           (unsigned long long)f->Rflags);

    if (f->Rip == 0) {
        uint64_t ret = 0;
        if (KiUserReadU64((uintptr_t)f->Rsp, &ret)) {
            KiFaultPrintf("Likely null indirect call. Return address from [RSP]:\n");
            KiPrintAddress(proc, "  ret=", (uintptr_t)ret);
            KiDumpBytes(proc, "Bytes near return address", (uintptr_t)ret);
            KiDumpIndirectCallSlot(proc, (uintptr_t)ret);
        } else {
            KiFaultPrintf("Likely null indirect call, but [RSP] is unreadable.\n");
        }
    } else {
        KiDumpBytes(proc, "Bytes near RIP", (uintptr_t)f->Rip);
    }

    KiDumpStack(proc, (uintptr_t)f->Rsp);
#if 0
    KiDumpRbpCallStack(proc, thread, f);
#endif
}

static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
                               uintptr_t errCode, uintptr_t cr2) {
    const char* faultName = KeExceptionVectorName((uint32_t)vector);
    const char* procName  = proc ? proc->Name  : "unknown";
    uint32_t    pid       = proc ? proc->ProcessId : 0;
    const char* modName   = proc ? KiResolveModule(proc, rip) : "unknown";

    static const char banner[] = "\n*** APPLICATION CRASH ***\n";
    KiOutputWrite(HAL_OUTPUT_STREAM_CRASH, 0,
                  0x00FFAA00u, 0x00000000u, banner, sizeof(banner) - 1u);

    if (vector == 14)
        KiFaultPrintf("Process: %s (PID %u) | Fault: %s\n"
               "Module:  %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
               procName, pid, faultName,
               modName,
               (unsigned long long)rip,
               (unsigned long long)errCode,
               (unsigned long long)cr2);
    else
        KiFaultPrintf("Process: %s (PID %u) | Fault: %s\n"
               "Module:  %s | RIP: %016llx | Code: %04llx\n",
               procName, pid, faultName,
               modName,
               (unsigned long long)rip,
               (unsigned long long)errCode);
}

// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
    int exitCode = (int)exitStatus;
    if (proc)   (void)KeTerminateProcess(proc, exitCode);
    if (thread) KeExitThread(thread, exitCode);

    // KeExitThread normally switches away and never resumes this context. If
    // a fault path re-enters here during scheduler bring-up or with no runnable
    // peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
    // dead thread to the scheduler with interrupts enabled so waiters such as
    // the foreground shell can observe the process exit event.
    while (true) {
        KeEnableInterrupts();
        KeYield();
        KeWaitForInterrupt();
    }
}

_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
    THREAD*  thread = KeGetCurrentThread();
    PROCESS* proc   = thread ? thread->Owner : (PROCESS*)0;

    KiPrintCrashHeader(proc, vector, rip, errCode, cr2);
    KiDumpModules(proc);

    KiFinishFaultedProcess(proc, thread, KeExceptionProcessExitStatus((uint32_t)vector));
}

// Panics after printing a kernel-mode CPU exception report.
static _Noreturn void KiPanicKernelFault(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* frame) {
    KiBeginFatalReport();

    int vector = (int)frame->Vector;
    const char* name = KeExceptionVectorName((uint32_t)vector);

    DbgPrint("FAULT: kernel exception vector=%d name=%s rip=%016llx rsp=%016llx err=%04llx cr2=%016llx thread=0x%016llX pid=%u proc=%s\n",
             vector,
             name,
             (unsigned long long)frame->Rip,
             (unsigned long long)frame->Rsp,
             (unsigned long long)frame->ErrorCode,
             (unsigned long long)frame->Cr2,
             thread,
             proc ? proc->ProcessId : 0,
             proc ? proc->Name : "<none>");
    KiDumpFaultFrame(proc, thread, frame);
    if (vector == 14) {
        MmDebugDescribeAddress((uintptr_t)frame->Cr2);
        if (frame->Rip) MmDebugDescribeAddress((uintptr_t)frame->Rip);
        if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
        MmVadDebugDumpProcess(proc);
    }

    if (vector == 14) {
        KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | RSP: %016llx | Code: %04llx | CR2: %016llx\n",
                name,
                (unsigned long long)frame->Rip,
                (unsigned long long)frame->Rsp,
                (unsigned long long)frame->ErrorCode,
                (unsigned long long)frame->Cr2);
    } else if (KeExceptionVectorHasErrorCode((uint32_t)vector)) {
        const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip;
        KePanic(KE_PANIC_CODE_FAULT,
                "PANIC: %s | RIP: %016llx | Code: %04llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n",
                name,
                (unsigned long long)frame->Rip,
                (unsigned long long)frame->ErrorCode,
                (unsigned long long)frame->Cs,
                (unsigned long long)frame->Rflags,
                (unsigned long long)frame->Rsp,
                ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]);
    } else {
        const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip;
        KePanic(KE_PANIC_CODE_FAULT,
                "PANIC: %s | RIP: %016llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n",
                name,
                (unsigned long long)frame->Rip,
                (unsigned long long)frame->Cs,
                (unsigned long long)frame->Rflags,
                (unsigned long long)frame->Rsp,
                ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]);
    }
}

// Terminates the current process after printing an unhandled user exception report.
static _Noreturn void KiTerminateUserFault(PROCESS* proc, THREAD* thread, const KFAULT_FRAME* frame) {
    KiPrintCrashHeader(proc, (int)frame->Vector,
                       (uintptr_t)frame->Rip,
                       (uintptr_t)frame->ErrorCode,
                       (uintptr_t)frame->Cr2);
    KiDumpFaultFrame(proc, thread, frame);
    KiDumpModules(proc);

    KiFinishFaultedProcess(proc, thread, KeExceptionProcessExitStatus((uint32_t)frame->Vector));
}

// Dispatches a CPU fault frame and returns only when a page fault was repaired.
__declspec(dllexport) KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) {
    // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
    // the interrupted context on a secondary CPU once another CPU owns panic.
    if (KiPanicIsActive())
        KiPanicSecondaryHalt();

    THREAD*  thread = KeGetCurrentThread();
    PROCESS* proc   = thread ? thread->Owner : (PROCESS*)0;

    if (!KeFaultFrameIsValid(frame)) {
        KePanic(KE_PANIC_CODE_FAULT, "PANIC: bad KFAULT_FRAME %016llx\n",
                (unsigned long long)(uintptr_t)frame);
    }

    KSTATUS repaired = KeExceptionResolveRecoverableFault(frame);
    if (KSUCCESS(repaired))
        return NTLSTATUS_SUCCESS;

    if (!KeFaultFrameFromUser(frame))
        KiPanicKernelFault(proc, thread, frame);

    DbgPrint("USERFAULT: pid=%u tid=%u vector=%llu rip=0x%016llX rsp=0x%016llX cr2=0x%016llX error=0x%016llX rcx=0x%016llX rbx=0x%016llX rax=0x%016llX\n",
             proc ? proc->ProcessId : 0,
             thread ? thread->ThreadId : 0,
             frame->Vector,
             (uint64_t)frame->Rip,
             (uint64_t)frame->Rsp,
             (uint64_t)frame->Cr2,
             (uint64_t)frame->ErrorCode,
             (uint64_t)frame->Rcx,
             (uint64_t)frame->Rbx,
             (uint64_t)frame->Rax);

    KSTATUS delivered = KeDeliverUserException(frame);
    if (KSUCCESS(delivered))
        return NTLSTATUS_SUCCESS;

    if (delivered != NTLSTATUS_NOT_FOUND)
        DbgPrint("USER/EXC: delivery failed vector=%llu rip=%016llx rsp=%016llx status=%u\n",
                 frame->Vector,
                 (unsigned long long)frame->Rip,
                 (unsigned long long)frame->Rsp,
                 delivered);

    KiTerminateUserFault(proc, thread, frame);
}

// KeFaultDispatch - single C entry point for all three IDT fault stubs.

// The asm stubs collect the CPU exception frame values and call here.
// KeFaultDispatchFrame owns the mode split and default unhandled policy.
__declspec(dllexport) _Noreturn void KeFaultDispatch(uint32_t vector, uintptr_t rip,
                                                     uintptr_t errCode, uintptr_t cr2,
                                                     uintptr_t cs) {
    KFAULT_FRAME frame;
    memset(&frame, 0, sizeof(frame));
    frame.Magic = KFAULT_FRAME_MAGIC_VALUE;
    frame.Vector = (uint64_t)vector;
    frame.ErrorCode = errCode;
    frame.Cr2 = cr2;
    frame.Rip = rip;
    frame.Cs = cs;
    (void)KeFaultDispatchFrame(&frame);
    KeStopProcessor();
}