nt-lite/code/kernel/ke_debug.c

// Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.

#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <kernel/ke_debug.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_work.h>
#include <kernel/mm_virt.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor

int gDebugEnabled = 0;
static uint32_t gDebugFlags = 0;
void (*PrintChar)(char, uint32_t, uint32_t);

#define KE_DEBUG_LOG_BUFFER_SIZE    (50u * 1024u)
#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE      1024u
#define KE_DEBUG_SCREEN_DRAIN_SIZE  512u

static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char*    gDebugLogBuffer;
static size_t   gDebugLogSize;
static size_t   gDebugLogWrite;
static size_t   gDebugLogUsed;
static char*    gDebugScreenBuffer;
static size_t   gDebugScreenSize;
static size_t   gDebugScreenRead;
static size_t   gDebugScreenWrite;
static size_t   gDebugScreenUsed;
static volatile long gDebugLogFlushing;
static volatile long gDebugConsoleOwnsSerial;
static volatile long gDebugConsoleWorkReady;
static KWORK_ITEM gDebugConsoleWorkItem;

// Implements the debug Log Append Locked operation.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
    if (!gDebugLogBuffer || gDebugLogSize == 0 || !text || len == 0)
        return;

    if (len >= gDebugLogSize) {
        text += len - (gDebugLogSize - 1u);
        len = gDebugLogSize - 1u;
    }

    size_t first = gDebugLogSize - gDebugLogWrite;
    if (first > len) first = len;
    memcpy(gDebugLogBuffer + gDebugLogWrite, text, first);
    if (len > first)
        memcpy(gDebugLogBuffer, text + first, len - first);

    gDebugLogWrite = (gDebugLogWrite + len) % gDebugLogSize;
    if (gDebugLogUsed + len >= gDebugLogSize)
        gDebugLogUsed = gDebugLogSize;
    else
        gDebugLogUsed += len;
}

// Implements the debug Screen Append Locked operation.
static void KiDebugScreenAppendLocked(const char* text, size_t len) {
    if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0)
        return;

    if (len >= gDebugScreenSize) {
        text += len - (gDebugScreenSize - 1u);
        len = gDebugScreenSize - 1u;
    }

    while (gDebugScreenUsed + len > gDebugScreenSize) {
        gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize;
        gDebugScreenUsed--;
    }

    size_t first = gDebugScreenSize - gDebugScreenWrite;
    if (first > len) first = len;
    memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first);
    if (len > first)
        memcpy(gDebugScreenBuffer, text + first, len - first);

    gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize;
    gDebugScreenUsed += len;
}

// Implements the debug Screen Drain Locked operation.
static size_t KiDebugScreenDrainLocked(char* out, size_t max) {
    if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0)
        return 0;

    size_t len = gDebugScreenUsed;
    if (len > max) len = max;

    size_t first = gDebugScreenSize - gDebugScreenRead;
    if (first > len) first = len;
    memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first);
    if (len > first)
        memcpy(out + first, gDebugScreenBuffer, len - first);

    gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize;
    gDebugScreenUsed -= len;
    return len;
}

// Ensures the asynchronous screen ring exists before console output starts using it.
static bool KiDebugEnsureScreenBuffer(void) {
    if (gDebugScreenBuffer && gDebugScreenSize != 0)
        return true;

    char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r'));
    if (!buffer)
        return false;
    memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE);

    bool installed = false;
    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
    if (!gDebugScreenBuffer) {
        gDebugScreenBuffer = buffer;
        gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE;
        gDebugScreenRead = 0;
        gDebugScreenWrite = 0;
        gDebugScreenUsed = 0;
        installed = true;
    }
    KeReleaseSpinlock(&gDebugPrintLock, flags);

    if (!installed)
        MmFreePool(buffer);
    return true;
}

// Implements the debug Serial Write Char operation.
static void KiDebugSerialWriteChar(char c) {
    while ((__inbyte(0x3FD) & 0x20) == 0) { }
    if (c == '\n') {
        __outbyte(0x3F8, '\r');
        while ((__inbyte(0x3FD) & 0x20) == 0) { }
    }
    __outbyte(0x3F8, (uint8_t)c);
}

// Implements the debug Serial Write Locked operation.
static void KiDebugSerialWriteLocked(const char* text, size_t len) {
    if (!text)
        return;
    for (size_t i = 0; i < len; i++)
        KiDebugSerialWriteChar(text[i]);
}

// Implements the debug Console Write operation.
static void KiDebugConsoleWrite(const char* text, size_t len) {
    if (!PrintChar || !text)
        return;
    for (size_t i = 0; i < len; i++)
        PrintChar(text[i], 0xCCCCCC, 0x012456);
}

// Implements the debug Console Work operation.
static void KiDebugConsoleWork(void* context) {
    (void)context;
    char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE];

    for (;;) {
        uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
        size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk));
        KeReleaseSpinlock(&gDebugPrintLock, flags);

        if (len == 0)
            break;
        KiDebugConsoleWrite(chunk, len);
    }
}

// Queues the debug Console Work operation if the passive work queue is ready.
static void KiDebugQueueConsoleWork(void) {
    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted())
        (void)KeQueueWorkItem(&gDebugConsoleWorkItem);
}

static volatile long gPanicInProgress = 0;
static volatile uintptr_t gPanicOwnerToken = 0;
static volatile long gPanicStoppedCpuCount = 0;

// Implements the panic Secondary Halt operation.
static _Noreturn void KiPanicSecondaryHalt(void) {
    _disable();
    _InterlockedIncrement(&gPanicStoppedCpuCount);
    for (;;) __halt();
}

// Implements the panic Current Token operation.
static uintptr_t KiPanicCurrentToken(void) {
    THREAD* thread = KeGetCurrentThread();
    if (thread)
        return (uintptr_t)thread;
    KCPU* cpu = KeGetCurrentCpu();
    if (cpu)
        return (uintptr_t)cpu;
    return 1u;
}

// Implements the panic Is Active operation.
static int KiPanicIsActive(void) {
    return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0;
}

// Implements the panic Current Cpu Owns Report operation.
static int KiPanicCurrentCpuOwnsReport(void) {
    uintptr_t token = KiPanicCurrentToken();
    return token != 0 && gPanicOwnerToken == token;
}

// Implements the begin Fatal Report operation.
static void KiBeginFatalReport(void) {
    _disable();
    uintptr_t token = KiPanicCurrentToken();
    if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) == 0) {
        gPanicOwnerToken = token;
        return;
    }
    if (gPanicOwnerToken != token)
        KiPanicSecondaryHalt();
}

// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the
// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely
// unexpected NMI is not handled here and will fall through to the normal fault
// path in the assembly stub.
__declspec(dllexport) int KeHandleNmiIpi(void) {
    if (KiPanicIsActive())
        KiPanicSecondaryHalt();

    if (MmPteHandleShootdownNmi())
        return 1;

    return 0;
}

// Implements the kernel Put Char operation.
void KiKernelPutChar(char c) {
    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
        KiDebugSerialWriteChar(c);
    if (PrintChar)
        PrintChar(c, 0xCCCCCC, 0x012456);
}

// Initializes debug.
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
    gDebugFlags = flags;
    gDebugEnabled = (flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) != 0;
    PrintChar = PrintPtr;
    printf_set_putchar(KiKernelPutChar);

    if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0))
        KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0);

    if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
        gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
        if (gDebugLogBuffer) {
            gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
            gDebugLogWrite = 0;
            gDebugLogUsed = 0;
            memset(gDebugLogBuffer, 0, gDebugLogSize);
        }
    }

    if (flags & KE_DEBUG_CONSOLE)
        (void)KiDebugEnsureScreenBuffer();
}

// Returns debug Flags.
uint32_t KeGetDebugFlags(void) {
    return gDebugFlags;
}

// Updates debug Console Writer.
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
    if (!PrintPtr)
        return;
    PrintChar = PrintPtr;
    printf_set_putchar(KiKernelPutChar);

    // After the display manager owns debug drawing, DbgPrint and user console
    // writes mirror to COM1 themselves. The display writer is screen-only so
    // asynchronous screen flushing does not duplicate serial bytes.
    (void)KiDebugEnsureScreenBuffer();
    _InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1);
    KiDebugQueueConsoleWork();
}

// Implements the dbg Print operation.
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
    if (!gDebugEnabled || !fmt) return;

    char line[KE_DEBUG_LOG_LINE_SIZE];
    va_list va;
    va_start(va, fmt);
    int n = vsnprintf_(line, sizeof(line), fmt, va);
    va_end(va);

    size_t len = 0;
    if (n > 0) {
        len = (size_t)n;
        if (len >= sizeof(line))
            len = sizeof(line) - 1u;
    }

    if (KiPanicIsActive()) {
        if (!KiPanicCurrentCpuOwnsReport())
            KiPanicSecondaryHalt();
        if (gDebugFlags & KE_DEBUG_CONSOLE) {
            if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
                KiDebugSerialWriteLocked(line, len);
            KiDebugConsoleWrite(line, len);
        }
        return;
    }

    bool queueConsole = false;
    bool writeConsoleSync = false;

    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
        KiDebugSerialWriteLocked(line, len);
    if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
        KiDebugLogAppendLocked(line, len);

    if (gDebugFlags & KE_DEBUG_CONSOLE) {
        if (KeIsWorkQueueStarted() && gDebugScreenBuffer) {
            KiDebugScreenAppendLocked(line, len);
            queueConsole = true;
        } else {
            writeConsoleSync = true;
        }
    }
    KeReleaseSpinlock(&gDebugPrintLock, flags);

    if (queueConsole)
        KiDebugQueueConsoleWork();
    else if (writeConsoleSync)
        KiDebugConsoleWrite(line, len);
}

// Flushes debug Log To File.
__declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
    if (!gDebugLogBuffer || gDebugLogSize == 0 || !SysSvIsValid(path) || path.Length == 0)
        return STATUS_INVALID_PARAMETER;
    if (!KeCanWait())
        return STATUS_INVALID_IRQL;

    if (_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 1, 0) != 0)
        return STATUS_PENDING;

    size_t used;
    size_t start;
    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
    used = gDebugLogUsed;
    start = (gDebugLogUsed == gDebugLogSize) ? gDebugLogWrite : 0;
    KeReleaseSpinlock(&gDebugPrintLock, flags);

    if (used == 0) {
        _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
        return STATUS_SUCCESS;
    }

    (void)IoDeleteFile(path);

    FILE_OBJECT* file = (FILE_OBJECT*)0;
    KSTATUS st = IoCreateFile(path, &file);
    if (!KSUCCESS(st)) {
        _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
        return st;
    }

    size_t written = 0;
    size_t totalWritten = 0;
    size_t first = gDebugLogSize - start;
    if (first > used) first = used;
    st = IoWriteFile(file, gDebugLogBuffer + start, first, &written);
    totalWritten += written;

    if (KSUCCESS(st) && used > first) {
        written = 0;
        st = IoWriteFile(file, gDebugLogBuffer, used - first, &written);
        totalWritten += written;
    }

    IoCloseFile(file);

    _InterlockedExchange((volatile long*)&gDebugLogFlushing, 0);
    return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}


// Attempts to persist the debug log during a panic when the filesystem is usable.
static void KiTryFlushPanicDebugLog(void) {
    if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
        return;
    if (!KeCanWait() || !KeSystemRootReady())
        return;

    char path[1024];
    if (!KeBuildSystemRootPath("kernel-panic.log", path, sizeof(path)))
        return;

    KSTATUS st = KeFlushDebugLogToFile(SysSvFromCstr(path));
    if (!KSUCCESS(st))
        DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
}

// KePanic.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
    KiBeginFatalReport();

    // Stop every other CPU for the final panic banner. Kernel fault dispatch
    // claims panic ownership before its detailed dump, so this does not steal
    // ownership from the CPU that found the fatal error.
    HalApicSendPanicIpi();

    if (PrintChar) {
        // Print a bright-red banner so the framebuffer is unmistakable.
        static const char banner[] = "\n*** KERNEL PANIC ***\n";
        const char* p;
        KCPU* cpu = KeGetCurrentCpu();
        THREAD* thread = KeGetCurrentThread();

        for (p = banner; *p; p++)
            PrintChar(*p, 0xFF2222, 0x000000);

        printf("Panic code: %d\n", code);
        if (cpu) {
            printf("Panic CPU: %u LAPIC=%u",
                   (unsigned)cpu->CpuIndex,
                   (unsigned)cpu->LapicId);
            if (thread)
                printf(" thread=0x%016llX tid=%llu",
                       thread,
                       (unsigned long long)thread->ThreadId);
            printf("\n");
        }

        va_list va;
        va_start(va, fmt);
        vprintf(fmt, va);
        va_end(va);
    }

    KiTryFlushPanicDebugLog();

    for (;;) __halt();
}

// KePrintConsole - write user-visible console text using the default console palette.
// User text mirrors to COM1 immediately, then queues expensive screen drawing
// through the passive debug-console worker when that path is available.
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
    if (!buf || len == 0) return;

    bool queueConsole = false;
    bool writeConsoleSync = false;
    bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer();

    uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
        KiDebugSerialWriteLocked(buf, len);

    if (PrintChar) {
        if (!KiPanicIsActive() && canQueueScreen) {
            KiDebugScreenAppendLocked(buf, len);
            queueConsole = true;
        } else {
            writeConsoleSync = true;
        }
    }
    KeReleaseSpinlock(&gDebugPrintLock, flags);

    if (queueConsole)
        KiDebugQueueConsoleWork();
    else if (writeConsoleSync)
        KiDebugConsoleWrite(buf, len);
}

// KeBugCheckProcess - user-mode fault handler

// Resolves the faulting RIP to a module name, prints a yellow crash banner,
// kills the offending process, then exits the current thread so the scheduler
// can continue running everything else.

// Return the short exception name for a CPU exception vector.
// Keep this table complete for vectors 0..31 so faults never collapse into
// a vague "general protection" bucket just because the IDT entry existed
// first or the crash printer lacked a name.
static const char* KiFaultVectorName(int vector) {
    switch (vector) {
        case 0:  return "#DE Divide Error";
        case 1:  return "#DB Debug";
        case 2:  return "NMI Interrupt";
        case 3:  return "#BP Breakpoint";
        case 4:  return "#OF Overflow";
        case 5:  return "#BR Bound Range Exceeded";
        case 6:  return "#UD Invalid Opcode";
        case 7:  return "#NM Device Not Available";
        case 8:  return "#DF Double Fault";
        case 9:  return "Coprocessor Segment Overrun";
        case 10: return "#TS Invalid TSS";
        case 11: return "#NP Segment Not Present";
        case 12: return "#SS Stack-Segment Fault";
        case 13: return "#GP General Protection";
        case 14: return "#PF Page Fault";
        case 15: return "Reserved Exception 15";
        case 16: return "#MF x87 Floating-Point Exception";
        case 17: return "#AC Alignment Check";
        case 18: return "#MC Machine Check";
        case 19: return "#XM SIMD Floating-Point Exception";
        case 20: return "#VE Virtualization Exception";
        case 21: return "#CP Control Protection Exception";
        case 22: return "Reserved Exception 22";
        case 23: return "Reserved Exception 23";
        case 24: return "Reserved Exception 24";
        case 25: return "Reserved Exception 25";
        case 26: return "Reserved Exception 26";
        case 27: return "Reserved Exception 27";
        case 28: return "#HV Hypervisor Injection Exception";
        case 29: return "#VC VMM Communication Exception";
        case 30: return "#SX Security Exception";
        case 31: return "Reserved Exception 31";
        default: return "Unknown Exception";
    }
}

// Implements the fault Has Error Code operation.
static int KiFaultHasErrorCode(int vector) {
    switch (vector) {
        case 8:   // #DF
        case 10:  // #TS
        case 11:  // #NP
        case 12:  // #SS
        case 13:  // #GP
        case 14:  // #PF
        case 17:  // #AC
        case 21:  // #CP
        case 29:  // #VC
        case 30:  // #SX
            return 1;
        default:
            return 0;
    }
}

// Walk the process module list to find which image owns an address.
// Returns the module name string, or "unknown" if the address is outside all
// known image ranges. Base/size are optional and are useful for offset dumps.
static const char* KiResolveModuleEx(PROCESS* proc, uintptr_t addr,
                                     uintptr_t* baseOut, size_t* sizeOut) {
    if (baseOut) *baseOut = 0;
    if (sizeOut) *sizeOut = 0;

    if (!proc) return "unknown";

    if (proc->ImageBase && proc->ImageSize &&
        addr >= proc->ImageBase && addr < proc->ImageBase + proc->ImageSize) {
        if (baseOut) *baseOut = proc->ImageBase;
        if (sizeOut) *sizeOut = proc->ImageSize;
        return proc->Name;
    }

    for (uint32_t i = 0; i < proc->ModuleCount; i++) {
        uintptr_t base = proc->Modules[i].Base;
        size_t    size = proc->Modules[i].Size;
        if (base && size && addr >= base && addr < base + size) {
            if (baseOut) *baseOut = base;
            if (sizeOut) *sizeOut = size;
            return proc->Modules[i].Name;
        }
    }

    return "unknown";
}

// Implements the resolve Module operation.
static const char* KiResolveModule(PROCESS* proc, uintptr_t addr) {
    return KiResolveModuleEx(proc, addr, (uintptr_t*)0, (size_t*)0);
}

// Prints address.
static void KiPrintAddress(PROCESS* proc, const char* label, uintptr_t addr) {
    uintptr_t base = 0;
    size_t size = 0;
    const char* mod = KiResolveModuleEx(proc, addr, &base, &size);

    if (base) {
        printf("%s%016llx <%s+0x%016llX>\n",
               label,
               (unsigned long long)addr,
               mod,
               (unsigned long long)(addr - base));
    } else {
        printf("%s%016llx <unknown>\n",
               label,
               (unsigned long long)addr);
    }
}

// Implements the user Read U64 operation.
static int KiUserReadU64(uintptr_t addr, uint64_t* out) {
    if (!out) return 0;
    if (!addr) return 0;
    if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(uint64_t))))
        return 0;
    *out = *(const uint64_t*)addr;
    return 1;
}

// Implements the user Read U8 operation.
static int KiUserReadU8(uintptr_t addr, uint8_t* out) {
    if (!out) return 0;
    if (!addr) return 0;
    if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(uint8_t))))
        return 0;
    *out = *(const uint8_t*)addr;
    return 1;
}

// Dumps bytes.
static void KiDumpBytes(PROCESS* proc, const char* label, uintptr_t center) {
    if (!center) {
        printf("%s: <null>\n", label);
        return;
    }

    printf("%s around ", label);
    KiPrintAddress(proc, "", center);

    uintptr_t start = center >= 16 ? center - 16 : 0;
    for (int row = 0; row < 2; row++) {
        uintptr_t addr = start + (uintptr_t)row * 16u;
        printf("  %016llx:", (unsigned long long)addr);
        for (int i = 0; i < 16; i++) {
            uint8_t b = 0;
            if (KiUserReadU8(addr + (uintptr_t)i, &b))
                printf(" %02x", b);
            else
                printf(" ??");
        }
        printf("\n");
    }
}

// Implements the user Read I32 operation.
static int KiUserReadI32(uintptr_t addr, int32_t* out) {
    if (!out) return 0;
    if (!addr) return 0;
    if (!KSUCCESS(MmProbeUserRead((const void*)addr, sizeof(int32_t))))
        return 0;
    *out = *(const int32_t*)addr;
    return 1;
}

// Dumps indirect Call Slot.
static void KiDumpIndirectCallSlot(PROCESS* proc, uintptr_t ret) {
    if (!ret) return;

    // Most x64 import/function-pointer calls are one of:
    //   FF 15 disp32       call qword ptr [rip+disp32]
    //   48 FF 15 disp32    rex.w call qword ptr [rip+disp32]
    // The saved return address points immediately after the instruction.
    uintptr_t insn = 0;
    int32_t disp = 0;
    uint8_t b0 = 0, b1 = 0, b2 = 0;

    if (KiUserReadU8(ret - 6, &b0) &&
        KiUserReadU8(ret - 5, &b1) &&
        b0 == 0xff && b1 == 0x15 &&
        // Implements the user Read I32 operation.
        KiUserReadI32(ret - 4, &disp)) {
        insn = ret - 6;
    } else if (KiUserReadU8(ret - 7, &b0) &&
               KiUserReadU8(ret - 6, &b1) &&
               KiUserReadU8(ret - 5, &b2) &&
               b0 == 0x48 && b1 == 0xff && b2 == 0x15 &&
               // Implements the user Read I32 operation.
               KiUserReadI32(ret - 4, &disp)) {
        insn = ret - 7;
    } else {
        printf("No RIP-relative indirect call pattern found before return address.\n");
        return;
    }

    uintptr_t slot = ret + (intptr_t)disp;
    uint64_t target = 0;

    printf("Decoded null indirect call:\n");
    KiPrintAddress(proc, "  insn=", insn);
    KiPrintAddress(proc, "  slot=", slot);

    if (KiUserReadU64(slot, &target)) {
        KiPrintAddress(proc, "  *slot=", (uintptr_t)target);
    } else {
        printf("  *slot=<unreadable>\n");
    }

    KiDumpBytes(proc, "Bytes near indirect slot", slot);
}

// Dumps modules.
static void KiDumpModules(PROCESS* proc) {
    if (!proc) return;

    printf("Images:\n");
    if (proc->ImageBase && proc->ImageSize) {
        printf("  %-31s %016llx-%016llx size=%llx\n",
               proc->Name,
               (unsigned long long)proc->ImageBase,
               (unsigned long long)(proc->ImageBase + proc->ImageSize),
               (unsigned long long)proc->ImageSize);
    }

    for (uint32_t i = 0; i < proc->ModuleCount; i++) {
        uintptr_t base = proc->Modules[i].Base;
        size_t size = proc->Modules[i].Size;
        printf("  %-31s %016llx-%016llx size=%llx\n",
               proc->Modules[i].Name,
               (unsigned long long)base,
               (unsigned long long)(base + size),
               (unsigned long long)size);
    }
}

// Dumps user Stack.
static void KiDumpUserStack(PROCESS* proc, uintptr_t rsp) {
    printf("User stack @ %016llx:\n", (unsigned long long)rsp);
    if (!rsp) {
        printf("  <no user RSP captured>\n");
        return;
    }

    for (int i = 0; i < 12; i++) {
        uintptr_t slot = rsp + (uintptr_t)i * 8u;
        uint64_t value = 0;
        if (KiUserReadU64(slot, &value)) {
            uintptr_t base = 0;
            size_t size = 0;
            const char* mod = KiResolveModuleEx(proc, (uintptr_t)value, &base, &size);
            if (base) {
                printf("  [%02d] %016llx : %016llx <%s+0x%016llX>\n",
                       i,
                       (unsigned long long)slot,
                       (unsigned long long)value,
                       mod,
                       (unsigned long long)((uintptr_t)value - base));
            } else {
                printf("  [%02d] %016llx : %016llx\n",
                       i,
                       (unsigned long long)slot,
                       (unsigned long long)value);
            }
        } else {
            printf("  [%02d] %016llx : <unreadable>\n",
                   i,
                   (unsigned long long)slot);
        }
    }
}

// Dumps fault Frame.
static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
    if (!f) return;

    printf("Regs:\n");
    printf("  RAX=%016llx RBX=%016llx RCX=%016llx RDX=%016llx\n",
           (unsigned long long)f->Rax, (unsigned long long)f->Rbx,
           (unsigned long long)f->Rcx, (unsigned long long)f->Rdx);
    printf("  RSI=%016llx RDI=%016llx RBP=%016llx RSP=%016llx\n",
           (unsigned long long)f->Rsi, (unsigned long long)f->Rdi,
           (unsigned long long)f->Rbp, (unsigned long long)f->Rsp);
    printf("  R8 =%016llx R9 =%016llx R10=%016llx R11=%016llx\n",
           (unsigned long long)f->R8,  (unsigned long long)f->R9,
           (unsigned long long)f->R10, (unsigned long long)f->R11);
    printf("  R12=%016llx R13=%016llx R14=%016llx R15=%016llx\n",
           (unsigned long long)f->R12, (unsigned long long)f->R13,
           (unsigned long long)f->R14, (unsigned long long)f->R15);
    printf("  CS=%04llx SS=%04llx RFLAGS=%016llx\n",
           (unsigned long long)f->Cs,
           (unsigned long long)f->Ss,
           (unsigned long long)f->Rflags);

    if (f->Rip == 0) {
        uint64_t ret = 0;
        if (KiUserReadU64((uintptr_t)f->Rsp, &ret)) {
            printf("Likely null indirect call. Return address from [RSP]:\n");
            KiPrintAddress(proc, "  ret=", (uintptr_t)ret);
            KiDumpBytes(proc, "Bytes near return address", (uintptr_t)ret);
            KiDumpIndirectCallSlot(proc, (uintptr_t)ret);
        } else {
            printf("Likely null indirect call, but [RSP] is unreadable.\n");
        }
    } else {
        KiDumpBytes(proc, "Bytes near RIP", (uintptr_t)f->Rip);
    }

    KiDumpUserStack(proc, (uintptr_t)f->Rsp);
}

// Prints crash Char.
static void KiPrintCrashChar(char c) {
    if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
        KiDebugSerialWriteChar(c);
    if (PrintChar)
        PrintChar(c, 0xFFAA00, 0x000000);
}

static KSTATUS KiFaultExitStatus(int vector) {
    switch (vector) {
        case 0:  return STATUS_PROCESS_DIVIDE_BY_ZERO;
        case 3:  return STATUS_PROCESS_BREAKPOINT;
        case 6:  return STATUS_PROCESS_ILLEGAL_OPCODE;
        case 13: return STATUS_PROCESS_GENERAL_PROTECTION;
        case 14: return STATUS_PROCESS_PAGE_FAULT;
        case 17: return STATUS_PROCESS_ALIGNMENT_FAULT;
        default: return STATUS_PROCESS_UNKNOWN_FAULT;
    }
}

static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
                               uintptr_t errCode, uintptr_t cr2) {
    const char* faultName = KiFaultVectorName(vector);
    const char* procName  = proc ? proc->Name  : "unknown";
    uint32_t    pid       = proc ? proc->ProcessId : 0;
    const char* modName   = proc ? KiResolveModule(proc, rip) : "unknown";

    static const char banner[] = "\n*** APPLICATION CRASH ***\n";
    for (const char* p = banner; *p; p++)
        KiPrintCrashChar(*p);

    printf_set_putchar(KiPrintCrashChar);

    if (vector == 14)
        printf("Process: %s (PID %u) | Fault: %s\n"
               "Module:  %s | RIP: %016llx | Code: %04llx | CR2: %016llx\n",
               procName, pid, faultName,
               modName,
               (unsigned long long)rip,
               (unsigned long long)errCode,
               (unsigned long long)cr2);
    else
        printf("Process: %s (PID %u) | Fault: %s\n"
               "Module:  %s | RIP: %016llx | Code: %04llx\n",
               procName, pid, faultName,
               modName,
               (unsigned long long)rip,
               (unsigned long long)errCode);
}

// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
    printf_set_putchar(KiKernelPutChar);

    int exitCode = (int)exitStatus;
    if (proc)   (void)KeTerminateProcess(proc, exitCode);
    if (thread) KeExitThread(thread, exitCode);

    // KeExitThread normally switches away and never resumes this context.  If
    // a fault path re-enters here during scheduler bring-up or with no runnable
    // peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
    // dead thread to the scheduler with interrupts enabled so waiters such as
    // the foreground shell can observe the process exit event.
    while (true) {
        _enable();
        KeYield();
        __halt();
    }
}

_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
    THREAD*  thread = KeGetCurrentThread();
    PROCESS* proc   = thread ? thread->Owner : (PROCESS*)0;

    KiPrintCrashHeader(proc, vector, rip, errCode, cr2);
    KiDumpModules(proc);

    KiFinishFaultedProcess(proc, thread, KiFaultExitStatus(vector));
}

// Dispatches a CPU fault frame and returns only when a user #PF was repaired.
__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
    // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
    // the interrupted context on a secondary CPU once another CPU owns panic.
    if (KiPanicIsActive())
        KiPanicSecondaryHalt();

    THREAD*  thread = KeGetCurrentThread();
    PROCESS* proc   = thread ? thread->Owner : (PROCESS*)0;

    if (!frame || frame->Magic != 0x3146544C5541464BULL) {
        KePanic(KE_PANIC_CODE_FAULT, "PANIC: bad KFAULT_FRAME %016llx\n",
                (unsigned long long)(uintptr_t)frame);
    }

    if (frame->Vector == 14) {
        KSTATUS handled = MmVadResolvePageFaultByAddress((uintptr_t)frame->Cr2, frame->ErrorCode);
        if (KSUCCESS(handled))
            return STATUS_SUCCESS;
    }

    if ((frame->Cs & 3) == 0) {
        KiBeginFatalReport();

        int vector = (int)frame->Vector;
        const char* name = KiFaultVectorName(vector);

        DbgPrint("FAULT: kernel exception vector=%d name=%s rip=%016llx rsp=%016llx err=%04llx cr2=%016llx thread=%p pid=%u proc=%s\n",
                 vector,
                 name,
                 (unsigned long long)frame->Rip,
                 (unsigned long long)frame->Rsp,
                 (unsigned long long)frame->ErrorCode,
                 (unsigned long long)frame->Cr2,
                 thread,
                 proc ? proc->ProcessId : 0,
                 proc ? proc->Name : "<none>");
        KiDumpFaultFrame(proc, frame);
        if (vector == 14) {
            MmDebugDescribeAddress((uintptr_t)frame->Cr2);
            if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
            MmVadDebugDumpProcess(proc);
        }

        if (vector == 14) {
            KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | RSP: %016llx | Code: %04llx | CR2: %016llx\n",
                    name,
                    (unsigned long long)frame->Rip,
                    (unsigned long long)frame->Rsp,
                    (unsigned long long)frame->ErrorCode,
                    (unsigned long long)frame->Cr2);
        } else if (KiFaultHasErrorCode(vector)) {
            const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip;
            KePanic(KE_PANIC_CODE_FAULT,
                    "PANIC: %s | RIP: %016llx | Code: %04llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n",
                    name,
                    (unsigned long long)frame->Rip,
                    (unsigned long long)frame->ErrorCode,
                    (unsigned long long)frame->Cs,
                    (unsigned long long)frame->Rflags,
                    (unsigned long long)frame->Rsp,
                    ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]);
        } else {
            const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip;
            KePanic(KE_PANIC_CODE_FAULT,
                    "PANIC: %s | RIP: %016llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n",
                    name,
                    (unsigned long long)frame->Rip,
                    (unsigned long long)frame->Cs,
                    (unsigned long long)frame->Rflags,
                    (unsigned long long)frame->Rsp,
                    ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]);
        }
    }

    KiPrintCrashHeader(proc, (int)frame->Vector,
                       (uintptr_t)frame->Rip,
                       (uintptr_t)frame->ErrorCode,
                       (uintptr_t)frame->Cr2);
    KiDumpFaultFrame(proc, frame);
    KiDumpModules(proc);

    KiFinishFaultedProcess(proc, thread, KiFaultExitStatus((int)frame->Vector));
}

// KeFaultDispatch - single C entry point for all three IDT fault stubs.

// The asm stubs (ke_syscall.asm) collect the CPU exception frame values and
// call here. We check the CS RPL to decide: kernel fault → panic and halt,
// user fault → KeBugCheckProcess (kill process, resume scheduler).
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
                                                     uintptr_t errCode, uintptr_t cr2,
                                                     uintptr_t cs) {
    KFAULT_FRAME frame;
    memset(&frame, 0, sizeof(frame));
    frame.Magic = 0x3146544C5541464BULL;
    frame.Vector = (uint64_t)vector;
    frame.ErrorCode = errCode;
    frame.Cr2 = cr2;
    frame.Rip = rip;
    frame.Cs = cs;
    (void)KeFaultDispatchFrame(&frame);
    while (1) __halt();
}