nt-lite/code/kernel/ke_exception.c

// Kernel exception classification and recoverable fault policy.

#include <kernel/ke_exception.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
#include <string.h>


#define KI_EXCEPTION_MAX_DISPATCH_DEPTH KE_USER_EXCEPTION_MAX_DEPTH
#define KI_EXCEPTION_USER_ENTRY_SHADOW 40u
#define KI_EXCEPTION_FRAME_ALIGNMENT 16u
#define KI_USER_CS 0x1BULL
#define KI_USER_SS 0x23ULL
#define KI_RFLAGS_FIXED 0x2ULL
#define KI_RFLAGS_IOPL 0x3000ULL
#define KI_RFLAGS_NT 0x4000ULL
#define KI_RFLAGS_VM 0x20000ULL

// Returns whether one user address can be used as a control-flow target or pointer.
static bool KiExceptionUserAddressValid(uintptr_t address) {
    return address >= USER_BASE && address < USER_LIMIT;
}

// Returns whether one byte range stays wholly inside user address space.
static bool KiExceptionUserRangeValid(uintptr_t base, size_t size) {
    if (size == 0)
        return false;
    if (base < USER_BASE || base >= USER_LIMIT)
        return false;
    if (size > (size_t)(USER_LIMIT - base))
        return false;
    return true;
}

// Returns whether a saved user RFLAGS value can be restored safely.
static bool KiExceptionUserRflagsValid(uint64_t rflags) {
    if ((rflags & KI_RFLAGS_FIXED) == 0)
        return false;
    if ((rflags & (KI_RFLAGS_IOPL | KI_RFLAGS_NT | KI_RFLAGS_VM)) != 0)
        return false;
    return true;
}

// Rounds an address down to the alignment required by the user-mode ABI.
static uintptr_t KiExceptionAlignDown(uintptr_t value, uintptr_t alignment) {
    return value & ~(alignment - 1u);
}

// Terminates the current process from exception-return policy code.
static _Noreturn void KiTerminateCurrentExceptionProcess(KSTATUS exitStatus) {
    THREAD* thread = KeGetCurrentThread();
    PROCESS* process = thread ? thread->Owner : (PROCESS*)0;
    int32_t exitCode = (int32_t)exitStatus;

    if (process)
        (void)KeTerminateProcess(process, exitCode);
    if (thread)
        KeExitThread(thread, exitCode);

    while (true) {
        KeYield();
    }
}

// Records a delivered exception against the process that owns user policy.
static void KiRecordExceptionDelivery(PROCESS* process, const SYS_EXCEPTION_RECORD* record,
                                      uint32_t nestedDepth, bool userRaised) {
    if (!process || !record)
        return;

    uintptr_t flags = KeAcquireSpinlock(&process->ExceptionLock);
    process->ExceptionDeliveredCount++;
    if (userRaised)
        process->ExceptionUserRaisedCount++;
    if (nestedDepth > process->ExceptionMaxNestedDispatch)
        process->ExceptionMaxNestedDispatch = nestedDepth;
    process->LastExceptionCode = record->Code;
    process->LastExceptionAddress = record->ExceptionAddress;
    process->LastExceptionFaultAddress = record->FaultAddress;
    KeReleaseSpinlock(&process->ExceptionLock, flags);
}

// Records one completed exception continuation.
static void KiRecordExceptionContinued(PROCESS* process) {
    if (!process)
        return;
    uintptr_t flags = KeAcquireSpinlock(&process->ExceptionLock);
    process->ExceptionContinuedCount++;
    KeReleaseSpinlock(&process->ExceptionLock, flags);
}

// Records that user policy declined or failed to handle an exception.
static void KiRecordExceptionUnhandled(PROCESS* process) {
    if (!process)
        return;
    uintptr_t flags = KeAcquireSpinlock(&process->ExceptionLock);
    process->ExceptionUnhandledCount++;
    KeReleaseSpinlock(&process->ExceptionLock, flags);
}

// Records a malformed or forbidden exception continuation request.
static void KiRecordExceptionInvalidContinue(PROCESS* process) {
    if (!process)
        return;
    uintptr_t flags = KeAcquireSpinlock(&process->ExceptionLock);
    process->ExceptionInvalidContinueCount++;
    KeReleaseSpinlock(&process->ExceptionLock, flags);
}


// Returns the process exit status for one unhandled native exception record.
static KSTATUS KiExceptionUnhandledExitStatus(const SYS_EXCEPTION_RECORD* record) {
    if (!record)
        return NTLSTATUS_PROCESS_UNKNOWN_FAULT;

    switch (record->Code) {
        case SYS_EXCEPTION_CODE_INT_DIVIDE_BY_ZERO:
            return NTLSTATUS_PROCESS_DIVIDE_BY_ZERO;
        case SYS_EXCEPTION_CODE_BREAKPOINT:
            return NTLSTATUS_PROCESS_BREAKPOINT;
        case SYS_EXCEPTION_CODE_ILLEGAL_INSTRUCTION:
            return NTLSTATUS_PROCESS_ILLEGAL_OPCODE;
        case SYS_EXCEPTION_CODE_PRIVILEGED_INSTRUCTION:
            return NTLSTATUS_PROCESS_GENERAL_PROTECTION;
        case SYS_EXCEPTION_CODE_ACCESS_VIOLATION:
            return NTLSTATUS_PROCESS_PAGE_FAULT;
        case SYS_EXCEPTION_CODE_DATATYPE_MISALIGNMENT:
            return NTLSTATUS_PROCESS_ALIGNMENT_FAULT;
        case SYS_EXCEPTION_CODE_USER_RAISED:
            return NTLSTATUS_PROCESS_USER_EXCEPTION;
        default:
            return NTLSTATUS_PROCESS_UNKNOWN_FAULT;
    }
}

// Maps an architectural vector to the native structured-exception code.
static uint32_t KiExceptionCodeFromVector(uint32_t vector) {
    switch (vector) {
        case 0:  return SYS_EXCEPTION_CODE_INT_DIVIDE_BY_ZERO;
        case 1:  return SYS_EXCEPTION_CODE_SINGLE_STEP;
        case 3:  return SYS_EXCEPTION_CODE_BREAKPOINT;
        case 4:  return SYS_EXCEPTION_CODE_INT_OVERFLOW;
        case 5:  return SYS_EXCEPTION_CODE_ARRAY_BOUNDS_EXCEEDED;
        case 6:  return SYS_EXCEPTION_CODE_ILLEGAL_INSTRUCTION;
        case 13: return SYS_EXCEPTION_CODE_PRIVILEGED_INSTRUCTION;
        case 14: return SYS_EXCEPTION_CODE_ACCESS_VIOLATION;
        case 17: return SYS_EXCEPTION_CODE_DATATYPE_MISALIGNMENT;
        default: return SYS_EXCEPTION_CODE_UNKNOWN;
    }
}

// Fills the user-mode context record from a captured CPU fault frame.
static void KiCaptureUserContextFromFault(const KFAULT_FRAME* frame, SYS_USER_CONTEXT* context) {
    memset(context, 0, sizeof(*context));
    context->Size = sizeof(*context);
    context->Version = SYS_EXCEPTION_ABI_VERSION;
    context->Flags = SYS_CONTEXT_FLAG_ALL;
    context->Rip = frame->Rip;
    context->Rsp = frame->Rsp;
    context->Rflags = frame->Rflags;
    context->Rax = frame->Rax;
    context->Rcx = frame->Rcx;
    context->Rdx = frame->Rdx;
    context->Rbx = frame->Rbx;
    context->Rbp = frame->Rbp;
    context->Rsi = frame->Rsi;
    context->Rdi = frame->Rdi;
    context->R8 = frame->R8;
    context->R9 = frame->R9;
    context->R10 = frame->R10;
    context->R11 = frame->R11;
    context->R12 = frame->R12;
    context->R13 = frame->R13;
    context->R14 = frame->R14;
    context->R15 = frame->R15;
    context->SegCs = (uint32_t)frame->Cs;
    context->SegSs = (uint32_t)frame->Ss;
}

// Fills the user-mode context record from the current syscall return frame.
static void KiCaptureUserContextFromTrap(const KTRAP_FRAME* frame, SYS_USER_CONTEXT* context) {
    memset(context, 0, sizeof(*context));
    context->Size = sizeof(*context);
    context->Version = SYS_EXCEPTION_ABI_VERSION;
    context->Flags = SYS_CONTEXT_FLAG_ALL;
    context->Rip = frame->Rip;
    context->Rsp = frame->Rsp;
    context->Rflags = frame->Rflags;
    context->Rax = NTLSTATUS_SUCCESS;
    context->Rcx = frame->Rcx;
    context->Rdx = frame->Rdx;
    context->Rbx = frame->Rbx;
    context->Rbp = frame->Rbp;
    context->Rsi = frame->Rsi;
    context->Rdi = frame->Rdi;
    context->R8 = frame->R8;
    context->R9 = frame->R9;
    context->R10 = frame->R10;
    context->R11 = frame->R11;
    context->R12 = frame->R12;
    context->R13 = frame->R13;
    context->R14 = frame->R14;
    context->R15 = frame->R15;
    context->SegCs = (uint32_t)frame->Cs;
    context->SegSs = (uint32_t)frame->Ss;
}

// Fills the native exception record from an architectural user fault.
static void KiCaptureExceptionRecordFromFault(const KFAULT_FRAME* frame, SYS_EXCEPTION_RECORD* record) {
    memset(record, 0, sizeof(*record));
    record->Size = sizeof(*record);
    record->Version = SYS_EXCEPTION_ABI_VERSION;
    record->Code = KiExceptionCodeFromVector((uint32_t)frame->Vector);
    record->Flags = SYS_EXCEPTION_FLAG_FIRST_CHANCE;
    if ((frame->Vector == 8) || (frame->Vector == 18))
        record->Flags |= SYS_EXCEPTION_FLAG_NONCONTINUABLE;
    record->ExceptionAddress = (uintptr_t)frame->Rip;
    record->FaultAddress = frame->Vector == 14 ? (uintptr_t)frame->Cr2 : (uintptr_t)frame->Rip;
    record->ErrorCode = frame->ErrorCode;

    if (frame->Vector == 14) {
        uint64_t accessKind = 0;
        if (frame->ErrorCode & 0x10u)
            accessKind = 8;
        else if (frame->ErrorCode & 0x2u)
            accessKind = 1;
        record->ParameterCount = 2;
        record->Parameters[0] = accessKind;
        record->Parameters[1] = frame->Cr2;
    }
}

// Captures a stable process dispatcher snapshot outside the process lock.
static KSTATUS KiSnapshotExceptionDispatcher(PROCESS* process,
                                             SYS_EXCEPTION_DISPATCHER_REQUEST* out) {
    if (!process || !out)
        return NTLSTATUS_INVALID_PARAMETER;

    memset(out, 0, sizeof(*out));
    out->Size = sizeof(*out);

    uintptr_t flags = KeAcquireSpinlock(&process->ExceptionLock);
    out->Flags = process->ExceptionDispatchFlags;
    out->DispatcherRoutine = (SYS_USER_EXCEPTION_DISPATCHER)process->ExceptionDispatcher;
    out->DispatcherContext = (void*)process->ExceptionDispatcherContext;
    out->ExceptionStackBase = (void*)process->ExceptionStackBase;
    out->ExceptionStackSize = process->ExceptionStackSize;
    KeReleaseSpinlock(&process->ExceptionLock, flags);

    if (!out->DispatcherRoutine)
        return NTLSTATUS_NOT_FOUND;
    return NTLSTATUS_SUCCESS;
}

// Selects the stack top used to inject one exception dispatcher frame.
static KSTATUS KiSelectExceptionStackTop(THREAD* thread,
                                         const SYS_EXCEPTION_DISPATCHER_REQUEST* dispatcher,
                                         const KFAULT_FRAME* fault,
                                         uintptr_t* stackTopOut) {
    if (!thread || !dispatcher || !fault || !stackTopOut)
        return NTLSTATUS_INVALID_PARAMETER;

    if (dispatcher->ExceptionStackBase) {
        uintptr_t base = (uintptr_t)dispatcher->ExceptionStackBase;
        uintptr_t top = base + dispatcher->ExceptionStackSize;
        uintptr_t stride = KiExceptionAlignDown(sizeof(SYS_EXCEPTION_FRAME) +
                                                KI_EXCEPTION_USER_ENTRY_SHADOW +
                                                KI_EXCEPTION_FRAME_ALIGNMENT - 1u,
                                                KI_EXCEPTION_FRAME_ALIGNMENT);
        uintptr_t needed = (uintptr_t)(thread->UserExceptionDepth + 1u) * stride;
        if (needed > dispatcher->ExceptionStackSize)
            return NTLSTATUS_BUFFER_TOO_SMALL;
        *stackTopOut = top - (uintptr_t)thread->UserExceptionDepth * stride;
        return NTLSTATUS_SUCCESS;
    }

    if (!KiExceptionUserAddressValid((uintptr_t)fault->Rsp))
        return NTLSTATUS_ACCESS_DENIED;
    *stackTopOut = (uintptr_t)fault->Rsp;
    return NTLSTATUS_SUCCESS;
}

// Selects an exception stack using an explicit user RSP.
static KSTATUS KiSelectExceptionStackTopFromRsp(THREAD* thread,
                                                const SYS_EXCEPTION_DISPATCHER_REQUEST* dispatcher,
                                                uintptr_t rsp,
                                                uintptr_t* stackTopOut) {
    if (!thread || !dispatcher || !stackTopOut)
        return NTLSTATUS_INVALID_PARAMETER;

    if (dispatcher->ExceptionStackBase) {
        uintptr_t base = (uintptr_t)dispatcher->ExceptionStackBase;
        uintptr_t top = base + dispatcher->ExceptionStackSize;
        uintptr_t stride = KiExceptionAlignDown(sizeof(SYS_EXCEPTION_FRAME) +
                                                KI_EXCEPTION_USER_ENTRY_SHADOW +
                                                KI_EXCEPTION_FRAME_ALIGNMENT - 1u,
                                                KI_EXCEPTION_FRAME_ALIGNMENT);
        uintptr_t needed = (uintptr_t)(thread->UserExceptionDepth + 1u) * stride;
        if (needed > dispatcher->ExceptionStackSize)
            return NTLSTATUS_BUFFER_TOO_SMALL;
        *stackTopOut = top - (uintptr_t)thread->UserExceptionDepth * stride;
        return NTLSTATUS_SUCCESS;
    }

    if (!KiExceptionUserAddressValid(rsp))
        return NTLSTATUS_ACCESS_DENIED;
    *stackTopOut = rsp;
    return NTLSTATUS_SUCCESS;
}

// Copies the native exception delivery frame to the selected user stack.
static KSTATUS KiBuildUserExceptionFrame(THREAD* thread,
                                         const SYS_EXCEPTION_DISPATCHER_REQUEST* dispatcher,
                                         KFAULT_FRAME* fault,
                                         uintptr_t* frameAddressOut,
                                         uintptr_t* entryRspOut) {
    uintptr_t stackTop = 0;
    KSTATUS status = KiSelectExceptionStackTop(thread, dispatcher, fault, &stackTop);
    if (!KSUCCESS(status))
        return status;

    uintptr_t frameAddress = KiExceptionAlignDown(stackTop - sizeof(SYS_EXCEPTION_FRAME),
                                                  KI_EXCEPTION_FRAME_ALIGNMENT);
    uintptr_t entryRsp = frameAddress - KI_EXCEPTION_USER_ENTRY_SHADOW;
    if (!KiExceptionUserRangeValid(entryRsp,
                                   KI_EXCEPTION_USER_ENTRY_SHADOW + sizeof(SYS_EXCEPTION_FRAME)))
        return NTLSTATUS_ACCESS_DENIED;

    SYS_EXCEPTION_FRAME userFrame;
    memset(&userFrame, 0, sizeof(userFrame));
    userFrame.Size = sizeof(userFrame);
    userFrame.Version = SYS_EXCEPTION_ABI_VERSION;
    KiCaptureExceptionRecordFromFault(fault, &userFrame.ExceptionRecord);
    KiCaptureUserContextFromFault(fault, &userFrame.ContextRecord);
    userFrame.DispatcherContext = dispatcher->DispatcherContext;

    uint8_t shadow[KI_EXCEPTION_USER_ENTRY_SHADOW];
    memset(shadow, 0, sizeof(shadow));
    status = CopyToUser((void*)entryRsp, shadow, sizeof(shadow));
    if (!KSUCCESS(status))
        return status;

    status = CopyToUser((void*)frameAddress, &userFrame, sizeof(userFrame));
    if (!KSUCCESS(status))
        return status;

    *frameAddressOut = frameAddress;
    *entryRspOut = entryRsp;
    return NTLSTATUS_SUCCESS;
}

// Copies an already-classified user exception frame to the selected user stack.
static KSTATUS KiBuildUserRaisedExceptionFrame(THREAD* thread,
                                               const SYS_EXCEPTION_DISPATCHER_REQUEST* dispatcher,
                                               const KTRAP_FRAME* trapFrame,
                                               const SYS_EXCEPTION_RECORD* record,
                                               uintptr_t* frameAddressOut,
                                               uintptr_t* entryRspOut) {
    uintptr_t stackTop = 0;
    KSTATUS status = KiSelectExceptionStackTopFromRsp(thread, dispatcher, trapFrame->Rsp, &stackTop);
    if (!KSUCCESS(status))
        return status;

    uintptr_t frameAddress = KiExceptionAlignDown(stackTop - sizeof(SYS_EXCEPTION_FRAME),
                                                  KI_EXCEPTION_FRAME_ALIGNMENT);
    uintptr_t entryRsp = frameAddress - KI_EXCEPTION_USER_ENTRY_SHADOW;
    if (!KiExceptionUserRangeValid(entryRsp,
                                   KI_EXCEPTION_USER_ENTRY_SHADOW + sizeof(SYS_EXCEPTION_FRAME)))
        return NTLSTATUS_ACCESS_DENIED;

    SYS_EXCEPTION_FRAME userFrame;
    memset(&userFrame, 0, sizeof(userFrame));
    userFrame.Size = sizeof(userFrame);
    userFrame.Version = SYS_EXCEPTION_ABI_VERSION;
    userFrame.ExceptionRecord = *record;
    KiCaptureUserContextFromTrap(trapFrame, &userFrame.ContextRecord);
    userFrame.DispatcherContext = dispatcher->DispatcherContext;

    uint8_t shadow[KI_EXCEPTION_USER_ENTRY_SHADOW];
    memset(shadow, 0, sizeof(shadow));
    status = CopyToUser((void*)entryRsp, shadow, sizeof(shadow));
    if (!KSUCCESS(status))
        return status;

    status = CopyToUser((void*)frameAddress, &userFrame, sizeof(userFrame));
    if (!KSUCCESS(status))
        return status;

    *frameAddressOut = frameAddress;
    *entryRspOut = entryRsp;
    return NTLSTATUS_SUCCESS;
}

// Validates a user context before using it as an exception resume target.
static KSTATUS KiValidateContinueContext(const SYS_USER_CONTEXT* context) {
    if (!context)
        return NTLSTATUS_INVALID_PARAMETER;
    if (context->Size != sizeof(*context) || context->Version != SYS_EXCEPTION_ABI_VERSION)
        return NTLSTATUS_INVALID_PARAMETER;
    if (context->Flags != SYS_CONTEXT_FLAG_ALL)
        return NTLSTATUS_INVALID_PARAMETER;
    if (!KiExceptionUserAddressValid((uintptr_t)context->Rip) ||
        !KiExceptionUserAddressValid((uintptr_t)context->Rsp))
        return NTLSTATUS_ACCESS_DENIED;
    if (!KiExceptionUserRflagsValid(context->Rflags))
        return NTLSTATUS_ACCESS_DENIED;
    if (context->SegCs != KI_USER_CS || context->SegSs != KI_USER_SS)
        return NTLSTATUS_ACCESS_DENIED;
    return NTLSTATUS_SUCCESS;
}

// Applies a validated user context to the current syscall return trap frame.
static void KiApplyContinueContextToTrapFrame(KTRAP_FRAME* frame,
                                              const SYS_USER_CONTEXT* context) {
    frame->Rip = context->Rip;
    frame->Rsp = context->Rsp;
    frame->Rflags = context->Rflags;
    frame->Cs = context->SegCs;
    frame->Ss = context->SegSs;
    frame->Rax = context->Rax;
    frame->Rcx = context->Rcx;
    frame->Rdx = context->Rdx;
    frame->Rbx = context->Rbx;
    frame->Rbp = context->Rbp;
    frame->Rsi = context->Rsi;
    frame->Rdi = context->Rdi;
    frame->R8 = context->R8;
    frame->R9 = context->R9;
    frame->R10 = context->R10;
    frame->R11 = context->R11;
    frame->R12 = context->R12;
    frame->R13 = context->R13;
    frame->R14 = context->R14;
    frame->R15 = context->R15;
}

// Redirects a captured user fault to the registered native exception dispatcher.
KSTATUS KeDeliverUserException(KFAULT_FRAME* frame) {
    if (!frame || !KeFaultFrameIsValid(frame) || !KeFaultFrameFromUser(frame))
        return NTLSTATUS_INVALID_PARAMETER;

    THREAD* thread = KeGetCurrentThread();
    PROCESS* process = thread ? thread->Owner : (PROCESS*)0;
    if (!thread || !process)
        return NTLSTATUS_FAILURE;
    if (thread->UserExceptionDepth >= KI_EXCEPTION_MAX_DISPATCH_DEPTH)
        return NTLSTATUS_PROCESS_UNKNOWN_FAULT;

    SYS_EXCEPTION_DISPATCHER_REQUEST dispatcher;
    KSTATUS status = KiSnapshotExceptionDispatcher(process, &dispatcher);
    if (!KSUCCESS(status))
        return status;

    uintptr_t frameAddress = 0;
    uintptr_t entryRsp = 0;
    status = KiBuildUserExceptionFrame(thread, &dispatcher, frame, &frameAddress, &entryRsp);
    if (!KSUCCESS(status))
        return status;

    SYS_EXCEPTION_RECORD deliveredRecord;
    KiCaptureExceptionRecordFromFault(frame, &deliveredRecord);

    uint32_t depth = thread->UserExceptionDepth;
    if ((frame->Vector == 8) || (frame->Vector == 18))
        thread->UserExceptionNoncontinuableMask |= (1u << depth);
    thread->UserExceptionExitStatus[depth] = KiExceptionUnhandledExitStatus(&deliveredRecord);

    KiRecordExceptionDelivery(process, &deliveredRecord, depth + 1u, false);
    thread->UserExceptionDepth = depth + 1u;
    thread->UserExceptionFrameUserAddress = frameAddress;

    frame->Rip = (uint64_t)(uintptr_t)dispatcher.DispatcherRoutine;
    frame->Rsp = entryRsp;
    frame->Rcx = frameAddress;
    frame->Rdx = 0;
    frame->R8 = 0;
    frame->R9 = 0;
    frame->R10 = 0;
    frame->R11 = 0;
    frame->Rax = NTLSTATUS_SUCCESS;
    frame->Cs = KI_USER_CS;
    frame->Ss = KI_USER_SS;
    frame->Rflags |= KI_RFLAGS_FIXED;
    frame->Rflags &= ~(KI_RFLAGS_IOPL | KI_RFLAGS_NT | KI_RFLAGS_VM);
    return NTLSTATUS_SUCCESS;
}

// Raises a user-authored structured exception from the current syscall frame.
uint64_t KeRaiseUserException(const SYS_RAISE_EXCEPTION_REQUEST* request, const uint64_t* parameters) {
    if (!request)
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->Size != sizeof(*request))
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->Flags & ~SYS_RAISE_EXCEPTION_FLAG_NONCONTINUABLE)
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->ParameterCount > SYS_EXCEPTION_MAXIMUM_PARAMETERS)
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->ParameterCount && !parameters)
        return NTLSTATUS_INVALID_PARAMETER;

    THREAD* thread = KeGetCurrentThread();
    PROCESS* process = thread ? thread->Owner : (PROCESS*)0;
    if (!thread || !process)
        return NTLSTATUS_FAILURE;
    if (thread->UserExceptionDepth >= KI_EXCEPTION_MAX_DISPATCH_DEPTH)
        return NTLSTATUS_PROCESS_UNKNOWN_FAULT;

    KTRAP_FRAME* trapFrame = KeGetCurrentTrapFrame();
    if (!trapFrame || !KeTrapFrameFromUser(trapFrame) || trapFrame->Kind != KTRAP_KIND_SYSCALL)
        return NTLSTATUS_FAILURE;

    SYS_EXCEPTION_DISPATCHER_REQUEST dispatcher;
    KSTATUS status = KiSnapshotExceptionDispatcher(process, &dispatcher);
    if (!KSUCCESS(status))
        return status;

    SYS_EXCEPTION_RECORD record;
    memset(&record, 0, sizeof(record));
    record.Size = sizeof(record);
    record.Version = SYS_EXCEPTION_ABI_VERSION;
    record.Code = SYS_EXCEPTION_CODE_USER_RAISED;
    record.UserCode = request->UserCode;
    record.Flags = SYS_EXCEPTION_FLAG_FIRST_CHANCE;
    if (request->Flags & SYS_RAISE_EXCEPTION_FLAG_NONCONTINUABLE)
        record.Flags |= SYS_EXCEPTION_FLAG_NONCONTINUABLE;
    record.ExceptionAddress = (uintptr_t)trapFrame->Rip;
    record.FaultAddress = 0;
    record.ParameterCount = request->ParameterCount;
    for (uint32_t i = 0; i < request->ParameterCount; i++)
        record.Parameters[i] = parameters[i];

    uintptr_t frameAddress = 0;
    uintptr_t entryRsp = 0;
    status = KiBuildUserRaisedExceptionFrame(thread, &dispatcher, trapFrame, &record,
                                             &frameAddress, &entryRsp);
    if (!KSUCCESS(status))
        return status;

    uint32_t depth = thread->UserExceptionDepth;
    if (record.Flags & SYS_EXCEPTION_FLAG_NONCONTINUABLE)
        thread->UserExceptionNoncontinuableMask |= (1u << depth);
    thread->UserExceptionExitStatus[depth] = KiExceptionUnhandledExitStatus(&record);

    KiRecordExceptionDelivery(process, &record, depth + 1u, true);
    thread->UserExceptionDepth = depth + 1u;
    thread->UserExceptionFrameUserAddress = frameAddress;

    trapFrame->Rip = (uint64_t)(uintptr_t)dispatcher.DispatcherRoutine;
    trapFrame->Rsp = entryRsp;
    trapFrame->Rcx = frameAddress;
    trapFrame->Rdx = 0;
    trapFrame->R8 = 0;
    trapFrame->R9 = 0;
    trapFrame->R10 = 0;
    trapFrame->R11 = 0;
    trapFrame->Rax = NTLSTATUS_SUCCESS;
    trapFrame->Cs = KI_USER_CS;
    trapFrame->Ss = KI_USER_SS;
    trapFrame->Rflags |= KI_RFLAGS_FIXED;
    trapFrame->Rflags &= ~(KI_RFLAGS_IOPL | KI_RFLAGS_NT | KI_RFLAGS_VM);
    return NTLSTATUS_SUCCESS;
}

// Completes native exception dispatch by applying the user-selected disposition.
uint64_t KeContinueException(const SYS_CONTINUE_EXCEPTION_REQUEST* request) {
    if (!request)
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->Size != sizeof(*request) || request->Flags != SYS_EXCEPTION_CONTINUE_FLAG_NONE)
        return NTLSTATUS_INVALID_PARAMETER;

    THREAD* thread = KeGetCurrentThread();
    PROCESS* process = thread ? thread->Owner : (PROCESS*)0;
    if (!thread || thread->UserExceptionDepth == 0) {
        KiRecordExceptionInvalidContinue(process);
        return NTLSTATUS_INVALID_PARAMETER;
    }

    uint32_t activeDepth = thread->UserExceptionDepth - 1u;
    uint32_t activeBit = 1u << activeDepth;
    KSTATUS activeExitStatus = thread->UserExceptionExitStatus[activeDepth];
    if (activeExitStatus == NTLSTATUS_SUCCESS)
        activeExitStatus = NTLSTATUS_PROCESS_UNKNOWN_FAULT;

    if (request->Disposition == SYS_EXCEPTION_DISPOSITION_CONTINUE_SEARCH ||
        request->Disposition == SYS_EXCEPTION_DISPOSITION_TERMINATE_PROCESS) {
        KiRecordExceptionUnhandled(process);
        thread->UserExceptionDepth = activeDepth;
        thread->UserExceptionNoncontinuableMask &= ~activeBit;
        thread->UserExceptionFrameUserAddress = 0;
        thread->UserExceptionExitStatus[activeDepth] = NTLSTATUS_SUCCESS;
        KiTerminateCurrentExceptionProcess(activeExitStatus);
    }

    if (request->Disposition != SYS_EXCEPTION_DISPOSITION_CONTINUE_EXECUTION) {
        KiRecordExceptionInvalidContinue(process);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    if (thread->UserExceptionNoncontinuableMask & activeBit) {
        KiRecordExceptionInvalidContinue(process);
        thread->UserExceptionDepth = activeDepth;
        thread->UserExceptionNoncontinuableMask &= ~activeBit;
        thread->UserExceptionFrameUserAddress = 0;
        thread->UserExceptionExitStatus[activeDepth] = NTLSTATUS_SUCCESS;
        KiTerminateCurrentExceptionProcess(NTLSTATUS_PROCESS_UNKNOWN_FAULT);
    }
    if (!request->Context) {
        KiRecordExceptionInvalidContinue(process);
        return NTLSTATUS_INVALID_PARAMETER;
    }

    SYS_USER_CONTEXT context;
    KSTATUS status = CopyFromUser(&context, request->Context, sizeof(context));
    if (!KSUCCESS(status))
        return status;

    status = KiValidateContinueContext(&context);
    if (!KSUCCESS(status)) {
        KiRecordExceptionInvalidContinue(process);
        KiTerminateCurrentExceptionProcess(status);
    }

    KTRAP_FRAME* trapFrame = KeGetCurrentTrapFrame();
    if (!trapFrame || !KeTrapFrameFromUser(trapFrame) || trapFrame->Kind != KTRAP_KIND_SYSCALL) {
        KiRecordExceptionInvalidContinue(process);
        return NTLSTATUS_FAILURE;
    }

    KiApplyContinueContextToTrapFrame(trapFrame, &context);
    KiRecordExceptionContinued(process);
    thread->UserExceptionDepth = activeDepth;
    thread->UserExceptionNoncontinuableMask &= ~activeBit;
    thread->UserExceptionFrameUserAddress = 0;
    thread->UserExceptionExitStatus[activeDepth] = NTLSTATUS_SUCCESS;
    return context.Rax;
}

// Returns whether a fault frame was initialized by the assembly stubs.
bool KeFaultFrameIsValid(const KFAULT_FRAME* frame) {
    return frame && frame->Magic == KFAULT_FRAME_MAGIC_VALUE;
}

// Returns whether a fault frame captured an interrupted user-mode context.
bool KeFaultFrameFromUser(const KFAULT_FRAME* frame) {
    return KeFaultFrameIsValid(frame) && ((frame->Cs & 3u) == 3u);
}

// Returns the short architectural name for one CPU exception vector.
const char* KeExceptionVectorName(uint32_t vector) {
    switch (vector) {
        case 0:  return "#DE Divide Error";
        case 1:  return "#DB Debug";
        case 2:  return "NMI Interrupt";
        case 3:  return "#BP Breakpoint";
        case 4:  return "#OF Overflow";
        case 5:  return "#BR Bound Range Exceeded";
        case 6:  return "#UD Invalid Opcode";
        case 7:  return "#NM Device Not Available";
        case 8:  return "#DF Double Fault";
        case 9:  return "Coprocessor Segment Overrun";
        case 10: return "#TS Invalid TSS";
        case 11: return "#NP Segment Not Present";
        case 12: return "#SS Stack-Segment Fault";
        case 13: return "#GP General Protection";
        case 14: return "#PF Page Fault";
        case 15: return "Reserved Exception 15";
        case 16: return "#MF x87 Floating-Point Exception";
        case 17: return "#AC Alignment Check";
        case 18: return "#MC Machine Check";
        case 19: return "#XM SIMD Floating-Point Exception";
        case 20: return "#VE Virtualization Exception";
        case 21: return "#CP Control Protection Exception";
        case 22: return "Reserved Exception 22";
        case 23: return "Reserved Exception 23";
        case 24: return "Reserved Exception 24";
        case 25: return "Reserved Exception 25";
        case 26: return "Reserved Exception 26";
        case 27: return "Reserved Exception 27";
        case 28: return "#HV Hypervisor Injection Exception";
        case 29: return "#VC VMM Communication Exception";
        case 30: return "#SX Security Exception";
        case 31: return "Reserved Exception 31";
        default: return "Unknown Exception";
    }
}

// Returns whether the CPU pushes an error code for one exception vector.
bool KeExceptionVectorHasErrorCode(uint32_t vector) {
    switch (vector) {
        case 8:
        case 10:
        case 11:
        case 12:
        case 13:
        case 14:
        case 17:
        case 21:
        case 29:
        case 30:
            return true;
        default:
            return false;
    }
}

// Maps an unhandled user-mode CPU exception to the legacy process exit status.
KSTATUS KeExceptionProcessExitStatus(uint32_t vector) {
    switch (vector) {
        case 0:  return NTLSTATUS_PROCESS_DIVIDE_BY_ZERO;
        case 3:  return NTLSTATUS_PROCESS_BREAKPOINT;
        case 6:  return NTLSTATUS_PROCESS_ILLEGAL_OPCODE;
        case 13: return NTLSTATUS_PROCESS_GENERAL_PROTECTION;
        case 14: return NTLSTATUS_PROCESS_PAGE_FAULT;
        case 17: return NTLSTATUS_PROCESS_ALIGNMENT_FAULT;
        default: return NTLSTATUS_PROCESS_UNKNOWN_FAULT;
    }
}

// Handles faults that are still kernel-owned repair cases, not user exceptions.
KSTATUS KeExceptionResolveRecoverableFault(const KFAULT_FRAME* frame) {
    if (!KeFaultFrameIsValid(frame))
        return NTLSTATUS_INVALID_PARAMETER;
    if (frame->Vector != 14)
        return NTLSTATUS_NOT_SUPPORTED;

    return MmVadResolvePageFaultByAddress((uintptr_t)frame->Cr2, frame->ErrorCode);
}

// Validates a user exception dispatcher registration before publishing it.
static KSTATUS KiValidateExceptionDispatcherRequest(const SYS_EXCEPTION_DISPATCHER_REQUEST* request) {
    if (!request)
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->Size != sizeof(*request))
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->Flags != SYS_EXCEPTION_DISPATCH_FLAG_NONE)
        return NTLSTATUS_INVALID_PARAMETER;

    if (!request->DispatcherRoutine) {
        if (request->ExceptionStackBase || request->ExceptionStackSize)
            return NTLSTATUS_INVALID_PARAMETER;
        return NTLSTATUS_SUCCESS;
    }

    if (!KiExceptionUserAddressValid((uintptr_t)request->DispatcherRoutine))
        return NTLSTATUS_ACCESS_DENIED;

    if ((request->ExceptionStackBase == 0) != (request->ExceptionStackSize == 0))
        return NTLSTATUS_INVALID_PARAMETER;
    if (request->ExceptionStackBase) {
        if (!KiExceptionUserRangeValid((uintptr_t)request->ExceptionStackBase, request->ExceptionStackSize))
            return NTLSTATUS_ACCESS_DENIED;
        if (request->ExceptionStackSize < sizeof(SYS_EXCEPTION_FRAME) + 64u)
            return NTLSTATUS_BUFFER_TOO_SMALL;
    }

    return NTLSTATUS_SUCCESS;
}

// Registers or clears the current process's native user exception dispatcher.
KSTATUS KeSetExceptionDispatcher(PROCESS* process, const SYS_EXCEPTION_DISPATCHER_REQUEST* request) {
    if (!process || !request)
        return NTLSTATUS_INVALID_PARAMETER;

    KSTATUS status = KiValidateExceptionDispatcherRequest(request);
    if (!KSUCCESS(status))
        return status;

    uintptr_t flags = KeAcquireSpinlock(&process->ExceptionLock);
    if (!request->DispatcherRoutine) {
        process->ExceptionDispatcher = 0;
        process->ExceptionDispatcherContext = 0;
        process->ExceptionStackBase = 0;
        process->ExceptionStackSize = 0;
        process->ExceptionDispatchFlags = SYS_EXCEPTION_DISPATCH_FLAG_NONE;
    } else {
        process->ExceptionDispatcher = (uintptr_t)request->DispatcherRoutine;
        process->ExceptionDispatcherContext = (uintptr_t)request->DispatcherContext;
        process->ExceptionStackBase = (uintptr_t)request->ExceptionStackBase;
        process->ExceptionStackSize = request->ExceptionStackSize;
        process->ExceptionDispatchFlags = request->Flags;
    }
    KeReleaseSpinlock(&process->ExceptionLock, flags);
    return NTLSTATUS_SUCCESS;
}

// Copies a stable snapshot of the current process's exception dispatcher registration.
KSTATUS KeGetExceptionDispatcher(PROCESS* process, SYS_EXCEPTION_DISPATCHER_REQUEST* request) {
    if (!process || !request)
        return NTLSTATUS_INVALID_PARAMETER;

    memset(request, 0, sizeof(*request));
    request->Size = sizeof(*request);

    uintptr_t flags = KeAcquireSpinlock(&process->ExceptionLock);
    request->Flags = process->ExceptionDispatchFlags;
    request->DispatcherRoutine = (SYS_USER_EXCEPTION_DISPATCHER)process->ExceptionDispatcher;
    request->DispatcherContext = (void*)process->ExceptionDispatcherContext;
    request->ExceptionStackBase = (void*)process->ExceptionStackBase;
    request->ExceptionStackSize = process->ExceptionStackSize;
    KeReleaseSpinlock(&process->ExceptionLock, flags);
    return NTLSTATUS_SUCCESS;
}