; ke_syscall.asm - INT 0x2F, fault, and IRQ kernel entries (MASM64)
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
; slots used by user APC return redirection.

EXTERN KeSyscallDispatch:PROC
EXTERN KiEnterTrapFrame:PROC
EXTERN KiExitTrapFrame:PROC
EXTERN KiDeliverKernelApcs:PROC
EXTERN KeFaultDispatchFrame:PROC

KTRAP_FRAME_SIZE      EQU 240
KTRAP_FRAME_MAGIC     EQU 4650545241544B31h
KTRAP_KIND_SYSCALL    EQU 1
KTRAP_KIND_INTERRUPT  EQU 2
KTRAP_MODE_KERNEL     EQU 0
KTRAP_MODE_USER       EQU 1

TF_Magic        EQU 0
TF_Kind         EQU 8
TF_Vector       EQU 12
TF_PreviousMode EQU 16
TF_Rax          EQU 24
TF_Rcx          EQU 32
TF_Rdx          EQU 40
TF_Rbx          EQU 48
TF_Rbp          EQU 56
TF_Rsi          EQU 64
TF_Rdi          EQU 72
TF_R8           EQU 80
TF_R9           EQU 88
TF_R10          EQU 96
TF_R11          EQU 104
TF_R12          EQU 112
TF_R13          EQU 120
TF_R14          EQU 128
TF_R15          EQU 136
TF_Rip          EQU 144
TF_Cs           EQU 152
TF_Rflags       EQU 160
TF_Rsp          EQU 168
TF_Ss           EQU 176
TF_RipSlot      EQU 184
TF_CsSlot       EQU 192
TF_RflagsSlot   EQU 200
TF_RspSlot      EQU 208
TF_SsSlot       EQU 216

.CODE

KeSyscallEntry PROC
    swapgs             ; GS_BASE: user TEB -> KCPU, KERNEL_GS_BASE: KCPU -> user TEB
    ; CPU frame at entry, ring 3 -> ring 0:
    ;   [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS,
    ;   [rsp+18] user RSP, [rsp+20] SS
    ; Save RBX because we use it as the trap-frame pointer and user-mode code
    ; sees the syscall stub as a normal function call.
    push    rbx
    sub     rsp, KTRAP_FRAME_SIZE
    mov     rbx, rsp

    mov     [rbx + TF_Rax], rax
    mov     rax, KTRAP_FRAME_MAGIC
    mov     [rbx + TF_Magic], rax
    mov     dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
    mov     dword ptr [rbx + TF_Vector], 02Fh
    mov     dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER

    mov     [rbx + TF_Rcx], rcx
    mov     [rbx + TF_Rdx], rdx
    mov     rax, [rbx + KTRAP_FRAME_SIZE]
    mov     [rbx + TF_Rbx], rax
    mov     [rbx + TF_Rbp], rbp
    mov     [rbx + TF_Rsi], rsi
    mov     [rbx + TF_Rdi], rdi
    mov     [rbx + TF_R8],  r8
    mov     [rbx + TF_R9],  r9
    mov     [rbx + TF_R10], r10
    mov     [rbx + TF_R11], r11
    mov     [rbx + TF_R12], r12
    mov     [rbx + TF_R13], r13
    mov     [rbx + TF_R14], r14
    mov     [rbx + TF_R15], r15

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 8]
    mov     [rbx + TF_RipSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rip], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 16]
    mov     [rbx + TF_CsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Cs], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 24]
    mov     [rbx + TF_RflagsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rflags], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 32]
    mov     [rbx + TF_RspSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rsp], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 40]
    mov     [rbx + TF_SsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Ss], rdx

    sub     rsp, 32
    mov     rcx, rbx
    call    KiEnterTrapFrame
    add     rsp, 32

    ; The interrupt gate protects the ring transition and SWAPGS/trap-frame
    ; construction. Once the thread context is fully established, syscalls are
    ; ordinary interruptible PASSIVE_LEVEL execution rather than one giant
    ; interrupt-disabled critical section.
    sti

    ; Re-order original user arguments into KeSyscallDispatch(nr,a0..a3).
    ; Original registers were saved in the trap frame before any C calls.
    sub     rsp, 48
    mov     rax, [rbx + TF_R9]
    mov     [rsp + 32], rax     ; a3

    mov     r9,  [rbx + TF_R8]  ; a2
    mov     r8,  [rbx + TF_Rdx] ; a1
    mov     rdx, [rbx + TF_Rcx] ; a0
    mov     rcx, [rbx + TF_Rax] ; nr
    call    KeSyscallDispatch
    add     rsp, 48

    mov     [rbx + TF_Rax], rax

    ; Syscall return is a scheduler-safe kernel APC delivery boundary. Keep IF
    ; set while normal routines run, then close the tiny machine-exit window
    ; before committing the editable IRET frame and restoring user GS.
    sub     rsp, 32
    call    KiDeliverKernelApcs
    add     rsp, 32
    cli

    sub     rsp, 32
    mov     rcx, rbx
    call    KiExitTrapFrame
    add     rsp, 32

    ; Restore the editable trap-frame register image. Normal syscalls only
    ; require RAX, but exception continuation may redirect the IRET to an
    ; arbitrary saved user context that needs the full GPR set.
    mov     rax, [rbx + TF_Rax]
    mov     rcx, [rbx + TF_Rcx]
    mov     rdx, [rbx + TF_Rdx]
    mov     rbp, [rbx + TF_Rbp]
    mov     rsi, [rbx + TF_Rsi]
    mov     rdi, [rbx + TF_Rdi]
    mov     r8,  [rbx + TF_R8]
    mov     r9,  [rbx + TF_R9]
    mov     r10, [rbx + TF_R10]
    mov     r11, [rbx + TF_R11]
    mov     r12, [rbx + TF_R12]
    mov     r13, [rbx + TF_R13]
    mov     r14, [rbx + TF_R14]
    mov     r15, [rbx + TF_R15]
    lea     rsp, [rbx + KTRAP_FRAME_SIZE + 8]
    mov     rbx, [rbx + TF_Rbx]
    swapgs             ; GS_BASE: KCPU -> user TEB before returning to ring 3
    iretq
KeSyscallEntry ENDP

; ---------------------------------------------------------------------------
; Fault stubs - snapshot a full user fault frame before entering C.
;
; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless
; by itself. The captured user RSP lets the C crash dumper read [RSP], which is
; usually the return address immediately after the bad CALL. The GPR snapshot
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
;   KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) ; rcx
; ---------------------------------------------------------------------------

KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
KFAULT_SIZE  EQU 0C0h

KF_Magic     EQU 000h
KF_Vector    EQU 008h
KF_ErrorCode EQU 010h
KF_Cr2       EQU 018h
KF_Rip       EQU 020h
KF_Cs        EQU 028h
KF_Rflags    EQU 030h
KF_Rsp       EQU 038h
KF_Ss        EQU 040h
KF_Rax       EQU 048h
KF_Rcx       EQU 050h
KF_Rdx       EQU 058h
KF_Rbx       EQU 060h
KF_Rbp       EQU 068h
KF_Rsi       EQU 070h
KF_Rdi       EQU 078h
KF_R8        EQU 080h
KF_R9        EQU 088h
KF_R10       EQU 090h
KF_R11       EQU 098h
KF_R12       EQU 0A0h
KF_R13       EQU 0A8h
KF_R14       EQU 0B0h
KF_R15       EQU 0B8h

; Asynchronous kernel entry must preserve the full interrupted x87/SSE state.
; MS x64 calls may freely clobber XMM0-XMM5, and an interrupt or fault can
; arrive between any two user or kernel instructions rather than at a call ABI.
; Reserve one 512-byte FXSAVE image plus alignment slack on the interrupted stack.
ASYNC_FPU_SCRATCH EQU 528

ASYNC_FPU_SAVE MACRO
    sub  rsp, ASYNC_FPU_SCRATCH
    lea  r12, [rsp + 15]
    and  r12, -16
    fxsave64 [r12]
ENDM

ASYNC_FPU_RESTORE MACRO
    fxrstor64 [r12]
ENDM

FAULT_RESUME_NOERR MACRO
    LOCAL noerr_skip_user_slots, noerr_no_swapgs, noerr_unhandled_halt
    test eax, eax
    jnz  noerr_unhandled_halt

    mov  rdx, rbx
    mov  rax, [rdx + KF_Rip]
    mov  [rdx + KFAULT_SIZE + 16], rax
    mov  rax, [rdx + KF_Cs]
    mov  [rdx + KFAULT_SIZE + 24], rax
    mov  rax, [rdx + KF_Rflags]
    mov  [rdx + KFAULT_SIZE + 32], rax

    mov  rax, [rdx + KF_Cs]
    test rax, 3
    jz   noerr_skip_user_slots
    mov  rax, [rdx + KF_Rsp]
    mov  [rdx + KFAULT_SIZE + 40], rax
    mov  rax, [rdx + KF_Ss]
    mov  [rdx + KFAULT_SIZE + 48], rax

noerr_skip_user_slots:
    mov  rcx, [rdx + KF_Rcx]
    mov  r8,  [rdx + KF_R8]
    mov  r9,  [rdx + KF_R9]
    mov  rsi, [rdx + KF_Rsi]
    mov  rdi, [rdx + KF_Rdi]
    mov  rbp, [rdx + KF_Rbp]
    mov  r12, [rdx + KF_R12]
    mov  r13, [rdx + KF_R13]
    mov  r14, [rdx + KF_R14]
    mov  r15, [rdx + KF_R15]
    lea  rsp, [rdx + KFAULT_SIZE + 16]
    mov  rax, [rdx + KF_Rax]
    mov  r11, [rdx + KF_R11]
    mov  r10, [rdx + KF_Cs]
    test r10, 3
    jz   noerr_no_swapgs
    mov  r10, [rdx + KF_R10]
    mov  rbx, [rdx + KF_Rbx]
    mov  rdx, [rdx + KF_Rdx]
    swapgs
    iretq
noerr_no_swapgs:
    mov  r10, [rdx + KF_R10]
    mov  rbx, [rdx + KF_Rbx]
    mov  rdx, [rdx + KF_Rdx]
    iretq

noerr_unhandled_halt:
    cli
@@:
    hlt
    jmp @B
ENDM

FAULT_RESUME_ERR MACRO
    LOCAL err_skip_user_slots, err_no_swapgs, err_unhandled_halt
    test eax, eax
    jnz  err_unhandled_halt

    mov  rdx, rbx
    mov  rax, [rdx + KF_Rip]
    mov  [rdx + KFAULT_SIZE + 24], rax
    mov  rax, [rdx + KF_Cs]
    mov  [rdx + KFAULT_SIZE + 32], rax
    mov  rax, [rdx + KF_Rflags]
    mov  [rdx + KFAULT_SIZE + 40], rax

    mov  rax, [rdx + KF_Cs]
    test rax, 3
    jz   err_skip_user_slots
    mov  rax, [rdx + KF_Rsp]
    mov  [rdx + KFAULT_SIZE + 48], rax
    mov  rax, [rdx + KF_Ss]
    mov  [rdx + KFAULT_SIZE + 56], rax

err_skip_user_slots:
    mov  rcx, [rdx + KF_Rcx]
    mov  r8,  [rdx + KF_R8]
    mov  r9,  [rdx + KF_R9]
    mov  rsi, [rdx + KF_Rsi]
    mov  rdi, [rdx + KF_Rdi]
    mov  rbp, [rdx + KF_Rbp]
    mov  r12, [rdx + KF_R12]
    mov  r13, [rdx + KF_R13]
    mov  r14, [rdx + KF_R14]
    mov  r15, [rdx + KF_R15]
    lea  rsp, [rdx + KFAULT_SIZE + 24]
    mov  rax, [rdx + KF_Rax]
    mov  r11, [rdx + KF_R11]
    mov  r10, [rdx + KF_Cs]
    test r10, 3
    jz   err_no_swapgs
    mov  r10, [rdx + KF_R10]
    mov  rbx, [rdx + KF_Rbx]
    mov  rdx, [rdx + KF_Rdx]
    swapgs
    iretq
err_no_swapgs:
    mov  r10, [rdx + KF_R10]
    mov  rbx, [rdx + KF_Rbx]
    mov  rdx, [rdx + KF_Rdx]
    iretq

err_unhandled_halt:
    cli
@@:
    hlt
    jmp @B
ENDM

; ---------------------------------------------------------
; Divide Error (#0) - NO error code pushed by CPU
; User exception frame: [E]=RIP [E+8]=CS [E+16]=RFLAGS [E+24]=RSP [E+32]=SS
; ---------------------------------------------------------
KeFaultDE PROC
    push rax                    ; [rsp+8] after next push = original RAX
    push r11                    ; [rsp]   = original R11
    mov  r11, rsp               ; saved pair; CPU frame starts at [r11+16]
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp               ; rax = KFAULT_FRAME*
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], 0
    mov  qword ptr [rax + KF_ErrorCode], 0
    mov  qword ptr [rax + KF_Cr2], 0

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   de_kernel_frame
    mov  r10, [rdx+24]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+32]
    mov  [rax + KF_Ss], r10
    swapgs
    jmp  de_call

de_kernel_frame:
    lea  r10, [rdx+24]       ; same-privilege no-error frame has RIP/CS/RFLAGS
    mov  [rax + KF_Rsp], r10
    mov  qword ptr [rax + KF_Ss], 0

de_call:
    mov  rbx, rax
    ASYNC_FPU_SAVE
    and  rsp, -16
    sub  rsp, 32
    mov  rcx, rbx
    call KeFaultDispatchFrame
    ASYNC_FPU_RESTORE
    FAULT_RESUME_NOERR
KeFaultDE ENDP

; ---------------------------------------------------------
; General Protection Fault (#13) - error code pushed by CPU
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
; ---------------------------------------------------------
KeFaultGP PROC
    push rax
    push r11
    mov  r11, rsp
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], 13
    mov  qword ptr [rax + KF_Cr2], 0

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_ErrorCode], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+24]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   gp_kernel_frame
    mov  r10, [rdx+32]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+40]
    mov  [rax + KF_Ss], r10
    swapgs
    jmp  gp_call

gp_kernel_frame:
    lea  r10, [rdx+32]       ; same-privilege fault: original RSP is above ERR/RIP/CS/RFLAGS
    mov  [rax + KF_Rsp], r10
    mov  qword ptr [rax + KF_Ss], 0

gp_call:
    mov  rbx, rax
    ASYNC_FPU_SAVE
    and  rsp, -16
    sub  rsp, 32
    mov  rcx, rbx
    call KeFaultDispatchFrame
    ASYNC_FPU_RESTORE
    FAULT_RESUME_ERR
KeFaultGP ENDP

; ---------------------------------------------------------
; Page Fault (#14) - error code pushed by CPU
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
; ---------------------------------------------------------
KeFaultPF PROC
    push rax
    push r11
    mov  r11, rsp
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], 14

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, cr2
    mov  [rax + KF_Cr2], r10
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_ErrorCode], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+24]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   pf_kernel_frame
    mov  r10, [rdx+32]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+40]
    mov  [rax + KF_Ss], r10
    swapgs
    jmp  pf_call

pf_kernel_frame:
    lea  r10, [rdx+32]       ; same-privilege #PF frame has ERR/RIP/CS/RFLAGS
    mov  [rax + KF_Rsp], r10
    mov  qword ptr [rax + KF_Ss], 0

pf_call:
    mov  rbx, rax               ; Keep the KFAULT_FRAME pointer across C.
    ASYNC_FPU_SAVE
    and  rsp, -16
    sub  rsp, 32
    mov  rcx, rbx
    call KeFaultDispatchFrame
    ASYNC_FPU_RESTORE
    FAULT_RESUME_ERR
KeFaultPF ENDP

; ---------------------------------------------------------
; Generic CPU exception stubs.
; Each vector gets its own IDT entry and vector number so faults like #UD
; do not fall through an absent descriptor and reappear as #GP.
; ---------------------------------------------------------
FAULT_NOERR MACRO StubName, VecNum
    LOCAL noerr_kernel_frame, noerr_call
StubName PROC
    push rax
    push r11
    mov  r11, rsp
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], VecNum
    mov  qword ptr [rax + KF_ErrorCode], 0
    mov  qword ptr [rax + KF_Cr2], 0

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   noerr_kernel_frame
    mov  r10, [rdx+24]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+32]
    mov  [rax + KF_Ss], r10
    swapgs
    jmp  noerr_call

noerr_kernel_frame:
    lea  r10, [rdx+24]       ; same-privilege no-error frame has RIP/CS/RFLAGS
    mov  [rax + KF_Rsp], r10
    mov  qword ptr [rax + KF_Ss], 0

noerr_call:
    mov  rbx, rax
    ASYNC_FPU_SAVE
    and  rsp, -16
    sub  rsp, 32
    mov  rcx, rbx
    call KeFaultDispatchFrame
    ASYNC_FPU_RESTORE
    FAULT_RESUME_NOERR
StubName ENDP
ENDM

FAULT_ERR MACRO StubName, VecNum
    LOCAL err_kernel_frame, err_call
StubName PROC
    push rax
    push r11
    mov  r11, rsp
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], VecNum
    mov  qword ptr [rax + KF_Cr2], 0

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_ErrorCode], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+24]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   err_kernel_frame
    mov  r10, [rdx+32]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+40]
    mov  [rax + KF_Ss], r10
    swapgs
    jmp  err_call

err_kernel_frame:
    lea  r10, [rdx+32]       ; same-privilege error-code frame has ERR/RIP/CS/RFLAGS
    mov  [rax + KF_Rsp], r10
    mov  qword ptr [rax + KF_Ss], 0

err_call:
    mov  rbx, rax
    ASYNC_FPU_SAVE
    and  rsp, -16
    sub  rsp, 32
    mov  rcx, rbx
    call KeFaultDispatchFrame
    ASYNC_FPU_RESTORE
    FAULT_RESUME_ERR
StubName ENDP
ENDM

EXTERN KeHandleNmiIpi:PROC
PUBLIC KeNmiIpiStub
KeNmiIpiStub PROC
    ; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
    ; the NMI was a panic/TLB-control event and is fully handled. If it
    ; returns 0, restore the interrupted context and fall through to the normal
    ; #NMI fault stub so unexpected NMIs still panic with a useful frame.
    push    rax
    push    rcx
    push    rdx
    push    r8
    push    r9
    push    r10
    push    r11
    push    r12
    push    r13
    mov     r13, rsp

    ASYNC_FPU_SAVE
    and     rsp, -16
    sub     rsp, 32
    call    KeHandleNmiIpi
    ASYNC_FPU_RESTORE
    mov     rsp, r13

    test    eax, eax
    jz      short NmiUnhandled

    pop     r13
    pop     r12
    pop     r11
    pop     r10
    pop     r9
    pop     r8
    pop     rdx
    pop     rcx
    pop     rax
    iretq

NmiUnhandled:
    pop     r13
    pop     r12
    pop     r11
    pop     r10
    pop     r9
    pop     r8
    pop     rdx
    pop     rcx
    pop     rax
    jmp     KeFaultNMI
KeNmiIpiStub ENDP

FAULT_NOERR KeFaultDB,    1
FAULT_NOERR KeFaultNMI,   2
FAULT_NOERR KeFaultBP,    3
FAULT_NOERR KeFaultOF,    4
FAULT_NOERR KeFaultBR,    5
FAULT_NOERR KeFaultUD,    6
FAULT_NOERR KeFaultNM,    7
FAULT_ERR   KeFaultDF,    8
FAULT_NOERR KeFaultCSO,   9
FAULT_ERR   KeFaultTS,    10
FAULT_ERR   KeFaultNP,    11
FAULT_ERR   KeFaultSS,    12
FAULT_NOERR KeFaultRSV15, 15
FAULT_NOERR KeFaultMF,    16
FAULT_ERR   KeFaultAC,    17
FAULT_NOERR KeFaultMC,    18
FAULT_NOERR KeFaultXM,    19
FAULT_NOERR KeFaultVE,    20
FAULT_ERR   KeFaultCP,    21
FAULT_NOERR KeFaultRSV22, 22
FAULT_NOERR KeFaultRSV23, 23
FAULT_NOERR KeFaultRSV24, 24
FAULT_NOERR KeFaultRSV25, 25
FAULT_NOERR KeFaultRSV26, 26
FAULT_NOERR KeFaultRSV27, 27
FAULT_NOERR KeFaultHV,    28
FAULT_ERR   KeFaultVC,    29
FAULT_ERR   KeFaultSX,    30
FAULT_NOERR KeFaultRSV31, 31

EXTERN KeIrqDispatch:PROC
EXTERN KiEnterInterrupt:PROC
EXTERN KiExitInterrupt:PROC

IrqStub MACRO num
PUBLIC KeIrqStub&num
KeIrqStub&num PROC
    push    rax
    mov     al, (030h + num)
    jmp     KeIrqCommon
KeIrqStub&num ENDP
ENDM

IrqStub 0
IrqStub 1
IrqStub 2
IrqStub 3
IrqStub 4
IrqStub 5
IrqStub 6
IrqStub 7
IrqStub 8
IrqStub 9
IrqStub 10
IrqStub 11
IrqStub 12
IrqStub 13
IrqStub 14
IrqStub 15
IrqStub 16
IrqStub 17
IrqStub 18
IrqStub 19
IrqStub 20
IrqStub 21
IrqStub 22
IrqStub 23
IrqStub 24
IrqStub 25
IrqStub 26
IrqStub 27
IrqStub 28
IrqStub 29
IrqStub 30
IrqStub 31
IrqStub 32
IrqStub 33
IrqStub 34
IrqStub 35
IrqStub 36
IrqStub 37
IrqStub 38
IrqStub 39
IrqStub 40
IrqStub 41
IrqStub 42
IrqStub 43
IrqStub 44
IrqStub 45
IrqStub 46
IrqStub 47
IrqStub 48
IrqStub 49
IrqStub 50
IrqStub 51
IrqStub 52
IrqStub 53
IrqStub 54
IrqStub 55
IrqStub 56
IrqStub 57
IrqStub 58
IrqStub 59
IrqStub 60
IrqStub 61
IrqStub 62
IrqStub 63
IrqStub 64
IrqStub 65
IrqStub 66
IrqStub 67
IrqStub 68
IrqStub 69
IrqStub 70
IrqStub 71
IrqStub 72
IrqStub 73
IrqStub 74
IrqStub 75
IrqStub 76
IrqStub 77
IrqStub 78
IrqStub 79
IrqStub 80
IrqStub 81
IrqStub 82
IrqStub 83
IrqStub 84
IrqStub 85
IrqStub 86
IrqStub 87
IrqStub 88
IrqStub 89
IrqStub 90
IrqStub 91
IrqStub 92
IrqStub 93
IrqStub 94
IrqStub 95
IrqStub 96
IrqStub 97
IrqStub 98
IrqStub 99
IrqStub 100
IrqStub 101
IrqStub 102
IrqStub 103
IrqStub 104
IrqStub 105
IrqStub 106
IrqStub 107
IrqStub 108
IrqStub 109
IrqStub 110
IrqStub 111
IrqStub 112
IrqStub 113
IrqStub 114
IrqStub 115
IrqStub 116
IrqStub 117
IrqStub 118
IrqStub 119
IrqStub 120
IrqStub 121
IrqStub 122
IrqStub 123
IrqStub 124
IrqStub 125
IrqStub 126
IrqStub 127
IrqStub 128
IrqStub 129
IrqStub 130
IrqStub 131
IrqStub 132
IrqStub 133
IrqStub 134
IrqStub 135
IrqStub 136
IrqStub 137
IrqStub 138
IrqStub 139
IrqStub 140
IrqStub 141
IrqStub 142
IrqStub 143
IrqStub 144
IrqStub 145
IrqStub 146
IrqStub 147
IrqStub 148
IrqStub 149
IrqStub 150
IrqStub 151
IrqStub 152
IrqStub 153
IrqStub 154
IrqStub 155
IrqStub 156
IrqStub 157
IrqStub 158
IrqStub 159

PUBLIC KeLocalTimerStub
KeLocalTimerStub PROC
    push    rax
    mov     al, 0E0h
    jmp     KeIrqCommon
KeLocalTimerStub ENDP

PUBLIC KeSpuriousStub
KeSpuriousStub PROC
    push    rax
    mov     al, 0FFh
    jmp     KeIrqCommon
KeSpuriousStub ENDP

PUBLIC KeRescheduleIpiStub
KeRescheduleIpiStub PROC
    push    rax
    mov     al, 0F0h
    jmp     KeIrqCommon
KeRescheduleIpiStub ENDP

KeIrqCommon PROC
    ; al = IDT vector, original rax saved by the vector stub.
    push    rcx
    push    rdx
    push    r8
    push    r9
    push    r10
    push    r11
    push    rbx

    sub     rsp, KTRAP_FRAME_SIZE
    mov     rbx, rsp

    movzx   edx, al
    mov     rax, KTRAP_FRAME_MAGIC
    mov     [rbx + TF_Magic], rax
    mov     dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT
    mov     [rbx + TF_Vector], edx

    ; Saved volatile registers live immediately above the trap frame.
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 56]
    mov     [rbx + TF_Rax], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 48]
    mov     [rbx + TF_Rcx], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 40]
    mov     [rbx + TF_Rdx], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 0]
    mov     [rbx + TF_Rbx], rax
    mov     [rbx + TF_Rbp], rbp
    mov     [rbx + TF_Rsi], rsi
    mov     [rbx + TF_Rdi], rdi
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 32]
    mov     [rbx + TF_R8], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 24]
    mov     [rbx + TF_R9], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 16]
    mov     [rbx + TF_R10], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 8]
    mov     [rbx + TF_R11], rax
    mov     [rbx + TF_R12], r12
    mov     [rbx + TF_R13], r13
    mov     [rbx + TF_R14], r14
    mov     [rbx + TF_R15], r15

    ; CPU interrupt frame starts after the 8 saved registers.
    lea     rax, [rbx + KTRAP_FRAME_SIZE + 64]
    mov     [rbx + TF_RipSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rip], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 72]
    mov     [rbx + TF_CsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Cs], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 80]
    mov     [rbx + TF_RflagsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rflags], rdx

    mov     rax, [rbx + TF_Cs]
    and     eax, 3
    cmp     eax, 3
    jne     short IrqKernelModeFrame

    mov     dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
    lea     rax, [rbx + KTRAP_FRAME_SIZE + 88]
    mov     [rbx + TF_RspSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rsp], rdx
    lea     rax, [rbx + KTRAP_FRAME_SIZE + 96]
    mov     [rbx + TF_SsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Ss], rdx
    swapgs             ; interrupted ring 3: install kernel GS/KCPU
    jmp     short IrqFrameReady

IrqKernelModeFrame:
    mov     dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_KERNEL
    mov     qword ptr [rbx + TF_Rsp], 0
    mov     qword ptr [rbx + TF_Ss], 0
    mov     qword ptr [rbx + TF_RspSlot], 0
    mov     qword ptr [rbx + TF_SsSlot], 0

IrqFrameReady:
    ASYNC_FPU_SAVE
    and     rsp, -16
    sub     rsp, 32
    mov     rcx, rbx
    call    KiEnterTrapFrame
    call    KiEnterInterrupt
    mov     ecx, [rbx + TF_Vector]
    call    KeIrqDispatch
    call    KiExitInterrupt
    mov     rcx, rbx
    call    KiExitTrapFrame
    ASYNC_FPU_RESTORE

    mov     eax, [rbx + TF_PreviousMode]
    cmp     eax, KTRAP_MODE_USER
    jne     short IrqNoSwapGsOut
    swapgs             ; returning to ring 3: restore user GS/TEB
IrqNoSwapGsOut:
    mov     r12, [rbx + TF_R12]
    lea     rsp, [rbx + KTRAP_FRAME_SIZE]
    pop     rbx
    pop     r11
    pop     r10
    pop     r9
    pop     r8
    pop     rdx
    pop     rcx
    pop     rax
    iretq
KeIrqCommon ENDP

END