; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64)
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack.  KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
; slots, which is the foundation for future user APC return redirection.

EXTERN KeSyscallDispatch:PROC
EXTERN KiEnterTrapFrame:PROC
EXTERN KiExitTrapFrame:PROC
EXTERN KeFaultDispatchFrame:PROC

KTRAP_FRAME_SIZE      EQU 240
KTRAP_FRAME_MAGIC     EQU 4650545241544B31h
KTRAP_KIND_SYSCALL    EQU 1
KTRAP_KIND_INTERRUPT  EQU 2
KTRAP_MODE_KERNEL     EQU 0
KTRAP_MODE_USER       EQU 1

TF_Magic        EQU 0
TF_Kind         EQU 8
TF_Vector       EQU 12
TF_PreviousMode EQU 16
TF_Reserved     EQU 20
TF_Rax          EQU 24
TF_Rcx          EQU 32
TF_Rdx          EQU 40
TF_Rbx          EQU 48
TF_Rbp          EQU 56
TF_Rsi          EQU 64
TF_Rdi          EQU 72
TF_R8           EQU 80
TF_R9           EQU 88
TF_R10          EQU 96
TF_R11          EQU 104
TF_R12          EQU 112
TF_R13          EQU 120
TF_R14          EQU 128
TF_R15          EQU 136
TF_Rip          EQU 144
TF_Cs           EQU 152
TF_Rflags       EQU 160
TF_Rsp          EQU 168
TF_Ss           EQU 176
TF_RipSlot      EQU 184
TF_CsSlot       EQU 192
TF_RflagsSlot   EQU 200
TF_RspSlot      EQU 208
TF_SsSlot       EQU 216

.CODE

KeSyscallEntry PROC
    ; CPU frame at entry, ring 3 -> ring 0:
    ;   [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS,
    ;   [rsp+18] user RSP, [rsp+20] SS
    ; Save RBX because we use it as the trap-frame pointer and user-mode code
    ; sees the syscall stub as a normal function call.
    push    rbx
    sub     rsp, KTRAP_FRAME_SIZE
    mov     rbx, rsp

    mov     [rbx + TF_Rax], rax
    mov     rax, KTRAP_FRAME_MAGIC
    mov     [rbx + TF_Magic], rax
    mov     dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
    mov     dword ptr [rbx + TF_Vector], 40h
    mov     dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
    mov     dword ptr [rbx + TF_Reserved], 0

    mov     [rbx + TF_Rcx], rcx
    mov     [rbx + TF_Rdx], rdx
    mov     rax, [rbx + KTRAP_FRAME_SIZE]
    mov     [rbx + TF_Rbx], rax
    mov     [rbx + TF_Rbp], rbp
    mov     [rbx + TF_Rsi], rsi
    mov     [rbx + TF_Rdi], rdi
    mov     [rbx + TF_R8],  r8
    mov     [rbx + TF_R9],  r9
    mov     [rbx + TF_R10], r10
    mov     [rbx + TF_R11], r11
    mov     [rbx + TF_R12], r12
    mov     [rbx + TF_R13], r13
    mov     [rbx + TF_R14], r14
    mov     [rbx + TF_R15], r15

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 8]
    mov     [rbx + TF_RipSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rip], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 16]
    mov     [rbx + TF_CsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Cs], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 24]
    mov     [rbx + TF_RflagsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rflags], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 32]
    mov     [rbx + TF_RspSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rsp], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 40]
    mov     [rbx + TF_SsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Ss], rdx

    sub     rsp, 32
    mov     rcx, rbx
    call    KiEnterTrapFrame
    add     rsp, 32

    ; Re-order original user arguments into KeSyscallDispatch(nr,a0..a5).
    ; Original registers were saved in the trap frame before any C calls.
    sub     rsp, 64
    mov     rax, [rbx + TF_R9]
    mov     [rsp + 32], rax     ; a3
    mov     rax, [rbx + TF_R10]
    mov     [rsp + 40], rax     ; a4
    mov     rax, [rbx + TF_R11]
    mov     [rsp + 48], rax     ; a5

    mov     r9,  [rbx + TF_R8]  ; a2
    mov     r8,  [rbx + TF_Rdx] ; a1
    mov     rdx, [rbx + TF_Rcx] ; a0
    mov     rcx, [rbx + TF_Rax] ; nr
    call    KeSyscallDispatch
    add     rsp, 64

    mov     [rbx + TF_Rax], rax

    sub     rsp, 32
    mov     rcx, rbx
    call    KiExitTrapFrame
    add     rsp, 32

    mov     rax, [rbx + TF_Rax]
    add     rsp, KTRAP_FRAME_SIZE
    pop     rbx
    iretq
KeSyscallEntry ENDP

; ---------------------------------------------------------------------------
; Fault stubs - snapshot a full user fault frame before entering C.
;
; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless
; by itself.  The captured user RSP lets the C crash dumper read [RSP], which is
; usually the return address immediately after the bad CALL.  The GPR snapshot
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
;   void KeFaultDispatchFrame(const KFAULT_FRAME* frame)  ; rcx
; ---------------------------------------------------------------------------

KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
KFAULT_SIZE  EQU 0C0h

KF_Magic     EQU 000h
KF_Vector    EQU 008h
KF_ErrorCode EQU 010h
KF_Cr2       EQU 018h
KF_Rip       EQU 020h
KF_Cs        EQU 028h
KF_Rflags    EQU 030h
KF_Rsp       EQU 038h
KF_Ss        EQU 040h
KF_Rax       EQU 048h
KF_Rcx       EQU 050h
KF_Rdx       EQU 058h
KF_Rbx       EQU 060h
KF_Rbp       EQU 068h
KF_Rsi       EQU 070h
KF_Rdi       EQU 078h
KF_R8        EQU 080h
KF_R9        EQU 088h
KF_R10       EQU 090h
KF_R11       EQU 098h
KF_R12       EQU 0A0h
KF_R13       EQU 0A8h
KF_R14       EQU 0B0h
KF_R15       EQU 0B8h

FAULT_CALL_SHADOW EQU 28h

; ---------------------------------------------------------
; Divide Error (#0) - NO error code pushed by CPU
; User exception frame: [E]=RIP [E+8]=CS [E+16]=RFLAGS [E+24]=RSP [E+32]=SS
; ---------------------------------------------------------
KeFaultDE PROC
    push rax                    ; [rsp+8] after next push = original RAX
    push r11                    ; [rsp]   = original R11
    mov  r11, rsp               ; saved pair; CPU frame starts at [r11+16]
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp               ; rax = KFAULT_FRAME*
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], 0
    mov  qword ptr [rax + KF_ErrorCode], 0
    mov  qword ptr [rax + KF_Cr2], 0

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   de_kernel_frame
    mov  r10, [rdx+24]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+32]
    mov  [rax + KF_Ss], r10
    jmp  de_call

de_kernel_frame:
    mov  qword ptr [rax + KF_Rsp], 0
    mov  qword ptr [rax + KF_Ss], 0

de_call:
    and  rsp, -16
    sub  rsp, FAULT_CALL_SHADOW
    mov  rcx, rax
    call KeFaultDispatchFrame
KeFaultDE ENDP

; ---------------------------------------------------------
; General Protection Fault (#13) - error code pushed by CPU
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
; ---------------------------------------------------------
KeFaultGP PROC
    push rax
    push r11
    mov  r11, rsp
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], 13
    mov  qword ptr [rax + KF_Cr2], 0

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_ErrorCode], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+24]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   gp_kernel_frame
    mov  r10, [rdx+32]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+40]
    mov  [rax + KF_Ss], r10
    jmp  gp_call

gp_kernel_frame:
    mov  qword ptr [rax + KF_Rsp], 0
    mov  qword ptr [rax + KF_Ss], 0

gp_call:
    and  rsp, -16
    sub  rsp, FAULT_CALL_SHADOW
    mov  rcx, rax
    call KeFaultDispatchFrame
KeFaultGP ENDP

; ---------------------------------------------------------
; Page Fault (#14) - error code pushed by CPU
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
; ---------------------------------------------------------
KeFaultPF PROC
    push rax
    push r11
    mov  r11, rsp
    sub  rsp, KFAULT_SIZE
    mov  rax, rsp
    mov  [rax + KF_R10], r10

    mov  r10, KFAULT_MAGIC
    mov  [rax + KF_Magic], r10
    mov  qword ptr [rax + KF_Vector], 14

    mov  [rax + KF_Rcx], rcx
    mov  [rax + KF_Rdx], rdx
    mov  [rax + KF_Rbx], rbx
    mov  [rax + KF_Rbp], rbp
    mov  [rax + KF_Rsi], rsi
    mov  [rax + KF_Rdi], rdi
    mov  [rax + KF_R8],  r8
    mov  [rax + KF_R9],  r9
    mov  r10, cr2
    mov  [rax + KF_Cr2], r10
    mov  r10, [r11]
    mov  [rax + KF_R11], r10
    mov  r10, [r11+8]
    mov  [rax + KF_Rax], r10
    mov  [rax + KF_R12], r12
    mov  [rax + KF_R13], r13
    mov  [rax + KF_R14], r14
    mov  [rax + KF_R15], r15

    lea  rdx, [r11+16]
    mov  r10, [rdx]
    mov  [rax + KF_ErrorCode], r10
    mov  r10, [rdx+8]
    mov  [rax + KF_Rip], r10
    mov  r10, [rdx+16]
    mov  [rax + KF_Cs], r10
    mov  r10, [rdx+24]
    mov  [rax + KF_Rflags], r10

    mov  r10, [rax + KF_Cs]
    test r10, 3
    jz   pf_kernel_frame
    mov  r10, [rdx+32]
    mov  [rax + KF_Rsp], r10
    mov  r10, [rdx+40]
    mov  [rax + KF_Ss], r10
    jmp  pf_call

pf_kernel_frame:
    mov  qword ptr [rax + KF_Rsp], 0
    mov  qword ptr [rax + KF_Ss], 0

pf_call:
    and  rsp, -16
    sub  rsp, FAULT_CALL_SHADOW
    mov  rcx, rax
    call KeFaultDispatchFrame
KeFaultPF ENDP

EXTERN KeIrqDispatch:PROC
EXTERN KiEnterInterrupt:PROC
EXTERN KiExitInterrupt:PROC

IrqStub MACRO num
PUBLIC KeIrqStub&num
KeIrqStub&num PROC
    push    rax
    mov     al, num
    jmp     KeIrqCommon
KeIrqStub&num ENDP
ENDM

IrqStub 0
IrqStub 1
IrqStub 2
IrqStub 3
IrqStub 4
IrqStub 5
IrqStub 6
IrqStub 7
IrqStub 8
IrqStub 9
IrqStub 10
IrqStub 11
IrqStub 12
IrqStub 13
IrqStub 14
IrqStub 15
IrqStub 16
IrqStub 17
IrqStub 18
IrqStub 19
IrqStub 20
IrqStub 21
IrqStub 22
IrqStub 23

KeIrqCommon PROC
    ; al = irq number, original rax saved by the per-IRQ stub.
    push    rcx
    push    rdx
    push    r8
    push    r9
    push    r10
    push    r11
    push    rbx

    sub     rsp, KTRAP_FRAME_SIZE
    mov     rbx, rsp

    movzx   edx, al
    mov     rax, KTRAP_FRAME_MAGIC
    mov     [rbx + TF_Magic], rax
    mov     dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT
    mov     [rbx + TF_Vector], edx
    mov     dword ptr [rbx + TF_Reserved], 0

    ; Saved volatile registers live immediately above the trap frame.
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 56]
    mov     [rbx + TF_Rax], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 48]
    mov     [rbx + TF_Rcx], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 40]
    mov     [rbx + TF_Rdx], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 0]
    mov     [rbx + TF_Rbx], rax
    mov     [rbx + TF_Rbp], rbp
    mov     [rbx + TF_Rsi], rsi
    mov     [rbx + TF_Rdi], rdi
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 32]
    mov     [rbx + TF_R8], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 24]
    mov     [rbx + TF_R9], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 16]
    mov     [rbx + TF_R10], rax
    mov     rax, [rbx + KTRAP_FRAME_SIZE + 8]
    mov     [rbx + TF_R11], rax
    mov     [rbx + TF_R12], r12
    mov     [rbx + TF_R13], r13
    mov     [rbx + TF_R14], r14
    mov     [rbx + TF_R15], r15

    ; CPU interrupt frame starts after the 8 saved registers.
    lea     rax, [rbx + KTRAP_FRAME_SIZE + 64]
    mov     [rbx + TF_RipSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rip], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 72]
    mov     [rbx + TF_CsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Cs], rdx

    lea     rax, [rbx + KTRAP_FRAME_SIZE + 80]
    mov     [rbx + TF_RflagsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rflags], rdx

    mov     rax, [rbx + TF_Cs]
    and     eax, 3
    cmp     eax, 3
    jne     short IrqKernelModeFrame

    mov     dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
    lea     rax, [rbx + KTRAP_FRAME_SIZE + 88]
    mov     [rbx + TF_RspSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Rsp], rdx
    lea     rax, [rbx + KTRAP_FRAME_SIZE + 96]
    mov     [rbx + TF_SsSlot], rax
    mov     rdx, [rax]
    mov     [rbx + TF_Ss], rdx
    jmp     short IrqFrameReady

IrqKernelModeFrame:
    mov     dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_KERNEL
    mov     qword ptr [rbx + TF_Rsp], 0
    mov     qword ptr [rbx + TF_Ss], 0
    mov     qword ptr [rbx + TF_RspSlot], 0
    mov     qword ptr [rbx + TF_SsSlot], 0

IrqFrameReady:
    sub     rsp, 40         ; 32-byte shadow space + alignment/local slack
    mov     rcx, rbx
    call    KiEnterTrapFrame
    call    KiEnterInterrupt
    mov     ecx, [rbx + TF_Vector]
    call    KeIrqDispatch
    call    KiExitInterrupt
    mov     rcx, rbx
    call    KiExitTrapFrame
    add     rsp, 40

    add     rsp, KTRAP_FRAME_SIZE
    pop     rbx
    pop     r11
    pop     r10
    pop     r9
    pop     r8
    pop     rdx
    pop     rcx
    pop     rax
    iretq
KeIrqCommon ENDP

END