1088 lines
26 KiB
NASM
1088 lines
26 KiB
NASM
; ke_syscall.asm - INT 0x2F, fault, and IRQ kernel entries (MASM64)
|
|
;
|
|
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
|
|
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
|
|
; slots used by user APC return redirection.
|
|
|
|
EXTERN KeSyscallDispatch:PROC
|
|
EXTERN KiEnterTrapFrame:PROC
|
|
EXTERN KiExitTrapFrame:PROC
|
|
EXTERN KiDeliverKernelApcs:PROC
|
|
EXTERN KeFaultDispatchFrame:PROC
|
|
|
|
KTRAP_FRAME_SIZE EQU 240
|
|
KTRAP_FRAME_MAGIC EQU 4650545241544B31h
|
|
KTRAP_KIND_SYSCALL EQU 1
|
|
KTRAP_KIND_INTERRUPT EQU 2
|
|
KTRAP_MODE_KERNEL EQU 0
|
|
KTRAP_MODE_USER EQU 1
|
|
|
|
TF_Magic EQU 0
|
|
TF_Kind EQU 8
|
|
TF_Vector EQU 12
|
|
TF_PreviousMode EQU 16
|
|
TF_Rax EQU 24
|
|
TF_Rcx EQU 32
|
|
TF_Rdx EQU 40
|
|
TF_Rbx EQU 48
|
|
TF_Rbp EQU 56
|
|
TF_Rsi EQU 64
|
|
TF_Rdi EQU 72
|
|
TF_R8 EQU 80
|
|
TF_R9 EQU 88
|
|
TF_R10 EQU 96
|
|
TF_R11 EQU 104
|
|
TF_R12 EQU 112
|
|
TF_R13 EQU 120
|
|
TF_R14 EQU 128
|
|
TF_R15 EQU 136
|
|
TF_Rip EQU 144
|
|
TF_Cs EQU 152
|
|
TF_Rflags EQU 160
|
|
TF_Rsp EQU 168
|
|
TF_Ss EQU 176
|
|
TF_RipSlot EQU 184
|
|
TF_CsSlot EQU 192
|
|
TF_RflagsSlot EQU 200
|
|
TF_RspSlot EQU 208
|
|
TF_SsSlot EQU 216
|
|
|
|
.CODE
|
|
|
|
KeSyscallEntry PROC
|
|
swapgs ; GS_BASE: user TEB -> KCPU, KERNEL_GS_BASE: KCPU -> user TEB
|
|
; CPU frame at entry, ring 3 -> ring 0:
|
|
; [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS,
|
|
; [rsp+18] user RSP, [rsp+20] SS
|
|
; Save RBX because we use it as the trap-frame pointer and user-mode code
|
|
; sees the syscall stub as a normal function call.
|
|
push rbx
|
|
sub rsp, KTRAP_FRAME_SIZE
|
|
mov rbx, rsp
|
|
|
|
mov [rbx + TF_Rax], rax
|
|
mov rax, KTRAP_FRAME_MAGIC
|
|
mov [rbx + TF_Magic], rax
|
|
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
|
|
mov dword ptr [rbx + TF_Vector], 02Fh
|
|
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
|
|
|
|
mov [rbx + TF_Rcx], rcx
|
|
mov [rbx + TF_Rdx], rdx
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE]
|
|
mov [rbx + TF_Rbx], rax
|
|
mov [rbx + TF_Rbp], rbp
|
|
mov [rbx + TF_Rsi], rsi
|
|
mov [rbx + TF_Rdi], rdi
|
|
mov [rbx + TF_R8], r8
|
|
mov [rbx + TF_R9], r9
|
|
mov [rbx + TF_R10], r10
|
|
mov [rbx + TF_R11], r11
|
|
mov [rbx + TF_R12], r12
|
|
mov [rbx + TF_R13], r13
|
|
mov [rbx + TF_R14], r14
|
|
mov [rbx + TF_R15], r15
|
|
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 8]
|
|
mov [rbx + TF_RipSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Rip], rdx
|
|
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 16]
|
|
mov [rbx + TF_CsSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Cs], rdx
|
|
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 24]
|
|
mov [rbx + TF_RflagsSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Rflags], rdx
|
|
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 32]
|
|
mov [rbx + TF_RspSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Rsp], rdx
|
|
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 40]
|
|
mov [rbx + TF_SsSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Ss], rdx
|
|
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KiEnterTrapFrame
|
|
add rsp, 32
|
|
|
|
; The interrupt gate protects the ring transition and SWAPGS/trap-frame
|
|
; construction. Once the thread context is fully established, syscalls are
|
|
; ordinary interruptible PASSIVE_LEVEL execution rather than one giant
|
|
; interrupt-disabled critical section.
|
|
sti
|
|
|
|
; Re-order original user arguments into KeSyscallDispatch(nr,a0..a3).
|
|
; Original registers were saved in the trap frame before any C calls.
|
|
sub rsp, 48
|
|
mov rax, [rbx + TF_R9]
|
|
mov [rsp + 32], rax ; a3
|
|
|
|
mov r9, [rbx + TF_R8] ; a2
|
|
mov r8, [rbx + TF_Rdx] ; a1
|
|
mov rdx, [rbx + TF_Rcx] ; a0
|
|
mov rcx, [rbx + TF_Rax] ; nr
|
|
call KeSyscallDispatch
|
|
add rsp, 48
|
|
|
|
mov [rbx + TF_Rax], rax
|
|
|
|
; Syscall return is a scheduler-safe kernel APC delivery boundary. Keep IF
|
|
; set while normal routines run, then close the tiny machine-exit window
|
|
; before committing the editable IRET frame and restoring user GS.
|
|
sub rsp, 32
|
|
call KiDeliverKernelApcs
|
|
add rsp, 32
|
|
cli
|
|
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KiExitTrapFrame
|
|
add rsp, 32
|
|
|
|
; Restore the editable trap-frame register image. Normal syscalls only
|
|
; require RAX, but exception continuation may redirect the IRET to an
|
|
; arbitrary saved user context that needs the full GPR set.
|
|
mov rax, [rbx + TF_Rax]
|
|
mov rcx, [rbx + TF_Rcx]
|
|
mov rdx, [rbx + TF_Rdx]
|
|
mov rbp, [rbx + TF_Rbp]
|
|
mov rsi, [rbx + TF_Rsi]
|
|
mov rdi, [rbx + TF_Rdi]
|
|
mov r8, [rbx + TF_R8]
|
|
mov r9, [rbx + TF_R9]
|
|
mov r10, [rbx + TF_R10]
|
|
mov r11, [rbx + TF_R11]
|
|
mov r12, [rbx + TF_R12]
|
|
mov r13, [rbx + TF_R13]
|
|
mov r14, [rbx + TF_R14]
|
|
mov r15, [rbx + TF_R15]
|
|
lea rsp, [rbx + KTRAP_FRAME_SIZE + 8]
|
|
mov rbx, [rbx + TF_Rbx]
|
|
swapgs ; GS_BASE: KCPU -> user TEB before returning to ring 3
|
|
iretq
|
|
KeSyscallEntry ENDP
|
|
|
|
; ---------------------------------------------------------------------------
|
|
; Fault stubs - snapshot a full user fault frame before entering C.
|
|
;
|
|
; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless
|
|
; by itself. The captured user RSP lets the C crash dumper read [RSP], which is
|
|
; usually the return address immediately after the bad CALL. The GPR snapshot
|
|
; also shows which register held the null target.
|
|
;
|
|
; KeFaultDispatchFrame signature (MS x64):
|
|
; KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) ; rcx
|
|
; ---------------------------------------------------------------------------
|
|
|
|
KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
|
|
KFAULT_SIZE EQU 0C0h
|
|
|
|
KF_Magic EQU 000h
|
|
KF_Vector EQU 008h
|
|
KF_ErrorCode EQU 010h
|
|
KF_Cr2 EQU 018h
|
|
KF_Rip EQU 020h
|
|
KF_Cs EQU 028h
|
|
KF_Rflags EQU 030h
|
|
KF_Rsp EQU 038h
|
|
KF_Ss EQU 040h
|
|
KF_Rax EQU 048h
|
|
KF_Rcx EQU 050h
|
|
KF_Rdx EQU 058h
|
|
KF_Rbx EQU 060h
|
|
KF_Rbp EQU 068h
|
|
KF_Rsi EQU 070h
|
|
KF_Rdi EQU 078h
|
|
KF_R8 EQU 080h
|
|
KF_R9 EQU 088h
|
|
KF_R10 EQU 090h
|
|
KF_R11 EQU 098h
|
|
KF_R12 EQU 0A0h
|
|
KF_R13 EQU 0A8h
|
|
KF_R14 EQU 0B0h
|
|
KF_R15 EQU 0B8h
|
|
|
|
; Asynchronous kernel entry must preserve the full interrupted x87/SSE state.
|
|
; MS x64 calls may freely clobber XMM0-XMM5, and an interrupt or fault can
|
|
; arrive between any two user or kernel instructions rather than at a call ABI.
|
|
; Reserve one 512-byte FXSAVE image plus alignment slack on the interrupted stack.
|
|
ASYNC_FPU_SCRATCH EQU 528
|
|
|
|
ASYNC_FPU_SAVE MACRO
|
|
sub rsp, ASYNC_FPU_SCRATCH
|
|
lea r12, [rsp + 15]
|
|
and r12, -16
|
|
fxsave64 [r12]
|
|
ENDM
|
|
|
|
ASYNC_FPU_RESTORE MACRO
|
|
fxrstor64 [r12]
|
|
ENDM
|
|
|
|
FAULT_RESUME_NOERR MACRO
|
|
LOCAL noerr_skip_user_slots, noerr_no_swapgs, noerr_unhandled_halt
|
|
test eax, eax
|
|
jnz noerr_unhandled_halt
|
|
|
|
mov rdx, rbx
|
|
mov rax, [rdx + KF_Rip]
|
|
mov [rdx + KFAULT_SIZE + 16], rax
|
|
mov rax, [rdx + KF_Cs]
|
|
mov [rdx + KFAULT_SIZE + 24], rax
|
|
mov rax, [rdx + KF_Rflags]
|
|
mov [rdx + KFAULT_SIZE + 32], rax
|
|
|
|
mov rax, [rdx + KF_Cs]
|
|
test rax, 3
|
|
jz noerr_skip_user_slots
|
|
mov rax, [rdx + KF_Rsp]
|
|
mov [rdx + KFAULT_SIZE + 40], rax
|
|
mov rax, [rdx + KF_Ss]
|
|
mov [rdx + KFAULT_SIZE + 48], rax
|
|
|
|
noerr_skip_user_slots:
|
|
mov rcx, [rdx + KF_Rcx]
|
|
mov r8, [rdx + KF_R8]
|
|
mov r9, [rdx + KF_R9]
|
|
mov rsi, [rdx + KF_Rsi]
|
|
mov rdi, [rdx + KF_Rdi]
|
|
mov rbp, [rdx + KF_Rbp]
|
|
mov r12, [rdx + KF_R12]
|
|
mov r13, [rdx + KF_R13]
|
|
mov r14, [rdx + KF_R14]
|
|
mov r15, [rdx + KF_R15]
|
|
lea rsp, [rdx + KFAULT_SIZE + 16]
|
|
mov rax, [rdx + KF_Rax]
|
|
mov r11, [rdx + KF_R11]
|
|
mov r10, [rdx + KF_Cs]
|
|
test r10, 3
|
|
jz noerr_no_swapgs
|
|
mov r10, [rdx + KF_R10]
|
|
mov rbx, [rdx + KF_Rbx]
|
|
mov rdx, [rdx + KF_Rdx]
|
|
swapgs
|
|
iretq
|
|
noerr_no_swapgs:
|
|
mov r10, [rdx + KF_R10]
|
|
mov rbx, [rdx + KF_Rbx]
|
|
mov rdx, [rdx + KF_Rdx]
|
|
iretq
|
|
|
|
noerr_unhandled_halt:
|
|
cli
|
|
@@:
|
|
hlt
|
|
jmp @B
|
|
ENDM
|
|
|
|
FAULT_RESUME_ERR MACRO
|
|
LOCAL err_skip_user_slots, err_no_swapgs, err_unhandled_halt
|
|
test eax, eax
|
|
jnz err_unhandled_halt
|
|
|
|
mov rdx, rbx
|
|
mov rax, [rdx + KF_Rip]
|
|
mov [rdx + KFAULT_SIZE + 24], rax
|
|
mov rax, [rdx + KF_Cs]
|
|
mov [rdx + KFAULT_SIZE + 32], rax
|
|
mov rax, [rdx + KF_Rflags]
|
|
mov [rdx + KFAULT_SIZE + 40], rax
|
|
|
|
mov rax, [rdx + KF_Cs]
|
|
test rax, 3
|
|
jz err_skip_user_slots
|
|
mov rax, [rdx + KF_Rsp]
|
|
mov [rdx + KFAULT_SIZE + 48], rax
|
|
mov rax, [rdx + KF_Ss]
|
|
mov [rdx + KFAULT_SIZE + 56], rax
|
|
|
|
err_skip_user_slots:
|
|
mov rcx, [rdx + KF_Rcx]
|
|
mov r8, [rdx + KF_R8]
|
|
mov r9, [rdx + KF_R9]
|
|
mov rsi, [rdx + KF_Rsi]
|
|
mov rdi, [rdx + KF_Rdi]
|
|
mov rbp, [rdx + KF_Rbp]
|
|
mov r12, [rdx + KF_R12]
|
|
mov r13, [rdx + KF_R13]
|
|
mov r14, [rdx + KF_R14]
|
|
mov r15, [rdx + KF_R15]
|
|
lea rsp, [rdx + KFAULT_SIZE + 24]
|
|
mov rax, [rdx + KF_Rax]
|
|
mov r11, [rdx + KF_R11]
|
|
mov r10, [rdx + KF_Cs]
|
|
test r10, 3
|
|
jz err_no_swapgs
|
|
mov r10, [rdx + KF_R10]
|
|
mov rbx, [rdx + KF_Rbx]
|
|
mov rdx, [rdx + KF_Rdx]
|
|
swapgs
|
|
iretq
|
|
err_no_swapgs:
|
|
mov r10, [rdx + KF_R10]
|
|
mov rbx, [rdx + KF_Rbx]
|
|
mov rdx, [rdx + KF_Rdx]
|
|
iretq
|
|
|
|
err_unhandled_halt:
|
|
cli
|
|
@@:
|
|
hlt
|
|
jmp @B
|
|
ENDM
|
|
|
|
; ---------------------------------------------------------
|
|
; Divide Error (#0) - NO error code pushed by CPU
|
|
; User exception frame: [E]=RIP [E+8]=CS [E+16]=RFLAGS [E+24]=RSP [E+32]=SS
|
|
; ---------------------------------------------------------
|
|
KeFaultDE PROC
|
|
push rax ; [rsp+8] after next push = original RAX
|
|
push r11 ; [rsp] = original R11
|
|
mov r11, rsp ; saved pair; CPU frame starts at [r11+16]
|
|
sub rsp, KFAULT_SIZE
|
|
mov rax, rsp ; rax = KFAULT_FRAME*
|
|
mov [rax + KF_R10], r10
|
|
|
|
mov r10, KFAULT_MAGIC
|
|
mov [rax + KF_Magic], r10
|
|
mov qword ptr [rax + KF_Vector], 0
|
|
mov qword ptr [rax + KF_ErrorCode], 0
|
|
mov qword ptr [rax + KF_Cr2], 0
|
|
|
|
mov [rax + KF_Rcx], rcx
|
|
mov [rax + KF_Rdx], rdx
|
|
mov [rax + KF_Rbx], rbx
|
|
mov [rax + KF_Rbp], rbp
|
|
mov [rax + KF_Rsi], rsi
|
|
mov [rax + KF_Rdi], rdi
|
|
mov [rax + KF_R8], r8
|
|
mov [rax + KF_R9], r9
|
|
mov r10, [r11]
|
|
mov [rax + KF_R11], r10
|
|
mov r10, [r11+8]
|
|
mov [rax + KF_Rax], r10
|
|
mov [rax + KF_R12], r12
|
|
mov [rax + KF_R13], r13
|
|
mov [rax + KF_R14], r14
|
|
mov [rax + KF_R15], r15
|
|
|
|
lea rdx, [r11+16]
|
|
mov r10, [rdx]
|
|
mov [rax + KF_Rip], r10
|
|
mov r10, [rdx+8]
|
|
mov [rax + KF_Cs], r10
|
|
mov r10, [rdx+16]
|
|
mov [rax + KF_Rflags], r10
|
|
|
|
mov r10, [rax + KF_Cs]
|
|
test r10, 3
|
|
jz de_kernel_frame
|
|
mov r10, [rdx+24]
|
|
mov [rax + KF_Rsp], r10
|
|
mov r10, [rdx+32]
|
|
mov [rax + KF_Ss], r10
|
|
swapgs
|
|
jmp de_call
|
|
|
|
de_kernel_frame:
|
|
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
|
|
mov [rax + KF_Rsp], r10
|
|
mov qword ptr [rax + KF_Ss], 0
|
|
|
|
de_call:
|
|
mov rbx, rax
|
|
ASYNC_FPU_SAVE
|
|
and rsp, -16
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KeFaultDispatchFrame
|
|
ASYNC_FPU_RESTORE
|
|
FAULT_RESUME_NOERR
|
|
KeFaultDE ENDP
|
|
|
|
; ---------------------------------------------------------
|
|
; General Protection Fault (#13) - error code pushed by CPU
|
|
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
|
|
; ---------------------------------------------------------
|
|
KeFaultGP PROC
|
|
push rax
|
|
push r11
|
|
mov r11, rsp
|
|
sub rsp, KFAULT_SIZE
|
|
mov rax, rsp
|
|
mov [rax + KF_R10], r10
|
|
|
|
mov r10, KFAULT_MAGIC
|
|
mov [rax + KF_Magic], r10
|
|
mov qword ptr [rax + KF_Vector], 13
|
|
mov qword ptr [rax + KF_Cr2], 0
|
|
|
|
mov [rax + KF_Rcx], rcx
|
|
mov [rax + KF_Rdx], rdx
|
|
mov [rax + KF_Rbx], rbx
|
|
mov [rax + KF_Rbp], rbp
|
|
mov [rax + KF_Rsi], rsi
|
|
mov [rax + KF_Rdi], rdi
|
|
mov [rax + KF_R8], r8
|
|
mov [rax + KF_R9], r9
|
|
mov r10, [r11]
|
|
mov [rax + KF_R11], r10
|
|
mov r10, [r11+8]
|
|
mov [rax + KF_Rax], r10
|
|
mov [rax + KF_R12], r12
|
|
mov [rax + KF_R13], r13
|
|
mov [rax + KF_R14], r14
|
|
mov [rax + KF_R15], r15
|
|
|
|
lea rdx, [r11+16]
|
|
mov r10, [rdx]
|
|
mov [rax + KF_ErrorCode], r10
|
|
mov r10, [rdx+8]
|
|
mov [rax + KF_Rip], r10
|
|
mov r10, [rdx+16]
|
|
mov [rax + KF_Cs], r10
|
|
mov r10, [rdx+24]
|
|
mov [rax + KF_Rflags], r10
|
|
|
|
mov r10, [rax + KF_Cs]
|
|
test r10, 3
|
|
jz gp_kernel_frame
|
|
mov r10, [rdx+32]
|
|
mov [rax + KF_Rsp], r10
|
|
mov r10, [rdx+40]
|
|
mov [rax + KF_Ss], r10
|
|
swapgs
|
|
jmp gp_call
|
|
|
|
gp_kernel_frame:
|
|
lea r10, [rdx+32] ; same-privilege fault: original RSP is above ERR/RIP/CS/RFLAGS
|
|
mov [rax + KF_Rsp], r10
|
|
mov qword ptr [rax + KF_Ss], 0
|
|
|
|
gp_call:
|
|
mov rbx, rax
|
|
ASYNC_FPU_SAVE
|
|
and rsp, -16
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KeFaultDispatchFrame
|
|
ASYNC_FPU_RESTORE
|
|
FAULT_RESUME_ERR
|
|
KeFaultGP ENDP
|
|
|
|
; ---------------------------------------------------------
|
|
; Page Fault (#14) - error code pushed by CPU
|
|
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
|
|
; ---------------------------------------------------------
|
|
KeFaultPF PROC
|
|
push rax
|
|
push r11
|
|
mov r11, rsp
|
|
sub rsp, KFAULT_SIZE
|
|
mov rax, rsp
|
|
mov [rax + KF_R10], r10
|
|
|
|
mov r10, KFAULT_MAGIC
|
|
mov [rax + KF_Magic], r10
|
|
mov qword ptr [rax + KF_Vector], 14
|
|
|
|
mov [rax + KF_Rcx], rcx
|
|
mov [rax + KF_Rdx], rdx
|
|
mov [rax + KF_Rbx], rbx
|
|
mov [rax + KF_Rbp], rbp
|
|
mov [rax + KF_Rsi], rsi
|
|
mov [rax + KF_Rdi], rdi
|
|
mov [rax + KF_R8], r8
|
|
mov [rax + KF_R9], r9
|
|
mov r10, cr2
|
|
mov [rax + KF_Cr2], r10
|
|
mov r10, [r11]
|
|
mov [rax + KF_R11], r10
|
|
mov r10, [r11+8]
|
|
mov [rax + KF_Rax], r10
|
|
mov [rax + KF_R12], r12
|
|
mov [rax + KF_R13], r13
|
|
mov [rax + KF_R14], r14
|
|
mov [rax + KF_R15], r15
|
|
|
|
lea rdx, [r11+16]
|
|
mov r10, [rdx]
|
|
mov [rax + KF_ErrorCode], r10
|
|
mov r10, [rdx+8]
|
|
mov [rax + KF_Rip], r10
|
|
mov r10, [rdx+16]
|
|
mov [rax + KF_Cs], r10
|
|
mov r10, [rdx+24]
|
|
mov [rax + KF_Rflags], r10
|
|
|
|
mov r10, [rax + KF_Cs]
|
|
test r10, 3
|
|
jz pf_kernel_frame
|
|
mov r10, [rdx+32]
|
|
mov [rax + KF_Rsp], r10
|
|
mov r10, [rdx+40]
|
|
mov [rax + KF_Ss], r10
|
|
swapgs
|
|
jmp pf_call
|
|
|
|
pf_kernel_frame:
|
|
lea r10, [rdx+32] ; same-privilege #PF frame has ERR/RIP/CS/RFLAGS
|
|
mov [rax + KF_Rsp], r10
|
|
mov qword ptr [rax + KF_Ss], 0
|
|
|
|
pf_call:
|
|
mov rbx, rax ; Keep the KFAULT_FRAME pointer across C.
|
|
ASYNC_FPU_SAVE
|
|
and rsp, -16
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KeFaultDispatchFrame
|
|
ASYNC_FPU_RESTORE
|
|
FAULT_RESUME_ERR
|
|
KeFaultPF ENDP
|
|
|
|
; ---------------------------------------------------------
|
|
; Generic CPU exception stubs.
|
|
; Each vector gets its own IDT entry and vector number so faults like #UD
|
|
; do not fall through an absent descriptor and reappear as #GP.
|
|
; ---------------------------------------------------------
|
|
FAULT_NOERR MACRO StubName, VecNum
|
|
LOCAL noerr_kernel_frame, noerr_call
|
|
StubName PROC
|
|
push rax
|
|
push r11
|
|
mov r11, rsp
|
|
sub rsp, KFAULT_SIZE
|
|
mov rax, rsp
|
|
mov [rax + KF_R10], r10
|
|
|
|
mov r10, KFAULT_MAGIC
|
|
mov [rax + KF_Magic], r10
|
|
mov qword ptr [rax + KF_Vector], VecNum
|
|
mov qword ptr [rax + KF_ErrorCode], 0
|
|
mov qword ptr [rax + KF_Cr2], 0
|
|
|
|
mov [rax + KF_Rcx], rcx
|
|
mov [rax + KF_Rdx], rdx
|
|
mov [rax + KF_Rbx], rbx
|
|
mov [rax + KF_Rbp], rbp
|
|
mov [rax + KF_Rsi], rsi
|
|
mov [rax + KF_Rdi], rdi
|
|
mov [rax + KF_R8], r8
|
|
mov [rax + KF_R9], r9
|
|
mov r10, [r11]
|
|
mov [rax + KF_R11], r10
|
|
mov r10, [r11+8]
|
|
mov [rax + KF_Rax], r10
|
|
mov [rax + KF_R12], r12
|
|
mov [rax + KF_R13], r13
|
|
mov [rax + KF_R14], r14
|
|
mov [rax + KF_R15], r15
|
|
|
|
lea rdx, [r11+16]
|
|
mov r10, [rdx]
|
|
mov [rax + KF_Rip], r10
|
|
mov r10, [rdx+8]
|
|
mov [rax + KF_Cs], r10
|
|
mov r10, [rdx+16]
|
|
mov [rax + KF_Rflags], r10
|
|
|
|
mov r10, [rax + KF_Cs]
|
|
test r10, 3
|
|
jz noerr_kernel_frame
|
|
mov r10, [rdx+24]
|
|
mov [rax + KF_Rsp], r10
|
|
mov r10, [rdx+32]
|
|
mov [rax + KF_Ss], r10
|
|
swapgs
|
|
jmp noerr_call
|
|
|
|
noerr_kernel_frame:
|
|
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
|
|
mov [rax + KF_Rsp], r10
|
|
mov qword ptr [rax + KF_Ss], 0
|
|
|
|
noerr_call:
|
|
mov rbx, rax
|
|
ASYNC_FPU_SAVE
|
|
and rsp, -16
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KeFaultDispatchFrame
|
|
ASYNC_FPU_RESTORE
|
|
FAULT_RESUME_NOERR
|
|
StubName ENDP
|
|
ENDM
|
|
|
|
FAULT_ERR MACRO StubName, VecNum
|
|
LOCAL err_kernel_frame, err_call
|
|
StubName PROC
|
|
push rax
|
|
push r11
|
|
mov r11, rsp
|
|
sub rsp, KFAULT_SIZE
|
|
mov rax, rsp
|
|
mov [rax + KF_R10], r10
|
|
|
|
mov r10, KFAULT_MAGIC
|
|
mov [rax + KF_Magic], r10
|
|
mov qword ptr [rax + KF_Vector], VecNum
|
|
mov qword ptr [rax + KF_Cr2], 0
|
|
|
|
mov [rax + KF_Rcx], rcx
|
|
mov [rax + KF_Rdx], rdx
|
|
mov [rax + KF_Rbx], rbx
|
|
mov [rax + KF_Rbp], rbp
|
|
mov [rax + KF_Rsi], rsi
|
|
mov [rax + KF_Rdi], rdi
|
|
mov [rax + KF_R8], r8
|
|
mov [rax + KF_R9], r9
|
|
mov r10, [r11]
|
|
mov [rax + KF_R11], r10
|
|
mov r10, [r11+8]
|
|
mov [rax + KF_Rax], r10
|
|
mov [rax + KF_R12], r12
|
|
mov [rax + KF_R13], r13
|
|
mov [rax + KF_R14], r14
|
|
mov [rax + KF_R15], r15
|
|
|
|
lea rdx, [r11+16]
|
|
mov r10, [rdx]
|
|
mov [rax + KF_ErrorCode], r10
|
|
mov r10, [rdx+8]
|
|
mov [rax + KF_Rip], r10
|
|
mov r10, [rdx+16]
|
|
mov [rax + KF_Cs], r10
|
|
mov r10, [rdx+24]
|
|
mov [rax + KF_Rflags], r10
|
|
|
|
mov r10, [rax + KF_Cs]
|
|
test r10, 3
|
|
jz err_kernel_frame
|
|
mov r10, [rdx+32]
|
|
mov [rax + KF_Rsp], r10
|
|
mov r10, [rdx+40]
|
|
mov [rax + KF_Ss], r10
|
|
swapgs
|
|
jmp err_call
|
|
|
|
err_kernel_frame:
|
|
lea r10, [rdx+32] ; same-privilege error-code frame has ERR/RIP/CS/RFLAGS
|
|
mov [rax + KF_Rsp], r10
|
|
mov qword ptr [rax + KF_Ss], 0
|
|
|
|
err_call:
|
|
mov rbx, rax
|
|
ASYNC_FPU_SAVE
|
|
and rsp, -16
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KeFaultDispatchFrame
|
|
ASYNC_FPU_RESTORE
|
|
FAULT_RESUME_ERR
|
|
StubName ENDP
|
|
ENDM
|
|
|
|
EXTERN KeHandleNmiIpi:PROC
|
|
PUBLIC KeNmiIpiStub
|
|
KeNmiIpiStub PROC
|
|
; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
|
|
; the NMI was a panic/TLB-control event and is fully handled. If it
|
|
; returns 0, restore the interrupted context and fall through to the normal
|
|
; #NMI fault stub so unexpected NMIs still panic with a useful frame.
|
|
push rax
|
|
push rcx
|
|
push rdx
|
|
push r8
|
|
push r9
|
|
push r10
|
|
push r11
|
|
push r12
|
|
push r13
|
|
mov r13, rsp
|
|
|
|
ASYNC_FPU_SAVE
|
|
and rsp, -16
|
|
sub rsp, 32
|
|
call KeHandleNmiIpi
|
|
ASYNC_FPU_RESTORE
|
|
mov rsp, r13
|
|
|
|
test eax, eax
|
|
jz short NmiUnhandled
|
|
|
|
pop r13
|
|
pop r12
|
|
pop r11
|
|
pop r10
|
|
pop r9
|
|
pop r8
|
|
pop rdx
|
|
pop rcx
|
|
pop rax
|
|
iretq
|
|
|
|
NmiUnhandled:
|
|
pop r13
|
|
pop r12
|
|
pop r11
|
|
pop r10
|
|
pop r9
|
|
pop r8
|
|
pop rdx
|
|
pop rcx
|
|
pop rax
|
|
jmp KeFaultNMI
|
|
KeNmiIpiStub ENDP
|
|
|
|
FAULT_NOERR KeFaultDB, 1
|
|
FAULT_NOERR KeFaultNMI, 2
|
|
FAULT_NOERR KeFaultBP, 3
|
|
FAULT_NOERR KeFaultOF, 4
|
|
FAULT_NOERR KeFaultBR, 5
|
|
FAULT_NOERR KeFaultUD, 6
|
|
FAULT_NOERR KeFaultNM, 7
|
|
FAULT_ERR KeFaultDF, 8
|
|
FAULT_NOERR KeFaultCSO, 9
|
|
FAULT_ERR KeFaultTS, 10
|
|
FAULT_ERR KeFaultNP, 11
|
|
FAULT_ERR KeFaultSS, 12
|
|
FAULT_NOERR KeFaultRSV15, 15
|
|
FAULT_NOERR KeFaultMF, 16
|
|
FAULT_ERR KeFaultAC, 17
|
|
FAULT_NOERR KeFaultMC, 18
|
|
FAULT_NOERR KeFaultXM, 19
|
|
FAULT_NOERR KeFaultVE, 20
|
|
FAULT_ERR KeFaultCP, 21
|
|
FAULT_NOERR KeFaultRSV22, 22
|
|
FAULT_NOERR KeFaultRSV23, 23
|
|
FAULT_NOERR KeFaultRSV24, 24
|
|
FAULT_NOERR KeFaultRSV25, 25
|
|
FAULT_NOERR KeFaultRSV26, 26
|
|
FAULT_NOERR KeFaultRSV27, 27
|
|
FAULT_NOERR KeFaultHV, 28
|
|
FAULT_ERR KeFaultVC, 29
|
|
FAULT_ERR KeFaultSX, 30
|
|
FAULT_NOERR KeFaultRSV31, 31
|
|
|
|
EXTERN KeIrqDispatch:PROC
|
|
EXTERN KiEnterInterrupt:PROC
|
|
EXTERN KiExitInterrupt:PROC
|
|
|
|
IrqStub MACRO num
|
|
PUBLIC KeIrqStub&num
|
|
KeIrqStub&num PROC
|
|
push rax
|
|
mov al, (030h + num)
|
|
jmp KeIrqCommon
|
|
KeIrqStub&num ENDP
|
|
ENDM
|
|
|
|
IrqStub 0
|
|
IrqStub 1
|
|
IrqStub 2
|
|
IrqStub 3
|
|
IrqStub 4
|
|
IrqStub 5
|
|
IrqStub 6
|
|
IrqStub 7
|
|
IrqStub 8
|
|
IrqStub 9
|
|
IrqStub 10
|
|
IrqStub 11
|
|
IrqStub 12
|
|
IrqStub 13
|
|
IrqStub 14
|
|
IrqStub 15
|
|
IrqStub 16
|
|
IrqStub 17
|
|
IrqStub 18
|
|
IrqStub 19
|
|
IrqStub 20
|
|
IrqStub 21
|
|
IrqStub 22
|
|
IrqStub 23
|
|
IrqStub 24
|
|
IrqStub 25
|
|
IrqStub 26
|
|
IrqStub 27
|
|
IrqStub 28
|
|
IrqStub 29
|
|
IrqStub 30
|
|
IrqStub 31
|
|
IrqStub 32
|
|
IrqStub 33
|
|
IrqStub 34
|
|
IrqStub 35
|
|
IrqStub 36
|
|
IrqStub 37
|
|
IrqStub 38
|
|
IrqStub 39
|
|
IrqStub 40
|
|
IrqStub 41
|
|
IrqStub 42
|
|
IrqStub 43
|
|
IrqStub 44
|
|
IrqStub 45
|
|
IrqStub 46
|
|
IrqStub 47
|
|
IrqStub 48
|
|
IrqStub 49
|
|
IrqStub 50
|
|
IrqStub 51
|
|
IrqStub 52
|
|
IrqStub 53
|
|
IrqStub 54
|
|
IrqStub 55
|
|
IrqStub 56
|
|
IrqStub 57
|
|
IrqStub 58
|
|
IrqStub 59
|
|
IrqStub 60
|
|
IrqStub 61
|
|
IrqStub 62
|
|
IrqStub 63
|
|
IrqStub 64
|
|
IrqStub 65
|
|
IrqStub 66
|
|
IrqStub 67
|
|
IrqStub 68
|
|
IrqStub 69
|
|
IrqStub 70
|
|
IrqStub 71
|
|
IrqStub 72
|
|
IrqStub 73
|
|
IrqStub 74
|
|
IrqStub 75
|
|
IrqStub 76
|
|
IrqStub 77
|
|
IrqStub 78
|
|
IrqStub 79
|
|
IrqStub 80
|
|
IrqStub 81
|
|
IrqStub 82
|
|
IrqStub 83
|
|
IrqStub 84
|
|
IrqStub 85
|
|
IrqStub 86
|
|
IrqStub 87
|
|
IrqStub 88
|
|
IrqStub 89
|
|
IrqStub 90
|
|
IrqStub 91
|
|
IrqStub 92
|
|
IrqStub 93
|
|
IrqStub 94
|
|
IrqStub 95
|
|
IrqStub 96
|
|
IrqStub 97
|
|
IrqStub 98
|
|
IrqStub 99
|
|
IrqStub 100
|
|
IrqStub 101
|
|
IrqStub 102
|
|
IrqStub 103
|
|
IrqStub 104
|
|
IrqStub 105
|
|
IrqStub 106
|
|
IrqStub 107
|
|
IrqStub 108
|
|
IrqStub 109
|
|
IrqStub 110
|
|
IrqStub 111
|
|
IrqStub 112
|
|
IrqStub 113
|
|
IrqStub 114
|
|
IrqStub 115
|
|
IrqStub 116
|
|
IrqStub 117
|
|
IrqStub 118
|
|
IrqStub 119
|
|
IrqStub 120
|
|
IrqStub 121
|
|
IrqStub 122
|
|
IrqStub 123
|
|
IrqStub 124
|
|
IrqStub 125
|
|
IrqStub 126
|
|
IrqStub 127
|
|
IrqStub 128
|
|
IrqStub 129
|
|
IrqStub 130
|
|
IrqStub 131
|
|
IrqStub 132
|
|
IrqStub 133
|
|
IrqStub 134
|
|
IrqStub 135
|
|
IrqStub 136
|
|
IrqStub 137
|
|
IrqStub 138
|
|
IrqStub 139
|
|
IrqStub 140
|
|
IrqStub 141
|
|
IrqStub 142
|
|
IrqStub 143
|
|
IrqStub 144
|
|
IrqStub 145
|
|
IrqStub 146
|
|
IrqStub 147
|
|
IrqStub 148
|
|
IrqStub 149
|
|
IrqStub 150
|
|
IrqStub 151
|
|
IrqStub 152
|
|
IrqStub 153
|
|
IrqStub 154
|
|
IrqStub 155
|
|
IrqStub 156
|
|
IrqStub 157
|
|
IrqStub 158
|
|
IrqStub 159
|
|
|
|
PUBLIC KeLocalTimerStub
|
|
KeLocalTimerStub PROC
|
|
push rax
|
|
mov al, 0E0h
|
|
jmp KeIrqCommon
|
|
KeLocalTimerStub ENDP
|
|
|
|
PUBLIC KeSpuriousStub
|
|
KeSpuriousStub PROC
|
|
push rax
|
|
mov al, 0FFh
|
|
jmp KeIrqCommon
|
|
KeSpuriousStub ENDP
|
|
|
|
PUBLIC KeRescheduleIpiStub
|
|
KeRescheduleIpiStub PROC
|
|
push rax
|
|
mov al, 0F0h
|
|
jmp KeIrqCommon
|
|
KeRescheduleIpiStub ENDP
|
|
|
|
KeIrqCommon PROC
|
|
; al = IDT vector, original rax saved by the vector stub.
|
|
push rcx
|
|
push rdx
|
|
push r8
|
|
push r9
|
|
push r10
|
|
push r11
|
|
push rbx
|
|
|
|
sub rsp, KTRAP_FRAME_SIZE
|
|
mov rbx, rsp
|
|
|
|
movzx edx, al
|
|
mov rax, KTRAP_FRAME_MAGIC
|
|
mov [rbx + TF_Magic], rax
|
|
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT
|
|
mov [rbx + TF_Vector], edx
|
|
|
|
; Saved volatile registers live immediately above the trap frame.
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 56]
|
|
mov [rbx + TF_Rax], rax
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 48]
|
|
mov [rbx + TF_Rcx], rax
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 40]
|
|
mov [rbx + TF_Rdx], rax
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 0]
|
|
mov [rbx + TF_Rbx], rax
|
|
mov [rbx + TF_Rbp], rbp
|
|
mov [rbx + TF_Rsi], rsi
|
|
mov [rbx + TF_Rdi], rdi
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 32]
|
|
mov [rbx + TF_R8], rax
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 24]
|
|
mov [rbx + TF_R9], rax
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 16]
|
|
mov [rbx + TF_R10], rax
|
|
mov rax, [rbx + KTRAP_FRAME_SIZE + 8]
|
|
mov [rbx + TF_R11], rax
|
|
mov [rbx + TF_R12], r12
|
|
mov [rbx + TF_R13], r13
|
|
mov [rbx + TF_R14], r14
|
|
mov [rbx + TF_R15], r15
|
|
|
|
; CPU interrupt frame starts after the 8 saved registers.
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 64]
|
|
mov [rbx + TF_RipSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Rip], rdx
|
|
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 72]
|
|
mov [rbx + TF_CsSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Cs], rdx
|
|
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 80]
|
|
mov [rbx + TF_RflagsSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Rflags], rdx
|
|
|
|
mov rax, [rbx + TF_Cs]
|
|
and eax, 3
|
|
cmp eax, 3
|
|
jne short IrqKernelModeFrame
|
|
|
|
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 88]
|
|
mov [rbx + TF_RspSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Rsp], rdx
|
|
lea rax, [rbx + KTRAP_FRAME_SIZE + 96]
|
|
mov [rbx + TF_SsSlot], rax
|
|
mov rdx, [rax]
|
|
mov [rbx + TF_Ss], rdx
|
|
swapgs ; interrupted ring 3: install kernel GS/KCPU
|
|
jmp short IrqFrameReady
|
|
|
|
IrqKernelModeFrame:
|
|
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_KERNEL
|
|
mov qword ptr [rbx + TF_Rsp], 0
|
|
mov qword ptr [rbx + TF_Ss], 0
|
|
mov qword ptr [rbx + TF_RspSlot], 0
|
|
mov qword ptr [rbx + TF_SsSlot], 0
|
|
|
|
IrqFrameReady:
|
|
ASYNC_FPU_SAVE
|
|
and rsp, -16
|
|
sub rsp, 32
|
|
mov rcx, rbx
|
|
call KiEnterTrapFrame
|
|
call KiEnterInterrupt
|
|
mov ecx, [rbx + TF_Vector]
|
|
call KeIrqDispatch
|
|
call KiExitInterrupt
|
|
mov rcx, rbx
|
|
call KiExitTrapFrame
|
|
ASYNC_FPU_RESTORE
|
|
|
|
mov eax, [rbx + TF_PreviousMode]
|
|
cmp eax, KTRAP_MODE_USER
|
|
jne short IrqNoSwapGsOut
|
|
swapgs ; returning to ring 3: restore user GS/TEB
|
|
IrqNoSwapGsOut:
|
|
mov r12, [rbx + TF_R12]
|
|
lea rsp, [rbx + KTRAP_FRAME_SIZE]
|
|
pop rbx
|
|
pop r11
|
|
pop r10
|
|
pop r9
|
|
pop r8
|
|
pop rdx
|
|
pop rcx
|
|
pop rax
|
|
iretq
|
|
KeIrqCommon ENDP
|
|
|
|
END
|