; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64) ; ; Syscall entry now builds an editable KTRAP_FRAME on the current kernel ; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET ; slots used by user APC return redirection. EXTERN KeSyscallDispatch:PROC EXTERN KiEnterTrapFrame:PROC EXTERN KiExitTrapFrame:PROC EXTERN KeFaultDispatchFrame:PROC KTRAP_FRAME_SIZE EQU 240 KTRAP_FRAME_MAGIC EQU 4650545241544B31h KTRAP_KIND_SYSCALL EQU 1 KTRAP_KIND_INTERRUPT EQU 2 KTRAP_MODE_KERNEL EQU 0 KTRAP_MODE_USER EQU 1 TF_Magic EQU 0 TF_Kind EQU 8 TF_Vector EQU 12 TF_PreviousMode EQU 16 TF_Rax EQU 24 TF_Rcx EQU 32 TF_Rdx EQU 40 TF_Rbx EQU 48 TF_Rbp EQU 56 TF_Rsi EQU 64 TF_Rdi EQU 72 TF_R8 EQU 80 TF_R9 EQU 88 TF_R10 EQU 96 TF_R11 EQU 104 TF_R12 EQU 112 TF_R13 EQU 120 TF_R14 EQU 128 TF_R15 EQU 136 TF_Rip EQU 144 TF_Cs EQU 152 TF_Rflags EQU 160 TF_Rsp EQU 168 TF_Ss EQU 176 TF_RipSlot EQU 184 TF_CsSlot EQU 192 TF_RflagsSlot EQU 200 TF_RspSlot EQU 208 TF_SsSlot EQU 216 .CODE KeSyscallEntry PROC swapgs ; GS_BASE: user TEB -> KCPU, KERNEL_GS_BASE: KCPU -> user TEB ; CPU frame at entry, ring 3 -> ring 0: ; [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS, ; [rsp+18] user RSP, [rsp+20] SS ; Save RBX because we use it as the trap-frame pointer and user-mode code ; sees the syscall stub as a normal function call. push rbx sub rsp, KTRAP_FRAME_SIZE mov rbx, rsp mov [rbx + TF_Rax], rax mov rax, KTRAP_FRAME_MAGIC mov [rbx + TF_Magic], rax mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL mov dword ptr [rbx + TF_Vector], 40h mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER mov [rbx + TF_Rcx], rcx mov [rbx + TF_Rdx], rdx mov rax, [rbx + KTRAP_FRAME_SIZE] mov [rbx + TF_Rbx], rax mov [rbx + TF_Rbp], rbp mov [rbx + TF_Rsi], rsi mov [rbx + TF_Rdi], rdi mov [rbx + TF_R8], r8 mov [rbx + TF_R9], r9 mov [rbx + TF_R10], r10 mov [rbx + TF_R11], r11 mov [rbx + TF_R12], r12 mov [rbx + TF_R13], r13 mov [rbx + TF_R14], r14 mov [rbx + TF_R15], r15 lea rax, [rbx + KTRAP_FRAME_SIZE + 8] mov [rbx + TF_RipSlot], rax mov rdx, [rax] mov [rbx + TF_Rip], rdx lea rax, [rbx + KTRAP_FRAME_SIZE + 16] mov [rbx + TF_CsSlot], rax mov rdx, [rax] mov [rbx + TF_Cs], rdx lea rax, [rbx + KTRAP_FRAME_SIZE + 24] mov [rbx + TF_RflagsSlot], rax mov rdx, [rax] mov [rbx + TF_Rflags], rdx lea rax, [rbx + KTRAP_FRAME_SIZE + 32] mov [rbx + TF_RspSlot], rax mov rdx, [rax] mov [rbx + TF_Rsp], rdx lea rax, [rbx + KTRAP_FRAME_SIZE + 40] mov [rbx + TF_SsSlot], rax mov rdx, [rax] mov [rbx + TF_Ss], rdx sub rsp, 32 mov rcx, rbx call KiEnterTrapFrame add rsp, 32 ; Re-order original user arguments into KeSyscallDispatch(nr,a0..a5). ; Original registers were saved in the trap frame before any C calls. sub rsp, 64 mov rax, [rbx + TF_R9] mov [rsp + 32], rax ; a3 mov rax, [rbx + TF_R10] mov [rsp + 40], rax ; a4 mov rax, [rbx + TF_R11] mov [rsp + 48], rax ; a5 mov r9, [rbx + TF_R8] ; a2 mov r8, [rbx + TF_Rdx] ; a1 mov rdx, [rbx + TF_Rcx] ; a0 mov rcx, [rbx + TF_Rax] ; nr call KeSyscallDispatch add rsp, 64 mov [rbx + TF_Rax], rax sub rsp, 32 mov rcx, rbx call KiExitTrapFrame add rsp, 32 mov rax, [rbx + TF_Rax] add rsp, KTRAP_FRAME_SIZE pop rbx swapgs ; GS_BASE: KCPU -> user TEB before returning to ring 3 iretq KeSyscallEntry ENDP ; --------------------------------------------------------------------------- ; Fault stubs - snapshot a full user fault frame before entering C. ; ; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless ; by itself. The captured user RSP lets the C crash dumper read [RSP], which is ; usually the return address immediately after the bad CALL. The GPR snapshot ; also shows which register held the null target. ; ; KeFaultDispatchFrame signature (MS x64): ; void KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx ; --------------------------------------------------------------------------- KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish KFAULT_SIZE EQU 0C0h KF_Magic EQU 000h KF_Vector EQU 008h KF_ErrorCode EQU 010h KF_Cr2 EQU 018h KF_Rip EQU 020h KF_Cs EQU 028h KF_Rflags EQU 030h KF_Rsp EQU 038h KF_Ss EQU 040h KF_Rax EQU 048h KF_Rcx EQU 050h KF_Rdx EQU 058h KF_Rbx EQU 060h KF_Rbp EQU 068h KF_Rsi EQU 070h KF_Rdi EQU 078h KF_R8 EQU 080h KF_R9 EQU 088h KF_R10 EQU 090h KF_R11 EQU 098h KF_R12 EQU 0A0h KF_R13 EQU 0A8h KF_R14 EQU 0B0h KF_R15 EQU 0B8h FAULT_CALL_SHADOW EQU 28h ; --------------------------------------------------------- ; Divide Error (#0) - NO error code pushed by CPU ; User exception frame: [E]=RIP [E+8]=CS [E+16]=RFLAGS [E+24]=RSP [E+32]=SS ; --------------------------------------------------------- KeFaultDE PROC push rax ; [rsp+8] after next push = original RAX push r11 ; [rsp] = original R11 mov r11, rsp ; saved pair; CPU frame starts at [r11+16] sub rsp, KFAULT_SIZE mov rax, rsp ; rax = KFAULT_FRAME* mov [rax + KF_R10], r10 mov r10, KFAULT_MAGIC mov [rax + KF_Magic], r10 mov qword ptr [rax + KF_Vector], 0 mov qword ptr [rax + KF_ErrorCode], 0 mov qword ptr [rax + KF_Cr2], 0 mov [rax + KF_Rcx], rcx mov [rax + KF_Rdx], rdx mov [rax + KF_Rbx], rbx mov [rax + KF_Rbp], rbp mov [rax + KF_Rsi], rsi mov [rax + KF_Rdi], rdi mov [rax + KF_R8], r8 mov [rax + KF_R9], r9 mov r10, [r11] mov [rax + KF_R11], r10 mov r10, [r11+8] mov [rax + KF_Rax], r10 mov [rax + KF_R12], r12 mov [rax + KF_R13], r13 mov [rax + KF_R14], r14 mov [rax + KF_R15], r15 lea rdx, [r11+16] mov r10, [rdx] mov [rax + KF_Rip], r10 mov r10, [rdx+8] mov [rax + KF_Cs], r10 mov r10, [rdx+16] mov [rax + KF_Rflags], r10 mov r10, [rax + KF_Cs] test r10, 3 jz de_kernel_frame mov r10, [rdx+24] mov [rax + KF_Rsp], r10 mov r10, [rdx+32] mov [rax + KF_Ss], r10 swapgs jmp de_call de_kernel_frame: lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 de_call: and rsp, -16 sub rsp, FAULT_CALL_SHADOW mov rcx, rax call KeFaultDispatchFrame KeFaultDE ENDP ; --------------------------------------------------------- ; General Protection Fault (#13) - error code pushed by CPU ; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS ; --------------------------------------------------------- KeFaultGP PROC push rax push r11 mov r11, rsp sub rsp, KFAULT_SIZE mov rax, rsp mov [rax + KF_R10], r10 mov r10, KFAULT_MAGIC mov [rax + KF_Magic], r10 mov qword ptr [rax + KF_Vector], 13 mov qword ptr [rax + KF_Cr2], 0 mov [rax + KF_Rcx], rcx mov [rax + KF_Rdx], rdx mov [rax + KF_Rbx], rbx mov [rax + KF_Rbp], rbp mov [rax + KF_Rsi], rsi mov [rax + KF_Rdi], rdi mov [rax + KF_R8], r8 mov [rax + KF_R9], r9 mov r10, [r11] mov [rax + KF_R11], r10 mov r10, [r11+8] mov [rax + KF_Rax], r10 mov [rax + KF_R12], r12 mov [rax + KF_R13], r13 mov [rax + KF_R14], r14 mov [rax + KF_R15], r15 lea rdx, [r11+16] mov r10, [rdx] mov [rax + KF_ErrorCode], r10 mov r10, [rdx+8] mov [rax + KF_Rip], r10 mov r10, [rdx+16] mov [rax + KF_Cs], r10 mov r10, [rdx+24] mov [rax + KF_Rflags], r10 mov r10, [rax + KF_Cs] test r10, 3 jz gp_kernel_frame mov r10, [rdx+32] mov [rax + KF_Rsp], r10 mov r10, [rdx+40] mov [rax + KF_Ss], r10 swapgs jmp gp_call gp_kernel_frame: lea r10, [rdx+32] ; same-privilege fault: original RSP is above ERR/RIP/CS/RFLAGS mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 gp_call: and rsp, -16 sub rsp, FAULT_CALL_SHADOW mov rcx, rax call KeFaultDispatchFrame KeFaultGP ENDP ; --------------------------------------------------------- ; Page Fault (#14) - error code pushed by CPU ; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS ; --------------------------------------------------------- KeFaultPF PROC push rax push r11 mov r11, rsp sub rsp, KFAULT_SIZE mov rax, rsp mov [rax + KF_R10], r10 mov r10, KFAULT_MAGIC mov [rax + KF_Magic], r10 mov qword ptr [rax + KF_Vector], 14 mov [rax + KF_Rcx], rcx mov [rax + KF_Rdx], rdx mov [rax + KF_Rbx], rbx mov [rax + KF_Rbp], rbp mov [rax + KF_Rsi], rsi mov [rax + KF_Rdi], rdi mov [rax + KF_R8], r8 mov [rax + KF_R9], r9 mov r10, cr2 mov [rax + KF_Cr2], r10 mov r10, [r11] mov [rax + KF_R11], r10 mov r10, [r11+8] mov [rax + KF_Rax], r10 mov [rax + KF_R12], r12 mov [rax + KF_R13], r13 mov [rax + KF_R14], r14 mov [rax + KF_R15], r15 lea rdx, [r11+16] mov r10, [rdx] mov [rax + KF_ErrorCode], r10 mov r10, [rdx+8] mov [rax + KF_Rip], r10 mov r10, [rdx+16] mov [rax + KF_Cs], r10 mov r10, [rdx+24] mov [rax + KF_Rflags], r10 mov r10, [rax + KF_Cs] test r10, 3 jz pf_kernel_frame mov r10, [rdx+32] mov [rax + KF_Rsp], r10 mov r10, [rdx+40] mov [rax + KF_Ss], r10 swapgs jmp pf_call pf_kernel_frame: lea r10, [rdx+32] ; same-privilege #PF frame has ERR/RIP/CS/RFLAGS mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 pf_call: and rsp, -16 sub rsp, FAULT_CALL_SHADOW mov rcx, rax call KeFaultDispatchFrame KeFaultPF ENDP ; --------------------------------------------------------- ; Generic CPU exception stubs. ; Each vector gets its own IDT entry and vector number so faults like #UD ; do not fall through an absent descriptor and reappear as #GP. ; --------------------------------------------------------- FAULT_NOERR MACRO StubName, VecNum LOCAL noerr_kernel_frame, noerr_call StubName PROC push rax push r11 mov r11, rsp sub rsp, KFAULT_SIZE mov rax, rsp mov [rax + KF_R10], r10 mov r10, KFAULT_MAGIC mov [rax + KF_Magic], r10 mov qword ptr [rax + KF_Vector], VecNum mov qword ptr [rax + KF_ErrorCode], 0 mov qword ptr [rax + KF_Cr2], 0 mov [rax + KF_Rcx], rcx mov [rax + KF_Rdx], rdx mov [rax + KF_Rbx], rbx mov [rax + KF_Rbp], rbp mov [rax + KF_Rsi], rsi mov [rax + KF_Rdi], rdi mov [rax + KF_R8], r8 mov [rax + KF_R9], r9 mov r10, [r11] mov [rax + KF_R11], r10 mov r10, [r11+8] mov [rax + KF_Rax], r10 mov [rax + KF_R12], r12 mov [rax + KF_R13], r13 mov [rax + KF_R14], r14 mov [rax + KF_R15], r15 lea rdx, [r11+16] mov r10, [rdx] mov [rax + KF_Rip], r10 mov r10, [rdx+8] mov [rax + KF_Cs], r10 mov r10, [rdx+16] mov [rax + KF_Rflags], r10 mov r10, [rax + KF_Cs] test r10, 3 jz noerr_kernel_frame mov r10, [rdx+24] mov [rax + KF_Rsp], r10 mov r10, [rdx+32] mov [rax + KF_Ss], r10 swapgs jmp noerr_call noerr_kernel_frame: lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 noerr_call: and rsp, -16 sub rsp, FAULT_CALL_SHADOW mov rcx, rax call KeFaultDispatchFrame StubName ENDP ENDM FAULT_ERR MACRO StubName, VecNum LOCAL err_kernel_frame, err_call StubName PROC push rax push r11 mov r11, rsp sub rsp, KFAULT_SIZE mov rax, rsp mov [rax + KF_R10], r10 mov r10, KFAULT_MAGIC mov [rax + KF_Magic], r10 mov qword ptr [rax + KF_Vector], VecNum mov qword ptr [rax + KF_Cr2], 0 mov [rax + KF_Rcx], rcx mov [rax + KF_Rdx], rdx mov [rax + KF_Rbx], rbx mov [rax + KF_Rbp], rbp mov [rax + KF_Rsi], rsi mov [rax + KF_Rdi], rdi mov [rax + KF_R8], r8 mov [rax + KF_R9], r9 mov r10, [r11] mov [rax + KF_R11], r10 mov r10, [r11+8] mov [rax + KF_Rax], r10 mov [rax + KF_R12], r12 mov [rax + KF_R13], r13 mov [rax + KF_R14], r14 mov [rax + KF_R15], r15 lea rdx, [r11+16] mov r10, [rdx] mov [rax + KF_ErrorCode], r10 mov r10, [rdx+8] mov [rax + KF_Rip], r10 mov r10, [rdx+16] mov [rax + KF_Cs], r10 mov r10, [rdx+24] mov [rax + KF_Rflags], r10 mov r10, [rax + KF_Cs] test r10, 3 jz err_kernel_frame mov r10, [rdx+32] mov [rax + KF_Rsp], r10 mov r10, [rdx+40] mov [rax + KF_Ss], r10 swapgs jmp err_call err_kernel_frame: lea r10, [rdx+32] ; same-privilege error-code frame has ERR/RIP/CS/RFLAGS mov [rax + KF_Rsp], r10 mov qword ptr [rax + KF_Ss], 0 err_call: and rsp, -16 sub rsp, FAULT_CALL_SHADOW mov rcx, rax call KeFaultDispatchFrame StubName ENDP ENDM EXTERN KeHandleNmiIpi:PROC PUBLIC KeNmiIpiStub KeNmiIpiStub PROC ; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1, ; the NMI was a panic/TLB-control event and is fully handled. If it ; returns 0, restore the interrupted context and fall through to the normal ; #NMI fault stub so unexpected NMIs still panic with a useful frame. push rax push rcx push rdx push r8 push r9 push r10 push r11 sub rsp, 32 call KeHandleNmiIpi add rsp, 32 test eax, eax jz short NmiUnhandled pop r11 pop r10 pop r9 pop r8 pop rdx pop rcx pop rax iretq NmiUnhandled: pop r11 pop r10 pop r9 pop r8 pop rdx pop rcx pop rax jmp KeFaultNMI KeNmiIpiStub ENDP FAULT_NOERR KeFaultDB, 1 FAULT_NOERR KeFaultNMI, 2 FAULT_NOERR KeFaultBP, 3 FAULT_NOERR KeFaultOF, 4 FAULT_NOERR KeFaultBR, 5 FAULT_NOERR KeFaultUD, 6 FAULT_NOERR KeFaultNM, 7 FAULT_ERR KeFaultDF, 8 FAULT_NOERR KeFaultCSO, 9 FAULT_ERR KeFaultTS, 10 FAULT_ERR KeFaultNP, 11 FAULT_ERR KeFaultSS, 12 FAULT_NOERR KeFaultRSV15, 15 FAULT_NOERR KeFaultMF, 16 FAULT_ERR KeFaultAC, 17 FAULT_NOERR KeFaultMC, 18 FAULT_NOERR KeFaultXM, 19 FAULT_NOERR KeFaultVE, 20 FAULT_ERR KeFaultCP, 21 FAULT_NOERR KeFaultRSV22, 22 FAULT_NOERR KeFaultRSV23, 23 FAULT_NOERR KeFaultRSV24, 24 FAULT_NOERR KeFaultRSV25, 25 FAULT_NOERR KeFaultRSV26, 26 FAULT_NOERR KeFaultRSV27, 27 FAULT_NOERR KeFaultHV, 28 FAULT_ERR KeFaultVC, 29 FAULT_ERR KeFaultSX, 30 FAULT_NOERR KeFaultRSV31, 31 EXTERN KeIrqDispatch:PROC EXTERN KiEnterInterrupt:PROC EXTERN KiExitInterrupt:PROC IrqStub MACRO num PUBLIC KeIrqStub&num KeIrqStub&num PROC push rax mov al, num jmp KeIrqCommon KeIrqStub&num ENDP ENDM IrqStub 0 IrqStub 1 IrqStub 2 IrqStub 3 IrqStub 4 IrqStub 5 IrqStub 6 IrqStub 7 IrqStub 8 IrqStub 9 IrqStub 10 IrqStub 11 IrqStub 12 IrqStub 13 IrqStub 14 IrqStub 15 IrqStub 16 IrqStub 17 IrqStub 18 IrqStub 19 IrqStub 20 IrqStub 21 IrqStub 22 IrqStub 23 PUBLIC KeRescheduleIpiStub KeRescheduleIpiStub PROC push rax mov al, 0F0h jmp KeIrqCommon KeRescheduleIpiStub ENDP KeIrqCommon PROC ; al = irq number, original rax saved by the per-IRQ stub. push rcx push rdx push r8 push r9 push r10 push r11 push rbx sub rsp, KTRAP_FRAME_SIZE mov rbx, rsp movzx edx, al mov rax, KTRAP_FRAME_MAGIC mov [rbx + TF_Magic], rax mov dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT mov [rbx + TF_Vector], edx ; Saved volatile registers live immediately above the trap frame. mov rax, [rbx + KTRAP_FRAME_SIZE + 56] mov [rbx + TF_Rax], rax mov rax, [rbx + KTRAP_FRAME_SIZE + 48] mov [rbx + TF_Rcx], rax mov rax, [rbx + KTRAP_FRAME_SIZE + 40] mov [rbx + TF_Rdx], rax mov rax, [rbx + KTRAP_FRAME_SIZE + 0] mov [rbx + TF_Rbx], rax mov [rbx + TF_Rbp], rbp mov [rbx + TF_Rsi], rsi mov [rbx + TF_Rdi], rdi mov rax, [rbx + KTRAP_FRAME_SIZE + 32] mov [rbx + TF_R8], rax mov rax, [rbx + KTRAP_FRAME_SIZE + 24] mov [rbx + TF_R9], rax mov rax, [rbx + KTRAP_FRAME_SIZE + 16] mov [rbx + TF_R10], rax mov rax, [rbx + KTRAP_FRAME_SIZE + 8] mov [rbx + TF_R11], rax mov [rbx + TF_R12], r12 mov [rbx + TF_R13], r13 mov [rbx + TF_R14], r14 mov [rbx + TF_R15], r15 ; CPU interrupt frame starts after the 8 saved registers. lea rax, [rbx + KTRAP_FRAME_SIZE + 64] mov [rbx + TF_RipSlot], rax mov rdx, [rax] mov [rbx + TF_Rip], rdx lea rax, [rbx + KTRAP_FRAME_SIZE + 72] mov [rbx + TF_CsSlot], rax mov rdx, [rax] mov [rbx + TF_Cs], rdx lea rax, [rbx + KTRAP_FRAME_SIZE + 80] mov [rbx + TF_RflagsSlot], rax mov rdx, [rax] mov [rbx + TF_Rflags], rdx mov rax, [rbx + TF_Cs] and eax, 3 cmp eax, 3 jne short IrqKernelModeFrame mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER lea rax, [rbx + KTRAP_FRAME_SIZE + 88] mov [rbx + TF_RspSlot], rax mov rdx, [rax] mov [rbx + TF_Rsp], rdx lea rax, [rbx + KTRAP_FRAME_SIZE + 96] mov [rbx + TF_SsSlot], rax mov rdx, [rax] mov [rbx + TF_Ss], rdx swapgs ; interrupted ring 3: install kernel GS/KCPU jmp short IrqFrameReady IrqKernelModeFrame: mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_KERNEL mov qword ptr [rbx + TF_Rsp], 0 mov qword ptr [rbx + TF_Ss], 0 mov qword ptr [rbx + TF_RspSlot], 0 mov qword ptr [rbx + TF_SsSlot], 0 IrqFrameReady: sub rsp, 40 ; 32-byte shadow space + alignment/local slack mov rcx, rbx call KiEnterTrapFrame call KiEnterInterrupt mov ecx, [rbx + TF_Vector] call KeIrqDispatch call KiExitInterrupt mov rcx, rbx call KiExitTrapFrame add rsp, 40 mov eax, [rbx + TF_PreviousMode] cmp eax, KTRAP_MODE_USER jne short IrqNoSwapGsOut swapgs ; returning to ring 3: restore user GS/TEB IrqNoSwapGsOut: add rsp, KTRAP_FRAME_SIZE pop rbx pop r11 pop r10 pop r9 pop r8 pop rdx pop rcx pop rax iretq KeIrqCommon ENDP END