maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
d36f19509c72ba98336e2d83d22e7fce55c1e759
nt-lite/code/kernel/ke_syscall.asm
T
maksw 73abd6f90d isolate POSIX builds from the host UCRT, wire lwext4 and ntfs-3g as ext4.sys and ntfs.sys, split the FAT driver into directory/file/async/driver/mount units, add posixsrv and dwm plus input/console/desktop syscall clients and a pile of new smoke targets, rename serial.sys to cereal.sys, name HDA/USB/lwIP/AHCI workers, harden ATA probing/reset/removal around actual hardware deadlines, absent-channel detection, diagnostics and explicit worker lifetime, redo ACPI S5 shutdown around GAS/MMIO and HW-reduced firmware, scan SSDTs for _S5 and deal with Intel SMI sleep trapping, make serial and SMP timeouts use real monotonic deadlines, add thread names and fix terminal thread teardown, scheduler and APC prepare/commit races, make advisory file locks queryable and preserve split ranges and add mutable file append state, add shared-memory pipes with kernel-assisted wait/wake, introduce capability-scoped input and display managers, split surface map rights into read/write, add explicit unmap and replace present with commit/generation/damage tracking while keeping physical presentation under the display manager, decouple native exception classes from Win32 exception numbers and translate them in kernel32, add opaque process startup data, bump the resource protocol to v3 with namespace directories, symlinks and enumeration and service control to v2 with generic READY handle exports, add the new POSIX errno/status surface and rebuild fds around posixsrv-owned shared open descriptions, descriptor-local flags, exact inheritance snapshots, dup/fcntl/record locks and shared-ring pipes, add waitpid, POSIX credentials/groups/sessions/passwd/group lookups and move sockets onto stable open descriptions, port the shells and apps to the new console/input services and surface commit API, teach init to load services.ini and start/supervise RSSRV, DWM and generic services under restricted capabilities while publishing their endpoints through RSSRV, split RSSRV into namespace, volume, display, session, input, console and resource components and make /SystemRoot a real symlink to the independently published system volume, shift the syscall table again, expand lock/resource/surface/POSIX/service/thread smoke coverage, make docs scream ASK QUESTIONS and DISCUSS at LLMs, replace MIT No Attribution with WTFPL+.
2026-09-16 10:33:20 +02:00

938 lines
24 KiB
NASM
Raw Blame History

; ke_syscall.asm - INT 0x40, fault, and IRQ kernel entries (MASM64)
;
; Syscall entry now builds an editable KTRAP_FRAME on the current kernel
; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET
; slots used by user APC return redirection.
EXTERN KeSyscallDispatch:PROC
EXTERN KiEnterTrapFrame:PROC
EXTERN KiExitTrapFrame:PROC
EXTERN KiDeliverKernelApcs:PROC
EXTERN KeFaultDispatchFrame:PROC
KTRAP_FRAME_SIZE EQU 240
KTRAP_FRAME_MAGIC EQU 4650545241544B31h
KTRAP_KIND_SYSCALL EQU 1
KTRAP_KIND_INTERRUPT EQU 2
KTRAP_MODE_KERNEL EQU 0
KTRAP_MODE_USER EQU 1
TF_Magic EQU 0
TF_Kind EQU 8
TF_Vector EQU 12
TF_PreviousMode EQU 16
TF_Rax EQU 24
TF_Rcx EQU 32
TF_Rdx EQU 40
TF_Rbx EQU 48
TF_Rbp EQU 56
TF_Rsi EQU 64
TF_Rdi EQU 72
TF_R8 EQU 80
TF_R9 EQU 88
TF_R10 EQU 96
TF_R11 EQU 104
TF_R12 EQU 112
TF_R13 EQU 120
TF_R14 EQU 128
TF_R15 EQU 136
TF_Rip EQU 144
TF_Cs EQU 152
TF_Rflags EQU 160
TF_Rsp EQU 168
TF_Ss EQU 176
TF_RipSlot EQU 184
TF_CsSlot EQU 192
TF_RflagsSlot EQU 200
TF_RspSlot EQU 208
TF_SsSlot EQU 216
.CODE
KeSyscallEntry PROC
swapgs ; GS_BASE: user TEB -> KCPU, KERNEL_GS_BASE: KCPU -> user TEB
; CPU frame at entry, ring 3 -> ring 0:
; [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS,
; [rsp+18] user RSP, [rsp+20] SS
; Save RBX because we use it as the trap-frame pointer and user-mode code
; sees the syscall stub as a normal function call.
push rbx
sub rsp, KTRAP_FRAME_SIZE
mov rbx, rsp
mov [rbx + TF_Rax], rax
mov rax, KTRAP_FRAME_MAGIC
mov [rbx + TF_Magic], rax
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL
mov dword ptr [rbx + TF_Vector], 40h
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
mov [rbx + TF_Rcx], rcx
mov [rbx + TF_Rdx], rdx
mov rax, [rbx + KTRAP_FRAME_SIZE]
mov [rbx + TF_Rbx], rax
mov [rbx + TF_Rbp], rbp
mov [rbx + TF_Rsi], rsi
mov [rbx + TF_Rdi], rdi
mov [rbx + TF_R8], r8
mov [rbx + TF_R9], r9
mov [rbx + TF_R10], r10
mov [rbx + TF_R11], r11
mov [rbx + TF_R12], r12
mov [rbx + TF_R13], r13
mov [rbx + TF_R14], r14
mov [rbx + TF_R15], r15
lea rax, [rbx + KTRAP_FRAME_SIZE + 8]
mov [rbx + TF_RipSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rip], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 16]
mov [rbx + TF_CsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Cs], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 24]
mov [rbx + TF_RflagsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rflags], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 32]
mov [rbx + TF_RspSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rsp], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 40]
mov [rbx + TF_SsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Ss], rdx
sub rsp, 32
mov rcx, rbx
call KiEnterTrapFrame
add rsp, 32
; The interrupt gate protects the ring transition and SWAPGS/trap-frame
; construction. Once the thread context is fully established, syscalls are
; ordinary interruptible PASSIVE_LEVEL execution rather than one giant
; interrupt-disabled critical section.
sti
; Re-order original user arguments into KeSyscallDispatch(nr,a0..a3).
; Original registers were saved in the trap frame before any C calls.
sub rsp, 48
mov rax, [rbx + TF_R9]
mov [rsp + 32], rax ; a3
mov r9, [rbx + TF_R8] ; a2
mov r8, [rbx + TF_Rdx] ; a1
mov rdx, [rbx + TF_Rcx] ; a0
mov rcx, [rbx + TF_Rax] ; nr
call KeSyscallDispatch
add rsp, 48
mov [rbx + TF_Rax], rax
; Syscall return is a scheduler-safe kernel APC delivery boundary. Keep IF
; set while normal routines run, then close the tiny machine-exit window
; before committing the editable IRET frame and restoring user GS.
sub rsp, 32
call KiDeliverKernelApcs
add rsp, 32
cli
sub rsp, 32
mov rcx, rbx
call KiExitTrapFrame
add rsp, 32
; Restore the editable trap-frame register image. Normal syscalls only
; require RAX, but exception continuation may redirect the IRET to an
; arbitrary saved user context that needs the full GPR set.
mov rax, [rbx + TF_Rax]
mov rcx, [rbx + TF_Rcx]
mov rdx, [rbx + TF_Rdx]
mov rbp, [rbx + TF_Rbp]
mov rsi, [rbx + TF_Rsi]
mov rdi, [rbx + TF_Rdi]
mov r8, [rbx + TF_R8]
mov r9, [rbx + TF_R9]
mov r10, [rbx + TF_R10]
mov r11, [rbx + TF_R11]
mov r12, [rbx + TF_R12]
mov r13, [rbx + TF_R13]
mov r14, [rbx + TF_R14]
mov r15, [rbx + TF_R15]
lea rsp, [rbx + KTRAP_FRAME_SIZE + 8]
mov rbx, [rbx + TF_Rbx]
swapgs ; GS_BASE: KCPU -> user TEB before returning to ring 3
iretq
KeSyscallEntry ENDP
; ---------------------------------------------------------------------------
; Fault stubs - snapshot a full user fault frame before entering C.
;
; For null indirect calls/jumps the CPU reports RIP=0, which is almost useless
; by itself. The captured user RSP lets the C crash dumper read [RSP], which is
; usually the return address immediately after the bad CALL. The GPR snapshot
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
; KSTATUS KeFaultDispatchFrame(KFAULT_FRAME* frame) ; rcx
; ---------------------------------------------------------------------------
KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
KFAULT_SIZE EQU 0C0h
KF_Magic EQU 000h
KF_Vector EQU 008h
KF_ErrorCode EQU 010h
KF_Cr2 EQU 018h
KF_Rip EQU 020h
KF_Cs EQU 028h
KF_Rflags EQU 030h
KF_Rsp EQU 038h
KF_Ss EQU 040h
KF_Rax EQU 048h
KF_Rcx EQU 050h
KF_Rdx EQU 058h
KF_Rbx EQU 060h
KF_Rbp EQU 068h
KF_Rsi EQU 070h
KF_Rdi EQU 078h
KF_R8 EQU 080h
KF_R9 EQU 088h
KF_R10 EQU 090h
KF_R11 EQU 098h
KF_R12 EQU 0A0h
KF_R13 EQU 0A8h
KF_R14 EQU 0B0h
KF_R15 EQU 0B8h
; Asynchronous kernel entry must preserve the full interrupted x87/SSE state.
; MS x64 calls may freely clobber XMM0-XMM5, and an interrupt or fault can
; arrive between any two user or kernel instructions rather than at a call ABI.
; Reserve one 512-byte FXSAVE image plus alignment slack on the interrupted stack.
ASYNC_FPU_SCRATCH EQU 528
ASYNC_FPU_SAVE MACRO
sub rsp, ASYNC_FPU_SCRATCH
lea r12, [rsp + 15]
and r12, -16
fxsave64 [r12]
ENDM
ASYNC_FPU_RESTORE MACRO
fxrstor64 [r12]
ENDM
FAULT_RESUME_NOERR MACRO
LOCAL noerr_skip_user_slots, noerr_no_swapgs, noerr_unhandled_halt
test eax, eax
jnz noerr_unhandled_halt
mov rdx, rbx
mov rax, [rdx + KF_Rip]
mov [rdx + KFAULT_SIZE + 16], rax
mov rax, [rdx + KF_Cs]
mov [rdx + KFAULT_SIZE + 24], rax
mov rax, [rdx + KF_Rflags]
mov [rdx + KFAULT_SIZE + 32], rax
mov rax, [rdx + KF_Cs]
test rax, 3
jz noerr_skip_user_slots
mov rax, [rdx + KF_Rsp]
mov [rdx + KFAULT_SIZE + 40], rax
mov rax, [rdx + KF_Ss]
mov [rdx + KFAULT_SIZE + 48], rax
noerr_skip_user_slots:
mov rcx, [rdx + KF_Rcx]
mov r8, [rdx + KF_R8]
mov r9, [rdx + KF_R9]
mov rsi, [rdx + KF_Rsi]
mov rdi, [rdx + KF_Rdi]
mov rbp, [rdx + KF_Rbp]
mov r12, [rdx + KF_R12]
mov r13, [rdx + KF_R13]
mov r14, [rdx + KF_R14]
mov r15, [rdx + KF_R15]
lea rsp, [rdx + KFAULT_SIZE + 16]
mov rax, [rdx + KF_Rax]
mov r11, [rdx + KF_R11]
mov r10, [rdx + KF_Cs]
test r10, 3
jz noerr_no_swapgs
mov r10, [rdx + KF_R10]
mov rbx, [rdx + KF_Rbx]
mov rdx, [rdx + KF_Rdx]
swapgs
iretq
noerr_no_swapgs:
mov r10, [rdx + KF_R10]
mov rbx, [rdx + KF_Rbx]
mov rdx, [rdx + KF_Rdx]
iretq
noerr_unhandled_halt:
cli
@@:
hlt
jmp @B
ENDM
FAULT_RESUME_ERR MACRO
LOCAL err_skip_user_slots, err_no_swapgs, err_unhandled_halt
test eax, eax
jnz err_unhandled_halt
mov rdx, rbx
mov rax, [rdx + KF_Rip]
mov [rdx + KFAULT_SIZE + 24], rax
mov rax, [rdx + KF_Cs]
mov [rdx + KFAULT_SIZE + 32], rax
mov rax, [rdx + KF_Rflags]
mov [rdx + KFAULT_SIZE + 40], rax
mov rax, [rdx + KF_Cs]
test rax, 3
jz err_skip_user_slots
mov rax, [rdx + KF_Rsp]
mov [rdx + KFAULT_SIZE + 48], rax
mov rax, [rdx + KF_Ss]
mov [rdx + KFAULT_SIZE + 56], rax
err_skip_user_slots:
mov rcx, [rdx + KF_Rcx]
mov r8, [rdx + KF_R8]
mov r9, [rdx + KF_R9]
mov rsi, [rdx + KF_Rsi]
mov rdi, [rdx + KF_Rdi]
mov rbp, [rdx + KF_Rbp]
mov r12, [rdx + KF_R12]
mov r13, [rdx + KF_R13]
mov r14, [rdx + KF_R14]
mov r15, [rdx + KF_R15]
lea rsp, [rdx + KFAULT_SIZE + 24]
mov rax, [rdx + KF_Rax]
mov r11, [rdx + KF_R11]
mov r10, [rdx + KF_Cs]
test r10, 3
jz err_no_swapgs
mov r10, [rdx + KF_R10]
mov rbx, [rdx + KF_Rbx]
mov rdx, [rdx + KF_Rdx]
swapgs
iretq
err_no_swapgs:
mov r10, [rdx + KF_R10]
mov rbx, [rdx + KF_Rbx]
mov rdx, [rdx + KF_Rdx]
iretq
err_unhandled_halt:
cli
@@:
hlt
jmp @B
ENDM
; ---------------------------------------------------------
; Divide Error (#0) - NO error code pushed by CPU
; User exception frame: [E]=RIP [E+8]=CS [E+16]=RFLAGS [E+24]=RSP [E+32]=SS
; ---------------------------------------------------------
KeFaultDE PROC
push rax ; [rsp+8] after next push = original RAX
push r11 ; [rsp] = original R11
mov r11, rsp ; saved pair; CPU frame starts at [r11+16]
sub rsp, KFAULT_SIZE
mov rax, rsp ; rax = KFAULT_FRAME*
mov [rax + KF_R10], r10
mov r10, KFAULT_MAGIC
mov [rax + KF_Magic], r10
mov qword ptr [rax + KF_Vector], 0
mov qword ptr [rax + KF_ErrorCode], 0
mov qword ptr [rax + KF_Cr2], 0
mov [rax + KF_Rcx], rcx
mov [rax + KF_Rdx], rdx
mov [rax + KF_Rbx], rbx
mov [rax + KF_Rbp], rbp
mov [rax + KF_Rsi], rsi
mov [rax + KF_Rdi], rdi
mov [rax + KF_R8], r8
mov [rax + KF_R9], r9
mov r10, [r11]
mov [rax + KF_R11], r10
mov r10, [r11+8]
mov [rax + KF_Rax], r10
mov [rax + KF_R12], r12
mov [rax + KF_R13], r13
mov [rax + KF_R14], r14
mov [rax + KF_R15], r15
lea rdx, [r11+16]
mov r10, [rdx]
mov [rax + KF_Rip], r10
mov r10, [rdx+8]
mov [rax + KF_Cs], r10
mov r10, [rdx+16]
mov [rax + KF_Rflags], r10
mov r10, [rax + KF_Cs]
test r10, 3
jz de_kernel_frame
mov r10, [rdx+24]
mov [rax + KF_Rsp], r10
mov r10, [rdx+32]
mov [rax + KF_Ss], r10
swapgs
jmp de_call
de_kernel_frame:
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
de_call:
mov rbx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_NOERR
KeFaultDE ENDP
; ---------------------------------------------------------
; General Protection Fault (#13) - error code pushed by CPU
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
; ---------------------------------------------------------
KeFaultGP PROC
push rax
push r11
mov r11, rsp
sub rsp, KFAULT_SIZE
mov rax, rsp
mov [rax + KF_R10], r10
mov r10, KFAULT_MAGIC
mov [rax + KF_Magic], r10
mov qword ptr [rax + KF_Vector], 13
mov qword ptr [rax + KF_Cr2], 0
mov [rax + KF_Rcx], rcx
mov [rax + KF_Rdx], rdx
mov [rax + KF_Rbx], rbx
mov [rax + KF_Rbp], rbp
mov [rax + KF_Rsi], rsi
mov [rax + KF_Rdi], rdi
mov [rax + KF_R8], r8
mov [rax + KF_R9], r9
mov r10, [r11]
mov [rax + KF_R11], r10
mov r10, [r11+8]
mov [rax + KF_Rax], r10
mov [rax + KF_R12], r12
mov [rax + KF_R13], r13
mov [rax + KF_R14], r14
mov [rax + KF_R15], r15
lea rdx, [r11+16]
mov r10, [rdx]
mov [rax + KF_ErrorCode], r10
mov r10, [rdx+8]
mov [rax + KF_Rip], r10
mov r10, [rdx+16]
mov [rax + KF_Cs], r10
mov r10, [rdx+24]
mov [rax + KF_Rflags], r10
mov r10, [rax + KF_Cs]
test r10, 3
jz gp_kernel_frame
mov r10, [rdx+32]
mov [rax + KF_Rsp], r10
mov r10, [rdx+40]
mov [rax + KF_Ss], r10
swapgs
jmp gp_call
gp_kernel_frame:
lea r10, [rdx+32] ; same-privilege fault: original RSP is above ERR/RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
gp_call:
mov rbx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_ERR
KeFaultGP ENDP
; ---------------------------------------------------------
; Page Fault (#14) - error code pushed by CPU
; User exception frame: [E]=ERR [E+8]=RIP [E+16]=CS [E+24]=RFLAGS [E+32]=RSP [E+40]=SS
; ---------------------------------------------------------
KeFaultPF PROC
push rax
push r11
mov r11, rsp
sub rsp, KFAULT_SIZE
mov rax, rsp
mov [rax + KF_R10], r10
mov r10, KFAULT_MAGIC
mov [rax + KF_Magic], r10
mov qword ptr [rax + KF_Vector], 14
mov [rax + KF_Rcx], rcx
mov [rax + KF_Rdx], rdx
mov [rax + KF_Rbx], rbx
mov [rax + KF_Rbp], rbp
mov [rax + KF_Rsi], rsi
mov [rax + KF_Rdi], rdi
mov [rax + KF_R8], r8
mov [rax + KF_R9], r9
mov r10, cr2
mov [rax + KF_Cr2], r10
mov r10, [r11]
mov [rax + KF_R11], r10
mov r10, [r11+8]
mov [rax + KF_Rax], r10
mov [rax + KF_R12], r12
mov [rax + KF_R13], r13
mov [rax + KF_R14], r14
mov [rax + KF_R15], r15
lea rdx, [r11+16]
mov r10, [rdx]
mov [rax + KF_ErrorCode], r10
mov r10, [rdx+8]
mov [rax + KF_Rip], r10
mov r10, [rdx+16]
mov [rax + KF_Cs], r10
mov r10, [rdx+24]
mov [rax + KF_Rflags], r10
mov r10, [rax + KF_Cs]
test r10, 3
jz pf_kernel_frame
mov r10, [rdx+32]
mov [rax + KF_Rsp], r10
mov r10, [rdx+40]
mov [rax + KF_Ss], r10
swapgs
jmp pf_call
pf_kernel_frame:
lea r10, [rdx+32] ; same-privilege #PF frame has ERR/RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
pf_call:
mov rbx, rax ; Keep the KFAULT_FRAME pointer across C.
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_ERR
KeFaultPF ENDP
; ---------------------------------------------------------
; Generic CPU exception stubs.
; Each vector gets its own IDT entry and vector number so faults like #UD
; do not fall through an absent descriptor and reappear as #GP.
; ---------------------------------------------------------
FAULT_NOERR MACRO StubName, VecNum
LOCAL noerr_kernel_frame, noerr_call
StubName PROC
push rax
push r11
mov r11, rsp
sub rsp, KFAULT_SIZE
mov rax, rsp
mov [rax + KF_R10], r10
mov r10, KFAULT_MAGIC
mov [rax + KF_Magic], r10
mov qword ptr [rax + KF_Vector], VecNum
mov qword ptr [rax + KF_ErrorCode], 0
mov qword ptr [rax + KF_Cr2], 0
mov [rax + KF_Rcx], rcx
mov [rax + KF_Rdx], rdx
mov [rax + KF_Rbx], rbx
mov [rax + KF_Rbp], rbp
mov [rax + KF_Rsi], rsi
mov [rax + KF_Rdi], rdi
mov [rax + KF_R8], r8
mov [rax + KF_R9], r9
mov r10, [r11]
mov [rax + KF_R11], r10
mov r10, [r11+8]
mov [rax + KF_Rax], r10
mov [rax + KF_R12], r12
mov [rax + KF_R13], r13
mov [rax + KF_R14], r14
mov [rax + KF_R15], r15
lea rdx, [r11+16]
mov r10, [rdx]
mov [rax + KF_Rip], r10
mov r10, [rdx+8]
mov [rax + KF_Cs], r10
mov r10, [rdx+16]
mov [rax + KF_Rflags], r10
mov r10, [rax + KF_Cs]
test r10, 3
jz noerr_kernel_frame
mov r10, [rdx+24]
mov [rax + KF_Rsp], r10
mov r10, [rdx+32]
mov [rax + KF_Ss], r10
swapgs
jmp noerr_call
noerr_kernel_frame:
lea r10, [rdx+24] ; same-privilege no-error frame has RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
noerr_call:
mov rbx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_NOERR
StubName ENDP
ENDM
FAULT_ERR MACRO StubName, VecNum
LOCAL err_kernel_frame, err_call
StubName PROC
push rax
push r11
mov r11, rsp
sub rsp, KFAULT_SIZE
mov rax, rsp
mov [rax + KF_R10], r10
mov r10, KFAULT_MAGIC
mov [rax + KF_Magic], r10
mov qword ptr [rax + KF_Vector], VecNum
mov qword ptr [rax + KF_Cr2], 0
mov [rax + KF_Rcx], rcx
mov [rax + KF_Rdx], rdx
mov [rax + KF_Rbx], rbx
mov [rax + KF_Rbp], rbp
mov [rax + KF_Rsi], rsi
mov [rax + KF_Rdi], rdi
mov [rax + KF_R8], r8
mov [rax + KF_R9], r9
mov r10, [r11]
mov [rax + KF_R11], r10
mov r10, [r11+8]
mov [rax + KF_Rax], r10
mov [rax + KF_R12], r12
mov [rax + KF_R13], r13
mov [rax + KF_R14], r14
mov [rax + KF_R15], r15
lea rdx, [r11+16]
mov r10, [rdx]
mov [rax + KF_ErrorCode], r10
mov r10, [rdx+8]
mov [rax + KF_Rip], r10
mov r10, [rdx+16]
mov [rax + KF_Cs], r10
mov r10, [rdx+24]
mov [rax + KF_Rflags], r10
mov r10, [rax + KF_Cs]
test r10, 3
jz err_kernel_frame
mov r10, [rdx+32]
mov [rax + KF_Rsp], r10
mov r10, [rdx+40]
mov [rax + KF_Ss], r10
swapgs
jmp err_call
err_kernel_frame:
lea r10, [rdx+32] ; same-privilege error-code frame has ERR/RIP/CS/RFLAGS
mov [rax + KF_Rsp], r10
mov qword ptr [rax + KF_Ss], 0
err_call:
mov rbx, rax
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KeFaultDispatchFrame
ASYNC_FPU_RESTORE
FAULT_RESUME_ERR
StubName ENDP
ENDM
EXTERN KeHandleNmiIpi:PROC
PUBLIC KeNmiIpiStub
KeNmiIpiStub PROC
; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1,
; the NMI was a panic/TLB-control event and is fully handled. If it
; returns 0, restore the interrupted context and fall through to the normal
; #NMI fault stub so unexpected NMIs still panic with a useful frame.
push rax
push rcx
push rdx
push r8
push r9
push r10
push r11
push r12
push r13
mov r13, rsp
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
call KeHandleNmiIpi
ASYNC_FPU_RESTORE
mov rsp, r13
test eax, eax
jz short NmiUnhandled
pop r13
pop r12
pop r11
pop r10
pop r9
pop r8
pop rdx
pop rcx
pop rax
iretq
NmiUnhandled:
pop r13
pop r12
pop r11
pop r10
pop r9
pop r8
pop rdx
pop rcx
pop rax
jmp KeFaultNMI
KeNmiIpiStub ENDP
FAULT_NOERR KeFaultDB, 1
FAULT_NOERR KeFaultNMI, 2
FAULT_NOERR KeFaultBP, 3
FAULT_NOERR KeFaultOF, 4
FAULT_NOERR KeFaultBR, 5
FAULT_NOERR KeFaultUD, 6
FAULT_NOERR KeFaultNM, 7
FAULT_ERR KeFaultDF, 8
FAULT_NOERR KeFaultCSO, 9
FAULT_ERR KeFaultTS, 10
FAULT_ERR KeFaultNP, 11
FAULT_ERR KeFaultSS, 12
FAULT_NOERR KeFaultRSV15, 15
FAULT_NOERR KeFaultMF, 16
FAULT_ERR KeFaultAC, 17
FAULT_NOERR KeFaultMC, 18
FAULT_NOERR KeFaultXM, 19
FAULT_NOERR KeFaultVE, 20
FAULT_ERR KeFaultCP, 21
FAULT_NOERR KeFaultRSV22, 22
FAULT_NOERR KeFaultRSV23, 23
FAULT_NOERR KeFaultRSV24, 24
FAULT_NOERR KeFaultRSV25, 25
FAULT_NOERR KeFaultRSV26, 26
FAULT_NOERR KeFaultRSV27, 27
FAULT_NOERR KeFaultHV, 28
FAULT_ERR KeFaultVC, 29
FAULT_ERR KeFaultSX, 30
FAULT_NOERR KeFaultRSV31, 31
EXTERN KeIrqDispatch:PROC
EXTERN KiEnterInterrupt:PROC
EXTERN KiExitInterrupt:PROC
IrqStub MACRO num
PUBLIC KeIrqStub&num
KeIrqStub&num PROC
push rax
mov al, num
jmp KeIrqCommon
KeIrqStub&num ENDP
ENDM
IrqStub 0
IrqStub 1
IrqStub 2
IrqStub 3
IrqStub 4
IrqStub 5
IrqStub 6
IrqStub 7
IrqStub 8
IrqStub 9
IrqStub 10
IrqStub 11
IrqStub 12
IrqStub 13
IrqStub 14
IrqStub 15
IrqStub 16
IrqStub 17
IrqStub 18
IrqStub 19
IrqStub 20
IrqStub 21
IrqStub 22
IrqStub 23
PUBLIC KeRescheduleIpiStub
KeRescheduleIpiStub PROC
push rax
mov al, 0F0h
jmp KeIrqCommon
KeRescheduleIpiStub ENDP
KeIrqCommon PROC
; al = irq number, original rax saved by the per-IRQ stub.
push rcx
push rdx
push r8
push r9
push r10
push r11
push rbx
sub rsp, KTRAP_FRAME_SIZE
mov rbx, rsp
movzx edx, al
mov rax, KTRAP_FRAME_MAGIC
mov [rbx + TF_Magic], rax
mov dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT
mov [rbx + TF_Vector], edx
; Saved volatile registers live immediately above the trap frame.
mov rax, [rbx + KTRAP_FRAME_SIZE + 56]
mov [rbx + TF_Rax], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 48]
mov [rbx + TF_Rcx], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 40]
mov [rbx + TF_Rdx], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 0]
mov [rbx + TF_Rbx], rax
mov [rbx + TF_Rbp], rbp
mov [rbx + TF_Rsi], rsi
mov [rbx + TF_Rdi], rdi
mov rax, [rbx + KTRAP_FRAME_SIZE + 32]
mov [rbx + TF_R8], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 24]
mov [rbx + TF_R9], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 16]
mov [rbx + TF_R10], rax
mov rax, [rbx + KTRAP_FRAME_SIZE + 8]
mov [rbx + TF_R11], rax
mov [rbx + TF_R12], r12
mov [rbx + TF_R13], r13
mov [rbx + TF_R14], r14
mov [rbx + TF_R15], r15
; CPU interrupt frame starts after the 8 saved registers.
lea rax, [rbx + KTRAP_FRAME_SIZE + 64]
mov [rbx + TF_RipSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rip], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 72]
mov [rbx + TF_CsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Cs], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 80]
mov [rbx + TF_RflagsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rflags], rdx
mov rax, [rbx + TF_Cs]
and eax, 3
cmp eax, 3
jne short IrqKernelModeFrame
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER
lea rax, [rbx + KTRAP_FRAME_SIZE + 88]
mov [rbx + TF_RspSlot], rax
mov rdx, [rax]
mov [rbx + TF_Rsp], rdx
lea rax, [rbx + KTRAP_FRAME_SIZE + 96]
mov [rbx + TF_SsSlot], rax
mov rdx, [rax]
mov [rbx + TF_Ss], rdx
swapgs ; interrupted ring 3: install kernel GS/KCPU
jmp short IrqFrameReady
IrqKernelModeFrame:
mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_KERNEL
mov qword ptr [rbx + TF_Rsp], 0
mov qword ptr [rbx + TF_Ss], 0
mov qword ptr [rbx + TF_RspSlot], 0
mov qword ptr [rbx + TF_SsSlot], 0
IrqFrameReady:
ASYNC_FPU_SAVE
and rsp, -16
sub rsp, 32
mov rcx, rbx
call KiEnterTrapFrame
call KiEnterInterrupt
mov ecx, [rbx + TF_Vector]
call KeIrqDispatch
call KiExitInterrupt
mov rcx, rbx
call KiExitTrapFrame
ASYNC_FPU_RESTORE
mov eax, [rbx + TF_PreviousMode]
cmp eax, KTRAP_MODE_USER
jne short IrqNoSwapGsOut
swapgs ; returning to ring 3: restore user GS/TEB
IrqNoSwapGsOut:
mov r12, [rbx + TF_R12]
lea rsp, [rbx + KTRAP_FRAME_SIZE]
pop rbx
pop r11
pop r10
pop r9
pop r8
pop rdx
pop rcx
pop rax
iretq
KeIrqCommon ENDP
END
Reference in new issue View Git Blame Copy Permalink