From bfe2b55142b1f8f20c4ee24fdb4290cdd3f88eba Mon Sep 17 00:00:00 2001 From: Maksymilian Wilczynski Date: Sat, 30 May 2026 11:18:46 +0200 Subject: [PATCH] parallel compile, SMP, swapgs, cleanup, remove useless Reserved fields --- OVMF/OVMF_VARS_4M_d.fd | Bin 540672 -> 540672 bytes build.bat | 86 +-- code/drivers/net/lwip/lwip_stack.c | 1 - code/kernel/hal/hal_apic.c | 98 ++- code/kernel/hal/hal_main.c | 96 ++- code/kernel/hal/hal_mouse.c | 3 +- code/kernel/include/kernel/hal/hal_apic.h | 44 +- code/kernel/include/kernel/io_file.h | 2 +- code/kernel/include/kernel/ke_cpu.h | 29 +- code/kernel/include/kernel/ke_debug.h | 2 + code/kernel/include/kernel/ke_idt.h | 32 +- code/kernel/include/kernel/ke_irp.h | 4 +- code/kernel/include/kernel/ke_sched.h | 9 + code/kernel/include/kernel/ke_selftest.h | 7 +- code/kernel/include/kernel/ke_smp.h | 16 + code/kernel/include/kernel/ke_thread.h | 22 +- code/kernel/include/kernel/ke_trap.h | 11 +- code/kernel/include/kernel/ldr_image.h | 2 +- code/kernel/include/kernel/mm_phys.h | 1 + code/kernel/include/kernel/mm_virt.h | 7 + code/kernel/ke_cpu.c | 43 +- code/kernel/ke_debug.c | 109 +++- code/kernel/ke_dpc.c | 4 +- code/kernel/ke_gdt_reload.asm | 97 +-- code/kernel/ke_idt.c | 90 +-- code/kernel/ke_irq.c | 28 +- code/kernel/ke_sched.c | 392 ++++++++++-- code/kernel/ke_selftest.c | 84 ++- code/kernel/ke_smp.c | 660 +++++++++++++++++++++ code/kernel/ke_switch.asm | 68 ++- code/kernel/ke_syscall.asm | 67 ++- code/kernel/ke_syscall.c | 8 +- code/kernel/ke_thread.c | 76 ++- code/kernel/ke_trap.c | 1 - code/kernel/kmain.c | 24 +- code/kernel/ldr_image.c | 24 - code/kernel/mm_phys.c | 20 + code/kernel/mm_virt.c | 227 ++++++- code/public/ntlite/ddi/net_abi.h | 3 +- code/public/ntlite/syscall.h | 7 +- code/user/apps/shell_native/shell_native.c | 34 ++ readme.md | 19 +- 42 files changed, 2131 insertions(+), 426 deletions(-) create mode 100644 code/kernel/include/kernel/ke_smp.h create mode 100644 code/kernel/ke_smp.c diff --git a/OVMF/OVMF_VARS_4M_d.fd b/OVMF/OVMF_VARS_4M_d.fd index d8e8b069d5ef88f0b8ae0cf822a74036f83a7ab5..9ae5bd7adc95b4905023bacf6d09d6dd4dadc194 100644 GIT binary patch delta 155 zcmV;M0A&AwfFXc@A+TmClV&9%li(->lVHM@lQJYGlf3A8lkLM~x@}2vgTP!vei%pZSs&GpW!HC)y(8yY%!1d_Tn#Fa5klESOk)?1$Z-#vM z5M?kO3&Er4a##_4kJEKP&%cMZI$4uk~9f7fyNlKxVO~SOsprs=W zeH(Q|E^caa;c2bU`MN|;pjQO6%yEI`@$a#`>7UkC!*pvF5!>Hh)w6p%)+=+b&#@YW z-`0NPvKoZ$)NcFRIF?^U?GN1WEqK^b1Cv`C;h**V2r~s6`9XYbu+}25zuipKKH_V* zZl*U=7=n0*Dmk(WO1e$NIt+KxyYQD;rl|40{tAV4i3~kisf1OD(Mlz(N}Q@x!m7kr zr4m*pPFE^nRRSuNuqtr|E3pPjaYiq|z^1e#N~LT8#!=eFlDL}qK*&xHXu1YmUAg@q zbL``V9Z*QAFw@2btCZ#TaZqxrFao17XzF^;!u9+CRQhHSrv>q4@O?bw*?Tm1PRs&n zO{G7>y|z}5r+kS({Jt@iYNUpe-bZOYbVgQy)>B^} zBvquEFcQEy3cZ?XoxMbZpeEwbzktzN8>ZF62a%Y~c6+n;tB}|;dEG(Na5oBldC)nHB-QdFU zpr*UAJ=RE}p(nb?vkKqofBTR`VhuF(wnH+Jr5}3ed3h17qV;6Dv$`6SBXBB-6iVC` z|IG5}XOr`?i%{ljcPfnJJZjmm%yDX)#QafU_L%6U=7ZKTSERBH6O`iQJmT|q)ALE* ze54Eug1Yw|^iUL8gDqt4cF;pow~sw-HUiQVlrHx$TwZ(L*);5MJO)~8X3pu)bFRa+ zq+37u3kmk1(A@@1$2wR98qvFni0^VkGUfFZg_Nj z9y!=dKjQP6p=BoOxC$Hl+jZs(Ilqj2?Dy{th?ny@3I2%$KQF;ANbpZ3_-7J)T!NP* z_(ci+xdi`0f`2K&zmnh+68vjFekMRKaqL|!Go@wd?R!@nn8v1xOIUFs^!IKXR@kr- O7*>MAO3j2)tNaU-1j=9l diff --git a/build.bat b/build.bat index e68d032..16b91ef 100644 --- a/build.bat +++ b/build.bat @@ -66,14 +66,11 @@ ml64 /nologo /c /Fo"obj\syscall\stubs.obj" "%USER%\syscall\syscall_stubs :: ---- Compile shared freestanding support ---- cl /nologo /c /Fo"obj\libc.obj" "%COMMON%\libc.c" || exit /b 1 -cl %CFLAGS% /c /Fo"obj\printf.obj" "%COMMON%\printf.c" || exit /b 1 -cl %CFLAGS% /c /Fo"obj\ini_parser.obj" "%COMMON%\ini_parser.c" || exit /b 1 +cl %CFLAGS% /MP /c /Fo"obj\\" "%COMMON%\printf.c" "%COMMON%\ini_parser.c" || exit /b 1 :: ---- Compile bootloader ---- echo Compiling bootloader... -for %%f in (%SRC%\boot\uefi\*.c) do ( - cl %CFLAGS% /c /I"%SRC%\boot\uefi\include" /Fo"obj\boot\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /I"%SRC%\boot\uefi\include" /Fo"obj\boot\\" "%SRC%\boot\uefi\*.c" || exit /b 1 :: ---- Link bootloader ---- echo Linking bootx64.efi... @@ -82,9 +79,7 @@ link %LFLAGS% /SUBSYSTEM:EFI_APPLICATION /ENTRY:UefiMain /OUT:"bin\bootx64.efi" :: ---- Compile HAL ---- echo Compiling hal... -for %%f in (%HAL%\*.c) do ( - cl %CFLAGS% /c /Fo"obj\hal\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\hal\\" "%HAL%\*.c" || exit /b 1 :: ---- Link hal.dll ---- echo Linking hal.dll... @@ -92,9 +87,7 @@ link %LFLAGS% /DLL /OUT:"bin\hal.dll" /ENTRY:HalInitialize /IGNORE:4216 obj\prin :: ---- Compile kernel ---- echo Compiling kernel... -for %%f in (%KERNEL%\*.c) do ( - cl %KERNEL_CFLAGS% /c /Fo"obj\kernel\%%~nf.obj" "%%f" || exit /b 1 -) +cl %KERNEL_CFLAGS% /MP /c /Fo"obj\kernel\\" "%KERNEL%\*.c" || exit /b 1 :: ---- Link kernel.exe ---- echo Linking kernel.exe... @@ -108,22 +101,19 @@ link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL ^ :: ---- Compile win32.dll ---- echo Compiling win32.dll... -for %%f in (%USER%\personalities\win32\*.c) do ( - cl %CFLAGS% /c /Fo"obj\win32\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\win32\\" "%USER%\personalities\win32\*.c" || exit /b 1 link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL ^ /OUT:"bin\win32.dll" bin\syscall.lib obj\libc.obj obj\win32\*.obj || exit /b 1 :: ---- Compile posix.dll ---- echo Compiling posix.dll... -for %%f in (%USER%\personalities\posix\*.c) do ( - cl %CFLAGS% /c /Fo"obj\posix\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\posix\\" "%USER%\personalities\posix\*.c" || exit /b 1 link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL ^ /OUT:"bin\posix.dll" bin\syscall.lib obj\libc.obj obj\posix\*.obj || exit /b 1 :: ---- Compile drivers ---- echo Compiling drivers... + echo Compiling ide.sys... cl %CFLAGS% /c /Fo"obj\drivers\ide.obj" "%DRIVERS%\storage\ide\ide.c" || exit /b 1 link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:"bin\ide.sys" /ENTRY:DriverEntry ^ @@ -140,15 +130,15 @@ link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:"bin\fat32.sys" /ENTRY:Dr bin\hal.lib bin\kernel.lib obj\libc.obj obj\drivers\fat32.obj || exit /b 1 echo Compiling lwip.sys... +set "LWIP_FILES=" for /f "usebackq tokens=* delims=" %%f in ("%LWIP_ROOT%\lwip_sources.txt") do ( set "LWIP_FILE=%%f" if not "!LWIP_FILE!"=="" if not "!LWIP_FILE:~0,1!"=="#" ( set "LWIP_WIN_FILE=!LWIP_FILE:/=\!" - for %%g in ("!LWIP_WIN_FILE!") do ( - cl %LWIP_DRIVER_CFLAGS% /c /Fo"obj\lwip\%%~ng.obj" "%SRC%\!LWIP_WIN_FILE!" || exit /b 1 - ) + set "LWIP_FILES=!LWIP_FILES! "%SRC%\!LWIP_WIN_FILE!"" ) ) +cl %LWIP_DRIVER_CFLAGS% /MP /c /Fo"obj\lwip\\" !LWIP_FILES! || exit /b 1 link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:"bin\lwip.sys" /ENTRY:DriverEntry ^ bin\hal.lib bin\kernel.lib obj\libc.obj obj\lwip\*.obj || exit /b 1 @@ -169,113 +159,85 @@ link /nologo /SUBSYSTEM:NATIVE /NODEFAULTLIB /DLL /OUT:"bin\rtl8139.sys" /ENTRY: :: ---- Compile init ---- echo Compiling init... -for %%f in (%USER%\system\init\*.c) do ( - cl %CFLAGS% /c /Fo"obj\init\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\init\\" "%USER%\system\init\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:_start ^ /OUT:"bin\init.exe" /PDB:"bin\init.pdb" bin\syscall.lib obj\libc.obj obj\init\init.obj || exit /b 1 :: ---- Compile shell_win32 ---- echo Compiling shell_win32... -for %%f in (%USER%\apps\shell_win32\*.c) do ( - cl %CFLAGS% /c /Fo"obj\shell_win32\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\shell_win32\\" "%USER%\apps\shell_win32\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\shell_win32.exe" /PDB:"bin\shell_win32.pdb" bin\win32.lib obj\libc.obj obj\shell_win32\*.obj || exit /b 1 :: ---- Compile shell_posix ---- echo Compiling shell_posix... -for %%f in (%USER%\apps\shell_posix\*.c) do ( - cl %CFLAGS% /c /Fo"obj\shell_posix\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\shell_posix\\" "%USER%\apps\shell_posix\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\shell_posix.exe" /PDB:"bin\shell_posix.pdb" bin\posix.lib obj\libc.obj obj\shell_posix\*.obj || exit /b 1 :: ---- Compile shell_native ---- echo Compiling shell_native... -for %%f in (%USER%\apps\shell_native\*.c) do ( - cl %CFLAGS% /c /Fo"obj\shell_native\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\shell_native\\" "%USER%\apps\shell_native\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\shell_native.exe" /PDB:"bin\shell_native.pdb" bin\syscall.lib obj\libc.obj obj\shell_native\*.obj || exit /b 1 :: ---- Compile async_apc ---- echo Compiling async_apc... -for %%f in (%USER%\tests\async_apc\*.c) do ( - cl %CFLAGS% /c /Fo"obj\async_apc\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\async_apc\\" "%USER%\tests\async_apc\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\async_apc.exe" /PDB:"bin\async_apc.pdb" bin\syscall.lib obj\libc.obj obj\async_apc\*.obj || exit /b 1 :: ---- Compile async_cancel ---- echo Compiling async_cancel... -for %%f in (%USER%\tests\async_cancel\*.c) do ( - cl %CFLAGS% /c /Fo"obj\async_cancel\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\async_cancel\\" "%USER%\tests\async_cancel\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\async_cancel.exe" /PDB:"bin\async_cancel.pdb" bin\syscall.lib obj\libc.obj obj\async_cancel\*.obj || exit /b 1 :: ---- Compile edit ---- echo Compiling edit... -for %%f in (%USER%\apps\edit\*.c) do ( - cl %CFLAGS% /c /Fo"obj\edit\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\edit\\" "%USER%\apps\edit\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\edit.exe" /PDB:"bin\edit.pdb" bin\syscall.lib obj\libc.obj obj\edit\*.obj || exit /b 1 :: ---- Compile httpget ---- echo Compiling httpget... -for %%f in (%USER%\apps\httpget\*.c) do ( - cl %CFLAGS% /c /Fo"obj\httpget\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\httpget\\" "%USER%\apps\httpget\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\httpget.exe" /PDB:"bin\httpget.pdb" bin\syscall.lib obj\libc.obj obj\httpget\*.obj || exit /b 1 :: ---- Compile tcpserve ---- echo Compiling tcpserve... -for %%f in (%USER%\apps\tcpserve\*.c) do ( - cl %CFLAGS% /c /Fo"obj\tcpserve\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\tcpserve\\" "%USER%\apps\tcpserve\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\tcpserve.exe" /PDB:"bin\tcpserve.pdb" bin\syscall.lib obj\libc.obj obj\tcpserve\*.obj || exit /b 1 :: ---- Compile udp ---- echo Compiling udp... -for %%f in (%USER%\apps\udp\*.c) do ( - cl %CFLAGS% /c /Fo"obj\udp\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\udp\\" "%USER%\apps\udp\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\udp.exe" /PDB:"bin\udp.pdb" bin\syscall.lib obj\libc.obj obj\udp\*.obj || exit /b 1 :: ---- Compile resolve ---- echo Compiling resolve... -for %%f in (%USER%\apps\resolve\*.c) do ( - cl %CFLAGS% /c /Fo"obj\resolve\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\resolve\\" "%USER%\apps\resolve\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\resolve.exe" /PDB:"bin\resolve.pdb" bin\syscall.lib obj\libc.obj obj\resolve\*.obj || exit /b 1 :: ---- Compile ping ---- echo Compiling ping... -for %%f in (%USER%\apps\ping\*.c) do ( - cl %CFLAGS% /c /Fo"obj\ping\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\ping\\" "%USER%\apps\ping\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\ping.exe" /PDB:"bin\ping.pdb" bin\syscall.lib obj\libc.obj obj\ping\*.obj || exit /b 1 :: ---- Compile doom ---- echo Compiling doom... -for %%f in (%USER%\apps\doom\*.c) do ( - cl %CFLAGS% /c /Fo"obj\doom\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\doom\\" "%USER%\apps\doom\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\doom.exe" /PDB:"bin\doom.pdb" bin\syscall.lib obj\libc.obj obj\doom\*.obj || exit /b 1 :: ---- Compile video player ---- echo Compiling video... -for %%f in (%USER%\apps\video\*.c) do ( - cl %CFLAGS% /c /Fo"obj\video\%%~nf.obj" "%%f" || exit /b 1 -) +cl %CFLAGS% /MP /c /Fo"obj\video\\" "%USER%\apps\video\*.c" || exit /b 1 link %LFLAGS% /SUBSYSTEM:NATIVE /ENTRY:main ^ /OUT:"bin\video.exe" /PDB:"bin\video.pdb" bin\win32.lib obj\libc.obj %USER%\apps\video\bink2w64.lib obj\video\*.obj || exit /b 1 diff --git a/code/drivers/net/lwip/lwip_stack.c b/code/drivers/net/lwip/lwip_stack.c index f46e6c6..8830eb5 100644 --- a/code/drivers/net/lwip/lwip_stack.c +++ b/code/drivers/net/lwip/lwip_stack.c @@ -78,7 +78,6 @@ typedef struct LWIP_PROVIDER_SOCKET { uint8_t ShutdownRead; uint8_t ShutdownWrite; uint8_t NonBlocking; - uint8_t Reserved0; uint32_t ReceiveTimeoutMs; uint32_t SendTimeoutMs; diff --git a/code/kernel/hal/hal_apic.c b/code/kernel/hal/hal_apic.c index ab5cda2..9e57b4c 100644 --- a/code/kernel/hal/hal_apic.c +++ b/code/kernel/hal/hal_apic.c @@ -2,18 +2,25 @@ #include #include #include +#include extern HAL_API gApi; // ---- Local APIC ---- static volatile uint32_t* gLapic = (volatile uint32_t *)0; #define LAPIC_REG(off) (*(volatile uint32_t *)((uint8_t *)gLapic + (off))) +#define LAPIC_ID 0x020 #define LAPIC_EOI 0x0B0 #define LAPIC_SPURIOUS 0x0F0 +#define LAPIC_ICR_LO 0x300 +#define LAPIC_ICR_HI 0x310 #define LAPIC_ENABLE 0x100 // bit in spurious vector register +#define APIC_VECTOR_RESCHEDULE 0xF0u + // ---- I/O APIC ---- static volatile uint32_t* gIoapic = (volatile uint32_t *)0; +static uint32_t gBspLapicId = 0; #define IOAPIC_IDX 0x00 #define IOAPIC_DATA 0x10 @@ -26,18 +33,99 @@ static void IoapicWrite(uint8_t reg, uint32_t val) { *(volatile uint32_t *)((uint8_t *)gIoapic + IOAPIC_DATA) = val; } + +static void HalApicStallFallback(uint32_t loops) { + for (volatile uint32_t i = 0; i < loops; i++) { + _mm_pause(); + } +} + +static void HalApicStallUs(uint32_t us) { + if (gApi.Timer.Stall) gApi.Timer.Stall(us); + else HalApicStallFallback(us * 128u + 1u); +} + +static void HalApicWaitIcrIdle(void) { + for (uint32_t i = 0; i < 1000000u; i++) { + if (!(LAPIC_REG(LAPIC_ICR_LO) & (1u << 12))) + return; + _mm_pause(); + } +} + +static void HalApicSendIpi(uint32_t lapicId, uint32_t low) { + HalApicWaitIcrIdle(); + LAPIC_REG(LAPIC_ICR_HI) = lapicId << 24; + LAPIC_REG(LAPIC_ICR_LO) = low; + HalApicWaitIcrIdle(); +} + // IRQ n -> redirection table at register 0x10 + 2*n (low), 0x11 + 2*n (high). #define IOREDTBL_LO(n) (0x10 + 2*(n)) #define IOREDTBL_HI(n) (0x11 + 2*(n)) // Interrupt vector base: IRQ 0 -> vector 0x20, IRQ 1 -> 0x21, ... #define IRQ_VECTOR_BASE 0x20 +__declspec(dllexport) uint32_t HalGetBspLapicId(void) { + return gBspLapicId; +} + +__declspec(dllexport) void HalApicEnableCurrentCpu(void) { + if (!gLapic) return; + LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF; +} + + +__declspec(dllexport) void HalApicSendPanicIpi(void) { + if (!gLapic) return; + + // Stop-the-world panic IPI. Use NMI delivery instead of a normal fixed + // vector so CPUs running with IF=0 inside spinlock/high-IRQL regions still + // enter the panic path. The NMI vector is already installed in the IDT on + // every CPU that has reached kernel-owned state. + HalApicWaitIcrIdle(); + LAPIC_REG(LAPIC_ICR_HI) = 0; + LAPIC_REG(LAPIC_ICR_LO) = (3u << 18) | (4u << 8); // all-excluding-self, NMI + HalApicWaitIcrIdle(); +} + +__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId) { + if (!gLapic) return; + HalApicSendIpi(lapicId, APIC_VECTOR_RESCHEDULE); // fixed delivery, physical destination +} + +__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId) { + if (!gLapic) return; + + // TLB shootdown must reach CPUs that may be spinning with IF=0 inside + // high-IRQL/spinlock regions, so use NMI delivery rather than a fixed + // vector. The NMI stub handles active shootdowns and returns; unexpected + // NMIs still go through the normal fault/panic path. + HalApicSendIpi(lapicId, (4u << 8)); // NMI delivery, physical destination +} + +__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector) { + if (!gLapic || !startupVector || lapicId == gBspLapicId) + return STATUS_INVALID_PARAMETER; + + // INIT assert/deassert, then two SIPIs. The target begins at startupVector << 12. + HalApicSendIpi(lapicId, 0x0000C500u); // INIT, level assert, level-triggered + HalApicStallUs(10000); + HalApicSendIpi(lapicId, 0x00008500u); // INIT, level deassert + HalApicStallUs(200); + HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI + HalApicStallUs(200); + HalApicSendIpi(lapicId, 0x00000600u | startupVector); // SIPI retry + HalApicStallUs(200); + return STATUS_SUCCESS; +} + void HalApicRouteIrq(uint8_t irq) { // Full initial setup of the redirection entry. // Sets the vector and clears the mask bit in one write. uint32_t vector = IRQ_VECTOR_BASE + irq; // e.g. IRQ 1 -> vector 0x21 + IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24); // external IRQs target the BSP IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed - IoapicWrite(IOREDTBL_HI(irq), 0); // deliver to LAPIC id 0 } void HalApicMaskIrq(uint8_t irq) { @@ -50,8 +138,8 @@ void HalApicUnmaskIrq(uint8_t irq) { uint32_t lo = IoapicRead(IOREDTBL_LO(irq)); lo &= ~(1 << 16); // clear mask bit lo = (lo & ~0xFF) | (IRQ_VECTOR_BASE + irq); + IoapicWrite(IOREDTBL_HI(irq), gBspLapicId << 24); IoapicWrite(IOREDTBL_LO(irq), lo); - IoapicWrite(IOREDTBL_HI(irq), 0); // LAPIC id 0 } void HalApicSendEoi(void) { @@ -62,6 +150,10 @@ void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) { gLapic = (volatile uint32_t *)(gApi.PhysToVirt(lapicPhys)); gIoapic = (volatile uint32_t *)(gApi.PhysToVirt(ioapicPhys)); + // The BSP is the CPU executing HAL init. Keep all external/device IRQs + // routed to this LAPIC; CPU-local IPIs use the local APIC directly. + gBspLapicId = (LAPIC_REG(LAPIC_ID) >> 24) & 0xFF; + // Enable Local APIC, set spurious vector to 0xFF. LAPIC_REG(LAPIC_SPURIOUS) = LAPIC_ENABLE | 0xFF; @@ -69,6 +161,6 @@ void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) { uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF; for (uint8_t i = 0; i <= maxRedir; i++) { IoapicWrite(IOREDTBL_LO(i), (1 << 16) | (IRQ_VECTOR_BASE + i)); - IoapicWrite(IOREDTBL_HI(i), 0); + IoapicWrite(IOREDTBL_HI(i), gBspLapicId << 24); } } diff --git a/code/kernel/hal/hal_main.c b/code/kernel/hal/hal_main.c index dfa9c62..5d2dc8c 100644 --- a/code/kernel/hal/hal_main.c +++ b/code/kernel/hal/hal_main.c @@ -40,7 +40,7 @@ typedef struct { uint32_t Length; uint64_t XsdtAddress; uint8_t ExtendedChecksum; - uint8_t Reserved[3]; + uint8_t Padding[3]; } RSDP; // ACPI SDT Header @@ -81,6 +81,7 @@ typedef struct { static uintptr_t gLapicPhys = 0xFEE00000; static uintptr_t gIoapicPhys = 0xFEC00000; static uintptr_t gHpetPhys = 0xFED00000; +static HAL_CPU_TOPOLOGY gCpuTopology; // ACPI power-off state (filled in by ParseFadt / ScanDsdtForS5) static uint16_t gPm1aCntBlk = 0; @@ -89,7 +90,7 @@ static uint8_t gSlpTypA = 5; // S5 SLP_TYP for PM1a; 5 is correct for QEMU static uint8_t gSlpTypB = 5; // S5 SLP_TYP for PM1b int debugEnabled = 0; -int _fltused; // fuckass +int _fltused; // MSVC floating-point helper anchor static void hal_putchar(char c) { gApi.Screen.PrintChar(c, 0x00FF00, 0x000000); } @@ -114,7 +115,7 @@ typedef struct { ACPISDTHeader Header; // 0 (36 bytes) uint32_t FirmwareCtrl; // 36 uint32_t Dsdt; // 40 - physical addr of DSDT - uint8_t Reserved1; // 44 + uint8_t Ignored44; // 44 uint8_t PreferredPmProfile; // 45 uint16_t SciInt; // 46 uint32_t SmiCmd; // 48 @@ -201,16 +202,51 @@ static void ParseFadt(const FADT* fadt) { } } + +static void HalAddCpuRecord(uint32_t uid, uint32_t lapicId, uint32_t flags) { + if (gCpuTopology.CpuCount >= HAL_MAX_CPUS) + return; + + for (uint32_t i = 0; i < gCpuTopology.CpuCount; i++) { + if (gCpuTopology.Cpus[i].LapicId == lapicId) + return; + } + + HAL_CPU_INFO* cpu = &gCpuTopology.Cpus[gCpuTopology.CpuCount++]; + memset(cpu, 0, sizeof(*cpu)); + cpu->ProcessorUid = uid; + cpu->LapicId = lapicId; + cpu->Flags = flags; + cpu->Enabled = (flags & 1u) ? 1u : 0u; +} + static void ParseMadt(MADT* madt) { gLapicPhys = madt->LocalAPICAddress; uint8_t* p = (uint8_t *)(madt + 1); uint8_t* end = (uint8_t *)madt + madt->Header.Length; - while (p < end) { + while (p + 2 <= end) { uint8_t type = p[0], len = p[1]; - if (type == 1) { // I/O APIC entry + if (!len || p + len > end) break; + + if (type == 0 && len >= 8) { // Processor Local APIC + uint32_t uid = p[2]; + uint32_t lapicId = p[3]; + uint32_t flags = *(uint32_t *)(p + 4); + if (flags & 1u) + HalAddCpuRecord(uid, lapicId, flags); + } else if (type == 1 && len >= 12) { // I/O APIC entry gIoapicPhys = *(uint32_t *)(p + 4); + } else if (type == 5 && len >= 12) { // Local APIC Address Override + gLapicPhys = (uintptr_t)*(uint64_t *)(p + 4); + } else if (type == 9 && len >= 16) { // Processor Local x2APIC + uint32_t lapicId = *(uint32_t *)(p + 4); + uint32_t flags = *(uint32_t *)(p + 8); + uint32_t uid = *(uint32_t *)(p + 12); + if ((flags & 1u) && lapicId < 256u) + HalAddCpuRecord(uid, lapicId, flags); } - p += len ? len : 1; + + p += len; } } @@ -313,6 +349,44 @@ static void HalShutdown(void) { while (true) __halt(); } +// --------------------------------------------------------------------------- +__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out) { + if (!out) return STATUS_INVALID_PARAMETER; + + memset(out, 0, sizeof(*out)); + *out = gCpuTopology; + + if (!out->CpuCount) { + out->CpuCount = 1; + out->BspLapicId = HalGetBspLapicId(); + out->Cpus[0].LapicId = out->BspLapicId; + out->Cpus[0].ProcessorUid = 0; + out->Cpus[0].Flags = 1; + out->Cpus[0].Enabled = 1; + out->Cpus[0].IsBsp = 1; + return STATUS_SUCCESS; + } + + uint32_t bsp = HalGetBspLapicId(); + out->BspLapicId = bsp; + int foundBsp = 0; + for (uint32_t i = 0; i < out->CpuCount; i++) { + out->Cpus[i].IsBsp = (out->Cpus[i].LapicId == bsp) ? 1u : 0u; + if (out->Cpus[i].IsBsp) foundBsp = 1; + } + + if (!foundBsp && out->CpuCount < HAL_MAX_CPUS) { + HAL_CPU_INFO* cpu = &out->Cpus[out->CpuCount++]; + memset(cpu, 0, sizeof(*cpu)); + cpu->LapicId = bsp; + cpu->ProcessorUid = bsp; + cpu->Flags = 1; + cpu->Enabled = 1; + cpu->IsBsp = 1; + } + return STATUS_SUCCESS; +} + // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) { if (gInitialised) return STATUS_ALREADY_EXISTS; @@ -357,12 +431,10 @@ __declspec(dllexport) KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) { gApi.Version = HAL_API_VERSION; gInitialised = 1; - // This shouldn't be here but i have no other way. - // After i send 0xF4 and move the mouse the controller sends an interrupts that doesn't fire - // and because the CPU missed the IRQ 12 edge transition (cli) later when we expect it to fire, it doesn't - // leaving us with no mouse and no keyboard. The dreaded 8042 will hold the CLOCK line low forever. - // To combat this we just go about our day executing the rest of the init and just before the end - // of init we just iterate 256 times to grab whatever the controller left for us ("Drain and Pray"). + // The 8042 can leave output bytes pending while the keyboard/mouse drivers + // are being brought up with interrupts temporarily disabled. Drain the + // controller before returning from HAL init so stale status/data bytes do + // not suppress the next keyboard or mouse interrupt edge. for (int i = 0; i < 256; i++) { if (__inbyte(0x64) & 0x01) { __inbyte(0x60); // Consume diff --git a/code/kernel/hal/hal_mouse.c b/code/kernel/hal/hal_mouse.c index ca76fd5..71f2a35 100644 --- a/code/kernel/hal/hal_mouse.c +++ b/code/kernel/hal/hal_mouse.c @@ -271,8 +271,7 @@ void HalMouseInitialize(HAL_MOUSE* out, int screenW, int screenH) { ack = MouseRecvByte(); } while (ack != 0xFAu && --retries > 0); - // Thanks to the increased Ps2WaitOutput timeout, we will actually - // wait long enough for the physical hardware self-test to finish here. + // The output timeout covers the physical hardware self-test latency. MouseRecvByte(); // 0xAA - BAT passed MouseRecvByte(); // 0x00 - device ID diff --git a/code/kernel/include/kernel/hal/hal_apic.h b/code/kernel/include/kernel/hal/hal_apic.h index b0339e4..81d9d13 100644 --- a/code/kernel/include/kernel/hal/hal_apic.h +++ b/code/kernel/include/kernel/hal/hal_apic.h @@ -1,9 +1,35 @@ -// hal_apic.h -#pragma once -#include - -void HalApicRouteIrq(uint8_t irq); -void HalApicMaskIrq(uint8_t irq); -void HalApicUnmaskIrq(uint8_t irq); -void HalApicSendEoi(void); -void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys); +// hal_apic.h +#pragma once +#include +#include + +void HalApicRouteIrq(uint8_t irq); +void HalApicMaskIrq(uint8_t irq); +void HalApicUnmaskIrq(uint8_t irq); +void HalApicSendEoi(void); +void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys); + + +#define HAL_MAX_CPUS 32u + +typedef struct HAL_CPU_INFO { + uint32_t ProcessorUid; + uint32_t LapicId; + uint32_t Flags; + uint8_t Enabled; + uint8_t IsBsp; +} HAL_CPU_INFO; + +typedef struct HAL_CPU_TOPOLOGY { + uint32_t CpuCount; + uint32_t BspLapicId; + HAL_CPU_INFO Cpus[HAL_MAX_CPUS]; +} HAL_CPU_TOPOLOGY; + +__declspec(dllexport) KSTATUS HalGetProcessorTopology(HAL_CPU_TOPOLOGY* out); +__declspec(dllexport) uint32_t HalGetBspLapicId(void); +__declspec(dllexport) void HalApicEnableCurrentCpu(void); +__declspec(dllexport) void HalApicSendPanicIpi(void); +__declspec(dllexport) void HalApicSendRescheduleIpi(uint32_t lapicId); +__declspec(dllexport) void HalApicSendTlbShootdownIpi(uint32_t lapicId); +__declspec(dllexport) KSTATUS HalStartApplicationProcessor(uint32_t lapicId, uint8_t startupVector); diff --git a/code/kernel/include/kernel/io_file.h b/code/kernel/include/kernel/io_file.h index f5d2c49..cb180d7 100644 --- a/code/kernel/include/kernel/io_file.h +++ b/code/kernel/include/kernel/io_file.h @@ -23,7 +23,7 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext, size_t information); // --------------------------------------------------------------------------- -// Filesystem driver vtable - filled in by fat32.sys (or any future fs driver) +// Filesystem driver vtable - filled in by a filesystem driver. // --------------------------------------------------------------------------- typedef struct IO_FILESYSTEM { // Open a file by absolute path (e.g. "apps\\shell.exe"). diff --git a/code/kernel/include/kernel/ke_cpu.h b/code/kernel/include/kernel/ke_cpu.h index ae60737..9e6da58 100644 --- a/code/kernel/include/kernel/ke_cpu.h +++ b/code/kernel/include/kernel/ke_cpu.h @@ -1,9 +1,8 @@ #pragma once // ke_cpu.h - per-CPU execution state // -// SMP phase 1 still runs only the bootstrap processor, but the kernel now -// represents CPU-local state explicitly. AP startup can fill additional KCPU -// entries later without first untangling scheduler/IRQL/TSS globals. +// Every processor that reaches kernel-owned state has a KCPU record. The record +// owns CPU-local scheduler, IRQL, GDT, and TSS state. #include #include @@ -15,23 +14,42 @@ struct THREAD; typedef struct KCPU { + // GS:0 points back at the owning KCPU while executing in ring 0. + // User mode keeps its TEB in GS_BASE; swapgs restores this pointer on + // entry from user mode and parks the user TEB in IA32_KERNEL_GS_BASE. + struct KCPU* Self; + uint32_t CpuIndex; uint32_t LapicId; + uint32_t AcpiProcessorId; + uint32_t BootProcessorIndex; uint8_t Present; + uint8_t Started; + uint8_t Parked; uint8_t Online; uint8_t IsBsp; - uint8_t Reserved0; struct THREAD* CurrentThread; struct THREAD* IdleThread; struct THREAD* OutgoingThread; + // Do not store the scheduler post-switch unlock cookie here. It belongs + // to the suspended THREAD continuation that resumes after KeSwitchContext(). + KIRQL CurrentIrql; uint32_t InterruptDepth; volatile long ReschedulePending; + // AP scheduler admission handshake. APs set SchedulerActive just before + // they leave the raw kernel park loop and enter their per-CPU idle thread. + volatile long SchedulerCommand; + volatile long SchedulerActive; + TSS64 Tss; + uint64_t Gdt[KE_GDT_ENTRIES]; + KGDTR Gdtr; + void* KernelStack; uintptr_t KernelStackTop; @@ -43,6 +61,9 @@ extern KCPU gCpus[KE_MAX_CPUS]; extern uint32_t gCpuCount; void KeInitializeBootstrapCpu(void); +void KeLoadKernelGsBase(KCPU* cpu); +void KeSetUserGsBase(uintptr_t userGsBase); +uintptr_t KeGetUserGsBase(void); KCPU* KeGetCurrentCpu(void); KCPU* KeGetCpuByIndex(uint32_t index); uint32_t KeGetCurrentCpuIndex(void); diff --git a/code/kernel/include/kernel/ke_debug.h b/code/kernel/include/kernel/ke_debug.h index 99f84d0..5d87d80 100644 --- a/code/kernel/include/kernel/ke_debug.h +++ b/code/kernel/include/kernel/ke_debug.h @@ -16,9 +16,11 @@ typedef enum KE_PANIC_CODE { KE_PANIC_CODE_TRAP = 4, KE_PANIC_CODE_DRIVER = 5, KE_PANIC_CODE_INIT = 6, + KE_PANIC_CODE_SCHEDULER= 7, } KE_PANIC_CODE; __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...); +__declspec(dllexport) int KeHandleNmiIpi(void); // --------------------------------------------------------------------------- // Fault dispatch - called from the IDT fault stubs (ke_syscall.asm). diff --git a/code/kernel/include/kernel/ke_idt.h b/code/kernel/include/kernel/ke_idt.h index 560267b..30ad0f7 100644 --- a/code/kernel/include/kernel/ke_idt.h +++ b/code/kernel/include/kernel/ke_idt.h @@ -5,25 +5,36 @@ // 64-bit TSS - minimal; we only care about RSP0. #pragma pack(push, 1) typedef struct TSS64 { - uint32_t Reserved0; + uint32_t MustBeZero0; uint64_t RSP0; // loaded into RSP on ring-0 entry uint64_t RSP1; uint64_t RSP2; - uint64_t Reserved1; + uint64_t MustBeZero1; uint64_t IST[7]; - uint64_t Reserved2; - uint16_t Reserved3; + uint64_t MustBeZero2; + uint16_t MustBeZero3; uint16_t IoMapBase; } TSS64; #pragma pack(pop) -// Compatibility pointer used by ke_switch.asm. -// It points at CPU0's TSS until the context-switch path grows a true -// per-CPU base lookup. -extern TSS64* gTss; +#define KE_GDT_ENTRIES 8u + +#pragma pack(push, 1) +typedef struct KGDTR { + uint16_t Limit; + uint64_t Base; +} KGDTR; +#pragma pack(pop) + +struct KCPU; void KeInitializeGdt(void); +void KeInitializeGdtForCpu(struct KCPU* cpu); +void KeLoadGdtForCurrentCpu(struct KCPU* cpu); +void KeSetCurrentTssRsp0(uintptr_t rsp0); + void KeInitializeIdt(void); +void KeLoadIdtForCurrentCpu(void); // Called from hal_timer.c after HPET/APIC timer fires. void KeTimerInterrupt(void); @@ -36,3 +47,8 @@ extern void KeFaultDE(void); // #DE divide error extern void KeFaultGP(void); // #GP general protection extern void KeFaultPF(void); // #PF page fault extern void KeFaultUD(void); // #UD invalid opcode +extern void KeNmiIpiStub(void); // returning NMI SMP-control stub + +// CPU-local IPI vector. This uses the common interrupt entry machinery but is +// not an external/device IRQ and is never routed through the I/O APIC. +#define KE_VECTOR_RESCHEDULE_IPI 0xF0u diff --git a/code/kernel/include/kernel/ke_irp.h b/code/kernel/include/kernel/ke_irp.h index 9669aa8..bd6c993 100644 --- a/code/kernel/include/kernel/ke_irp.h +++ b/code/kernel/include/kernel/ke_irp.h @@ -37,8 +37,8 @@ struct KAPC; typedef struct IRP IRP; // Completion callback: invoked by IoCompleteRequest() in the completing -// context. Today that is the driver worker or sync fallback path; future -// DPC/APC work will keep all completion policy centralized here. +// context. Driver worker, synchronous fallback, DPC, and APC completion paths +// all centralize policy here. typedef void (*IRP_COMPLETION_FN)(IRP *irp, void *ctx); // ------------------------------------------------------------------------- diff --git a/code/kernel/include/kernel/ke_sched.h b/code/kernel/include/kernel/ke_sched.h index c3c3bc1..056b591 100644 --- a/code/kernel/include/kernel/ke_sched.h +++ b/code/kernel/include/kernel/ke_sched.h @@ -3,6 +3,8 @@ // Three run queues (Low / Normal / High). IRQ 0 drives KeSchedulerTick. #include +struct KCPU; + #define SCHED_TICK_MS 10u #define SCHED_HIGH_QUANTUM_TICKS 2u #define SCHED_NORMAL_QUANTUM_TICKS 4u @@ -26,6 +28,11 @@ typedef struct KE_SCHED_STATS { // Called once by KernelMain before any threads are created. void KeInitializeScheduler(void); +// Split global scheduler setup from per-CPU idle-thread creation so AP bring-up +// can add one idle thread per CPU without reinitializing the global queues. +void KiInitializeSchedulerGlobal(void); +KSTATUS KiCreateIdleThreadForCpu(struct KCPU* cpu); + // Add / remove a thread from the run queues. void KeReadyThread(THREAD* thread); void KeUnreadyThread(THREAD* thread); @@ -44,6 +51,8 @@ void KiRequestReschedule(void); int KiIsReschedulePending(void); void KiScheduleFromInterrupt(void); void KiSchedulerNoteVoluntaryBlock(THREAD* thread); +void KiSchedulerMarkThreadWaiting(THREAD* thread); +void KiSchedulerWakeThread(THREAD* thread, KSTATUS status); // Voluntarily yield the CPU. Picks the next runnable thread immediately. __declspec(dllexport) void KeYield(void); diff --git a/code/kernel/include/kernel/ke_selftest.h b/code/kernel/include/kernel/ke_selftest.h index 528c2aa..83ac960 100644 --- a/code/kernel/include/kernel/ke_selftest.h +++ b/code/kernel/include/kernel/ke_selftest.h @@ -1,9 +1,8 @@ #pragma once -// ke_selftest.h - temporary kernel bring-up self-tests. -// -// These tests intentionally live behind one call site so they can be walled -// off or compiled out later without touching kmain.c again. +// ke_selftest.h - kernel bring-up and regression self-tests. #include +void KiRunCoreHardeningSelfTests(void); +void KiRunEarlyKernelSelfTests(void); void KiRunKernelSelfTests(const HAL_API* hal); diff --git a/code/kernel/include/kernel/ke_smp.h b/code/kernel/include/kernel/ke_smp.h new file mode 100644 index 0000000..69c5310 --- /dev/null +++ b/code/kernel/include/kernel/ke_smp.h @@ -0,0 +1,16 @@ +#pragma once +#include +#include + +// Start application processors with hardware INIT/SIPI and park them in +// kernel-owned state. APs are not runnable until scheduler admission succeeds. +KSTATUS KeStartApplicationProcessors(void); + +// Create one scheduler idle thread per AP and release APs from the raw kernel +// park loop into their idle threads. +KSTATUS KeAdmitApplicationProcessorsToScheduler(void); + +// Run explicit AP-affine kernel workers that exercise the global scheduler, +// reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver +// threads remain BSP-affine unless their affinity is explicitly widened. +KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void); diff --git a/code/kernel/include/kernel/ke_thread.h b/code/kernel/include/kernel/ke_thread.h index d935761..4f226b5 100644 --- a/code/kernel/include/kernel/ke_thread.h +++ b/code/kernel/include/kernel/ke_thread.h @@ -70,7 +70,12 @@ typedef struct THREAD { uintptr_t UserEntry; uintptr_t UserArg; - uintptr_t SchedLockFlags; // Stores RFLAGS while thread is context switched out + // Saved scheduler-lock cookie for this thread's suspended switch + // continuation. This is deliberately thread-owned, not CPU-owned: + // after KeSwitchContext() resumes a saved KiSchedule() continuation, the + // resumed thread must restore the IF/IRQL state captured when that same + // thread switched out. + uintptr_t SwitchLockCookie; void* UserStackBase; // Tracks user stack allocation to prevent memory leaks // Signaled (manual-reset) when the thread exits. @@ -90,6 +95,15 @@ typedef struct THREAD { uint32_t LastRunTicks; uint32_t RunningCpu; + // Scheduler hardening state. Queued means membership in a scheduler ready + // queue only; wait-object membership is tracked by WaitListHead. Idle + // threads are CPU-local and never enter a global ready queue. + uint8_t Queued; + uint8_t IsIdleThread; + uint16_t SchedulerFlags; + uint32_t LastCpu; + uint64_t AffinityMask; + // Set by wait primitives before blocking and completed by the object // that wakes the thread. Keep this field after the assembly-visible // context-switch fields: ke_switch.asm hard-codes offsets through @@ -130,6 +144,9 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui // given priority. __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, THREAD_PRIORITY priority, THREAD** out); +__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, + THREAD_PRIORITY priority, uint64_t affinityMask, + THREAD** out); // Mark a thread dead and deschedule it permanently. __declspec(dllexport) void KeExitThread(THREAD* thread, int code); @@ -158,5 +175,8 @@ void KiWakeThreadForUserApc(THREAD* thread); // Assembly stub - implemented in ke_switch.asm. void KeSwitchContext(THREAD* from, THREAD* to); +__declspec(noreturn) void KeStartInitialThread(THREAD* thread); void KeInitThreadFpuContext(THREAD* thread); +void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask); +uint64_t KeGetDefaultThreadAffinityMask(void); diff --git a/code/kernel/include/kernel/ke_trap.h b/code/kernel/include/kernel/ke_trap.h index 28eec6d..8d6eae6 100644 --- a/code/kernel/include/kernel/ke_trap.h +++ b/code/kernel/include/kernel/ke_trap.h @@ -1,10 +1,9 @@ #pragma once // ke_trap.h - trap-frame metadata for syscall/interrupt return paths // -// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally -// an editable return frame: future user APC delivery can rewrite Rip/Rsp in the -// frame, and KiExitTrapFrame will commit those values back to the hardware IRET -// slots before returning to user mode. +// KTRAP_FRAME is built by ke_syscall.asm on kernel entry. It is intentionally +// an editable return frame: user APC delivery can rewrite Rip/Rsp in the frame, +// and KiExitTrapFrame commits those values back to the hardware IRET slots. #include #include @@ -18,12 +17,11 @@ #define KTRAP_MODE_KERNEL 0u #define KTRAP_MODE_USER 1u -typedef struct KTRAP_FRAME { +typedef __declspec(align(16)) struct KTRAP_FRAME { uint64_t Magic; uint32_t Kind; uint32_t Vector; uint32_t PreviousMode; - uint32_t Reserved; uint64_t Rax; uint64_t Rcx; @@ -54,7 +52,6 @@ typedef struct KTRAP_FRAME { uint64_t* SsSlot; struct KTRAP_FRAME* PreviousFrame; - uint64_t Reserved2; } KTRAP_FRAME; typedef struct SYS_TRAP_FRAME_INFO_K { diff --git a/code/kernel/include/kernel/ldr_image.h b/code/kernel/include/kernel/ldr_image.h index d097db7..cffafa5 100644 --- a/code/kernel/include/kernel/ldr_image.h +++ b/code/kernel/include/kernel/ldr_image.h @@ -36,6 +36,6 @@ KSTATUS LdrLoadModuleForProcess(struct PROCESS* process, const char* filename, u // Called by KernelMain to register all modules from BOOT_INFO so that // LdrGetExport can find them. -struct BOOT_INFO; // forward declare for the compiler to shut the fuck up +struct BOOT_INFO; // forward declaration void LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo); void LdrRegisterModule(const char* name, uintptr_t base); diff --git a/code/kernel/include/kernel/mm_phys.h b/code/kernel/include/kernel/mm_phys.h index 1b1dd14..b8009d8 100644 --- a/code/kernel/include/kernel/mm_phys.h +++ b/code/kernel/include/kernel/mm_phys.h @@ -18,6 +18,7 @@ bool MmInitializePhysical(const MEMORY_MAP* map, // Allocate / free a single 4 KB frame. // MmAllocateFrame returns 0 on failure. PHYSICAL_ADDRESS MmAllocateFrame(void); +PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive); void MmFreeFrame(PHYSICAL_ADDRESS frame); // Allocate 'count' physically contiguous 4 KB frames. diff --git a/code/kernel/include/kernel/mm_virt.h b/code/kernel/include/kernel/mm_virt.h index 3501795..513c3c4 100644 --- a/code/kernel/include/kernel/mm_virt.h +++ b/code/kernel/include/kernel/mm_virt.h @@ -53,6 +53,13 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint // Returns 0 if the address is not mapped. __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt); +// TLB/cache coherency hooks for the shared kernel address space. Local flushes +// use INVLPG directly; kernel mapping changes broadcast an NMI shootdown to +// other online CPUs and wait for acknowledgements. +void MmFlushTlbLocal(uintptr_t virt); +void MmFlushTlbForKernelMapping(uintptr_t virt); +int MmHandleTlbShootdownNmi(void); + // --------------------------------------------------------------------------- // User/kernel copy boundary helpers diff --git a/code/kernel/ke_cpu.c b/code/kernel/ke_cpu.c index fa4c450..2c5f4e4 100644 --- a/code/kernel/ke_cpu.c +++ b/code/kernel/ke_cpu.c @@ -1,26 +1,58 @@ // ke_cpu.c - bootstrap/per-CPU state plumbing +#include #include +#include #include #define BOOTSTRAP_CPU_STACK_SIZE 0x4000u +#define IA32_GS_BASE 0xC0000101u +#define IA32_KERNEL_GS_BASE 0xC0000102u KCPU gCpus[KE_MAX_CPUS]; uint32_t gCpuCount = 0; static uint8_t gBootstrapCpuStack[BOOTSTRAP_CPU_STACK_SIZE]; +static volatile long gKernelGsReady = 0; + +extern KCPU* KeReadCurrentCpuFromGs(void); + +void KeLoadKernelGsBase(KCPU* cpu) { + if (!cpu) + cpu = &gCpus[0]; + + cpu->Self = cpu; + __writemsr(IA32_GS_BASE, (uint64_t)(uintptr_t)cpu); + gKernelGsReady = 1; +} + +void KeSetUserGsBase(uintptr_t userGsBase) { + // While running in the kernel, GS_BASE is the current KCPU. The user TEB + // base is parked in IA32_KERNEL_GS_BASE so swapgs can exchange it with the + // KCPU pointer on return to ring 3. + __writemsr(IA32_KERNEL_GS_BASE, (uint64_t)userGsBase); +} + +uintptr_t KeGetUserGsBase(void) { + return (uintptr_t)__readmsr(IA32_KERNEL_GS_BASE); +} void KeInitializeBootstrapCpu(void) { memset(gCpus, 0, sizeof(gCpus)); KCPU* cpu = &gCpus[0]; + cpu->Self = cpu; cpu->CpuIndex = 0; cpu->LapicId = 0; cpu->Present = 1; + cpu->Started = 1; + cpu->Parked = 0; cpu->Online = 1; cpu->IsBsp = 1; cpu->CurrentIrql = PASSIVE_LEVEL; cpu->InterruptDepth = 0; cpu->ReschedulePending = 0; + cpu->SchedulerCommand = 0; + cpu->SchedulerActive = 0; cpu->KernelStack = gBootstrapCpuStack; cpu->KernelStackTop = (uintptr_t)(gBootstrapCpuStack + sizeof(gBootstrapCpuStack)); memset(&cpu->Tss, 0, sizeof(cpu->Tss)); @@ -28,11 +60,18 @@ void KeInitializeBootstrapCpu(void) { cpu->Tss.RSP0 = cpu->KernelStackTop; gCpuCount = 1; + KeLoadKernelGsBase(cpu); + KeSetUserGsBase(0); } KCPU* KeGetCurrentCpu(void) { - // SMP phase 1: only CPU0 is online. Later this becomes LAPIC-ID or - // per-CPU-base based lookup once the bootloader parks APs for us. + if (gKernelGsReady) { + KCPU* cpu = KeReadCurrentCpuFromGs(); + if (cpu >= &gCpus[0] && cpu < &gCpus[KE_MAX_CPUS] && cpu->Self == cpu) + return cpu; + } + + // Early bootstrap fallback before GS_BASE has been installed. return &gCpus[0]; } diff --git a/code/kernel/ke_debug.c b/code/kernel/ke_debug.c index 1b2996a..8d9e99d 100644 --- a/code/kernel/ke_debug.c +++ b/code/kernel/ke_debug.c @@ -3,18 +3,48 @@ #include #include #include +#include +#include #include +#include #include #include #include #include #include #include -int _fltused; // fuckass +int _fltused; // MSVC floating-point helper anchor int gDebugEnabled = 0; void (*PrintChar)(char, uint32_t, uint32_t); +static volatile long gPanicInProgress = 0; +static volatile long gPanicStoppedCpuCount = 0; + +static _Noreturn void KiPanicSecondaryHalt(void) { + _disable(); + _InterlockedIncrement(&gPanicStoppedCpuCount); + for (;;) __halt(); +} + +static int KiPanicIsActive(void) { + return _InterlockedCompareExchange((volatile long*)&gPanicInProgress, 0, 0) != 0; +} + +// Returning NMI handler used for SMP control IPIs. Panic NMIs stop the +// target CPU permanently; TLB shootdown NMIs flush and return. A genuinely +// unexpected NMI is not handled here and will fall through to the normal fault +// path in the assembly stub. +__declspec(dllexport) int KeHandleNmiIpi(void) { + if (KiPanicIsActive()) + KiPanicSecondaryHalt(); + + if (MmHandleTlbShootdownNmi()) + return 1; + + return 0; +} + void KiKernelPutChar(char c) { PrintChar(c, 0x00FF00, 0x000000); } @@ -39,26 +69,48 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) { // KePanic __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) { - // Silence! Stop the world! + // Silence this CPU first. _disable(); - // Print a bright-red banner so the framebuffer is unmistakable. - // We call PrintChar directly so we can set a red foreground. - // (DbgPrint uses green; we want this visually distinct.) - static const char banner[] = "\n*** KERNEL PANIC ***\n"; - for (const char* p = banner; *p; p++) - PrintChar(*p, 0xFF2222, 0x000000); + // Only one CPU owns the visible panic report. CPUs that fault while the + // system is already panicking immediately join the stopped set. + if (_InterlockedCompareExchange((volatile long*)&gPanicInProgress, 1, 0) != 0) { + KiPanicSecondaryHalt(); + } - // Print the panic code first, then the caller's message through the normal vprintf path. - printf("Panic code: %d\n", code); + // Stop every other CPU as early as possible. This uses an NMI IPI so it + // still reaches CPUs with IF cleared inside spinlock/high-IRQL regions. + HalApicSendPanicIpi(); - va_list va; - va_start(va, fmt); - vprintf(fmt, va); - va_end(va); + if (PrintChar) { + // Print a bright-red banner so the framebuffer is unmistakable. + static const char banner[] = "\n*** KERNEL PANIC ***\n"; + const char* p; + KCPU* cpu = KeGetCurrentCpu(); + THREAD* thread = KeGetCurrentThread(); - // Die! - while (true) __halt(); + for (p = banner; *p; p++) + PrintChar(*p, 0xFF2222, 0x000000); + + printf("Panic code: %d\n", code); + if (cpu) { + printf("Panic CPU: %u LAPIC=%u", + (unsigned)cpu->CpuIndex, + (unsigned)cpu->LapicId); + if (thread) + printf(" thread=%p tid=%llu", + thread, + (unsigned long long)thread->ThreadId); + printf("\n"); + } + + va_list va; + va_start(va, fmt); + vprintf(fmt, va); + va_end(va); + } + + for (;;) __halt(); } // KePrintConsole - write user-visible console text in white-on-black. @@ -434,6 +486,11 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u } __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) { + // Panic NMIs are broadcast to stop all other CPUs. Do not try to decode + // the interrupted context on a secondary CPU once another CPU owns panic. + if (KiPanicIsActive()) + KiPanicSecondaryHalt(); + THREAD* thread = KeGetCurrentThread(); PROCESS* proc = thread ? thread->Owner : (PROCESS*)0; @@ -453,14 +510,26 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr (unsigned long long)frame->ErrorCode, (unsigned long long)frame->Cr2); } else if (KiFaultHasErrorCode(vector)) { - KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx | Code: %04llx\n", + const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip; + KePanic(KE_PANIC_CODE_FAULT, + "PANIC: %s | RIP: %016llx | Code: %04llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n", name, (unsigned long long)frame->Rip, - (unsigned long long)frame->ErrorCode); + (unsigned long long)frame->ErrorCode, + (unsigned long long)frame->Cs, + (unsigned long long)frame->Rflags, + (unsigned long long)frame->Rsp, + ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]); } else { - KePanic(KE_PANIC_CODE_FAULT, "PANIC: %s | RIP: %016llx\n", + const uint8_t* ip = (const uint8_t*)(uintptr_t)frame->Rip; + KePanic(KE_PANIC_CODE_FAULT, + "PANIC: %s | RIP: %016llx | CS=%04llx RFLAGS=%016llx RSP=%016llx | bytes=%02x %02x %02x %02x %02x %02x %02x %02x\n", name, - (unsigned long long)frame->Rip); + (unsigned long long)frame->Rip, + (unsigned long long)frame->Cs, + (unsigned long long)frame->Rflags, + (unsigned long long)frame->Rsp, + ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7]); } } diff --git a/code/kernel/ke_dpc.c b/code/kernel/ke_dpc.c index 621c3f0..242c310 100644 --- a/code/kernel/ke_dpc.c +++ b/code/kernel/ke_dpc.c @@ -64,8 +64,8 @@ void KiDrainDpcQueue(void) { if (!gDpcInitialized) return; - // Never run DPC routines above DISPATCH_LEVEL. If a future nested IRQ path - // reaches here at DIRQL, leave the work pending for the outermost exit. + // Never run DPC routines above DISPATCH_LEVEL. If a nested IRQ reaches here + // at DIRQL, leave the work pending for the outermost exit. if (KeGetCurrentIrql() > DISPATCH_LEVEL) return; diff --git a/code/kernel/ke_gdt_reload.asm b/code/kernel/ke_gdt_reload.asm index aa7c11d..0b68c2a 100644 --- a/code/kernel/ke_gdt_reload.asm +++ b/code/kernel/ke_gdt_reload.asm @@ -1,45 +1,52 @@ -; ke_helpers.asm -.code - -; void KeFlushGdt(GDTR* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel); -KeFlushGdt PROC - ; RCX = pGdtr (pointer to limit/base struct) - ; RDX = codeSel - ; R8 = dataSel - ; R9 = tssSel - - ; 1. Load the GDT - lgdt fword ptr [rcx] - - ; 2. Reload Data Segments - mov ax, r8w - mov ds, ax - mov es, ax - mov ss, ax - mov fs, ax - mov gs, ax - - ; 3. Load Task Register - mov ax, r9w - ltr ax - - ; 4. Reload Code Segment (Far Return) - ; In MASM64, use OFFSET to get the address of the label. - ; We push the selector, then the offset, then issue retff / retfq. - - push rdx ; Push new Code Selector (RDX) - lea rax, [reload_cs] ; RIP-relative address of the label - push rax ; Push "return" address - db 048h, 0CBh ; rex_w retf (effectively retfq) - -reload_cs: - ret -KeFlushGdt ENDP - -; void KeFlushIdt(IDTR* pIdtr); -KeFlushIdt PROC - lidt fword ptr [rcx] - ret -KeFlushIdt ENDP - -END +; ke_helpers.asm +.code + +; void KeFlushGdt(GDTR* pGdtr, uint16_t codeSel, uint16_t dataSel, uint16_t tssSel); +KeFlushGdt PROC + ; RCX = pGdtr (pointer to limit/base struct) + ; RDX = codeSel + ; R8 = dataSel + ; R9 = tssSel + + ; 1. Load the GDT + lgdt fword ptr [rcx] + + ; 2. Reload Data Segments + mov ax, r8w + mov ds, ax + mov es, ax + mov ss, ax + mov fs, ax + mov gs, ax + + ; 3. Load Task Register + mov ax, r9w + ltr ax + + ; 4. Reload Code Segment (Far Return) + ; In MASM64, use OFFSET to get the address of the label. + ; We push the selector, then the offset, then issue retff / retfq. + + push rdx ; Push new Code Selector (RDX) + lea rax, [reload_cs] ; RIP-relative address of the label + push rax ; Push "return" address + db 048h, 0CBh ; rex_w retf (effectively retfq) + +reload_cs: + ret +KeFlushGdt ENDP + +; void KeFlushIdt(IDTR* pIdtr); +KeFlushIdt PROC + lidt fword ptr [rcx] + ret +KeFlushIdt ENDP + +; KCPU* KeReadCurrentCpuFromGs(void); +; GS:0 contains KCPU.Self while executing in the kernel. +KeReadCurrentCpuFromGs PROC + mov rax, qword ptr gs:[0] + ret +KeReadCurrentCpuFromGs ENDP + +END diff --git a/code/kernel/ke_idt.c b/code/kernel/ke_idt.c index 959b0ad..42e8433 100644 --- a/code/kernel/ke_idt.c +++ b/code/kernel/ke_idt.c @@ -41,9 +41,11 @@ extern void KeIrqStub20(void); extern void KeIrqStub21(void); extern void KeIrqStub22(void); extern void KeIrqStub23(void); +extern void KeRescheduleIpiStub(void); extern void KeFaultDB(void); extern void KeFaultNMI(void); +extern void KeNmiIpiStub(void); extern void KeFaultBP(void); extern void KeFaultOF(void); extern void KeFaultBR(void); @@ -77,16 +79,9 @@ extern void KeFaultRSV31(void); // --------------------------------------------------------------------------- // GDT // --------------------------------------------------------------------------- -#define GDT_ENTRIES 8 -static uint64_t gGdt[GDT_ENTRIES]; - -#pragma pack(push, 1) -typedef struct { uint16_t Limit; uint64_t Base; } GDTR; -#pragma pack(pop) - -static void KiGdtSetEntry(int idx, uint32_t base, uint32_t limit, +static void KiGdtSetEntry(uint64_t* gdt, int idx, uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) { - gGdt[idx] = + gdt[idx] = ((uint64_t)(limit & 0xFFFF)) | ((uint64_t)(base & 0xFFFFFF) << 16) | ((uint64_t)access << 40) | @@ -95,54 +90,71 @@ static void KiGdtSetEntry(int idx, uint32_t base, uint32_t limit, ((uint64_t)(base >> 24 & 0xFF)<< 56); } -static void KiGdtSetTss(int idx, uintptr_t base, uint32_t limit) { +static void KiGdtSetTss(uint64_t* gdt, int idx, uintptr_t base, uint32_t limit) { // TSS descriptor is 16 bytes (two GDT slots). - gGdt[idx] = 0; - gGdt[idx+1] = 0; + gdt[idx] = 0; + gdt[idx+1] = 0; - gGdt[idx] = + gdt[idx] = ((uint64_t)(limit & 0xFFFF)) | ((uint64_t)(base & 0xFFFFFF) << 16) | (0x89ULL << 40) | // P=1,DPL=0,type=TSS64 avail ((uint64_t)((limit>>16)&0xF) << 48) | ((uint64_t)(base>>24 & 0xFF) << 56); - gGdt[idx+1] = (base >> 32); + gdt[idx+1] = (base >> 32); } -TSS64* gTss = (TSS64*)0; +void KeInitializeGdtForCpu(KCPU* cpu) { + if (!cpu) return; -void KeInitializeGdt(void) { - KCPU* cpu = KeGetCurrentCpu(); TSS64* tss = &cpu->Tss; - gTss = tss; + uint64_t* gdt = cpu->Gdt; - memset(gGdt, 0, sizeof(gGdt)); + memset(gdt, 0, sizeof(cpu->Gdt)); // 0x00: Null - KiGdtSetEntry(0, 0, 0, 0, 0); + KiGdtSetEntry(gdt, 0, 0, 0, 0, 0); // 0x08: Ring-0 Code (Access 0x9A, Flags 0xA for 64-bit) - KiGdtSetEntry(1, 0, 0xFFFFF, 0x9A, 0xA); + KiGdtSetEntry(gdt, 1, 0, 0xFFFFF, 0x9A, 0xA); // 0x10: Ring-0 Data - KiGdtSetEntry(2, 0, 0xFFFFF, 0x92, 0xC); + KiGdtSetEntry(gdt, 2, 0, 0xFFFFF, 0x92, 0xC); // 0x18: Ring-3 Code - KiGdtSetEntry(3, 0, 0xFFFFF, 0xFA, 0xA); + KiGdtSetEntry(gdt, 3, 0, 0xFFFFF, 0xFA, 0xA); // 0x20: Ring-3 Data - KiGdtSetEntry(4, 0, 0xFFFFF, 0xF2, 0xC); - - // 0x28: TSS (Occupies two slots: 0x28 and 0x30). - // The TSS storage is per-CPU; phase 1 initializes CPU0 only. + KiGdtSetEntry(gdt, 4, 0, 0xFFFFF, 0xF2, 0xC); + + // 0x28: CPU-local TSS (occupies slots 0x28 and 0x30). tss->IoMapBase = sizeof(TSS64); if (!tss->RSP0) tss->RSP0 = cpu->KernelStackTop; - KiGdtSetTss(5, (uintptr_t)tss, sizeof(TSS64) - 1); + KiGdtSetTss(gdt, 5, (uintptr_t)tss, sizeof(TSS64) - 1); - GDTR gdtr = { sizeof(gGdt) - 1, (uint64_t)gGdt }; + cpu->Gdtr.Limit = sizeof(cpu->Gdt) - 1; + cpu->Gdtr.Base = (uint64_t)cpu->Gdt; +} - // Call the MASM64 proc to reload everything - // Selectors: CS=0x08, Data=0x10, TSS=0x28 - KeFlushGdt(&gdtr, 0x08, 0x10, 0x28); +void KeLoadGdtForCurrentCpu(KCPU* cpu) { + if (!cpu) cpu = KeGetCurrentCpu(); + KeFlushGdt(&cpu->Gdtr, 0x08, 0x10, 0x28); + + // Reloading GS as a segment register during GDT reload can disturb the + // hidden GS base. Reinstall the kernel per-CPU base immediately after + // the descriptor reload so KeGetCurrentCpu() remains GS-backed. + KeLoadKernelGsBase(cpu); +} + +void KeInitializeGdt(void) { + KCPU* cpu = KeGetCurrentCpu(); + KeInitializeGdtForCpu(cpu); + KeLoadGdtForCurrentCpu(cpu); +} + +void KeSetCurrentTssRsp0(uintptr_t rsp0) { + TSS64* tss = KeGetCurrentTss(); + if (tss) + tss->RSP0 = rsp0; } // --------------------------------------------------------------------------- @@ -158,7 +170,7 @@ typedef struct { uint8_t TypeAttr; uint16_t OffsetMid; uint32_t OffsetHigh; - uint32_t Reserved; + uint32_t MustBeZero; } IDT_ENTRY; #pragma pack(pop) @@ -176,14 +188,14 @@ static void KiIdtSetGate(int vec, uintptr_t handler, gIdt[vec].TypeAttr = typeAttr; gIdt[vec].OffsetMid = (uint16_t)((handler >> 16) & 0xFFFF); gIdt[vec].OffsetHigh = (uint32_t)((handler >> 32) & 0xFFFFFFFF); - gIdt[vec].Reserved = 0; + gIdt[vec].MustBeZero = 0; } void KeInitializeIdt(void) { memset(gIdt, 0, sizeof(gIdt)); static void* faultStubs[32] = { - (void*)KeFaultDE, (void*)KeFaultDB, (void*)KeFaultNMI, + (void*)KeFaultDE, (void*)KeFaultDB, (void*)KeNmiIpiStub, (void*)KeFaultBP, (void*)KeFaultOF, (void*)KeFaultBR, (void*)KeFaultUD, (void*)KeFaultNM, (void*)KeFaultDF, (void*)KeFaultCSO, (void*)KeFaultTS, (void*)KeFaultNP, @@ -214,8 +226,12 @@ void KeInitializeIdt(void) { for (int i = 0; i < NUM_IRQS; i++) KiIdtSetGate(0x20 + i, (uintptr_t)irqStubs[i], 0x8E, 0); + KiIdtSetGate(KE_VECTOR_RESCHEDULE_IPI, (uintptr_t)KeRescheduleIpiStub, 0x8E, 0); + + KeLoadIdtForCurrentCpu(); +} + +void KeLoadIdtForCurrentCpu(void) { IDTR idtr = { sizeof(gIdt) - 1, (uint64_t)gIdt }; - - // Call the MASM64 proc to load IDT KeFlushIdt(&idtr); } diff --git a/code/kernel/ke_irq.c b/code/kernel/ke_irq.c index 7ee4edf..a535fc2 100644 --- a/code/kernel/ke_irq.c +++ b/code/kernel/ke_irq.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -19,10 +20,10 @@ static IRQ_HANDLER_LIST gIrqHandlers[MAX_IRQS]; static const HAL_API *gHal; static KIRQL KiIrqToIrql(uint8_t irq) { - KIRQL irql = (KIRQL)(DIRQL_BASE + irq); - if (irql >= HIGH_LEVEL) - irql = (KIRQL)(HIGH_LEVEL - 1); - return irql; + uint32_t level = (uint32_t)DIRQL_BASE + (uint32_t)irq; + if (level >= HIGH_LEVEL) + level = (uint32_t)(HIGH_LEVEL - 1); + return (KIRQL)level; } void KeIrqInitialize(const HAL_API *hal) { @@ -51,15 +52,18 @@ void KeIrqDispatch(uint8_t irq) { KIRQL targetIrql = KiIrqToIrql(irq); KIRQL currentIrql = KeGetCurrentIrql(); - // Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected - // while IF is clear, but this keeps the software state sane if that changes - // later. + // Do not ever lower IRQL on interrupt entry. Nested IRQs are not expected + // while IF is clear, but preserving monotonic IRQL keeps nesting safe. if (targetIrql < currentIrql) targetIrql = currentIrql; KIRQL oldIrql = KeRaiseIrql(targetIrql); - if (irq < MAX_IRQS) { + if (irq == KE_VECTOR_RESCHEDULE_IPI) { + // Nothing to do here. The sender has already set this CPU's + // ReschedulePending flag. KiExitInterrupt consumes it after EOI and + // after the interrupted frame unwinds to PASSIVE_LEVEL. + } else if (irq < MAX_IRQS) { IRQ_HANDLER_LIST* list = &gIrqHandlers[irq]; for (uint8_t i = 0; i < list->Count; i++) { if (list->Handlers[i]) @@ -85,10 +89,10 @@ void KiExitInterrupt(void) { if (cpu->InterruptDepth) cpu->InterruptDepth--; - // A reschedule is only legal after the interrupt unwound back to the - // interrupted thread's original PASSIVE context. If a future path ever - // permits nested interrupting of DISPATCH/DIRQL code, leave the request - // pending instead of switching while a spinlock/high-IRQL region is active. + // A reschedule is only legal after the interrupt unwinds back to the + // interrupted thread's PASSIVE context. If an interrupt hit DISPATCH/DIRQL + // code, leave the request pending instead of switching out of a spinlock or + // high-IRQL region. if (cpu->InterruptDepth == 0 && KeGetCurrentIrql() == PASSIVE_LEVEL) { KiDrainDpcQueue(); if (KeGetCurrentIrql() == PASSIVE_LEVEL) diff --git a/code/kernel/ke_sched.c b/code/kernel/ke_sched.c index be6b685..ab525ab 100644 --- a/code/kernel/ke_sched.c +++ b/code/kernel/ke_sched.c @@ -1,6 +1,7 @@ // ke_sched.c - MLFQ scheduler #include #include +#include #include #include #include @@ -8,14 +9,64 @@ #include #include #include +#include #include #include +#include static LIST_ENTRY gRunQueue[PriorityCount]; static SPINLOCK gSchedLock; static uint32_t gAgingCounter = 0; static KE_SCHED_STATS gSchedStats; +static PROCESS* gIdleProcess; + +static uint64_t KiDefaultThreadAffinityMask(void) { + // Normal kernel, user, and driver-created threads are BSP-affine by + // default. Code that is explicitly audited for AP execution can widen its + // affinity before the thread enters the ready queue. + return 1ULL; +} + +static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) { + if (!thread || !cpu) return 0; + if (thread->IsIdleThread) return thread == cpu->IdleThread; + if (cpu->CpuIndex >= 64) return 0; + return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0; +} + +static void KiRequestRescheduleCpuLocked(KCPU* target) { + if (!target || !target->Online) + return; + + _InterlockedExchange(&target->ReschedulePending, 1); + + KCPU* self = KeGetCurrentCpu(); + if (self && target->CpuIndex != self->CpuIndex) { + HalApicSendRescheduleIpi(target->LapicId); + } +} + +static void KiPokeRunnableCpusLocked(THREAD* thread) { + if (!thread || thread->IsIdleThread) + return; + + for (uint32_t i = 0; i < gCpuCount; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present || !cpu->Started || !cpu->Online) + continue; + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + + THREAD* cur = cpu->CurrentThread; + if (!cur || cur == cpu->IdleThread) { + KiRequestRescheduleCpuLocked(cpu); + } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) { + gSchedStats.PriorityPreemptions++; + KiRequestRescheduleCpuLocked(cpu); + } + } +} static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) { switch (priority) { @@ -60,29 +111,133 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) { KeReleaseSpinlock(&gSchedLock, flags); } -static void KiIdleLoop(void) { - while (true) { __halt(); } + +static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) { + // The scheduler lock is intentionally held across KeSwitchContext(). The + // unlock cookie belongs to the thread continuation that will resume after + // the switch, not to the physical CPU. Another dispatch may resume this + // thread from an unrelated interrupt, so using a KCPU slot here would + // restore the outgoing CPU context's IF/IRQL into the incoming thread. + if (thread) + thread->SwitchLockCookie = cookie; } -void KeInitializeScheduler(void) { - KCPU* cpu = KeGetCurrentCpu(); +static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) { + return thread ? thread->SwitchLockCookie : 0x202ULL; +} + +static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) { + KePanic(KE_PANIC_CODE_SCHEDULER, + "scheduler invariant failed: %s: thread=0x%llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n", + where, + (uint64_t)thread, + thread ? thread->ThreadId : 0, + thread ? (unsigned)thread->State : 0, + thread ? (unsigned)thread->Queued : 0, + thread ? (unsigned)thread->RunningCpu : 0, + thread ? (unsigned)thread->IsIdleThread : 0, + why ? why : "?"); +} + +static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread) { + if (!thread) + KiSchedulerBug(where, thread, "null thread"); + if (thread->IsIdleThread) + KiSchedulerBug(where, thread, "idle thread cannot enter ready queue"); + if (thread->State == ThreadDead) + KiSchedulerBug(where, thread, "dead thread cannot enter ready queue"); + if (thread->Queued) + KiSchedulerBug(where, thread, "thread already in ready queue"); + if (thread->RunningCpu != KE_INVALID_CPU) + KiSchedulerBug(where, thread, "thread still reserved/running on a CPU"); +} + +static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) { + if (!thread) + KiSchedulerBug(where, thread, "null picked thread"); + if (thread->IsIdleThread) + return; + if (thread->Queued) + KiSchedulerBug(where, thread, "picked thread still marked queued"); + if (thread->RunningCpu != KE_INVALID_CPU) + KiSchedulerBug(where, thread, "picked thread already running on a CPU"); + if (thread->State != ThreadReady) + KiSchedulerBug(where, thread, "picked thread was not ready"); +} + +static void KiInsertReadyLocked(THREAD* thread) { + KiAssertReadyQueueCandidateLocked("KiInsertReadyLocked", thread); + + int p = (int)thread->Priority; + if (p < 0) p = 0; + if (p >= PriorityCount) p = PriorityCount - 1; + + thread->State = ThreadReady; + thread->Queued = 1; + ListInsertTail(&gRunQueue[p], &thread->SchedulerLink); +} + +static int KiAnyRunnableForCpuLocked(KCPU* cpu); + +static int KiAnyRunnableForCurrentCpu(void) { + KCPU* cpu = KeGetCurrentCpu(); + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + int any = KiAnyRunnableForCpuLocked(cpu); + KeReleaseSpinlock(&gSchedLock, flags); + return any; +} + +static void KiIdleLoop(void) { + for (;;) { + KCPU* cpu = KeGetCurrentCpu(); + cpu->SchedulerTicks++; + + // AP idle threads sleep until an interrupt or reschedule IPI wakes the + // CPU. Ready paths set ReschedulePending and target the CPU's LAPIC; + // KiExitInterrupt consumes the request after EOI. + if (KiAnyRunnableForCurrentCpu()) { + KeYield(); + continue; + } + + // HLT only wakes for maskable IPIs when IF=1. A CPU that reaches + // the idle thread through an unusual switch path must not sleep with + // interrupts disabled, otherwise reschedule/TLB shootdown IPIs can + // be delayed forever. + _enable(); + __halt(); + } +} + +void KiInitializeSchedulerGlobal(void) { for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]); gSchedLock = (SPINLOCK)SPINLOCK_INIT; - PROCESS* idleProc; - KSTATUS s = KeCreateProcess("idle", &idleProc); - if (!KSUCCESS(s)) return; + KSTATUS s = KeCreateProcess("idle", &gIdleProcess); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create idle process, status=%u\n", s); PROCESS* systemProc; s = KeCreateSystemProcess(&systemProc); - if (!KSUCCESS(s)) return; + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create system process, status=%u\n", s); (void)systemProc; +} + +KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) { + if (!cpu || !gIdleProcess) + return STATUS_INVALID_PARAMETER; + if (cpu->IdleThread) + return STATUS_SUCCESS; THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD)); + if (!idle) + return STATUS_NO_MEMORY; + idle->Header.Destroy = (void *)0; - idle->ThreadId = 0; - idle->State = ThreadReady; + idle->ThreadId = cpu->CpuIndex ? (0x80000000u | cpu->CpuIndex) : 0; + idle->State = ThreadRunning; idle->Priority = PriorityLow; idle->QuantaConsumed = 0; idle->QuantumTicks = 1; @@ -90,12 +245,16 @@ void KeInitializeScheduler(void) { idle->CpuBurstTicks = 0; idle->LastRunTicks = 0; idle->RunningCpu = cpu->CpuIndex; - idle->Owner = idleProc; + idle->Queued = 0; + idle->IsIdleThread = 1; + idle->SchedulerFlags = 0; + idle->LastCpu = cpu->CpuIndex; + idle->AffinityMask = (cpu->CpuIndex < 64) ? (1ULL << cpu->CpuIndex) : 0ULL; + idle->Owner = gIdleProcess; idle->WaitObject = (OBJECT *)0; idle->WaitStatus = STATUS_SUCCESS; idle->WaitListHead = (LIST_ENTRY*)0; idle->AlertableWait = 0; - idle->SchedLockFlags = 0x202ULL; // IF=1 idle->UserStackBase = (void*)0; idle->Teb = (void*)0; idle->TebSize = 0; @@ -106,6 +265,10 @@ void KeInitializeScheduler(void) { LIST_INIT(&idle->SchedulerLink); void* kstack = MmAllocatePool(KSTACK_SIZE); + if (!kstack) { + ObDereferenceObject(&idle->Header); + return STATUS_NO_MEMORY; + } idle->KernelStackBase = kstack; idle->KernelStackTop = (uintptr_t)kstack + KSTACK_SIZE; idle->TssRsp0 = idle->KernelStackTop; @@ -121,22 +284,34 @@ void KeInitializeScheduler(void) { // The hand-built CPU idle thread is not created through KeCreateThread(), // so register it manually with the idle process. This keeps PID 0 visible // as the real idle process while kernel worker threads live under PID 1. - ObReferenceObject(&idleProc->Header); - uintptr_t idleFlags = KeAcquireSpinlock(&idleProc->HandleLock); - ListInsertTail(&idleProc->ThreadList, &idle->ProcessLink); - idleProc->ThreadCount++; - KeReleaseSpinlock(&idleProc->HandleLock, idleFlags); + ObReferenceObject(&gIdleProcess->Header); + uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock); + ListInsertTail(&gIdleProcess->ThreadList, &idle->ProcessLink); + gIdleProcess->ThreadCount++; + KeReleaseSpinlock(&gIdleProcess->HandleLock, idleFlags); - cpu->IdleThread = idle; + cpu->IdleThread = idle; cpu->CurrentThread = idle; cpu->OutgoingThread = (THREAD*)0; + idle->SwitchLockCookie = 0x202ULL; + return STATUS_SUCCESS; +} + +void KeInitializeScheduler(void) { + KiInitializeSchedulerGlobal(); + + KCPU* cpu = KeGetCurrentCpu(); + KSTATUS s = KiCreateIdleThreadForCpu(cpu); + if (!KSUCCESS(s)) + KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create CPU%u idle thread, status=%u\n", + (unsigned)cpu->CpuIndex, s); } // Safely release the lock and clean up any dead threads natively post-switch void KeSchedulerPostSwitch(void) { KCPU* cpu = KeGetCurrentCpu(); THREAD* cur = cpu->CurrentThread; - KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags); + KeReleaseSpinlock(&gSchedLock, KiPostSwitchUnlockCookie(cur)); KiDeliverKernelApcs(); @@ -148,24 +323,29 @@ void KeSchedulerPostSwitch(void) { } } -void KeReadyThread(THREAD* thread) { - if (!thread) return; - thread->State = ThreadReady; - int p = (int)thread->Priority; - if (p < 0) p = 0; - if (p >= PriorityCount) p = PriorityCount - 1; +void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask) { + if (!thread || thread->IsIdleThread) return; + + if (!affinityMask) + affinityMask = 1ULL; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - ListInsertTail(&gRunQueue[p], &thread->SchedulerLink); + thread->AffinityMask = affinityMask; + if (thread->Queued) + KiPokeRunnableCpusLocked(thread); + KeReleaseSpinlock(&gSchedLock, flags); +} - KCPU* cpu = KeGetCurrentCpu(); - THREAD* cur = cpu->CurrentThread; - if (!cur || cur == cpu->IdleThread) { - KiRequestReschedule(); - } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) { - gSchedStats.PriorityPreemptions++; - KiRequestReschedule(); - } +uint64_t KeGetDefaultThreadAffinityMask(void) { + return KiDefaultThreadAffinityMask(); +} + +void KeReadyThread(THREAD* thread) { + if (!thread) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + KiInsertReadyLocked(thread); + KiPokeRunnableCpusLocked(thread); KeReleaseSpinlock(&gSchedLock, flags); } @@ -173,35 +353,58 @@ void KeReadyThread(THREAD* thread) { void KeUnreadyThread(THREAD* thread) { if (!thread) return; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - ListRemove(&thread->SchedulerLink); + if (thread->Queued) { + ListRemove(&thread->SchedulerLink); + thread->Queued = 0; + } KeReleaseSpinlock(&gSchedLock, flags); } -static int KiAnyRunnableLocked(void) { + +static int KiAnyRunnableForCpuLocked(KCPU* cpu) { + if (!cpu) cpu = KeGetCurrentCpu(); + for (int p = 0; p < PriorityCount; p++) { - if (!LIST_IS_EMPTY(&gRunQueue[p])) - return 1; + LIST_ENTRY* head = &gRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (KiThreadCanRunOnCpuLocked(thread, cpu)) + return 1; + } } return 0; } -static THREAD* KiPickNext(void) { +static THREAD* KiPickNextForCpu(KCPU* cpu) { + if (!cpu) cpu = KeGetCurrentCpu(); + for (int p = PriorityCount - 1; p >= 0; p--) { - if (!LIST_IS_EMPTY(&gRunQueue[p])) { - LIST_ENTRY* node = gRunQueue[p].Fwd; + LIST_ENTRY* head = &gRunQueue[p]; + for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) { + THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink); + if (!KiThreadCanRunOnCpuLocked(thread, cpu)) + continue; + ListRemove(node); - return CONTAINING_RECORD(node, THREAD, SchedulerLink); + thread->Queued = 0; + KiAssertPickedThreadLocked("KiPickNextForCpu", thread); + return thread; } } - return KeGetCurrentCpu()->IdleThread; + + return cpu->IdleThread; } + void KiRequestReschedule(void) { - KeGetCurrentCpu()->ReschedulePending = 1; + KCPU* cpu = KeGetCurrentCpu(); + if (cpu) + _InterlockedExchange(&cpu->ReschedulePending, 1); } int KiIsReschedulePending(void) { - return KeGetCurrentCpu()->ReschedulePending != 0; + KCPU* cpu = KeGetCurrentCpu(); + return cpu && _InterlockedCompareExchange(&cpu->ReschedulePending, 0, 0) != 0; } static void KiSchedule(void) { @@ -210,16 +413,16 @@ static void KiSchedule(void) { THREAD* idle = cpu->IdleThread; if (!cur) return; - cpu->ReschedulePending = 0; + _InterlockedExchange(&cpu->ReschedulePending, 0); uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - cur->SchedLockFlags = flags; + KiArmPostSwitchUnlock(cur, flags); // The currently running thread is not present in a run queue. If it is // still runnable and nobody else is ready, keep executing it. Otherwise a // lone CPU-bound thread gets alternated with the idle thread on every timer // tick, effectively turning a periodic preemption check into a 50% throttle. - if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableLocked()) { + if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableForCpuLocked(cpu)) { if (cur->QuantumRemaining == 0) KiRefillQuantumLocked(cur); gSchedStats.KeepCurrentDecisions++; @@ -227,22 +430,21 @@ static void KiSchedule(void) { return; } - THREAD* next = KiPickNext(); + THREAD* next = KiPickNextForCpu(cpu); if (next == idle) gSchedStats.IdleSelections++; if (next == cur) { - // Only re-queue if still running normally; a ThreadWaiting/ThreadDead - // caller has already placed SchedulerLink elsewhere (or is discarding - // the thread). - if (cur != idle && cur->State == ThreadRunning) - ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); + if (cur != idle) + KiSchedulerBug("KiSchedule", cur, "current thread unexpectedly appeared in ready queue"); KeReleaseSpinlock(&gSchedLock, flags); return; } + KiAssertPickedThreadLocked("KiSchedule", next); next->State = ThreadRunning; KiRefillQuantumLocked(next); cpu->CurrentThread = next; next->RunningCpu = cpu->CpuIndex; + next->LastCpu = cpu->CpuIndex; // Only re-queue the outgoing thread if it is still in the running state. // If KeBlockThread set State = ThreadWaiting, SchedulerLink was already @@ -250,11 +452,14 @@ static void KiSchedule(void) { // corrupt both lists simultaneously (same intrusive node in two lists). // If KeExitThread set State = ThreadDead, the thread must not be re-queued. if (cur != idle && cur->State == ThreadRunning) { + cur->RunningCpu = KE_INVALID_CPU; + cur->LastCpu = cpu->CpuIndex; cur->State = ThreadReady; - ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink); + KiInsertReadyLocked(cur); + } else if (cur) { + cur->RunningCpu = KE_INVALID_CPU; + cur->LastCpu = cpu->CpuIndex; } - - if (cur) cur->RunningCpu = KE_INVALID_CPU; cpu->OutgoingThread = cur; cpu->ContextSwitches++; gSchedStats.ContextSwitches++; @@ -267,6 +472,28 @@ void KiScheduleFromInterrupt(void) { KiSchedule(); } + +void KiSchedulerMarkThreadWaiting(THREAD* thread) { + if (!thread) return; + + KCPU* cpu = KeGetCurrentCpu(); + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + + if (thread->IsIdleThread) + KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "idle thread cannot block"); + if (thread->Queued) + KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "queued thread cannot block"); + if (thread != cpu->CurrentThread) + KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "only the current thread can prepare a blocking wait"); + if (thread->RunningCpu != cpu->CpuIndex) + KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread running CPU does not match current CPU"); + if (thread->State != ThreadRunning) + KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread was not running before wait"); + + thread->State = ThreadWaiting; + KeReleaseSpinlock(&gSchedLock, flags); +} + void KiSchedulerNoteVoluntaryBlock(THREAD* thread) { if (!thread) return; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); @@ -275,6 +502,51 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) { KeReleaseSpinlock(&gSchedLock, flags); } + +void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) { + if (!thread) return; + + uintptr_t flags = KeAcquireSpinlock(&gSchedLock); + + // Complete the wait while holding the scheduler lock. On SMP a wake can + // race the prepare->commit window on another CPU: the target thread is + // already on the object's wait list and has State=ThreadWaiting, but it is + // still executing with RunningCpu set until it actually calls KeYield(). + // That case must complete in-place, not enqueue the still-running thread. + if (thread->State != ThreadWaiting) { + thread->WaitStatus = status; + KeReleaseSpinlock(&gSchedLock, flags); + return; + } + + thread->WaitObject = (OBJECT*)0; + thread->WaitStatus = status; + thread->WaitListHead = (LIST_ENTRY*)0; + thread->AlertableWait = 0; + + thread->CpuBurstTicks = 0; + thread->QuantaConsumed = 0; + thread->QuantumRemaining = 0; + + if (thread->RunningCpu != KE_INVALID_CPU) { + // Prepare->commit race: the waiter is still running somewhere. Its + // KeCommitBlockThread() will see State != ThreadWaiting and return the + // completed wait status without ever entering the ready queue. + thread->State = ThreadRunning; + KeReleaseSpinlock(&gSchedLock, flags); + return; + } + + if (thread->Priority < PriorityHigh) { + thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1); + } + + KiInsertReadyLocked(thread); + KiPokeRunnableCpusLocked(thread); + + KeReleaseSpinlock(&gSchedLock, flags); +} + void KeSchedulerTick(void) { KCPU* cpu = KeGetCurrentCpu(); THREAD* cur = cpu->CurrentThread; @@ -282,14 +554,14 @@ void KeSchedulerTick(void) { if (!cur) return; uintptr_t flags = KeAcquireSpinlock(&gSchedLock); - cur->SchedLockFlags = flags; + KiArmPostSwitchUnlock(cur, flags); gSchedStats.Ticks++; int requestReschedule = 0; if (cur == idle) { - if (KiAnyRunnableLocked()) + if (KiAnyRunnableForCpuLocked(cpu)) requestReschedule = 1; } else { cur->CpuBurstTicks++; @@ -312,7 +584,7 @@ void KeSchedulerTick(void) { cur->QuantumRemaining--; if (cur->QuantumRemaining == 0) { - if (KiAnyRunnableLocked()) { + if (KiAnyRunnableForCpuLocked(cpu)) { gSchedStats.QuantumExpirations++; requestReschedule = 1; } else { diff --git a/code/kernel/ke_selftest.c b/code/kernel/ke_selftest.c index 9cb9766..ed20938 100644 --- a/code/kernel/ke_selftest.c +++ b/code/kernel/ke_selftest.c @@ -1,10 +1,11 @@ -// ke_selftest.c - temporary kernel bring-up self-tests for execution/I/O model. +// ke_selftest.c - kernel bring-up and regression self-tests. #include #include #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ #include #include #include +#include // ------------------------------------------------------------------------- // Kernel APC self-test @@ -747,6 +749,86 @@ static void KiRunTrapFrameSelfTest(void) { DbgPrint("TRAP selftest: PASS\n"); } + + +void KiRunCoreHardeningSelfTests(void) { + DbgPrint("CORE selftest: begin\n"); + + if (gCpuCount == 0 || gCpuCount > KE_MAX_CPUS) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid CPU count %u\n", gCpuCount); + + KCPU* cpu = KeGetCurrentCpu(); + if (cpu != &gCpus[0]) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU mismatch cpu=0x%llX cpu0=0x%llX\n", + (uint64_t)cpu, (uint64_t)&gCpus[0]); + if (cpu->Self != cpu) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GS/KCPU self pointer mismatch self=0x%llX cpu=0x%llX\n", + (uint64_t)cpu->Self, (uint64_t)cpu); + if (KeGetCurrentCpuIndex() != 0) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current CPU index is not BSP/0\n"); + if (!cpu->Present || !cpu->Started || !cpu->Online || !cpu->IsBsp) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP lifecycle flags invalid P/S/O/B=%u/%u/%u/%u\n", + cpu->Present, cpu->Started, cpu->Online, cpu->IsBsp); + if (cpu->Parked) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: BSP is marked parked\n"); + if (KeGetCurrentIrql() != PASSIVE_LEVEL) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: expected PASSIVE_LEVEL, got %u\n", KeGetCurrentIrql()); + if (KeGetCurrentTss() != &cpu->Tss) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current TSS is not CPU-local\n"); + if (!cpu->Tss.RSP0 || cpu->Tss.IoMapBase != sizeof(TSS64)) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: invalid BSP TSS rsp0=0x%llX iomap=%u\n", + cpu->Tss.RSP0, cpu->Tss.IoMapBase); + if (!cpu->Gdtr.Base || cpu->Gdtr.Base != (uint64_t)cpu->Gdt) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: GDT is not CPU-local base=0x%llX gdt=0x%llX\n", + cpu->Gdtr.Base, (uint64_t)cpu->Gdt); + + for (uint32_t i = 1; i < gCpuCount; i++) { + KCPU* ap = &gCpus[i]; + if (!ap->Present || !ap->Started || ap->Parked || !ap->Online || ap->IsBsp) + KePanic(KE_PANIC_CODE_SELFTEST, + "CORE selftest: AP%u lifecycle invalid P/S/Park/O/B=%u/%u/%u/%u/%u\n", + i, ap->Present, ap->Started, ap->Parked, ap->Online, ap->IsBsp); + if (!ap->SchedulerCommand || !ap->SchedulerActive) + KePanic(KE_PANIC_CODE_SELFTEST, + "CORE selftest: AP%u scheduler admission invalid command=%ld active=%ld\n", + i, ap->SchedulerCommand, ap->SchedulerActive); + if (!ap->KernelStack || !ap->KernelStackTop) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u missing kernel stack\n", i); + if (!ap->Tss.RSP0 || ap->Tss.IoMapBase != sizeof(TSS64)) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u invalid TSS rsp0=0x%llX iomap=%u\n", + i, ap->Tss.RSP0, ap->Tss.IoMapBase); + if (!ap->Gdtr.Base || ap->Gdtr.Base != (uint64_t)ap->Gdt) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u GDT not CPU-local\n", i); + if (!ap->IdleThread || !ap->CurrentThread || ap->CurrentThread != ap->IdleThread) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: AP%u idle/current thread missing\n", i); + if (!ap->IdleThread->IsIdleThread || ap->IdleThread->Queued || ap->IdleThread->RunningCpu != i) + KePanic(KE_PANIC_CODE_SELFTEST, + "CORE selftest: AP%u idle invariant failed queued=%u idle=%u runningCpu=%u\n", + i, ap->IdleThread->Queued, ap->IdleThread->IsIdleThread, ap->IdleThread->RunningCpu); + } + + THREAD* cur = KeGetCurrentThread(); + if (!cur || cur != cpu->CurrentThread) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: current thread mismatch\n"); + if (!cpu->IdleThread || cur != cpu->IdleThread) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: scheduler did not start on CPU0 idle thread\n"); + if (!cur->IsIdleThread || cur->Queued || cur->RunningCpu != cpu->CpuIndex) + KePanic(KE_PANIC_CODE_SELFTEST, "CORE selftest: idle invariant failed queued=%u idle=%u runningCpu=%u\n", + cur->Queued, cur->IsIdleThread, cur->RunningCpu); + + DbgPrint("CORE selftest: PASS\n"); +} + + +void KiRunEarlyKernelSelfTests(void) { + KSTATUS s = KeRunApplicationProcessorSchedulerSelfTests(); + if (!KSUCCESS(s)) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SMP scheduler selftest failed status=0x%x\n", + s); + } +} + void KiRunKernelSelfTests(const HAL_API* hal) { DbgPrint("Kernel selftests: begin\n"); KiRunKernelApcSelfTest(); diff --git a/code/kernel/ke_smp.c b/code/kernel/ke_smp.c new file mode 100644 index 0000000..eede9d5 --- /dev/null +++ b/code/kernel/ke_smp.c @@ -0,0 +1,660 @@ +// ke_smp.c - SMP application-processor startup and scheduler admission +// +// The bootloader is intentionally not involved in AP ownership. The kernel +// asks the HAL for MADT-discovered LAPIC IDs, copies a small low-memory x86 AP +// startup blob, sends INIT/SIPI through the local APIC, and waits for each AP +// to enter KiApKernelEntry() and park in kernel-owned state. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AP_TRAMPOLINE_MAX_PHYS 0x000A0000ULL +#define AP_TRAMPOLINE_BOOT_OFF 0x0800u +#define AP_STACK_PAGES 8u +#define AP_START_TIMEOUT_SPINS 20000000u +#define AP_SCHED_TIMEOUT_SPINS 20000000u + +#define AP_CR4_PAE 0x00000020u +#define AP_CR4_OSFXSR 0x00000200u +#define AP_CR4_OSXMMEXCPT 0x00000400u +#define AP_CR4_REQUIRED (AP_CR4_PAE | AP_CR4_OSFXSR | AP_CR4_OSXMMEXCPT) +#define AP_CR0_PE 0x00000001u +#define AP_CR0_MP 0x00000002u +#define AP_CR0_EM 0x00000004u +#define AP_CR0_TS 0x00000008u +#define AP_CR0_NE 0x00000020u +#define AP_CR0_PG 0x80000000u + +#define AP_BOOT_MAGIC 0x3436544F4F425041ULL /* 'APBOOT64' */ +#define AP_STATE_EMPTY 0u +#define AP_STATE_READY 1u +#define AP_STATE_LONG_MODE 2u +#define AP_STATE_KERNEL_ENTRY 3u +#define AP_STATE_KERNEL_PARKED 4u +#define AP_STATE_FAILED 0xFFFFFFFFu + +#define AP_STAGE_NONE 0x00000000u +#define AP_STAGE_LONG64 0x4C4D3634u /* 'LM64' */ +#define AP_STAGE_ENTRY 0x454E5452u /* 'ENTR' */ +#define AP_STAGE_GS 0x47534241u /* 'GSBA' */ +#define AP_STAGE_GDT 0x47445421u /* 'GDT!' */ +#define AP_STAGE_IDT 0x49445421u /* 'IDT!' */ +#define AP_STAGE_PARK 0x5041524Bu /* 'PARK' */ + +#pragma pack(push, 1) +typedef struct AP_BOOT_BLOCK { + uint64_t Magic; + uint64_t KernelCr3; + uint64_t LongModeEntry; + uint64_t Kcpu; + uint64_t StackTop; + volatile uint64_t Heartbeat; + volatile uint32_t State; + volatile uint32_t Stage; + volatile uint32_t ApIndex; +} AP_BOOT_BLOCK; +#pragma pack(pop) + +_Static_assert(offsetof(AP_BOOT_BLOCK, KernelCr3) == 8, "AP_BOOT_BLOCK.KernelCr3 offset"); +_Static_assert(offsetof(AP_BOOT_BLOCK, StackTop) == 32, "AP_BOOT_BLOCK.StackTop offset"); +_Static_assert(offsetof(AP_BOOT_BLOCK, State) == 48, "AP_BOOT_BLOCK.State offset"); +_Static_assert(offsetof(AP_BOOT_BLOCK, Stage) == 52, "AP_BOOT_BLOCK.Stage offset"); + +static PHYSICAL_ADDRESS gApTrampolinePhys; +static AP_BOOT_BLOCK* gApBootBlock; + +static void KiEmit8(uint8_t** p, uint8_t v) { *(*p)++ = v; } +static void KiEmit16(uint8_t** p, uint16_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); } +static void KiEmit32(uint8_t** p, uint32_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); } +static void KiEmit64(uint8_t** p, uint64_t v) { memcpy(*p, &v, sizeof(v)); *p += sizeof(v); } + +static void KiPatch16(uint8_t* base, size_t off, uint16_t v) { memcpy(base + off, &v, sizeof(v)); } +static void KiPatch32(uint8_t* base, size_t off, uint32_t v) { memcpy(base + off, &v, sizeof(v)); } +static void KiPatch64(uint8_t* base, size_t off, uint64_t v) { memcpy(base + off, &v, sizeof(v)); } + +static uint64_t KiGdtDescriptor(uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) { + return ((uint64_t)(limit & 0xFFFF)) | + ((uint64_t)(base & 0xFFFFFF) << 16) | + ((uint64_t)access << 40) | + ((uint64_t)((limit >> 16) & 0xF) << 48) | + ((uint64_t)(flags & 0xF) << 52) | + ((uint64_t)((base >> 24) & 0xFF) << 56); +} + +static KSTATUS KiBuildApTrampoline(void) { + if (gApTrampolinePhys) + return STATUS_SUCCESS; + + PHYSICAL_ADDRESS frame = MmAllocateFrameBelow(AP_TRAMPOLINE_MAX_PHYS); + if (!frame || frame < PAGE_SIZE || (frame & (PAGE_SIZE - 1))) + return STATUS_NO_MEMORY; + + uint8_t* page = (uint8_t*)(PHYS_MAP_BASE + frame); + memset(page, 0, PAGE_SIZE); + + uint32_t physBase = (uint32_t)frame; + uint8_t* p = page; + + // --- 16-bit real-mode SIPI entry --- + KiEmit8(&p, 0xFA); // cli + KiEmit8(&p, 0x8C); KiEmit8(&p, 0xC8); // mov ax, cs + KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD8); // mov ds, ax + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x01); KiEmit8(&p, 0x16); + size_t gdtr16Patch = (size_t)(p - page); KiEmit16(&p, 0); // lgdt [gdtr] + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x20); KiEmit8(&p, 0xC0); // mov eax, cr0 + KiEmit8(&p, 0x66); KiEmit8(&p, 0x83); KiEmit8(&p, 0xC8); KiEmit8(&p, 0x01); // or eax, 1 + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xC0); // mov cr0, eax + KiEmit8(&p, 0x66); KiEmit8(&p, 0xEA); // jmp far 32-bit offset + size_t prot32Patch = (size_t)(p - page); KiEmit32(&p, 0); + KiEmit16(&p, 0x0008); // selector 32-bit code + + while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90); + uint32_t prot32Off = (uint32_t)(p - page); + + // --- 32-bit protected-mode transition to long mode --- + KiEmit8(&p, 0x66); KiEmit8(&p, 0xB8); KiEmit16(&p, 0x0010); // mov ax, 0x10 + KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD8); // mov ds, ax + KiEmit8(&p, 0x8E); KiEmit8(&p, 0xC0); // mov es, ax + KiEmit8(&p, 0x8E); KiEmit8(&p, 0xD0); // mov ss, ax + KiEmit8(&p, 0x8E); KiEmit8(&p, 0xE0); // mov fs, ax + KiEmit8(&p, 0x8E); KiEmit8(&p, 0xE8); // mov gs, ax + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x20); KiEmit8(&p, 0xE0); // mov eax, cr4 + KiEmit8(&p, 0x0D); KiEmit32(&p, AP_CR4_REQUIRED); // or eax, PAE|OSFXSR|OSXMMEXCPT + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xE0); // mov cr4, eax + KiEmit8(&p, 0xB9); KiEmit32(&p, 0xC0000080u); // mov ecx, IA32_EFER + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x32); // rdmsr + KiEmit8(&p, 0x0D); KiEmit32(&p, 0x00000900u); // or eax, LME|NXE + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x30); // wrmsr + KiEmit8(&p, 0xA1); size_t cr3Patch = (size_t)(p - page); KiEmit32(&p, 0); // mov eax,[cr3] + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xD8); // mov cr3, eax + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x20); KiEmit8(&p, 0xC0); // mov eax, cr0 + KiEmit8(&p, 0x25); KiEmit32(&p, ~(AP_CR0_EM | AP_CR0_TS)); // and eax, ~(EM|TS) + KiEmit8(&p, 0x0D); KiEmit32(&p, AP_CR0_PG | AP_CR0_PE | AP_CR0_MP | AP_CR0_NE); // or eax, PG|PE|MP|NE + KiEmit8(&p, 0x0F); KiEmit8(&p, 0x22); KiEmit8(&p, 0xC0); // mov cr0, eax + KiEmit8(&p, 0xEA); size_t long64Patch = (size_t)(p - page); KiEmit32(&p, 0); + KiEmit16(&p, 0x0018); // selector 64-bit code + + while (((uintptr_t)(p - page)) & 0x0F) KiEmit8(&p, 0x90); + uint32_t long64Off = (uint32_t)(p - page); + + // --- 64-bit entry. Jump into high-half KiApKernelEntry(KCPU*). --- + KiEmit8(&p, 0x48); KiEmit8(&p, 0xBB); size_t bootBlockPatch = (size_t)(p - page); KiEmit64(&p, 0); // mov rbx, bootblock + KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Stage)); KiEmit32(&p, AP_STAGE_LONG64); + KiEmit8(&p, 0xC7); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, State)); KiEmit32(&p, AP_STATE_LONG_MODE); + KiEmit8(&p, 0x48); KiEmit8(&p, 0x8B); KiEmit8(&p, 0xA3); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, StackTop)); // mov rsp,[rbx+StackTop] + KiEmit8(&p, 0x48); KiEmit8(&p, 0x83); KiEmit8(&p, 0xE4); KiEmit8(&p, 0xF0); // and rsp,-16 + KiEmit8(&p, 0x48); KiEmit8(&p, 0x83); KiEmit8(&p, 0xEC); KiEmit8(&p, 0x28); // sub rsp,28h + KiEmit8(&p, 0x48); KiEmit8(&p, 0x8B); KiEmit8(&p, 0x8B); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, Kcpu)); // mov rcx,[rbx+Kcpu] + KiEmit8(&p, 0x48); KiEmit8(&p, 0x8B); KiEmit8(&p, 0x83); KiEmit32(&p, (uint32_t)offsetof(AP_BOOT_BLOCK, LongModeEntry)); // mov rax,[rbx+LongModeEntry] + KiEmit8(&p, 0xFF); KiEmit8(&p, 0xE0); // jmp rax + + while (((uintptr_t)(p - page)) & 0x07) KiEmit8(&p, 0); + uint32_t gdtOff = (uint32_t)(p - page); + uint64_t* gdt = (uint64_t*)p; + gdt[0] = 0; + gdt[1] = KiGdtDescriptor(0, 0xFFFFF, 0x9A, 0xC); // 32-bit code + gdt[2] = KiGdtDescriptor(0, 0xFFFFF, 0x92, 0xC); // 32-bit data + gdt[3] = KiGdtDescriptor(0, 0xFFFFF, 0x9A, 0xA); // 64-bit code + gdt[4] = KiGdtDescriptor(0, 0xFFFFF, 0x92, 0xC); // 64-bit data + p += 5 * sizeof(uint64_t); + + uint32_t gdtrOff = (uint32_t)(p - page); + KiEmit16(&p, (uint16_t)(5 * sizeof(uint64_t) - 1)); + KiEmit32(&p, physBase + gdtOff); + + if ((uintptr_t)(p - page) >= AP_TRAMPOLINE_BOOT_OFF) + return STATUS_DEVICE_ERROR; + + AP_BOOT_BLOCK* boot = (AP_BOOT_BLOCK*)(page + AP_TRAMPOLINE_BOOT_OFF); + memset(boot, 0, sizeof(*boot)); + boot->Magic = AP_BOOT_MAGIC; + + KiPatch16(page, gdtr16Patch, (uint16_t)gdtrOff); + KiPatch32(page, prot32Patch, physBase + prot32Off); + KiPatch32(page, cr3Patch, physBase + AP_TRAMPOLINE_BOOT_OFF + (uint32_t)offsetof(AP_BOOT_BLOCK, KernelCr3)); + KiPatch32(page, long64Patch, physBase + long64Off); + KiPatch64(page, bootBlockPatch, (uint64_t)(physBase + AP_TRAMPOLINE_BOOT_OFF)); + + gApTrampolinePhys = frame; + gApBootBlock = boot; + return STATUS_SUCCESS; +} + +_Noreturn void KiApKernelEntry(KCPU* cpu) { + _disable(); + + if (!cpu) { + while (1) __halt(); + } + + // APs start from the SIPI reset state, so do not assume the firmware/BSP + // CR4 feature bits are present. The scheduler and context-switch paths + // use FXSAVE/FXRSTOR on every CPU; without OSFXSR, those instructions + // raise #UD as soon as an AP enters its idle thread. + uintptr_t cr4 = __readcr4(); + if ((cr4 & AP_CR4_REQUIRED) != AP_CR4_REQUIRED) + __writecr4(cr4 | AP_CR4_REQUIRED); + + if (gApBootBlock) { + gApBootBlock->Stage = AP_STAGE_ENTRY; + gApBootBlock->State = AP_STATE_KERNEL_ENTRY; + gApBootBlock->Heartbeat++; + } + + cpu->Self = cpu; + KeLoadKernelGsBase(cpu); + KeSetUserGsBase(0); + if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_GS; + + HalApicEnableCurrentCpu(); + + KeInitializeGdtForCpu(cpu); + KeLoadGdtForCurrentCpu(cpu); + if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_GDT; + + KeLoadIdtForCurrentCpu(); + if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_IDT; + + cpu->Started = 1; + cpu->Parked = 1; + cpu->Online = 0; + cpu->CurrentIrql = PASSIVE_LEVEL; + cpu->InterruptDepth = 0; + cpu->ReschedulePending = 0; + + if (gApBootBlock) { + gApBootBlock->Stage = AP_STAGE_PARK; + gApBootBlock->State = AP_STATE_KERNEL_PARKED; + } + + for (;;) { + cpu->SchedulerTicks++; + + if (_InterlockedCompareExchange(&cpu->SchedulerCommand, 0, 0) != 0) { + THREAD* idle = cpu->IdleThread; + if (idle) { + cpu->CurrentThread = idle; + idle->State = ThreadRunning; + idle->RunningCpu = cpu->CpuIndex; + idle->Queued = 0; + cpu->Parked = 0; + _InterlockedExchange(&cpu->SchedulerActive, 1); + _enable(); + KeStartInitialThread(idle); + } + } + + _mm_pause(); + } +} + +static void KiPrepareCpuRecord(uint32_t cpuIndex, const HAL_CPU_INFO* info) { + KCPU* cpu = &gCpus[cpuIndex]; + memset(cpu, 0, sizeof(*cpu)); + cpu->Self = cpu; + cpu->CpuIndex = cpuIndex; + cpu->LapicId = info->LapicId; + cpu->AcpiProcessorId = info->ProcessorUid; + cpu->BootProcessorIndex = cpuIndex; + cpu->Present = info->Enabled ? 1 : 0; + cpu->Started = 0; + cpu->Parked = 0; + cpu->Online = 0; + cpu->IsBsp = info->IsBsp ? 1 : 0; + cpu->CurrentIrql = PASSIVE_LEVEL; + cpu->InterruptDepth = 0; + cpu->ReschedulePending = 0; + cpu->SchedulerCommand = 0; + cpu->SchedulerActive = 0; +} + +static KSTATUS KiAllocateApStack(KCPU* cpu) { + PHYSICAL_ADDRESS stackPhys = MmAllocateContiguous(AP_STACK_PAGES); + if (!stackPhys) return STATUS_NO_MEMORY; + + cpu->KernelStack = (void*)(PHYS_MAP_BASE + stackPhys); + cpu->KernelStackTop = (uintptr_t)cpu->KernelStack + AP_STACK_PAGES * PAGE_SIZE; + memset(cpu->KernelStack, 0, AP_STACK_PAGES * PAGE_SIZE); + memset(&cpu->Tss, 0, sizeof(cpu->Tss)); + cpu->Tss.IoMapBase = sizeof(TSS64); + cpu->Tss.RSP0 = cpu->KernelStackTop; + KeInitializeGdtForCpu(cpu); + return STATUS_SUCCESS; +} + +static void KiFillBootBlockForCpu(KCPU* cpu, uint32_t apIndex) { + memset(gApBootBlock, 0, sizeof(*gApBootBlock)); + gApBootBlock->Magic = AP_BOOT_MAGIC; + gApBootBlock->KernelCr3 = (uint64_t)__readcr3(); + gApBootBlock->LongModeEntry = (uint64_t)(uintptr_t)KiApKernelEntry; + gApBootBlock->Kcpu = (uint64_t)(uintptr_t)cpu; + gApBootBlock->StackTop = cpu->KernelStackTop; + gApBootBlock->State = AP_STATE_READY; + gApBootBlock->Stage = AP_STAGE_NONE; + gApBootBlock->ApIndex = apIndex; +} + +KSTATUS KeStartApplicationProcessors(void) { + HAL_CPU_TOPOLOGY topo; + KSTATUS st = HalGetProcessorTopology(&topo); + if (!KSUCCESS(st)) return st; + + if (topo.CpuCount == 0 || topo.CpuCount > KE_MAX_CPUS) + return STATUS_DEVICE_ERROR; + + uint32_t bspLapic = HalGetBspLapicId(); + KCPU* bsp = KeGetCurrentCpu(); + bsp->LapicId = bspLapic; + bsp->IsBsp = 1; + bsp->Present = 1; + bsp->Started = 1; + bsp->Online = 1; + bsp->Parked = 0; + bsp->SchedulerCommand = 0; + bsp->SchedulerActive = 1; + + uint32_t count = 1; + for (uint32_t i = 0; i < topo.CpuCount && count < KE_MAX_CPUS; i++) { + HAL_CPU_INFO* info = &topo.Cpus[i]; + if (!info->Enabled) continue; + if (info->LapicId == bspLapic) { + bsp->AcpiProcessorId = info->ProcessorUid; + continue; + } + KiPrepareCpuRecord(count, info); + count++; + } + gCpuCount = count; + + DbgPrint("SMP: discovered %u CPU(s), BSP LAPIC=%u\n", gCpuCount, bspLapic); + if (gCpuCount <= 1) { + DbgPrint("SMP: no application processors present\n"); + return STATUS_SUCCESS; + } + + if ((__readcr3() >> 32) != 0) + return STATUS_NOT_SUPPORTED; + + st = KiBuildApTrampoline(); + if (!KSUCCESS(st)) return st; + + uint8_t sipiVector = (uint8_t)(gApTrampolinePhys >> 12); + uint32_t claimed = 0; + + for (uint32_t i = 1; i < gCpuCount; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present) continue; + + st = KiAllocateApStack(cpu); + if (!KSUCCESS(st)) return st; + + KiFillBootBlockForCpu(cpu, i); + DbgPrint("SMP: starting AP%u LAPIC=%u vector=0x%x\n", i, cpu->LapicId, sipiVector); + + st = HalStartApplicationProcessor(cpu->LapicId, sipiVector); + if (!KSUCCESS(st)) return st; + + uint64_t lastHeartbeat = gApBootBlock->Heartbeat; + int parked = 0; + for (uint32_t spin = 0; spin < AP_START_TIMEOUT_SPINS; spin++) { + if (gApBootBlock->State == AP_STATE_KERNEL_PARKED) { + parked = 1; + break; + } + if ((spin & 0xFFFFu) == 0) { + uint64_t hb = gApBootBlock->Heartbeat; + lastHeartbeat = hb; + } + _mm_pause(); + } + + if (!parked) { + KePanic(KE_PANIC_CODE_INIT, + "SMP: AP%u failed to reach kernel park state lapic=%u state=%u stage=0x%x heartbeat=%llu last=%llu\n", + i, cpu->LapicId, gApBootBlock->State, gApBootBlock->Stage, + gApBootBlock->Heartbeat, lastHeartbeat); + } + + cpu->Started = 1; + cpu->Parked = 1; + cpu->Online = 0; + claimed++; + DbgPrint("SMP: AP%u reached kernel park\n", i); + } + + DbgPrint("SMP: kernel claimed %u APs\n", claimed); + return STATUS_SUCCESS; +} + +KSTATUS KeAdmitApplicationProcessorsToScheduler(void) { + if (gCpuCount <= 1) { + DbgPrint("SMP: no AP scheduler admission needed\n"); + return STATUS_SUCCESS; + } + + uint32_t admitted = 0; + for (uint32_t i = 1; i < gCpuCount; i++) { + KCPU* cpu = &gCpus[i]; + if (!cpu->Present) continue; + + if (!cpu->Started || !cpu->Parked || cpu->Online) { + KePanic(KE_PANIC_CODE_INIT, + "SMP: AP%u cannot enter scheduler P/S/Park/O=%u/%u/%u/%u\n", + i, cpu->Present, cpu->Started, cpu->Parked, cpu->Online); + } + + KSTATUS st = KiCreateIdleThreadForCpu(cpu); + if (!KSUCCESS(st)) + return st; + + cpu->Online = 1; + _ReadWriteBarrier(); + _InterlockedExchange(&cpu->SchedulerCommand, 1); + + int active = 0; + for (uint32_t spin = 0; spin < AP_SCHED_TIMEOUT_SPINS; spin++) { + if (_InterlockedCompareExchange(&cpu->SchedulerActive, 0, 0) != 0) { + active = 1; + break; + } + _mm_pause(); + } + + if (!active) { + KePanic(KE_PANIC_CODE_INIT, + "SMP: AP%u failed scheduler admission lapic=%u parked=%u online=%u command=%ld active=%ld ticks=%llu\n", + i, cpu->LapicId, cpu->Parked, cpu->Online, + cpu->SchedulerCommand, cpu->SchedulerActive, cpu->SchedulerTicks); + } + + admitted++; + DbgPrint("SMP: AP%u entered scheduler idle thread\n", i); + } + + DbgPrint("SMP: scheduler admitted %u APs\n", admitted); + return STATUS_SUCCESS; +} + +#define SMP_AP_WORKER_ROUNDS 64u +#define SMP_AP_WORKER_TIMEOUT_SPINS 4000000u +#define SMP_AP_TEST_MAGIC 0x524B575041504D53ULL /* 'SMPAPWKR' */ + +typedef struct SMP_AP_WORK_TEST SMP_AP_WORK_TEST; + +typedef struct SMP_AP_WORKER_ARG { + uint64_t Magic; + uint32_t CpuIndex; + uint32_t WorkerIndex; + struct SMP_AP_WORK_TEST* Test; + EVENT* StepEvent; + volatile long Ready; + volatile long WaitArmed; + volatile long Progress; + volatile long Done; + volatile long Failure; + uint32_t WrongCpu; + uint32_t WrongRunningCpu; +} SMP_AP_WORKER_ARG; + +struct SMP_AP_WORK_TEST { + uint64_t Magic; + uint32_t WorkerCount; + volatile long ReadyCount; + volatile long DoneCount; + volatile long FailureCount; + volatile long CpuHits[KE_MAX_CPUS]; + SMP_AP_WORKER_ARG Workers[KE_MAX_CPUS]; +}; + +static void KiSmpApControlledWorker(void* context) { + SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context; + if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) { + KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%llX\n", (uint64_t)arg); + } + + _InterlockedExchange(&arg->Ready, 1); + _InterlockedIncrement(&arg->Test->ReadyCount); + + for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) { + _InterlockedIncrement(&arg->WaitArmed); + KSTATUS waitStatus = KeWaitForEvent(arg->StepEvent, 0); + if (!KSUCCESS(waitStatus)) { + _InterlockedExchange(&arg->Failure, 1); + break; + } + + KCPU* cpu = KeGetCurrentCpu(); + THREAD* self = KeGetCurrentThread(); + if (!cpu || cpu->CpuIndex != arg->CpuIndex) { + arg->WrongCpu = cpu ? cpu->CpuIndex : KE_INVALID_CPU; + _InterlockedExchange(&arg->Failure, 1); + break; + } + if (!self || self->RunningCpu != cpu->CpuIndex) { + arg->WrongRunningCpu = self ? self->RunningCpu : KE_INVALID_CPU; + _InterlockedExchange(&arg->Failure, 1); + break; + } + if (self && (self->AffinityMask & (1ULL << cpu->CpuIndex)) == 0) { + arg->WrongCpu = cpu->CpuIndex; + _InterlockedExchange(&arg->Failure, 1); + break; + } + + if (cpu->CpuIndex < KE_MAX_CPUS) + _InterlockedIncrement(&arg->Test->CpuHits[cpu->CpuIndex]); + + _InterlockedIncrement(&arg->Progress); + if ((round & 3u) == 0) + KeYield(); + } + + if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) { + _InterlockedIncrement(&arg->Test->FailureCount); + } + + _InterlockedExchange(&arg->Done, 1); + _InterlockedIncrement(&arg->Test->DoneCount); +} + +static int KiWaitForWorkerCounter(volatile long* counter, long expected, const char* what) { + for (uint32_t spin = 0; spin < SMP_AP_WORKER_TIMEOUT_SPINS; spin++) { + if (_InterlockedCompareExchange(counter, 0, 0) >= expected) + return 1; + if ((spin & 0x3FFu) == 0) + KeYield(); + else + _mm_pause(); + } + DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%ld got=%ld\n", + what ? what : "counter", expected, _InterlockedCompareExchange(counter, 0, 0)); + return 0; +} + +KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) { + if (gCpuCount <= 1) { + DbgPrint("SMP scheduler selftest: SKIP, no APs\n"); + return STATUS_SUCCESS; + } + + DbgPrint("SMP scheduler selftest: begin AP-affine workers\n"); + + SMP_AP_WORK_TEST* test = (SMP_AP_WORK_TEST*)MmAllocatePool(sizeof(SMP_AP_WORK_TEST)); + if (!test) return STATUS_NO_MEMORY; + memset(test, 0, sizeof(*test)); + test->Magic = SMP_AP_TEST_MAGIC; + + uint32_t workerCount = 0; + for (uint32_t cpuIndex = 1; cpuIndex < gCpuCount; cpuIndex++) { + KCPU* cpu = &gCpus[cpuIndex]; + if (!cpu->Present || !cpu->Started || !cpu->Online || cpu->Parked) + continue; + if (cpuIndex >= 64) + continue; + + SMP_AP_WORKER_ARG* arg = &test->Workers[workerCount]; + arg->Magic = SMP_AP_TEST_MAGIC; + arg->CpuIndex = cpuIndex; + arg->WorkerIndex = workerCount; + arg->Test = test; + + KSTATUS st = KeEventCreate(EventAutoReset, 0, &arg->StepEvent); + if (!KSUCCESS(st)) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SMP scheduler selftest: AP%u step event create failed status=0x%x\n", + cpuIndex, st); + } + + THREAD* thread = (THREAD*)0; + st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker, + (uintptr_t)arg, + PriorityNormal, + (1ULL << cpuIndex), + &thread); + if (!KSUCCESS(st)) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SMP scheduler selftest: AP%u worker create failed status=0x%x\n", + cpuIndex, st); + } + + workerCount++; + } + test->WorkerCount = workerCount; + + if (workerCount == 0) { + MmFreePool(test); + DbgPrint("SMP scheduler selftest: SKIP, no online AP workers\n"); + return STATUS_SUCCESS; + } + + if (!KiWaitForWorkerCounter(&test->ReadyCount, (long)workerCount, "AP worker ready")) { + KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers failed to become ready ready=%ld workers=%u\n", + test->ReadyCount, workerCount); + } + + for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) { + for (uint32_t w = 0; w < workerCount; w++) { + SMP_AP_WORKER_ARG* arg = &test->Workers[w]; + if (!KiWaitForWorkerCounter(&arg->WaitArmed, (long)(round + 1), "AP worker wait-armed")) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SMP scheduler selftest: AP%u did not arm wait for round %u armed=%ld progress=%ld\n", + arg->CpuIndex, round, arg->WaitArmed, arg->Progress); + } + KeSignalEvent(arg->StepEvent); + } + + for (uint32_t w = 0; w < workerCount; w++) { + SMP_AP_WORKER_ARG* arg = &test->Workers[w]; + if (!KiWaitForWorkerCounter(&arg->Progress, (long)(round + 1), "AP worker progress")) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SMP scheduler selftest: AP%u no progress round=%u progress=%ld failure=%ld wrongCpu=%u wrongRunningCpu=%u\n", + arg->CpuIndex, round, arg->Progress, arg->Failure, + arg->WrongCpu, arg->WrongRunningCpu); + } + if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SMP scheduler selftest: AP%u worker failed round=%u wrongCpu=%u wrongRunningCpu=%u progress=%ld\n", + arg->CpuIndex, round, arg->WrongCpu, arg->WrongRunningCpu, arg->Progress); + } + } + } + + if (!KiWaitForWorkerCounter(&test->DoneCount, (long)workerCount, "AP worker done")) { + KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%ld workers=%u\n", + test->DoneCount, workerCount); + } + + if (_InterlockedCompareExchange(&test->FailureCount, 0, 0) != 0) { + KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%ld\n", + test->FailureCount); + } + + for (uint32_t w = 0; w < workerCount; w++) { + SMP_AP_WORKER_ARG* arg = &test->Workers[w]; + long hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0; + if (hits <= 0) { + KePanic(KE_PANIC_CODE_SELFTEST, + "SMP scheduler selftest: AP%u recorded no CPU hits\n", arg->CpuIndex); + } + if (arg->StepEvent) + ObDereferenceObject(&arg->StepEvent->Header); + } + + DbgPrint("SMP scheduler selftest: PASS workers=%u rounds=%u\n", + workerCount, SMP_AP_WORKER_ROUNDS); + MmFreePool(test); + return STATUS_SUCCESS; +} diff --git a/code/kernel/ke_switch.asm b/code/kernel/ke_switch.asm index 5e1396a..3e2a60b 100644 --- a/code/kernel/ke_switch.asm +++ b/code/kernel/ke_switch.asm @@ -14,7 +14,7 @@ THREAD_Teb EQU 112 THREAD_FpuContext EQU 168 TSS_RSP0_OFFSET EQU 4 -EXTERN gTss:QWORD ; phase-1 pointer to current CPU TSS, exported from ke_idt.c +EXTERN KeSetCurrentTssRsp0 : PROC .CODE @@ -37,12 +37,24 @@ KeSwitchContext PROC ; from->SavedContext = rsp mov [rcx + THREAD_SavedContext], rsp + ; Update this CPU's TSS.RSP0 before switching to the incoming stack. + ; The helper hides whether current-CPU storage is still CPU0-only, LAPIC-ID + ; indexed, or eventually reached through a SWAPGS-backed per-CPU base. + mov rbx, rdx + mov rcx, [rbx + THREAD_TssRsp0] + sub rsp, 40 + call KeSetCurrentTssRsp0 + add rsp, 40 + mov rdx, rbx + ; rsp = to->SavedContext mov rsp, [rdx + THREAD_SavedContext] - ; Update MSR_GS_BASE (0xC0000101) with to->Teb - mov rax, [rdx + THREAD_Teb] ; Fetch the incoming TEB pointer - mov ecx, 0C0000101h ; IA32_GS_BASE MSR + ; While executing in ring 0, GS_BASE is this CPU's KCPU. Park the + ; incoming thread's user TEB in IA32_KERNEL_GS_BASE; user-mode entry/exit + ; swaps it with the kernel GS base using swapgs. + mov rax, [rdx + THREAD_Teb] ; Fetch the incoming user GS/TEB pointer + mov ecx, 0C0000102h ; IA32_KERNEL_GS_BASE MSR mov r8, rdx ; Save RDX ('to' thread) as WRMSR clobbers EDX mov rdx, rax ; Copy TEB pointer to RDX shr rdx, 32 ; EDX = High 32 bits of TEB @@ -50,12 +62,6 @@ KeSwitchContext PROC wrmsr ; Write to MSR mov rdx, r8 ; Restore RDX - ; Update TSS.RSP0 so that the next interrupt/syscall uses the - ; incoming thread's kernel stack top. - mov rax, [rdx + THREAD_TssRsp0] - mov r8, gTss - mov qword ptr [r8 + TSS_RSP0_OFFSET], rax - mov rax, [rdx + THREAD_FpuContext] fxrstor64 [rax] @@ -69,6 +75,47 @@ KeSwitchContext PROC ret KeSwitchContext ENDP + +; ------------------------------------------------------------------------- +; void KeStartInitialThread(THREAD* to) +; RCX = to +; +; Used by AP scheduler admission. The AP is already in kernel long mode on a +; AP bootstrap stack; this helper installs the idle thread's stack/TSS +; state and returns directly into that thread's saved context. There is no +; outgoing THREAD and no scheduler lock to release. +; ------------------------------------------------------------------------- +KeStartInitialThread PROC + mov rbx, rcx + + ; Update this CPU's TSS.RSP0 for the incoming thread. + mov rcx, [rbx + THREAD_TssRsp0] + sub rsp, 40 + call KeSetCurrentTssRsp0 + add rsp, 40 + + ; Park the incoming thread's user TEB in IA32_KERNEL_GS_BASE. Idle/kernel + ; threads have a NULL TEB, which is fine. + mov rax, [rbx + THREAD_Teb] + mov ecx, 0C0000102h + mov rdx, rax + shr rdx, 32 + wrmsr + + ; Load the thread's saved kernel context and jump into it. + mov rsp, [rbx + THREAD_SavedContext] + mov rax, [rbx + THREAD_FpuContext] + fxrstor64 [rax] + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + ret +KeStartInitialThread ENDP + ; ------------------------------------------------------------------------- ; KeThreadEntry - landing pad for brand-new threads. ; @@ -93,6 +140,7 @@ KeThreadEntry PROC call KeSchedulerPostSwitch add rsp, 32 pop rcx ; load user arg + swapgs ; GS_BASE: KCPU -> user TEB, KERNEL_GS_BASE: user TEB -> KCPU iretq ; -> ring 3 KeThreadEntry ENDP diff --git a/code/kernel/ke_syscall.asm b/code/kernel/ke_syscall.asm index 03f3f0c..54b5e37 100644 --- a/code/kernel/ke_syscall.asm +++ b/code/kernel/ke_syscall.asm @@ -2,7 +2,7 @@ ; ; Syscall entry now builds an editable KTRAP_FRAME on the current kernel ; stack. KiExitTrapFrame commits Rip/Cs/Rflags/Rsp/Ss back to the CPU IRET -; slots, which is the foundation for future user APC return redirection. +; slots used by user APC return redirection. EXTERN KeSyscallDispatch:PROC EXTERN KiEnterTrapFrame:PROC @@ -20,7 +20,6 @@ TF_Magic EQU 0 TF_Kind EQU 8 TF_Vector EQU 12 TF_PreviousMode EQU 16 -TF_Reserved EQU 20 TF_Rax EQU 24 TF_Rcx EQU 32 TF_Rdx EQU 40 @@ -50,6 +49,7 @@ TF_SsSlot EQU 216 .CODE KeSyscallEntry PROC + swapgs ; GS_BASE: user TEB -> KCPU, KERNEL_GS_BASE: KCPU -> user TEB ; CPU frame at entry, ring 3 -> ring 0: ; [rsp+00] RIP, [rsp+08] CS, [rsp+10] RFLAGS, ; [rsp+18] user RSP, [rsp+20] SS @@ -65,7 +65,6 @@ KeSyscallEntry PROC mov dword ptr [rbx + TF_Kind], KTRAP_KIND_SYSCALL mov dword ptr [rbx + TF_Vector], 40h mov dword ptr [rbx + TF_PreviousMode], KTRAP_MODE_USER - mov dword ptr [rbx + TF_Reserved], 0 mov [rbx + TF_Rcx], rcx mov [rbx + TF_Rdx], rdx @@ -140,6 +139,7 @@ KeSyscallEntry PROC mov rax, [rbx + TF_Rax] add rsp, KTRAP_FRAME_SIZE pop rbx + swapgs ; GS_BASE: KCPU -> user TEB before returning to ring 3 iretq KeSyscallEntry ENDP @@ -235,6 +235,7 @@ KeFaultDE PROC mov [rax + KF_Rsp], r10 mov r10, [rdx+32] mov [rax + KF_Ss], r10 + swapgs jmp de_call de_kernel_frame: @@ -299,6 +300,7 @@ KeFaultGP PROC mov [rax + KF_Rsp], r10 mov r10, [rdx+40] mov [rax + KF_Ss], r10 + swapgs jmp gp_call gp_kernel_frame: @@ -364,6 +366,7 @@ KeFaultPF PROC mov [rax + KF_Rsp], r10 mov r10, [rdx+40] mov [rax + KF_Ss], r10 + swapgs jmp pf_call pf_kernel_frame: @@ -431,6 +434,7 @@ StubName PROC mov [rax + KF_Rsp], r10 mov r10, [rdx+32] mov [rax + KF_Ss], r10 + swapgs jmp noerr_call noerr_kernel_frame: @@ -494,6 +498,7 @@ StubName PROC mov [rax + KF_Rsp], r10 mov r10, [rdx+40] mov [rax + KF_Ss], r10 + swapgs jmp err_call err_kernel_frame: @@ -508,6 +513,48 @@ err_call: StubName ENDP ENDM +EXTERN KeHandleNmiIpi:PROC +PUBLIC KeNmiIpiStub +KeNmiIpiStub PROC + ; Returning NMI path for SMP control IPIs. If KeHandleNmiIpi returns 1, + ; the NMI was a panic/TLB-control event and is fully handled. If it + ; returns 0, restore the interrupted context and fall through to the normal + ; #NMI fault stub so unexpected NMIs still panic with a useful frame. + push rax + push rcx + push rdx + push r8 + push r9 + push r10 + push r11 + + sub rsp, 32 + call KeHandleNmiIpi + add rsp, 32 + + test eax, eax + jz short NmiUnhandled + + pop r11 + pop r10 + pop r9 + pop r8 + pop rdx + pop rcx + pop rax + iretq + +NmiUnhandled: + pop r11 + pop r10 + pop r9 + pop r8 + pop rdx + pop rcx + pop rax + jmp KeFaultNMI +KeNmiIpiStub ENDP + FAULT_NOERR KeFaultDB, 1 FAULT_NOERR KeFaultNMI, 2 FAULT_NOERR KeFaultBP, 3 @@ -576,6 +623,13 @@ IrqStub 21 IrqStub 22 IrqStub 23 +PUBLIC KeRescheduleIpiStub +KeRescheduleIpiStub PROC + push rax + mov al, 0F0h + jmp KeIrqCommon +KeRescheduleIpiStub ENDP + KeIrqCommon PROC ; al = irq number, original rax saved by the per-IRQ stub. push rcx @@ -594,7 +648,6 @@ KeIrqCommon PROC mov [rbx + TF_Magic], rax mov dword ptr [rbx + TF_Kind], KTRAP_KIND_INTERRUPT mov [rbx + TF_Vector], edx - mov dword ptr [rbx + TF_Reserved], 0 ; Saved volatile registers live immediately above the trap frame. mov rax, [rbx + KTRAP_FRAME_SIZE + 56] @@ -651,6 +704,7 @@ KeIrqCommon PROC mov [rbx + TF_SsSlot], rax mov rdx, [rax] mov [rbx + TF_Ss], rdx + swapgs ; interrupted ring 3: install kernel GS/KCPU jmp short IrqFrameReady IrqKernelModeFrame: @@ -672,6 +726,11 @@ IrqFrameReady: call KiExitTrapFrame add rsp, 40 + mov eax, [rbx + TF_PreviousMode] + cmp eax, KTRAP_MODE_USER + jne short IrqNoSwapGsOut + swapgs ; returning to ring 3: restore user GS/TEB +IrqNoSwapGsOut: add rsp, KTRAP_FRAME_SIZE pop rbx pop r11 diff --git a/code/kernel/ke_syscall.c b/code/kernel/ke_syscall.c index 96eded6..9d10063 100644 --- a/code/kernel/ke_syscall.c +++ b/code/kernel/ke_syscall.c @@ -69,7 +69,6 @@ typedef struct SYS_ASYNC_FILE_REQUEST_K { uint32_t FileHandle; uint32_t EventHandle; uint32_t Flags; - uint32_t Reserved; uint64_t Offset; void* Buffer; size_t Size; @@ -805,7 +804,7 @@ static uint64_t SyscallAsyncFileCommon(uint64_t a0, bool write) { KSTATUS s = CopyFromUser(&req, (const void*)(uintptr_t)a0, sizeof(req)); if (!KSUCCESS(s)) return s; - if (req.Flags || req.Reserved) return STATUS_INVALID_PARAMETER; + if (req.Flags) return STATUS_INVALID_PARAMETER; if (!req.Iosb) return STATUS_INVALID_PARAMETER; if (!req.Buffer && req.Size) return STATUS_INVALID_PARAMETER; if (req.Size > SYSCALL_IO_MAX_BYTES) return STATUS_INVALID_PARAMETER; @@ -1989,7 +1988,6 @@ typedef struct SYS_SOCKET_SEND_TO_REQUEST_K { const void* Buffer; size_t Length; uint32_t Flags; - uint32_t Reserved; const SYS_SOCKET_ADDRESS_IPV4_K* RemoteAddress; size_t RemoteAddressSize; size_t* SentOut; @@ -1999,7 +1997,6 @@ typedef struct SYS_SOCKET_RECV_FROM_REQUEST_K { void* Buffer; size_t Length; uint32_t Flags; - uint32_t Reserved; SYS_SOCKET_ADDRESS_IPV4_K* RemoteAddressOut; size_t* RemoteAddressSizeInOut; size_t* ReceivedOut; @@ -2015,7 +2012,6 @@ typedef struct SYS_RESOLVE_HOST_REQUEST_K { typedef struct SYS_SOCKET_OPTION_REQUEST_K { uint32_t Option; - uint32_t Reserved; void* Value; size_t ValueSize; size_t* ActualSizeOut; @@ -2269,7 +2265,6 @@ static uint64_t SyscallSendTo(uint64_t a0, uint64_t a1, uint64_t a2, SYS_SOCKET_SEND_TO_REQUEST_K req; KSTATUS s = CopyFromUser(&req, (const void*)(uintptr_t)a1, sizeof(req)); if (!KSUCCESS(s)) return s; - if (req.Reserved != 0) return STATUS_INVALID_PARAMETER; if (req.Length && !req.Buffer) return STATUS_INVALID_PARAMETER; if (!req.RemoteAddress) return STATUS_INVALID_PARAMETER; if (req.RemoteAddressSize < sizeof(SYS_SOCKET_ADDRESS_IPV4_K)) return STATUS_BUFFER_TOO_SMALL; @@ -2313,7 +2308,6 @@ static uint64_t SyscallRecvFrom(uint64_t a0, uint64_t a1, uint64_t a2, SYS_SOCKET_RECV_FROM_REQUEST_K req; KSTATUS s = CopyFromUser(&req, (const void*)(uintptr_t)a1, sizeof(req)); if (!KSUCCESS(s)) return s; - if (req.Reserved != 0) return STATUS_INVALID_PARAMETER; if (req.Length && !req.Buffer) return STATUS_INVALID_PARAMETER; if (req.Length > SYSCALL_IO_MAX_BYTES) return STATUS_INVALID_PARAMETER; diff --git a/code/kernel/ke_thread.c b/code/kernel/ke_thread.c index a33c4f5..70a3eb2 100644 --- a/code/kernel/ke_thread.c +++ b/code/kernel/ke_thread.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,7 +17,7 @@ #define FPU_MXCSR_DEFAULT 0x1F80u // These offsets are consumed directly by ke_switch.asm. Keep the asserts -// here so future THREAD layout edits fail at build time instead of corrupting +// here so THREAD layout edits fail at build time instead of corrupting // context switches at runtime. #define STATIC_ASSERT_OFFSET(type, field, expected) \ _Static_assert(offsetof(type, field) == (expected), \ @@ -93,11 +94,18 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui thread->CpuBurstTicks = 0; thread->LastRunTicks = 0; thread->RunningCpu = KE_INVALID_CPU; + thread->Queued = 0; + thread->IsIdleThread = 0; + thread->SchedulerFlags = 0; + thread->LastCpu = KE_INVALID_CPU; + thread->AffinityMask = KeGetDefaultThreadAffinityMask(); + LIST_INIT(&thread->ProcessLink); + LIST_INIT(&thread->SchedulerLink); thread->WaitObject = (OBJECT *)0; thread->WaitStatus = STATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SchedLockFlags = 0x202ULL; + thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); thread->TrapFrame = (KTRAP_FRAME*)0; @@ -226,8 +234,10 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code) { // --------------------------------------------------------------------------- extern void KeKernelThreadEntry(void); -__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, - THREAD_PRIORITY priority, THREAD** out) { +static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, + THREAD_PRIORITY priority, + uint64_t affinityMask, + THREAD** out) { // Kernel worker threads are owned by the system process. The scheduler's // hand-built CPU idle thread stays attached to PID 0 (idle), while driver, // async I/O, and init worker threads are grouped under PID 1 (system). @@ -252,11 +262,18 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, thread->CpuBurstTicks = 0; thread->LastRunTicks = 0; thread->RunningCpu = KE_INVALID_CPU; + thread->Queued = 0; + thread->IsIdleThread = 0; + thread->SchedulerFlags = 0; + thread->LastCpu = KE_INVALID_CPU; + thread->AffinityMask = affinityMask ? affinityMask : KeGetDefaultThreadAffinityMask(); + LIST_INIT(&thread->ProcessLink); + LIST_INIT(&thread->SchedulerLink); thread->WaitObject = (OBJECT*)0; thread->WaitStatus = STATUS_SUCCESS; thread->WaitListHead = (LIST_ENTRY*)0; thread->AlertableWait = 0; - thread->SchedLockFlags = 0x202ULL; + thread->SwitchLockCookie = 0x202ULL; KiInitializeApcState(thread); thread->TrapFrame = (KTRAP_FRAME*)0; thread->UserStackBase = (void*)0; // no user stack @@ -300,14 +317,26 @@ __declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, return STATUS_SUCCESS; } +__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg, + THREAD_PRIORITY priority, THREAD** out) { + return KiCreateKernelThreadWithAffinity(fn, arg, priority, + KeGetDefaultThreadAffinityMask(), out); +} + +__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg, + THREAD_PRIORITY priority, uint64_t affinityMask, + THREAD** out) { + return KiCreateKernelThreadWithAffinity(fn, arg, priority, affinityMask, out); +} + void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) { // Called with the synchronization object's spinlock held, so the caller is // normally at DISPATCH_LEVEL here. This function only marks intent to // block; the actual scheduling point is KeCommitBlockThread at PASSIVE. KeAssertIrqlAtOrBelow(DISPATCH_LEVEL); if (!thread) return; + KiSchedulerMarkThreadWaiting(thread); - thread->State = ThreadWaiting; thread->WaitObject = waitObject; thread->WaitStatus = STATUS_PENDING; thread->WaitListHead = (LIST_ENTRY*)0; @@ -375,40 +404,7 @@ KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable) { void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) { if (!thread) return; - - // If the selected waiter is still the currently executing thread, the - // signal landed in the prepare->commit window. Complete the wait in-place - // instead of inserting the running thread into the ready queue. - if (thread == KeGetCurrentThread()) { - KiFinishWait(thread, status); - thread->CpuBurstTicks = 0; - thread->QuantaConsumed = 0; - thread->QuantumRemaining = 0; - if (thread->State == ThreadWaiting) - thread->State = ThreadRunning; - return; - } - - // The same object can be signaled after a timeout/cancel path exists, or a - // buggy caller can try to wake a non-waiting thread. Do not enqueue such a - // thread twice. - if (thread->State != ThreadWaiting) { - thread->WaitStatus = status; - return; - } - - KiFinishWait(thread, status); - thread->State = ThreadReady; - - thread->CpuBurstTicks = 0; - thread->QuantaConsumed = 0; - thread->QuantumRemaining = 0; - - if (thread->Priority < PriorityHigh) { - thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1); - } - - KeReadyThread(thread); + KiSchedulerWakeThread(thread, status); } void KeUnblockThread(THREAD* thread) { diff --git a/code/kernel/ke_trap.c b/code/kernel/ke_trap.c index ed8612b..a429bac 100644 --- a/code/kernel/ke_trap.c +++ b/code/kernel/ke_trap.c @@ -41,7 +41,6 @@ STATIC_ASSERT_OFFSET(KTRAP_FRAME, RflagsSlot, 200); STATIC_ASSERT_OFFSET(KTRAP_FRAME, RspSlot, 208); STATIC_ASSERT_OFFSET(KTRAP_FRAME, SsSlot, 216); STATIC_ASSERT_OFFSET(KTRAP_FRAME, PreviousFrame, 224); -STATIC_ASSERT_OFFSET(KTRAP_FRAME, Reserved2, 232); _Static_assert(sizeof(KTRAP_FRAME) == 240, "KTRAP_FRAME size must match ke_syscall.asm"); int KeTrapFrameFromUser(const KTRAP_FRAME* frame) { diff --git a/code/kernel/kmain.c b/code/kernel/kmain.c index 1965375..199a5de 100644 --- a/code/kernel/kmain.c +++ b/code/kernel/kmain.c @@ -21,6 +21,7 @@ #include #include #include +#include #include typedef struct { @@ -380,6 +381,8 @@ void SystemInitThread(void* arg) { DbgPrint("SystemInitThread: Thread started. Igniting the system...\n"); + KiRunEarlyKernelSelfTests(); + KiInitializeDrivers(args->bootInfo, args->hal); KiResolveSystemImports(args->bootInfo); @@ -436,7 +439,7 @@ extern void KernelMain(BOOT_INFO* bootInfo) { } PHYSICAL_ADDRESS pml4 = (PHYSICAL_ADDRESS)__readcr3(); - MmInitializeVirtual(pml4, 4 * 1024 * 1024 /* 4 MB initial pool */); + MmInitializeVirtual(pml4, 1 * 1024 * 1024 /* 1 MB initial pool */); // // Bootstrap CPU and architecture setup (GDT/IDT/Faults) @@ -472,7 +475,17 @@ extern void KernelMain(BOOT_INFO* bootInfo) { const HAL_API* hal = HalGetApi(); gKmainHal = hal; - KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1); // Enable debug + KeInitDebug(hal->Screen.PrintChar, bootInfo->Debug & 1); + + // CPU startup intentionally lives after HAL initialization but before IRQs + // and scheduler startup. HAL owns ACPI/MADT discovery and LAPIC INIT/SIPI; + // the kernel owns KCPU records, per-CPU stacks, GDT/TSS/IDT state, and the + // AP park loop. External device IRQs remain routed to the BSP during this + // step; APs only become runnable after scheduler admission below. + KSTATUS smpStatus = KeStartApplicationProcessors(); + if (!KSUCCESS(smpStatus)) { + KePanic(KE_PANIC_CODE_INIT, "SMP: failed to start application processors, status=0x%x\n", smpStatus); + } KeIrqInitialize(hal); KeInitializeDpcSystem(); @@ -501,6 +514,13 @@ extern void KernelMain(BOOT_INFO* bootInfo) { // KeInitializeScheduler(); + KSTATUS smpSchedStatus = KeAdmitApplicationProcessorsToScheduler(); + if (!KSUCCESS(smpSchedStatus)) { + KePanic(KE_PANIC_CODE_INIT, "SMP: failed to admit APs to scheduler, status=0x%x\n", smpSchedStatus); + } + + KiRunCoreHardeningSelfTests(); + // Pack the arguments for the init thread SYSTEM_INIT_ARGS* initArgs = MmAllocatePool(sizeof(SYSTEM_INIT_ARGS)); initArgs->bootInfo = bootInfo; diff --git a/code/kernel/ldr_image.c b/code/kernel/ldr_image.c index 3b26721..da84d48 100644 --- a/code/kernel/ldr_image.c +++ b/code/kernel/ldr_image.c @@ -454,28 +454,6 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) { ); } -static void KiDebugLibtccAllocatorSlot(const char* phase, const char* filename, uintptr_t imageBase) { - if (!filename || !imageBase) return; - if (_stricmp(filename, "libtcc.dll") != 0) return; - - uint64_t slot = *(uint64_t*)(imageBase + 0x4A010); - uint64_t expectedDefault = (uint64_t)(imageBase + 0x25F0); - - DbgPrint("LDR libtcc %s: base=%p slot[+4A010]=%p expected-default=%p\n", - phase, - (void*)imageBase, - (void*)(uintptr_t)slot, - (void*)(uintptr_t)expectedDefault); - - DbgPrint("LDR libtcc data[+4A000..+4A028]: %016llx %016llx %016llx %016llx %016llx %016llx\n", - (unsigned long long)*(uint64_t*)(imageBase + 0x4A000), - (unsigned long long)*(uint64_t*)(imageBase + 0x4A008), - (unsigned long long)*(uint64_t*)(imageBase + 0x4A010), - (unsigned long long)*(uint64_t*)(imageBase + 0x4A018), - (unsigned long long)*(uint64_t*)(imageBase + 0x4A020), - (unsigned long long)*(uint64_t*)(imageBase + 0x4A028)); -} - static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) { if (!process || !name || !base) return STATUS_INVALID_PARAMETER; @@ -543,7 +521,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_ return s; } - KiDebugLibtccAllocatorSlot("after load/reloc before imports", filename, imageBase); s = LdrResolveImportsForProcess(process, imageBase); if (!KSUCCESS(s)) { @@ -557,7 +534,6 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_ return s; } - KiDebugLibtccAllocatorSlot("after imports", filename, imageBase); s = LdrRegisterProcessModule(process, filename, imageBase); if (!KSUCCESS(s)) { diff --git a/code/kernel/mm_phys.c b/code/kernel/mm_phys.c index cada76f..c9ad2c2 100644 --- a/code/kernel/mm_phys.c +++ b/code/kernel/mm_phys.c @@ -101,6 +101,26 @@ bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase) { // --------------------------------------------------------------------------- // Public API // --------------------------------------------------------------------------- +PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive) { + if (maxExclusive <= PAGE_SIZE) return (PHYSICAL_ADDRESS)0; + + uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock); + for (size_t i = gFrameStack.Top; i > 0; i--) { + size_t idx = i - 1; + PHYSICAL_ADDRESS frame = gFrameStack.Stack[idx]; + if (frame && frame < maxExclusive && !(frame & (PAGE_SIZE - 1))) { + size_t tail = gFrameStack.Top - idx - 1; + for (size_t j = 0; j < tail; j++) + gFrameStack.Stack[idx + j] = gFrameStack.Stack[idx + 1 + j]; + gFrameStack.Top--; + KeReleaseSpinlock(&gFrameStack.Lock, flags); + return frame; + } + } + KeReleaseSpinlock(&gFrameStack.Lock, flags); + return (PHYSICAL_ADDRESS)0; +} + PHYSICAL_ADDRESS MmAllocateFrame(void) { uintptr_t flags = KeAcquireSpinlock(&gFrameStack.Lock); if (gFrameStack.Top == 0) { diff --git a/code/kernel/mm_virt.c b/code/kernel/mm_virt.c index 0a82b7c..0269736 100644 --- a/code/kernel/mm_virt.c +++ b/code/kernel/mm_virt.c @@ -14,9 +14,13 @@ // Backed on demand from the PMM. No per-process heaps. #include #include +#include #include +#include +#include #include #include +#include // --------------------------------------------------------------------------- // Helpers - physical window access @@ -34,6 +38,14 @@ static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) { // PML4 / page-table helpers // --------------------------------------------------------------------------- static PHYSICAL_ADDRESS gPml4Phys = 0; +static SPINLOCK gPageTableLock = SPINLOCK_INIT; + +static volatile long gTlbShootdownLock = 0; +static volatile long gTlbShootdownActive = 0; +static volatile long gTlbShootdownAcks = 0; +static uintptr_t gTlbShootdownVirt = 0; + +#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u // Extract the page-table index for each level from a virtual address. #define PML4_IDX(va) (((va) >> 39) & 0x1FF) @@ -64,38 +76,150 @@ static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys, return childPhys; } +// --------------------------------------------------------------------------- +// TLB/cache coherency hooks +// --------------------------------------------------------------------------- +void MmFlushTlbLocal(uintptr_t virt) { + __invlpg((void*)virt); +} + +static void KiAcquireTlbShootdownLock(void) { + // Do not use the normal spinlock here: it masks interrupts while spinning. + // A CPU spinning with IF=0 on the shootdown lock could not acknowledge a + // concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is + // meant to avoid. + while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0) + _mm_pause(); + _ReadWriteBarrier(); +} + +static void KiReleaseTlbShootdownLock(void) { + _ReadWriteBarrier(); + _InterlockedExchange(&gTlbShootdownLock, 0); +} + +void MmFlushTlbForKernelMapping(uintptr_t virt) { + MmFlushTlbLocal(virt); + + if (gCpuCount <= 1) + return; + + KCPU* self = KeGetCurrentCpu(); + uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU; + + KiAcquireTlbShootdownLock(); + + long targets = 0; + gTlbShootdownVirt = virt; + _InterlockedExchange(&gTlbShootdownAcks, 0); + _ReadWriteBarrier(); + _InterlockedExchange(&gTlbShootdownActive, 1); + _ReadWriteBarrier(); + + for (uint32_t i = 0; i < gCpuCount; i++) { + KCPU* cpu = &gCpus[i]; + if (i == selfIndex) + continue; + if (!cpu->Present || !cpu->Started || !cpu->Online) + continue; + targets++; + HalApicSendTlbShootdownIpi(cpu->LapicId); + } + + for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) { + if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets) + break; + _mm_pause(); + } + + long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0); + _InterlockedExchange(&gTlbShootdownActive, 0); + _ReadWriteBarrier(); + KiReleaseTlbShootdownLock(); + + if (acks < targets) { + KePanic(KE_PANIC_CODE_GENERIC, + "TLB shootdown timeout: va=0x%llX targets=%ld acks=%ld selfCpu=%u\n", + (uint64_t)virt, targets, acks, selfIndex); + } +} + +static int KiHandleTlbShootdown(void) { + if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0) + return 0; + + uintptr_t va = gTlbShootdownVirt; + _ReadWriteBarrier(); + MmFlushTlbLocal(va); + _ReadWriteBarrier(); + _InterlockedIncrement(&gTlbShootdownAcks); + return 1; +} + +int MmHandleTlbShootdownNmi(void) { + return KiHandleTlbShootdown(); +} + // --------------------------------------------------------------------------- // Public mapping // --------------------------------------------------------------------------- __declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags) { unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0; + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag); - if (!pdpt) return STATUS_NO_MEMORY; + if (!pdpt) { + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NO_MEMORY; + } PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag); - if (!pdt) return STATUS_NO_MEMORY; + if (!pdt) { + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NO_MEMORY; + } PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag); - if (!pt) return STATUS_NO_MEMORY; + if (!pt) { + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NO_MEMORY; + } uint64_t* pte = KiPtePtr(pt, PT_IDX(virt)); *pte = phys | flags; - __invlpg((void *)virt); + + KeReleaseSpinlock(&gPageTableLock, lockFlags); + MmFlushTlbForKernelMapping(virt); return STATUS_SUCCESS; } + __declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt) { + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt)); - if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(*e4 & PAGE_PRESENT)) { + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NOT_FOUND; + } uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt)); - if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(*e3 & PAGE_PRESENT)) { + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NOT_FOUND; + } uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt)); - if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(*e2 & PAGE_PRESENT)) { + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return STATUS_NOT_FOUND; + } uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt)); *pte = 0; - __invlpg((void *)virt); + + KeReleaseSpinlock(&gPageTableLock, lockFlags); + MmFlushTlbForKernelMapping(virt); return STATUS_SUCCESS; } + __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags) { if (!size) return STATUS_SUCCESS; @@ -103,60 +227,92 @@ __declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint uintptr_t base = virt & PAGE_MASK; uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK; + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + KSTATUS status = STATUS_SUCCESS; + uintptr_t flushEnd = base; + for (uintptr_t va = base; va < end; va += PAGE_SIZE) { uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va)); - if (!(*e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(*e4 & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va)); - if (!(*e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(*e3 & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va)); - if (!(*e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(*e2 & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va)); - if (!(*pte & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(*pte & PAGE_PRESENT)) { + status = STATUS_NOT_FOUND; + break; + } // Preserve the physical address, replace everything else. PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK; *pte = phys | flags; - __invlpg((void *)va); + flushEnd = va + PAGE_SIZE; } - return STATUS_SUCCESS; + KeReleaseSpinlock(&gPageTableLock, lockFlags); + + for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE) + MmFlushTlbForKernelMapping(va); + + return status; } + __declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt) { if (!gPml4Phys) return 0; // Not initialized + PHYSICAL_ADDRESS result = 0; + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + uint64_t pml4_idx = (virt >> 39) & 0x1FF; uint64_t pdpt_idx = (virt >> 30) & 0x1FF; uint64_t pd_idx = (virt >> 21) & 0x1FF; uint64_t pt_idx = (virt >> 12) & 0x1FF; uint64_t* pml4 = KiPhysToVirt(gPml4Phys); - if (!(pml4[pml4_idx] & PAGE_PRESENT)) return 0; + if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out; uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK); - if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) return 0; + if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out; // Check for 1GB huge pages (bit 7 - PS) - if (pdpt[pdpt_idx] & 0x80) { - return (pdpt[pdpt_idx] & 0x000FFFFFC0000000ULL) + (virt & 0x3FFFFFFF); + if (pdpt[pdpt_idx] & PAGE_PS) { + result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF); + goto out; } uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK); - if (!(pd[pd_idx] & PAGE_PRESENT)) return 0; + if (!(pd[pd_idx] & PAGE_PRESENT)) goto out; // Check for 2MB huge pages (bit 7 - PS) - if (pd[pd_idx] & 0x80) { - return (pd[pd_idx] & 0x000FFFFFFFE00000ULL) + (virt & 0x1FFFFF); + if (pd[pd_idx] & PAGE_PS) { + result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF); + goto out; } uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK); - if (!(pt[pt_idx] & PAGE_PRESENT)) return 0; + if (!(pt[pt_idx] & PAGE_PRESENT)) goto out; // Extract the physical frame and add the offset within the page - return (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF); + result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF); + +out: + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return result; } + // --------------------------------------------------------------------------- // Page query and user/kernel copy boundary helpers // --------------------------------------------------------------------------- @@ -165,9 +321,12 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, uint64_t* flagsOut) { if (!gPml4Phys) return STATUS_NOT_FOUND; + KSTATUS status = STATUS_NOT_FOUND; + uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock); + uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys); uint64_t e4 = pml4[PML4_IDX(virt)]; - if (!(e4 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(e4 & PAGE_PRESENT)) goto out; uint64_t effectiveWrite = e4 & PAGE_WRITE; uint64_t effectiveUser = e4 & PAGE_USER; @@ -175,7 +334,7 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK); uint64_t e3 = pdpt[PDPT_IDX(virt)]; - if (!(e3 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(e3 & PAGE_PRESENT)) goto out; effectiveWrite &= e3 & PAGE_WRITE; effectiveUser &= e3 & PAGE_USER; effectiveNx |= e3 & PAGE_NX; @@ -189,12 +348,13 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, if (!effectiveUser) flags &= ~PAGE_USER; *flagsOut = flags; } - return STATUS_SUCCESS; + status = STATUS_SUCCESS; + goto out; } uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK); uint64_t e2 = pd[PDT_IDX(virt)]; - if (!(e2 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(e2 & PAGE_PRESENT)) goto out; effectiveWrite &= e2 & PAGE_WRITE; effectiveUser &= e2 & PAGE_USER; effectiveNx |= e2 & PAGE_NX; @@ -208,12 +368,13 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, if (!effectiveUser) flags &= ~PAGE_USER; *flagsOut = flags; } - return STATUS_SUCCESS; + status = STATUS_SUCCESS; + goto out; } uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK); uint64_t e1 = pt[PT_IDX(virt)]; - if (!(e1 & PAGE_PRESENT)) return STATUS_NOT_FOUND; + if (!(e1 & PAGE_PRESENT)) goto out; effectiveWrite &= e1 & PAGE_WRITE; effectiveUser &= e1 & PAGE_USER; effectiveNx |= e1 & PAGE_NX; @@ -226,9 +387,14 @@ __declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, if (!effectiveUser) flags &= ~PAGE_USER; *flagsOut = flags; } - return STATUS_SUCCESS; + status = STATUS_SUCCESS; + +out: + KeReleaseSpinlock(&gPageTableLock, lockFlags); + return status; } + static int KiUserRangeValid(uintptr_t addr, size_t size) { if (size == 0) return 1; if (addr < USER_BASE) return 0; @@ -581,6 +747,7 @@ static uintptr_t gUserImageNext = USER_LIMIT; // --------------------------------------------------------------------------- void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize) { gPml4Phys = bootPml4Phys; + gPageTableLock = (SPINLOCK)SPINLOCK_INIT; // Kernel pool heap. gPoolHeap.FreeList = (BLOCK_HDR *)0; diff --git a/code/public/ntlite/ddi/net_abi.h b/code/public/ntlite/ddi/net_abi.h index 6755c1f..fc7dc68 100644 --- a/code/public/ntlite/ddi/net_abi.h +++ b/code/public/ntlite/ddi/net_abi.h @@ -17,7 +17,7 @@ #define SYS_SHUTDOWN_WRITE 1u #define SYS_SHUTDOWN_BOTH 2u -#define SYS_SOCKET_ADDRESS_IPV4_SIZE 16u +#define SYS_SOCKET_ADDRESS_IPV4_SIZE 8u #define SYS_RESOLVE_HOST_NAME_MAX 255u #define SYS_SOCKET_OPT_RECV_TIMEOUT_MS 1u @@ -35,7 +35,6 @@ typedef struct SYS_SOCKET_ADDRESS_IPV4_K { uint16_t Family; // SYS_AF_INET uint16_t Port; // host byte order uint8_t Address[4]; // dotted-quad bytes, e.g. {10,0,2,2} - uint8_t Reserved[8]; } SYS_SOCKET_ADDRESS_IPV4_K; typedef struct SYS_POLLFD_K { diff --git a/code/public/ntlite/syscall.h b/code/public/ntlite/syscall.h index 9535211..e2b55f2 100644 --- a/code/public/ntlite/syscall.h +++ b/code/public/ntlite/syscall.h @@ -79,8 +79,7 @@ typedef void (*SYS_USER_APC_ROUTINE)(void* normalContext, typedef struct SYS_ASYNC_FILE_REQUEST { uint32_t FileHandle; uint32_t EventHandle; // optional; 0 means no event signal - uint32_t Flags; // reserved, must be 0 for now - uint32_t Reserved; + uint32_t Flags; uint64_t Offset; // explicit byte offset; does not touch file pointer void* Buffer; size_t Size; @@ -320,14 +319,12 @@ typedef struct SYS_SOCKET_ADDRESS_IPV4 { uint16_t Family; // SYS_AF_INET uint16_t Port; // host byte order uint8_t Address[4]; // dotted-quad bytes, e.g. {10,0,2,2} - uint8_t Reserved[8]; } SYS_SOCKET_ADDRESS_IPV4; typedef struct SYS_SOCKET_SEND_TO_REQUEST { const void* Buffer; size_t Length; uint32_t Flags; - uint32_t Reserved; const SYS_SOCKET_ADDRESS_IPV4* RemoteAddress; size_t RemoteAddressSize; size_t* SentOut; @@ -337,7 +334,6 @@ typedef struct SYS_SOCKET_RECV_FROM_REQUEST { void* Buffer; size_t Length; uint32_t Flags; - uint32_t Reserved; SYS_SOCKET_ADDRESS_IPV4* RemoteAddressOut; size_t* RemoteAddressSizeInOut; size_t* ReceivedOut; @@ -354,7 +350,6 @@ typedef struct SYS_RESOLVE_HOST_REQUEST { typedef struct SYS_SOCKET_OPTION_REQUEST { uint32_t Option; - uint32_t Reserved; void* Value; size_t ValueSize; size_t* ActualSizeOut; diff --git a/code/user/apps/shell_native/shell_native.c b/code/user/apps/shell_native/shell_native.c index db732a7..afc8970 100644 --- a/code/user/apps/shell_native/shell_native.c +++ b/code/user/apps/shell_native/shell_native.c @@ -5,6 +5,10 @@ #include #define MAX_CMD 256 +#define MAX_PIDS 64 // Maximum number of PIDs we will fetch at once + +// External Syscall Prototypes +uint32_t SysGetProcessList(uint32_t* pidsOut, size_t maxCount, size_t* countOut); // --------------------------------------------------------------------------- // Output helpers @@ -91,6 +95,7 @@ static void cmd_help() { print("Available commands:\n" " help - Show this list\n" " mem - Show memory usage\n" + " ps - List running process IDs\n" " dir [path] - List directory contents\n" " cd - Change directory\n" " type - Display file contents\n" @@ -122,6 +127,34 @@ static void cmd_mem(void) { print("\n"); } +static void cmd_ps(void) { + uint32_t pids[MAX_PIDS]; + size_t count = 0; + + uint32_t status = SysGetProcessList(pids, MAX_PIDS, &count); + if (status != 0) { + print("ps: failed to retrieve process list\n"); + return; + } + + print("Running Processes (Count: "); + printUInt((uint32_t)count); + print("):\n"); + + size_t displayCount = count < MAX_PIDS ? count : MAX_PIDS; + for (size_t i = 0; i < displayCount; i++) { + print(" PID: "); + printUInt(pids[i]); + print("\n"); + } + + if (count > MAX_PIDS) { + print(" ... and "); + printUInt((uint32_t)(count - MAX_PIDS)); + print(" more processes.\n"); + } +} + static void cmd_write(const char* args) { char filename[MAX_CMD]; int i = 0; while (args[i] != ' ' && args[i] != '\0' && i < MAX_CMD - 1) { @@ -247,6 +280,7 @@ static int runCommand(const char* cmd) { } } if (strcmp(cmd, "mem") == 0) { cmd_mem(); return 0; } + if (strcmp(cmd, "ps") == 0) { cmd_ps(); return 0; } if (strcmp(cmd, "dir") == 0) { cmd_dir("."); return 0; } if (strncmp(cmd, "dir ", 4) == 0) { cmd_dir(cmd + 4); return 0; } if (strncmp(cmd, "cd ", 3) == 0) { if (set_cwd(cmd + 3) != 0) print("cd: failed\n"); return 0; } diff --git a/readme.md b/readme.md index fb21be4..490ef75 100644 --- a/readme.md +++ b/readme.md @@ -16,7 +16,7 @@ The codebase is still small enough to understand, but large enough to hurt your - UEFI bootloader - higher-half x86-64 kernel -- real SMP +- real SMP: hardware AP startup, per-CPU state, scheduler-owned idle threads, reschedule IPIs, and TLB shootdowns - physical memory manager - virtual memory manager with kernel pool and user heap - W^X-style user page protections @@ -60,7 +60,7 @@ The codebase is still small enough to understand, but large enough to hurt your Single address space for the whole system. No separate kernel/user page tables. Isolation comes from compilers, page permissions and privilege levels, not from switching CR3. -Kernel lives in the higher half. User processes only see what they are explicitly given. Kernel pages are supervisor-only so user-mode code cannot touch them. User pages are isolated from each other through the allocator and process creation path, not by hardware address-space separation. One rogue pointer can't crash the kernel but it might corrupt other processes. But what are the chances? +Kernel lives in the higher half. User processes only see what they are explicitly given. Kernel pages are supervisor-only so user-mode code cannot touch them. User pages are isolated from each other through the allocator and process creation path, not by hardware address-space separation. One rogue pointer can't crash the kernel but it *might* corrupt other processes. But what are the chances? User heap and stack mappings are non-executable, and the PE loader applies final section protections after loading. Writable sections are left writable and non-executable; executable sections are mapped executable without write access where possible. @@ -77,6 +77,14 @@ DIRQL hardware IRQ handlers; acknowledge device, queue bottom half, le The timer interrupt don't context-switch directly inside the IRQ handler. Interrupts unwind through a common exit path, which can drain DPCs, deliver APCs when legal, and perform deferred scheduling. +### SMP + +Once an AP reaches the kernel, it gets its own `KCPU`, kernel stack, GDT, TSS, GS base, idle thread, and scheduler state. The BSP is not special because the code wants it to be; it is special because hardware, firmware, and device interrupts keep making it special. + +External device interrupts are routed to the BSP. CPU-local things such as exceptions, NMIs, syscalls, reschedule IPIs, and TLB shootdowns happen on the CPU that receives them. + +TLB shootdowns use NMI delivery because fixed-vector IPIs are not good enough when another CPU may be spinning with interrupts masked. Unexpected NMIs still panic; shootdown NMIs return only while a shootdown is active. + ### DPCs and APCs Hard IRQ handlers stay small. They acknowledge hardware, capture status, and queue DPCs. @@ -274,17 +282,20 @@ Q: Does it run Doom? A: `doom.exe` Q: If I write a program now, will it run in the future? -A: Not a chance. WE DO in fact BREAK USERSPACE. +A: Not a chance. WE DO - in fact - BREAK USERSPACE. Q: Can you read and write files? A: As far as I'm aware. Q: Does it have async I/O? -A: You didn't read the features list? +A: The best you'll get. Q: Does the networking actually work? A: It didn't crash yet. +Q: Does it really have SMP? +A: Watch out. It bites. + Q: Is the Win32 layer real? A: Depends how you frame it. Some apps will run, most will not.