clean up and standardize types across the codebase, unify atomic operations, improve UTF-8 handling throughout the system, remove obsolete input and named queue interfaces, redesign process preparation and creation, tighten IPC and service boundaries, improve error handling and reliability, minor documentation cleanup, fix doom surfaces, fix panic in ini parsing.

code/kernel/ke_smp.c

@@ -4,6 +4,7 @@
 // The kernel asks the HAL for platform processor identities, prepares the x86
 // bootstrap state, asks the HAL to start each processor, and waits for it to
 // enter KiApKernelEntry() and park in kernel-owned state.
+#include <ntlite/atomic.h>
 #include <stdint.h>
 #include <stddef.h>
 #include <stdbool.h>
@@ -323,12 +324,12 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
     KeLoadIdtForCurrentCpu();
     if (gApBootBlock) gApBootBlock->Stage = AP_STAGE_IDT;
 
-    _InterlockedExchange(&cpu->State, KeCpuStarted);
+    NtlAtomicExchange32(&cpu->State, KeCpuStarted);
     cpu->CurrentIrql = PASSIVE_LEVEL;
     cpu->InterruptDepth = 0;
     cpu->ReschedulePending = 0;
 
-    _InterlockedExchange(&cpu->State, KeCpuParked);
+    NtlAtomicExchange32(&cpu->State, KeCpuParked);
     if (gApBootBlock) {
         gApBootBlock->Stage = AP_STAGE_PARK;
         gApBootBlock->State = AP_STATE_KERNEL_PARKED;
@@ -337,7 +338,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
     while (true) {
         cpu->SchedulerTicks++;
 
-        if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
+        if (NtlAtomicLoad32(&cpu->State) == KeCpuAdmissionRequested) {
             THREAD* idle = cpu->IdleThread;
             if (idle) {
                 cpu->CurrentThread = idle;
@@ -349,7 +350,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
                 // a current idle thread before they may be delivered.
                 KeConfigureCurrentProcessorTimer();
 
-                _InterlockedExchange(&cpu->State, KeCpuSchedulerActive);
+                NtlAtomicExchange32(&cpu->State, KeCpuSchedulerActive);
                 KeEnableInterrupts();
                 KeStartInitialThread(idle);
             }
@@ -360,7 +361,7 @@ _Noreturn void KiApKernelEntry(KCPU* cpu) {
         // changes between the poll and HLT: check once with interrupts
         // disabled, then use STI;HLT as a single sleep transition.
         uintptr_t interruptFlags = KeDisableInterrupts();
-        if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuAdmissionRequested) {
+        if (NtlAtomicLoad32(&cpu->State) == KeCpuAdmissionRequested) {
             KeRestoreInterrupts(interruptFlags);
             continue;
         }
@@ -742,7 +743,7 @@ KSTATUS KeStartApplicationProcessors(void) {
         if (!KSUCCESS(st)) {
             DbgPrint("SMP: AP%u hardware=%u stack allocation failed status=0x%x; disabling AP and continuing\n",
                      i, cpu->HardwareId, st);
-            _InterlockedExchange(&cpu->State, KeCpuAbsent);
+            NtlAtomicExchange32(&cpu->State, KeCpuAbsent);
             continue;
         }
 
@@ -755,7 +756,7 @@ KSTATUS KeStartApplicationProcessors(void) {
         if (!KSUCCESS(st)) {
             DbgPrint("SMP: AP%u hardware=%u startup IPI failed status=0x%x; disabling AP and continuing\n",
                      i, cpu->HardwareId, st);
-            _InterlockedExchange(&cpu->State, KeCpuAbsent);
+            NtlAtomicExchange32(&cpu->State, KeCpuAbsent);
             continue;
         }
 
@@ -777,11 +778,11 @@ KSTATUS KeStartApplicationProcessors(void) {
             DbgPrint("SMP: AP%u failed to reach kernel park state hardware=%u state=%u stage=0x%x(%s) heartbeat=%llu last=%llu; disabling AP and continuing\n",
                      i, cpu->HardwareId, gApBootBlock->State, gApBootBlock->Stage,
                      KiApStageName(gApBootBlock->Stage), gApBootBlock->Heartbeat, lastHeartbeat);
-            _InterlockedExchange(&cpu->State, KeCpuAbsent);
+            NtlAtomicExchange32(&cpu->State, KeCpuAbsent);
             continue;
         }
 
-        if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
+        if (NtlAtomicLoad32(&cpu->State) != KeCpuParked) {
             KePanic(KE_PANIC_CODE_INIT,
                     "SMP: AP%u boot block parked without CPU state transition state=%ld\n",
                     i, cpu->State);
@@ -812,7 +813,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
         KCPU* cpu = &gCpus[i];
         if (cpu->State == KeCpuAbsent) continue;
 
-        if (_InterlockedCompareExchange(&cpu->State, 0, 0) != KeCpuParked) {
+        if (NtlAtomicLoad32(&cpu->State) != KeCpuParked) {
             KePanic(KE_PANIC_CODE_INIT,
                     "SMP: AP%u cannot enter scheduler from state=%ld\n",
                     i, cpu->State);
@@ -824,7 +825,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
             return st;
         KiSmpPoolCheckpoint("smp after AP idle create", i);
 
-        _InterlockedExchange(&cpu->State, KeCpuAdmissionRequested);
+        NtlAtomicExchange32(&cpu->State, KeCpuAdmissionRequested);
         KiSmpPoolCheckpoint("smp before AP wake", i);
 
         // Parked secondary processors sleep with no periodic timer armed yet. The
@@ -834,11 +835,11 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
         // the AP has installed its idle thread.
         KeSendProcessorInterrupt(cpu->HardwareId, KE_VECTOR_RESCHEDULE_IPI);
 
-        int active = 0;
+        bool active = false;
         uint64_t schedulerDeadline = KiSmpDeadlineAfter(AP_SCHED_TIMEOUT_US);
         for (;;) {
-            if (_InterlockedCompareExchange(&cpu->State, 0, 0) == KeCpuSchedulerActive) {
-                active = 1;
+            if (NtlAtomicLoad32(&cpu->State) == KeCpuSchedulerActive) {
+                active = true;
                 break;
             }
             if (KiSmpQueryTicks() >= schedulerDeadline)
@@ -852,7 +853,7 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
                     i, cpu->HardwareId, cpu->State, cpu->SchedulerTicks);
         }
 
-        _InterlockedExchange(&cpu->State, KeCpuOnline);
+        NtlAtomicExchange32(&cpu->State, KeCpuOnline);
         KiSmpPoolCheckpoint("smp after AP scheduler active", i);
         admitted++;
         if (KiSmpDiagnosticsEnabled())
@@ -875,11 +876,11 @@ typedef struct SMP_AP_WORKER_ARG {
     uint32_t WorkerIndex;
     struct SMP_AP_WORK_TEST* Test;
     EVENT* StepEvent;
-    volatile long Ready;
-    volatile long WaitArmed;
-    volatile long Progress;
-    volatile long Done;
-    volatile long Failure;
+    volatile int32_t Ready;
+    volatile int32_t WaitArmed;
+    volatile int32_t Progress;
+    volatile int32_t Done;
+    volatile int32_t Failure;
     uint32_t WrongCpu;
     uint32_t WrongRunningCpu;
 } SMP_AP_WORKER_ARG;
@@ -887,10 +888,10 @@ typedef struct SMP_AP_WORKER_ARG {
 struct SMP_AP_WORK_TEST {
     uint64_t Magic;
     uint32_t WorkerCount;
-    volatile long ReadyCount;
-    volatile long DoneCount;
-    volatile long FailureCount;
-    volatile long CpuHits[KE_MAX_CPUS];
+    volatile int32_t ReadyCount;
+    volatile int32_t DoneCount;
+    volatile int32_t FailureCount;
+    volatile int32_t CpuHits[KE_MAX_CPUS];
     SMP_AP_WORKER_ARG Workers[KE_MAX_CPUS];
 };
 
@@ -901,14 +902,14 @@ static void KiSmpApControlledWorker(void* context) {
         KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%016llX\n", (uint64_t)arg);
     }
 
-    _InterlockedExchange(&arg->Ready, 1);
-    _InterlockedIncrement(&arg->Test->ReadyCount);
+    NtlAtomicExchange32(&arg->Ready, 1);
+    NtlAtomicIncrement32(&arg->Test->ReadyCount);
 
     for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
-        _InterlockedIncrement(&arg->WaitArmed);
+        NtlAtomicIncrement32(&arg->WaitArmed);
         KSTATUS waitStatus = KeWaitForEvent(arg->StepEvent, 0);
         if (!KSUCCESS(waitStatus)) {
-            _InterlockedExchange(&arg->Failure, 1);
+            NtlAtomicExchange32(&arg->Failure, 1);
             break;
         }
 
@@ -916,49 +917,49 @@ static void KiSmpApControlledWorker(void* context) {
         THREAD* self = KeGetCurrentThread();
         if (!cpu || cpu->CpuIndex != arg->CpuIndex) {
             arg->WrongCpu = cpu ? cpu->CpuIndex : KE_INVALID_CPU;
-            _InterlockedExchange(&arg->Failure, 1);
+            NtlAtomicExchange32(&arg->Failure, 1);
             break;
         }
         if (!self || self->RunningCpu != cpu->CpuIndex) {
             arg->WrongRunningCpu = self ? self->RunningCpu : KE_INVALID_CPU;
-            _InterlockedExchange(&arg->Failure, 1);
+            NtlAtomicExchange32(&arg->Failure, 1);
             break;
         }
         if (self && (self->AffinityMask & (1ULL << cpu->CpuIndex)) == 0) {
             arg->WrongCpu = cpu->CpuIndex;
-            _InterlockedExchange(&arg->Failure, 1);
+            NtlAtomicExchange32(&arg->Failure, 1);
             break;
         }
 
         if (cpu->CpuIndex < KE_MAX_CPUS)
-            _InterlockedIncrement(&arg->Test->CpuHits[cpu->CpuIndex]);
+            NtlAtomicIncrement32(&arg->Test->CpuHits[cpu->CpuIndex]);
 
-        _InterlockedIncrement(&arg->Progress);
+        NtlAtomicIncrement32(&arg->Progress);
         if ((round & 3u) == 0)
             KeYield();
     }
 
-    if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) {
-        _InterlockedIncrement(&arg->Test->FailureCount);
+    if (NtlAtomicLoad32(&arg->Failure) != 0) {
+        NtlAtomicIncrement32(&arg->Test->FailureCount);
     }
 
-    _InterlockedExchange(&arg->Done, 1);
-    _InterlockedIncrement(&arg->Test->DoneCount);
+    NtlAtomicExchange32(&arg->Done, 1);
+    NtlAtomicIncrement32(&arg->Test->DoneCount);
 }
 
 // Waits for for Worker Counter.
-static int KiWaitForWorkerCounter(volatile long* counter, long expected, const char* what) {
+static bool KiWaitForWorkerCounter(volatile int32_t* counter, int32_t expected, const char* what) {
     uint64_t deadline = KiSmpDeadlineAfter(SMP_AP_WORKER_TIMEOUT_US);
     for (;;) {
-        if (_InterlockedCompareExchange(counter, 0, 0) >= expected)
-            return 1;
+        if (NtlAtomicLoad32(counter) >= expected)
+            return true;
         if (KiSmpQueryTicks() >= deadline)
             break;
         KeYield();
     }
-    DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%ld got=%ld\n",
-             what ? what : "counter", expected, _InterlockedCompareExchange(counter, 0, 0));
-    return 0;
+    DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%d got=%ld\n",
+             what ? what : "counter", expected, NtlAtomicLoad32(counter));
+    return false;
 }
 
 // Runs application Processor Scheduler Self Tests.
@@ -1019,7 +1020,7 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
         return NTLSTATUS_SUCCESS;
     }
 
-    if (!KiWaitForWorkerCounter(&test->ReadyCount, (long)workerCount, "AP worker ready")) {
+    if (!KiWaitForWorkerCounter(&test->ReadyCount, (int32_t)workerCount, "AP worker ready")) {
         KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers failed to become ready ready=%ld workers=%u\n",
                 test->ReadyCount, workerCount);
     }
@@ -1027,7 +1028,7 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
     for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
         for (uint32_t w = 0; w < workerCount; w++) {
             SMP_AP_WORKER_ARG* arg = &test->Workers[w];
-            if (!KiWaitForWorkerCounter(&arg->WaitArmed, (long)(round + 1), "AP worker wait-armed")) {
+            if (!KiWaitForWorkerCounter(&arg->WaitArmed, (int32_t)(round + 1), "AP worker wait-armed")) {
                 KePanic(KE_PANIC_CODE_SELFTEST,
                         "SMP scheduler selftest: AP%u did not arm wait for round %u armed=%ld progress=%ld\n",
                         arg->CpuIndex, round, arg->WaitArmed, arg->Progress);
@@ -1037,13 +1038,13 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
 
         for (uint32_t w = 0; w < workerCount; w++) {
             SMP_AP_WORKER_ARG* arg = &test->Workers[w];
-            if (!KiWaitForWorkerCounter(&arg->Progress, (long)(round + 1), "AP worker progress")) {
+            if (!KiWaitForWorkerCounter(&arg->Progress, (int32_t)(round + 1), "AP worker progress")) {
                 KePanic(KE_PANIC_CODE_SELFTEST,
                         "SMP scheduler selftest: AP%u no progress round=%u progress=%ld failure=%ld wrongCpu=%u wrongRunningCpu=%u\n",
                         arg->CpuIndex, round, arg->Progress, arg->Failure,
                         arg->WrongCpu, arg->WrongRunningCpu);
             }
-            if (_InterlockedCompareExchange(&arg->Failure, 0, 0) != 0) {
+            if (NtlAtomicLoad32(&arg->Failure) != 0) {
                 KePanic(KE_PANIC_CODE_SELFTEST,
                         "SMP scheduler selftest: AP%u worker failed round=%u wrongCpu=%u wrongRunningCpu=%u progress=%ld\n",
                         arg->CpuIndex, round, arg->WrongCpu, arg->WrongRunningCpu, arg->Progress);
@@ -1051,19 +1052,19 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
         }
     }
 
-    if (!KiWaitForWorkerCounter(&test->DoneCount, (long)workerCount, "AP worker done")) {
-        KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%ld workers=%u\n",
+    if (!KiWaitForWorkerCounter(&test->DoneCount, (int32_t)workerCount, "AP worker done")) {
+        KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%d workers=%u\n",
                 test->DoneCount, workerCount);
     }
 
-    if (_InterlockedCompareExchange(&test->FailureCount, 0, 0) != 0) {
-        KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%ld\n",
+    if (NtlAtomicLoad32(&test->FailureCount) != 0) {
+        KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%d\n",
                 test->FailureCount);
     }
 
     for (uint32_t w = 0; w < workerCount; w++) {
         SMP_AP_WORKER_ARG* arg = &test->Workers[w];
-        long hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0;
+        int32_t hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0;
         if (hits <= 0) {
             KePanic(KE_PANIC_CODE_SELFTEST,
                     "SMP scheduler selftest: AP%u recorded no CPU hits\n", arg->CpuIndex);