parallel compile, SMP, swapgs, cleanup, remove useless Reserved fields

code/kernel/ke_sched.c

@@ -1,6 +1,7 @@
 // ke_sched.c - MLFQ scheduler
 #include <stddef.h>
 #include <stdbool.h>
+#include <intrin.h>
 #include <kernel/ke_sched.h>
 #include <kernel/ke_cpu.h>
 #include <kernel/ke_process.h>
@@ -8,14 +9,64 @@
 #include <kernel/ke_spinlock.h>
 #include <kernel/ke_irql.h>
 #include <kernel/ke_apc.h>
+#include <kernel/ke_debug.h>
 #include <kernel/ke_list.h>
 #include <kernel/mm_virt.h>
+#include <kernel/hal/hal_apic.h>
 
 static LIST_ENTRY gRunQueue[PriorityCount];
 static SPINLOCK   gSchedLock;
 
 static uint32_t gAgingCounter = 0;
 static KE_SCHED_STATS gSchedStats;
+static PROCESS* gIdleProcess;
+
+static uint64_t KiDefaultThreadAffinityMask(void) {
+    // Normal kernel, user, and driver-created threads are BSP-affine by
+    // default. Code that is explicitly audited for AP execution can widen its
+    // affinity before the thread enters the ready queue.
+    return 1ULL;
+}
+
+static int KiThreadCanRunOnCpuLocked(const THREAD* thread, const KCPU* cpu) {
+    if (!thread || !cpu) return 0;
+    if (thread->IsIdleThread) return thread == cpu->IdleThread;
+    if (cpu->CpuIndex >= 64) return 0;
+    return (thread->AffinityMask & (1ULL << cpu->CpuIndex)) != 0;
+}
+
+static void KiRequestRescheduleCpuLocked(KCPU* target) {
+    if (!target || !target->Online)
+        return;
+
+    _InterlockedExchange(&target->ReschedulePending, 1);
+
+    KCPU* self = KeGetCurrentCpu();
+    if (self && target->CpuIndex != self->CpuIndex) {
+        HalApicSendRescheduleIpi(target->LapicId);
+    }
+}
+
+static void KiPokeRunnableCpusLocked(THREAD* thread) {
+    if (!thread || thread->IsIdleThread)
+        return;
+
+    for (uint32_t i = 0; i < gCpuCount; i++) {
+        KCPU* cpu = &gCpus[i];
+        if (!cpu->Present || !cpu->Started || !cpu->Online)
+            continue;
+        if (!KiThreadCanRunOnCpuLocked(thread, cpu))
+            continue;
+
+        THREAD* cur = cpu->CurrentThread;
+        if (!cur || cur == cpu->IdleThread) {
+            KiRequestRescheduleCpuLocked(cpu);
+        } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) {
+            gSchedStats.PriorityPreemptions++;
+            KiRequestRescheduleCpuLocked(cpu);
+        }
+    }
+}
 
 static uint32_t KiBaseQuantumForPriority(THREAD_PRIORITY priority) {
     switch (priority) {
@@ -60,29 +111,133 @@ void KeGetSchedulerStats(KE_SCHED_STATS* out) {
     KeReleaseSpinlock(&gSchedLock, flags);
 }
 
-static void KiIdleLoop(void) {
-    while (true) { __halt(); }
+
+static void KiArmPostSwitchUnlock(THREAD* thread, uintptr_t cookie) {
+    // The scheduler lock is intentionally held across KeSwitchContext().  The
+    // unlock cookie belongs to the thread continuation that will resume after
+    // the switch, not to the physical CPU. Another dispatch may resume this
+    // thread from an unrelated interrupt, so using a KCPU slot here would
+    // restore the outgoing CPU context's IF/IRQL into the incoming thread.
+    if (thread)
+        thread->SwitchLockCookie = cookie;
 }
 
-void KeInitializeScheduler(void) {
-    KCPU* cpu = KeGetCurrentCpu();
+static uintptr_t KiPostSwitchUnlockCookie(THREAD* thread) {
+    return thread ? thread->SwitchLockCookie : 0x202ULL;
+}
 
+
+static void KiSchedulerBug(const char* where, THREAD* thread, const char* why) {
+    KePanic(KE_PANIC_CODE_SCHEDULER,
+            "scheduler invariant failed: %s: thread=0x%llX tid=%u state=%u queued=%u runningCpu=%u idle=%u: %s\n",
+            where,
+            (uint64_t)thread,
+            thread ? thread->ThreadId : 0,
+            thread ? (unsigned)thread->State : 0,
+            thread ? (unsigned)thread->Queued : 0,
+            thread ? (unsigned)thread->RunningCpu : 0,
+            thread ? (unsigned)thread->IsIdleThread : 0,
+            why ? why : "?");
+}
+
+static void KiAssertReadyQueueCandidateLocked(const char* where, THREAD* thread) {
+    if (!thread)
+        KiSchedulerBug(where, thread, "null thread");
+    if (thread->IsIdleThread)
+        KiSchedulerBug(where, thread, "idle thread cannot enter ready queue");
+    if (thread->State == ThreadDead)
+        KiSchedulerBug(where, thread, "dead thread cannot enter ready queue");
+    if (thread->Queued)
+        KiSchedulerBug(where, thread, "thread already in ready queue");
+    if (thread->RunningCpu != KE_INVALID_CPU)
+        KiSchedulerBug(where, thread, "thread still reserved/running on a CPU");
+}
+
+static void KiAssertPickedThreadLocked(const char* where, THREAD* thread) {
+    if (!thread)
+        KiSchedulerBug(where, thread, "null picked thread");
+    if (thread->IsIdleThread)
+        return;
+    if (thread->Queued)
+        KiSchedulerBug(where, thread, "picked thread still marked queued");
+    if (thread->RunningCpu != KE_INVALID_CPU)
+        KiSchedulerBug(where, thread, "picked thread already running on a CPU");
+    if (thread->State != ThreadReady)
+        KiSchedulerBug(where, thread, "picked thread was not ready");
+}
+
+static void KiInsertReadyLocked(THREAD* thread) {
+    KiAssertReadyQueueCandidateLocked("KiInsertReadyLocked", thread);
+
+    int p = (int)thread->Priority;
+    if (p < 0) p = 0;
+    if (p >= PriorityCount) p = PriorityCount - 1;
+
+    thread->State = ThreadReady;
+    thread->Queued = 1;
+    ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
+}
+
+static int KiAnyRunnableForCpuLocked(KCPU* cpu);
+
+static int KiAnyRunnableForCurrentCpu(void) {
+    KCPU* cpu = KeGetCurrentCpu();
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    int any = KiAnyRunnableForCpuLocked(cpu);
+    KeReleaseSpinlock(&gSchedLock, flags);
+    return any;
+}
+
+static void KiIdleLoop(void) {
+    for (;;) {
+        KCPU* cpu = KeGetCurrentCpu();
+        cpu->SchedulerTicks++;
+
+        // AP idle threads sleep until an interrupt or reschedule IPI wakes the
+        // CPU. Ready paths set ReschedulePending and target the CPU's LAPIC;
+        // KiExitInterrupt consumes the request after EOI.
+        if (KiAnyRunnableForCurrentCpu()) {
+            KeYield();
+            continue;
+        }
+
+        // HLT only wakes for maskable IPIs when IF=1.  A CPU that reaches
+        // the idle thread through an unusual switch path must not sleep with
+        // interrupts disabled, otherwise reschedule/TLB shootdown IPIs can
+        // be delayed forever.
+        _enable();
+        __halt();
+    }
+}
+
+void KiInitializeSchedulerGlobal(void) {
     for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
     gSchedLock = (SPINLOCK)SPINLOCK_INIT;
 
-    PROCESS* idleProc;
-    KSTATUS s = KeCreateProcess("idle", &idleProc);
-    if (!KSUCCESS(s)) return;
+    KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create idle process, status=%u\n", s);
 
     PROCESS* systemProc;
     s = KeCreateSystemProcess(&systemProc);
-    if (!KSUCCESS(s)) return;
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create system process, status=%u\n", s);
     (void)systemProc;
+}
+
+KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
+    if (!cpu || !gIdleProcess)
+        return STATUS_INVALID_PARAMETER;
+    if (cpu->IdleThread)
+        return STATUS_SUCCESS;
 
     THREAD* idle = (THREAD *)ObAllocateObject(ObjectTypeThread, sizeof(THREAD));
+    if (!idle)
+        return STATUS_NO_MEMORY;
+
     idle->Header.Destroy = (void *)0;
-    idle->ThreadId        = 0;
-    idle->State           = ThreadReady;
+    idle->ThreadId        = cpu->CpuIndex ? (0x80000000u | cpu->CpuIndex) : 0;
+    idle->State           = ThreadRunning;
     idle->Priority        = PriorityLow;
     idle->QuantaConsumed  = 0;
     idle->QuantumTicks     = 1;
@@ -90,12 +245,16 @@ void KeInitializeScheduler(void) {
     idle->CpuBurstTicks    = 0;
     idle->LastRunTicks     = 0;
     idle->RunningCpu       = cpu->CpuIndex;
-    idle->Owner           = idleProc;
+    idle->Queued           = 0;
+    idle->IsIdleThread     = 1;
+    idle->SchedulerFlags   = 0;
+    idle->LastCpu          = cpu->CpuIndex;
+    idle->AffinityMask     = (cpu->CpuIndex < 64) ? (1ULL << cpu->CpuIndex) : 0ULL;
+    idle->Owner           = gIdleProcess;
     idle->WaitObject      = (OBJECT *)0;
     idle->WaitStatus      = STATUS_SUCCESS;
     idle->WaitListHead    = (LIST_ENTRY*)0;
     idle->AlertableWait   = 0;
-    idle->SchedLockFlags  = 0x202ULL; // IF=1
     idle->UserStackBase   = (void*)0;
     idle->Teb             = (void*)0;
     idle->TebSize         = 0;
@@ -106,6 +265,10 @@ void KeInitializeScheduler(void) {
     LIST_INIT(&idle->SchedulerLink);
 
     void* kstack = MmAllocatePool(KSTACK_SIZE);
+    if (!kstack) {
+        ObDereferenceObject(&idle->Header);
+        return STATUS_NO_MEMORY;
+    }
     idle->KernelStackBase = kstack;
     idle->KernelStackTop  = (uintptr_t)kstack + KSTACK_SIZE;
     idle->TssRsp0         = idle->KernelStackTop;
@@ -121,22 +284,34 @@ void KeInitializeScheduler(void) {
     // The hand-built CPU idle thread is not created through KeCreateThread(),
     // so register it manually with the idle process.  This keeps PID 0 visible
     // as the real idle process while kernel worker threads live under PID 1.
-    ObReferenceObject(&idleProc->Header);
-    uintptr_t idleFlags = KeAcquireSpinlock(&idleProc->HandleLock);
-    ListInsertTail(&idleProc->ThreadList, &idle->ProcessLink);
-    idleProc->ThreadCount++;
-    KeReleaseSpinlock(&idleProc->HandleLock, idleFlags);
+    ObReferenceObject(&gIdleProcess->Header);
+    uintptr_t idleFlags = KeAcquireSpinlock(&gIdleProcess->HandleLock);
+    ListInsertTail(&gIdleProcess->ThreadList, &idle->ProcessLink);
+    gIdleProcess->ThreadCount++;
+    KeReleaseSpinlock(&gIdleProcess->HandleLock, idleFlags);
 
-    cpu->IdleThread    = idle;
+    cpu->IdleThread = idle;
     cpu->CurrentThread = idle;
     cpu->OutgoingThread = (THREAD*)0;
+    idle->SwitchLockCookie = 0x202ULL;
+    return STATUS_SUCCESS;
+}
+
+void KeInitializeScheduler(void) {
+    KiInitializeSchedulerGlobal();
+
+    KCPU* cpu = KeGetCurrentCpu();
+    KSTATUS s = KiCreateIdleThreadForCpu(cpu);
+    if (!KSUCCESS(s))
+        KePanic(KE_PANIC_CODE_INIT, "scheduler: failed to create CPU%u idle thread, status=%u\n",
+                (unsigned)cpu->CpuIndex, s);
 }
 
 // Safely release the lock and clean up any dead threads natively post-switch
 void KeSchedulerPostSwitch(void) {
     KCPU* cpu = KeGetCurrentCpu();
     THREAD* cur = cpu->CurrentThread;
-    KeReleaseSpinlock(&gSchedLock, cur->SchedLockFlags);
+    KeReleaseSpinlock(&gSchedLock, KiPostSwitchUnlockCookie(cur));
 
     KiDeliverKernelApcs();
 
@@ -148,24 +323,29 @@ void KeSchedulerPostSwitch(void) {
     }
 }
 
-void KeReadyThread(THREAD* thread) {
-    if (!thread) return;
-    thread->State = ThreadReady;
-    int p = (int)thread->Priority;
-    if (p < 0) p = 0;
-    if (p >= PriorityCount) p = PriorityCount - 1;
+void KeSetThreadAffinityMask(THREAD* thread, uint64_t affinityMask) {
+    if (!thread || thread->IsIdleThread) return;
+
+    if (!affinityMask)
+        affinityMask = 1ULL;
 
     uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
-    ListInsertTail(&gRunQueue[p], &thread->SchedulerLink);
+    thread->AffinityMask = affinityMask;
+    if (thread->Queued)
+        KiPokeRunnableCpusLocked(thread);
+    KeReleaseSpinlock(&gSchedLock, flags);
+}
 
-    KCPU* cpu = KeGetCurrentCpu();
-    THREAD* cur = cpu->CurrentThread;
-    if (!cur || cur == cpu->IdleThread) {
-        KiRequestReschedule();
-    } else if (cur->State == ThreadRunning && thread->Priority > cur->Priority) {
-        gSchedStats.PriorityPreemptions++;
-        KiRequestReschedule();
-    }
+uint64_t KeGetDefaultThreadAffinityMask(void) {
+    return KiDefaultThreadAffinityMask();
+}
+
+void KeReadyThread(THREAD* thread) {
+    if (!thread) return;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+    KiInsertReadyLocked(thread);
+    KiPokeRunnableCpusLocked(thread);
 
     KeReleaseSpinlock(&gSchedLock, flags);
 }
@@ -173,35 +353,58 @@ void KeReadyThread(THREAD* thread) {
 void KeUnreadyThread(THREAD* thread) {
     if (!thread) return;
     uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
-    ListRemove(&thread->SchedulerLink);
+    if (thread->Queued) {
+        ListRemove(&thread->SchedulerLink);
+        thread->Queued = 0;
+    }
     KeReleaseSpinlock(&gSchedLock, flags);
 }
 
-static int KiAnyRunnableLocked(void) {
+
+static int KiAnyRunnableForCpuLocked(KCPU* cpu) {
+    if (!cpu) cpu = KeGetCurrentCpu();
+
     for (int p = 0; p < PriorityCount; p++) {
-        if (!LIST_IS_EMPTY(&gRunQueue[p]))
-            return 1;
+        LIST_ENTRY* head = &gRunQueue[p];
+        for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
+            THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+            if (KiThreadCanRunOnCpuLocked(thread, cpu))
+                return 1;
+        }
     }
     return 0;
 }
 
-static THREAD* KiPickNext(void) {
+static THREAD* KiPickNextForCpu(KCPU* cpu) {
+    if (!cpu) cpu = KeGetCurrentCpu();
+
     for (int p = PriorityCount - 1; p >= 0; p--) {
-        if (!LIST_IS_EMPTY(&gRunQueue[p])) {
-            LIST_ENTRY* node = gRunQueue[p].Fwd;
+        LIST_ENTRY* head = &gRunQueue[p];
+        for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
+            THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
+            if (!KiThreadCanRunOnCpuLocked(thread, cpu))
+                continue;
+
             ListRemove(node);
-            return CONTAINING_RECORD(node, THREAD, SchedulerLink);
+            thread->Queued = 0;
+            KiAssertPickedThreadLocked("KiPickNextForCpu", thread);
+            return thread;
         }
     }
-    return KeGetCurrentCpu()->IdleThread;
+
+    return cpu->IdleThread;
 }
 
+
 void KiRequestReschedule(void) {
-    KeGetCurrentCpu()->ReschedulePending = 1;
+    KCPU* cpu = KeGetCurrentCpu();
+    if (cpu)
+        _InterlockedExchange(&cpu->ReschedulePending, 1);
 }
 
 int KiIsReschedulePending(void) {
-    return KeGetCurrentCpu()->ReschedulePending != 0;
+    KCPU* cpu = KeGetCurrentCpu();
+    return cpu && _InterlockedCompareExchange(&cpu->ReschedulePending, 0, 0) != 0;
 }
 
 static void KiSchedule(void) {
@@ -210,16 +413,16 @@ static void KiSchedule(void) {
     THREAD* idle = cpu->IdleThread;
     if (!cur) return;
 
-    cpu->ReschedulePending = 0;
+    _InterlockedExchange(&cpu->ReschedulePending, 0);
 
     uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
-    cur->SchedLockFlags = flags;
+    KiArmPostSwitchUnlock(cur, flags);
 
     // The currently running thread is not present in a run queue.  If it is
     // still runnable and nobody else is ready, keep executing it.  Otherwise a
     // lone CPU-bound thread gets alternated with the idle thread on every timer
     // tick, effectively turning a periodic preemption check into a 50% throttle.
-    if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableLocked()) {
+    if (cur != idle && cur->State == ThreadRunning && !KiAnyRunnableForCpuLocked(cpu)) {
         if (cur->QuantumRemaining == 0)
             KiRefillQuantumLocked(cur);
         gSchedStats.KeepCurrentDecisions++;
@@ -227,22 +430,21 @@ static void KiSchedule(void) {
         return;
     }
 
-    THREAD* next = KiPickNext();
+    THREAD* next = KiPickNextForCpu(cpu);
     if (next == idle) gSchedStats.IdleSelections++;
     if (next == cur) {
-        // Only re-queue if still running normally; a ThreadWaiting/ThreadDead
-        // caller has already placed SchedulerLink elsewhere (or is discarding
-        // the thread).
-        if (cur != idle && cur->State == ThreadRunning)
-            ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
+        if (cur != idle)
+            KiSchedulerBug("KiSchedule", cur, "current thread unexpectedly appeared in ready queue");
         KeReleaseSpinlock(&gSchedLock, flags);
         return;
     }
 
+    KiAssertPickedThreadLocked("KiSchedule", next);
     next->State    = ThreadRunning;
     KiRefillQuantumLocked(next);
     cpu->CurrentThread = next;
     next->RunningCpu = cpu->CpuIndex;
+    next->LastCpu = cpu->CpuIndex;
 
     // Only re-queue the outgoing thread if it is still in the running state.
     // If KeBlockThread set State = ThreadWaiting, SchedulerLink was already
@@ -250,11 +452,14 @@ static void KiSchedule(void) {
     // corrupt both lists simultaneously (same intrusive node in two lists).
     // If KeExitThread set State = ThreadDead, the thread must not be re-queued.
     if (cur != idle && cur->State == ThreadRunning) {
+        cur->RunningCpu = KE_INVALID_CPU;
+        cur->LastCpu = cpu->CpuIndex;
         cur->State = ThreadReady;
-        ListInsertTail(&gRunQueue[cur->Priority], &cur->SchedulerLink);
+        KiInsertReadyLocked(cur);
+    } else if (cur) {
+        cur->RunningCpu = KE_INVALID_CPU;
+        cur->LastCpu = cpu->CpuIndex;
     }
-
-    if (cur) cur->RunningCpu = KE_INVALID_CPU;
     cpu->OutgoingThread = cur;
     cpu->ContextSwitches++;
     gSchedStats.ContextSwitches++;
@@ -267,6 +472,28 @@ void KiScheduleFromInterrupt(void) {
     KiSchedule();
 }
 
+
+void KiSchedulerMarkThreadWaiting(THREAD* thread) {
+    if (!thread) return;
+
+    KCPU* cpu = KeGetCurrentCpu();
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+
+    if (thread->IsIdleThread)
+        KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "idle thread cannot block");
+    if (thread->Queued)
+        KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "queued thread cannot block");
+    if (thread != cpu->CurrentThread)
+        KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "only the current thread can prepare a blocking wait");
+    if (thread->RunningCpu != cpu->CpuIndex)
+        KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread running CPU does not match current CPU");
+    if (thread->State != ThreadRunning)
+        KiSchedulerBug("KiSchedulerMarkThreadWaiting", thread, "thread was not running before wait");
+
+    thread->State = ThreadWaiting;
+    KeReleaseSpinlock(&gSchedLock, flags);
+}
+
 void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
     if (!thread) return;
     uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
@@ -275,6 +502,51 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
     KeReleaseSpinlock(&gSchedLock, flags);
 }
 
+
+void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
+    if (!thread) return;
+
+    uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
+
+    // Complete the wait while holding the scheduler lock.  On SMP a wake can
+    // race the prepare->commit window on another CPU: the target thread is
+    // already on the object's wait list and has State=ThreadWaiting, but it is
+    // still executing with RunningCpu set until it actually calls KeYield().
+    // That case must complete in-place, not enqueue the still-running thread.
+    if (thread->State != ThreadWaiting) {
+        thread->WaitStatus = status;
+        KeReleaseSpinlock(&gSchedLock, flags);
+        return;
+    }
+
+    thread->WaitObject    = (OBJECT*)0;
+    thread->WaitStatus    = status;
+    thread->WaitListHead  = (LIST_ENTRY*)0;
+    thread->AlertableWait = 0;
+
+    thread->CpuBurstTicks = 0;
+    thread->QuantaConsumed = 0;
+    thread->QuantumRemaining = 0;
+
+    if (thread->RunningCpu != KE_INVALID_CPU) {
+        // Prepare->commit race: the waiter is still running somewhere.  Its
+        // KeCommitBlockThread() will see State != ThreadWaiting and return the
+        // completed wait status without ever entering the ready queue.
+        thread->State = ThreadRunning;
+        KeReleaseSpinlock(&gSchedLock, flags);
+        return;
+    }
+
+    if (thread->Priority < PriorityHigh) {
+        thread->Priority = (THREAD_PRIORITY)(thread->Priority + 1);
+    }
+
+    KiInsertReadyLocked(thread);
+    KiPokeRunnableCpusLocked(thread);
+
+    KeReleaseSpinlock(&gSchedLock, flags);
+}
+
 void KeSchedulerTick(void) {
     KCPU* cpu = KeGetCurrentCpu();
     THREAD* cur = cpu->CurrentThread;
@@ -282,14 +554,14 @@ void KeSchedulerTick(void) {
     if (!cur) return;
 
     uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
-    cur->SchedLockFlags = flags;
+    KiArmPostSwitchUnlock(cur, flags);
 
     gSchedStats.Ticks++;
 
     int requestReschedule = 0;
 
     if (cur == idle) {
-        if (KiAnyRunnableLocked())
+        if (KiAnyRunnableForCpuLocked(cpu))
             requestReschedule = 1;
     } else {
         cur->CpuBurstTicks++;
@@ -312,7 +584,7 @@ void KeSchedulerTick(void) {
             cur->QuantumRemaining--;
 
         if (cur->QuantumRemaining == 0) {
-            if (KiAnyRunnableLocked()) {
+            if (KiAnyRunnableForCpuLocked(cpu)) {
                 gSchedStats.QuantumExpirations++;
                 requestReschedule = 1;
             } else {