replace ad hoc driver loading with a proper driver manager, new DDK, new PnP manager with device matching, asynchronous startup, resource ownership and removal, add shared PCI capability helpers, replace typed, rigid services with a proper service manager, implement proper system shutdown and restart, shutting down unloads drivers and shuts down services, add simple power button, remove heavyweight boot self-tests and SMP stress runs, add monotonic timestamps to kernel debug output, fruitless USB and PS/2 fixes, make test builds opt in.
This commit is contained in:
maksw committed
1 parent
c0ad63c27f
commit
d36f19509c
160 files changed
+6563
-7366
No files matched your search
@@ -863,218 +863,3 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
|
||||
DbgPrint("SMP: scheduler admitted %u APs\n", admitted);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
#define SMP_AP_WORKER_ROUNDS 64u
|
||||
#define SMP_AP_WORKER_TIMEOUT_US 2000000ull
|
||||
#define SMP_AP_TEST_MAGIC 0x524B575041504D53ULL // 'SMPAPWKR'
|
||||
|
||||
typedef struct SMP_AP_WORK_TEST SMP_AP_WORK_TEST;
|
||||
|
||||
typedef struct SMP_AP_WORKER_ARG {
|
||||
uint64_t Magic;
|
||||
uint32_t CpuIndex;
|
||||
uint32_t WorkerIndex;
|
||||
struct SMP_AP_WORK_TEST* Test;
|
||||
EVENT* StepEvent;
|
||||
volatile int32_t Ready;
|
||||
volatile int32_t WaitArmed;
|
||||
volatile int32_t Progress;
|
||||
volatile int32_t Done;
|
||||
volatile int32_t Failure;
|
||||
uint32_t WrongCpu;
|
||||
uint32_t WrongRunningCpu;
|
||||
} SMP_AP_WORKER_ARG;
|
||||
|
||||
struct SMP_AP_WORK_TEST {
|
||||
uint64_t Magic;
|
||||
uint32_t WorkerCount;
|
||||
volatile int32_t ReadyCount;
|
||||
volatile int32_t DoneCount;
|
||||
volatile int32_t FailureCount;
|
||||
volatile int32_t CpuHits[KE_MAX_CPUS];
|
||||
SMP_AP_WORKER_ARG Workers[KE_MAX_CPUS];
|
||||
};
|
||||
|
||||
// Implements the smp Ap Controlled Worker operation.
|
||||
static void KiSmpApControlledWorker(void* context) {
|
||||
SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context;
|
||||
if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%016llX\n", (uint64_t)arg);
|
||||
}
|
||||
|
||||
NtlAtomicExchange32(&arg->Ready, 1);
|
||||
NtlAtomicIncrement32(&arg->Test->ReadyCount);
|
||||
|
||||
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
|
||||
NtlAtomicIncrement32(&arg->WaitArmed);
|
||||
KSTATUS waitStatus = KeWaitForEvent(arg->StepEvent, 0);
|
||||
if (!KSUCCESS(waitStatus)) {
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
|
||||
KCPU* cpu = KeGetCurrentCpu();
|
||||
THREAD* self = KeGetCurrentThread();
|
||||
if (!cpu || cpu->CpuIndex != arg->CpuIndex) {
|
||||
arg->WrongCpu = cpu ? cpu->CpuIndex : KE_INVALID_CPU;
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
if (!self || self->RunningCpu != cpu->CpuIndex) {
|
||||
arg->WrongRunningCpu = self ? self->RunningCpu : KE_INVALID_CPU;
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
if (self && (self->AffinityMask & (1ULL << cpu->CpuIndex)) == 0) {
|
||||
arg->WrongCpu = cpu->CpuIndex;
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
|
||||
if (cpu->CpuIndex < KE_MAX_CPUS)
|
||||
NtlAtomicIncrement32(&arg->Test->CpuHits[cpu->CpuIndex]);
|
||||
|
||||
NtlAtomicIncrement32(&arg->Progress);
|
||||
if ((round & 3u) == 0)
|
||||
KeYield();
|
||||
}
|
||||
|
||||
if (NtlAtomicLoad32(&arg->Failure) != 0) {
|
||||
NtlAtomicIncrement32(&arg->Test->FailureCount);
|
||||
}
|
||||
|
||||
NtlAtomicExchange32(&arg->Done, 1);
|
||||
NtlAtomicIncrement32(&arg->Test->DoneCount);
|
||||
}
|
||||
|
||||
// Waits for for Worker Counter.
|
||||
static bool KiWaitForWorkerCounter(volatile int32_t* counter, int32_t expected, const char* what) {
|
||||
uint64_t deadline = KiSmpDeadlineAfter(SMP_AP_WORKER_TIMEOUT_US);
|
||||
for (;;) {
|
||||
if (NtlAtomicLoad32(counter) >= expected)
|
||||
return true;
|
||||
if (KiSmpQueryTicks() >= deadline)
|
||||
break;
|
||||
KeYield();
|
||||
}
|
||||
DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%d got=%ld\n",
|
||||
what ? what : "counter", expected, NtlAtomicLoad32(counter));
|
||||
return false;
|
||||
}
|
||||
|
||||
// Runs application Processor Scheduler Self Tests.
|
||||
KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
|
||||
if (gCpuCount <= 1) {
|
||||
DbgPrint("SMP scheduler selftest: SKIP, no APs\n");
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
DbgPrint("SMP scheduler selftest: begin AP-affine workers\n");
|
||||
|
||||
SMP_AP_WORK_TEST* test = (SMP_AP_WORK_TEST*)MmAllocatePool(sizeof(SMP_AP_WORK_TEST));
|
||||
if (!test) return NTLSTATUS_NO_MEMORY;
|
||||
memset(test, 0, sizeof(*test));
|
||||
test->Magic = SMP_AP_TEST_MAGIC;
|
||||
|
||||
uint32_t workerCount = 0;
|
||||
for (uint32_t cpuIndex = 1; cpuIndex < gCpuCount; cpuIndex++) {
|
||||
KCPU* cpu = &gCpus[cpuIndex];
|
||||
if (cpu->State != KeCpuOnline)
|
||||
continue;
|
||||
if (cpuIndex >= 64)
|
||||
continue;
|
||||
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[workerCount];
|
||||
arg->Magic = SMP_AP_TEST_MAGIC;
|
||||
arg->CpuIndex = cpuIndex;
|
||||
arg->WorkerIndex = workerCount;
|
||||
arg->Test = test;
|
||||
|
||||
KSTATUS st = KeEventCreate(EventAutoReset, 0, &arg->StepEvent);
|
||||
if (!KSUCCESS(st)) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u step event create failed status=0x%x\n",
|
||||
cpuIndex, st);
|
||||
}
|
||||
|
||||
KE_THREAD_CREATE_PARAMETERS workerParameters =
|
||||
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "selftest.smp");
|
||||
workerParameters.AffinityMask = 1ULL << cpuIndex;
|
||||
st = KeCreateKernelThread((uintptr_t)KiSmpApControlledWorker,
|
||||
(uintptr_t)arg,
|
||||
&workerParameters,
|
||||
(THREAD**)0);
|
||||
if (!KSUCCESS(st)) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u worker create failed status=0x%x\n",
|
||||
cpuIndex, st);
|
||||
}
|
||||
|
||||
workerCount++;
|
||||
}
|
||||
test->WorkerCount = workerCount;
|
||||
|
||||
if (workerCount == 0) {
|
||||
MmFreePool(test);
|
||||
DbgPrint("SMP scheduler selftest: SKIP, no online AP workers\n");
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (!KiWaitForWorkerCounter(&test->ReadyCount, (int32_t)workerCount, "AP worker ready")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers failed to become ready ready=%ld workers=%u\n",
|
||||
test->ReadyCount, workerCount);
|
||||
}
|
||||
|
||||
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
|
||||
for (uint32_t w = 0; w < workerCount; w++) {
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
|
||||
if (!KiWaitForWorkerCounter(&arg->WaitArmed, (int32_t)(round + 1), "AP worker wait-armed")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u did not arm wait for round %u armed=%ld progress=%ld\n",
|
||||
arg->CpuIndex, round, arg->WaitArmed, arg->Progress);
|
||||
}
|
||||
KeSignalEvent(arg->StepEvent);
|
||||
}
|
||||
|
||||
for (uint32_t w = 0; w < workerCount; w++) {
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
|
||||
if (!KiWaitForWorkerCounter(&arg->Progress, (int32_t)(round + 1), "AP worker progress")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u no progress round=%u progress=%ld failure=%ld wrongCpu=%u wrongRunningCpu=%u\n",
|
||||
arg->CpuIndex, round, arg->Progress, arg->Failure,
|
||||
arg->WrongCpu, arg->WrongRunningCpu);
|
||||
}
|
||||
if (NtlAtomicLoad32(&arg->Failure) != 0) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u worker failed round=%u wrongCpu=%u wrongRunningCpu=%u progress=%ld\n",
|
||||
arg->CpuIndex, round, arg->WrongCpu, arg->WrongRunningCpu, arg->Progress);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!KiWaitForWorkerCounter(&test->DoneCount, (int32_t)workerCount, "AP worker done")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%d workers=%u\n",
|
||||
test->DoneCount, workerCount);
|
||||
}
|
||||
|
||||
if (NtlAtomicLoad32(&test->FailureCount) != 0) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%d\n",
|
||||
test->FailureCount);
|
||||
}
|
||||
|
||||
for (uint32_t w = 0; w < workerCount; w++) {
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
|
||||
int32_t hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0;
|
||||
if (hits <= 0) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u recorded no CPU hits\n", arg->CpuIndex);
|
||||
}
|
||||
if (arg->StepEvent)
|
||||
ObDereferenceObject(&arg->StepEvent->Header);
|
||||
}
|
||||
|
||||
DbgPrint("SMP scheduler selftest: PASS workers=%u rounds=%u\n",
|
||||
workerCount, SMP_AP_WORKER_ROUNDS);
|
||||
MmFreePool(test);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
Reference in new issue
Block a user