replace ad hoc driver loading with a proper driver manager, new DDK, new PnP manager with device matching, asynchronous startup, resource ownership and removal, add shared PCI capability helpers, replace typed, rigid services with a proper service manager, implement proper system shutdown and restart, shutting down unloads drivers and shuts down services, add simple power button, remove heavyweight boot self-tests and SMP stress runs, add monotonic timestamps to kernel debug output, fruitless USB and PS/2 fixes, make test builds opt in.
This commit is contained in:
maksw committed
1 parent
c0ad63c27f
commit
d36f19509c
160 files changed
+6563
-7366
No files matched your search
@@ -0,0 +1,422 @@
|
||||
// Strict drivers.ini v2 parsing, validation and dependency evaluation.
|
||||
|
||||
#include <kernel/dm_config.h>
|
||||
#include <string.h>
|
||||
|
||||
#define DM_DRIVER_IMAGE_SET 1u
|
||||
#define DM_DRIVER_START_SET 2u
|
||||
#define DM_DRIVER_CLASS_SET 4u
|
||||
#define DM_DRIVER_CRITICAL_SET 8u
|
||||
#define DM_DRIVER_REQUIRES_SET 16u
|
||||
#define DM_DRIVER_AFTER_SET 32u
|
||||
#define DM_CLASS_REQUIRES_SET 1u
|
||||
#define DM_CLASS_AFTER_SET 2u
|
||||
|
||||
static bool DmFail(char* error, size_t capacity, const char* message) {
|
||||
if (error && capacity) {
|
||||
size_t i = 0;
|
||||
while (message[i] && i + 1 < capacity) {
|
||||
error[i] = message[i];
|
||||
++i;
|
||||
}
|
||||
error[i] = '\0';
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool DmSpace(char c) {
|
||||
return c == ' ' || c == '\t' || c == '\r';
|
||||
}
|
||||
|
||||
static void DmTrim(const char** begin, const char** end) {
|
||||
while (*begin < *end && DmSpace(**begin)) ++*begin;
|
||||
while (*end > *begin && DmSpace((*end)[-1])) --*end;
|
||||
}
|
||||
|
||||
static bool DmEqual(const char* begin, const char* end, const char* value) {
|
||||
while (begin < end && *value && *begin == *value) {
|
||||
++begin;
|
||||
++value;
|
||||
}
|
||||
return begin == end && !*value;
|
||||
}
|
||||
|
||||
static bool DmIdentifier(const char* begin, const char* end) {
|
||||
if (begin == end || (size_t)(end - begin) >= DM_ID_MAX) return false;
|
||||
for (const char* p = begin; p < end; ++p) {
|
||||
char c = *p;
|
||||
if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
||||
(c >= '0' && c <= '9') || c == '.' || c == '-' || c == '_')) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// PE import module names are compared without ASCII case distinctions.
|
||||
static bool DmImageEqual(const char* a, const char* b) {
|
||||
while (*a && *b) {
|
||||
char x = *a++;
|
||||
char y = *b++;
|
||||
if (x >= 'A' && x <= 'Z') x = (char)(x + ('a' - 'A'));
|
||||
if (y >= 'A' && y <= 'Z') y = (char)(y + ('a' - 'A'));
|
||||
if (x != y) return false;
|
||||
}
|
||||
return !*a && !*b;
|
||||
}
|
||||
|
||||
static bool DmCopy(char* out, size_t capacity, const char* begin, const char* end) {
|
||||
size_t size = (size_t)(end - begin);
|
||||
if (!size || size >= capacity) return false;
|
||||
memcpy(out, begin, size);
|
||||
out[size] = '\0';
|
||||
return true;
|
||||
}
|
||||
|
||||
static int DmFindDriver(const DM_CONFIG* config, const char* id) {
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i)
|
||||
if (!strcmp(config->Drivers[i].Id, id)) return (int)i;
|
||||
return -1;
|
||||
}
|
||||
|
||||
static int DmFindClass(const DM_CONFIG* config, const char* id) {
|
||||
for (uint32_t i = 0; i < config->ClassCount; ++i)
|
||||
if (!strcmp(config->Classes[i].Id, id)) return (int)i;
|
||||
return -1;
|
||||
}
|
||||
|
||||
static bool DmParseEdges(DM_EDGES* edges, const char* begin, const char* end,
|
||||
DM_DEPENDENCY_KIND kind) {
|
||||
if (begin == end) return true;
|
||||
while (begin < end) {
|
||||
const char* finish = begin;
|
||||
while (finish < end && *finish != ',') ++finish;
|
||||
const char* tokenEnd = finish;
|
||||
DmTrim(&begin, &tokenEnd);
|
||||
if (!DmIdentifier(begin, tokenEnd) || edges->Count == DM_MAX_EDGES) return false;
|
||||
for (uint32_t i = 0; i < edges->Count; ++i) {
|
||||
if (edges->Items[i].Kind == kind &&
|
||||
DmEqual(begin, tokenEnd, edges->Items[i].Id)) return false;
|
||||
}
|
||||
DM_DEPENDENCY* edge = &edges->Items[edges->Count++];
|
||||
edge->Kind = kind;
|
||||
if (!DmCopy(edge->Id, sizeof(edge->Id), begin, tokenEnd)) return false;
|
||||
if (finish == end) return true;
|
||||
begin = finish + 1;
|
||||
DmTrim(&begin, &end);
|
||||
if (begin == end) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Validates references and boot-phase satisfiability without flattening class alternatives.
|
||||
static bool DmValidateEdges(const DM_CONFIG* config, const DM_EDGES* edges,
|
||||
const DM_DRIVER_CONFIG* source, bool required) {
|
||||
for (uint32_t i = 0; i < edges->Count; ++i) {
|
||||
const DM_DEPENDENCY* edge = &edges->Items[i];
|
||||
if (edge->Kind == DmDependencyDriver) {
|
||||
int target = DmFindDriver(config, edge->Id);
|
||||
if (target < 0) return false;
|
||||
if (source && source->Phase == DmStartBoot &&
|
||||
config->Drivers[target].Phase != DmStartBoot) return false;
|
||||
} else {
|
||||
if (DmFindClass(config, edge->Id) < 0) return false;
|
||||
uint32_t bootProviders = 0;
|
||||
for (uint32_t j = 0; j < config->DriverCount; ++j) {
|
||||
if (strcmp(config->Drivers[j].Class, edge->Id)) continue;
|
||||
if (config->Drivers[j].Phase == DmStartBoot) ++bootProviders;
|
||||
}
|
||||
if (required && source && source->Phase == DmStartBoot && !bootProviders)
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool DmEdgesOverlap(const DM_EDGES* requires, const DM_EDGES* after) {
|
||||
for (uint32_t i = 0; i < requires->Count; ++i)
|
||||
for (uint32_t j = 0; j < after->Count; ++j)
|
||||
if (requires->Items[i].Kind == after->Items[j].Kind &&
|
||||
!strcmp(requires->Items[i].Id, after->Items[j].Id)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool DmVisitClass(uint32_t index, uint32_t count,
|
||||
uint8_t color[DM_MAX_CLASSES],
|
||||
uint8_t graph[DM_MAX_CLASSES][DM_MAX_CLASSES]) {
|
||||
color[index] = 1;
|
||||
for (uint32_t j = 0; j < count; ++j) {
|
||||
if (!graph[index][j]) continue;
|
||||
if (color[j] == 1 ||
|
||||
(color[j] == 0 && !DmVisitClass(j, count, color, graph))) return false;
|
||||
}
|
||||
color[index] = 2;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DmConfigParse(DM_CONFIG* config, const char* bytes, size_t size,
|
||||
char* error, size_t errorCapacity) {
|
||||
if (!config || !bytes || size == 0 || size > DM_CONFIG_MAX_BYTES)
|
||||
return DmFail(error, errorCapacity, "invalid configuration size");
|
||||
memset(config, 0, sizeof(*config));
|
||||
DM_DRIVER_CONFIG* driver = NULL;
|
||||
DM_CLASS_CONFIG* cls = NULL;
|
||||
bool seenDriverSection = false;
|
||||
size_t pos = 0;
|
||||
while (pos < size) {
|
||||
size_t line = pos;
|
||||
while (pos < size && bytes[pos] != '\n') ++pos;
|
||||
const char* begin = bytes + line;
|
||||
const char* end = bytes + pos;
|
||||
if (pos < size) ++pos;
|
||||
DmTrim(&begin, &end);
|
||||
if (begin == end || *begin == ';' || *begin == '#') continue;
|
||||
if (*begin == '[') {
|
||||
if (end[-1] != ']' || (size_t)(end - begin) < 4)
|
||||
return DmFail(error, errorCapacity, "invalid section");
|
||||
const char* name = begin + 1;
|
||||
const char* nameEnd = end - 1;
|
||||
driver = NULL;
|
||||
cls = NULL;
|
||||
if ((size_t)(nameEnd - name) > 7 && !memcmp(name, "driver.", 7)) {
|
||||
name += 7;
|
||||
if (!DmIdentifier(name, nameEnd) || config->DriverCount == DM_MAX_DRIVERS)
|
||||
return DmFail(error, errorCapacity, "invalid driver id or driver limit");
|
||||
driver = &config->Drivers[config->DriverCount++];
|
||||
seenDriverSection = true;
|
||||
if (!DmCopy(driver->Id, sizeof(driver->Id), name, nameEnd)) return false;
|
||||
for (uint32_t i = 0; i + 1 < config->DriverCount; ++i)
|
||||
if (!strcmp(config->Drivers[i].Id, driver->Id))
|
||||
return DmFail(error, errorCapacity, "duplicate driver id");
|
||||
} else if ((size_t)(nameEnd - name) > 6 && !memcmp(name, "class.", 6)) {
|
||||
name += 6;
|
||||
if (!DmIdentifier(name, nameEnd) || config->ClassCount == DM_MAX_CLASSES)
|
||||
return DmFail(error, errorCapacity, "invalid class id or class limit");
|
||||
if (seenDriverSection)
|
||||
return DmFail(error, errorCapacity, "class declared after driver");
|
||||
cls = &config->Classes[config->ClassCount++];
|
||||
if (!DmCopy(cls->Id, sizeof(cls->Id), name, nameEnd)) return false;
|
||||
for (uint32_t i = 0; i + 1 < config->ClassCount; ++i)
|
||||
if (!strcmp(config->Classes[i].Id, cls->Id))
|
||||
return DmFail(error, errorCapacity, "duplicate class id");
|
||||
} else return DmFail(error, errorCapacity, "unknown section");
|
||||
continue;
|
||||
}
|
||||
if (!driver && !cls) return DmFail(error, errorCapacity, "key outside section");
|
||||
const char* equal = begin;
|
||||
while (equal < end && *equal != '=') ++equal;
|
||||
if (equal == end) return DmFail(error, errorCapacity, "missing equals");
|
||||
const char* keyEnd = equal;
|
||||
const char* value = equal + 1;
|
||||
DmTrim(&begin, &keyEnd);
|
||||
DmTrim(&value, &end);
|
||||
for (const char* p = value; p < end; ++p) {
|
||||
if (*p == ';' || *p == '#') {
|
||||
end = p;
|
||||
DmTrim(&value, &end);
|
||||
break;
|
||||
}
|
||||
}
|
||||
uint8_t mask = 0;
|
||||
if (driver) {
|
||||
if (DmEqual(begin, keyEnd, "image")) {
|
||||
mask = DM_DRIVER_IMAGE_SET;
|
||||
if (!DmCopy(driver->Image, sizeof(driver->Image), value, end))
|
||||
return DmFail(error, errorCapacity, "invalid image");
|
||||
for (const char* p = value; p < end; ++p)
|
||||
if (!((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') ||
|
||||
(*p >= '0' && *p <= '9') ||
|
||||
*p == '-' || *p == '_' || *p == '.'))
|
||||
return DmFail(error, errorCapacity, "invalid image filename");
|
||||
if (end - value < 5 || memcmp(end - 4, ".sys", 4))
|
||||
return DmFail(error, errorCapacity, "image must be a .sys file");
|
||||
} else if (DmEqual(begin, keyEnd, "start")) {
|
||||
mask = DM_DRIVER_START_SET;
|
||||
if (DmEqual(value, end, "boot")) driver->Phase = DmStartBoot;
|
||||
else if (DmEqual(value, end, "runtime")) driver->Phase = DmStartRuntime;
|
||||
else if (DmEqual(value, end, "disabled")) {
|
||||
driver->Phase = DmStartDisabled;
|
||||
driver->State = DmRunDisabled;
|
||||
}
|
||||
else return DmFail(error, errorCapacity, "invalid start phase");
|
||||
} else if (DmEqual(begin, keyEnd, "class")) {
|
||||
mask = DM_DRIVER_CLASS_SET;
|
||||
if (value != end && (!DmIdentifier(value, end) ||
|
||||
!DmCopy(driver->Class, sizeof(driver->Class), value, end)))
|
||||
return DmFail(error, errorCapacity, "invalid class");
|
||||
} else if (DmEqual(begin, keyEnd, "requires")) {
|
||||
mask = DM_DRIVER_REQUIRES_SET;
|
||||
if (!DmParseEdges(&driver->Requires, value, end, DmDependencyDriver))
|
||||
return DmFail(error, errorCapacity, "invalid requires");
|
||||
} else if (DmEqual(begin, keyEnd, "after")) {
|
||||
mask = DM_DRIVER_AFTER_SET;
|
||||
if (!DmParseEdges(&driver->After, value, end, DmDependencyDriver))
|
||||
return DmFail(error, errorCapacity, "invalid after");
|
||||
} else if (DmEqual(begin, keyEnd, "critical")) {
|
||||
mask = DM_DRIVER_CRITICAL_SET;
|
||||
if (value == end) driver->Critical = false;
|
||||
else if (DmEqual(value, end, "true")) driver->Critical = true;
|
||||
else if (DmEqual(value, end, "false")) driver->Critical = false;
|
||||
else return DmFail(error, errorCapacity, "invalid critical value");
|
||||
} else return DmFail(error, errorCapacity, "unknown driver key");
|
||||
if (driver->Present & mask) return DmFail(error, errorCapacity, "duplicate driver key");
|
||||
driver->Present |= mask;
|
||||
} else {
|
||||
if (DmEqual(begin, keyEnd, "requires")) {
|
||||
mask = DM_CLASS_REQUIRES_SET;
|
||||
if (!DmParseEdges(&cls->Requires, value, end, DmDependencyClass))
|
||||
return DmFail(error, errorCapacity, "invalid class requires");
|
||||
} else if (DmEqual(begin, keyEnd, "after")) {
|
||||
mask = DM_CLASS_AFTER_SET;
|
||||
if (!DmParseEdges(&cls->After, value, end, DmDependencyClass))
|
||||
return DmFail(error, errorCapacity, "invalid class after");
|
||||
} else return DmFail(error, errorCapacity, "unknown class key");
|
||||
if (cls->Present & mask) return DmFail(error, errorCapacity, "duplicate class key");
|
||||
cls->Present |= mask;
|
||||
}
|
||||
}
|
||||
if (!config->DriverCount) return DmFail(error, errorCapacity, "no drivers");
|
||||
// Class graph is checked independently even when a class has no providers.
|
||||
uint8_t classGraph[DM_MAX_CLASSES][DM_MAX_CLASSES] = {{0}};
|
||||
for (uint32_t i = 0; i < config->ClassCount; ++i) {
|
||||
const DM_CLASS_CONFIG* cls = &config->Classes[i];
|
||||
if (DmEdgesOverlap(&cls->Requires, &cls->After))
|
||||
return DmFail(error, errorCapacity, "conflicting class dependency");
|
||||
const DM_EDGES* lists[] = { &cls->Requires, &cls->After };
|
||||
for (uint32_t k = 0; k < 2; ++k)
|
||||
for (uint32_t j = 0; j < lists[k]->Count; ++j) {
|
||||
int index = DmFindClass(config, lists[k]->Items[j].Id);
|
||||
if (index < 0) return DmFail(error, errorCapacity, "unknown class dependency");
|
||||
classGraph[i][index] = 1;
|
||||
}
|
||||
}
|
||||
uint8_t classColors[DM_MAX_CLASSES] = {0};
|
||||
for (uint32_t i = 0; i < config->ClassCount; ++i)
|
||||
if (!classColors[i] && !DmVisitClass(i, config->ClassCount,
|
||||
classColors, classGraph))
|
||||
return DmFail(error, errorCapacity, "class dependency cycle");
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i) {
|
||||
DM_DRIVER_CONFIG* d = &config->Drivers[i];
|
||||
if ((d->Present & (DM_DRIVER_IMAGE_SET | DM_DRIVER_START_SET)) !=
|
||||
(DM_DRIVER_IMAGE_SET | DM_DRIVER_START_SET))
|
||||
return DmFail(error, errorCapacity, "driver needs image and start");
|
||||
for (uint32_t j = 0; j < i; ++j)
|
||||
if (d->Phase != DmStartDisabled &&
|
||||
config->Drivers[j].Phase != DmStartDisabled &&
|
||||
DmImageEqual(config->Drivers[j].Image, d->Image))
|
||||
return DmFail(error, errorCapacity, "duplicate image");
|
||||
if (d->Class[0] && DmFindClass(config, d->Class) < 0)
|
||||
return DmFail(error, errorCapacity, "undeclared driver class");
|
||||
if (DmEdgesOverlap(&d->Requires, &d->After))
|
||||
return DmFail(error, errorCapacity, "conflicting driver dependency");
|
||||
if (!DmValidateEdges(config, &d->Requires, d, true) ||
|
||||
!DmValidateEdges(config, &d->After, d, false))
|
||||
return DmFail(error, errorCapacity, "invalid driver dependency or phase");
|
||||
if (d->Class[0] && d->Phase != DmStartDisabled) {
|
||||
DM_CLASS_CONFIG* c = &config->Classes[DmFindClass(config, d->Class)];
|
||||
if (!DmValidateEdges(config, &c->Requires, d, true) ||
|
||||
!DmValidateEdges(config, &c->After, d, false))
|
||||
return DmFail(error, errorCapacity, "invalid class dependency or phase");
|
||||
}
|
||||
}
|
||||
// An optimistic dry run models requires= as an OR over class providers and
|
||||
// after= as a settlement barrier. It detects real combined deadlocks without
|
||||
// rejecting a class merely because one alternative provider depends on it.
|
||||
for (uint32_t phase = DmStartBoot; phase <= DmStartRuntime; ++phase) {
|
||||
config->ActivePhase = (DM_START_PHASE)phase;
|
||||
for (;;) {
|
||||
bool progress = false;
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i) {
|
||||
DM_DRIVER_CONFIG* d = &config->Drivers[i];
|
||||
if (d->Phase != phase || d->State != DmRunWaiting) continue;
|
||||
if (DmDriverCanStart(config, i)) {
|
||||
d->State = DmRunStarted;
|
||||
progress = true;
|
||||
} else if (DmDriverIsBlocked(config, i)) {
|
||||
d->State = DmRunBlocked;
|
||||
progress = true;
|
||||
}
|
||||
}
|
||||
if (!progress) break;
|
||||
}
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i)
|
||||
if (config->Drivers[i].Phase == phase &&
|
||||
config->Drivers[i].State == DmRunWaiting)
|
||||
return DmFail(error, errorCapacity, "unschedulable dependency cycle");
|
||||
}
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i)
|
||||
if (config->Drivers[i].Phase != DmStartDisabled)
|
||||
config->Drivers[i].State = DmRunWaiting;
|
||||
config->ActivePhase = DmStartBoot;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DmClassSettled(const DM_CONFIG* config, const char* classId) {
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i) {
|
||||
const DM_DRIVER_CONFIG* driver = &config->Drivers[i];
|
||||
if (!strcmp(driver->Class, classId) &&
|
||||
(config->ActivePhase == DmStartRuntime || driver->Phase == DmStartBoot) &&
|
||||
driver->State == DmRunWaiting) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DmClassAvailable(const DM_CONFIG* config, const char* classId) {
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i) {
|
||||
const DM_DRIVER_CONFIG* driver = &config->Drivers[i];
|
||||
if (!strcmp(driver->Class, classId) && driver->State == DmRunStarted) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool DmEdgesReady(const DM_CONFIG* config, const DM_EDGES* edges, bool required) {
|
||||
for (uint32_t i = 0; i < edges->Count; ++i) {
|
||||
const DM_DEPENDENCY* edge = &edges->Items[i];
|
||||
if (edge->Kind == DmDependencyClass) {
|
||||
if (required ? !DmClassAvailable(config, edge->Id) :
|
||||
!DmClassSettled(config, edge->Id)) return false;
|
||||
} else {
|
||||
int index = DmFindDriver(config, edge->Id);
|
||||
if (index < 0) return false;
|
||||
DM_RUN_STATE state = config->Drivers[index].State;
|
||||
if (required ? state != DmRunStarted : state == DmRunWaiting) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DmDriverCanStart(const DM_CONFIG* config, uint32_t index) {
|
||||
if (!config || index >= config->DriverCount) return false;
|
||||
const DM_DRIVER_CONFIG* driver = &config->Drivers[index];
|
||||
if (driver->State != DmRunWaiting ||
|
||||
!DmEdgesReady(config, &driver->Requires, true) ||
|
||||
!DmEdgesReady(config, &driver->After, false)) return false;
|
||||
if (!driver->Class[0]) return true;
|
||||
const DM_CLASS_CONFIG* cls = &config->Classes[DmFindClass(config, driver->Class)];
|
||||
return DmEdgesReady(config, &cls->Requires, true) &&
|
||||
DmEdgesReady(config, &cls->After, false);
|
||||
}
|
||||
|
||||
static bool DmEdgesImpossible(const DM_CONFIG* config, const DM_EDGES* edges) {
|
||||
for (uint32_t i = 0; i < edges->Count; ++i) {
|
||||
const DM_DEPENDENCY* edge = &edges->Items[i];
|
||||
if (edge->Kind == DmDependencyClass) {
|
||||
if (DmClassSettled(config, edge->Id) &&
|
||||
!DmClassAvailable(config, edge->Id)) return true;
|
||||
} else {
|
||||
int index = DmFindDriver(config, edge->Id);
|
||||
if (index < 0 || config->Drivers[index].State == DmRunFailed ||
|
||||
config->Drivers[index].State == DmRunBlocked ||
|
||||
config->Drivers[index].State == DmRunDisabled) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool DmDriverIsBlocked(const DM_CONFIG* config, uint32_t index) {
|
||||
if (!config || index >= config->DriverCount) return false;
|
||||
const DM_DRIVER_CONFIG* driver = &config->Drivers[index];
|
||||
if (driver->State != DmRunWaiting) return false;
|
||||
if (DmEdgesImpossible(config, &driver->Requires)) return true;
|
||||
if (!driver->Class[0]) return false;
|
||||
return DmEdgesImpossible(config,
|
||||
&config->Classes[DmFindClass(config, driver->Class)].Requires);
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
// Driver module identity and versioned declarative module exports.
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stddef.h>
|
||||
#include <string.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddk/module.h>
|
||||
#include <kernel/driver_module.h>
|
||||
#include <kernel/ldr_image.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/io_pnp.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
|
||||
#define DM_NAME_MAX 64u
|
||||
|
||||
typedef enum DM_MODULE_STATE {
|
||||
DmModuleStarting,
|
||||
DmModuleStarted,
|
||||
DmModuleFailed,
|
||||
DmModuleStopping,
|
||||
DmModuleStopped,
|
||||
} DM_MODULE_STATE;
|
||||
|
||||
struct NTL_MODULE {
|
||||
struct NTL_MODULE* Next;
|
||||
char Name[DM_NAME_MAX];
|
||||
uintptr_t ImageBase;
|
||||
DM_MODULE_STATE State;
|
||||
NTL_MODULE_PREPARE_SHUTDOWN PrepareShutdown;
|
||||
NTL_MODULE_COMMIT_SHUTDOWN CommitShutdown;
|
||||
};
|
||||
|
||||
static NTL_MODULE* gDriverModules;
|
||||
|
||||
static bool DmImageContains(uintptr_t base, size_t imageSize, uintptr_t ptr, size_t bytes) {
|
||||
return ptr >= base && ptr - base <= imageSize && bytes <= imageSize - (ptr - base);
|
||||
}
|
||||
|
||||
__declspec(dllexport) const char* NtlModuleGetName(const NTL_MODULE* module) {
|
||||
return module ? module->Name : (const char*)0;
|
||||
}
|
||||
|
||||
static bool DmValidImageString(uintptr_t base, size_t size, const char* text,
|
||||
size_t maxLength) {
|
||||
if (!DmImageContains(base, size, (uintptr_t)text, 1u)) return false;
|
||||
for (size_t i = 0; i < maxLength; ++i) {
|
||||
if (!DmImageContains(base, size, (uintptr_t)text + i, 1u)) return false;
|
||||
if (!text[i])
|
||||
return i && SysSvIsUtf8Text(SysSvFromParts(text, i));
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool DmValidIoDriver(uintptr_t imageBase, size_t imageSize, const DRIVER_OBJECT* driver) {
|
||||
if (!DmImageContains(imageBase, imageSize, (uintptr_t)driver, sizeof(*driver)) ||
|
||||
driver->Header.Type != ObjectTypeDriver ||
|
||||
driver->Header.LifetimeState != ObjectAlive ||
|
||||
!driver->Name.Length ||
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)driver->Name.Data,
|
||||
driver->Name.Length) ||
|
||||
!SysSvIsUtf8Text(driver->Name) ||
|
||||
(driver->DriverAddDevice &&
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)driver->DriverAddDevice, 1u)) ||
|
||||
(driver->DriverUnload &&
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)driver->DriverUnload, 1u)))
|
||||
return false;
|
||||
|
||||
for (uint32_t i = 0; i < IRP_MJ_MAXIMUM_FUNCTION; ++i) {
|
||||
if (driver->MajorFunction[i] &&
|
||||
!DmImageContains(imageBase, imageSize,
|
||||
(uintptr_t)driver->MajorFunction[i], 1u))
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!driver->IdTable) return true;
|
||||
if (!driver->DriverAddDevice ||
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)driver->IdTable,
|
||||
sizeof(*driver->IdTable)) ||
|
||||
!driver->IdTable->Count || driver->IdTable->Count > 32u ||
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)driver->IdTable->Ids,
|
||||
driver->IdTable->Count * sizeof(IO_DEVICE_ID)))
|
||||
return false;
|
||||
|
||||
for (size_t i = 0; i < driver->IdTable->Count; ++i) {
|
||||
const IO_DEVICE_ID* id = &driver->IdTable->Ids[i];
|
||||
if ((id->BusType != IoBusPci && id->BusType != IoBusUsb &&
|
||||
id->BusType != IoBusUnknown) ||
|
||||
!id->Id.Length ||
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)id->Id.Data,
|
||||
id->Id.Length) ||
|
||||
!SysSvIsUtf8Text(id->Id))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase) {
|
||||
if (!name || !*name || !imageBase)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
size_t length = 0;
|
||||
while (length < DM_NAME_MAX && name[length])
|
||||
length++;
|
||||
if (!length || length == DM_NAME_MAX)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
for (NTL_MODULE* it = gDriverModules; it; it = it->Next) {
|
||||
if (strcmp(it->Name, name) == 0 || it->ImageBase == imageBase)
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
|
||||
size_t imageSize = LdrGetImageSize(imageBase);
|
||||
if (!imageSize)
|
||||
return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
|
||||
const NTL_MODULE_EXPORT* manifest =
|
||||
(const NTL_MODULE_EXPORT*)LdrGetImageExport(imageBase, NTL_DDK_MODULE_EXPORT_NAME);
|
||||
NTL_MODULE_INITIALIZE initialize = (NTL_MODULE_INITIALIZE)0;
|
||||
|
||||
if (!manifest) return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
{
|
||||
if (!DmImageContains(imageBase, imageSize, (uintptr_t)manifest, sizeof(*manifest)) ||
|
||||
manifest->Signature != NTL_DDK_MODULE_SIGNATURE ||
|
||||
manifest->Version != NTL_DDK_MODULE_ABI_VERSION ||
|
||||
manifest->Size != sizeof(*manifest) || manifest->Flags != 0u ||
|
||||
manifest->Reserved != 0u || manifest->Reserved2 != 0u ||
|
||||
(manifest->Initialize && !DmImageContains(imageBase, imageSize,
|
||||
(uintptr_t)manifest->Initialize, 1u)) ||
|
||||
(manifest->PrepareShutdown && !DmImageContains(imageBase, imageSize,
|
||||
(uintptr_t)manifest->PrepareShutdown, 1u)) ||
|
||||
(manifest->CommitShutdown && !DmImageContains(imageBase, imageSize,
|
||||
(uintptr_t)manifest->CommitShutdown, 1u)))
|
||||
return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
if ((manifest->DeviceCount && !manifest->Devices) ||
|
||||
manifest->DeviceCount > 64u ||
|
||||
(manifest->DeviceCount &&
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)manifest->Devices,
|
||||
(size_t)manifest->DeviceCount * sizeof(NTL_DEVICE_DESC))))
|
||||
return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
for (uint32_t i = 0; i < manifest->DeviceCount; ++i) {
|
||||
const NTL_DEVICE_DESC* d = &manifest->Devices[i];
|
||||
if (d->Size != sizeof(*d) || d->Flags || !d->Name || !d->Start ||
|
||||
!DmValidImageString(imageBase, imageSize, d->Name, 64u) ||
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)d->Start, 1u) ||
|
||||
(d->Stop && !DmImageContains(imageBase, imageSize, (uintptr_t)d->Stop, 1u)) ||
|
||||
(d->CancelStart && !DmImageContains(imageBase, imageSize,
|
||||
(uintptr_t)d->CancelStart, 1u)) ||
|
||||
!d->MatchCount || d->MatchCount > 32u ||
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)d->Matches,
|
||||
(size_t)d->MatchCount * sizeof(NTL_DEVICE_MATCH)))
|
||||
return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
for (uint32_t j = 0; j < d->MatchCount; ++j) {
|
||||
const NTL_DEVICE_MATCH* m = &d->Matches[j];
|
||||
if ((m->Bus != NtlBusPci && m->Bus != NtlBusUsb) ||
|
||||
!DmValidImageString(imageBase, imageSize, m->Id, 96u))
|
||||
return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
}
|
||||
}
|
||||
if ((manifest->IoDriverCount && !manifest->IoDrivers) ||
|
||||
manifest->IoDriverCount > 64u ||
|
||||
(manifest->IoDriverCount &&
|
||||
!DmImageContains(imageBase, imageSize, (uintptr_t)manifest->IoDrivers,
|
||||
(size_t)manifest->IoDriverCount * sizeof(*manifest->IoDrivers))))
|
||||
return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
for (uint32_t i = 0; i < manifest->IoDriverCount; ++i) {
|
||||
if (!DmImageContains(imageBase, imageSize,
|
||||
(uintptr_t)manifest->IoDrivers[i], sizeof(DRIVER_OBJECT)))
|
||||
return NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
}
|
||||
initialize = manifest->Initialize;
|
||||
}
|
||||
|
||||
NTL_MODULE* module = (NTL_MODULE*)MmAllocatePool(sizeof(*module));
|
||||
if (!module)
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
|
||||
memset(module, 0, sizeof(*module));
|
||||
memcpy(module->Name, name, length);
|
||||
module->ImageBase = imageBase;
|
||||
module->State = DmModuleStarting;
|
||||
module->PrepareShutdown = manifest->PrepareShutdown;
|
||||
module->CommitShutdown = manifest->CommitShutdown;
|
||||
module->Next = gDriverModules;
|
||||
gDriverModules = module;
|
||||
|
||||
KSTATUS status = initialize ? initialize(module) : NTLSTATUS_SUCCESS;
|
||||
if (!KSUCCESS(status) && !KFAILED(status))
|
||||
status = NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
if (KSUCCESS(status) && manifest) {
|
||||
for (uint32_t i = 0; i < manifest->IoDriverCount; ++i) {
|
||||
if (!DmValidIoDriver(imageBase, imageSize, manifest->IoDrivers[i])) {
|
||||
status = NTLSTATUS_INVALID_IMAGE_FORMAT;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (KSUCCESS(status))
|
||||
status = IoPnpPublishModule(module, manifest->Devices, manifest->DeviceCount,
|
||||
manifest->IoDrivers, manifest->IoDriverCount);
|
||||
module->State = KSUCCESS(status) ? DmModuleStarted : DmModuleFailed;
|
||||
return status;
|
||||
}
|
||||
|
||||
KSTATUS DmPrepareShutdown(void) {
|
||||
KSTATUS firstError = NTLSTATUS_SUCCESS;
|
||||
for (NTL_MODULE* module = gDriverModules; module; module = module->Next) {
|
||||
if (module->State != DmModuleStarted) continue;
|
||||
module->State = DmModuleStopping;
|
||||
if (!module->PrepareShutdown) continue;
|
||||
KSTATUS st = module->PrepareShutdown(module);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("shutdown: prepare %s failed 0x%x\n", module->Name, st);
|
||||
if (KSUCCESS(firstError)) firstError = st;
|
||||
}
|
||||
}
|
||||
return firstError;
|
||||
}
|
||||
|
||||
void DmCommitShutdown(void) {
|
||||
for (NTL_MODULE* module = gDriverModules; module; module = module->Next) {
|
||||
if (module->State != DmModuleStopping) continue;
|
||||
if (module->CommitShutdown) module->CommitShutdown(module);
|
||||
module->State = DmModuleStopped;
|
||||
}
|
||||
}
|
||||
+77
-12
@@ -5,6 +5,7 @@
|
||||
#include <kernel/hal/hal_api.h>
|
||||
#include <kernel/hal/hal_timer.h>
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <intrin.h>
|
||||
|
||||
|
||||
@@ -34,6 +35,12 @@ static volatile uint32_t* gIoapic = (volatile uint32_t*)0;
|
||||
static uint32_t gBspLapicId = 0;
|
||||
static uint8_t gX2ApicSupported = 0;
|
||||
static uint8_t gX2ApicEnabled = 0;
|
||||
static uint32_t gIoapicGsiBase = 0;
|
||||
static uint32_t gIoapicRedirectionCount = 0;
|
||||
static uint32_t gIsaGsi[16];
|
||||
static uint16_t gIsaFlags[16];
|
||||
static uint16_t gSciIrq = UINT16_MAX;
|
||||
static bool gIsaOverridesInitialized = false;
|
||||
#define IOAPIC_IDX 0x00
|
||||
#define IOAPIC_DATA 0x10
|
||||
|
||||
@@ -188,29 +195,86 @@ KSTATUS HalApicComposeMsi(uint8_t vector, HAL_MSI_MESSAGE* messageOut) {
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Initializes the identity ISA map before processing MADT source overrides.
|
||||
static void HalApicInitializeIsaMap(void) {
|
||||
if (gIsaOverridesInitialized) return;
|
||||
for (uint32_t irq = 0; irq < 16; ++irq) {
|
||||
gIsaGsi[irq] = irq;
|
||||
gIsaFlags[irq] = 0;
|
||||
}
|
||||
gIsaOverridesInitialized = true;
|
||||
}
|
||||
|
||||
// Records the global-system-interrupt base of the currently supported IOAPIC.
|
||||
void HalApicSetGsiBase(uint32_t gsiBase) { gIoapicGsiBase = gsiBase; }
|
||||
|
||||
// Preserves MADT interrupt-source overrides for legacy ISA devices.
|
||||
void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags) {
|
||||
if (irq >= 16) return;
|
||||
HalApicInitializeIsaMap();
|
||||
gIsaGsi[irq] = gsi;
|
||||
gIsaFlags[irq] = flags;
|
||||
}
|
||||
|
||||
// Marks the FADT SCI as a low-active, level-triggered interrupt by default.
|
||||
void HalApicSetSciIrq(uint16_t irq) { gSciIrq = irq; }
|
||||
|
||||
// Resolves one logical IRQ to a physical pin on the selected IOAPIC.
|
||||
static bool HalApicResolveIrq(uint8_t irq, uint32_t* pinOut, uint32_t* flagsOut) {
|
||||
HalApicInitializeIsaMap();
|
||||
uint32_t gsi = irq < 16 ? gIsaGsi[irq] : irq;
|
||||
if (gsi < gIoapicGsiBase || gsi - gIoapicGsiBase >= gIoapicRedirectionCount)
|
||||
return false;
|
||||
uint32_t flags = 0;
|
||||
if (irq == gSciIrq) flags = (1u << 13) | (1u << 15);
|
||||
if (irq < 16) {
|
||||
uint16_t override = gIsaFlags[irq];
|
||||
// MADT 00 is bus-conformant; 11 selects active low or level triggered.
|
||||
if ((override & 3u) == 3u) flags |= (1u << 13);
|
||||
else if ((override & 3u) == 1u) flags &= ~(1u << 13);
|
||||
if (((override >> 2) & 3u) == 3u) flags |= (1u << 15);
|
||||
else if (((override >> 2) & 3u) == 1u) flags &= ~(1u << 15);
|
||||
}
|
||||
*pinOut = gsi - gIoapicGsiBase;
|
||||
*flagsOut = flags;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Reports whether a platform IRQ can be routed by the current IOAPIC.
|
||||
bool HalApicCanRouteIrq(uint8_t irq) {
|
||||
uint32_t pin, flags;
|
||||
return irq < 24 && HalApicResolveIrq(irq, &pin, &flags);
|
||||
}
|
||||
|
||||
// Implements the apic Route Irq operation.
|
||||
void HalApicRouteIrq(uint8_t irq) {
|
||||
// Full initial setup of the redirection entry.
|
||||
// Sets the vector and clears the mask bit in one write.
|
||||
uint32_t vector = IRQ_VECTOR_BASE + irq; // e.g. IRQ 1 -> vector 0x21
|
||||
IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24); // external IRQs target the BSP
|
||||
IoapicWrite(IOREDTBL_LO(irq), vector); // unmasked (bit 16 = 0), edge, fixed
|
||||
uint32_t pin, flags;
|
||||
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
|
||||
uint32_t vector = IRQ_VECTOR_BASE + irq;
|
||||
IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
|
||||
IoapicWrite(IOREDTBL_LO(pin), vector | flags);
|
||||
}
|
||||
|
||||
// Implements the apic Mask Irq operation.
|
||||
void HalApicMaskIrq(uint8_t irq) {
|
||||
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
|
||||
uint32_t pin, flags;
|
||||
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
|
||||
uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
|
||||
lo |= (1 << 16);
|
||||
IoapicWrite(IOREDTBL_LO(irq), lo);
|
||||
IoapicWrite(IOREDTBL_LO(pin), lo);
|
||||
}
|
||||
|
||||
// Implements the apic Unmask Irq operation.
|
||||
void HalApicUnmaskIrq(uint8_t irq) {
|
||||
uint32_t lo = IoapicRead(IOREDTBL_LO(irq));
|
||||
lo &= ~(1 << 16); // clear mask bit
|
||||
lo = (lo & ~0xFF) | (IRQ_VECTOR_BASE + irq);
|
||||
IoapicWrite(IOREDTBL_HI(irq), (gBspLapicId & 0xFFu) << 24);
|
||||
IoapicWrite(IOREDTBL_LO(irq), lo);
|
||||
uint32_t pin, flags;
|
||||
if (!HalApicResolveIrq(irq, &pin, &flags)) return;
|
||||
uint32_t lo = IoapicRead(IOREDTBL_LO(pin));
|
||||
lo = (lo & ~(0xFFu | (1u << 16) | (1u << 13) | (1u << 15))) |
|
||||
(IRQ_VECTOR_BASE + irq) | flags;
|
||||
IoapicWrite(IOREDTBL_HI(pin), (gBspLapicId & 0xFFu) << 24);
|
||||
IoapicWrite(IOREDTBL_LO(pin), lo);
|
||||
}
|
||||
|
||||
// Implements the apic Send Eoi operation.
|
||||
@@ -240,8 +304,9 @@ void HalApicInitialize(uintptr_t lapicPhys, uintptr_t ioapicPhys) {
|
||||
|
||||
// Mask all IOAPIC lines initially.
|
||||
uint32_t maxRedir = (IoapicRead(1) >> 16) & 0xFF;
|
||||
for (uint8_t i = 0; i <= maxRedir; i++) {
|
||||
IoapicWrite(IOREDTBL_LO(i), (1 << 16) | (IRQ_VECTOR_BASE + i));
|
||||
gIoapicRedirectionCount = maxRedir + 1u;
|
||||
for (uint32_t i = 0; i <= maxRedir; i++) {
|
||||
IoapicWrite(IOREDTBL_LO(i), (1u << 16) | IRQ_VECTOR_BASE);
|
||||
IoapicWrite(IOREDTBL_HI(i), (gBspLapicId & 0xFFu) << 24);
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,12 @@ typedef struct ACPI_FIXED_REGISTER {
|
||||
|
||||
static ACPI_FIXED_REGISTER gPm1aCnt;
|
||||
static ACPI_FIXED_REGISTER gPm1bCnt;
|
||||
static ACPI_FIXED_REGISTER gPm1aStatus;
|
||||
static ACPI_FIXED_REGISTER gPm1bStatus;
|
||||
static ACPI_FIXED_REGISTER gPm1aEnable;
|
||||
static ACPI_FIXED_REGISTER gPm1bEnable;
|
||||
static uint16_t gSciIrq = 0;
|
||||
static bool gFixedPowerButtonArmed = false;
|
||||
static ACPI_FIXED_REGISTER gSleepControl;
|
||||
static uint32_t gLegacyPm1aPort = 0;
|
||||
static uint32_t gLegacyPm1bPort = 0;
|
||||
@@ -172,6 +178,14 @@ void* HalPhysicalToVirtual(uint64_t phys) {
|
||||
// FADT (Fixed ACPI Description Table) offsets used here. ACPI 2.0+
|
||||
// systems may put the real DSDT and PM1 control registers in the extended
|
||||
// X_* fields, so do not rely only on the legacy 32-bit fields.
|
||||
#define FADT_SCI_INT_OFFSET 46u
|
||||
#define FADT_PM1A_EVT_BLK_OFFSET 56u
|
||||
#define FADT_PM1B_EVT_BLK_OFFSET 60u
|
||||
#define FADT_PM1_EVT_LEN_OFFSET 88u
|
||||
#define FADT_X_PM1A_EVT_BLK_OFFSET 148u
|
||||
#define FADT_X_PM1B_EVT_BLK_OFFSET 160u
|
||||
#define FADT_FLAG_POWER_BUTTON (1u << 4)
|
||||
#define ACPI_PM1_POWER_BUTTON_BIT (1u << 8)
|
||||
#define FADT_DSDT_OFFSET 40u
|
||||
#define FADT_SMI_CMD_OFFSET 48u
|
||||
#define FADT_ACPI_ENABLE_OFFSET 52u
|
||||
@@ -184,6 +198,8 @@ void* HalPhysicalToVirtual(uint64_t phys) {
|
||||
#define FADT_X_PM1A_CNT_BLK_OFFSET 172u
|
||||
#define FADT_X_PM1B_CNT_BLK_OFFSET 184u
|
||||
#define FADT_SLEEP_CONTROL_OFFSET 244u
|
||||
#define FADT_RESET_REG_OFFSET 116u
|
||||
#define FADT_RESET_VALUE_OFFSET 128u
|
||||
#define FADT_FLAG_HW_REDUCED_ACPI (1u << 20)
|
||||
#define INTEL_PCH_SMI_EN_OFFSET 0x30u
|
||||
#define INTEL_PCH_SMI_ON_SLP_EN (1u << 4)
|
||||
@@ -198,6 +214,9 @@ typedef struct {
|
||||
} ACPI_GAS;
|
||||
#pragma pack(pop)
|
||||
|
||||
static ACPI_FIXED_REGISTER gResetRegister;
|
||||
static uint8_t gResetValue;
|
||||
|
||||
// Implements the acpi Read U32 operation.
|
||||
static uint32_t AcpiReadU32(const void* table, uint32_t tableLen, uint32_t off) {
|
||||
uint32_t value = 0;
|
||||
@@ -291,6 +310,77 @@ static void AcpiWriteFixedRegister(const ACPI_FIXED_REGISTER* reg, uint32_t valu
|
||||
else *(volatile uint32_t*)address = value;
|
||||
}
|
||||
|
||||
// Selects a PM1 event block as two adjacent equal-width status and enable registers.
|
||||
static void AcpiConfigurePm1EventBlock(const void* fadt, uint32_t len,
|
||||
uint32_t legacyOffset, uint32_t gasOffset,
|
||||
uint8_t halfLength, ACPI_FIXED_REGISTER* status,
|
||||
ACPI_FIXED_REGISTER* enable) {
|
||||
ACPI_FIXED_REGISTER block = {0};
|
||||
uint32_t legacy = AcpiReadU32(fadt, len, legacyOffset);
|
||||
if (legacy && legacy <= 0xFFFFu) {
|
||||
block.AddressSpaceId = 1;
|
||||
block.Address = legacy;
|
||||
}
|
||||
ACPI_FIXED_REGISTER extended = {0};
|
||||
if (AcpiReadGasRegister(fadt, len, gasOffset, &extended)) block = extended;
|
||||
if (!block.Address || (block.AddressSpaceId == 1 &&
|
||||
block.Address + halfLength * 2u - 1u > 0xFFFFu)) return;
|
||||
block.WidthBytes = halfLength;
|
||||
*status = block;
|
||||
block.Address += halfLength;
|
||||
*enable = block;
|
||||
}
|
||||
|
||||
// Reports the fixed-feature power-button interrupt supported by this HAL.
|
||||
static KSTATUS AcpiQueryFixedButtonIrq(uint8_t* irqOut) {
|
||||
if (!irqOut) return NTLSTATUS_INVALID_PARAMETER;
|
||||
// A FADT control-method power button requires AML evaluation, not PM1 polling.
|
||||
if (gHwReducedAcpi || (gFadtFlags & FADT_FLAG_POWER_BUTTON) ||
|
||||
!gSciIrq || gSciIrq >= 16u || !gPm1aStatus.Address || !gPm1aEnable.Address ||
|
||||
!HalApicCanRouteIrq((uint8_t)gSciIrq))
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
*irqOut = (uint8_t)gSciIrq;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Discards any stale press then enables the PM1 fixed power-button SCI.
|
||||
static KSTATUS AcpiArmFixedButton(void) {
|
||||
uint8_t irq;
|
||||
KSTATUS status = AcpiQueryFixedButtonIrq(&irq);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
// No SCI_EN means firmware still owns ACPI events. Never steal ownership
|
||||
// through SMI_CMD just to install a power-button handler during boot.
|
||||
if (!(AcpiReadFixedRegister(&gPm1aCnt) & 1u))
|
||||
return NTLSTATUS_NOT_SUPPORTED;
|
||||
AcpiWriteFixedRegister(&gPm1aStatus, ACPI_PM1_POWER_BUTTON_BIT);
|
||||
if (gPm1bStatus.Address)
|
||||
AcpiWriteFixedRegister(&gPm1bStatus, ACPI_PM1_POWER_BUTTON_BIT);
|
||||
uint32_t enabled = AcpiReadFixedRegister(&gPm1aEnable);
|
||||
AcpiWriteFixedRegister(&gPm1aEnable, enabled | ACPI_PM1_POWER_BUTTON_BIT);
|
||||
if (gPm1bEnable.Address) {
|
||||
enabled = AcpiReadFixedRegister(&gPm1bEnable);
|
||||
AcpiWriteFixedRegister(&gPm1bEnable, enabled | ACPI_PM1_POWER_BUTTON_BIT);
|
||||
}
|
||||
gFixedPowerButtonArmed = true;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Acknowledges the fixed-button source only; the SCI may carry unrelated events.
|
||||
static bool AcpiConsumeFixedButton(void) {
|
||||
if (!gFixedPowerButtonArmed) return false;
|
||||
bool pressed = false;
|
||||
if (AcpiReadFixedRegister(&gPm1aStatus) & ACPI_PM1_POWER_BUTTON_BIT) {
|
||||
AcpiWriteFixedRegister(&gPm1aStatus, ACPI_PM1_POWER_BUTTON_BIT);
|
||||
pressed = true;
|
||||
}
|
||||
if (gPm1bStatus.Address &&
|
||||
(AcpiReadFixedRegister(&gPm1bStatus) & ACPI_PM1_POWER_BUTTON_BIT)) {
|
||||
AcpiWriteFixedRegister(&gPm1bStatus, ACPI_PM1_POWER_BUTTON_BIT);
|
||||
pressed = true;
|
||||
}
|
||||
return pressed;
|
||||
}
|
||||
|
||||
// Detects whether PM1 control is using the standard Intel PCH PMBASE layout.
|
||||
static void AcpiDetectIntelPchPmBase(void) {
|
||||
gIntelPchPmBase = 0;
|
||||
@@ -490,10 +580,20 @@ static void ParseFadt(const ACPISDTHeader* header) {
|
||||
|
||||
memset(&gPm1aCnt, 0, sizeof(gPm1aCnt));
|
||||
memset(&gPm1bCnt, 0, sizeof(gPm1bCnt));
|
||||
memset(&gPm1aStatus, 0, sizeof(gPm1aStatus));
|
||||
memset(&gPm1bStatus, 0, sizeof(gPm1bStatus));
|
||||
memset(&gPm1aEnable, 0, sizeof(gPm1aEnable));
|
||||
memset(&gPm1bEnable, 0, sizeof(gPm1bEnable));
|
||||
gSciIrq = 0;
|
||||
if (len >= FADT_SCI_INT_OFFSET + sizeof(gSciIrq))
|
||||
memcpy(&gSciIrq, (const uint8_t*)fadt + FADT_SCI_INT_OFFSET, sizeof(gSciIrq));
|
||||
memset(&gSleepControl, 0, sizeof(gSleepControl));
|
||||
gLegacyPm1aPort = 0;
|
||||
gLegacyPm1bPort = 0;
|
||||
gFadtFlags = AcpiReadU32(fadt, len, FADT_FLAGS_OFFSET);
|
||||
if (AcpiReadByteGasRegister(fadt, len, FADT_RESET_REG_OFFSET, &gResetRegister) &&
|
||||
FADT_RESET_VALUE_OFFSET < len)
|
||||
gResetValue = *((const uint8_t*)fadt + FADT_RESET_VALUE_OFFSET);
|
||||
gHwReducedAcpi = (gFadtFlags & FADT_FLAG_HW_REDUCED_ACPI) != 0;
|
||||
|
||||
uint8_t pm1CntLength = FADT_PM1_CNT_LEN_OFFSET < len
|
||||
@@ -528,6 +628,16 @@ static void ParseFadt(const ACPISDTHeader* header) {
|
||||
if (gHwReducedAcpi)
|
||||
(void)AcpiReadByteGasRegister(fadt, len, FADT_SLEEP_CONTROL_OFFSET, &gSleepControl);
|
||||
|
||||
uint8_t eventLength = FADT_PM1_EVT_LEN_OFFSET < len ?
|
||||
*((const uint8_t*)fadt + FADT_PM1_EVT_LEN_OFFSET) : 0u;
|
||||
if (!gHwReducedAcpi && (eventLength == 4u || eventLength == 8u)) {
|
||||
uint8_t half = eventLength / 2u;
|
||||
AcpiConfigurePm1EventBlock(fadt, len, FADT_PM1A_EVT_BLK_OFFSET,
|
||||
FADT_X_PM1A_EVT_BLK_OFFSET, half, &gPm1aStatus, &gPm1aEnable);
|
||||
AcpiConfigurePm1EventBlock(fadt, len, FADT_PM1B_EVT_BLK_OFFSET,
|
||||
FADT_X_PM1B_EVT_BLK_OFFSET, half, &gPm1bStatus, &gPm1bEnable);
|
||||
}
|
||||
|
||||
gSmiCommandPort = AcpiReadU32(fadt, len, FADT_SMI_CMD_OFFSET);
|
||||
gAcpiEnableValue = FADT_ACPI_ENABLE_OFFSET < len
|
||||
? *((const uint8_t*)fadt + FADT_ACPI_ENABLE_OFFSET)
|
||||
@@ -599,6 +709,15 @@ static void ParseMadt(MADT* madt) {
|
||||
HalAddCpuRecord(uid, lapicId, flags);
|
||||
} else if (type == 1 && len >= 12) { // I/O APIC entry
|
||||
gIoapicPhys = *(uint32_t*)(p + 4);
|
||||
uint32_t base;
|
||||
memcpy(&base, p + 8, sizeof(base));
|
||||
HalApicSetGsiBase(base);
|
||||
} else if (type == 2 && len >= 10 && p[2] == 0) { // ISA source override
|
||||
uint32_t gsi;
|
||||
uint16_t flags;
|
||||
memcpy(&gsi, p + 4, sizeof(gsi));
|
||||
memcpy(&flags, p + 8, sizeof(flags));
|
||||
HalApicSetIsaOverride(p[3], gsi, flags);
|
||||
} else if (type == 5 && len >= 12) { // Local APIC Address Override
|
||||
gLapicPhys = (uintptr_t)*(uint64_t*)(p + 4);
|
||||
} else if (type == 9 && len >= 16) { // Processor Local x2APIC
|
||||
@@ -796,6 +915,19 @@ static void HalShutdown(void) {
|
||||
HalStopProcessor();
|
||||
}
|
||||
|
||||
static void HalReset(void) {
|
||||
(void)HalDisableInterrupts();
|
||||
if (gResetRegister.Address && gResetValue)
|
||||
AcpiWriteFixedRegister(&gResetRegister, gResetValue);
|
||||
if (gApi.Timer.Stall) gApi.Timer.Stall(100000u);
|
||||
__outbyte(0xCF9u, 0x06u);
|
||||
if (gApi.Timer.Stall) gApi.Timer.Stall(100000u);
|
||||
__outbyte(0x64u, 0xFEu);
|
||||
if (gApi.Timer.Stall) gApi.Timer.Stall(100000u);
|
||||
HalDbgPrint("HAL: reset failed\n");
|
||||
HalStopProcessor();
|
||||
}
|
||||
|
||||
static KSTATUS HalQueryProcessorTopology(HAL_PROCESSOR_TOPOLOGY* out) {
|
||||
if (!out) return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
@@ -863,6 +995,7 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
|
||||
HalDbgPrint("HAL: LAPIC=%016llx IOAPIC=%016llx HPET=%016llx\n",
|
||||
gLapicPhys, gIoapicPhys, gHpetPhys);
|
||||
|
||||
HalApicSetSciIrq(gSciIrq);
|
||||
HalApicInitialize(gLapicPhys, gIoapicPhys);
|
||||
HalDbgPrint("HAL: local APIC mode=%s BSP_ID=%u\n",
|
||||
HalApicUsingX2Apic() ? "x2APIC" : "xAPIC",
|
||||
@@ -874,6 +1007,10 @@ HAL_ENTRY KSTATUS HalInitialize(const HAL_INIT_PARAMS* initParams) {
|
||||
HalRtcInitialize(&gApi.Rtc);
|
||||
HalPciInitialize(&gApi.Pci);
|
||||
gApi.Power.Shutdown = HalShutdown;
|
||||
gApi.Power.Reset = HalReset;
|
||||
gApi.Power.QueryFixedButtonIrq = AcpiQueryFixedButtonIrq;
|
||||
gApi.Power.ArmFixedButton = AcpiArmFixedButton;
|
||||
gApi.Power.ConsumeFixedButton = AcpiConsumeFixedButton;
|
||||
|
||||
AcpiDetectIntelPchPmBase();
|
||||
gInitialised = 1;
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// Kernel driver configuration and dependency graph contract.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define DM_MAX_DRIVERS 64u
|
||||
#define DM_MAX_CLASSES 32u
|
||||
#define DM_MAX_EDGES 16u
|
||||
#define DM_ID_MAX 64u
|
||||
#define DM_IMAGE_MAX 64u
|
||||
#define DM_CONFIG_MAX_BYTES (64u * 1024u)
|
||||
|
||||
typedef enum DM_START_PHASE {
|
||||
DmStartBoot,
|
||||
DmStartRuntime,
|
||||
DmStartDisabled,
|
||||
} DM_START_PHASE;
|
||||
|
||||
typedef enum DM_DEPENDENCY_KIND {
|
||||
DmDependencyClass,
|
||||
DmDependencyDriver,
|
||||
} DM_DEPENDENCY_KIND;
|
||||
|
||||
typedef struct DM_DEPENDENCY {
|
||||
DM_DEPENDENCY_KIND Kind;
|
||||
char Id[DM_ID_MAX];
|
||||
} DM_DEPENDENCY;
|
||||
|
||||
typedef struct DM_EDGES {
|
||||
DM_DEPENDENCY Items[DM_MAX_EDGES];
|
||||
uint32_t Count;
|
||||
} DM_EDGES;
|
||||
|
||||
typedef enum DM_RUN_STATE {
|
||||
DmRunWaiting,
|
||||
DmRunStarted,
|
||||
DmRunFailed,
|
||||
DmRunBlocked,
|
||||
DmRunDisabled,
|
||||
} DM_RUN_STATE;
|
||||
|
||||
typedef struct DM_DRIVER_CONFIG {
|
||||
char Id[DM_ID_MAX];
|
||||
char Image[DM_IMAGE_MAX];
|
||||
char Class[DM_ID_MAX];
|
||||
DM_EDGES Requires;
|
||||
DM_EDGES After;
|
||||
DM_START_PHASE Phase;
|
||||
DM_RUN_STATE State;
|
||||
bool Critical;
|
||||
uint8_t Present;
|
||||
} DM_DRIVER_CONFIG;
|
||||
|
||||
typedef struct DM_CLASS_CONFIG {
|
||||
char Id[DM_ID_MAX];
|
||||
DM_EDGES Requires;
|
||||
DM_EDGES After;
|
||||
uint8_t Present;
|
||||
} DM_CLASS_CONFIG;
|
||||
|
||||
typedef struct DM_CONFIG {
|
||||
DM_DRIVER_CONFIG Drivers[DM_MAX_DRIVERS];
|
||||
DM_CLASS_CONFIG Classes[DM_MAX_CLASSES];
|
||||
uint32_t DriverCount;
|
||||
uint32_t ClassCount;
|
||||
DM_START_PHASE ActivePhase;
|
||||
} DM_CONFIG;
|
||||
|
||||
// Parses and validates a complete immutable configuration before any driver starts.
|
||||
// Empty optional values and omitted optional keys have identical semantics.
|
||||
bool DmConfigParse(DM_CONFIG* config, const char* bytes, size_t size,
|
||||
char* error, size_t errorCapacity);
|
||||
|
||||
// Returns true once every configured provider in this class has reached a terminal state.
|
||||
bool DmClassSettled(const DM_CONFIG* config, const char* classId);
|
||||
bool DmClassAvailable(const DM_CONFIG* config, const char* classId);
|
||||
|
||||
// Tests the current requirements/ordering gates for a single waiting module.
|
||||
bool DmDriverCanStart(const DM_CONFIG* config, uint32_t index);
|
||||
|
||||
// Returns true once a waiting driver's required dependency cannot become available.
|
||||
bool DmDriverIsBlocked(const DM_CONFIG* config, uint32_t index);
|
||||
@@ -0,0 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <ntlite/status.h>
|
||||
|
||||
|
||||
KSTATUS DmStartLoadedDriver(const char* name, uintptr_t imageBase);
|
||||
KSTATUS DmPrepareShutdown(void);
|
||||
void DmCommitShutdown(void);
|
||||
@@ -185,9 +185,14 @@ typedef struct HAL_INTERRUPT_CONTROLLER {
|
||||
} HAL_INTERRUPT_CONTROLLER;
|
||||
|
||||
typedef struct HAL_POWER {
|
||||
// ACPI fixed-feature button events. Control-method devices require an AML owner.
|
||||
KSTATUS (*QueryFixedButtonIrq)(uint8_t* irqOut);
|
||||
KSTATUS (*ArmFixedButton)(void);
|
||||
bool (*ConsumeFixedButton)(void);
|
||||
// ACPI S5 soft-off. Disables interrupts, writes SLP_EN to PM1a/b_CNT,
|
||||
// and halts if firmware doesn't respond. Does not return.
|
||||
void (*Shutdown)(void);
|
||||
void (*Reset)(void);
|
||||
} HAL_POWER;
|
||||
|
||||
typedef struct HAL_API {
|
||||
@@ -226,7 +231,7 @@ typedef struct HAL_INIT_PARAMS {
|
||||
} HAL_INIT_PARAMS;
|
||||
|
||||
// Increment whenever the kernel/HAL table ABI changes.
|
||||
#define HAL_API_VERSION 2u
|
||||
#define HAL_API_VERSION 4u
|
||||
|
||||
// Returns the HAL table. Processor interrupt-state operations are valid before HalInitialize().
|
||||
HAL_ENTRY const HAL_API* HalGetApi(void);
|
||||
|
||||
@@ -7,6 +7,10 @@
|
||||
#include <kernel/hal/hal_api.h>
|
||||
|
||||
void HalApicRouteIrq(uint8_t irq);
|
||||
void HalApicSetGsiBase(uint32_t gsiBase);
|
||||
void HalApicSetIsaOverride(uint8_t irq, uint32_t gsi, uint16_t flags);
|
||||
void HalApicSetSciIrq(uint16_t irq);
|
||||
bool HalApicCanRouteIrq(uint8_t irq);
|
||||
void HalApicMaskIrq(uint8_t irq);
|
||||
void HalApicUnmaskIrq(uint8_t irq);
|
||||
void HalApicSendEoi(void);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Kernel DDK-table provider declarations.
|
||||
// Kernel driver service initialization.
|
||||
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
@@ -8,8 +8,5 @@
|
||||
struct HAL_API;
|
||||
typedef struct HAL_API HAL_API;
|
||||
|
||||
// Initializes the kernel-owned DDK table from platform services.
|
||||
void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase);
|
||||
|
||||
// Returns the kernel-owned DDK table passed to driver entrypoints.
|
||||
const NTL_DDK* IoGetDdk(void);
|
||||
void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase);
|
||||
@@ -4,6 +4,85 @@
|
||||
|
||||
#include <ntlite/ddk/dma.h>
|
||||
|
||||
typedef struct NTL_DDK_DMA {
|
||||
// Returns the one adapter and requirement set owned by a device object.
|
||||
KSTATUS (*GetAdapter)(struct DEVICE_OBJECT* device,
|
||||
const IO_DMA_REQUIREMENTS* requirements,
|
||||
IO_DMA_ADAPTER** outAdapter);
|
||||
|
||||
// Releases one caller-owned adapter reference.
|
||||
void (*PutAdapter)(IO_DMA_ADAPTER* adapter);
|
||||
|
||||
// Allocates coherent memory with stable CPU and base device addresses. The
|
||||
// caller must quiesce every device reference before releasing the buffer.
|
||||
KSTATUS (*AllocateCommonBuffer)(IO_DMA_ADAPTER* adapter,
|
||||
size_t size,
|
||||
size_t alignment,
|
||||
IO_DMA_BUFFER** outBuffer);
|
||||
|
||||
// Releases a coherent common buffer.
|
||||
void (*FreeCommonBuffer)(IO_DMA_BUFFER* buffer);
|
||||
|
||||
// Returns the stable CPU address of a common buffer.
|
||||
void* (*GetCommonBufferAddress)(const IO_DMA_BUFFER* buffer);
|
||||
|
||||
// Returns the stable device address of a common buffer.
|
||||
uint64_t (*GetCommonBufferDeviceAddress)(const IO_DMA_BUFFER* buffer);
|
||||
|
||||
// Returns the caller-requested size of a common buffer.
|
||||
size_t (*GetCommonBufferSize)(const IO_DMA_BUFFER* buffer);
|
||||
|
||||
// Stabilizes a kernel range for one or more streaming mappings. Pinning
|
||||
// preserves the pages, but does not transfer ownership to a device.
|
||||
KSTATUS (*PinKernelMemory)(void* address,
|
||||
size_t size,
|
||||
IO_DMA_DIRECTION direction,
|
||||
MM_PINNED_MEMORY** outMemory);
|
||||
|
||||
// Stabilizes a process-owned user range for one or more mappings. The
|
||||
// descriptor retains the process and range until the final unpin.
|
||||
KSTATUS (*PinUserMemory)(struct PROCESS* process,
|
||||
void* address,
|
||||
size_t size,
|
||||
IO_DMA_DIRECTION direction,
|
||||
MM_PINNED_MEMORY** outMemory);
|
||||
|
||||
// Releases one pinned-memory reference after all mappings leave.
|
||||
void (*UnpinMemory)(MM_PINNED_MEMORY* memory);
|
||||
|
||||
// Maps a pinned subrange and creates a constrained segment list. A new
|
||||
// mapping begins CPU-owned and requires external operation serialization.
|
||||
KSTATUS (*MapPinnedMemory)(IO_DMA_ADAPTER* adapter,
|
||||
MM_PINNED_MEMORY* memory,
|
||||
size_t offset,
|
||||
size_t length,
|
||||
IO_DMA_DIRECTION direction,
|
||||
uint32_t flags,
|
||||
IO_DMA_MAPPING** outMapping);
|
||||
|
||||
// Releases a mapping and performs a final receive-side bounce copy only
|
||||
// when ownership was transferred to the device and not returned to the CPU.
|
||||
void (*UnmapMemory)(IO_DMA_MAPPING* mapping);
|
||||
|
||||
// Transfers mapping ownership from the CPU to the device. Streaming DMA
|
||||
// must not begin before this operation succeeds.
|
||||
KSTATUS (*SynchronizeForDevice)(IO_DMA_MAPPING* mapping);
|
||||
|
||||
// Transfers mapping ownership from the device to the CPU. CPU readers must
|
||||
// not consume receive data before this operation succeeds.
|
||||
KSTATUS (*SynchronizeForCpu)(IO_DMA_MAPPING* mapping);
|
||||
|
||||
// Returns the immutable segment count owned by a mapping.
|
||||
uint32_t (*GetSegmentCount)(const IO_DMA_MAPPING* mapping);
|
||||
|
||||
// Returns the immutable segment array owned by a mapping.
|
||||
const IO_DMA_SEGMENT* (*GetSegments)(const IO_DMA_MAPPING* mapping);
|
||||
|
||||
// Copies current DMA counters into caller-owned storage.
|
||||
void (*QueryStats)(IO_DMA_STATS* outStats);
|
||||
} NTL_DDK_DMA;
|
||||
|
||||
|
||||
struct DEVICE_OBJECT;
|
||||
|
||||
// Returns the kernel-owned DDK DMA table.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <ntlite/ddk/device.h>
|
||||
|
||||
struct DEVICE_OBJECT;
|
||||
struct NTL_MODULE;
|
||||
struct DRIVER_OBJECT;
|
||||
|
||||
KSTATUS IoPnpInitialize(void);
|
||||
KSTATUS IoPnpPublishModule(struct NTL_MODULE* module,
|
||||
const NTL_DEVICE_DESC* descriptors, uint32_t count,
|
||||
struct DRIVER_OBJECT* const* ioDrivers, uint32_t ioCount);
|
||||
KSTATUS IoPnpPhysicalDeviceArrived(struct DEVICE_OBJECT* pdo);
|
||||
void IoPnpPhysicalDevicePublished(struct DEVICE_OBJECT* pdo);
|
||||
void IoPnpPhysicalDeviceRemoved(struct DEVICE_OBJECT* pdo);
|
||||
KSTATUS IoPnpWaitForIdle(uint64_t timeoutMs);
|
||||
KSTATUS IoPnpShutdown(uint64_t timeoutMs);
|
||||
@@ -5,6 +5,34 @@
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <ntlite/ddk/audio.h>
|
||||
|
||||
typedef struct NTL_DDK_AUDIO {
|
||||
// Registers one endpoint and returns a provider-owned reference valid until unregister returns.
|
||||
KSTATUS (*RegisterEndpoint)(const IO_AUDIO_ENDPOINT_REGISTRATION* registration,
|
||||
IO_AUDIO_ENDPOINT** outEndpoint);
|
||||
|
||||
// Stops the endpoint, removes it from enumeration, and consumes the provider-owned reference.
|
||||
void (*UnregisterEndpoint)(IO_AUDIO_ENDPOINT* endpoint, IO_AUDIO_REMOVAL_MODE mode);
|
||||
|
||||
// Reports one completed hardware period from ISR/DPC completion processing.
|
||||
// presentedFrames, hardwarePositionFrames, and interruptCount must be monotonic for the
|
||||
// endpoint lifetime. timestampTicks uses the DDK monotonic microsecond source.
|
||||
void (*NotifyPeriod)(IO_AUDIO_ENDPOINT* endpoint,
|
||||
uint32_t completedPeriodIndex,
|
||||
uint64_t presentedFrames,
|
||||
uint64_t hardwarePositionFrames,
|
||||
uint64_t interruptCount,
|
||||
uint64_t timestampTicks);
|
||||
|
||||
// Updates jack-derived endpoint availability without changing endpoint identity.
|
||||
void (*SetEndpointAvailable)(IO_AUDIO_ENDPOINT* endpoint, bool available);
|
||||
|
||||
// Fails an endpoint and wakes every stream owner.
|
||||
void (*ReportEndpointFailure)(IO_AUDIO_ENDPOINT* endpoint,
|
||||
KSTATUS status,
|
||||
uint32_t flags);
|
||||
} NTL_DDK_AUDIO;
|
||||
|
||||
#include <kernel/ob_object.h>
|
||||
#include <kernel/ke_event.h>
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
// Named queue registry hooks shared by process handle cleanup and syscalls.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <ntlite/handle.h>
|
||||
|
||||
struct PROCESS;
|
||||
struct OBJECT;
|
||||
|
||||
// Removes registry entries owned by the given process handle/object pair.
|
||||
void KeNamedQueueUnpublishHandle(struct PROCESS* process, SYS_HANDLE handle, struct OBJECT* object);
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
#include <ntlite/status.h>
|
||||
#include <stdint.h>
|
||||
|
||||
struct PROCESS;
|
||||
struct EVENT;
|
||||
|
||||
struct HAL_API;
|
||||
KSTATUS KePowerInitialize(void);
|
||||
void KePowerInitializeButton(const struct HAL_API* hal);
|
||||
void KePowerSetInitProcess(struct PROCESS* init);
|
||||
KSTATUS KePowerRequest(uint32_t action);
|
||||
KSTATUS KePowerRequestHardware(uint32_t action);
|
||||
KSTATUS KePowerGetRequest(struct PROCESS* caller, uint32_t* actionOut);
|
||||
KSTATUS KePowerComplete(struct PROCESS* caller, uint32_t action);
|
||||
struct EVENT* KePowerRequestEvent(struct PROCESS* caller);
|
||||
struct EVENT* KePowerCompletionEvent(void);
|
||||
uint32_t KePowerCommittedAction(void);
|
||||
@@ -1,10 +1,6 @@
|
||||
// Kernel ke selftest declarations shared by implementation files and kernel clients.
|
||||
// Debug-only bootstrap checks and focused kernel diagnostics.
|
||||
|
||||
#pragma once
|
||||
// ke_selftest.h - kernel bring-up and regression self-tests.
|
||||
|
||||
#include <kernel/hal/hal_api.h>
|
||||
|
||||
void KiRunCoreHardeningSelfTests(void);
|
||||
void KiRunEarlyKernelSelfTests(void);
|
||||
void KiRunKernelSelfTests(const HAL_API* hal);
|
||||
void KiCheckBootState(void);
|
||||
void KiRunKernelSelfTests(void);
|
||||
@@ -19,8 +19,3 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void);
|
||||
// KeStartApplicationProcessors calls this after MADT enumeration; it is also
|
||||
// safe to call again after failed AP startup disables a CPU record.
|
||||
void KeBuildProcessorTopology(void);
|
||||
|
||||
// Run explicit AP-affine kernel workers that exercise the global scheduler,
|
||||
// reschedule IPIs, yield, and event wait/wake. Normal kernel/user/driver
|
||||
// threads remain boot-processor-affine unless their affinity is explicitly widened.
|
||||
KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void);
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
#include <ntlite/service.h>
|
||||
#include <ntlite/status.h>
|
||||
#include <ntlite/security.h>
|
||||
#include <ntlite/ddk/object.h>
|
||||
|
||||
struct PORT;
|
||||
|
||||
@@ -20,5 +20,9 @@ KSTATUS KeSystemServiceReference(SYS_SERVICE_ID serviceId, struct PORT** listene
|
||||
// Removes every service entry that refers to a listener being destroyed.
|
||||
void KeSystemServiceUnregisterListener(struct PORT* listener);
|
||||
|
||||
// Returns the security-context claim required to register one service identity.
|
||||
SYS_SERVICE_CLAIMS KeSystemServiceRequiredClaim(SYS_SERVICE_ID serviceId);
|
||||
// Creates one kernel-issued, typed authority for a numeric registry identity.
|
||||
// Caller owns the returned object reference and may issue attenuated handles.
|
||||
KSTATUS KeSystemServiceCreateRegistrationAuthority(SYS_SERVICE_ID serviceId, OBJECT** authorityOut);
|
||||
|
||||
// Reads the bound identity from a referenced, type-checked authority.
|
||||
KSTATUS KeSystemServiceRegistrationIdentity(const OBJECT* authority, SYS_SERVICE_ID* serviceIdOut);
|
||||
@@ -11,25 +11,31 @@
|
||||
|
||||
#define SYS_PEB_MAGIC 0x30424550u // "PEB0"
|
||||
#define SYS_TEB_MAGIC 0x30424554u // "TEB0"
|
||||
#define SYS_ENV_VERSION 4u
|
||||
#define SYS_ENV_VERSION 5u
|
||||
|
||||
#ifndef SYS_USER_ENV_TYPES_DEFINED
|
||||
#define SYS_USER_ENV_TYPES_DEFINED 1
|
||||
|
||||
#define SYS_STARTUP_CAP_SYSTEM_VOLUME 1u
|
||||
#define SYS_STARTUP_CAP_SERVICE_CONTROL 2u
|
||||
#define SYS_STARTUP_CAP_SYSTEM_VOLUME_NAMESPACE 3u
|
||||
#define SYS_STARTUP_CAP_VOLUME_MANAGER 4u
|
||||
#define SYS_STARTUP_CAP_INPUT_MANAGER 5u
|
||||
#define SYS_STARTUP_CAP_DISPLAY_MANAGER 6u
|
||||
#define SYS_STARTUP_CAP_SESSION_CONTROL 7u
|
||||
#define SYS_STARTUP_CAPABILITY_MAX 32u
|
||||
|
||||
typedef struct SYS_STARTUP_CAPABILITY {
|
||||
uint32_t Tag;
|
||||
char Name[SYS_STARTUP_CAPABILITY_NAME_MAX + 1u];
|
||||
SYS_HANDLE Handle;
|
||||
} SYS_STARTUP_CAPABILITY;
|
||||
|
||||
// Kernel-side validation of canonical process startup capability names.
|
||||
static inline int SysStartupCapabilityNameValid(const char name[SYS_STARTUP_CAPABILITY_NAME_MAX + 1u]) {
|
||||
if (!name || !name[0]) return 0;
|
||||
int terminated = 0;
|
||||
for (uint32_t i = 0; i <= SYS_STARTUP_CAPABILITY_NAME_MAX; ++i) {
|
||||
char c = name[i];
|
||||
if (!c) { terminated = 1; continue; }
|
||||
if (terminated || !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
|
||||
(c >= '0' && c <= '9') || c == '_' || c == '-' || c == '.')) return 0;
|
||||
}
|
||||
return terminated;
|
||||
}
|
||||
|
||||
typedef struct SYS_PEB {
|
||||
uint32_t Magic;
|
||||
uint32_t Version;
|
||||
|
||||
@@ -20,6 +20,9 @@ KSTATUS LdrLoadImageFileForProcess(struct PROCESS* process, struct FILE_OBJECT*
|
||||
size_t LdrGetImageSize(uintptr_t base);
|
||||
size_t LdrGetMappedImageSize(struct PROCESS* process, uintptr_t base);
|
||||
|
||||
// Look up a named export in one already-mapped image.
|
||||
void* LdrGetImageExport(uintptr_t imageBase, const char* functionName);
|
||||
|
||||
// Look up a named export in a loaded module.
|
||||
// 'moduleName' must match a registered kernel module name such as "hal.dll".
|
||||
void* LdrGetExport(const char* moduleName, const char* functionName);
|
||||
@@ -40,4 +43,5 @@ KSTATUS LdrProtectImageForProcess(struct PROCESS* process, uintptr_t imageBase,
|
||||
struct BOOT_INFO; // forward declaration
|
||||
KSTATUS LdrRegisterModulesFromBootInfo(const struct BOOT_INFO* bootInfo);
|
||||
KSTATUS LdrRegisterModule(const char* name, uintptr_t base);
|
||||
KSTATUS LdrSetModuleExportsAvailable(const char* name, bool available);
|
||||
const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut, size_t* sizeOut);
|
||||
@@ -72,6 +72,7 @@ KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf,
|
||||
|
||||
// Flushes dirty cached pages for one file.
|
||||
KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file);
|
||||
KSTATUS MmFileCacheFlushAll(void);
|
||||
|
||||
// Drops cached clean pages overlapping the byte range. Dirty pages are flushed
|
||||
// by callers before destructive metadata operations.
|
||||
|
||||
@@ -13,8 +13,6 @@ typedef struct SECURITY_CONTEXT {
|
||||
uint32_t SecurityId;
|
||||
SYS_PRIVILEGES EffectivePrivileges;
|
||||
SYS_PRIVILEGES DelegablePrivileges;
|
||||
SYS_SERVICE_CLAIMS ServiceClaims;
|
||||
SYS_SERVICE_CLAIMS DelegableServiceClaims;
|
||||
uint32_t Flags;
|
||||
} SECURITY_CONTEXT;
|
||||
|
||||
@@ -24,19 +22,14 @@ KSTATUS SeInitializeSecurity(void);
|
||||
// Returns the bootstrap root context. The returned pointer is owned by the security subsystem.
|
||||
SECURITY_CONTEXT* SeGetRootSecurityContext(void);
|
||||
|
||||
// Derives one immutable child context without allowing privilege or service-claim amplification.
|
||||
// Derives one immutable child context without allowing ambient privilege amplification.
|
||||
KSTATUS SeDeriveSecurityContext(SECURITY_CONTEXT* source,
|
||||
SYS_PRIVILEGES effectivePrivileges,
|
||||
SYS_PRIVILEGES delegablePrivileges,
|
||||
SYS_SERVICE_CLAIMS serviceClaims,
|
||||
SYS_SERVICE_CLAIMS delegableServiceClaims,
|
||||
SECURITY_CONTEXT** contextOut);
|
||||
|
||||
// Returns whether one process currently carries all requested ambient privileges.
|
||||
bool SeProcessHasPrivilege(const struct PROCESS* process, SYS_PRIVILEGES privileges);
|
||||
|
||||
// Returns whether one security context carries all requested service claims.
|
||||
bool SeContextHasServiceClaims(const SECURITY_CONTEXT* context, SYS_SERVICE_CLAIMS claims);
|
||||
|
||||
// Copies one immutable security context into the public query layout.
|
||||
void SeQuerySecurityContext(const SECURITY_CONTEXT* context, SYS_SECURITY_CONTEXT_INFO* infoOut);
|
||||
@@ -193,4 +193,9 @@
|
||||
#define SYS_GET_SOCKET_OPTION 160
|
||||
#define SYS_POLL 161
|
||||
|
||||
#define SYS_MAX 162
|
||||
#define SYS_OPEN_POWER_EVENT 162
|
||||
#define SYS_REQUEST_SYSTEM_POWER 163
|
||||
#define SYS_QUERY_SYSTEM_POWER 164
|
||||
#define SYS_COMPLETE_SYSTEM_POWER 165
|
||||
|
||||
#define SYS_MAX 166
|
||||
+115
-108
@@ -1,135 +1,142 @@
|
||||
// Kernel DDK-table provider implementation.
|
||||
|
||||
#include <stdint.h>
|
||||
#include <ntlite/ddk/ddk.h>
|
||||
#include <kernel/hal/hal_api.h>
|
||||
#include <kernel/io_ddk.h>
|
||||
#include <kernel/io_dma.h>
|
||||
#include <kernel/ke_audio.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/mm_pte.h>
|
||||
|
||||
static const HAL_API* gIoDdkHal;
|
||||
static uintptr_t gIoDdkPhysicalWindowBase;
|
||||
static const HAL_API* gHal;
|
||||
static uintptr_t gPhysicalWindowBase;
|
||||
|
||||
// Returns the kernel direct-map alias for a physical address.
|
||||
static void* IoDdkPhysToVirt(uint64_t physicalAddress) {
|
||||
if (!gIoDdkPhysicalWindowBase || physicalAddress > UINTPTR_MAX - gIoDdkPhysicalWindowBase)
|
||||
return (void*)0;
|
||||
return (void*)(gIoDdkPhysicalWindowBase + (uintptr_t)physicalAddress);
|
||||
void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase) {
|
||||
gHal = hal;
|
||||
gPhysicalWindowBase = physicalWindowBase;
|
||||
}
|
||||
|
||||
// Maps one kernel page through the DDK memory surface.
|
||||
static KSTATUS IoDdkMapPage(uintptr_t virtualAddress,
|
||||
uint64_t physicalAddress,
|
||||
uint64_t pageFlags,
|
||||
NTL_DDK_MAP_MODE mode) {
|
||||
MM_PTE_MAP_MODE kernelMode = MmPteMapFailIfPresent;
|
||||
if (mode == NtlDdkMapReplaceExisting)
|
||||
kernelMode = MmPteMapReplaceExisting;
|
||||
return MmPteMapPage(virtualAddress, physicalAddress, pageFlags, kernelMode);
|
||||
__declspec(dllexport) uint64_t KeQueryTicks(void) {
|
||||
return gHal && gHal->Timer.QueryTicks ? gHal->Timer.QueryTicks() : 0;
|
||||
}
|
||||
|
||||
// Reads one aligned PCI configuration DWORD through the platform HAL.
|
||||
static uint32_t IoDdkPciRead32(uint16_t segment,
|
||||
uint8_t bus,
|
||||
uint8_t device,
|
||||
uint8_t function,
|
||||
uint16_t offset) {
|
||||
if (!gIoDdkHal || !gIoDdkHal->Pci.Read32)
|
||||
return 0xFFFFFFFFu;
|
||||
return gIoDdkHal->Pci.Read32(segment, bus, device, function, offset);
|
||||
__declspec(dllexport) void KeStallExecution(uint32_t microseconds) {
|
||||
if (gHal && gHal->Timer.Stall) gHal->Timer.Stall(microseconds);
|
||||
}
|
||||
|
||||
// Writes one aligned PCI configuration DWORD through the platform HAL.
|
||||
static void IoDdkPciWrite32(uint16_t segment,
|
||||
uint8_t bus,
|
||||
uint8_t device,
|
||||
uint8_t function,
|
||||
uint16_t offset,
|
||||
uint32_t value) {
|
||||
if (!gIoDdkHal || !gIoDdkHal->Pci.Write32)
|
||||
return;
|
||||
gIoDdkHal->Pci.Write32(segment, bus, device, function, offset, value);
|
||||
__declspec(dllexport) uint32_t KePciReadConfig32(uint16_t segment, uint8_t bus, uint8_t device, uint8_t function, uint16_t offset) {
|
||||
return gHal && gHal->Pci.Read32 ? gHal->Pci.Read32(segment, bus, device, function, offset) : UINT32_MAX;
|
||||
}
|
||||
|
||||
// Allocates one kernel IRQ slot for MSI delivery.
|
||||
static KSTATUS IoDdkAllocateMsiVector(uint8_t* outIrq) {
|
||||
return KeIrqAllocateMsiVector(outIrq);
|
||||
__declspec(dllexport) void KePciWriteConfig32(uint16_t segment, uint8_t bus, uint8_t device, uint8_t function, uint16_t offset, uint32_t value) {
|
||||
if (gHal && gHal->Pci.Write32) gHal->Pci.Write32(segment, bus, device, function, offset, value);
|
||||
}
|
||||
|
||||
|
||||
// Composes one platform MSI message for an allocated nt-lite IRQ slot.
|
||||
static KSTATUS IoDdkComposeMsiMessage(uint8_t irq, NTL_DDK_MSI_MESSAGE* messageOut) {
|
||||
if (!gIoDdkHal || !messageOut || irq >= 24u)
|
||||
__declspec(dllexport) KSTATUS KeIrqComposeMsiMessage(uint8_t irq, NTL_DDK_MSI_MESSAGE* out) {
|
||||
if (!gHal || !out || irq >= 24u || !gHal->InterruptController.ComposeMsi)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
HAL_MSI_MESSAGE message;
|
||||
KSTATUS status = gIoDdkHal->InterruptController.ComposeMsi((uint8_t)(0x20u + irq), &message);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
|
||||
messageOut->Address = message.Address;
|
||||
messageOut->Data = message.Data;
|
||||
messageOut->Reserved = 0;
|
||||
KSTATUS status = gHal->InterruptController.ComposeMsi((uint8_t)(0x20u + irq), &message);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
out->Address = message.Address;
|
||||
out->Data = message.Data;
|
||||
out->Reserved = 0;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
static const NTL_DDK_DEBUG gIoDdkDebug = {
|
||||
.Print = DbgPrint,
|
||||
.GetFlags = KeGetDebugFlags,
|
||||
};
|
||||
|
||||
static const NTL_DDK_INTERRUPT gIoDdkInterrupt = {
|
||||
.AllocateMsiVector = IoDdkAllocateMsiVector,
|
||||
.ComposeMsiMessage = IoDdkComposeMsiMessage,
|
||||
};
|
||||
|
||||
static const NTL_DDK_MEMORY gIoDdkMemory = {
|
||||
.PhysToVirt = IoDdkPhysToVirt,
|
||||
.MapPage = IoDdkMapPage,
|
||||
};
|
||||
|
||||
static const NTL_DDK_PLATFORM gIoDdkPlatform = {
|
||||
.PciRead32 = IoDdkPciRead32,
|
||||
.PciWrite32 = IoDdkPciWrite32,
|
||||
};
|
||||
|
||||
static const NTL_DDK_SCHEDULER gIoDdkScheduler = {
|
||||
.DelayThread = KeDelayThread,
|
||||
.Yield = KeYield,
|
||||
};
|
||||
|
||||
static NTL_DDK_TIMER gIoDdkTimer = {
|
||||
.QueryTicks = 0,
|
||||
.Stall = 0,
|
||||
};
|
||||
|
||||
static NTL_DDK gIoDdk = {
|
||||
.Version = NTL_DDK_VERSION,
|
||||
.Debug = &gIoDdkDebug,
|
||||
.Audio = (const NTL_DDK_AUDIO*)0,
|
||||
.Dma = (const NTL_DDK_DMA*)0,
|
||||
.Interrupt = &gIoDdkInterrupt,
|
||||
.Memory = &gIoDdkMemory,
|
||||
.Platform = &gIoDdkPlatform,
|
||||
.Scheduler = &gIoDdkScheduler,
|
||||
.Timer = &gIoDdkTimer,
|
||||
};
|
||||
|
||||
// Initializes the kernel-owned DDK table from platform services.
|
||||
void IoInitializeDdk(const HAL_API* hal, uintptr_t physicalWindowBase) {
|
||||
gIoDdkHal = hal;
|
||||
gIoDdkPhysicalWindowBase = physicalWindowBase;
|
||||
gIoDdk.Audio = KeAudioGetDdk();
|
||||
gIoDdk.Dma = IoGetDmaDdk();
|
||||
gIoDdkTimer.QueryTicks = hal ? hal->Timer.QueryTicks : 0;
|
||||
gIoDdkTimer.Stall = hal ? hal->Timer.Stall : 0;
|
||||
__declspec(dllexport) void* MmDdkPhysToVirt(uint64_t physicalAddress) {
|
||||
if (!gPhysicalWindowBase || physicalAddress > UINTPTR_MAX - gPhysicalWindowBase)
|
||||
return (void*)0;
|
||||
return (void*)(gPhysicalWindowBase + (uintptr_t)physicalAddress);
|
||||
}
|
||||
|
||||
// Returns the kernel-owned DDK table passed to driver entrypoints.
|
||||
const NTL_DDK* IoGetDdk(void) {
|
||||
return &gIoDdk;
|
||||
__declspec(dllexport) KSTATUS MmDdkMapPage(uintptr_t virtualAddress, uint64_t physicalAddress, uint64_t flags, NTL_DDK_MAP_MODE mode) {
|
||||
return MmPteMapPage(virtualAddress, physicalAddress, flags,
|
||||
mode == NtlDdkMapReplaceExisting ? MmPteMapReplaceExisting : MmPteMapFailIfPresent);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoDmaGetAdapter(struct DEVICE_OBJECT* device, const IO_DMA_REQUIREMENTS* requirements, IO_DMA_ADAPTER** outAdapter) {
|
||||
return IoGetDmaDdk()->GetAdapter(device, requirements, outAdapter);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoDmaPutAdapter(IO_DMA_ADAPTER* adapter) {
|
||||
IoGetDmaDdk()->PutAdapter(adapter);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoDmaAllocateCommonBuffer(IO_DMA_ADAPTER* adapter, size_t size, size_t alignment, IO_DMA_BUFFER** outBuffer) {
|
||||
return IoGetDmaDdk()->AllocateCommonBuffer(adapter, size, alignment, outBuffer);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoDmaFreeCommonBuffer(IO_DMA_BUFFER* buffer) {
|
||||
IoGetDmaDdk()->FreeCommonBuffer(buffer);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* IoDmaGetCommonBufferAddress(const IO_DMA_BUFFER* buffer) {
|
||||
return IoGetDmaDdk()->GetCommonBufferAddress(buffer);
|
||||
}
|
||||
|
||||
__declspec(dllexport) uint64_t IoDmaGetCommonBufferDeviceAddress(const IO_DMA_BUFFER* buffer) {
|
||||
return IoGetDmaDdk()->GetCommonBufferDeviceAddress(buffer);
|
||||
}
|
||||
|
||||
__declspec(dllexport) size_t IoDmaGetCommonBufferSize(const IO_DMA_BUFFER* buffer) {
|
||||
return IoGetDmaDdk()->GetCommonBufferSize(buffer);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoDmaPinKernelMemory(void* address, size_t size, IO_DMA_DIRECTION direction, MM_PINNED_MEMORY** outMemory) {
|
||||
return IoGetDmaDdk()->PinKernelMemory(address, size, direction, outMemory);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoDmaPinUserMemory(struct PROCESS* process, void* address, size_t size, IO_DMA_DIRECTION direction, MM_PINNED_MEMORY** outMemory) {
|
||||
return IoGetDmaDdk()->PinUserMemory(process, address, size, direction, outMemory);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoDmaUnpinMemory(MM_PINNED_MEMORY* memory) {
|
||||
IoGetDmaDdk()->UnpinMemory(memory);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoDmaMapPinnedMemory(IO_DMA_ADAPTER* adapter, MM_PINNED_MEMORY* memory, size_t offset, size_t length, IO_DMA_DIRECTION direction, uint32_t flags, IO_DMA_MAPPING** outMapping) {
|
||||
return IoGetDmaDdk()->MapPinnedMemory(adapter, memory, offset, length, direction, flags, outMapping);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoDmaUnmapMemory(IO_DMA_MAPPING* mapping) {
|
||||
IoGetDmaDdk()->UnmapMemory(mapping);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoDmaSynchronizeForDevice(IO_DMA_MAPPING* mapping) {
|
||||
return IoGetDmaDdk()->SynchronizeForDevice(mapping);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoDmaSynchronizeForCpu(IO_DMA_MAPPING* mapping) {
|
||||
return IoGetDmaDdk()->SynchronizeForCpu(mapping);
|
||||
}
|
||||
|
||||
__declspec(dllexport) uint32_t IoDmaGetSegmentCount(const IO_DMA_MAPPING* mapping) {
|
||||
return IoGetDmaDdk()->GetSegmentCount(mapping);
|
||||
}
|
||||
|
||||
__declspec(dllexport) const IO_DMA_SEGMENT* IoDmaGetSegments(const IO_DMA_MAPPING* mapping) {
|
||||
return IoGetDmaDdk()->GetSegments(mapping);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoDmaQueryStats(IO_DMA_STATS* outStats) {
|
||||
IoGetDmaDdk()->QueryStats(outStats);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, IO_AUDIO_ENDPOINT** outEndpoint) {
|
||||
return KeAudioGetDdk()->RegisterEndpoint(registration, outEndpoint);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoAudioUnregisterEndpoint(IO_AUDIO_ENDPOINT* endpoint, IO_AUDIO_REMOVAL_MODE mode) {
|
||||
KeAudioGetDdk()->UnregisterEndpoint(endpoint, mode);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoAudioNotifyPeriod(IO_AUDIO_ENDPOINT* endpoint, uint32_t completedPeriodIndex, uint64_t presentedFrames, uint64_t hardwarePositionFrames, uint64_t interruptCount, uint64_t timestampTicks) {
|
||||
KeAudioGetDdk()->NotifyPeriod(endpoint, completedPeriodIndex, presentedFrames, hardwarePositionFrames, interruptCount, timestampTicks);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoAudioSetEndpointAvailable(IO_AUDIO_ENDPOINT* endpoint, bool available) {
|
||||
KeAudioGetDdk()->SetEndpointAvailable(endpoint, available);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoAudioReportEndpointFailure(IO_AUDIO_ENDPOINT* endpoint, KSTATUS status, uint32_t flags) {
|
||||
KeAudioGetDdk()->ReportEndpointFailure(endpoint, status, flags);
|
||||
}
|
||||
+8
-135
@@ -11,72 +11,7 @@
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ob_object.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
|
||||
#define MAX_DRIVER_OBJECTS 64
|
||||
#define MAX_PDO_OBJECTS 128
|
||||
|
||||
// Implements the sv Or Unnamed operation.
|
||||
static SYS_STRING_VIEW IoSvOrUnnamed(SYS_STRING_VIEW name) {
|
||||
return SysSvIsEmpty(name) ? SYS_SV_LITERAL("<unnamed>") : name;
|
||||
}
|
||||
|
||||
// Implements the bus Type Name operation.
|
||||
static const char* IoBusTypeName(IO_BUS_TYPE busType) {
|
||||
switch (busType) {
|
||||
case IoBusPci: return "pci";
|
||||
case IoBusUsb: return "usb";
|
||||
default: return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
// Driver object registry / PDO AddDevice binding.
|
||||
|
||||
static DRIVER_OBJECT* gDriverObjects[MAX_DRIVER_OBJECTS];
|
||||
static uint32_t gDriverObjectCount;
|
||||
static DEVICE_OBJECT* gPhysicalDevices[MAX_PDO_OBJECTS];
|
||||
static uint32_t gPhysicalDeviceCount;
|
||||
|
||||
// Implements the driver Matches Pdo operation.
|
||||
static bool IoDriverMatchesPdo(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
|
||||
if (!driver || !driver->IdTable || !pdo || pdo->Type != DeviceObjectPhysical)
|
||||
return false;
|
||||
|
||||
for (size_t d = 0; d < driver->IdTable->Count; d++) {
|
||||
const IO_DEVICE_ID* want = &driver->IdTable->Ids[d];
|
||||
if (want->BusType != IoBusUnknown && want->BusType != pdo->u.Pdo.BusType)
|
||||
continue;
|
||||
for (uint8_t p = 0; p < pdo->u.Pdo.IdCount; p++) {
|
||||
if (SysSvEqual(want->Id, pdo->u.Pdo.Ids[p].Id))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Implements the try Add Device For Driver operation.
|
||||
static bool IoTryAddDeviceForDriver(DRIVER_OBJECT* driver, DEVICE_OBJECT* pdo) {
|
||||
if (!driver || !pdo || !driver->DriverAddDevice)
|
||||
return false;
|
||||
if (!IoDriverMatchesPdo(driver, pdo))
|
||||
return false;
|
||||
|
||||
KSTATUS st = driver->DriverAddDevice(driver, pdo);
|
||||
if (KSUCCESS(st)) {
|
||||
DbgPrint("io: AddDevice driver=%v pdo=%v bus=%s\n",
|
||||
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), IoBusTypeName(pdo->u.Pdo.BusType));
|
||||
|
||||
KSTATUS start = IoSendPnpIrpSync(pdo, IRP_MN_START_DEVICE);
|
||||
if (!KSUCCESS(start) && start != NTLSTATUS_NOT_SUPPORTED) {
|
||||
DbgPrint("io: START_DEVICE failed driver=%v pdo=%v st=0x%x\n",
|
||||
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), start);
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
DbgPrint("io: AddDevice failed driver=%v pdo=%v st=0x%x\n",
|
||||
IoSvOrUnnamed(driver->Name), IoSvOrUnnamed(pdo->Name), st);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
#include <kernel/io_pnp.h>
|
||||
|
||||
__declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
|
||||
SYS_STRING_VIEW name) {
|
||||
@@ -86,42 +21,12 @@ __declspec(dllexport) void IoInitializeDriverObject(DRIVER_OBJECT* driver,
|
||||
driver->Name = name;
|
||||
}
|
||||
|
||||
// Registers driver Object.
|
||||
__declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
|
||||
if (!driver) return NTLSTATUS_INVALID_PARAMETER;
|
||||
if (!SysSvIsUtf8Text(driver->Name)) return NTLSTATUS_INVALID_PARAMETER;
|
||||
if (gDriverObjectCount >= MAX_DRIVER_OBJECTS)
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
|
||||
if (driver->Header.Type != ObjectTypeDriver ||
|
||||
driver->Header.LifetimeState != ObjectAlive) {
|
||||
ObInitializeObjectHeader(&driver->Header, ObjectTypeDriver);
|
||||
}
|
||||
|
||||
gDriverObjects[gDriverObjectCount++] = driver;
|
||||
DbgPrint("io: registered driver object %v\n", IoSvOrUnnamed(driver->Name));
|
||||
|
||||
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++)
|
||||
(void)IoTryAddDeviceForDriver(driver, gPhysicalDevices[i]);
|
||||
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Device object lifetime.
|
||||
|
||||
static void IoDeviceDestroy(OBJECT* object) {
|
||||
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
|
||||
IoDmaDetachDevice(dev);
|
||||
|
||||
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
|
||||
if (gPhysicalDevices[i] == dev) {
|
||||
gPhysicalDevices[i] = gPhysicalDevices[gPhysicalDeviceCount - 1u];
|
||||
gPhysicalDevices[gPhysicalDeviceCount - 1u] = (DEVICE_OBJECT*)0;
|
||||
gPhysicalDeviceCount--;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (dev->LowerDevice)
|
||||
IoDetachDevice(dev);
|
||||
|
||||
@@ -232,27 +137,11 @@ __declspec(dllexport) KSTATUS IoRegisterPhysicalDevice(DEVICE_OBJECT* pdo) {
|
||||
if (!pdo || pdo->Type != DeviceObjectPhysical)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
|
||||
if (gPhysicalDevices[i] == pdo)
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
KSTATUS status = IoPnpPhysicalDeviceArrived(pdo);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
|
||||
if (gPhysicalDeviceCount >= MAX_PDO_OBJECTS)
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
IoPnpPhysicalDevicePublished(pdo);
|
||||
|
||||
gPhysicalDevices[gPhysicalDeviceCount++] = pdo;
|
||||
|
||||
bool anyMatched = false;
|
||||
for (uint32_t i = 0; i < gDriverObjectCount; i++) {
|
||||
if (IoTryAddDeviceForDriver(gDriverObjects[i], pdo))
|
||||
anyMatched = true;
|
||||
}
|
||||
|
||||
if (!anyMatched && pdo->u.Pdo.BusType == IoBusUsb) {
|
||||
SYS_STRING_VIEW id = pdo->u.Pdo.IdCount ? pdo->u.Pdo.Ids[0].Id : SYS_SV_LITERAL("<no-id>");
|
||||
DbgPrint("io: no function driver for pdo=%v id=%v bus=%s\n",
|
||||
IoSvOrUnnamed(pdo->Name), id, IoBusTypeName(pdo->u.Pdo.BusType));
|
||||
}
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
@@ -280,24 +169,6 @@ __declspec(dllexport) void IoDereferenceDeviceObject(DEVICE_OBJECT* device) {
|
||||
if (device) ObDereferenceObject(&device->Header);
|
||||
}
|
||||
|
||||
// Releases physical Device Registry Entry.
|
||||
static void IoRemovePhysicalDeviceRegistryEntry(DEVICE_OBJECT* device) {
|
||||
if (!device || device->Type != DeviceObjectPhysical)
|
||||
return;
|
||||
|
||||
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
|
||||
if (gPhysicalDevices[i] == device) {
|
||||
for (uint32_t j = i + 1; j < gPhysicalDeviceCount; j++)
|
||||
gPhysicalDevices[j - 1] = gPhysicalDevices[j];
|
||||
gPhysicalDeviceCount--;
|
||||
if (gPhysicalDeviceCount < MAX_PDO_OBJECTS)
|
||||
gPhysicalDevices[gPhysicalDeviceCount] = (DEVICE_OBJECT*)0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Implements the unlink Device From Parent operation.
|
||||
static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
|
||||
if (!device || !device->Parent)
|
||||
return;
|
||||
@@ -333,9 +204,11 @@ __declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
|
||||
device->PnpState = DevicePnpDeleted;
|
||||
KeReleaseSpinlock(&device->Lock, flags);
|
||||
|
||||
IoDmaDetachDevice(device);
|
||||
if (device->Type == DeviceObjectPhysical)
|
||||
IoPnpPhysicalDeviceRemoved(device);
|
||||
else
|
||||
IoDmaDetachDevice(device);
|
||||
IoUnlinkDeviceFromParent(device);
|
||||
IoRemovePhysicalDeviceRegistryEntry(device);
|
||||
IoDereferenceDeviceObject(device);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,594 @@
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
#include <ntlite/ddk/device.h>
|
||||
#include <kernel/io_pnp.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/ke_work.h>
|
||||
#include <kernel/ke_time.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
|
||||
#define PNP_MAX_DESCRIPTORS 64u
|
||||
#define PNP_MAX_DEVICES 128u
|
||||
#define PNP_WORKERS 2u
|
||||
|
||||
typedef enum PNP_BIND_STATE {
|
||||
PnpUnbound,
|
||||
PnpReserved,
|
||||
PnpStarting,
|
||||
PnpPending,
|
||||
PnpCancelling,
|
||||
PnpStarted,
|
||||
PnpFailed,
|
||||
PnpCleaning,
|
||||
PnpStopping,
|
||||
} PNP_BIND_STATE;
|
||||
|
||||
typedef struct PNP_DRIVER {
|
||||
const NTL_DEVICE_DESC* Descriptor;
|
||||
DRIVER_OBJECT* IoDriver;
|
||||
struct NTL_MODULE* Module;
|
||||
} PNP_DRIVER;
|
||||
|
||||
typedef struct PNP_NODE {
|
||||
DEVICE_OBJECT* Pdo;
|
||||
DEVICE_OBJECT* ParentPdo;
|
||||
NTL_DEVICE* Binding;
|
||||
const PNP_DRIVER* Driver;
|
||||
PNP_BIND_STATE State;
|
||||
bool Removed;
|
||||
bool Published;
|
||||
bool CancelIssued;
|
||||
} PNP_NODE;
|
||||
|
||||
struct NTL_DEVICE_RESOURCE {
|
||||
NTL_DEVICE_RESOURCE* Next;
|
||||
NTL_DEVICE* Owner;
|
||||
NTL_DEVICE_RESOURCE_RELEASE Release;
|
||||
void* Context;
|
||||
};
|
||||
|
||||
struct NTL_DEVICE {
|
||||
SPINLOCK Lock;
|
||||
PNP_NODE* Node;
|
||||
void* Context;
|
||||
NTL_DEVICE_RESOURCE* Resources;
|
||||
KSTATUS CompletionStatus;
|
||||
bool CompletionSet;
|
||||
bool Closing;
|
||||
};
|
||||
|
||||
static SPINLOCK gPnpLock = SPINLOCK_INIT;
|
||||
static PNP_DRIVER gPnpDrivers[PNP_MAX_DESCRIPTORS];
|
||||
static uint32_t gPnpDriverCount;
|
||||
static PNP_NODE* gPnpNodes[PNP_MAX_DEVICES];
|
||||
static uint32_t gPnpNodeCount;
|
||||
static EVENT* gPnpReady;
|
||||
static EVENT* gPnpChanged;
|
||||
static bool gPnpStarted;
|
||||
static bool gPnpShutdown;
|
||||
static KWORK_ITEM gPnpWakeItem;
|
||||
|
||||
static void IoPnpSignal(void* unused) {
|
||||
(void)unused;
|
||||
for (uint32_t i = 0; i < PNP_WORKERS; ++i)
|
||||
KeSignalEvent(gPnpReady);
|
||||
KeSignalEvent(gPnpChanged);
|
||||
}
|
||||
|
||||
static void IoPnpWake(void) {
|
||||
if (gPnpStarted)
|
||||
(void)KeQueueWorkItem(&gPnpWakeItem);
|
||||
}
|
||||
|
||||
static bool IoPnpMatches(const NTL_DEVICE_DESC* descriptor, const DEVICE_OBJECT* pdo) {
|
||||
for (uint32_t i = 0; i < descriptor->MatchCount; ++i) {
|
||||
const NTL_DEVICE_MATCH* match = &descriptor->Matches[i];
|
||||
if ((uint32_t)match->Bus != (uint32_t)pdo->u.Pdo.BusType)
|
||||
continue;
|
||||
size_t length = strlen(match->Id);
|
||||
for (uint8_t j = 0; j < pdo->u.Pdo.IdCount; ++j) {
|
||||
SYS_STRING_VIEW id = pdo->u.Pdo.Ids[j].Id;
|
||||
if (id.Length == length && !memcmp(id.Data, match->Id, length))
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool IoPnpMatchesIoDriver(const DRIVER_OBJECT* driver, const DEVICE_OBJECT* pdo) {
|
||||
if (!driver->IdTable || !driver->DriverAddDevice) return false;
|
||||
for (size_t i = 0; i < driver->IdTable->Count; ++i) {
|
||||
const IO_DEVICE_ID* wanted = &driver->IdTable->Ids[i];
|
||||
if (wanted->BusType != IoBusUnknown && wanted->BusType != pdo->u.Pdo.BusType)
|
||||
continue;
|
||||
for (uint8_t j = 0; j < pdo->u.Pdo.IdCount; ++j)
|
||||
if (SysSvEqual(wanted->Id, pdo->u.Pdo.Ids[j].Id)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static const PNP_DRIVER* IoPnpFindDriver(const DEVICE_OBJECT* pdo) {
|
||||
for (uint32_t i = 0; i < gPnpDriverCount; ++i) {
|
||||
if ((gPnpDrivers[i].Descriptor && IoPnpMatches(gPnpDrivers[i].Descriptor, pdo)) ||
|
||||
(gPnpDrivers[i].IoDriver && IoPnpMatchesIoDriver(gPnpDrivers[i].IoDriver, pdo)))
|
||||
return &gPnpDrivers[i];
|
||||
}
|
||||
return (const PNP_DRIVER*)0;
|
||||
}
|
||||
|
||||
static PNP_NODE* IoPnpFindNode(const DEVICE_OBJECT* pdo) {
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i)
|
||||
if (gPnpNodes[i]->Pdo == pdo)
|
||||
return gPnpNodes[i];
|
||||
return (PNP_NODE*)0;
|
||||
}
|
||||
|
||||
static DEVICE_OBJECT* IoPnpParentPdo(const DEVICE_OBJECT* pdo) {
|
||||
const DEVICE_OBJECT* parent = pdo->Parent;
|
||||
for (uint32_t i = 0; parent && i < 64u; ++i) {
|
||||
if (parent->Type == DeviceObjectPhysical)
|
||||
return (DEVICE_OBJECT*)parent;
|
||||
parent = parent->LowerDevice ? parent->LowerDevice : parent->Parent;
|
||||
}
|
||||
return (DEVICE_OBJECT*)0;
|
||||
}
|
||||
|
||||
static bool IoPnpHasChildren(const DEVICE_OBJECT* parent) {
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
|
||||
const DEVICE_OBJECT* ancestor = gPnpNodes[i]->ParentPdo;
|
||||
for (uint32_t depth = 0; ancestor && depth < PNP_MAX_DEVICES; ++depth) {
|
||||
if (ancestor == parent) return true;
|
||||
const PNP_NODE* node = IoPnpFindNode(ancestor);
|
||||
ancestor = node ? node->ParentPdo : (const DEVICE_OBJECT*)0;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static void IoPnpReleaseResources(NTL_DEVICE* device) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&device->Lock);
|
||||
device->Closing = true;
|
||||
NTL_DEVICE_RESOURCE* resource = device->Resources;
|
||||
device->Resources = (NTL_DEVICE_RESOURCE*)0;
|
||||
KeReleaseSpinlock(&device->Lock, flags);
|
||||
|
||||
while (resource) {
|
||||
NTL_DEVICE_RESOURCE* next = resource->Next;
|
||||
resource->Owner = (NTL_DEVICE*)0;
|
||||
resource->Release(resource->Context);
|
||||
MmFreePool(resource);
|
||||
resource = next;
|
||||
}
|
||||
}
|
||||
|
||||
static void IoPnpDispose(PNP_NODE* node) {
|
||||
NTL_DEVICE* device = node->Binding;
|
||||
if (device) {
|
||||
if (node->State == PnpStopping) {
|
||||
if (node->Driver->Descriptor && node->Driver->Descriptor->Stop)
|
||||
node->Driver->Descriptor->Stop(device);
|
||||
else if (node->Driver->IoDriver && !node->Pdo->DeletePending)
|
||||
(void)IoSendPnpIrpSync(node->Pdo, IRP_MN_REMOVE_DEVICE);
|
||||
}
|
||||
IoPnpReleaseResources(device);
|
||||
MmFreePool(device);
|
||||
}
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
|
||||
if (gPnpNodes[i] == node) {
|
||||
gPnpNodes[i] = gPnpNodes[--gPnpNodeCount];
|
||||
gPnpNodes[gPnpNodeCount] = (PNP_NODE*)0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoDereferenceDeviceObject(node->Pdo);
|
||||
MmFreePool(node);
|
||||
IoPnpWake();
|
||||
}
|
||||
|
||||
static void IoPnpStart(PNP_NODE* node) {
|
||||
const NTL_DEVICE_DESC* desc = node->Driver->Descriptor;
|
||||
size_t contextSize = desc ? desc->ContextSize : 0;
|
||||
NTL_DEVICE* device = (NTL_DEVICE*)MmAllocatePool(sizeof(*device) + contextSize);
|
||||
if (!device) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
node->State = PnpFailed;
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoPnpWake();
|
||||
return;
|
||||
}
|
||||
memset(device, 0, sizeof(*device) + contextSize);
|
||||
device->Lock = (SPINLOCK)SPINLOCK_INIT;
|
||||
device->Node = node;
|
||||
device->Context = (void*)(device + 1);
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
node->Binding = device;
|
||||
node->State = PnpStarting;
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
|
||||
KSTATUS result;
|
||||
if (desc) {
|
||||
result = desc->Start(device);
|
||||
} else {
|
||||
DRIVER_OBJECT* io = node->Driver->IoDriver;
|
||||
result = io->DriverAddDevice(io, node->Pdo);
|
||||
if (KSUCCESS(result)) {
|
||||
result = IoSendPnpIrpSync(node->Pdo, IRP_MN_START_DEVICE);
|
||||
if (result == NTLSTATUS_NOT_SUPPORTED) result = NTLSTATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
flags = KeAcquireSpinlock(&gPnpLock);
|
||||
if (device->CompletionSet)
|
||||
result = device->CompletionStatus;
|
||||
node->State = result == NTLSTATUS_PENDING ? PnpPending :
|
||||
KSUCCESS(result) ? PnpStarted : PnpFailed;
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
if (KFAILED(result))
|
||||
DbgPrint("pnp: start failed (0x%x)\n", result);
|
||||
IoPnpWake();
|
||||
}
|
||||
|
||||
static void IoPnpWorker(void* ignored) {
|
||||
(void)ignored;
|
||||
for (;;) {
|
||||
(void)KeWaitForEvent(gPnpReady, false);
|
||||
for (;;) {
|
||||
PNP_NODE* node = (PNP_NODE*)0;
|
||||
bool stopping = false;
|
||||
bool cleanup = false;
|
||||
bool cancel = false;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
|
||||
PNP_NODE* candidate = gPnpNodes[i];
|
||||
if (candidate->State == PnpPending && candidate->Removed &&
|
||||
!candidate->CancelIssued && (candidate->Driver->Descriptor && candidate->Driver->Descriptor->CancelStart)) {
|
||||
candidate->CancelIssued = true;
|
||||
candidate->State = PnpCancelling;
|
||||
node = candidate;
|
||||
cancel = true;
|
||||
break;
|
||||
}
|
||||
if (candidate->State == PnpFailed && candidate->Binding && !candidate->Removed) {
|
||||
candidate->State = PnpCleaning;
|
||||
node = candidate;
|
||||
cleanup = true;
|
||||
break;
|
||||
}
|
||||
if (gPnpShutdown && candidate->Removed &&
|
||||
IoPnpHasChildren(candidate->Pdo))
|
||||
continue;
|
||||
if (candidate->Removed && candidate->State != PnpReserved &&
|
||||
candidate->State != PnpStarting && candidate->State != PnpPending &&
|
||||
candidate->State != PnpCancelling && candidate->State != PnpStopping &&
|
||||
candidate->State != PnpCleaning) {
|
||||
candidate->State = PnpStopping;
|
||||
node = candidate;
|
||||
stopping = true;
|
||||
break;
|
||||
}
|
||||
if (candidate->State != PnpUnbound || candidate->Removed ||
|
||||
gPnpShutdown || !candidate->Published)
|
||||
continue;
|
||||
const PNP_DRIVER* driver = IoPnpFindDriver(candidate->Pdo);
|
||||
if (driver) {
|
||||
candidate->Driver = driver;
|
||||
candidate->State = PnpReserved;
|
||||
node = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
if (!node) break;
|
||||
KeSignalEvent(gPnpReady);
|
||||
if (stopping) {
|
||||
IoPnpDispose(node);
|
||||
} else if (cancel) {
|
||||
node->Driver->Descriptor->CancelStart(node->Binding);
|
||||
uintptr_t update = KeAcquireSpinlock(&gPnpLock);
|
||||
node->State = node->Binding->CompletionSet ?
|
||||
(KSUCCESS(node->Binding->CompletionStatus) ? PnpStarted : PnpFailed) :
|
||||
PnpPending;
|
||||
KeReleaseSpinlock(&gPnpLock, update);
|
||||
IoPnpWake();
|
||||
} else if (cleanup) {
|
||||
NTL_DEVICE* device = node->Binding;
|
||||
if (node->Driver->Descriptor && node->Driver->Descriptor->Stop)
|
||||
node->Driver->Descriptor->Stop(device);
|
||||
else if (node->Driver->IoDriver && node->Pdo->AttachedDevice)
|
||||
(void)IoSendPnpIrpSync(node->Pdo, IRP_MN_REMOVE_DEVICE);
|
||||
IoPnpReleaseResources(device);
|
||||
uintptr_t update = KeAcquireSpinlock(&gPnpLock);
|
||||
node->Binding = (NTL_DEVICE*)0;
|
||||
node->State = PnpFailed;
|
||||
KeReleaseSpinlock(&gPnpLock, update);
|
||||
MmFreePool(device);
|
||||
IoPnpWake();
|
||||
} else {
|
||||
IoPnpStart(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS IoPnpInitialize(void) {
|
||||
if (gPnpStarted) return NTLSTATUS_ALREADY_EXISTS;
|
||||
KSTATUS st = KeEventCreate(EventAutoReset, false, &gPnpReady);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
st = KeEventCreate(EventAutoReset, false, &gPnpChanged);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
for (uint32_t i = 0; i < PNP_WORKERS; ++i) {
|
||||
const KE_THREAD_CREATE_PARAMETERS params =
|
||||
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "pnp.device");
|
||||
st = KeCreateKernelThread((uintptr_t)IoPnpWorker, 0, ¶ms, (THREAD**)0);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
}
|
||||
KeInitializeWorkItem(&gPnpWakeItem, IoPnpSignal, (void*)0);
|
||||
gPnpStarted = true;
|
||||
IoPnpWake();
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS IoPnpPublishModule(struct NTL_MODULE* module,
|
||||
const NTL_DEVICE_DESC* descriptors, uint32_t count,
|
||||
DRIVER_OBJECT* const* ioDrivers, uint32_t ioCount) {
|
||||
if (!gPnpStarted || !module || (count && !descriptors) || (ioCount && !ioDrivers))
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
if (gPnpShutdown) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_DEVICE_REMOVED;
|
||||
}
|
||||
if (count > PNP_MAX_DESCRIPTORS - gPnpDriverCount ||
|
||||
ioCount > PNP_MAX_DESCRIPTORS - gPnpDriverCount - count) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
}
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
for (uint32_t j = 0; j < gPnpDriverCount; ++j) {
|
||||
SYS_STRING_VIEW existing = gPnpDrivers[j].IoDriver ?
|
||||
gPnpDrivers[j].IoDriver->Name :
|
||||
SysSvFromParts(gPnpDrivers[j].Descriptor->Name,
|
||||
strlen(gPnpDrivers[j].Descriptor->Name));
|
||||
SYS_STRING_VIEW incoming = SysSvFromParts(descriptors[i].Name,
|
||||
strlen(descriptors[i].Name));
|
||||
if (SysSvEqual(existing, incoming)) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
}
|
||||
for (uint32_t j = i + 1; j < count; ++j) {
|
||||
if (!strcmp(descriptors[i].Name, descriptors[j].Name)) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < ioCount; ++i) {
|
||||
DRIVER_OBJECT* driver = ioDrivers[i];
|
||||
for (uint32_t j = 0; j < gPnpDriverCount; ++j) {
|
||||
SYS_STRING_VIEW existing = gPnpDrivers[j].IoDriver ?
|
||||
gPnpDrivers[j].IoDriver->Name :
|
||||
SysSvFromParts(gPnpDrivers[j].Descriptor->Name,
|
||||
strlen(gPnpDrivers[j].Descriptor->Name));
|
||||
if (SysSvEqual(driver->Name, existing)) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
}
|
||||
for (uint32_t j = i + 1; j < ioCount; ++j) {
|
||||
if (SysSvEqual(driver->Name, ioDrivers[j]->Name)) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
}
|
||||
for (uint32_t j = 0; j < count; ++j) {
|
||||
if (SysSvEqual(driver->Name, SysSvFromParts(descriptors[j].Name,
|
||||
strlen(descriptors[j].Name)))) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
gPnpDrivers[gPnpDriverCount].Descriptor = &descriptors[i];
|
||||
gPnpDrivers[gPnpDriverCount].IoDriver = (DRIVER_OBJECT*)0;
|
||||
gPnpDrivers[gPnpDriverCount].Module = module;
|
||||
++gPnpDriverCount;
|
||||
}
|
||||
for (uint32_t i = 0; i < ioCount; ++i) {
|
||||
gPnpDrivers[gPnpDriverCount].Descriptor = (const NTL_DEVICE_DESC*)0;
|
||||
gPnpDrivers[gPnpDriverCount].IoDriver = ioDrivers[i];
|
||||
gPnpDrivers[gPnpDriverCount].Module = module;
|
||||
++gPnpDriverCount;
|
||||
}
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoPnpWake();
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS IoPnpPhysicalDeviceArrived(DEVICE_OBJECT* pdo) {
|
||||
if (!pdo || pdo->Type != DeviceObjectPhysical) return NTLSTATUS_INVALID_PARAMETER;
|
||||
if (!gPnpStarted) return NTLSTATUS_SUCCESS;
|
||||
PNP_NODE* node = (PNP_NODE*)MmAllocatePool(sizeof(*node));
|
||||
if (!node) return NTLSTATUS_NO_MEMORY;
|
||||
memset(node, 0, sizeof(*node));
|
||||
node->Pdo = pdo;
|
||||
node->ParentPdo = IoPnpParentPdo(pdo);
|
||||
IoReferenceDeviceObject(pdo);
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
if (gPnpShutdown) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoDereferenceDeviceObject(pdo);
|
||||
MmFreePool(node);
|
||||
return NTLSTATUS_DEVICE_REMOVED;
|
||||
}
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
|
||||
if (gPnpNodes[i]->Pdo == pdo) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoDereferenceDeviceObject(pdo);
|
||||
MmFreePool(node);
|
||||
return NTLSTATUS_ALREADY_EXISTS;
|
||||
}
|
||||
}
|
||||
if (gPnpNodeCount == PNP_MAX_DEVICES) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoDereferenceDeviceObject(pdo);
|
||||
MmFreePool(node);
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
}
|
||||
gPnpNodes[gPnpNodeCount++] = node;
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
void IoPnpPhysicalDevicePublished(DEVICE_OBJECT* pdo) {
|
||||
if (!pdo || !gPnpStarted) return;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
|
||||
if (gPnpNodes[i]->Pdo == pdo) {
|
||||
gPnpNodes[i]->Published = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoPnpWake();
|
||||
}
|
||||
|
||||
void IoPnpPhysicalDeviceRemoved(DEVICE_OBJECT* pdo) {
|
||||
if (!pdo || !gPnpStarted) return;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
|
||||
if (gPnpNodes[i]->Pdo == pdo) {
|
||||
gPnpNodes[i]->Removed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoPnpWake();
|
||||
}
|
||||
|
||||
KSTATUS IoPnpWaitForIdle(uint64_t timeoutMs) {
|
||||
if (!gPnpStarted) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uint64_t deadline = KeMonotonicDeadlineAfterMilliseconds(timeoutMs);
|
||||
for (;;) {
|
||||
bool idle = true;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i) {
|
||||
PNP_NODE* node = gPnpNodes[i];
|
||||
if (node->State == PnpReserved || node->State == PnpStarting ||
|
||||
node->State == PnpPending || node->State == PnpCancelling ||
|
||||
node->State == PnpCleaning ||
|
||||
node->State == PnpStopping ||
|
||||
(node->State == PnpFailed && node->Binding) ||
|
||||
(node->Removed && node->State != PnpPending) ||
|
||||
(!gPnpShutdown && node->State == PnpUnbound && node->Published &&
|
||||
IoPnpFindDriver(node->Pdo))) {
|
||||
idle = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
if (idle) return NTLSTATUS_SUCCESS;
|
||||
uint64_t remaining = KeMonotonicMillisecondsUntil(deadline);
|
||||
if (!remaining) return NTLSTATUS_TIMEOUT;
|
||||
uint32_t index = 0;
|
||||
KSTATUS st = KeWaitForMultipleEvents(&gPnpChanged, 1, false, remaining, &index);
|
||||
if (st == NTLSTATUS_TIMEOUT) return NTLSTATUS_TIMEOUT;
|
||||
if (!KSUCCESS(st)) return st;
|
||||
}
|
||||
}
|
||||
|
||||
KSTATUS IoPnpShutdown(uint64_t timeoutMs) {
|
||||
if (!gPnpStarted) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
gPnpShutdown = true;
|
||||
for (uint32_t i = 0; i < gPnpNodeCount; ++i)
|
||||
gPnpNodes[i]->Removed = true;
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoPnpWake();
|
||||
return IoPnpWaitForIdle(timeoutMs);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void* NtlDeviceContext(NTL_DEVICE* device) {
|
||||
return device ? device->Context : (void*)0;
|
||||
}
|
||||
|
||||
__declspec(dllexport) const char* NtlDeviceId(NTL_DEVICE* device, uint32_t index) {
|
||||
if (!device || index >= device->Node->Pdo->u.Pdo.IdCount) return (const char*)0;
|
||||
return device->Node->Pdo->u.Pdo.Ids[index].Id.Data;
|
||||
}
|
||||
|
||||
__declspec(dllexport) NTL_BUS_TYPE NtlDeviceBus(NTL_DEVICE* device) {
|
||||
return device ? (NTL_BUS_TYPE)device->Node->Pdo->u.Pdo.BusType : (NTL_BUS_TYPE)0;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS NtlDeviceCompleteStart(NTL_DEVICE* device, KSTATUS result) {
|
||||
if (!device || result == NTLSTATUS_PENDING || !device->Node) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPnpLock);
|
||||
PNP_NODE* node = device->Node;
|
||||
if (node->Binding != device || device->CompletionSet ||
|
||||
(node->State != PnpStarting && node->State != PnpPending &&
|
||||
node->State != PnpCancelling)) {
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
return NTLSTATUS_OBJECT_DEAD;
|
||||
}
|
||||
device->CompletionSet = true;
|
||||
device->CompletionStatus = result;
|
||||
if (node->State == PnpPending) {
|
||||
node->State = KSUCCESS(result) ? PnpStarted : PnpFailed;
|
||||
}
|
||||
KeReleaseSpinlock(&gPnpLock, flags);
|
||||
IoPnpWake();
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS NtlDeviceOwnResource(NTL_DEVICE* device, void* context,
|
||||
NTL_DEVICE_RESOURCE_RELEASE release,
|
||||
NTL_DEVICE_RESOURCE** out) {
|
||||
if (!device || !release || !out) return NTLSTATUS_INVALID_PARAMETER;
|
||||
*out = (NTL_DEVICE_RESOURCE*)0;
|
||||
NTL_DEVICE_RESOURCE* resource = (NTL_DEVICE_RESOURCE*)MmAllocatePool(sizeof(*resource));
|
||||
if (!resource) return NTLSTATUS_NO_MEMORY;
|
||||
resource->Owner = device;
|
||||
resource->Context = context;
|
||||
resource->Release = release;
|
||||
uintptr_t flags = KeAcquireSpinlock(&device->Lock);
|
||||
if (device->Closing) {
|
||||
KeReleaseSpinlock(&device->Lock, flags);
|
||||
MmFreePool(resource);
|
||||
return NTLSTATUS_OBJECT_DEAD;
|
||||
}
|
||||
resource->Next = device->Resources;
|
||||
device->Resources = resource;
|
||||
KeReleaseSpinlock(&device->Lock, flags);
|
||||
*out = resource;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS NtlDeviceReleaseResource(NTL_DEVICE_RESOURCE* resource) {
|
||||
if (!resource || !resource->Owner) return NTLSTATUS_INVALID_PARAMETER;
|
||||
NTL_DEVICE* device = resource->Owner;
|
||||
uintptr_t flags = KeAcquireSpinlock(&device->Lock);
|
||||
NTL_DEVICE_RESOURCE** link = &device->Resources;
|
||||
while (*link && *link != resource) link = &(*link)->Next;
|
||||
if (*link != resource) {
|
||||
KeReleaseSpinlock(&device->Lock, flags);
|
||||
return NTLSTATUS_OBJECT_DEAD;
|
||||
}
|
||||
*link = resource->Next;
|
||||
resource->Owner = (NTL_DEVICE*)0;
|
||||
KeReleaseSpinlock(&device->Lock, flags);
|
||||
resource->Release(resource->Context);
|
||||
MmFreePool(resource);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
+65
-10
@@ -38,6 +38,10 @@ static size_t gDebugLogSize;
|
||||
static size_t gDebugLogWrite;
|
||||
static size_t gDebugLogUsed;
|
||||
static volatile int32_t gDebugLogFlushing;
|
||||
static bool gDebugAtLineStart = true;
|
||||
static bool gDebugClockReady;
|
||||
static uint64_t gDebugClockOrigin;
|
||||
static uint64_t gDebugLastTick;
|
||||
|
||||
// Appends bytes to the in-memory debug log while its lock is held.
|
||||
static void KiDebugLogAppendLocked(const char* text, size_t len) {
|
||||
@@ -185,6 +189,30 @@ uint32_t KeGetDebugFlags(void) {
|
||||
return gDebugFlags;
|
||||
}
|
||||
|
||||
// Prefixes each logical debug line with elapsed monotonic seconds when available.
|
||||
static size_t KiDebugTimestamp(char* buffer, size_t capacity) {
|
||||
if (!gDebugHal || !gDebugHal->Timer.QueryTicks)
|
||||
return 0;
|
||||
|
||||
uint64_t tick = gDebugHal->Timer.QueryTicks();
|
||||
if (tick == 0)
|
||||
return 0;
|
||||
if (!gDebugClockReady) {
|
||||
gDebugClockOrigin = tick;
|
||||
gDebugLastTick = tick;
|
||||
gDebugClockReady = true;
|
||||
}
|
||||
if (tick < gDebugLastTick)
|
||||
tick = gDebugLastTick;
|
||||
gDebugLastTick = tick;
|
||||
|
||||
uint64_t elapsed = tick - gDebugClockOrigin;
|
||||
int count = snprintf_(buffer, capacity, "[%6llu.%06llu] ",
|
||||
(unsigned long long)(elapsed / 1000000ull),
|
||||
(unsigned long long)(elapsed % 1000000ull));
|
||||
return count > 0 && (size_t)count < capacity ? (size_t)count : 0;
|
||||
}
|
||||
|
||||
// Writes one formatted kernel diagnostic.
|
||||
__declspec(dllexport) void DbgPrint(const char* fmt, ...) {
|
||||
if (!gDebugEnabled || !fmt)
|
||||
@@ -202,23 +230,50 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
|
||||
if (length >= sizeof(line))
|
||||
length = sizeof(line) - 1u;
|
||||
}
|
||||
if (length == 0)
|
||||
return;
|
||||
|
||||
bool panic = KiPanicIsActive();
|
||||
if (panic && !KiPanicCurrentCpuOwnsReport())
|
||||
KiPanicSecondaryHalt();
|
||||
|
||||
if (!panic) {
|
||||
uintptr_t lockFlags = KeAcquireSpinlock(&gDebugPrintLock);
|
||||
if (!NtlAtomicLoad32(&gDebugLogFlushing))
|
||||
KiDebugLogAppendLocked(line, length);
|
||||
KeReleaseSpinlock(&gDebugPrintLock, lockFlags);
|
||||
uint32_t stream = panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_DEBUG;
|
||||
uint32_t flags = panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0;
|
||||
uint32_t foreground = panic ? 0x00FF2222u : 0x00CCCCCCu;
|
||||
uint32_t background = panic ? 0x00000000u : 0x00012456u;
|
||||
|
||||
uintptr_t lockFlags = 0;
|
||||
if (!panic)
|
||||
lockFlags = KeAcquireSpinlock(&gDebugPrintLock);
|
||||
|
||||
size_t offset = 0;
|
||||
while (offset < length) {
|
||||
size_t end = offset;
|
||||
while (end < length && line[end] != '\n')
|
||||
end++;
|
||||
if (end < length)
|
||||
end++;
|
||||
|
||||
if (panic || gDebugAtLineStart) {
|
||||
char prefix[48];
|
||||
size_t prefixLength = KiDebugTimestamp(prefix, sizeof(prefix));
|
||||
if (prefixLength) {
|
||||
if (!panic && !NtlAtomicLoad32(&gDebugLogFlushing))
|
||||
KiDebugLogAppendLocked(prefix, prefixLength);
|
||||
KiOutputWrite(stream, flags, foreground, background, prefix, prefixLength);
|
||||
}
|
||||
}
|
||||
|
||||
if (!panic && !NtlAtomicLoad32(&gDebugLogFlushing))
|
||||
KiDebugLogAppendLocked(line + offset, end - offset);
|
||||
KiOutputWrite(stream, flags, foreground, background, line + offset, end - offset);
|
||||
if (!panic)
|
||||
gDebugAtLineStart = line[end - 1u] == '\n';
|
||||
offset = end;
|
||||
}
|
||||
|
||||
KiOutputWrite(panic ? HAL_OUTPUT_STREAM_PANIC : HAL_OUTPUT_STREAM_DEBUG,
|
||||
panic ? HAL_OUTPUT_WRITE_EMERGENCY : 0,
|
||||
panic ? 0x00FF2222u : 0x00CCCCCCu,
|
||||
panic ? 0x00000000u : 0x00012456u,
|
||||
line, length);
|
||||
if (!panic)
|
||||
KeReleaseSpinlock(&gDebugPrintLock, lockFlags);
|
||||
}
|
||||
|
||||
// Writes one debug-log segment and rejects successful zero-progress writes.
|
||||
|
||||
+45
-11
@@ -1516,6 +1516,45 @@ DISPLAY_API KSTATUS KeDisplayRegisterDevice(DISPLAY_DEVICE* device) {
|
||||
return result;
|
||||
}
|
||||
|
||||
DISPLAY_API KSTATUS KeDisplayUnregisterDevice(DISPLAY_DEVICE* device) {
|
||||
if (!device || device == &gBootFbDevice)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
bool providerHeld = false;
|
||||
KSTATUS status = KiDisplayAcquireProviderMutex(&providerHeld);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
|
||||
DISPLAY_DEVICE** link = &gDisplayList;
|
||||
while (*link && *link != device)
|
||||
link = &(*link)->Next;
|
||||
|
||||
if (!*link) {
|
||||
KeReleaseSpinlock(&gDisplayLock, flags);
|
||||
KiDisplayReleaseProviderMutex(providerHeld);
|
||||
return NTLSTATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
*link = device->Next;
|
||||
device->Next = (DISPLAY_DEVICE*)0;
|
||||
if (gPrimaryDisplay == device) {
|
||||
gPrimaryDisplay = (DISPLAY_DEVICE*)0;
|
||||
gDisplayGeneration++;
|
||||
gCursorDrawn = false;
|
||||
gCursorUpdatePending = false;
|
||||
gCursorAppliedGeneration = gCursorRequestGeneration;
|
||||
gCursorCommittedX = DISPLAY_CURSOR_POS_INVALID;
|
||||
gCursorCommittedY = DISPLAY_CURSOR_POS_INVALID;
|
||||
gDirtyAll = false;
|
||||
gDirtyRectCount = 0;
|
||||
gDrawDepth = 0;
|
||||
}
|
||||
KeReleaseSpinlock(&gDisplayLock, flags);
|
||||
KiDisplayReleaseProviderMutex(providerHeld);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Implements the display Seed New Primary operation.
|
||||
static KSTATUS KiDisplaySeedNewPrimary(DISPLAY_DEVICE* oldDevice, DISPLAY_DEVICE* newDevice) {
|
||||
if (!oldDevice || !newDevice || oldDevice == newDevice)
|
||||
@@ -1570,6 +1609,11 @@ DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) {
|
||||
|
||||
const char* name = device->Name ? device->Name : "?";
|
||||
|
||||
bool providerHeld = false;
|
||||
KSTATUS result = KiDisplayAcquireProviderMutex(&providerHeld);
|
||||
if (!KSUCCESS(result))
|
||||
return result;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gDisplayLock);
|
||||
bool found = false;
|
||||
for (DISPLAY_DEVICE* it = gDisplayList; it; it = it->Next) {
|
||||
@@ -1580,6 +1624,7 @@ DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) {
|
||||
}
|
||||
if (!found) {
|
||||
KeReleaseSpinlock(&gDisplayLock, flags);
|
||||
KiDisplayReleaseProviderMutex(providerHeld);
|
||||
return NTLSTATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
@@ -1590,17 +1635,6 @@ DISPLAY_API KSTATUS KeDisplaySetPrimary(DISPLAY_DEVICE* device) {
|
||||
gCursorUpdatePending = gCursorVisible;
|
||||
KeReleaseSpinlock(&gDisplayLock, flags);
|
||||
|
||||
bool providerHeld = false;
|
||||
KSTATUS result = KiDisplayAcquireProviderMutex(&providerHeld);
|
||||
if (!KSUCCESS(result)) {
|
||||
flags = KeAcquireSpinlock(&gDisplayLock);
|
||||
gDisplayTransitionInProgress = false;
|
||||
gDisplayGeneration++;
|
||||
KeReleaseSpinlock(&gDisplayLock, flags);
|
||||
KiDisplayQueueCursorWork();
|
||||
return result;
|
||||
}
|
||||
|
||||
if (oldDevice && oldCursorDrawn && oldDevice->Ops && oldDevice->Ops->ShowCursor)
|
||||
(void)oldDevice->Ops->ShowCursor(oldDevice, false);
|
||||
|
||||
|
||||
@@ -13,6 +13,9 @@ static SPINLOCK gDpcLock;
|
||||
static volatile int32_t gDpcInitialized = 0;
|
||||
static volatile int32_t gDpcActive = 0;
|
||||
|
||||
// Bound a single interrupt-exit dispatch so self-requeuing devices yield the CPU.
|
||||
#define KE_DPC_DISPATCH_BUDGET 64u
|
||||
|
||||
// Initializes dpc System.
|
||||
void KeInitializeDpcSystem(void) {
|
||||
LIST_INIT(&gDpcQueue);
|
||||
@@ -83,7 +86,10 @@ void KiDrainDpcQueue(void) {
|
||||
if (oldIrql < DISPATCH_LEVEL)
|
||||
KeRaiseIrql(DISPATCH_LEVEL);
|
||||
|
||||
while (true) {
|
||||
// The xHCI event routine already bounds its own work, but an IRQ stream
|
||||
// can requeue it after every dispatch. Limit the whole drain, too. The
|
||||
// pending queue is picked up at the next timer or device interrupt exit.
|
||||
for (uint32_t dispatched = 0; dispatched < KE_DPC_DISPATCH_BUDGET; dispatched++) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gDpcLock);
|
||||
LIST_ENTRY* node = KeListPopHeadChecked(&gDpcQueue);
|
||||
if (!node) {
|
||||
|
||||
@@ -301,6 +301,9 @@ SYS_RIGHTS KeGetValidHandleRights(OBJECT_TYPE type) {
|
||||
return management | SYS_RIGHT_QUERY | SYS_DISPLAY_MANAGER_RIGHT_ENUMERATE |
|
||||
SYS_DISPLAY_MANAGER_RIGHT_MODESET | SYS_DISPLAY_MANAGER_RIGHT_PRESENT |
|
||||
SYS_DISPLAY_MANAGER_RIGHT_CURSOR | SYS_DISPLAY_MANAGER_RIGHT_CONSOLE;
|
||||
case ObjectTypeServiceRegistration:
|
||||
return management | SYS_RIGHT_QUERY |
|
||||
SYS_SERVICE_REGISTRATION_RIGHT_REGISTER | SYS_SERVICE_REGISTRATION_RIGHT_UNREGISTER;
|
||||
case ObjectTypeDevice:
|
||||
case ObjectTypeDriver:
|
||||
case ObjectTypeUsbDevice:
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
#include <ntlite/atomic.h>
|
||||
#include <ntlite/syscall.h>
|
||||
#include <kernel/ke_power.h>
|
||||
#include <kernel/ke_process.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/ke_irq.h>
|
||||
#include <kernel/ke_dpc.h>
|
||||
#include <kernel/ke_settings.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/hal/hal_api.h>
|
||||
#include <string.h>
|
||||
|
||||
static SPINLOCK gPowerLock = SPINLOCK_INIT;
|
||||
static EVENT* gPowerRequest;
|
||||
static EVENT* gPowerComplete;
|
||||
static PROCESS* gInit;
|
||||
static uint32_t gAction;
|
||||
static bool gCommitted;
|
||||
static const HAL_API* gButtonHal;
|
||||
static uint32_t gButtonAction;
|
||||
static KDPC gPowerButtonDpc;
|
||||
|
||||
KSTATUS KePowerInitialize(void) {
|
||||
if (gPowerRequest) return NTLSTATUS_ALREADY_EXISTS;
|
||||
KSTATUS st = KeEventCreate(EventManualReset, false, &gPowerRequest);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
st = KeEventCreate(EventManualReset, false, &gPowerComplete);
|
||||
return st;
|
||||
}
|
||||
|
||||
// Delivers the physical action after interrupt-exit processing.
|
||||
static void KiPowerButtonDpc(KDPC* dpc, void* context, void* arg1, void* arg2) {
|
||||
(void)dpc;
|
||||
(void)context;
|
||||
(void)arg1;
|
||||
(void)arg2;
|
||||
(void)KePowerRequestHardware(gButtonAction);
|
||||
}
|
||||
|
||||
// The ACPI SCI handler only acknowledges the fixed event and queues normal work.
|
||||
static void KiPowerButtonIsr(void) {
|
||||
if (!gButtonHal || !gButtonHal->Power.ConsumeFixedButton ||
|
||||
!gButtonHal->Power.ConsumeFixedButton()) return;
|
||||
(void)KeInsertQueueDpc(&gPowerButtonDpc, NULL, NULL);
|
||||
}
|
||||
|
||||
// Reads immutable boot-time button policy and attaches it to the fixed ACPI SCI.
|
||||
void KePowerInitializeButton(const HAL_API* hal) {
|
||||
char mode[24];
|
||||
KSTATUS status = KeGetSettingString("power", "button", mode, sizeof(mode));
|
||||
if (!KSUCCESS(status))
|
||||
KePanic(KE_PANIC_CODE_INIT, "power: missing [power] button, status=0x%x\n", status);
|
||||
if (strcmp(mode, "ignore") == 0) {
|
||||
DbgPrint("power: ACPI power button ignored by policy\n");
|
||||
return;
|
||||
}
|
||||
if (strcmp(mode, "shutdown") == 0) gButtonAction = SYS_POWER_ACTION_SHUTDOWN;
|
||||
else if (strcmp(mode, "restart") == 0) gButtonAction = SYS_POWER_ACTION_RESTART;
|
||||
else KePanic(KE_PANIC_CODE_INIT, "power: invalid [power] button=%s\n", mode);
|
||||
|
||||
if (!hal || !hal->Power.QueryFixedButtonIrq || !hal->Power.ArmFixedButton ||
|
||||
!hal->Power.ConsumeFixedButton) {
|
||||
DbgPrint("power: no ACPI fixed power button HAL support\n");
|
||||
return;
|
||||
}
|
||||
uint8_t irq = 0;
|
||||
status = hal->Power.QueryFixedButtonIrq(&irq);
|
||||
if (!KSUCCESS(status)) {
|
||||
DbgPrint("power: ACPI fixed button unavailable (0x%x); AML button not yet supported\n", status);
|
||||
return;
|
||||
}
|
||||
// The IOAPIC is still masked for this IRQ. Arm the firmware source first,
|
||||
// then install the kernel handler and finally unmask via KeIrqRegister.
|
||||
status = hal->Power.ArmFixedButton();
|
||||
if (!KSUCCESS(status)) {
|
||||
DbgPrint("power: cannot arm fixed button IRQ %u, status=0x%x\n", irq, status);
|
||||
return;
|
||||
}
|
||||
KeInitializeDpc(&gPowerButtonDpc, KiPowerButtonDpc, NULL);
|
||||
gButtonHal = hal;
|
||||
KeIrqRegister(irq, KiPowerButtonIsr);
|
||||
DbgPrint("power: ACPI fixed button armed on IRQ %u, action=%s\n", irq, mode);
|
||||
}
|
||||
|
||||
void KePowerSetInitProcess(PROCESS* init) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
|
||||
gInit = init;
|
||||
KeReleaseSpinlock(&gPowerLock, flags);
|
||||
}
|
||||
|
||||
// Records one power transition under the coordinator lock. The hardware event
|
||||
// may arrive before init exists; syscall-originated requests require init.
|
||||
static KSTATUS KiPowerQueueRequest(uint32_t action, bool requireInit) {
|
||||
if (action != SYS_POWER_ACTION_SHUTDOWN && action != SYS_POWER_ACTION_RESTART)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
|
||||
KSTATUS status = NTLSTATUS_SUCCESS;
|
||||
if (!gPowerRequest || (requireInit && !gInit)) status = NTLSTATUS_NOT_SUPPORTED;
|
||||
else if (gAction) status = NTLSTATUS_ALREADY_EXISTS;
|
||||
else gAction = action;
|
||||
KeReleaseSpinlock(&gPowerLock, flags);
|
||||
if (KSUCCESS(status)) KeSignalEvent(gPowerRequest);
|
||||
return status;
|
||||
}
|
||||
|
||||
// Accepts a privileged explicit power request only after init exists.
|
||||
KSTATUS KePowerRequest(uint32_t action) { return KiPowerQueueRequest(action, true); }
|
||||
|
||||
// Latches a physical power-button request even during init startup.
|
||||
KSTATUS KePowerRequestHardware(uint32_t action) { return KiPowerQueueRequest(action, false); }
|
||||
|
||||
EVENT* KePowerRequestEvent(PROCESS* caller) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
|
||||
EVENT* event = caller && caller == gInit ? gPowerRequest : (EVENT*)0;
|
||||
KeReleaseSpinlock(&gPowerLock, flags);
|
||||
return event;
|
||||
}
|
||||
|
||||
KSTATUS KePowerGetRequest(PROCESS* caller, uint32_t* actionOut) {
|
||||
if (!actionOut) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
|
||||
KSTATUS st = caller && caller == gInit ? NTLSTATUS_SUCCESS : NTLSTATUS_ACCESS_DENIED;
|
||||
if (KSUCCESS(st)) *actionOut = gAction;
|
||||
KeReleaseSpinlock(&gPowerLock, flags);
|
||||
return st;
|
||||
}
|
||||
|
||||
KSTATUS KePowerComplete(PROCESS* caller, uint32_t action) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
|
||||
KSTATUS st = caller && caller == gInit ? NTLSTATUS_SUCCESS : NTLSTATUS_ACCESS_DENIED;
|
||||
if (KSUCCESS(st) && (!gAction || action != gAction || gCommitted))
|
||||
st = NTLSTATUS_INVALID_PARAMETER;
|
||||
if (KSUCCESS(st)) gCommitted = true;
|
||||
KeReleaseSpinlock(&gPowerLock, flags);
|
||||
if (KSUCCESS(st)) KeSignalEvent(gPowerComplete);
|
||||
return st;
|
||||
}
|
||||
|
||||
EVENT* KePowerCompletionEvent(void) { return gPowerComplete; }
|
||||
|
||||
uint32_t KePowerCommittedAction(void) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPowerLock);
|
||||
uint32_t result = gCommitted ? gAction : 0;
|
||||
KeReleaseSpinlock(&gPowerLock, flags);
|
||||
return result;
|
||||
}
|
||||
@@ -629,7 +629,8 @@ KSTATUS KeExitProcess(PROCESS* process, int32_t exitCode) {
|
||||
// Begins termination of one process and wakes every sibling thread.
|
||||
KSTATUS KeTerminateProcess(PROCESS* process, int32_t exitCode) {
|
||||
if (!process) return NTLSTATUS_INVALID_PARAMETER;
|
||||
if (KeIsProcessTerminationProtected(process)) return NTLSTATUS_ACCESS_DENIED;
|
||||
if (KeIsProcessTerminationProtected(process) && KeGetCurrentProcess() != process)
|
||||
return NTLSTATUS_ACCESS_DENIED;
|
||||
|
||||
process->ExitCode = exitCode;
|
||||
NtlAtomicExchange32(&process->Terminating, 1);
|
||||
|
||||
+16
-1333
File diff suppressed because it is too large.
Load diff
@@ -863,218 +863,3 @@ KSTATUS KeAdmitApplicationProcessorsToScheduler(void) {
|
||||
DbgPrint("SMP: scheduler admitted %u APs\n", admitted);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
#define SMP_AP_WORKER_ROUNDS 64u
|
||||
#define SMP_AP_WORKER_TIMEOUT_US 2000000ull
|
||||
#define SMP_AP_TEST_MAGIC 0x524B575041504D53ULL // 'SMPAPWKR'
|
||||
|
||||
typedef struct SMP_AP_WORK_TEST SMP_AP_WORK_TEST;
|
||||
|
||||
typedef struct SMP_AP_WORKER_ARG {
|
||||
uint64_t Magic;
|
||||
uint32_t CpuIndex;
|
||||
uint32_t WorkerIndex;
|
||||
struct SMP_AP_WORK_TEST* Test;
|
||||
EVENT* StepEvent;
|
||||
volatile int32_t Ready;
|
||||
volatile int32_t WaitArmed;
|
||||
volatile int32_t Progress;
|
||||
volatile int32_t Done;
|
||||
volatile int32_t Failure;
|
||||
uint32_t WrongCpu;
|
||||
uint32_t WrongRunningCpu;
|
||||
} SMP_AP_WORKER_ARG;
|
||||
|
||||
struct SMP_AP_WORK_TEST {
|
||||
uint64_t Magic;
|
||||
uint32_t WorkerCount;
|
||||
volatile int32_t ReadyCount;
|
||||
volatile int32_t DoneCount;
|
||||
volatile int32_t FailureCount;
|
||||
volatile int32_t CpuHits[KE_MAX_CPUS];
|
||||
SMP_AP_WORKER_ARG Workers[KE_MAX_CPUS];
|
||||
};
|
||||
|
||||
// Implements the smp Ap Controlled Worker operation.
|
||||
static void KiSmpApControlledWorker(void* context) {
|
||||
SMP_AP_WORKER_ARG* arg = (SMP_AP_WORKER_ARG*)context;
|
||||
if (!arg || arg->Magic != SMP_AP_TEST_MAGIC || !arg->StepEvent) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP AP worker received bad context 0x%016llX\n", (uint64_t)arg);
|
||||
}
|
||||
|
||||
NtlAtomicExchange32(&arg->Ready, 1);
|
||||
NtlAtomicIncrement32(&arg->Test->ReadyCount);
|
||||
|
||||
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
|
||||
NtlAtomicIncrement32(&arg->WaitArmed);
|
||||
KSTATUS waitStatus = KeWaitForEvent(arg->StepEvent, 0);
|
||||
if (!KSUCCESS(waitStatus)) {
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
|
||||
KCPU* cpu = KeGetCurrentCpu();
|
||||
THREAD* self = KeGetCurrentThread();
|
||||
if (!cpu || cpu->CpuIndex != arg->CpuIndex) {
|
||||
arg->WrongCpu = cpu ? cpu->CpuIndex : KE_INVALID_CPU;
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
if (!self || self->RunningCpu != cpu->CpuIndex) {
|
||||
arg->WrongRunningCpu = self ? self->RunningCpu : KE_INVALID_CPU;
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
if (self && (self->AffinityMask & (1ULL << cpu->CpuIndex)) == 0) {
|
||||
arg->WrongCpu = cpu->CpuIndex;
|
||||
NtlAtomicExchange32(&arg->Failure, 1);
|
||||
break;
|
||||
}
|
||||
|
||||
if (cpu->CpuIndex < KE_MAX_CPUS)
|
||||
NtlAtomicIncrement32(&arg->Test->CpuHits[cpu->CpuIndex]);
|
||||
|
||||
NtlAtomicIncrement32(&arg->Progress);
|
||||
if ((round & 3u) == 0)
|
||||
KeYield();
|
||||
}
|
||||
|
||||
if (NtlAtomicLoad32(&arg->Failure) != 0) {
|
||||
NtlAtomicIncrement32(&arg->Test->FailureCount);
|
||||
}
|
||||
|
||||
NtlAtomicExchange32(&arg->Done, 1);
|
||||
NtlAtomicIncrement32(&arg->Test->DoneCount);
|
||||
}
|
||||
|
||||
// Waits for for Worker Counter.
|
||||
static bool KiWaitForWorkerCounter(volatile int32_t* counter, int32_t expected, const char* what) {
|
||||
uint64_t deadline = KiSmpDeadlineAfter(SMP_AP_WORKER_TIMEOUT_US);
|
||||
for (;;) {
|
||||
if (NtlAtomicLoad32(counter) >= expected)
|
||||
return true;
|
||||
if (KiSmpQueryTicks() >= deadline)
|
||||
break;
|
||||
KeYield();
|
||||
}
|
||||
DbgPrint("SMP scheduler selftest timeout waiting for %s expected=%d got=%ld\n",
|
||||
what ? what : "counter", expected, NtlAtomicLoad32(counter));
|
||||
return false;
|
||||
}
|
||||
|
||||
// Runs application Processor Scheduler Self Tests.
|
||||
KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
|
||||
if (gCpuCount <= 1) {
|
||||
DbgPrint("SMP scheduler selftest: SKIP, no APs\n");
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
DbgPrint("SMP scheduler selftest: begin AP-affine workers\n");
|
||||
|
||||
SMP_AP_WORK_TEST* test = (SMP_AP_WORK_TEST*)MmAllocatePool(sizeof(SMP_AP_WORK_TEST));
|
||||
if (!test) return NTLSTATUS_NO_MEMORY;
|
||||
memset(test, 0, sizeof(*test));
|
||||
test->Magic = SMP_AP_TEST_MAGIC;
|
||||
|
||||
uint32_t workerCount = 0;
|
||||
for (uint32_t cpuIndex = 1; cpuIndex < gCpuCount; cpuIndex++) {
|
||||
KCPU* cpu = &gCpus[cpuIndex];
|
||||
if (cpu->State != KeCpuOnline)
|
||||
continue;
|
||||
if (cpuIndex >= 64)
|
||||
continue;
|
||||
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[workerCount];
|
||||
arg->Magic = SMP_AP_TEST_MAGIC;
|
||||
arg->CpuIndex = cpuIndex;
|
||||
arg->WorkerIndex = workerCount;
|
||||
arg->Test = test;
|
||||
|
||||
KSTATUS st = KeEventCreate(EventAutoReset, 0, &arg->StepEvent);
|
||||
if (!KSUCCESS(st)) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u step event create failed status=0x%x\n",
|
||||
cpuIndex, st);
|
||||
}
|
||||
|
||||
KE_THREAD_CREATE_PARAMETERS workerParameters =
|
||||
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "selftest.smp");
|
||||
workerParameters.AffinityMask = 1ULL << cpuIndex;
|
||||
st = KeCreateKernelThread((uintptr_t)KiSmpApControlledWorker,
|
||||
(uintptr_t)arg,
|
||||
&workerParameters,
|
||||
(THREAD**)0);
|
||||
if (!KSUCCESS(st)) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u worker create failed status=0x%x\n",
|
||||
cpuIndex, st);
|
||||
}
|
||||
|
||||
workerCount++;
|
||||
}
|
||||
test->WorkerCount = workerCount;
|
||||
|
||||
if (workerCount == 0) {
|
||||
MmFreePool(test);
|
||||
DbgPrint("SMP scheduler selftest: SKIP, no online AP workers\n");
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
if (!KiWaitForWorkerCounter(&test->ReadyCount, (int32_t)workerCount, "AP worker ready")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers failed to become ready ready=%ld workers=%u\n",
|
||||
test->ReadyCount, workerCount);
|
||||
}
|
||||
|
||||
for (uint32_t round = 0; round < SMP_AP_WORKER_ROUNDS; round++) {
|
||||
for (uint32_t w = 0; w < workerCount; w++) {
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
|
||||
if (!KiWaitForWorkerCounter(&arg->WaitArmed, (int32_t)(round + 1), "AP worker wait-armed")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u did not arm wait for round %u armed=%ld progress=%ld\n",
|
||||
arg->CpuIndex, round, arg->WaitArmed, arg->Progress);
|
||||
}
|
||||
KeSignalEvent(arg->StepEvent);
|
||||
}
|
||||
|
||||
for (uint32_t w = 0; w < workerCount; w++) {
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
|
||||
if (!KiWaitForWorkerCounter(&arg->Progress, (int32_t)(round + 1), "AP worker progress")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u no progress round=%u progress=%ld failure=%ld wrongCpu=%u wrongRunningCpu=%u\n",
|
||||
arg->CpuIndex, round, arg->Progress, arg->Failure,
|
||||
arg->WrongCpu, arg->WrongRunningCpu);
|
||||
}
|
||||
if (NtlAtomicLoad32(&arg->Failure) != 0) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u worker failed round=%u wrongCpu=%u wrongRunningCpu=%u progress=%ld\n",
|
||||
arg->CpuIndex, round, arg->WrongCpu, arg->WrongRunningCpu, arg->Progress);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!KiWaitForWorkerCounter(&test->DoneCount, (int32_t)workerCount, "AP worker done")) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: workers did not finish done=%d workers=%u\n",
|
||||
test->DoneCount, workerCount);
|
||||
}
|
||||
|
||||
if (NtlAtomicLoad32(&test->FailureCount) != 0) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST, "SMP scheduler selftest: failure count=%d\n",
|
||||
test->FailureCount);
|
||||
}
|
||||
|
||||
for (uint32_t w = 0; w < workerCount; w++) {
|
||||
SMP_AP_WORKER_ARG* arg = &test->Workers[w];
|
||||
int32_t hits = (arg->CpuIndex < KE_MAX_CPUS) ? test->CpuHits[arg->CpuIndex] : 0;
|
||||
if (hits <= 0) {
|
||||
KePanic(KE_PANIC_CODE_SELFTEST,
|
||||
"SMP scheduler selftest: AP%u recorded no CPU hits\n", arg->CpuIndex);
|
||||
}
|
||||
if (arg->StepEvent)
|
||||
ObDereferenceObject(&arg->StepEvent->Header);
|
||||
}
|
||||
|
||||
DbgPrint("SMP scheduler selftest: PASS workers=%u rounds=%u\n",
|
||||
workerCount, SMP_AP_WORKER_ROUNDS);
|
||||
MmFreePool(test);
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
+69
-32
@@ -33,6 +33,7 @@
|
||||
#include <kernel/ke_cpu.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_process.h>
|
||||
#include <kernel/ke_power.h>
|
||||
#include <kernel/se_security.h>
|
||||
#include <kernel/ke_mutex.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
@@ -317,6 +318,40 @@ static size_t KiAlignUpSize(size_t value, size_t alignment) {
|
||||
|
||||
// Individual syscall handlers.
|
||||
|
||||
static uint64_t SyscallOpenPowerEvent(SYS_HANDLE* handleOut) {
|
||||
if (!handleOut) return NTLSTATUS_INVALID_PARAMETER;
|
||||
KSTATUS st = MmProbeUserWrite(handleOut, sizeof(*handleOut));
|
||||
if (!KSUCCESS(st)) return st;
|
||||
PROCESS* caller = KeGetCurrentProcess();
|
||||
EVENT* event = KePowerRequestEvent(caller);
|
||||
if (!event) return NTLSTATUS_ACCESS_DENIED;
|
||||
SYS_HANDLE handle = SYS_HANDLE_INVALID;
|
||||
st = KiAllocHandleForCurrentWithRights(&event->Header,
|
||||
SYS_RIGHT_WAIT | SYS_RIGHT_QUERY, &handle);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
st = KiCopyHandleToUser(handleOut, handle);
|
||||
if (!KSUCCESS(st)) (void)KeCloseHandle(caller, handle);
|
||||
return st;
|
||||
}
|
||||
|
||||
static uint64_t SyscallRequestSystemPower(uint32_t action) {
|
||||
KSTATUS st = KiRequirePrivilege(SYS_PRIV_SYSTEM_POWER);
|
||||
return KSUCCESS(st) ? KePowerRequest(action) : st;
|
||||
}
|
||||
|
||||
static uint64_t SyscallQuerySystemPower(uint32_t* actionOut) {
|
||||
if (!actionOut) return NTLSTATUS_INVALID_PARAMETER;
|
||||
KSTATUS st = MmProbeUserWrite(actionOut, sizeof(*actionOut));
|
||||
if (!KSUCCESS(st)) return st;
|
||||
uint32_t action = 0;
|
||||
st = KePowerGetRequest(KeGetCurrentProcess(), &action);
|
||||
return KSUCCESS(st) ? KiCopyU32ToUser(actionOut, action) : st;
|
||||
}
|
||||
|
||||
static uint64_t SyscallCompleteSystemPower(uint32_t action) {
|
||||
return KePowerComplete(KeGetCurrentProcess(), action);
|
||||
}
|
||||
|
||||
// Implements SYS_EXIT_PROCESS.
|
||||
static uint64_t SyscallExitProcess(int32_t exitCode) {
|
||||
THREAD* t = KeGetCurrentThread();
|
||||
@@ -2873,12 +2908,12 @@ static uint64_t SyscallStartProcess(const SYS_START_PROCESS_REQUEST* request) {
|
||||
}
|
||||
|
||||
for (uint32_t i = 0; i < req.StartupCapabilityCount && KSUCCESS(status); ++i) {
|
||||
if (!capabilities[i].Tag || capabilities[i].Handle == SYS_HANDLE_INVALID) {
|
||||
if (!SysStartupCapabilityNameValid(capabilities[i].Name) || capabilities[i].Handle == SYS_HANDLE_INVALID) {
|
||||
status = NTLSTATUS_INVALID_PARAMETER;
|
||||
break;
|
||||
}
|
||||
for (uint32_t j = 0; j < i; ++j) {
|
||||
if (capabilities[j].Tag == capabilities[i].Tag) {
|
||||
if (strcmp(capabilities[j].Name, capabilities[i].Name) == 0) {
|
||||
status = NTLSTATUS_OBJECT_NAME_COLLISION;
|
||||
break;
|
||||
}
|
||||
@@ -3414,48 +3449,48 @@ static uint64_t SyscallOpenPortPeerProcess(SYS_HANDLE endpointHandle,
|
||||
return status;
|
||||
}
|
||||
|
||||
// Implements SYS_REGISTER_SYSTEM_SERVICE.
|
||||
static uint64_t SyscallRegisterSystemService(SYS_SERVICE_ID serviceId, SYS_HANDLE listenerHandle) {
|
||||
if (serviceId == SYS_SERVICE_INVALID || serviceId > SYS_SERVICE_MAX)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
KSTATUS status = KiRequirePrivilege(SYS_PRIV_SYSTEM_SERVICE_REGISTER);
|
||||
// Implements SYS_REGISTER_SYSTEM_SERVICE using a typed, attenuable registration authority.
|
||||
static uint64_t SyscallRegisterSystemService(SYS_HANDLE registrationHandle, SYS_HANDLE listenerHandle) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
if (!process) return NTLSTATUS_FAILURE;
|
||||
|
||||
OBJECT* authority = (OBJECT*)0;
|
||||
KSTATUS status = KeReferenceHandle(process, registrationHandle, ObjectTypeServiceRegistration,
|
||||
SYS_SERVICE_REGISTRATION_RIGHT_REGISTER, &authority);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
SYS_SERVICE_CLAIMS claim = KeSystemServiceRequiredClaim(serviceId);
|
||||
if (!process || !process->SecurityContext || claim == 0 ||
|
||||
!SeContextHasServiceClaims(process->SecurityContext, claim))
|
||||
return NTLSTATUS_ACCESS_DENIED;
|
||||
SYS_SERVICE_ID serviceId = SYS_SERVICE_INVALID;
|
||||
status = KeSystemServiceRegistrationIdentity(authority, &serviceId);
|
||||
ObDereferenceObject(authority);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
|
||||
KSTATUS handleStatus = NTLSTATUS_SUCCESS;
|
||||
PORT* listener = (PORT*)KiHandleToObject(listenerHandle,
|
||||
ObjectTypePort,
|
||||
SYS_PORT_RIGHT_ACCEPT,
|
||||
&handleStatus);
|
||||
PORT* listener = (PORT*)KiHandleToObject(listenerHandle, ObjectTypePort,
|
||||
SYS_PORT_RIGHT_ACCEPT, &handleStatus);
|
||||
if (!listener) return handleStatus;
|
||||
status = KeSystemServiceRegister(serviceId, listener);
|
||||
ObDereferenceObject((OBJECT*)listener);
|
||||
return status;
|
||||
}
|
||||
|
||||
// Implements SYS_UNREGISTER_SYSTEM_SERVICE.
|
||||
static uint64_t SyscallUnregisterSystemService(SYS_SERVICE_ID serviceId, SYS_HANDLE listenerHandle) {
|
||||
if (serviceId == SYS_SERVICE_INVALID || serviceId > SYS_SERVICE_MAX)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
KSTATUS status = KiRequirePrivilege(SYS_PRIV_SYSTEM_SERVICE_REGISTER);
|
||||
// Implements SYS_UNREGISTER_SYSTEM_SERVICE using the same typed authority.
|
||||
static uint64_t SyscallUnregisterSystemService(SYS_HANDLE registrationHandle, SYS_HANDLE listenerHandle) {
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
if (!process) return NTLSTATUS_FAILURE;
|
||||
|
||||
OBJECT* authority = (OBJECT*)0;
|
||||
KSTATUS status = KeReferenceHandle(process, registrationHandle, ObjectTypeServiceRegistration,
|
||||
SYS_SERVICE_REGISTRATION_RIGHT_UNREGISTER, &authority);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
|
||||
PROCESS* process = KeGetCurrentProcess();
|
||||
SYS_SERVICE_CLAIMS claim = KeSystemServiceRequiredClaim(serviceId);
|
||||
if (!process || !process->SecurityContext || claim == 0 ||
|
||||
!SeContextHasServiceClaims(process->SecurityContext, claim))
|
||||
return NTLSTATUS_ACCESS_DENIED;
|
||||
SYS_SERVICE_ID serviceId = SYS_SERVICE_INVALID;
|
||||
status = KeSystemServiceRegistrationIdentity(authority, &serviceId);
|
||||
ObDereferenceObject(authority);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
|
||||
KSTATUS handleStatus = NTLSTATUS_SUCCESS;
|
||||
PORT* listener = (PORT*)KiHandleToObject(listenerHandle,
|
||||
ObjectTypePort,
|
||||
SYS_PORT_RIGHT_ACCEPT,
|
||||
&handleStatus);
|
||||
PORT* listener = (PORT*)KiHandleToObject(listenerHandle, ObjectTypePort,
|
||||
SYS_PORT_RIGHT_ACCEPT, &handleStatus);
|
||||
if (!listener) return handleStatus;
|
||||
status = KeSystemServiceUnregister(serviceId, listener);
|
||||
ObDereferenceObject((OBJECT*)listener);
|
||||
@@ -3700,8 +3735,6 @@ static uint64_t SyscallDeriveSecurityContext(const SYS_DERIVE_SECURITY_CONTEXT_R
|
||||
status = SeDeriveSecurityContext(source,
|
||||
req.EffectivePrivileges,
|
||||
req.DelegablePrivileges,
|
||||
req.ServiceClaims,
|
||||
req.DelegableServiceClaims,
|
||||
&derived);
|
||||
ObDereferenceObject(&source->Header);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
@@ -6197,6 +6230,10 @@ typedef struct SYSCALL_ENTRY {
|
||||
|
||||
static const SYSCALL_ENTRY gSyscallTable[SYS_MAX] = {
|
||||
// Process, threads, waits, APC, and exceptions.
|
||||
[SYS_OPEN_POWER_EVENT] = SYSCALL_ENTRY1(SyscallOpenPowerEvent),
|
||||
[SYS_REQUEST_SYSTEM_POWER] = SYSCALL_ENTRY1(SyscallRequestSystemPower),
|
||||
[SYS_QUERY_SYSTEM_POWER] = SYSCALL_ENTRY1(SyscallQuerySystemPower),
|
||||
[SYS_COMPLETE_SYSTEM_POWER] = SYSCALL_ENTRY1(SyscallCompleteSystemPower),
|
||||
[SYS_EXIT_PROCESS] = SYSCALL_ENTRY1(SyscallExitProcess),
|
||||
[SYS_EXIT_THREAD] = SYSCALL_ENTRY1(SyscallExitThread),
|
||||
[SYS_CREATE_PROCESS_OBJECT] = SYSCALL_ENTRY1(SyscallCreateProcessObject),
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
#include <kernel/ke_port.h>
|
||||
#include <kernel/ke_spinlock.h>
|
||||
#include <kernel/ob_object.h>
|
||||
#include <ntlite/security.h>
|
||||
#include <kernel/ob_object.h>
|
||||
|
||||
static SPINLOCK gSystemServiceLock = SPINLOCK_INIT;
|
||||
static PORT* gSystemServices[SYS_SERVICE_MAX + 1u];
|
||||
@@ -14,14 +14,35 @@ static bool KiSystemServiceIdValid(SYS_SERVICE_ID serviceId) {
|
||||
return serviceId > SYS_SERVICE_INVALID && serviceId <= SYS_SERVICE_MAX;
|
||||
}
|
||||
|
||||
// Returns the security claim that authorizes publication of one well-known service.
|
||||
SYS_SERVICE_CLAIMS KeSystemServiceRequiredClaim(SYS_SERVICE_ID serviceId) {
|
||||
switch (serviceId) {
|
||||
case SYS_SERVICE_RESOURCE:
|
||||
return SYS_SERVICE_CLAIM_RESOURCE;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
// Authority is an object capability, never a process privilege or service role.
|
||||
typedef struct KE_SERVICE_REGISTRATION_AUTHORITY {
|
||||
OBJECT Header;
|
||||
SYS_SERVICE_ID ServiceId;
|
||||
} KE_SERVICE_REGISTRATION_AUTHORITY;
|
||||
|
||||
// Creates a capability to register one numeric service identity.
|
||||
KSTATUS KeSystemServiceCreateRegistrationAuthority(SYS_SERVICE_ID serviceId, OBJECT** authorityOut) {
|
||||
if (!authorityOut) return NTLSTATUS_INVALID_PARAMETER;
|
||||
*authorityOut = (OBJECT*)0;
|
||||
if (!KiSystemServiceIdValid(serviceId)) return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
KE_SERVICE_REGISTRATION_AUTHORITY* authority =
|
||||
(KE_SERVICE_REGISTRATION_AUTHORITY*)ObAllocateObject(ObjectTypeServiceRegistration,
|
||||
sizeof(*authority));
|
||||
if (!authority) return NTLSTATUS_NO_MEMORY;
|
||||
authority->ServiceId = serviceId;
|
||||
*authorityOut = &authority->Header;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Resolves an authority's registry identity without consulting process identity.
|
||||
KSTATUS KeSystemServiceRegistrationIdentity(const OBJECT* object, SYS_SERVICE_ID* serviceIdOut) {
|
||||
if (!object || !serviceIdOut || object->Type != ObjectTypeServiceRegistration)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
const KE_SERVICE_REGISTRATION_AUTHORITY* authority = (const KE_SERVICE_REGISTRATION_AUTHORITY*)object;
|
||||
if (!KiSystemServiceIdValid(authority->ServiceId)) return NTLSTATUS_INVALID_PARAMETER;
|
||||
*serviceIdOut = authority->ServiceId;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// Registers one weak listener reference without creating a string or pathname namespace.
|
||||
|
||||
+243
-216
@@ -2,10 +2,12 @@
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include <ntlite/bootstrap_capabilities.h>
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#pragma intrinsic(_disable)
|
||||
#include <ntlite/boot_info.h>
|
||||
#include <ntlite/syscall.h>
|
||||
#include <kernel/ke_hal.h>
|
||||
#include <kernel/mm_phys.h>
|
||||
#include <kernel/mm_pfn.h>
|
||||
@@ -13,6 +15,8 @@
|
||||
#include <kernel/mm_swap.h>
|
||||
#include <kernel/ob_object.h>
|
||||
#include <kernel/ke_process.h>
|
||||
#include <kernel/ke_power.h>
|
||||
#include <kernel/mm_file_cache.h>
|
||||
#include <kernel/ke_thread.h>
|
||||
#include <kernel/ke_sched.h>
|
||||
#include <kernel/ke_cpu.h>
|
||||
@@ -23,6 +27,9 @@
|
||||
#include <ntlite/ddi/net_device.h>
|
||||
#include <kernel/ke_debug.h>
|
||||
#include <kernel/ldr_image.h>
|
||||
#include <kernel/driver_module.h>
|
||||
#include <kernel/io_pnp.h>
|
||||
#include <kernel/dm_config.h>
|
||||
#include <kernel/hal/hal_api.h>
|
||||
#include <kernel/io_file.h>
|
||||
#include <kernel/io_volume.h>
|
||||
@@ -38,6 +45,7 @@
|
||||
#include <kernel/ke_work.h>
|
||||
#include <kernel/ke_system_root.h>
|
||||
#include <kernel/se_security.h>
|
||||
#include <kernel/ke_system_service.h>
|
||||
#include <kernel/ke_boot_config.h>
|
||||
|
||||
typedef struct {
|
||||
@@ -123,8 +131,6 @@ static void KiInitializeSwapFromSettings(void) {
|
||||
}
|
||||
}
|
||||
|
||||
typedef KSTATUS (*DRIVER_ENTRY_FN)(const NTL_DDK* ddk, NTL_DRIVER_INIT* init);
|
||||
|
||||
// HAL pointer cached at boot for subsystems that still need platform services.
|
||||
static const HAL_API* gKmainHal;
|
||||
|
||||
@@ -168,231 +174,179 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
|
||||
KiReclaimBootBlob(bootInfo, list->FilesPhysical, list->PageCount);
|
||||
if (bootInfo->BootIni.PhysicalBase && bootInfo->BootIni.PageCount)
|
||||
KiReclaimBootBlob(bootInfo, bootInfo->BootIni.PhysicalBase, bootInfo->BootIni.PageCount);
|
||||
if (bootInfo->DriversIni.PhysicalBase && bootInfo->DriversIni.PageCount)
|
||||
KiReclaimBootBlob(bootInfo, bootInfo->DriversIni.PhysicalBase, bootInfo->DriversIni.PageCount);
|
||||
}
|
||||
|
||||
// Returns whether one drivers.ini line character is horizontal whitespace.
|
||||
static bool KiDriverConfigSpace(char c) {
|
||||
return c == ' ' || c == '\t' || c == '\r';
|
||||
}
|
||||
|
||||
// Trims horizontal whitespace from one immutable drivers.ini line span.
|
||||
static void KiDriverConfigTrim(const char** begin, const char** end) {
|
||||
while (*begin < *end && KiDriverConfigSpace(**begin))
|
||||
++*begin;
|
||||
while (*end > *begin && KiDriverConfigSpace((*end)[-1]))
|
||||
--*end;
|
||||
}
|
||||
|
||||
// Returns whether one immutable drivers.ini span exactly matches an ASCII string.
|
||||
static bool KiDriverConfigSpanEquals(const char* begin, const char* end, const char* text) {
|
||||
while (begin < end && *text) {
|
||||
if (*begin++ != *text++)
|
||||
return false;
|
||||
}
|
||||
return begin == end && *text == '\0';
|
||||
}
|
||||
|
||||
// Loads one runtime driver filename from the system-root drivers directory.
|
||||
static KSTATUS KiLoadRuntimeDriverByName(const char* name, const NTL_DDK* ddk) {
|
||||
char relative[512];
|
||||
size_t length = strlen(name);
|
||||
if (length + 9u > sizeof(relative))
|
||||
return NTLSTATUS_BUFFER_TOO_SMALL;
|
||||
|
||||
// Loads a runtime image from the selected system root, keeping PE identity separate
|
||||
// from the configured module id used for lifecycle and dependency accounting.
|
||||
static KSTATUS KiLoadRuntimeDriver(const DM_DRIVER_CONFIG* driver) {
|
||||
char relative[DM_IMAGE_MAX + 8u];
|
||||
size_t length = strlen(driver->Image);
|
||||
memcpy(relative, "drivers/", 8);
|
||||
memcpy(relative + 8, name, length + 1);
|
||||
memcpy(relative + 8, driver->Image, length + 1);
|
||||
FILE_OBJECT* file = NULL;
|
||||
KSTATUS status = KeOpenSystemRootFile(relative, &file);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
if (!KSUCCESS(status)) return status;
|
||||
|
||||
uint64_t driverSize = IoGetFileSize(file);
|
||||
if (driverSize == 0 || driverSize > SIZE_MAX) {
|
||||
uint64_t size = IoGetFileSize(file);
|
||||
if (!size || size > SIZE_MAX) {
|
||||
IoCloseFile(file);
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
|
||||
void* driverMemory = MmAllocatePool((size_t)driverSize);
|
||||
if (!driverMemory) {
|
||||
void* memory = MmAllocatePool((size_t)size);
|
||||
if (!memory) {
|
||||
IoCloseFile(file);
|
||||
return NTLSTATUS_NO_MEMORY;
|
||||
}
|
||||
|
||||
size_t got = 0;
|
||||
status = IoReadFile(file, driverMemory, (size_t)driverSize, &got);
|
||||
status = IoReadFile(file, memory, (size_t)size, &got);
|
||||
IoCloseFile(file);
|
||||
if (!KSUCCESS(status) || got != (size_t)driverSize) {
|
||||
MmFreePool(driverMemory);
|
||||
if (!KSUCCESS(status) || got != (size_t)size) {
|
||||
MmFreePool(memory);
|
||||
return KSUCCESS(status) ? NTLSTATUS_FAILURE : status;
|
||||
}
|
||||
|
||||
uintptr_t mappedBase = 0;
|
||||
status = LdrLoadKernelImage(driverMemory, (size_t)driverSize, &mappedBase);
|
||||
MmFreePool(driverMemory);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
|
||||
status = LdrResolveImports(mappedBase);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
|
||||
status = LdrRegisterModule(name, mappedBase);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
|
||||
DRIVER_ENTRY_FN entry = (DRIVER_ENTRY_FN)LdrGetEntryPoint(mappedBase);
|
||||
if (!entry)
|
||||
return NTLSTATUS_FAILURE;
|
||||
|
||||
NTL_DRIVER_INIT init = {0};
|
||||
return entry(ddk, &init);
|
||||
uintptr_t base = 0;
|
||||
status = LdrLoadKernelImage(memory, (size_t)size, &base);
|
||||
MmFreePool(memory);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
status = LdrResolveImports(base);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
status = LdrRegisterModule(driver->Image, base);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
status = DmStartLoadedDriver(driver->Id, base);
|
||||
if (KSUCCESS(status))
|
||||
status = LdrSetModuleExportsAvailable(driver->Image, true);
|
||||
return status;
|
||||
}
|
||||
|
||||
// Scans only the [runtime] section of drivers.ini and loads each filename immediately.
|
||||
static KSTATUS KiLoadRuntimeDriversFromConfig(const char* text, size_t length, const NTL_DDK* ddk) {
|
||||
int inRuntime = 0;
|
||||
size_t offset = 0;
|
||||
|
||||
while (offset < length) {
|
||||
size_t lineStart = offset;
|
||||
while (offset < length && text[offset] != '\n')
|
||||
++offset;
|
||||
size_t lineEnd = offset;
|
||||
if (offset < length)
|
||||
++offset;
|
||||
|
||||
const char* begin = text + lineStart;
|
||||
const char* end = text + lineEnd;
|
||||
KiDriverConfigTrim(&begin, &end);
|
||||
if (begin == end || *begin == ';' || *begin == '#')
|
||||
continue;
|
||||
|
||||
if (*begin == '[') {
|
||||
if (end[-1] != ']')
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
const char* sectionBegin = begin + 1;
|
||||
const char* sectionEnd = end - 1;
|
||||
KiDriverConfigTrim(§ionBegin, §ionEnd);
|
||||
inRuntime = KiDriverConfigSpanEquals(sectionBegin, sectionEnd, "runtime");
|
||||
continue;
|
||||
// Starts a preloaded boot driver by its configuration identity, not array position.
|
||||
static KSTATUS KiLoadBootDriver(const BOOT_INFO* bootInfo,
|
||||
const DM_DRIVER_CONFIG* driver) {
|
||||
const BOOT_FILE* files = (const BOOT_FILE*)(bootInfo->PhysicalMapBase +
|
||||
bootInfo->BootDrivers.FilesPhysical);
|
||||
const BOOT_FILE* found = NULL;
|
||||
for (uint32_t i = 0; i < bootInfo->BootDrivers.Count; ++i) {
|
||||
if (!strcmp(files[i].DriverId, driver->Id)) {
|
||||
found = &files[i];
|
||||
break;
|
||||
}
|
||||
|
||||
if (!inRuntime)
|
||||
continue;
|
||||
|
||||
size_t nameLength = (size_t)(end - begin);
|
||||
if (nameLength == 0 || nameLength >= BOOT_FILE_NAME_MAX)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
for (const char* p = begin; p < end; ++p) {
|
||||
if (*p == '=' || *p == '/' || *p == '\\')
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
}
|
||||
|
||||
char name[BOOT_FILE_NAME_MAX];
|
||||
memcpy(name, begin, nameLength);
|
||||
name[nameLength] = '\0';
|
||||
DbgPrint("Found runtime driver: %s\n", name);
|
||||
|
||||
KSTATUS status = KiLoadRuntimeDriverByName(name, ddk);
|
||||
if (!KSUCCESS(status))
|
||||
DbgPrint("KiInitializeDrivers: runtime driver %s failed, status=0x%x\n", name, status);
|
||||
}
|
||||
|
||||
return NTLSTATUS_SUCCESS;
|
||||
if (!found || strcmp(found->Image, driver->Image)) return NTLSTATUS_INVALID_PARAMETER;
|
||||
uintptr_t base = 0;
|
||||
KSTATUS status = LdrLoadKernelImage(
|
||||
(const void*)(bootInfo->PhysicalMapBase + found->PhysicalBase), found->Length, &base);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
status = LdrResolveImports(base);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
status = LdrRegisterModule(driver->Image, base);
|
||||
if (!KSUCCESS(status)) return status;
|
||||
status = DmStartLoadedDriver(driver->Id, base);
|
||||
if (KSUCCESS(status))
|
||||
status = LdrSetModuleExportsAvailable(driver->Image, true);
|
||||
return status;
|
||||
}
|
||||
|
||||
// Initializes drivers.
|
||||
// Validates that the firmware preloaded exactly the configured boot modules.
|
||||
static bool KiCheckBootDriverHandoff(const BOOT_INFO* bootInfo, const DM_CONFIG* config) {
|
||||
const BOOT_FILE* files = (const BOOT_FILE*)(bootInfo->PhysicalMapBase +
|
||||
bootInfo->BootDrivers.FilesPhysical);
|
||||
uint32_t expected = 0;
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i) {
|
||||
const DM_DRIVER_CONFIG* driver = &config->Drivers[i];
|
||||
if (driver->Phase != DmStartBoot) continue;
|
||||
++expected;
|
||||
uint32_t occurrences = 0;
|
||||
for (uint32_t j = 0; j < bootInfo->BootDrivers.Count; ++j) {
|
||||
if (!strcmp(files[j].DriverId, driver->Id) &&
|
||||
!strcmp(files[j].Image, driver->Image)) ++occurrences;
|
||||
}
|
||||
if (occurrences != 1) return false;
|
||||
}
|
||||
return expected == bootInfo->BootDrivers.Count;
|
||||
}
|
||||
|
||||
// Starts the lexicographically first eligible driver, independent of INI order.
|
||||
// Failed providers block requires= dependents, while after= waits for settlement.
|
||||
static void KiRunDriverPhase(DM_CONFIG* config, const BOOT_INFO* bootInfo,
|
||||
DM_START_PHASE phase) {
|
||||
config->ActivePhase = phase;
|
||||
for (;;) {
|
||||
int selected = -1;
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i) {
|
||||
const DM_DRIVER_CONFIG* d = &config->Drivers[i];
|
||||
if (d->Phase != phase || !DmDriverCanStart(config, i)) continue;
|
||||
if (selected < 0 || strcmp(d->Id, config->Drivers[selected].Id) < 0)
|
||||
selected = (int)i;
|
||||
}
|
||||
if (selected >= 0) {
|
||||
DM_DRIVER_CONFIG* d = &config->Drivers[selected];
|
||||
DbgPrint("Driver manager: starting %s (%s)\n", d->Id, d->Image);
|
||||
KSTATUS status = phase == DmStartBoot
|
||||
? KiLoadBootDriver(bootInfo, d) : KiLoadRuntimeDriver(d);
|
||||
d->State = KSUCCESS(status) ? DmRunStarted : DmRunFailed;
|
||||
if (!KSUCCESS(status)) {
|
||||
DbgPrint("Driver manager: failed %s, status=0x%x\n", d->Id, status);
|
||||
if (d->Critical)
|
||||
KePanic(KE_PANIC_CODE_INIT,
|
||||
"Critical driver %s failed (0x%x)\n", d->Id, status);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
bool changed = false;
|
||||
uint32_t waiting = 0;
|
||||
for (uint32_t i = 0; i < config->DriverCount; ++i) {
|
||||
DM_DRIVER_CONFIG* d = &config->Drivers[i];
|
||||
if (d->Phase != phase || d->State != DmRunWaiting) continue;
|
||||
++waiting;
|
||||
if (DmDriverIsBlocked(config, i)) {
|
||||
d->State = DmRunBlocked;
|
||||
DbgPrint("Driver manager: blocked %s by unsatisfied dependency\n", d->Id);
|
||||
if (d->Critical)
|
||||
KePanic(KE_PANIC_CODE_INIT, "Critical driver %s blocked\n", d->Id);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (changed) continue;
|
||||
if (waiting)
|
||||
KePanic(KE_PANIC_CODE_INIT,
|
||||
"Driver manager: no eligible drivers in phase %u (waiting=%u)\n",
|
||||
(unsigned)phase, waiting);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
static DM_CONFIG* gDriverConfig;
|
||||
|
||||
static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
|
||||
(void)hal;
|
||||
const NTL_DDK* ddk = IoGetDdk();
|
||||
DbgPrint("KiInitializeDrivers: Initializing drivers...\n");
|
||||
|
||||
// Boot drivers are page-backed raw files supplied by the bootloader.
|
||||
const BOOT_FILE* bootFiles = bootInfo->BootDrivers.FilesPhysical
|
||||
? (const BOOT_FILE*)(bootInfo->PhysicalMapBase + bootInfo->BootDrivers.FilesPhysical)
|
||||
: NULL;
|
||||
DM_CONFIG* config = (DM_CONFIG*)MmAllocatePool(sizeof(*config));
|
||||
if (!config)
|
||||
KePanic(KE_PANIC_CODE_INIT, "Driver manager: configuration allocation failed\n");
|
||||
const char* text = (const char*)(bootInfo->PhysicalMapBase +
|
||||
bootInfo->DriversIni.PhysicalBase);
|
||||
char error[96] = {0};
|
||||
if (!DmConfigParse(config, text, bootInfo->DriversIni.Length, error, sizeof(error)))
|
||||
KePanic(KE_PANIC_CODE_INIT, "Driver manager: invalid drivers.ini: %s\n", error);
|
||||
if (!KiCheckBootDriverHandoff(bootInfo, config))
|
||||
KePanic(KE_PANIC_CODE_INIT, "Driver manager: boot preloads do not match drivers.ini\n");
|
||||
|
||||
for (uint32_t i = 0; bootFiles && i < bootInfo->BootDrivers.Count; ++i) {
|
||||
const BOOT_FILE* file = &bootFiles[i];
|
||||
if (!file->PhysicalBase || !file->Length) {
|
||||
DbgPrint("KiInitializeDrivers: invalid boot driver descriptor at index %u\n", i);
|
||||
continue;
|
||||
}
|
||||
KSTATUS pnpStatus = IoPnpInitialize();
|
||||
if (!KSUCCESS(pnpStatus))
|
||||
KePanic(KE_PANIC_CODE_INIT, "PnP initialization failed (0x%x)\n", pnpStatus);
|
||||
|
||||
DbgPrint("KiInitializeDrivers: Loading boot driver %s\n", file->Name);
|
||||
void* rawPeBuffer = (void*)(bootInfo->PhysicalMapBase + file->PhysicalBase);
|
||||
KiRunDriverPhase(config, bootInfo, DmStartBoot);
|
||||
pnpStatus = IoPnpWaitForIdle(30000u);
|
||||
if (!KSUCCESS(pnpStatus))
|
||||
KePanic(KE_PANIC_CODE_INIT, "PnP boot startup did not settle (0x%x)\n", pnpStatus);
|
||||
|
||||
uintptr_t mappedBase = 0;
|
||||
KSTATUS st = LdrLoadKernelImage(rawPeBuffer, file->Length, &mappedBase);
|
||||
if (!KSUCCESS(st)) {
|
||||
DbgPrint("ERROR: Failed to map driver %s (0x%X)\n", file->Name, st);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!KSUCCESS(LdrResolveImports(mappedBase))) {
|
||||
DbgPrint("ERROR: Failed to resolve imports for %s\n", file->Name);
|
||||
continue;
|
||||
}
|
||||
|
||||
KSTATUS registerStatus = LdrRegisterModule(file->Name, mappedBase);
|
||||
if (!KSUCCESS(registerStatus)) {
|
||||
DbgPrint("ERROR: Failed to register driver %s (0x%X)\n", file->Name, registerStatus);
|
||||
continue;
|
||||
}
|
||||
|
||||
DRIVER_ENTRY_FN entry = (DRIVER_ENTRY_FN)LdrGetEntryPoint(mappedBase);
|
||||
if (entry) {
|
||||
NTL_DRIVER_INIT init = {0};
|
||||
KSTATUS ds = entry(ddk, &init);
|
||||
if (!KSUCCESS(ds))
|
||||
DbgPrint("WARNING: Driver %s returned failure status 0x%X\n", file->Name, ds);
|
||||
}
|
||||
}
|
||||
|
||||
// runtime drivers.
|
||||
KSTATUS s;
|
||||
FILE_OBJECT* f;
|
||||
|
||||
s = KeInitializeSystemRootVolume();
|
||||
if (s != NTLSTATUS_SUCCESS) {
|
||||
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to find system root volume (status %d)\n", s);
|
||||
}
|
||||
|
||||
|
||||
s = KeOpenSystemRootFile("drivers/drivers.ini", &f);
|
||||
if (s != NTLSTATUS_SUCCESS) {
|
||||
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini on system root (status %d)\n", s);
|
||||
}
|
||||
|
||||
uint64_t fileSize = IoGetFileSize(f);
|
||||
if (fileSize == 0 || fileSize >= SIZE_MAX) {
|
||||
IoCloseFile(f);
|
||||
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: drivers.ini has an invalid size.\n");
|
||||
}
|
||||
|
||||
size_t size = (size_t)fileSize;
|
||||
char* mem = (char*)MmAllocatePool(size + 1u);
|
||||
if (!mem) {
|
||||
IoCloseFile(f);
|
||||
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to allocate drivers.ini buffer.\n");
|
||||
}
|
||||
|
||||
size_t got = 0;
|
||||
KSTATUS readStatus = IoReadFile(f, mem, size, &got);
|
||||
IoCloseFile(f);
|
||||
if (!KSUCCESS(readStatus) || got != size) {
|
||||
MmFreePool(mem);
|
||||
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to read drivers.ini.\n");
|
||||
}
|
||||
mem[got] = '\0';
|
||||
|
||||
KSTATUS runtimeStatus = KiLoadRuntimeDriversFromConfig(mem, got, ddk);
|
||||
MmFreePool(mem);
|
||||
if (!KSUCCESS(runtimeStatus)) {
|
||||
KePanic(KE_PANIC_CODE_INIT,
|
||||
"KiInitializeDrivers: drivers.ini [runtime] is malformed, status=0x%x\n",
|
||||
runtimeStatus);
|
||||
}
|
||||
// Root volume readiness depends on real devices, not the driver module state.
|
||||
KSTATUS status = KeInitializeSystemRootVolume();
|
||||
if (!KSUCCESS(status))
|
||||
KePanic(KE_PANIC_CODE_INIT, "Driver manager: no system root (0x%x)\n", status);
|
||||
KiRunDriverPhase(config, bootInfo, DmStartRuntime);
|
||||
// Retain the authoritative graph and module states for the shutdown slice.
|
||||
gDriverConfig = config;
|
||||
}
|
||||
|
||||
// Phase Helper: Load and launch a process from the filesystem.
|
||||
@@ -442,6 +396,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
SYS_HANDLE volumeManagerHandle = SYS_HANDLE_INVALID;
|
||||
SYS_HANDLE inputManagerHandle = SYS_HANDLE_INVALID;
|
||||
SYS_HANDLE displayManagerHandle = SYS_HANDLE_INVALID;
|
||||
SYS_HANDLE registrationAuthorityHandle = SYS_HANDLE_INVALID;
|
||||
s = KeReferenceSystemRootVolume(&systemVolume);
|
||||
if (KSUCCESS(s)) {
|
||||
s = KeCreateHandle(proc, &systemVolume->Header,
|
||||
@@ -493,20 +448,33 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
ObDereferenceObject(&displayManager->Header);
|
||||
}
|
||||
}
|
||||
if (KSUCCESS(s)) {
|
||||
OBJECT* authority = (OBJECT*)0;
|
||||
s = KeSystemServiceCreateRegistrationAuthority(SYS_SERVICE_RESOURCE, &authority);
|
||||
if (KSUCCESS(s)) {
|
||||
s = KeCreateHandle(proc, authority,
|
||||
SYS_RIGHT_DUPLICATE | SYS_RIGHT_DERIVE | SYS_RIGHT_TRANSFER |
|
||||
SYS_RIGHT_QUERY | SYS_SERVICE_REGISTRATION_RIGHT_REGISTER |
|
||||
SYS_SERVICE_REGISTRATION_RIGHT_UNREGISTER,
|
||||
®istrationAuthorityHandle);
|
||||
ObDereferenceObject(authority);
|
||||
}
|
||||
}
|
||||
if (!KSUCCESS(s)) {
|
||||
DbgPrint("KiLaunchUserProcess: Failed to create bootstrap startup capabilities. Status: %d\n", s);
|
||||
IoCloseFile(f);
|
||||
ObDereferenceObject(&proc->Header);
|
||||
return 0;
|
||||
}
|
||||
SYS_STARTUP_CAPABILITY startupCapabilities[5] = {
|
||||
{ SYS_STARTUP_CAP_SYSTEM_VOLUME, systemVolumeHandle },
|
||||
{ SYS_STARTUP_CAP_SYSTEM_VOLUME_NAMESPACE, namespaceVolumeHandle },
|
||||
{ SYS_STARTUP_CAP_VOLUME_MANAGER, volumeManagerHandle },
|
||||
{ SYS_STARTUP_CAP_INPUT_MANAGER, inputManagerHandle },
|
||||
{ SYS_STARTUP_CAP_DISPLAY_MANAGER, displayManagerHandle },
|
||||
SYS_STARTUP_CAPABILITY startupCapabilities[6] = {
|
||||
{ NTL_BOOT_CAP_SYSTEM_VOLUME, systemVolumeHandle },
|
||||
{ NTL_BOOT_CAP_NAMESPACE_ROOT, namespaceVolumeHandle },
|
||||
{ NTL_BOOT_CAP_VOLUME_MANAGER, volumeManagerHandle },
|
||||
{ NTL_BOOT_CAP_INPUT_MANAGER, inputManagerHandle },
|
||||
{ NTL_BOOT_CAP_DISPLAY_POLICY, displayManagerHandle },
|
||||
{ NTL_BOOT_CAP_RESOURCE_REGISTRATION, registrationAuthorityHandle },
|
||||
};
|
||||
s = KeSetProcessStartupCapabilities(proc, startupCapabilities, 5);
|
||||
s = KeSetProcessStartupCapabilities(proc, startupCapabilities, 6);
|
||||
if (!KSUCCESS(s)) {
|
||||
DbgPrint("KiLaunchUserProcess: Failed to install startup capabilities. Status: %d\n", s);
|
||||
IoCloseFile(f);
|
||||
@@ -538,16 +506,20 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
proc->Flags |= KE_PROCESS_FLAG_TERMINATION_PROTECTED;
|
||||
proc->ImageBase = imageBase;
|
||||
proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
|
||||
|
||||
uintptr_t entryPoint = LdrGetEntryPoint(imageBase);
|
||||
|
||||
KePowerSetInitProcess(proc);
|
||||
|
||||
// Create the main thread last. KeCreateThread queues it before returning.
|
||||
const KE_THREAD_CREATE_PARAMETERS threadParameters =
|
||||
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, proc->Name);
|
||||
s = KeCreateThread(proc, (uintptr_t)entryPoint, 0, &threadParameters, &thread);
|
||||
if (!KSUCCESS(s)) {
|
||||
KePowerSetInitProcess((PROCESS*)0);
|
||||
DbgPrint("KiLaunchUserProcess: Failed to create thread. Status: %d\n", s);
|
||||
proc->ImageBase = 0;
|
||||
proc->ImageSize = 0;
|
||||
@@ -562,16 +534,33 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
|
||||
return proc;
|
||||
}
|
||||
|
||||
static void KiFlushRegisteredDisk(uint8_t disk, DEVICE_OBJECT* device, void* context) {
|
||||
(void)device;
|
||||
const char* phase = (const char*)context;
|
||||
KSTATUS st = IoFlushDisk(disk);
|
||||
if (!KSUCCESS(st) && st != NTLSTATUS_NOT_FOUND)
|
||||
DbgPrint("shutdown: %s disk %u flush status 0x%x\n", phase, (unsigned)disk, st);
|
||||
}
|
||||
|
||||
static void KiFlushRegisteredDisks(const char* phase) {
|
||||
KSTATUS st = IoEnumerateDiskDevices(KiFlushRegisteredDisk, (void*)phase);
|
||||
if (!KSUCCESS(st))
|
||||
DbgPrint("shutdown: %s disk enumeration failed 0x%x\n", phase, st);
|
||||
}
|
||||
|
||||
// Implements the system Init Thread operation.
|
||||
void SystemInitThread(void* arg) {
|
||||
SYSTEM_INIT_ARGS* args = (SYSTEM_INIT_ARGS*)arg;
|
||||
|
||||
DbgPrint("SystemInitThread: Thread started. Igniting the system...\n");
|
||||
|
||||
KiRunEarlyKernelSelfTests();
|
||||
|
||||
KiInitializeDrivers(args->BootInfo, args->Hal);
|
||||
|
||||
KSTATUS powerStatus = KePowerInitialize();
|
||||
if (!KSUCCESS(powerStatus))
|
||||
KePanic(KE_PANIC_CODE_INIT, "Power coordinator failed (0x%x)\n", powerStatus);
|
||||
|
||||
KePowerInitializeButton(args->Hal);
|
||||
KeTimeReloadSettings();
|
||||
|
||||
KiInitializeSwapFromSettings();
|
||||
@@ -579,7 +568,7 @@ void SystemInitThread(void* arg) {
|
||||
KiReclaimBootloaderMemory(args->BootInfo);
|
||||
|
||||
if (args->BootConfig.Debug & KE_DEBUG_SELFTEST)
|
||||
KiRunKernelSelfTests(args->Hal);
|
||||
KiRunKernelSelfTests();
|
||||
|
||||
// Launch the bootstrap service manager. Its allocated PID is not a system contract.
|
||||
PROCESS* initProcess = KiLaunchUserProcess("system/init.exe", args->BootConfig.CommandLine);
|
||||
@@ -588,12 +577,23 @@ void SystemInitThread(void* arg) {
|
||||
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n");
|
||||
}
|
||||
|
||||
KeWaitForEvent(initProcess->ExitEvent, 0);
|
||||
EVENT* exits[2] = { KePowerCompletionEvent(), initProcess->ExitEvent };
|
||||
uint32_t signaled = 0;
|
||||
KSTATUS waitStatus = KeWaitForMultipleEvents(exits, 2u, false,
|
||||
SYS_WAIT_TIMEOUT_INFINITE, &signaled);
|
||||
if (!KSUCCESS(waitStatus) || signaled != 0u)
|
||||
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init exited without power commit\n");
|
||||
(void)KeWaitForEvent(initProcess->ExitEvent, false);
|
||||
int exitCode = initProcess->ExitCode;
|
||||
if (exitCode != 0) {
|
||||
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init.exe exited with code %d.\n", exitCode);
|
||||
}
|
||||
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
|
||||
DbgPrint("SystemInitThread: power commit accepted; init exited with code %d.\n", exitCode);
|
||||
uint32_t action = KePowerCommittedAction();
|
||||
if (!action) KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: missing power action\n");
|
||||
KSTATUS flush = MmFileCacheFlushAll();
|
||||
if (!KSUCCESS(flush)) DbgPrint("shutdown: cache flush status 0x%x\n", flush);
|
||||
KiFlushRegisteredDisks("initial");
|
||||
ObDereferenceObject(&initProcess->Header);
|
||||
|
||||
if (args->BootConfig.Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
|
||||
@@ -601,6 +601,20 @@ void SystemInitThread(void* arg) {
|
||||
if (!KSUCCESS(logStatus))
|
||||
DbgPrint("debuglog: shutdown flush failed st=0x%x\n", logStatus);
|
||||
}
|
||||
flush = MmFileCacheFlushAll();
|
||||
if (!KSUCCESS(flush)) DbgPrint("shutdown: final cache flush status 0x%x\n", flush);
|
||||
KiFlushRegisteredDisks("final");
|
||||
KSTATUS prepare = DmPrepareShutdown();
|
||||
if (!KSUCCESS(prepare)) DbgPrint("shutdown: prepare status 0x%x\n", prepare);
|
||||
KSTATUS pnpShutdown = IoPnpShutdown(10000u);
|
||||
if (!KSUCCESS(pnpShutdown))
|
||||
DbgPrint("shutdown: device stop status 0x%x\n", pnpShutdown);
|
||||
DmCommitShutdown();
|
||||
if (action == SYS_POWER_ACTION_RESTART) {
|
||||
if (!gKmainHal->Power.Reset)
|
||||
KePanic(KE_PANIC_CODE_INIT, "shutdown: reset HAL callback unavailable\n");
|
||||
gKmainHal->Power.Reset();
|
||||
}
|
||||
gKmainHal->Power.Shutdown();
|
||||
MmFreePool(args);
|
||||
}
|
||||
@@ -620,9 +634,11 @@ static bool KiValidateBootFile(const BOOT_FILE* file) {
|
||||
return false;
|
||||
|
||||
uint64_t capacity = (uint64_t)file->PageCount * PAGE_SIZE;
|
||||
if ((uint64_t)file->Length + 1u > capacity || file->Name[0] == '\0')
|
||||
if ((uint64_t)file->Length + 1u > capacity)
|
||||
return false;
|
||||
return memchr(file->Name, '\0', sizeof(file->Name)) != NULL;
|
||||
return file->DriverId[0] != '\0' && file->Image[0] != '\0' &&
|
||||
memchr(file->DriverId, '\0', sizeof(file->DriverId)) &&
|
||||
memchr(file->Image, '\0', sizeof(file->Image));
|
||||
}
|
||||
|
||||
// Returns whether one fixed core-image descriptor fits inside its page-backed physical allocation.
|
||||
@@ -657,10 +673,20 @@ static bool KiValidateBootInfo(const BOOT_INFO* bootInfo) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint64_t driverIniCapacity = (uint64_t)bootInfo->DriversIni.PageCount * PAGE_SIZE;
|
||||
if (!KiValidateBootPageRange(bootInfo->DriversIni.PhysicalBase,
|
||||
bootInfo->DriversIni.PageCount) ||
|
||||
bootInfo->DriversIni.Length == 0 ||
|
||||
bootInfo->DriversIni.Length > DM_CONFIG_MAX_BYTES ||
|
||||
(uint64_t)bootInfo->DriversIni.Length + 1u > driverIniCapacity ||
|
||||
bootInfo->DriversIni.PhysicalBase > UINTPTR_MAX - bootInfo->PhysicalMapBase)
|
||||
return false;
|
||||
|
||||
uint64_t driverListCapacity = (uint64_t)bootInfo->BootDrivers.PageCount * PAGE_SIZE;
|
||||
uint64_t driverListBytes = (uint64_t)bootInfo->BootDrivers.Count * sizeof(BOOT_FILE);
|
||||
if (!KiValidateBootPageRange(bootInfo->BootDrivers.FilesPhysical, bootInfo->BootDrivers.PageCount) ||
|
||||
bootInfo->BootDrivers.Count == 0 || driverListBytes > driverListCapacity ||
|
||||
bootInfo->BootDrivers.Count == 0 || bootInfo->BootDrivers.Count > DM_MAX_DRIVERS ||
|
||||
driverListBytes > driverListCapacity ||
|
||||
bootInfo->BootDrivers.FilesPhysical > UINTPTR_MAX - bootInfo->PhysicalMapBase) {
|
||||
return false;
|
||||
}
|
||||
@@ -825,7 +851,8 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
|
||||
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to admit APs to scheduler, status=0x%x\n", smpSchedStatus);
|
||||
}
|
||||
|
||||
KiRunCoreHardeningSelfTests();
|
||||
if (bootConfig.Debug & KE_DEBUG_SELFTEST)
|
||||
KiCheckBootState();
|
||||
|
||||
// Pack the arguments for the init thread.
|
||||
SYSTEM_INIT_ARGS* initArgs = MmAllocatePool(sizeof(SYSTEM_INIT_ARGS));
|
||||
|
||||
+26
-5
@@ -170,6 +170,7 @@ typedef struct MODULE_ENTRY {
|
||||
char Name[LDR_KERNEL_MODULE_NAME_MAX];
|
||||
uintptr_t Base;
|
||||
size_t Size;
|
||||
bool ExportsAvailable;
|
||||
struct MODULE_ENTRY* Next;
|
||||
} MODULE_ENTRY;
|
||||
|
||||
@@ -182,7 +183,8 @@ static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t add
|
||||
}
|
||||
|
||||
// Allocates and inserts one global kernel-module registry entry.
|
||||
static KSTATUS KiRegisterGlobalModule(const char* name, uintptr_t base, size_t size) {
|
||||
static KSTATUS KiRegisterGlobalModule(const char* name, uintptr_t base, size_t size,
|
||||
bool exportsAvailable) {
|
||||
if (!name || !*name || !base || !size)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
@@ -201,6 +203,7 @@ static KSTATUS KiRegisterGlobalModule(const char* name, uintptr_t base, size_t s
|
||||
entry->Name[nameLength] = '\0';
|
||||
entry->Base = base;
|
||||
entry->Size = size;
|
||||
entry->ExportsAvailable = exportsAvailable;
|
||||
entry->Next = gModules;
|
||||
gModules = entry;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
@@ -213,19 +216,32 @@ KSTATUS LdrRegisterModulesFromBootInfo(const BOOT_INFO* bootInfo) {
|
||||
|
||||
KSTATUS status = KiRegisterGlobalModule("kernel.exe",
|
||||
bootInfo->KernelImage.VirtualBase,
|
||||
bootInfo->KernelImage.ImageSize);
|
||||
bootInfo->KernelImage.ImageSize, true);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
|
||||
return KiRegisterGlobalModule("hal.dll",
|
||||
bootInfo->HalImage.VirtualBase,
|
||||
bootInfo->HalImage.ImageSize);
|
||||
bootInfo->HalImage.ImageSize, true);
|
||||
}
|
||||
|
||||
// Registers one loaded kernel module without imposing a fixed registry capacity.
|
||||
KSTATUS LdrRegisterModule(const char* name, uintptr_t base) {
|
||||
size_t size = base ? LdrGetImageSize(base) : 0;
|
||||
return KiRegisterGlobalModule(name, base, size);
|
||||
return KiRegisterGlobalModule(name, base, size, false);
|
||||
}
|
||||
|
||||
// Makes an initialized module's exports visible to future import resolution.
|
||||
// Failed images remain mapped for diagnostics but must not provide exports.
|
||||
KSTATUS LdrSetModuleExportsAvailable(const char* name, bool available) {
|
||||
if (!name) return NTLSTATUS_INVALID_PARAMETER;
|
||||
for (MODULE_ENTRY* entry = gModules; entry; entry = entry->Next) {
|
||||
if (_stricmp(entry->Name, name) == 0) {
|
||||
entry->ExportsAvailable = available;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
return NTLSTATUS_NOT_FOUND;
|
||||
}
|
||||
|
||||
// Finds the loaded kernel module that owns an address.
|
||||
@@ -251,7 +267,7 @@ const char* LdrFindKernelModuleByAddress(uintptr_t address, uintptr_t* baseOut,
|
||||
static uintptr_t KiFindGlobalModuleBase(const char* name) {
|
||||
if (!name) return 0;
|
||||
for (MODULE_ENTRY* module = gModules; module; module = module->Next) {
|
||||
if (_stricmp(module->Name, name) == 0)
|
||||
if (module->ExportsAvailable && _stricmp(module->Name, name) == 0)
|
||||
return module->Base;
|
||||
}
|
||||
return 0;
|
||||
@@ -286,6 +302,11 @@ static void* KiGetExportAtBase(uintptr_t base, const char* functionName) {
|
||||
return (void*)0;
|
||||
}
|
||||
|
||||
// Resolves a named export without depending on the image's registration name.
|
||||
void* LdrGetImageExport(uintptr_t imageBase, const char* functionName) {
|
||||
return functionName ? KiGetExportAtBase(imageBase, functionName) : (void*)0;
|
||||
}
|
||||
|
||||
// Export lookup.
|
||||
void* LdrGetExport(const char* moduleName, const char* functionName) {
|
||||
uintptr_t base = KiFindGlobalModuleBase(moduleName);
|
||||
|
||||
@@ -813,6 +813,25 @@ KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf,
|
||||
return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
|
||||
KSTATUS MmFileCacheFlushAll(void) {
|
||||
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
|
||||
for (uint32_t i = 0; i < 65536u; ++i) {
|
||||
int didWork = 0;
|
||||
KSTATUS st = KiFileCacheFlushOneDirtyPage((FILE_OBJECT*)0, &didWork);
|
||||
if (st == NTLSTATUS_NOT_FOUND && !didWork) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
|
||||
bool clean = gFileCacheDirtyPages == 0;
|
||||
KeReleaseSpinlock(&gFileCacheLock, flags);
|
||||
if (clean) return NTLSTATUS_SUCCESS;
|
||||
KeYield();
|
||||
continue;
|
||||
}
|
||||
if (!KSUCCESS(st)) return st;
|
||||
if (!didWork) return NTLSTATUS_SUCCESS;
|
||||
}
|
||||
return NTLSTATUS_LIMIT_REACHED;
|
||||
}
|
||||
|
||||
KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file) {
|
||||
if (!file)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
+15
-6
@@ -794,12 +794,14 @@ void MmPfnQueryStats(MM_PFN_STATS* out) {
|
||||
MM_PFN_STATS stats;
|
||||
memset(&stats, 0, sizeof(stats));
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
||||
stats.TotalFrames = (uint64_t)gTotalFrames;
|
||||
stats.FreeFrames = (uint64_t)gFreeFrames;
|
||||
stats.UsedFrames = stats.TotalFrames - stats.FreeFrames;
|
||||
|
||||
for (size_t pfn = 0; pfn < gMaxPfn; pfn++) {
|
||||
// Detailed statistics are diagnostic snapshots. Scan bounded PFN batches
|
||||
// so an ordinary query cannot monopolize the global allocator lock on
|
||||
// machines with millions of physical frames. Counters are sampled at end.
|
||||
const size_t pfnCount = gMaxPfn;
|
||||
for (size_t base = 0; base < pfnCount;) {
|
||||
size_t end = pfnCount - base < 1024u ? pfnCount : base + 1024u;
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
||||
for (size_t pfn = base; pfn < end; ++pfn) {
|
||||
const MM_PFN* entry = &gPfnDatabase[pfn];
|
||||
if (entry->State == MmPfnStateInvalid)
|
||||
continue;
|
||||
@@ -847,8 +849,15 @@ void MmPfnQueryStats(MM_PFN_STATS* out) {
|
||||
case MmPfnUseMmio: stats.MmioFrames++; break;
|
||||
default: break;
|
||||
}
|
||||
}
|
||||
KeReleaseSpinlock(&gPfnLock, flags);
|
||||
base = end;
|
||||
}
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&gPfnLock);
|
||||
stats.TotalFrames = (uint64_t)gTotalFrames;
|
||||
stats.FreeFrames = (uint64_t)gFreeFrames;
|
||||
stats.UsedFrames = stats.TotalFrames - stats.FreeFrames;
|
||||
stats.AvailableFrames = stats.FreeFrames + stats.ReclaimableFrames;
|
||||
|
||||
stats.AllocationCount = gPfnAllocations;
|
||||
|
||||
@@ -39,6 +39,7 @@ static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
|
||||
case ObjectTypeInputManager: return MM_POOL_TAG('i', 'm', 'g', 'r');
|
||||
case ObjectTypeDisplayManager: return MM_POOL_TAG('d', 'm', 'g', 'r');
|
||||
case ObjectTypePipeEndpoint: return MM_POOL_TAG('p', 'i', 'p', 'e');
|
||||
case ObjectTypeServiceRegistration: return MM_POOL_TAG('s', 'r', 'e', 'g');
|
||||
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,20 +17,15 @@ static void KiSecurityContextDestroy(OBJECT* object) {
|
||||
// Allocates and initializes one immutable security context object.
|
||||
static KSTATUS KiCreateSecurityContext(SYS_PRIVILEGES effectivePrivileges,
|
||||
SYS_PRIVILEGES delegablePrivileges,
|
||||
SYS_SERVICE_CLAIMS serviceClaims,
|
||||
SYS_SERVICE_CLAIMS delegableServiceClaims,
|
||||
SECURITY_CONTEXT** contextOut) {
|
||||
if (!contextOut)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
*contextOut = (SECURITY_CONTEXT*)0;
|
||||
|
||||
if ((effectivePrivileges & ~SYS_PRIV_ALL) != 0 ||
|
||||
(delegablePrivileges & ~SYS_PRIV_ALL) != 0 ||
|
||||
(serviceClaims & ~SYS_SERVICE_CLAIM_ALL) != 0 ||
|
||||
(delegableServiceClaims & ~SYS_SERVICE_CLAIM_ALL) != 0)
|
||||
(delegablePrivileges & ~SYS_PRIV_ALL) != 0)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
if ((delegablePrivileges & ~effectivePrivileges) != 0 ||
|
||||
(delegableServiceClaims & ~serviceClaims) != 0)
|
||||
if ((delegablePrivileges & ~effectivePrivileges) != 0)
|
||||
return NTLSTATUS_ACCESS_DENIED;
|
||||
|
||||
SECURITY_CONTEXT* context = (SECURITY_CONTEXT*)ObAllocateObject(ObjectTypeSecurityContext,
|
||||
@@ -47,8 +42,6 @@ static KSTATUS KiCreateSecurityContext(SYS_PRIVILEGES effectivePrivileges,
|
||||
|
||||
context->EffectivePrivileges = effectivePrivileges;
|
||||
context->DelegablePrivileges = delegablePrivileges;
|
||||
context->ServiceClaims = serviceClaims;
|
||||
context->DelegableServiceClaims = delegableServiceClaims;
|
||||
context->Flags = SYS_SECURITY_CONTEXT_FLAG_NONE;
|
||||
*contextOut = context;
|
||||
return NTLSTATUS_SUCCESS;
|
||||
@@ -62,8 +55,6 @@ KSTATUS SeInitializeSecurity(void) {
|
||||
SECURITY_CONTEXT* context = (SECURITY_CONTEXT*)0;
|
||||
KSTATUS status = KiCreateSecurityContext(SYS_PRIV_ALL,
|
||||
SYS_PRIV_ALL,
|
||||
SYS_SERVICE_CLAIM_ALL,
|
||||
SYS_SERVICE_CLAIM_ALL,
|
||||
&context);
|
||||
if (!KSUCCESS(status))
|
||||
return status;
|
||||
@@ -81,22 +72,16 @@ SECURITY_CONTEXT* SeGetRootSecurityContext(void) {
|
||||
KSTATUS SeDeriveSecurityContext(SECURITY_CONTEXT* source,
|
||||
SYS_PRIVILEGES effectivePrivileges,
|
||||
SYS_PRIVILEGES delegablePrivileges,
|
||||
SYS_SERVICE_CLAIMS serviceClaims,
|
||||
SYS_SERVICE_CLAIMS delegableServiceClaims,
|
||||
SECURITY_CONTEXT** contextOut) {
|
||||
if (!source || !contextOut)
|
||||
return NTLSTATUS_INVALID_PARAMETER;
|
||||
|
||||
if ((effectivePrivileges & ~source->DelegablePrivileges) != 0 ||
|
||||
(delegablePrivileges & ~source->DelegablePrivileges) != 0 ||
|
||||
(serviceClaims & ~source->DelegableServiceClaims) != 0 ||
|
||||
(delegableServiceClaims & ~source->DelegableServiceClaims) != 0)
|
||||
(delegablePrivileges & ~source->DelegablePrivileges) != 0)
|
||||
return NTLSTATUS_ACCESS_DENIED;
|
||||
|
||||
return KiCreateSecurityContext(effectivePrivileges,
|
||||
delegablePrivileges,
|
||||
serviceClaims,
|
||||
delegableServiceClaims,
|
||||
contextOut);
|
||||
}
|
||||
|
||||
@@ -107,13 +92,6 @@ bool SeProcessHasPrivilege(const struct PROCESS* process, SYS_PRIVILEGES privile
|
||||
return (process->SecurityContext->EffectivePrivileges & privileges) == privileges;
|
||||
}
|
||||
|
||||
// Tests one security context's effective service-claim mask.
|
||||
bool SeContextHasServiceClaims(const SECURITY_CONTEXT* context, SYS_SERVICE_CLAIMS claims) {
|
||||
if (!context)
|
||||
return false;
|
||||
return (context->ServiceClaims & claims) == claims;
|
||||
}
|
||||
|
||||
// Exports immutable context metadata to one public structure.
|
||||
void SeQuerySecurityContext(const SECURITY_CONTEXT* context, SYS_SECURITY_CONTEXT_INFO* infoOut) {
|
||||
if (!context || !infoOut)
|
||||
@@ -124,7 +102,5 @@ void SeQuerySecurityContext(const SECURITY_CONTEXT* context, SYS_SECURITY_CONTEX
|
||||
infoOut->SecurityId = context->SecurityId;
|
||||
infoOut->EffectivePrivileges = context->EffectivePrivileges;
|
||||
infoOut->DelegablePrivileges = context->DelegablePrivileges;
|
||||
infoOut->ServiceClaims = context->ServiceClaims;
|
||||
infoOut->DelegableServiceClaims = context->DelegableServiceClaims;
|
||||
infoOut->Flags = context->Flags;
|
||||
}
|
||||
Reference in new issue
Block a user