87 lines
3.0 KiB
C
87 lines
3.0 KiB
C
// Kernel checked intrusive-list helpers. This file detects stale or duplicate membership at the mutation site.
|
|
|
|
#include <stdint.h>
|
|
#include <kernel/ke_list.h>
|
|
#include <kernel/ke_debug.h>
|
|
|
|
#define KE_LIST_VALIDATION_LIMIT 1048576u
|
|
|
|
// Returns whether checked list validation is enabled for this boot.
|
|
static bool KiListChecksEnabled(void) {
|
|
return (KeGetDebugFlags() & KE_DEBUG_LIST_CHECKS) != 0;
|
|
}
|
|
|
|
// Stops on a checked-list invariant violation with the involved addresses.
|
|
static _Noreturn void KiListPanic(const char* operation,
|
|
const LIST_ENTRY* head,
|
|
const LIST_ENTRY* node) {
|
|
KePanic(KE_PANIC_CODE_GENERIC,
|
|
"list: %s invariant failed head=0x%016llX node=0x%016llX fwd=0x%016llX bwd=0x%016llX\n",
|
|
operation ? operation : "unknown",
|
|
(uint64_t)(uintptr_t)head,
|
|
(uint64_t)(uintptr_t)node,
|
|
(uint64_t)(uintptr_t)(node ? node->Fwd : (LIST_ENTRY*)0),
|
|
(uint64_t)(uintptr_t)(node ? node->Bwd : (LIST_ENTRY*)0));
|
|
}
|
|
|
|
// Validates a list head and every forward/backward linkage reachable from it.
|
|
void KeListValidateHead(const LIST_ENTRY* head) {
|
|
if (!KiListChecksEnabled())
|
|
return;
|
|
if (!head || !head->Fwd || !head->Bwd)
|
|
KiListPanic("validate-head", head, head);
|
|
|
|
const LIST_ENTRY* node = head;
|
|
for (uint32_t count = 0; count < KE_LIST_VALIDATION_LIMIT; count++) {
|
|
if (!node->Fwd || !node->Bwd)
|
|
KiListPanic("validate-link", head, node);
|
|
if (node->Fwd->Bwd != node || node->Bwd->Fwd != node)
|
|
KiListPanic("validate-neighbor", head, node);
|
|
|
|
node = node->Fwd;
|
|
if (node == head)
|
|
return;
|
|
}
|
|
|
|
KiListPanic("validate-cycle", head, node);
|
|
}
|
|
|
|
// Inserts a detached node at the tail after validating both invariants.
|
|
void KeListInsertTailChecked(LIST_ENTRY* head, LIST_ENTRY* node) {
|
|
if (KiListChecksEnabled()) {
|
|
KeListValidateHead(head);
|
|
if (!node || node->Fwd != node || node->Bwd != node)
|
|
KiListPanic("insert-tail", head, node);
|
|
}
|
|
|
|
ListInsertTail(head, node);
|
|
KeListValidateHead(head);
|
|
}
|
|
|
|
// Inserts a detached node at the head after validating both invariants.
|
|
void KeListInsertHeadChecked(LIST_ENTRY* head, LIST_ENTRY* node) {
|
|
if (KiListChecksEnabled()) {
|
|
KeListValidateHead(head);
|
|
if (!node || node->Fwd != node || node->Bwd != node)
|
|
KiListPanic("insert-head", head, node);
|
|
}
|
|
|
|
ListInsertHead(head, node);
|
|
KeListValidateHead(head);
|
|
}
|
|
|
|
// Removes a linked node and rejects detached or locally corrupt entries.
|
|
void KeListRemoveChecked(LIST_ENTRY* node) {
|
|
if (KiListChecksEnabled()) {
|
|
if (!node || !node->Fwd || !node->Bwd ||
|
|
(node->Fwd == node && node->Bwd == node) ||
|
|
node->Fwd->Bwd != node || node->Bwd->Fwd != node) {
|
|
KiListPanic("remove", (LIST_ENTRY*)0, node);
|
|
}
|
|
}
|
|
|
|
ListRemove(node);
|
|
if (KiListChecksEnabled() && (node->Fwd != node || node->Bwd != node))
|
|
KiListPanic("remove-detach", (LIST_ENTRY*)0, node);
|
|
}
|