// Kernel checked intrusive-list helpers. This file detects stale or duplicate membership at the mutation site. #include #include #include #define KE_LIST_VALIDATION_LIMIT 1048576u // Returns whether checked list validation is enabled for this boot. static bool KiListChecksEnabled(void) { return (KeGetDebugFlags() & KE_DEBUG_LIST_CHECKS) != 0; } // Stops on a checked-list invariant violation with the involved addresses. static _Noreturn void KiListPanic(const char* operation, const LIST_ENTRY* head, const LIST_ENTRY* node) { KePanic(KE_PANIC_CODE_GENERIC, "list: %s invariant failed head=0x%016llX node=0x%016llX fwd=0x%016llX bwd=0x%016llX\n", operation ? operation : "unknown", (uint64_t)(uintptr_t)head, (uint64_t)(uintptr_t)node, (uint64_t)(uintptr_t)(node ? node->Fwd : (LIST_ENTRY*)0), (uint64_t)(uintptr_t)(node ? node->Bwd : (LIST_ENTRY*)0)); } // Validates a list head and every forward/backward linkage reachable from it. void KeListValidateHead(const LIST_ENTRY* head) { if (!KiListChecksEnabled()) return; if (!head || !head->Fwd || !head->Bwd) KiListPanic("validate-head", head, head); const LIST_ENTRY* node = head; for (uint32_t count = 0; count < KE_LIST_VALIDATION_LIMIT; count++) { if (!node->Fwd || !node->Bwd) KiListPanic("validate-link", head, node); if (node->Fwd->Bwd != node || node->Bwd->Fwd != node) KiListPanic("validate-neighbor", head, node); node = node->Fwd; if (node == head) return; } KiListPanic("validate-cycle", head, node); } // Inserts a detached node at the tail after validating both invariants. void KeListInsertTailChecked(LIST_ENTRY* head, LIST_ENTRY* node) { if (KiListChecksEnabled()) { KeListValidateHead(head); if (!node || node->Fwd != node || node->Bwd != node) KiListPanic("insert-tail", head, node); } ListInsertTail(head, node); KeListValidateHead(head); } // Inserts a detached node at the head after validating both invariants. void KeListInsertHeadChecked(LIST_ENTRY* head, LIST_ENTRY* node) { if (KiListChecksEnabled()) { KeListValidateHead(head); if (!node || node->Fwd != node || node->Bwd != node) KiListPanic("insert-head", head, node); } ListInsertHead(head, node); KeListValidateHead(head); } // Removes a linked node and rejects detached or locally corrupt entries. void KeListRemoveChecked(LIST_ENTRY* node) { if (KiListChecksEnabled()) { if (!node || !node->Fwd || !node->Bwd || (node->Fwd == node && node->Bwd == node) || node->Fwd->Bwd != node || node->Bwd->Fwd != node) { KiListPanic("remove", (LIST_ENTRY*)0, node); } } ListRemove(node); if (KiListChecksEnabled() && (node->Fwd != node || node->Bwd != node)) KiListPanic("remove-detach", (LIST_ENTRY*)0, node); }