maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

Browse Source
This commit is contained in:
maksw committed 2026-06-20 13:17:39 +02:00
1 parent 61ec2442eb
commit 2c4a68a75f
114 files changed
+18069 -2620
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/include/kernel/io_file.h
+60 -1
View File
@@ -25,6 +25,39 @@ typedef void (*IO_FS_ASYNC_COMPLETE)(void* completeContext,
KSTATUS status,
size_t information);
// Filesystem-level cache and traversal statistics.
typedef struct IO_FILESYSTEM_STATS {
uint64_t PathLookups;
uint64_t PathCacheHits;
uint64_t PathCacheMisses;
uint64_t PathCacheStores;
uint64_t PathCacheInvalidations;
uint64_t PathCacheGeneration;
uint64_t DirectorySearches;
uint64_t DirectoryClustersRead;
uint64_t DirectoryEntriesScanned;
uint64_t FatChainCacheLookups;
uint64_t FatChainCacheHits;
uint64_t FatChainCacheMisses;
uint64_t FatChainCacheStores;
uint64_t FatChainCacheInvalidations;
uint64_t FatChainWalkSteps;
uint64_t DataReadRuns;
uint64_t DataReadSectors;
uint64_t DataReadRunMerges;
uint64_t MaxDataReadSpanSectors;
} IO_FILESYSTEM_STATS;
// Stable identity for a file on a mounted filesystem.
// Filesystems that can provide this let higher layers cache file-backed
// objects by file identity instead of by one FILE_OBJECT instance.
typedef struct IO_FILE_IDENTITY {
uint64_t VolumeId;
uint64_t FileId;
uint64_t Generation;
uint64_t Size;
} IO_FILE_IDENTITY;
// Filesystem driver vtable - filled in by a filesystem driver.
typedef struct IO_FILESYSTEM {
// Open a file by absolute path (e.g. "apps\\shell.exe").
@@ -69,14 +102,21 @@ typedef struct IO_FILESYSTEM {
// Truncate or extend the file to exactly 'newSize' bytes.
KSTATUS (*Truncate)(struct IO_FILESYSTEM* fs, void* fileCtx,
uint64_t newSize);
uint64_t newSize, uint32_t fileFlags);
// Return the file size in bytes.
uint64_t (*GetSize)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Return a stable file identity for caches that outlive one open handle.
KSTATUS (*QueryIdentity)(struct IO_FILESYSTEM* fs, void* fileCtx,
IO_FILE_IDENTITY* out);
// Release any resources the driver allocated for fileCtx.
void (*Close)(struct IO_FILESYSTEM* fs, void* fileCtx);
// Query cache and traversal statistics exposed through kernel stats.
void (*QueryStats)(struct IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out);
// Optional driver-private data.
void* DriverCtx;
@@ -106,9 +146,13 @@ typedef struct IO_STATUS_BLOCK {
struct EVENT;
struct KAPC;
typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
typedef struct IO_ASYNC_FILE_REQUEST_OBJECT IO_ASYNC_FILE_REQUEST_OBJECT;
// FILE_OBJECT - the kernel object returned to callers.
#define IO_FILE_FLAG_NO_CACHE 0x00000001u
#define IO_FILE_FLAG_SKIP_ZERO_EXTEND 0x00000002u
typedef struct FILE_OBJECT {
OBJECT Header; // must be first
IO_FILESYSTEM* Fs;
@@ -116,6 +160,11 @@ typedef struct FILE_OBJECT {
SPINLOCK Lock; // protects cached Size/Offset updates
uint64_t Size;
uint64_t Offset; // current position for sequential reads
uint32_t Flags; // IO_FILE_FLAG_*
uint32_t ReadaheadScore;
uint64_t ReadaheadNextOffset;
uint64_t ReadaheadIssuedUntil;
uint64_t ReadaheadSuppressedUntil;
} FILE_OBJECT;
// DIR_OBJECT - kernel object for an open directory iterator.
@@ -137,6 +186,7 @@ __declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t driveIndex, IO_FILESY
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t driveIndex);
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex);
__declspec(dllexport) bool IoFilesystemPresent(uint8_t driveIndex);
__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out);
// Kernel-side API.
__declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out);
@@ -149,15 +199,23 @@ __declspec(dllexport) KSTATUS IoWriteFile (FILE_OBJECT* file, const void* buf,
__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
const void* buf, size_t size,
size_t* bytesWritten);
__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file);
__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf,
size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
struct EVENT* completionEvent, struct KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
size_t size, uint64_t offset, IO_STATUS_BLOCK* iosb,
struct EVENT* completionEvent, struct KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut);
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
uint32_t flags);
__declspec(dllexport) struct EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request);
__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* request,
IO_STATUS_BLOCK* iosbOut);
@@ -169,6 +227,7 @@ __declspec(dllexport) KSTATUS IoRemoveDir (SYS_STRING_VIEW path);
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize);
__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset);
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file);
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out);
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file);
// Directory enumeration API.
code/kernel/include/kernel/ke_debug.h
+11 -1
View File
@@ -12,6 +12,16 @@
#define KE_DEBUG_SELFTEST 0x00000002u
#define KE_DEBUG_FILE_LOG 0x00000004u
#define KE_DEBUG_POOL_CANARY 0x00000008u
#define KE_DEBUG_POOL_POISON 0x00000010u
#define KE_DEBUG_POOL_QUARANTINE 0x00000020u
#define KE_DEBUG_POOL_CALLERS 0x00000040u
#define KE_DEBUG_DISPLAY_ASSERTS 0x00000080u
#define KE_DEBUG_DISPLAY_TRACE 0x00000100u
#define KE_DEBUG_INPUT_TRACE 0x00000200u
#define KE_DEBUG_VM_ASSERTS 0x00000400u
#define KE_DEBUG_SYSCALL_ASSERTS 0x00000800u
void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags);
void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t));
uint32_t KeGetDebugFlags(void);
@@ -68,7 +78,7 @@ typedef struct KFAULT_FRAME {
uint64_t R15;
} KFAULT_FRAME;
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame);
__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame);
// Compatibility wrapper for any older assembly/object still calling this.
__declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
code/kernel/include/kernel/ke_display.h
+2
View File
@@ -42,7 +42,9 @@ DISPLAY_API void KeDisplaySuspendConsole(bool suspend);
DISPLAY_API KSTATUS KeDisplayGetMode(DISPLAY_MODE* outMode);
DISPLAY_API KSTATUS KeDisplayGetSize(int* outWidth, int* outHeight);
typedef KSTATUS (*KE_DISPLAY_ROW_SOURCE)(void* context, int rowIndex, uint32_t* scratch, size_t pixelCount);
DISPLAY_API KSTATUS KeDisplayBlit(int x, int y, int w, int h, const uint32_t* pixels);
DISPLAY_API KSTATUS KeDisplayBlitRows(int x, int y, int w, int h, KE_DISPLAY_ROW_SOURCE source, void* context, uint32_t* scratch);
DISPLAY_API KSTATUS KeDisplayFillRect(int x, int y, int w, int h, uint32_t color);
DISPLAY_API KSTATUS KeDisplayClear(uint32_t color);
DISPLAY_API KSTATUS KeDisplayCopyRect(int dstX, int dstY, int srcX, int srcY, int w, int h);
code/kernel/include/kernel/ke_event.h
+3
View File
@@ -16,9 +16,12 @@ typedef struct EVENT {
EVENT_TYPE Type;
uint32_t Signaled;
LIST_ENTRY WaitList;
LIST_ENTRY MultiWaitList;
} EVENT;
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out);
__declspec(dllexport) void KeSignalEvent(EVENT* event);
__declspec(dllexport) void KeClearEvent(EVENT* event);
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable);
__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event);
__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count, uint8_t alertable, uint64_t timeoutMs, uint32_t* signaledIndexOut);
code/kernel/include/kernel/ke_input.h
+12
View File
@@ -6,6 +6,13 @@
#include <ntlite/input.h>
#include <ntlite/ddi/hal_api.h>
#include <ntlite/status.h>
#include <kernel/ob_object.h>
#include <kernel/ke_event.h>
typedef struct INPUT_QUEUE {
OBJECT Header;
EVENT* WaitEvent;
} INPUT_QUEUE;
void KeInputInitialize(const HAL_API* hal);
@@ -16,7 +23,12 @@ __declspec(dllexport) void KeInputSubmitMouseAbsolute(const INPUT_MOUSE_ABSOLUTE
__declspec(dllexport) void KeInputSubmitMouseAbsoluteScaled(uint32_t rawX, uint32_t rawY, uint32_t maxX, uint32_t maxY, int wheel, uint8_t buttons);
INPUT_KEY KeInputKbPoll(void);
KSTATUS KeInputWaitKey(uint8_t alertable);
bool KeInputIsKeyDown(INPUT_KEY key);
bool KeInputPeekEvent(INPUT_KEY_RECORD* record);
bool KeInputReadEvent(INPUT_KEY_RECORD* record);
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out);
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue);
INPUT_KEY KeInputReadKey(void);
char KeInputReadChar(void);
uint32_t KeInputGetModifiers(void);
code/kernel/include/kernel/ke_mutex.h
+4 -4
View File
@@ -14,7 +14,7 @@ typedef struct MUTEX {
LIST_ENTRY WaitList;
} MUTEX;
KSTATUS KeMutexCreate(MUTEX **out);
KSTATUS KeAcquireMutex(MUTEX* mutex);
KSTATUS KeTryAcquireMutex(MUTEX* mutex);
void KeReleaseMutex(MUTEX* mutex);
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out);
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex);
__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex);
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex);
code/kernel/include/kernel/ke_process.h
+8 -2
View File
@@ -62,6 +62,7 @@ typedef struct PROCESS {
int ExitCode;
volatile long Terminating;
volatile long Exited;
volatile long ResourcesReleased;
// Main executable image. Used for cleanup.
uintptr_t ImageBase;
@@ -80,8 +81,8 @@ typedef struct PROCESS {
// Per-process user virtual regions. The actual VA arena is global because
// NT-Lite currently uses one shared address space; this list tracks
// ownership so leaked user mappings can be reclaimed on process exit.
LIST_ENTRY UserRegions;
SPINLOCK UserRegionLock;
LIST_ENTRY VadList;
SPINLOCK VadLock;
// Per-process loaded modules (user processes only).
PROCESS_MODULE Modules[PROCESS_MODULE_MAX];
@@ -102,6 +103,7 @@ KSTATUS KeSetupUserProcessEnvironment(PROCESS* process,
SYS_STRING_VIEW currentDirectory);
void KeExitProcess(PROCESS* process, int exitCode);
KSTATUS KeTerminateProcess(PROCESS* process, int exitCode);
void KeCompleteProcessExitIfLastThread(PROCESS* process);
int KeIsProcessTerminating(PROCESS* process);
KSTATUS KeGetProcessExitCode(PROCESS* process, int* exitCodeOut);
@@ -118,8 +120,12 @@ LIST_ENTRY* KeGetProcessList(void);
PROCESS* KeGetCurrentProcess(void);
PROCESS* KeGetSystemProcess(void);
struct SYS_PROCESS_INFO;
struct SYS_THREAD_INFO;
// Copy a snapshot of rich process information into caller-provided storage.
size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount);
// Copy a snapshot of one process's threads into caller-provided storage.
KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut);
// Find a live process by its ProcessId. Bumps refcount; caller must ObDereferenceObject.
PROCESS* KeGetProcessById(uint32_t pid);
code/kernel/include/kernel/ke_sched.h
+4
View File
@@ -58,6 +58,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread);
void KiSchedulerMarkThreadWaiting(THREAD* thread);
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status);
// Blocks the current thread until at least the requested delay has elapsed.
__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds);
KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus);
// Voluntarily yield the CPU. Picks the next runnable thread immediately.
__declspec(dllexport) void KeYield(void);
code/kernel/include/kernel/ke_system_root.h
+25
View File
@@ -0,0 +1,25 @@
// System root volume declarations. This file exposes the one selected kernel root volume to loader and process code.
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <ntlite/status.h>
#include <kernel/io_file.h>
#define KE_SYSTEM_ROOT_MAX_DRIVES 8
// Selects the mounted volume that contains the kernel/system tree.
KSTATUS KeInitializeSystemRootVolume(void);
// Returns whether the system root volume has been selected.
bool KeSystemRootReady(void);
// Returns the selected system root drive, or zero before discovery.
uint8_t KeGetSystemRootDrive(void);
// Builds an absolute path rooted at the selected system volume.
bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize);
// Opens a file rooted at the selected system volume.
KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out);
code/kernel/include/kernel/ke_thread.h
+28 -4
View File
@@ -14,6 +14,8 @@
#define KSTACK_SIZE (64 * 1024) // 64 KB kernel stack
#define USTACK_SIZE (512 * 1024) // 512 KB user stack
typedef struct THREAD THREAD;
typedef enum THREAD_PRIORITY {
PriorityLow = 0,
PriorityNormal = 1,
@@ -28,6 +30,10 @@ typedef enum THREAD_STATE {
ThreadDead = 3,
} THREAD_STATE;
typedef void (*KE_WAIT_RUNDOWN_ROUTINE)(struct THREAD* thread,
void* context,
KSTATUS status);
// Saved register context - matches the layout pushed by KeSwitchContext.
// The saved RSP points to this structure on the kernel stack.
typedef struct KCONTEXT {
@@ -36,7 +42,7 @@ typedef struct KCONTEXT {
uint64_t Rip; // return address back into KeSwitchContext
} KCONTEXT;
typedef struct THREAD {
struct THREAD {
OBJECT Header; // must be first
uint32_t ThreadId;
@@ -82,7 +88,7 @@ typedef struct THREAD {
// Signaled (manual-reset) when the thread exits.
// Allocated in KeCreateThread, signaled in KeExitThread before teardown.
// Waiters block on this via SYS_WAIT_THREAD -> KeWaitForEvent.
// Waiters block on this through waitable thread handles.
EVENT* ExitEvent;
void* FpuContext;
@@ -113,6 +119,12 @@ typedef struct THREAD {
KSTATUS WaitStatus;
LIST_ENTRY* WaitListHead;
uint8_t AlertableWait;
uint64_t SleepUntilTick;
KSTATUS TimeoutStatus;
// Optional cleanup hook for waits that use auxiliary wait blocks.
KE_WAIT_RUNDOWN_ROUTINE WaitRundownRoutine;
void* WaitRundownContext;
// Kernel APC state. APCs are queued to a target thread and delivered
// when that thread reaches PASSIVE_LEVEL with APC delivery enabled.
@@ -135,15 +147,22 @@ typedef struct THREAD {
uintptr_t UserApcDispatcher;
uint8_t UserApcDeliverable;
uint8_t UserApcDeliveryActive;
} THREAD;
};
// Creates and queues a user thread. On success, *out receives a referenced
// THREAD pointer owned by the caller; release it with ObDereferenceObject()
// after copying any needed IDs/state. The live scheduler/thread lifetime is
// separate and remains valid until the thread exits and is reaped.
__declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
// Create a kernel-mode-only thread. fn is called directly in ring 0 with
// arg; when fn returns the thread exits automatically. No user stack or TEB
// is allocated. The thread is owned by the system process and runs at the
// given priority.
// given priority. On success, non-NULL out receives a referenced THREAD
// pointer owned by the caller; release it with ObDereferenceObject() after
// copying any needed IDs/state or after joining/waiting on caller-owned
// completion state. The scheduler's live thread reference is separate.
__declspec(dllexport) KSTATUS KeCreateKernelThread(uintptr_t fn, uintptr_t arg,
THREAD_PRIORITY priority, THREAD** out);
__declspec(dllexport) KSTATUS KeCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
@@ -167,9 +186,14 @@ __declspec(dllexport) void KeExitThread(THREAD* thread, int code);
// waiting. Alertable waits are explicit at every call site.
void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead);
void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context);
void KeClearThreadWaitRundown(THREAD* thread);
void KeRundownThreadWait(THREAD* thread, KSTATUS status);
KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable);
KSTATUS KeBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable);
void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status);
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status);
void KeAbortThreadWait(THREAD* thread, KSTATUS status);
void KeUnblockThread(THREAD* thread);
KSTATUS KeAlertThread(THREAD* thread);
void KiWakeThreadForKernelApc(THREAD* thread);
code/kernel/include/kernel/ke_work.h
+24
View File
@@ -0,0 +1,24 @@
// Kernel passive-level work queue declarations.
#pragma once
#include <stdbool.h>
#include <kernel/ke_list.h>
#include <ntlite/status.h>
// Work items are caller-owned embedded objects. They may be queued from
// interrupt/DPC/spinlock context and run later on a persistent system worker
// thread at PASSIVE_LEVEL. Queueing the same item while it is already queued
// is coalesced.
typedef void (*KWORK_ROUTINE)(void* context);
typedef struct KWORK_ITEM {
LIST_ENTRY Link;
KWORK_ROUTINE Routine;
void* Context;
volatile long Queued;
} KWORK_ITEM;
void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context);
KSTATUS KeStartWorkQueue(void);
bool KeIsWorkQueueStarted(void);
bool KeQueueWorkItem(KWORK_ITEM* item);
code/kernel/include/kernel/ldr_image.h
+5 -4
View File
@@ -9,11 +9,12 @@
#include <ntlite/status.h>
struct PROCESS;
struct FILE_OBJECT;
// Map a PE32+ image from 'buffer' into the kernel virtual address space.
// On success, *baseOut receives the virtual base address.
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user);
KSTATUS LdrLoadImageForProcess(struct PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut);
// Map a PE32+ image from a bootloader/driver raw buffer into kernel space.
// User images must be loaded from FILE_OBJECTs so image sections have stable identity.
KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut);
KSTATUS LdrLoadImageFileForProcess(struct PROCESS* process, struct FILE_OBJECT* file, uintptr_t* baseOut);
// Return the SizeOfImage recorded in the PE header for an already-mapped image.
size_t LdrGetImageSize(uintptr_t base);
code/kernel/include/kernel/ldr_section.h
+67
View File
@@ -0,0 +1,67 @@
// Kernel image section declarations shared by loader and memory-management code.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <kernel/mm_phys.h>
struct PROCESS;
struct FILE_OBJECT;
typedef struct PROCESS PROCESS;
typedef struct FILE_OBJECT FILE_OBJECT;
typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT;
// Describes one PE section in image-address-space coordinates.
typedef struct IMAGE_SECTION_DESCRIPTOR {
char Name[9];
uint32_t VirtualAddress;
uint32_t VirtualSize;
uint32_t RawDataOffset;
uint32_t RawDataSize;
uint32_t Characteristics;
uint64_t PageProtection;
} IMAGE_SECTION_DESCRIPTOR;
// Keeps the parsed identity and layout for a PE image section object.
typedef struct IMAGE_SECTION_METADATA {
uint64_t ContentKey;
uint64_t VolumeId;
uint64_t FileId;
uint64_t FileGeneration;
uint64_t PreferredBase;
uint32_t EntryRva;
uint32_t HeaderSize;
size_t RawSize;
size_t ImageSize;
size_t PageCount;
uint32_t SectionCount;
} IMAGE_SECTION_METADATA;
// Creates or references an image section object backed by an open file.
KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file,
IMAGE_SECTION_OBJECT** sectionOut);
// Adds a reference to an existing image section object.
void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section);
// Releases a reference returned by LdrCreateImageSectionFromFile or LdrReferenceImageSection.
void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section);
// Returns immutable metadata for an image section object.
const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section);
// Returns the immutable PE section table owned by the image section object.
const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section,
uint32_t* countOut);
// Retains one prototype frame owned by an image section object.
KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section,
size_t pageIndex,
PHYSICAL_ADDRESS* frameOut);
// Maps an image section into a process as an image-backed VAD.
KSTATUS LdrMapImageSection(PROCESS* process,
IMAGE_SECTION_OBJECT* section,
void** baseOut);
code/kernel/include/kernel/mm_file_cache.h
+99
View File
@@ -0,0 +1,99 @@
// Kernel file-page cache declarations.
// Caches PAGE_SIZE file pages shared by reads, mmap faults, and writeback.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <ntlite/status.h>
struct FILE_OBJECT;
typedef struct FILE_OBJECT FILE_OBJECT;
typedef struct MM_FILE_CACHE_PAGE {
FILE_OBJECT* File;
uint64_t Offset;
PHYSICAL_ADDRESS Frame;
uint32_t ValidBytes;
uint8_t Dirty;
} MM_FILE_CACHE_PAGE;
typedef struct MM_FILE_CACHE_STATS {
uint64_t Lookups;
uint64_t Hits;
uint64_t Misses;
uint64_t Loads;
uint64_t LoadFailures;
uint64_t RaceDiscards;
uint64_t InflightWaits;
uint64_t Invalidations;
uint64_t DirtyPages;
uint64_t MaxDirtyPages;
uint64_t DirtyMarks;
uint64_t WritebackRequests;
uint64_t WritebackPages;
uint64_t WritebackBytes;
uint64_t WritebackFailures;
uint64_t FlushFileCalls;
uint64_t FlushFileFailures;
uint64_t MappedDirtySyncs;
uint64_t MappedDirtyPages;
uint64_t Drops;
uint64_t DropFailures;
uint64_t StandbyTransitions;
uint64_t StandbyHits;
uint64_t ReadAheadRequests;
uint64_t ReadAheadPagesRequested;
uint64_t ReadAheadPagesQueued;
uint64_t ReadAheadPagesLoaded;
uint64_t ReadAheadPagesSkipped;
uint64_t ReadAheadFailures;
uint64_t ReadAheadPagesThrottled;
uint64_t MaxReadAheadPagesPerRequest;
uint64_t BatchReads;
uint64_t BatchPagesRequested;
uint64_t BatchPagesLoaded;
uint64_t MaxBatchPagesRequested;
uint64_t MaxBatchPagesLoaded;
uint64_t ResidentPages;
uint64_t ActivePages;
uint64_t StandbyPages;
uint64_t MaxResidentPages;
} MM_FILE_CACHE_STATS;
// Returns a retained physical frame containing the file page at page-aligned
// offset. The caller owns one PFN reference and must call
// MmFileCacheReleasePage(). Bytes beyond ValidBytes are zero-filled.
KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAGE* outPage);
void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page);
// Returns a retained coherent cache page suitable for a shared writable mapping.
KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
uint64_t fileSizeBefore, int needExistingBytes,
MM_FILE_CACHE_PAGE* outPage);
// Marks one coherent cache page dirty after a shared writable mapping is written.
void MmFileCacheMarkDirty(FILE_OBJECT* file, uint64_t offset, uint32_t validBytes);
// Writes into coherent cached file pages and marks them dirty.
KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf,
size_t size, uint64_t fileSizeBefore, size_t* writtenOut);
// Flushes dirty cached pages for one file.
KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file);
// Drops cached clean pages overlapping the byte range. Dirty pages are flushed
// by callers before destructive metadata operations.
void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size);
void MmFileCacheInvalidateAll(FILE_OBJECT* file);
// Reclaims one cache-owned page if possible. Safe to call under memory pressure;
// it never writes back because v1 stores only clean file data.
KSTATUS MmFileCacheDropOnePageForPressure(void);
// Queues best-effort read-ahead for clean file pages. The queue owns a file
// reference until the worker has consumed the request.
KSTATUS MmFileCacheScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, uint32_t pages);
void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out);
code/kernel/include/kernel/mm_pfn.h
+134
View File
@@ -0,0 +1,134 @@
// Physical frame database declarations.
// The PFN layer owns frame state, reference counts, pin counts, and allocation
// policy. Higher VMM layers should ask for pages by purpose instead of treating
// the physical allocator as an untyped bitmap.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <kernel/mm_phys.h>
#define MM_PFN_INVALID_INDEX ((size_t)-1)
typedef enum MM_PFN_STATE {
MmPfnStateInvalid = 0,
MmPfnStateReserved,
MmPfnStateFree,
MmPfnStateZeroed,
MmPfnStateActive,
MmPfnStateStandby,
MmPfnStateModified,
} MM_PFN_STATE;
typedef enum MM_PFN_USAGE {
MmPfnUseNone = 0,
MmPfnUseKernel,
MmPfnUsePageTable,
MmPfnUseUser,
MmPfnUseCleanFileCache,
MmPfnUseImage,
MmPfnUseMappedFile,
MmPfnUseAnonymous,
MmPfnUseCow,
MmPfnUseSurface,
MmPfnUseDriver,
MmPfnUseDma,
MmPfnUseBootloader,
MmPfnUsePfnDatabase,
MmPfnUseMmio,
} MM_PFN_USAGE;
typedef struct MM_PFN {
uint16_t State;
uint16_t Usage;
uint16_t RefCount;
uint16_t PinCount;
uint32_t Flags;
uint32_t MappedCount;
void* Owner;
uint64_t BackingPage;
} MM_PFN;
typedef struct MM_PFN_STATS {
uint64_t TotalFrames;
uint64_t FreeFrames;
uint64_t UsedFrames;
uint64_t ReservedFrames;
uint64_t ZeroedFrames;
uint64_t ActiveFrames;
uint64_t StandbyFrames;
uint64_t ModifiedFrames;
uint64_t PinnedFrames;
uint64_t EvictableFrames;
uint64_t NonEvictableFrames;
uint64_t ReclaimableFrames;
uint64_t AvailableFrames;
uint64_t MappedFrames;
uint64_t KernelFrames;
uint64_t PageTableFrames;
uint64_t UserFrames;
uint64_t CleanFileCacheFrames;
uint64_t ActiveCleanFileCacheFrames;
uint64_t StandbyCleanFileCacheFrames;
uint64_t ImageFrames;
uint64_t ActiveImageFrames;
uint64_t StandbyImageFrames;
uint64_t MappedFileFrames;
uint64_t AnonymousFrames;
uint64_t CowFrames;
uint64_t SurfaceFrames;
uint64_t DriverFrames;
uint64_t DmaFrames;
uint64_t BootloaderFrames;
uint64_t PfnDatabaseFrames;
uint64_t MmioFrames;
uint64_t AllocationCount;
uint64_t FreeCount;
uint64_t RetainCount;
uint64_t ReleaseCount;
uint64_t PinCount;
uint64_t UnpinCount;
uint64_t FailedAllocations;
uint64_t FailedContiguousAllocations;
uint64_t FailedInvalidRange;
uint64_t FailedEmptyRange;
uint64_t FailedReserveDenied;
uint64_t FailedNoFrame;
uint64_t FailedContiguousInvalidRequest;
uint64_t FailedContiguousReserveDenied;
uint64_t FailedContiguousNoRun;
uint64_t ReclaimAttempts;
uint64_t ReclaimPagesRequested;
uint64_t ReclaimPagesSucceeded;
uint64_t ReclaimPagesFailed;
uint64_t ReclaimReentrantSkips;
} MM_PFN_STATS;
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocate(MM_PFN_USAGE usage);
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateBelow(PHYSICAL_ADDRESS maxExclusive, MM_PFN_USAGE usage);
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive,
MM_PFN_USAGE usage);
__declspec(dllexport) PHYSICAL_ADDRESS MmPfnAllocateContiguous(size_t count, MM_PFN_USAGE usage);
__declspec(dllexport) KSTATUS MmPfnFree(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRetain(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRelease(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnPin(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnUnpin(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnAddMapping(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnRemoveMapping(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnMoveToStandby(PHYSICAL_ADDRESS frame);
__declspec(dllexport) KSTATUS MmPfnActivate(PHYSICAL_ADDRESS frame);
__declspec(dllexport) MM_PFN* MmPfnGet(PHYSICAL_ADDRESS frame);
__declspec(dllexport) void* MmPfnToVirtual(PHYSICAL_ADDRESS frame);
size_t MmPfnIndex(PHYSICAL_ADDRESS frame);
PHYSICAL_ADDRESS MmPfnAddress(size_t pfn);
void MmPfnQueryStats(MM_PFN_STATS* out);
code/kernel/include/kernel/mm_phys.h
+8 -22
View File
@@ -1,8 +1,9 @@
// Kernel mm phys declarations shared by implementation files and kernel clients.
// Physical-memory declarations shared by the kernel and drivers.
// The physical layer publishes page-size constants, boot memory-map ingestion,
// and aggregate memory statistics. Frame ownership lives in the PFN database.
#pragma once
// mm_phys.h - physical memory manager (PMM)
// Bitmap allocator over 4 KB frames, seeded from MEMORY_MAP.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
@@ -22,22 +23,7 @@ typedef struct MM_PHYSICAL_STATS {
uint64_t FreeRunHistogram[8];
} MM_PHYSICAL_STATS;
// Called once by KernelMain with the MEMORY_MAP from BOOT_INFO.
bool MmInitializePhysical(const MEMORY_MAP* map,
uint64_t physicalWindowBase);
// Allocate / free a single 4 KB frame.
// MmAllocateFrame returns 0 on failure.
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateFrame(void);
PHYSICAL_ADDRESS MmAllocateFrameBelow(PHYSICAL_ADDRESS maxExclusive);
PHYSICAL_ADDRESS MmAllocateFrameInRange(PHYSICAL_ADDRESS minInclusive,
PHYSICAL_ADDRESS maxExclusive);
void MmFreeFrame(PHYSICAL_ADDRESS frame);
// Allocate 'count' physically contiguous 4 KB frames.
// Returns 0 on failure. Use sparingly (DMA, page tables).
__declspec(dllexport) PHYSICAL_ADDRESS MmAllocateContiguous(size_t count);
size_t MmFreeFrameCount(void);
size_t MmTotalFrameCount(void);
void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out);
bool MmInitializePhysical(const MEMORY_MAP* map, uint64_t physicalWindowBase);
size_t MmFreeFrameCount(void);
size_t MmTotalFrameCount(void);
void MmQueryPhysicalStats(MM_PHYSICAL_STATS* out);
code/kernel/include/kernel/mm_pte.h
+105
View File
@@ -0,0 +1,105 @@
// Kernel page-table manager declarations shared by the VMM and low-level kernel clients.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <ntlite/status.h>
// x86-64 PTE bits exposed to the rest of the kernel.
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
#define PAGE_WRITETHROUGH (1ULL << 3)
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_ACCESSED (1ULL << 5)
#define PAGE_DIRTY (1ULL << 6)
#define PAGE_NX (1ULL << 63)
// The physical direct map is a kernel-only alias used to reach page-table frames and early physical memory.
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
// Page-table telemetry separates cumulative operations from the current number
// of 4 KiB leaf mappings owned by this layer. Huge boot mappings may exist but
// are not counted as leaves here.
typedef enum MM_PTE_QUERY_PURPOSE {
MmPteQueryGeneric = 0,
MmPteQueryUserProbe,
MmPteQueryHeap,
MmPteQueryVad,
MmPteQueryDebug,
MmPteQuerySelfTest,
MmPteQueryPurposeCount,
} MM_PTE_QUERY_PURPOSE;
typedef struct MM_PTE_STATS {
uint64_t MapCalls;
uint64_t MapFailures;
uint64_t UnmapCalls;
uint64_t UnmapFailures;
uint64_t ProtectCalls;
uint64_t ProtectFailures;
uint64_t QueryCalls;
uint64_t QueryFailures;
uint64_t QueryByPurpose[MmPteQueryPurposeCount];
uint64_t QueryFailuresByPurpose[MmPteQueryPurposeCount];
uint64_t TranslateCalls;
uint64_t LocalFlushes;
uint64_t GlobalFlushes;
uint64_t ShootdownRequests;
uint64_t ShootdownTargetCpus;
uint64_t ShootdownSkippedNoTargets;
uint64_t ShootdownAcks;
uint64_t ShootdownTimeouts;
uint64_t NewMappingNoFlushes;
uint64_t ReplacedMappingFlushes;
uint64_t PageTablePagesAllocated;
uint64_t CurrentLeafMappings;
} MM_PTE_STATS;
typedef enum MM_PTE_MAP_MODE {
MmPteMapFailIfPresent = 0,
MmPteMapReplaceExisting = 1,
} MM_PTE_MAP_MODE;
// Initializes the page-table manager around the bootloader-created PML4.
void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys);
// Installs the shared PML4 into CR3.
void MmPteActivate(void);
// Maps a 4 KiB page into the shared address space.
__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode);
// Removes a 4 KiB mapping from the shared address space.
__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt);
// Replaces the flags on every mapped 4 KiB page in the range while preserving the physical frame.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags);
// Tests and clears the hardware accessed bit on one mapped 4 KiB leaf.
// Returns nonzero when the page had been accessed and was given a second chance.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt);
// Tests and clears the hardware dirty bit on one mapped 4 KiB leaf.
__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt);
// Translates a virtual address through the current shared page tables.
__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt);
// Returns the physical address and effective flags for a mapped virtual address.
__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose, uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
// Flushes one address from the local TLB.
void MmPteFlushLocal(uintptr_t virt);
// Flushes a kernel mapping from every online CPU that shares the kernel page tables.
void MmPteFlushKernelAddress(uintptr_t virt);
// Handles the fixed-vector NMI used for kernel TLB shootdowns.
int MmPteHandleShootdownNmi(void);
// Returns page-table operation counters used by memory telemetry.
void MmPteQueryStats(MM_PTE_STATS* out);
code/kernel/include/kernel/mm_swap.h
+31
View File
@@ -0,0 +1,31 @@
// Kernel swap manager declarations.
// Swap owns pagefile slot allocation and page-sized I/O for anonymous VAD pages.
#pragma once
#include <stdint.h>
#include <stddef.h>
#include <ntlite/status.h>
#include <ntlite/string_view.h>
#include <kernel/mm_phys.h>
typedef struct MM_SWAP_STATS {
uint64_t Enabled;
uint64_t TotalSlots;
uint64_t UsedSlots;
uint64_t FreeSlots;
uint64_t HighWaterSlots;
uint64_t PageOuts;
uint64_t PageIns;
uint64_t PageOutFailures;
uint64_t PageInFailures;
uint64_t SlotAllocations;
uint64_t SlotFrees;
uint64_t SlotAllocationFailures;
} MM_SWAP_STATS;
KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes);
int MmSwapIsEnabled(void);
KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut);
KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame);
void MmSwapFreeSlot(uint32_t slot);
void MmSwapQueryStats(MM_SWAP_STATS* out);
code/kernel/include/kernel/mm_virt.h
+140 -76
View File
@@ -2,50 +2,43 @@
#pragma once
// mm_virt.h - virtual memory manager (VMM)
// Manages the single shared PML4, kernel pool, and page-granular user VM regions.
// Manages the single shared PML4, kernel pool, and VAD-backed user VM ownership.
#include <stdint.h>
#include <stddef.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pte.h>
#include <ntlite/status.h>
struct PROCESS;
typedef struct PROCESS PROCESS;
struct FILE_OBJECT;
typedef struct FILE_OBJECT FILE_OBJECT;
struct IMAGE_SECTION_OBJECT;
typedef struct IMAGE_SECTION_OBJECT IMAGE_SECTION_OBJECT;
typedef enum MM_MAP_MODE {
MmMapFailIfPresent = 0,
MmMapReplaceExisting = 1,
} MM_MAP_MODE;
typedef enum MM_VAD_TYPE {
MmVadGeneric = 0,
MmVadHeap,
MmVadImage,
MmVadStack,
MmVadTeb,
MmVadPeb,
MmVadEnvironment,
MmVadMappedFile,
MmVadShared,
} MM_VAD_TYPE;
typedef enum USER_REGION_TYPE {
UserRegionGeneric = 0,
UserRegionHeap,
UserRegionImage,
UserRegionStack,
UserRegionTeb,
UserRegionPeb,
UserRegionEnvironment,
UserRegionMappedFile,
UserRegionShared,
} USER_REGION_TYPE;
#define USER_REGION_GUARD_LOW 0x00000001u
#define USER_REGION_GUARD_HIGH 0x00000002u
#define USER_REGION_ZERO 0x00000004u
// Page flags (x86-64 PTE bits).
#define PAGE_PRESENT (1ULL << 0)
#define PAGE_WRITE (1ULL << 1)
#define PAGE_USER (1ULL << 2)
#define PAGE_WRITETHROUGH (1ULL << 3)
#define PAGE_NOCACHE (1ULL << 4)
#define PAGE_NX (1ULL << 63)
#define MM_VAD_GUARD_LOW 0x00000001u
#define MM_VAD_GUARD_HIGH 0x00000002u
#define MM_VAD_ZERO 0x00000004u
#define MM_VAD_PREFETCH 0x00000008u
#define MM_VAD_SHARED_WRITE 0x00000010u
// Virtual address space constants.
#define USER_BASE 0x0000000000001000ULL
#define USER_LIMIT 0x00007FFFFFFFF000ULL
#define KERNEL_BASE 0xFFFF800000000000ULL
#define PHYS_MAP_BASE 0xFFFF880000000000ULL
#define POOL_BASE 0xFFFFC00000000000ULL
#define POOL_LIMIT 0xFFFFE00000000000ULL
@@ -55,37 +48,6 @@ typedef enum USER_REGION_TYPE {
// poolSize: initial reservation for kernel pool (bytes, page-aligned).
void MmInitializeVirtual(PHYSICAL_ADDRESS bootPml4Phys, size_t poolSize);
// Install the shared PML4 into CR3. Called once at the end of KernelMain init.
void MmActivate(void);
// Page mapping.
__declspec(dllexport) KSTATUS MmMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_MAP_MODE mode);
__declspec(dllexport) KSTATUS MmUnmapPage(uintptr_t virt);
// Modify protection flags on an already-mapped virtual range.
// 'flags' is the complete new set of PAGE_* bits - it replaces the old flags
// entirely (except the physical address, which is preserved).
// Returns STATUS_NOT_FOUND if any page in the range is not mapped.
__declspec(dllexport) KSTATUS MmSetPageProtect(uintptr_t virt, size_t size, uint64_t flags);
// Walk the page tables to translate a virtual address to physical.
// Returns 0 if the address is not mapped.
__declspec(dllexport) PHYSICAL_ADDRESS MmGetPhysicalAddress(uintptr_t virt);
// TLB/cache coherency hooks for the shared kernel address space. Local flushes
// use INVLPG directly; kernel mapping changes broadcast an NMI shootdown to
// other online CPUs and wait for acknowledgements.
void MmFlushTlbLocal(uintptr_t virt);
void MmFlushTlbForKernelMapping(uintptr_t virt);
int MmHandleTlbShootdownNmi(void);
// User/kernel copy boundary helpers
// MmQueryPage returns the physical address and effective PAGE_* bits for a
// mapped virtual address. The reported PAGE_USER/PAGE_WRITE bits account for
// every page-table level, not just the leaf entry.
__declspec(dllexport) KSTATUS MmQueryPage(uintptr_t virt, PHYSICAL_ADDRESS* physOut, uint64_t* flagsOut);
// Probe a user virtual range without copying it. These functions only accept
// lower-half user mappings with PAGE_USER set; MmProbeUserWrite additionally
// requires PAGE_WRITE.
@@ -97,31 +59,96 @@ __declspec(dllexport) KSTATUS MmProbeUserWrite(void* ptr, size_t size);
__declspec(dllexport) KSTATUS CopyFromUser(void* kernelDst, const void* userSrc, size_t size);
__declspec(dllexport) KSTATUS CopyToUser(void* userDst, const void* kernelSrc, size_t size);
typedef struct MM_PINNED_USER_PAGE {
uintptr_t UserPage;
size_t PageOffset;
PHYSICAL_ADDRESS Frame;
void* KernelAddress;
uint64_t PageFlags;
} MM_PINNED_USER_PAGE;
typedef struct MM_LOCKED_USER_BUFFER MM_LOCKED_USER_BUFFER;
// Pins one user page and returns a kernel direct-map alias for the requested
// address. Callers may copy through KernelAddress + PageOffset while the page is
// pinned. The pin is short-lived: never sleep, do file I/O, or hold it across
// arbitrary callbacks.
__declspec(dllexport) KSTATUS MmPinUserPageForRead(PROCESS* process, const void* userAddress, MM_PINNED_USER_PAGE* out);
__declspec(dllexport) KSTATUS MmPinUserPageForWrite(PROCESS* process, void* userAddress, MM_PINNED_USER_PAGE* out);
__declspec(dllexport) void MmUnpinUserPage(MM_PINNED_USER_PAGE* page);
// Locks a user buffer for asynchronous kernel access. The containing VADs stay
// alive and protected, and resident PFNs stay pinned until the buffer is
// unlocked. WriteAccess describes kernel writes to the user buffer.
__declspec(dllexport) KSTATUS MmLockUserBuffer(PROCESS* process, void* userAddress, size_t size, int writeAccess, MM_LOCKED_USER_BUFFER** out);
__declspec(dllexport) void MmUnlockUserBuffer(MM_LOCKED_USER_BUFFER* buffer);
// Kernel pool - ring 0 only, never PAGE_USER.
// Best-fit free-list allocator, thread-safe via spinlock.
#define MM_POOL_TAG(a, b, c, d) \
((uint32_t)(uint8_t)(a) | ((uint32_t)(uint8_t)(b) << 8) | \
((uint32_t)(uint8_t)(c) << 16) | ((uint32_t)(uint8_t)(d) << 24))
__declspec(dllexport) void* MmAllocatePool(size_t size);
__declspec(dllexport) void* MmAllocatePoolTag(size_t size, uint32_t tag);
__declspec(dllexport) void MmFreePool(void* ptr);
__declspec(dllexport) void* MmReallocatePool(void* ptr, size_t newSize);
// User virtual regions.
// The kernel has one shared user VA arena, but every user mapping is owned by
// a PROCESS so process teardown can reclaim leaked memory. Guard pages are
// reserved VA with no present PTE.
KSTATUS MmCreateUserRegion(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t regionFlags,
USER_REGION_TYPE type, void** outBase);
KSTATUS MmEnsureUserRegionMapped(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmProtectUserRegion(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmDestroyUserRegion(PROCESS* process, void* base);
void MmDestroyAllUserRegions(PROCESS* process);
KSTATUS MmQueryUserRegion(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, USER_REGION_TYPE* typeOut, uint32_t* regionFlagsOut);
void MmQueryProcessUserMemory(PROCESS* process, size_t* regionCountOut, size_t* reservedBytesOut, size_t* committedBytesOut);
size_t MmGetUserRegionSize(PROCESS* process, const void* base);
size_t MmGetUserRegionSizeByBase(const void* base);
// Virtual address descriptors.
// The single shared address space still has per-process VAD ownership so process
// teardown can reclaim leaked mappings. VADs track sparse per-page commitment,
// and committed pages are materialized by user faults or explicit prefaulting.
KSTATUS MmVadReserve(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t vadFlags,
MM_VAD_TYPE type, void** outBase);
KSTATUS MmVadCommit(PROCESS* process, void* base, size_t size,
uint64_t pageFlags, uint32_t commitFlags);
KSTATUS MmVadCreateCommitted(PROCESS* process, size_t size, size_t alignment,
uint64_t pageFlags, uint32_t vadFlags,
MM_VAD_TYPE type, void** outBase);
KSTATUS MmVadCreateFileMapping(PROCESS* process, FILE_OBJECT* file,
uint64_t fileOffset, size_t size,
uint64_t pageFlags, uint32_t vadFlags,
void** outBase);
KSTATUS MmVadCreateImageSectionMapping(PROCESS* process, IMAGE_SECTION_OBJECT* section,
uint64_t pageFlags, uint32_t vadFlags,
void** outBase);
KSTATUS MmVadEnsureResident(PROCESS* process, void* base, size_t size, uint64_t accessFlags);
KSTATUS MmVadMakeImageRangePrivate(PROCESS* process, void* base, size_t size);
KSTATUS MmVadSyncFileMappings(FILE_OBJECT* file);
KSTATUS MmVadCreateCowView(PROCESS* sourceProcess, void* sourceBase, size_t size,
PROCESS* targetProcess, void** outBase);
KSTATUS MmVadCreateCowViewFromFrames(PROCESS* targetProcess,
const PHYSICAL_ADDRESS* frames,
size_t pageCount,
size_t payloadSize,
uint64_t pageFlags,
uint32_t vadFlags,
MM_VAD_TYPE type,
void** outBase);
KSTATUS MmVadCreateSharedViewFromFrames(PROCESS* targetProcess,
const PHYSICAL_ADDRESS* frames,
size_t pageCount,
size_t payloadSize,
uint64_t pageFlags,
uint32_t vadFlags,
MM_VAD_TYPE type,
void** outBase);
KSTATUS MmVadResolveUserPageFault(PROCESS* process, uintptr_t faultAddress, uint64_t errorCode);
KSTATUS MmVadResolvePageFaultByAddress(uintptr_t faultAddress, uint64_t errorCode);
KSTATUS MmVadTrimOnePageForTest(PROCESS* process, void** baseOut);
KSTATUS MmVadTrimOnePageForPressure(void);
KSTATUS MmVadProtect(PROCESS* process, void* base, size_t size, uint64_t pageFlags);
KSTATUS MmVadDestroy(PROCESS* process, void* base);
void MmVadDestroyAllForProcess(PROCESS* process);
KSTATUS MmVadQuery(PROCESS* process, const void* address, void** baseOut, size_t* sizeOut, uint64_t* pageFlagsOut, MM_VAD_TYPE* typeOut, uint32_t* vadFlagsOut);
void MmVadQueryProcessMemory(PROCESS* process, size_t* vadCountOut, size_t* reservedBytesOut, size_t* committedBytesOut);
size_t MmVadGetSize(PROCESS* process, const void* base);
size_t MmVadGetSizeByBase(const void* base);
// Debug-only ownership/address diagnostics used by fault and launch paths.
void MmDebugDescribeAddress(uintptr_t addr);
void MmDebugDumpUserRegions(PROCESS* process);
void MmVadDebugDumpProcess(PROCESS* process);
__declspec(dllexport) void* MmAllocateUserMemoryForProcess(PROCESS* process, size_t size);
__declspec(dllexport) void MmFreeUserMemoryForProcess(PROCESS* process, void* ptr);
@@ -140,3 +167,40 @@ __declspec(dllexport) void MmFreeUserImage(void* ptr, size_t size);
size_t MmGetKernelPoolUsage(void);
size_t MmGetUserHeapUsage(void);
// Demand-paging telemetry used by the shell stats command.
typedef struct MM_VAD_STATS {
uint64_t ResolveFaultCalls;
uint64_t ResolvedFaults;
uint64_t FailedFaults;
uint64_t ProtectionFaults;
uint64_t NoVadFaults;
uint64_t UncommittedFaults;
uint64_t NoAccessFaults;
uint64_t DemandZeroPages;
uint64_t ZeroPageMaps;
uint64_t ZeroPageCowSplits;
uint64_t CowFaults;
uint64_t CowPagesCopied;
uint64_t CowViewsCreated;
uint64_t CowPagesShared;
uint64_t EnsureResidentCalls;
uint64_t EnsureResidentPages;
uint64_t AlreadyResidentPages;
uint64_t FileBackedFaults;
uint64_t FileBackedReads;
uint64_t FileBackedReadFailures;
uint64_t FileBackedDiscards;
uint64_t FileBackedSharedWritableFaults;
uint64_t FileBackedDirtyFaults;
uint64_t FileBackedDirtySyncs;
uint64_t ImageBackedFaults;
uint64_t ImageBackedZeroFaults;
uint64_t ImageBackedPrivateFaults;
uint64_t ImageFixupPrivateCalls;
uint64_t ImageFixupPrivatePages;
uint64_t SwapPageOuts;
uint64_t SwapPageIns;
} MM_VAD_STATS;
void MmVadQueryStats(MM_VAD_STATS* out);
code/kernel/include/kernel/ob_object.h
+4
View File
@@ -23,6 +23,10 @@ typedef enum OBJECT_TYPE {
ObjectTypeDriver = 12,
ObjectTypeUsbDevice = 13,
ObjectTypeUsbPipe = 14,
ObjectTypeFileMapping = 15,
ObjectTypeSurface = 16,
ObjectTypeInputQueue = 17,
ObjectTypeImageSection = 18,
} OBJECT_TYPE;
typedef struct OBJECT {
code/kernel/include/kernel/syscall_nr.h
+49 -38
View File
@@ -11,13 +11,13 @@
#define SYS_EXIT_PROCESS 0
#define SYS_EXIT_THREAD 1
#define SYS_CREATE_PROCESS 2
#define SYS_WAIT_FOR_PROCESS 3
#define SYS_WAIT_SINGLE 3
#define SYS_GET_PROCESS_EXIT_CODE 4
#define SYS_KILL_PROCESS 5
#define SYS_GET_PROCESS_LIST 6
#define SYS_GET_PROCESS_PID 7
#define SYS_CREATE_THREAD 8
#define SYS_WAIT_THREAD 9
#define SYS_RETIRED_WAIT_THREAD 9
#define SYS_GET_CURRENT_TID 10
#define SYS_GET_CURRENT_PID 11
#define SYS_SET_THREAD_PRIORITY 12
@@ -31,17 +31,17 @@
#define SYS_QUERY_VM 20
#define SYS_CLOSE_HANDLE 21
#define SYS_CREATE_MUTEX 22
#define SYS_ACQUIRE_MUTEX 23
#define SYS_RETIRED_ACQUIRE_MUTEX 23
#define SYS_TRY_ACQUIRE_MUTEX 24
#define SYS_RELEASE_MUTEX 25
#define SYS_CREATE_SEM 26
#define SYS_WAIT_SEM 27
#define SYS_RETIRED_WAIT_SEM 27
#define SYS_TRY_WAIT_SEM 28
#define SYS_SIGNAL_SEM 29
#define SYS_CREATE_EVENT 30
#define SYS_SIGNAL_EVENT 31
#define SYS_CLEAR_EVENT 32
#define SYS_WAIT_EVENT 33
#define SYS_RETIRED_WAIT_EVENT 33
#define SYS_CREATE_QUEUE 34
#define SYS_SEND_QUEUE 35
#define SYS_RECV_QUEUE 36
@@ -74,36 +74,47 @@
#define SYS_FILL_CONSOLE_CHARACTER 63
#define SYS_FILL_CONSOLE_ATTRIBUTE 64
#define SYS_PUT_SCREEN_CHAR 65
#define SYS_BLIT_SCREEN 66
#define SYS_CLEAR_SCREEN 67
#define SYS_GET_SCREEN_SIZE 68
#define SYS_QUERY_DISPLAY_INFO 69
#define SYS_POLL_KEYBOARD 70
#define SYS_IS_KEY_DOWN 71
#define SYS_READ_KEYBOARD_CHAR 72
#define SYS_READ_KEY 73
#define SYS_GET_KEY_MODIFIERS 74
#define SYS_POLL_MOUSE 75
#define SYS_READ_MOUSE 76
#define SYS_GET_MOUSE_POS 77
#define SYS_GET_TICK 78
#define SYS_PRINT_DEBUG 79
#define SYS_QUERY_TRAP_FRAME 80
#define SYS_CREATE_SOCKET 81
#define SYS_BIND_SOCKET 82
#define SYS_LISTEN_SOCKET 83
#define SYS_ACCEPT_SOCKET 84
#define SYS_CONNECT_SOCKET 85
#define SYS_SEND_SOCKET 86
#define SYS_RECV_SOCKET 87
#define SYS_SHUTDOWN_SOCKET 88
#define SYS_SEND_TO_SOCKET 89
#define SYS_RECV_FROM_SOCKET 90
#define SYS_RESOLVE_HOST 91
#define SYS_SET_SOCKET_OPTION 92
#define SYS_GET_SOCKET_OPTION 93
#define SYS_POLL 94
#define SYS_GET_MEMORY_STATUS 95
#define SYS_QUERY_KERNEL_STATS 96
#define SYS_GET_SYSTEM_TIME 97
#define SYS_MAX 98
#define SYS_CLEAR_SCREEN 66
#define SYS_GET_SCREEN_SIZE 67
#define SYS_QUERY_DISPLAY_INFO 68
#define SYS_POLL_KEYBOARD 69
#define SYS_IS_KEY_DOWN 70
#define SYS_READ_KEYBOARD_CHAR 71
#define SYS_READ_KEY 72
#define SYS_GET_KEY_MODIFIERS 73
#define SYS_POLL_MOUSE 74
#define SYS_READ_MOUSE 75
#define SYS_GET_MOUSE_POS 76
#define SYS_GET_TICK 77
#define SYS_PRINT_DEBUG 78
#define SYS_QUERY_TRAP_FRAME 79
#define SYS_CREATE_SOCKET 80
#define SYS_BIND_SOCKET 81
#define SYS_LISTEN_SOCKET 82
#define SYS_ACCEPT_SOCKET 83
#define SYS_CONNECT_SOCKET 84
#define SYS_SEND_SOCKET 85
#define SYS_RECV_SOCKET 86
#define SYS_SHUTDOWN_SOCKET 87
#define SYS_SEND_TO_SOCKET 88
#define SYS_RECV_FROM_SOCKET 89
#define SYS_RESOLVE_HOST 90
#define SYS_SET_SOCKET_OPTION 91
#define SYS_GET_SOCKET_OPTION 92
#define SYS_POLL 93
#define SYS_GET_MEMORY_STATUS 94
#define SYS_QUERY_KERNEL_STATS 95
#define SYS_GET_SYSTEM_TIME 96
#define SYS_CREATE_FILE_MAPPING 97
#define SYS_MAP_FILE 98
#define SYS_CREATE_SURFACE 99
#define SYS_MAP_SURFACE 100
#define SYS_QUERY_SURFACE 101
#define SYS_PRESENT_SURFACE 102
#define SYS_GET_THREAD_LIST 103
#define SYS_READ_INPUT_EVENT 104
#define SYS_PEEK_INPUT_EVENT 105
#define SYS_WAIT_MANY 106
#define SYS_OPEN_INPUT_QUEUE 107
#define SYS_FLUSH_FILE 108
#define SYS_MAX 109
code/kernel/io_device.c
+336 -6
View File
@@ -7,6 +7,7 @@
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
#include <kernel/ke_sched.h>
#define MAX_DRIVER_OBJECTS 64
#define MAX_PDO_OBJECTS 128
@@ -443,6 +444,170 @@ static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
#define IO_DISK_READAHEAD_MAX_SECTORS 128u
#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u
#define IO_DISK_READAHEAD_MIN_SECTORS 8u
#define IO_DISK_SECTOR_BYTES 512u
typedef struct IO_DISK_READAHEAD_STATE {
SPINLOCK Lock;
uint8_t* Buffer;
uint8_t Valid;
uint8_t Loading;
uint64_t Generation;
uint64_t BaseLba;
uint32_t Count;
uint64_t NextLba;
uint32_t SequentialScore;
IO_DISK_STATS Stats;
} IO_DISK_READAHEAD_STATE;
static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES];
// Updates a maximum statistic while holding the disk read-ahead lock.
static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) {
if (field && *field < value) *field = value;
}
// Prepares the per-disk read-ahead buffer.
static void IoDiskReadAheadPrepare(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
state->Lock = (SPINLOCK)SPINLOCK_INIT;
state->Valid = 0;
state->Loading = 0;
state->Generation = 0;
state->BaseLba = 0;
state->Count = 0;
state->NextLba = 0;
state->SequentialScore = 0;
memset(&state->Stats, 0, sizeof(state->Stats));
if (!state->Buffer) {
state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES,
MM_POOL_TAG('d', 'r', 'a', 'h'));
if (!state->Buffer)
state->Stats.ReadaheadAllocFailures++;
}
}
// Releases the per-disk read-ahead buffer.
static void IoDiskReadAheadRelease(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uint8_t* buffer = (uint8_t*)0;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (!state->Loading) {
buffer = state->Buffer;
state->Buffer = (uint8_t*)0;
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->SequentialScore = 0;
KeReleaseSpinlock(&state->Lock, flags);
break;
}
KeReleaseSpinlock(&state->Lock, flags);
KeYield();
}
if (buffer)
MmFreePool(buffer);
}
// Invalidates cached disk read-ahead data for one disk.
static void IoDiskReadAheadInvalidate(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Returns true when one range fully contains another range.
static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount,
uint64_t innerBase, uint32_t innerCount) {
uint64_t outerEnd = outerBase + outerCount;
uint64_t innerEnd = innerBase + innerCount;
if (outerEnd < outerBase || innerEnd < innerBase)
return false;
return innerBase >= outerBase && innerEnd <= outerEnd;
}
// Attempts to satisfy a disk read from the per-disk read-ahead buffer.
static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) {
if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0)
return false;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool hit = state->Buffer && state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, lba, count);
if (hit) {
size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES;
size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES;
memcpy(buffer, state->Buffer + offset, bytes);
state->Stats.ReadaheadHits++;
state->Stats.ReadaheadBytesServed += bytes;
} else {
state->Stats.ReadaheadMisses++;
}
KeReleaseSpinlock(&state->Lock, flags);
return hit;
}
// Records request size and sequential access state for one disk read.
static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES || count == 0)
return 0;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadRequests++;
state->Stats.ReadSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count);
if (lba == state->NextLba) {
if (state->SequentialScore < 32)
state->SequentialScore++;
} else {
state->SequentialScore = 1;
}
state->NextLba = lba + count;
uint32_t score = state->SequentialScore;
if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE)
state->Stats.ReadaheadSequentialDetections++;
KeReleaseSpinlock(&state->Lock, flags);
return score;
}
// Records one disk write and invalidates stale read-ahead data.
static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.WriteRequests++;
state->Stats.WriteSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Implements the notify Disk Change operation.
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
@@ -477,6 +642,8 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
if (pnpStatus == STATUS_NOT_SUPPORTED)
device->PnpState = DevicePnpStarted;
IoDiskReadAheadPrepare(driveIndex);
IoReferenceDeviceObject(device);
gDiskDevices[driveIndex] = device;
IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
@@ -490,6 +657,8 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return STATUS_NOT_FOUND;
IoDiskReadAheadInvalidate(driveIndex);
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
@@ -518,6 +687,7 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoDiskReadAheadRelease(driveIndex);
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
@@ -632,6 +802,112 @@ __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
return st;
}
// Validates that a sector range fits the disk and IRP byte length.
static KSTATUS IoValidateDiskSectorRange(DEVICE_OBJECT* dev, uint64_t lba, uint32_t count) {
if (!dev || count == 0) return STATUS_INVALID_PARAMETER;
if (count > (0xffffffffu / IO_DISK_SECTOR_BYTES)) return STATUS_INVALID_PARAMETER;
if (lba > dev->u.Disk.SectorCount) return STATUS_INVALID_PARAMETER;
if ((uint64_t)count > dev->u.Disk.SectorCount - lba) return STATUS_INVALID_PARAMETER;
return STATUS_SUCCESS;
}
// Submits one raw disk read IRP.
static KSTATUS IoReadDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
uint32_t count, void* buffer) {
if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
return IoSubmitIrpSync(dev, &irp);
}
// Submits one raw disk write IRP.
static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
uint32_t count, void* buffer) {
if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, buffer);
return IoSubmitIrpSync(dev, &irp);
}
// Chooses a conservative read-ahead span from the current sequential score.
static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) {
uint32_t sectors = currentCount;
if (sectors < IO_DISK_READAHEAD_MIN_SECTORS)
sectors = IO_DISK_READAHEAD_MIN_SECTORS;
if (score >= 8 && sectors < 128u)
sectors = 128u;
else if (score >= 4 && sectors < 64u)
sectors = 64u;
else if (score >= 2 && sectors < 32u)
sectors = 32u;
if (sectors > IO_DISK_READAHEAD_MAX_SECTORS)
sectors = IO_DISK_READAHEAD_MAX_SECTORS;
return sectors;
}
// Loads the next sequential disk range into the read-ahead buffer.
static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev,
uint64_t nextLba, uint32_t currentCount, uint32_t score) {
if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
if (!state->Buffer) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score);
uint64_t diskSectors = dev->u.Disk.SectorCount;
if (nextLba >= diskSectors)
return;
if ((uint64_t)sectors > diskSectors - nextLba)
sectors = (uint32_t)(diskSectors - nextLba);
if (sectors == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool covered = state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors);
if (covered) {
KeReleaseSpinlock(&state->Lock, flags);
return;
}
if (state->Loading) {
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint64_t generation = state->Generation;
state->Loading = 1;
state->Valid = 0;
state->Count = 0;
KeReleaseSpinlock(&state->Lock, flags);
KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer);
flags = KeAcquireSpinlock(&state->Lock);
state->Loading = 0;
if (KSUCCESS(st) && generation == state->Generation) {
state->BaseLba = nextLba;
state->Count = sectors;
state->Valid = 1;
state->Stats.ReadaheadStores++;
state->Stats.ReadaheadSectorsLoaded += sectors;
IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors);
} else {
state->Valid = 0;
state->Count = 0;
if (!KSUCCESS(st))
state->Stats.ReadaheadReadFailures++;
}
KeReleaseSpinlock(&state->Lock, flags);
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
@@ -641,9 +917,23 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
if (!KSUCCESS(rangeStatus)) {
IoDereferenceDeviceObject(dev);
return rangeStatus;
}
if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) {
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
}
KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
if (KSUCCESS(st))
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return st;
@@ -658,10 +948,50 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
if (!KSUCCESS(rangeStatus)) {
IoDereferenceDeviceObject(dev);
return rangeStatus;
}
KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
if (KSUCCESS(st))
IoDiskReadAheadNoteWrite(driveIndex, count);
IoDereferenceDeviceObject(dev);
return st;
}
// Queries aggregate disk I/O statistics.
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (gDiskDevices[i])
out->RegisteredDisks++;
out->ReadRequests += state->Stats.ReadRequests;
out->ReadSectors += state->Stats.ReadSectors;
out->WriteRequests += state->Stats.WriteRequests;
out->WriteSectors += state->Stats.WriteSectors;
if (out->MaxReadSectors < state->Stats.MaxReadSectors)
out->MaxReadSectors = state->Stats.MaxReadSectors;
if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
out->MaxWriteSectors = state->Stats.MaxWriteSectors;
out->ReadaheadHits += state->Stats.ReadaheadHits;
out->ReadaheadMisses += state->Stats.ReadaheadMisses;
out->ReadaheadStores += state->Stats.ReadaheadStores;
out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations;
out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed;
out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded;
out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections;
out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed;
out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures;
out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures;
if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors)
out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors;
KeReleaseSpinlock(&state->Lock, flags);
}
}
code/kernel/io_file.c
+283 -18
View File
@@ -4,6 +4,8 @@
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/mm_file_cache.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_userenv.h>
@@ -13,6 +15,7 @@
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <ntlite/syscall.h>
#define MAX_DRIVES 8
@@ -45,11 +48,15 @@ struct IO_ASYNC_FILE_REQUEST_OBJECT {
uint8_t CancelRequested;
FILE_OBJECT* File; // referenced until object destruction
EVENT* WaitEvent; // internal completion event
EVENT* CompletionEvent; // optional, referenced until object destruction
KAPC* CompletionApc; // caller-owned APC storage
IO_STATUS_BLOCK* Iosb; // caller/user-owned completion block
IO_STATUS_BLOCK IosbSnapshot;
MM_LOCKED_USER_BUFFER* LockedBuffer;
MM_LOCKED_USER_BUFFER* LockedIosb;
void* Buffer; // caller/user-owned; must remain valid until completion/cancel
void* Buffer; // caller-owned; user buffers must be locked until completion/cancel
size_t Size;
uint64_t Offset;
};
@@ -65,6 +72,10 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
IO_ASYNC_FILE_REQUEST_OBJECT* req =
CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header);
if (req->WaitEvent) {
ObDereferenceObject(&req->WaitEvent->Header);
req->WaitEvent = (EVENT*)0;
}
if (req->CompletionEvent) {
ObDereferenceObject(&req->CompletionEvent->Header);
req->CompletionEvent = (EVENT*)0;
@@ -73,6 +84,14 @@ static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
ObDereferenceObject(&req->File->Header);
req->File = (FILE_OBJECT*)0;
}
if (req->LockedBuffer) {
MmUnlockUserBuffer(req->LockedBuffer);
req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
}
if (req->LockedIosb) {
MmUnlockUserBuffer(req->LockedIosb);
req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
}
}
static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
@@ -80,6 +99,9 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
size_t information) {
if (!req) return;
req->IosbSnapshot.Status = status;
req->IosbSnapshot.Information = information;
uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
req->State = (status == STATUS_CANCELLED) ? IoAsyncStateCancelled : IoAsyncStateCompleted;
KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
@@ -99,6 +121,18 @@ static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
if (req->CompletionEvent)
KeSignalEvent(req->CompletionEvent);
if (req->WaitEvent)
KeSignalEvent(req->WaitEvent);
if (req->LockedBuffer) {
MmUnlockUserBuffer(req->LockedBuffer);
req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
}
if (req->LockedIosb) {
MmUnlockUserBuffer(req->LockedIosb);
req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
}
}
// Implements the io Async File Dequeue operation.
@@ -226,10 +260,24 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
if (!file || !iosb) return STATUS_INVALID_PARAMETER;
if (!buffer && size) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (!file || !iosb) {
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_INVALID_PARAMETER;
}
if (!buffer && size) {
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_INVALID_PARAMETER;
}
if (!KeCanWait()) {
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_INVALID_IRQL;
}
iosb->Status = STATUS_PENDING;
iosb->Information = 0;
@@ -239,6 +287,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
sizeof(IO_ASYNC_FILE_REQUEST_OBJECT));
if (!req) {
iosb->Status = STATUS_NO_MEMORY;
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
return STATUS_NO_MEMORY;
}
@@ -247,10 +297,23 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
req->Op = op;
req->State = IoAsyncStateQueued;
req->CancelRequested = 0;
KSTATUS waitStatus = KeEventCreate(EventManualReset, 0, &req->WaitEvent);
if (!KSUCCESS(waitStatus)) {
iosb->Status = waitStatus;
MmUnlockUserBuffer(lockedBuffer);
MmUnlockUserBuffer(lockedIosb);
ObDereferenceObject(&req->Header);
return waitStatus;
}
req->File = file;
req->CompletionEvent = completionEvent;
req->CompletionApc = completionApc;
req->Iosb = iosb;
req->IosbSnapshot.Status = STATUS_PENDING;
req->IosbSnapshot.Information = 0;
req->LockedBuffer = lockedBuffer;
req->LockedIosb = lockedIosb;
req->Buffer = buffer;
req->Size = size;
req->Offset = offset;
@@ -288,6 +351,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
ObDereferenceObject(&req->Header); // provider rejected synchronously
iosb->Status = ns;
req->IosbSnapshot.Status = ns;
req->IosbSnapshot.Information = 0;
ObDereferenceObject(&req->Header); // creator reference
return ns;
}
@@ -295,6 +360,8 @@ static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
KSTATUS s = KiIoEnsureAsyncFileIoStarted();
if (!KSUCCESS(s)) {
iosb->Status = s;
req->IosbSnapshot.Status = s;
req->IosbSnapshot.Information = 0;
ObDereferenceObject(&req->Header);
return s;
}
@@ -323,10 +390,12 @@ __declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset,
iosb, completionEvent, completionApc,
requestOut);
lockedBuffer, lockedIosb, requestOut);
}
__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
@@ -334,10 +403,12 @@ __declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const vo
IO_STATUS_BLOCK* iosb,
EVENT* completionEvent,
KAPC* completionApc,
MM_LOCKED_USER_BUFFER* lockedBuffer,
MM_LOCKED_USER_BUFFER* lockedIosb,
IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset,
iosb, completionEvent, completionApc,
requestOut);
lockedBuffer, lockedIosb, requestOut);
}
// Implements the async File Request Object operation.
@@ -345,6 +416,19 @@ __declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJ
return req ? &req->Header : (OBJECT*)0;
}
// Returns the wait event owned by an async file request.
__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
return req ? req->WaitEvent : (EVENT*)0;
}
// Waits for an async file request to complete according to wait flags.
__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, uint32_t flags) {
if (!req || !req->WaitEvent) return STATUS_INVALID_PARAMETER;
if (flags & ~SYS_WAIT_FLAG_ALERTABLE) return STATUS_INVALID_PARAMETER;
return KeWaitForEvent(req->WaitEvent, (flags & SYS_WAIT_FLAG_ALERTABLE) ? 1 : 0);
}
// Cancels async File Request.
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
if (!req) return STATUS_INVALID_PARAMETER;
@@ -377,13 +461,8 @@ __declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJ
__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
IO_STATUS_BLOCK* iosbOut) {
if (!req || !iosbOut) return STATUS_INVALID_PARAMETER;
if (req->Iosb) {
iosbOut->Status = req->Iosb->Status;
iosbOut->Information = req->Iosb->Information;
} else {
iosbOut->Status = STATUS_FAILURE;
iosbOut->Information = 0;
}
iosbOut->Status = req->IosbSnapshot.Status;
iosbOut->Information = req->IosbSnapshot.Information;
return STATUS_SUCCESS;
}
@@ -622,10 +701,51 @@ __declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
}
// Aggregates filesystem traversal statistics across mounted volumes.
__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
for (uint8_t i = 0; i < MAX_DRIVES; i++) {
IO_FILESYSTEM* fs = gVolumes[i].Fs;
if (!fs || !fs->QueryStats) continue;
IO_FILESYSTEM_STATS one;
memset(&one, 0, sizeof(one));
fs->QueryStats(fs, &one);
out->PathLookups += one.PathLookups;
out->PathCacheHits += one.PathCacheHits;
out->PathCacheMisses += one.PathCacheMisses;
out->PathCacheStores += one.PathCacheStores;
out->PathCacheInvalidations += one.PathCacheInvalidations;
if (one.PathCacheGeneration > out->PathCacheGeneration)
out->PathCacheGeneration = one.PathCacheGeneration;
out->DirectorySearches += one.DirectorySearches;
out->DirectoryClustersRead += one.DirectoryClustersRead;
out->DirectoryEntriesScanned += one.DirectoryEntriesScanned;
out->FatChainCacheLookups += one.FatChainCacheLookups;
out->FatChainCacheHits += one.FatChainCacheHits;
out->FatChainCacheMisses += one.FatChainCacheMisses;
out->FatChainCacheStores += one.FatChainCacheStores;
out->FatChainCacheInvalidations += one.FatChainCacheInvalidations;
out->FatChainWalkSteps += one.FatChainWalkSteps;
out->DataReadRuns += one.DataReadRuns;
out->DataReadSectors += one.DataReadSectors;
out->DataReadRunMerges += one.DataReadRunMerges;
if (one.MaxDataReadSpanSectors > out->MaxDataReadSpanSectors)
out->MaxDataReadSpanSectors = one.MaxDataReadSpanSectors;
}
}
// Destructor.
static void KiFileDestroy(OBJECT* obj) {
FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
if (f->Fs && f->FileCtx) f->Fs->Close(f->Fs, f->FileCtx);
if (f->Fs && f->FileCtx) {
(void)IoFlushFile(f);
MmFileCacheInvalidateAll(f);
f->Fs->Close(f->Fs, f->FileCtx);
}
}
// IoOpenFile.
@@ -655,11 +775,73 @@ __declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out
f->Lock = (SPINLOCK)SPINLOCK_INIT;
f->Size = fs->GetSize(fs, fileCtx);
f->Offset = 0;
f->Flags = 0;
f->ReadaheadScore = 0;
f->ReadaheadNextOffset = 0;
f->ReadaheadIssuedUntil = 0;
f->ReadaheadSuppressedUntil = 0;
*out = f;
return STATUS_SUCCESS;
}
static uint64_t KiAlignUpPage64(uint64_t value) {
return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
}
static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) {
if (!file || bytesRead < 512)
return;
if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0)
return;
uint64_t end = offset + bytesRead;
uint64_t start = 0;
uint32_t pages = 0;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
if (offset == file->ReadaheadNextOffset) {
if (file->ReadaheadScore < 32)
file->ReadaheadScore++;
} else {
file->ReadaheadScore = 1;
file->ReadaheadIssuedUntil = end;
}
file->ReadaheadNextOffset = end;
if (end < file->ReadaheadSuppressedUntil) {
pages = 0;
} else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) {
pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u);
start = KiAlignUpPage64(end);
if (start < file->ReadaheadIssuedUntil)
start = KiAlignUpPage64(file->ReadaheadIssuedUntil);
if (start >= fileSize) {
pages = 0;
} else {
uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE);
if (maxEnd > fileSize)
pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE);
file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE);
}
}
KeReleaseSpinlock(&file->Lock, flags);
if (pages) {
KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages);
if (ras == STATUS_QUEUE_FULL) {
flags = KeAcquireSpinlock(&file->Lock);
uint64_t suppressUntil = end + (1024u * 1024u);
if (suppressUntil < end || suppressUntil > fileSize)
suppressUntil = fileSize;
if (file->ReadaheadSuppressedUntil < suppressUntil)
file->ReadaheadSuppressedUntil = suppressUntil;
KeReleaseSpinlock(&file->Lock, flags);
}
}
}
// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -679,7 +861,50 @@ __declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
size_t remaining = (size_t)(fileSize - offset);
if (size > remaining) size = remaining;
return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead);
if ((file->Flags & IO_FILE_FLAG_NO_CACHE) || size < 512)
return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead);
size_t total = 0;
uint8_t* dst = (uint8_t*)buf;
while (total < size) {
uint64_t absolute = offset + total;
uint64_t pageOffset = absolute & ~(uint64_t)(PAGE_SIZE - 1);
size_t inPage = (size_t)(absolute - pageOffset);
size_t chunk = PAGE_SIZE - inPage;
if (chunk > size - total) chunk = size - total;
MM_FILE_CACHE_PAGE page;
KSTATUS s = MmFileCacheGetPage(file, pageOffset, &page);
if (!KSUCCESS(s)) {
if (total) {
*bytesRead = total;
return STATUS_SUCCESS;
}
return s;
}
if (inPage >= page.ValidBytes) {
MmFileCacheReleasePage(&page);
break;
}
size_t available = page.ValidBytes - inPage;
if (chunk > available) chunk = available;
void* srcPage = MmPfnToVirtual(page.Frame);
if (!srcPage) {
MmFileCacheReleasePage(&page);
return total ? STATUS_SUCCESS : STATUS_FAILURE;
}
memcpy(dst + total, (const uint8_t*)srcPage + inPage, chunk);
MmFileCacheReleasePage(&page);
total += chunk;
if (chunk == 0) break;
}
*bytesRead = total;
KiIoMaybeScheduleReadAhead(file, offset, total, fileSize);
return STATUS_SUCCESS;
}
__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
@@ -691,10 +916,21 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
if (!file->Fs->Write) return STATUS_NOT_SUPPORTED;
if (size == 0) return STATUS_SUCCESS;
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
uint64_t oldSize = file->Size;
KeReleaseSpinlock(&file->Lock, flags);
size_t written = 0;
KSTATUS s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
KSTATUS s;
if (file->Flags & IO_FILE_FLAG_NO_CACHE) {
s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
if (KSUCCESS(s) && written)
MmFileCacheInvalidate(file, offset, written);
} else {
s = MmFileCacheWrite(file, offset, buf, size, oldSize, &written);
}
if (KSUCCESS(s)) {
uintptr_t flags = KeAcquireSpinlock(&file->Lock);
flags = KeAcquireSpinlock(&file->Lock);
uint64_t end = offset + written;
if (end > file->Size) file->Size = end;
KeReleaseSpinlock(&file->Lock, flags);
@@ -704,6 +940,16 @@ __declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
return s;
}
// Flushes dirty mapped and cached pages for one file object.
__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) {
if (!file) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
KSTATUS sync = MmVadSyncFileMappings(file);
KSTATUS flush = MmFileCacheFlushFile(file);
return KSUCCESS(sync) ? flush : sync;
}
// IoReadFile - sequential read at current offset.
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
size_t size, size_t* bytesRead) {
@@ -768,6 +1014,11 @@ __declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** o
f->Lock = (SPINLOCK)SPINLOCK_INIT;
f->Size = 0;
f->Offset = 0;
f->Flags = 0;
f->ReadaheadScore = 0;
f->ReadaheadNextOffset = 0;
f->ReadaheadIssuedUntil = 0;
f->ReadaheadSuppressedUntil = 0;
*out = f;
return STATUS_SUCCESS;
@@ -848,8 +1099,12 @@ __declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (!file->Fs->Truncate) return STATUS_NOT_SUPPORTED;
KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize);
KSTATUS flush = IoFlushFile(file);
if (!KSUCCESS(flush)) return flush;
KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize, file->Flags);
if (KSUCCESS(s)) {
MmFileCacheInvalidate(file, newSize, (size_t)-1);
file->Size = newSize;
// Pull the file pointer back if it now sits past EOF.
if (file->Offset > file->Size)
@@ -875,6 +1130,16 @@ __declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
return size;
}
// Queries the filesystem identity for a file object.
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) {
if (!file || !out)
return STATUS_INVALID_PARAMETER;
memset(out, 0, sizeof(*out));
if (!file->Fs || !file->Fs->QueryIdentity)
return STATUS_NOT_SUPPORTED;
return file->Fs->QueryIdentity(file->Fs, file->FileCtx, out);
}
// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
if (file) ObDereferenceObject(&file->Header);
code/kernel/ke_apc.c
+1 -1
View File
@@ -204,7 +204,7 @@ __declspec(dllexport) KSTATUS KeQueueUserApc(THREAD* thread,
if (thread->State == ThreadDead)
return STATUS_OBJECT_DEAD;
KUSER_APC* apc = (KUSER_APC*)MmAllocatePool(sizeof(KUSER_APC));
KUSER_APC* apc = (KUSER_APC*)MmAllocatePoolTag(sizeof(KUSER_APC), MM_POOL_TAG('a', 'p', 'c', ' '));
if (!apc)
return STATUS_NO_MEMORY;
code/kernel/ke_debug.c
+254 -22
View File
@@ -1,20 +1,24 @@
// Kernel debugging and bugcheck support. This file owns panic reporting, process fault reporting, and diagnostic queries.
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>
#include <ntlite/ddi/hal_api.h>
#include <kernel/hal/hal_apic.h>
#include <intrin.h>
#pragma intrinsic(_disable)
#pragma intrinsic(_enable)
#include <kernel/ke_debug.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_work.h>
#include <kernel/mm_virt.h>
#include <kernel/io_file.h>
#include <kernel/ke_system_root.h>
#include <common/printf.h>
int _fltused; // MSVC floating-point helper anchor
@@ -22,15 +26,25 @@ int gDebugEnabled = 0;
static uint32_t gDebugFlags = 0;
void (*PrintChar)(char, uint32_t, uint32_t);
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_DEBUG_LOG_BUFFER_SIZE (50u * 1024u)
#define KE_DEBUG_SCREEN_BUFFER_SIZE (16u * 1024u)
#define KE_DEBUG_LOG_LINE_SIZE 1024u
#define KE_DEBUG_SCREEN_DRAIN_SIZE 512u
static SPINLOCK gDebugPrintLock = SPINLOCK_INIT;
static char* gDebugLogBuffer;
static size_t gDebugLogSize;
static size_t gDebugLogWrite;
static size_t gDebugLogUsed;
static char* gDebugScreenBuffer;
static size_t gDebugScreenSize;
static size_t gDebugScreenRead;
static size_t gDebugScreenWrite;
static size_t gDebugScreenUsed;
static volatile long gDebugLogFlushing;
static volatile long gDebugConsoleOwnsSerial;
static volatile long gDebugConsoleWorkReady;
static KWORK_ITEM gDebugConsoleWorkItem;
// Implements the debug Log Append Locked operation.
static void KiDebugLogAppendLocked(const char* text, size_t len) {
@@ -55,6 +69,95 @@ static void KiDebugLogAppendLocked(const char* text, size_t len) {
gDebugLogUsed += len;
}
// Implements the debug Screen Append Locked operation.
static void KiDebugScreenAppendLocked(const char* text, size_t len) {
if (!gDebugScreenBuffer || gDebugScreenSize == 0 || !text || len == 0)
return;
if (len >= gDebugScreenSize) {
text += len - (gDebugScreenSize - 1u);
len = gDebugScreenSize - 1u;
}
while (gDebugScreenUsed + len > gDebugScreenSize) {
gDebugScreenRead = (gDebugScreenRead + 1u) % gDebugScreenSize;
gDebugScreenUsed--;
}
size_t first = gDebugScreenSize - gDebugScreenWrite;
if (first > len) first = len;
memcpy(gDebugScreenBuffer + gDebugScreenWrite, text, first);
if (len > first)
memcpy(gDebugScreenBuffer, text + first, len - first);
gDebugScreenWrite = (gDebugScreenWrite + len) % gDebugScreenSize;
gDebugScreenUsed += len;
}
// Implements the debug Screen Drain Locked operation.
static size_t KiDebugScreenDrainLocked(char* out, size_t max) {
if (!out || max == 0 || !gDebugScreenBuffer || gDebugScreenSize == 0 || gDebugScreenUsed == 0)
return 0;
size_t len = gDebugScreenUsed;
if (len > max) len = max;
size_t first = gDebugScreenSize - gDebugScreenRead;
if (first > len) first = len;
memcpy(out, gDebugScreenBuffer + gDebugScreenRead, first);
if (len > first)
memcpy(out + first, gDebugScreenBuffer, len - first);
gDebugScreenRead = (gDebugScreenRead + len) % gDebugScreenSize;
gDebugScreenUsed -= len;
return len;
}
// Ensures the asynchronous screen ring exists before console output starts using it.
static bool KiDebugEnsureScreenBuffer(void) {
if (gDebugScreenBuffer && gDebugScreenSize != 0)
return true;
char* buffer = (char*)MmAllocatePoolTag(KE_DEBUG_SCREEN_BUFFER_SIZE, MM_POOL_TAG('d', 's', 'c', 'r'));
if (!buffer)
return false;
memset(buffer, 0, KE_DEBUG_SCREEN_BUFFER_SIZE);
bool installed = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (!gDebugScreenBuffer) {
gDebugScreenBuffer = buffer;
gDebugScreenSize = KE_DEBUG_SCREEN_BUFFER_SIZE;
gDebugScreenRead = 0;
gDebugScreenWrite = 0;
gDebugScreenUsed = 0;
installed = true;
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (!installed)
MmFreePool(buffer);
return true;
}
// Implements the debug Serial Write Char operation.
static void KiDebugSerialWriteChar(char c) {
while ((__inbyte(0x3FD) & 0x20) == 0) { }
if (c == '\n') {
__outbyte(0x3F8, '\r');
while ((__inbyte(0x3FD) & 0x20) == 0) { }
}
__outbyte(0x3F8, (uint8_t)c);
}
// Implements the debug Serial Write Locked operation.
static void KiDebugSerialWriteLocked(const char* text, size_t len) {
if (!text)
return;
for (size_t i = 0; i < len; i++)
KiDebugSerialWriteChar(text[i]);
}
// Implements the debug Console Write operation.
static void KiDebugConsoleWrite(const char* text, size_t len) {
if (!PrintChar || !text)
@@ -63,6 +166,28 @@ static void KiDebugConsoleWrite(const char* text, size_t len) {
PrintChar(text[i], 0xCCCCCC, 0x012456);
}
// Implements the debug Console Work operation.
static void KiDebugConsoleWork(void* context) {
(void)context;
char chunk[KE_DEBUG_SCREEN_DRAIN_SIZE];
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
size_t len = KiDebugScreenDrainLocked(chunk, sizeof(chunk));
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (len == 0)
break;
KiDebugConsoleWrite(chunk, len);
}
}
// Queues the debug Console Work operation if the passive work queue is ready.
static void KiDebugQueueConsoleWork(void) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 1) && KeIsWorkQueueStarted())
(void)KeQueueWorkItem(&gDebugConsoleWorkItem);
}
static volatile long gPanicInProgress = 0;
static volatile uintptr_t gPanicOwnerToken = 0;
static volatile long gPanicStoppedCpuCount = 0;
@@ -116,7 +241,7 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
if (KiPanicIsActive())
KiPanicSecondaryHalt();
if (MmHandleTlbShootdownNmi())
if (MmPteHandleShootdownNmi())
return 1;
return 0;
@@ -124,6 +249,8 @@ __declspec(dllexport) int KeHandleNmiIpi(void) {
// Implements the kernel Put Char operation.
void KiKernelPutChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xCCCCCC, 0x012456);
}
@@ -135,8 +262,11 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
if (!_InterlockedCompareExchange((volatile long*)&gDebugConsoleWorkReady, 1, 0))
KeInitializeWorkItem(&gDebugConsoleWorkItem, KiDebugConsoleWork, (void*)0);
if ((flags & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) && !gDebugLogBuffer) {
gDebugLogBuffer = (char*)MmAllocatePool(KE_DEBUG_LOG_BUFFER_SIZE);
gDebugLogBuffer = (char*)MmAllocatePoolTag(KE_DEBUG_LOG_BUFFER_SIZE, MM_POOL_TAG('d', 'l', 'o', 'g'));
if (gDebugLogBuffer) {
gDebugLogSize = KE_DEBUG_LOG_BUFFER_SIZE;
gDebugLogWrite = 0;
@@ -144,6 +274,9 @@ void KeInitDebug(void (*PrintPtr)(char, uint32_t, uint32_t), uint32_t flags) {
memset(gDebugLogBuffer, 0, gDebugLogSize);
}
}
if (flags & KE_DEBUG_CONSOLE)
(void)KiDebugEnsureScreenBuffer();
}
// Returns debug Flags.
@@ -157,6 +290,13 @@ void KeSetDebugConsoleWriter(void (*PrintPtr)(char, uint32_t, uint32_t)) {
return;
PrintChar = PrintPtr;
printf_set_putchar(KiKernelPutChar);
// After the display manager owns debug drawing, DbgPrint and user console
// writes mirror to COM1 themselves. The display writer is screen-only so
// asynchronous screen flushing does not duplicate serial bytes.
(void)KiDebugEnsureScreenBuffer();
_InterlockedExchange((volatile long*)&gDebugConsoleOwnsSerial, 1);
KiDebugQueueConsoleWork();
}
// Implements the dbg Print operation.
@@ -179,17 +319,37 @@ __declspec(dllexport) void DbgPrint(const char* fmt, ...) {
if (KiPanicIsActive()) {
if (!KiPanicCurrentCpuOwnsReport())
KiPanicSecondaryHalt();
if (gDebugFlags & KE_DEBUG_CONSOLE)
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
KiDebugConsoleWrite(line, len);
}
return;
}
bool queueConsole = false;
bool writeConsoleSync = false;
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(line, len);
if (!_InterlockedCompareExchange((volatile long*)&gDebugLogFlushing, 0, 0))
KiDebugLogAppendLocked(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE)
KiDebugConsoleWrite(line, len);
if (gDebugFlags & KE_DEBUG_CONSOLE) {
if (KeIsWorkQueueStarted() && gDebugScreenBuffer) {
KiDebugScreenAppendLocked(line, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(line, len);
}
// Flushes debug Log To File.
@@ -242,6 +402,23 @@ __declspec(dllexport) KSTATUS KeFlushDebugLogToFile(SYS_STRING_VIEW path) {
return (KSUCCESS(st) && totalWritten == used) ? STATUS_SUCCESS : st;
}
// Attempts to persist the debug log during a panic when the filesystem is usable.
static void KiTryFlushPanicDebugLog(void) {
if (!(gDebugFlags & KE_DEBUG_FILE_LOG))
return;
if (!KeCanWait() || !KeSystemRootReady())
return;
char path[1024];
if (!KeBuildSystemRootPath("kernel-panic.log", path, sizeof(path)))
return;
KSTATUS st = KeFlushDebugLogToFile(SysSvFromCstr(path));
if (!KSUCCESS(st))
DbgPrint("debuglog: panic flush to %s failed st=0x%x\n", path, st);
}
// KePanic.
__declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
KiBeginFatalReport();
@@ -279,15 +456,39 @@ __declspec(dllexport) _Noreturn void KePanic(int code, const char* fmt, ...) {
va_end(va);
}
KiTryFlushPanicDebugLog();
for (;;) __halt();
}
// KePrintConsole - write user-visible console text using the default console palette.
// Panic output keeps a distinct color so fatal paths remain visually obvious.
// User text mirrors to COM1 immediately, then queues expensive screen drawing
// through the passive debug-console worker when that path is available.
__declspec(dllexport) void KePrintConsole(const char* buf, size_t len) {
if (!buf || !PrintChar) return;
for (size_t i = 0; i < len; i++)
PrintChar(buf[i], 0xCCCCCC, 0x012456);
if (!buf || len == 0) return;
bool queueConsole = false;
bool writeConsoleSync = false;
bool canQueueScreen = PrintChar && KeIsWorkQueueStarted() && KiDebugEnsureScreenBuffer();
uintptr_t flags = KeAcquireSpinlock(&gDebugPrintLock);
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteLocked(buf, len);
if (PrintChar) {
if (!KiPanicIsActive() && canQueueScreen) {
KiDebugScreenAppendLocked(buf, len);
queueConsole = true;
} else {
writeConsoleSync = true;
}
}
KeReleaseSpinlock(&gDebugPrintLock, flags);
if (queueConsole)
KiDebugQueueConsoleWork();
else if (writeConsoleSync)
KiDebugConsoleWrite(buf, len);
}
// KeBugCheckProcess - user-mode fault handler
@@ -613,8 +814,22 @@ static void KiDumpFaultFrame(PROCESS* proc, const KFAULT_FRAME* f) {
// Prints crash Char.
static void KiPrintCrashChar(char c) {
if (_InterlockedCompareExchange((volatile long*)&gDebugConsoleOwnsSerial, 1, 1))
KiDebugSerialWriteChar(c);
if (PrintChar)
PrintChar(c, 0xFFAA00, 0x000000); // amber - distinct from green debug / red panic
PrintChar(c, 0xFFAA00, 0x000000);
}
static KSTATUS KiFaultExitStatus(int vector) {
switch (vector) {
case 0: return STATUS_PROCESS_DIVIDE_BY_ZERO;
case 3: return STATUS_PROCESS_BREAKPOINT;
case 6: return STATUS_PROCESS_ILLEGAL_OPCODE;
case 13: return STATUS_PROCESS_GENERAL_PROTECTION;
case 14: return STATUS_PROCESS_PAGE_FAULT;
case 17: return STATUS_PROCESS_ALIGNMENT_FAULT;
default: return STATUS_PROCESS_UNKNOWN_FAULT;
}
}
static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
@@ -648,13 +863,23 @@ static void KiPrintCrashHeader(PROCESS* proc, int vector, uintptr_t rip,
}
// Implements the finish Faulted Process operation.
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread) {
static _Noreturn void KiFinishFaultedProcess(PROCESS* proc, THREAD* thread, KSTATUS exitStatus) {
printf_set_putchar(KiKernelPutChar);
if (proc) KeExitProcess(proc, -1);
if (thread) KeExitThread(thread, -1);
int exitCode = (int)exitStatus;
if (proc) (void)KeTerminateProcess(proc, exitCode);
if (thread) KeExitThread(thread, exitCode);
while (1) __halt();
// KeExitThread normally switches away and never resumes this context. If
// a fault path re-enters here during scheduler bring-up or with no runnable
// peer yet, do not fall back to a local CLI/HLT dead stop: keep offering the
// dead thread to the scheduler with interrupts enabled so waiters such as
// the foreground shell can observe the process exit event.
while (true) {
_enable();
KeYield();
__halt();
}
}
_Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, uintptr_t cr2) {
@@ -664,11 +889,11 @@ _Noreturn void KeBugCheckProcess(int vector, uintptr_t rip, uintptr_t errCode, u
KiPrintCrashHeader(proc, vector, rip, errCode, cr2);
KiDumpModules(proc);
KiFinishFaultedProcess(proc, thread);
KiFinishFaultedProcess(proc, thread, KiFaultExitStatus(vector));
}
// Implements the fault Dispatch Frame operation.
__declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Dispatches a CPU fault frame and returns only when a user #PF was repaired.
__declspec(dllexport) KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) {
// Panic NMIs are broadcast to stop all other CPUs. Do not try to decode
// the interrupted context on a secondary CPU once another CPU owns panic.
if (KiPanicIsActive())
@@ -682,6 +907,12 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
(unsigned long long)(uintptr_t)frame);
}
if (frame->Vector == 14) {
KSTATUS handled = MmVadResolvePageFaultByAddress((uintptr_t)frame->Cr2, frame->ErrorCode);
if (KSUCCESS(handled))
return STATUS_SUCCESS;
}
if ((frame->Cs & 3) == 0) {
KiBeginFatalReport();
@@ -702,7 +933,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
if (vector == 14) {
MmDebugDescribeAddress((uintptr_t)frame->Cr2);
if (frame->Rsp) MmDebugDescribeAddress((uintptr_t)frame->Rsp);
MmDebugDumpUserRegions(proc);
MmVadDebugDumpProcess(proc);
}
if (vector == 14) {
@@ -743,7 +974,7 @@ __declspec(dllexport) _Noreturn void KeFaultDispatchFrame(const KFAULT_FRAME* fr
KiDumpFaultFrame(proc, frame);
KiDumpModules(proc);
KiFinishFaultedProcess(proc, thread);
KiFinishFaultedProcess(proc, thread, KiFaultExitStatus((int)frame->Vector));
}
// KeFaultDispatch - single C entry point for all three IDT fault stubs.
@@ -762,5 +993,6 @@ __declspec(dllexport) _Noreturn void KeFaultDispatch(int vector, uintptr_t rip,
frame.Cr2 = cr2;
frame.Rip = rip;
frame.Cs = cs;
KeFaultDispatchFrame(&frame);
(void)KeFaultDispatchFrame(&frame);
while (1) __halt();
}
code/kernel/ke_display.c
+1150 -282
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_event.c
+204 -6
View File
@@ -1,17 +1,100 @@
// Kernel event primitive. This file owns manual-reset event state and waiter wakeups.
#include <stdbool.h>
#include <intrin.h>
#include <kernel/ke_event.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_apc.h>
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
#include <ntlite/syscall.h>
// Multi-wait context shared by the stack wait blocks for one wait call.
typedef struct KE_EVENT_MULTI_WAIT_CONTEXT KE_EVENT_MULTI_WAIT_CONTEXT;
typedef struct KE_EVENT_WAIT_BLOCK {
LIST_ENTRY Link;
KE_EVENT_MULTI_WAIT_CONTEXT* Context;
EVENT* Event;
uint32_t Index;
uint8_t Armed;
} KE_EVENT_WAIT_BLOCK;
struct KE_EVENT_MULTI_WAIT_CONTEXT {
THREAD* Thread;
KE_EVENT_WAIT_BLOCK* Blocks;
uint32_t Count;
volatile long Completed;
KSTATUS Status;
uint32_t SelectedIndex;
};
// Implements the event Destroy operation.
static void KiEventDestroy(OBJECT* obj) { (void)obj; }
// Implements the event Create operation.
// Completes a multi-wait block while the source event lock is held.
static void KiCompleteMultiWaitBlockLocked(KE_EVENT_WAIT_BLOCK* block) {
if (!block || !block->Armed)
return;
ListRemove(&block->Link);
block->Armed = 0;
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
if (!ctx)
return;
if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) != 0)
return;
ctx->Status = STATUS_SUCCESS;
ctx->SelectedIndex = block->Index;
KeUnblockThreadFromWaitListWithStatus(ctx->Thread, STATUS_SUCCESS);
}
// Removes any wait blocks that are still linked after the wait completed.
static void KiCleanupMultiWaitBlocks(KE_EVENT_MULTI_WAIT_CONTEXT* ctx) {
if (!ctx || !ctx->Blocks)
return;
for (uint32_t i = 0; i < ctx->Count; i++) {
KE_EVENT_WAIT_BLOCK* block = &ctx->Blocks[i];
EVENT* event = block->Event;
if (!event)
continue;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (block->Armed) {
ListRemove(&block->Link);
block->Armed = 0;
}
KeReleaseSpinlock(&event->Lock, flags);
}
}
// Removes a multi-wait from every event list during abort or thread teardown.
static void KiRundownMultiWait(THREAD* thread, void* context, KSTATUS status) {
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = (KE_EVENT_MULTI_WAIT_CONTEXT*)context;
if (!ctx)
return;
if (_InterlockedCompareExchange(&ctx->Completed, 1, 0) == 0) {
ctx->Status = status;
ctx->SelectedIndex = 0;
}
KiCleanupMultiWaitBlocks(ctx);
// Abort wakes a truly blocked waiter. Thread teardown may call this after a
// signal already made the thread ready; in that case only the list cleanup
// above is needed.
if (thread && thread->State == ThreadWaiting)
KeUnblockThreadFromWaitListWithStatus(thread, status);
}
// Creates event.
__declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialState, EVENT **out) {
if (!out) return STATUS_INVALID_PARAMETER;
EVENT* e = (EVENT *)ObAllocateObject(ObjectTypeEvent, sizeof(EVENT));
@@ -21,6 +104,7 @@ __declspec(dllexport) KSTATUS KeEventCreate(EVENT_TYPE type, uint32_t initialSta
e->Type = type;
e->Signaled = initialState ? 1 : 0;
LIST_INIT(&e->WaitList);
LIST_INIT(&e->MultiWaitList);
*out = e;
return STATUS_SUCCESS;
}
@@ -35,22 +119,39 @@ __declspec(dllexport) void KeSignalEvent(EVENT* event) {
while (!LIST_IS_EMPTY(&event->WaitList)) {
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(t, (LIST_ENTRY*)0);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
flags = KeAcquireSpinlock(&event->Lock);
}
while (!LIST_IS_EMPTY(&event->MultiWaitList)) {
LIST_ENTRY* node = event->MultiWaitList.Fwd;
KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link);
KiCompleteMultiWaitBlockLocked(block);
}
} else {
if (!LIST_IS_EMPTY(&event->WaitList)) {
// Auto-reset handoff: transfer the signal directly to exactly one
// waiter. Do not leave Signaled=1, because a later waiter can steal
// the disk/APC completion before the selected thread resumes.
while (!LIST_IS_EMPTY(&event->WaitList)) {
event->Signaled = 0;
LIST_ENTRY* node = ListPopHead(&event->WaitList);
THREAD* t = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(t, (LIST_ENTRY*)0);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadWithStatus(t, STATUS_SUCCESS);
return;
}
while (!LIST_IS_EMPTY(&event->MultiWaitList)) {
LIST_ENTRY* node = event->MultiWaitList.Fwd;
KE_EVENT_WAIT_BLOCK* block = CONTAINING_RECORD(node, KE_EVENT_WAIT_BLOCK, Link);
KE_EVENT_MULTI_WAIT_CONTEXT* ctx = block->Context;
if (ctx && _InterlockedCompareExchange(&ctx->Completed, 0, 0) == 0) {
event->Signaled = 0;
KiCompleteMultiWaitBlockLocked(block);
KeReleaseSpinlock(&event->Lock, flags);
return;
}
KiCompleteMultiWaitBlockLocked(block);
}
event->Signaled = 1;
}
KeReleaseSpinlock(&event->Lock, flags);
@@ -64,7 +165,22 @@ __declspec(dllexport) void KeClearEvent(EVENT* event) {
KeReleaseSpinlock(&event->Lock, flags);
}
// Waits for for Event.
// Attempts to consume an event signal without blocking.
__declspec(dllexport) KSTATUS KeTryWaitEvent(EVENT* event) {
if (!event) return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (!event->Signaled) {
KeReleaseSpinlock(&event->Lock, flags);
return STATUS_TIMEOUT;
}
if (event->Type == EventAutoReset)
event->Signaled = 0;
KeReleaseSpinlock(&event->Lock, flags);
return STATUS_SUCCESS;
}
// Waits for Event.
__declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
if (!event) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
@@ -85,3 +201,85 @@ __declspec(dllexport) KSTATUS KeWaitForEvent(EVENT* event, uint8_t alertable) {
return KeCommitBlockThread(self, alertable);
}
// Waits for any of a set of events, returning the selected index.
__declspec(dllexport) KSTATUS KeWaitForMultipleEvents(EVENT** events, uint32_t count,
uint8_t alertable, uint64_t timeoutMs,
uint32_t* signaledIndexOut) {
if (!events || count == 0 || count > SYS_WAIT_MANY_MAX_HANDLES)
return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
if (alertable && KeAreKernelApcsDisabled()) return STATUS_INVALID_PARAMETER;
for (uint32_t i = 0; i < count; i++) {
if (!events[i]) return STATUS_INVALID_PARAMETER;
KSTATUS s = KeTryWaitEvent(events[i]);
if (KSUCCESS(s)) {
if (signaledIndexOut) *signaledIndexOut = i;
return STATUS_SUCCESS;
}
}
if (timeoutMs == 0)
return STATUS_TIMEOUT;
KE_EVENT_WAIT_BLOCK stackBlocks[SYS_WAIT_MANY_MAX_HANDLES];
KE_EVENT_MULTI_WAIT_CONTEXT ctx;
ctx.Thread = KeGetCurrentThread();
ctx.Blocks = stackBlocks;
ctx.Count = count;
ctx.Completed = 0;
ctx.Status = STATUS_PENDING;
ctx.SelectedIndex = 0;
for (uint32_t i = 0; i < count; i++) {
stackBlocks[i].Context = &ctx;
stackBlocks[i].Event = events[i];
stackBlocks[i].Index = i;
stackBlocks[i].Armed = 0;
LIST_INIT(&stackBlocks[i].Link);
}
KePrepareBlockThread(ctx.Thread, (OBJECT*)0, alertable);
KeSetThreadWaitRundown(ctx.Thread, KiRundownMultiWait, &ctx);
if (timeoutMs != SYS_WAIT_TIMEOUT_INFINITE) {
KSTATUS ts = KeArmCurrentThreadTimeout(timeoutMs, STATUS_TIMEOUT);
if (!KSUCCESS(ts)) {
KeClearThreadWaitRundown(ctx.Thread);
KeUnblockThreadFromWaitListWithStatus(ctx.Thread, ts);
return ts;
}
}
for (uint32_t i = 0; i < count; i++) {
EVENT* event = events[i];
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0 && event->Signaled) {
if (event->Type == EventAutoReset)
event->Signaled = 0;
ctx.Status = STATUS_SUCCESS;
ctx.SelectedIndex = i;
_InterlockedExchange(&ctx.Completed, 1);
KeReleaseSpinlock(&event->Lock, flags);
KeUnblockThreadFromWaitListWithStatus(ctx.Thread, STATUS_SUCCESS);
break;
}
if (_InterlockedCompareExchange(&ctx.Completed, 0, 0) == 0) {
stackBlocks[i].Armed = 1;
ListInsertTail(&event->MultiWaitList, &stackBlocks[i].Link);
}
KeReleaseSpinlock(&event->Lock, flags);
}
KSTATUS waitStatus = KeCommitBlockThread(ctx.Thread, alertable);
if (_InterlockedCompareExchange(&ctx.Completed, 1, 0) == 0)
ctx.Status = waitStatus;
KiCleanupMultiWaitBlocks(&ctx);
KeClearThreadWaitRundown(ctx.Thread);
if (ctx.Status == STATUS_SUCCESS && signaledIndexOut)
*signaledIndexOut = ctx.SelectedIndex;
return ctx.Status;
}
code/kernel/ke_input.c
+94 -10
View File
@@ -8,6 +8,8 @@
#include <kernel/ke_input.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_display.h>
#include <kernel/ke_event.h>
#include <kernel/ob_object.h>
#define KEY_QUEUE_SIZE 256u
#define KEY_QUEUE_MASK (KEY_QUEUE_SIZE - 1u)
@@ -32,6 +34,7 @@ static volatile uint32_t gModifiers;
typedef struct INPUT_KEY_QUEUE_ENTRY {
INPUT_KEY key;
char ch;
uint32_t modifiers;
} INPUT_KEY_QUEUE_ENTRY;
static INPUT_KEY_QUEUE_ENTRY gKeyQueue[KEY_QUEUE_SIZE];
@@ -50,6 +53,8 @@ static volatile int gMouseBoundsH = 1080;
static volatile uint8_t gMouseButtons;
static volatile uint32_t gNextInputDeviceId = 1;
static EVENT* gKeyboardEvent;
static EVENT* gInputStateEvent;
static char KiInputKeyToChar(INPUT_KEY key);
@@ -77,14 +82,18 @@ static const KEY_CHAR_PAIR gCharMap[] = {
{ INPUT_KEY_KP_0, '0', '0' }, { INPUT_KEY_KP_PERIOD, '.', '.' },
};
// Implements the input Queue Key operation.
// Queues a keyboard event and wakes one blocked reader when the event was accepted.
static void KiInputQueueKey(INPUT_KEY key, char ch) {
if (key == INPUT_KEY_NONE) return;
int next = (gKeyTail + 1) & (int)KEY_QUEUE_MASK;
if (next == gKeyHead) return;
gKeyQueue[gKeyTail].key = key;
gKeyQueue[gKeyTail].ch = ch;
gKeyQueue[gKeyTail].modifiers = gModifiers;
gKeyTail = next;
if (gKeyboardEvent)
KeSignalEvent(gKeyboardEvent);
}
// Implements the input Queue Has Space operation.
@@ -211,6 +220,17 @@ void KeInputInitialize(const HAL_API* hal) {
gMouseHead = gMouseTail = 0;
gMouseButtons = 0;
if (!gKeyboardEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gKeyboardEvent);
if (!KSUCCESS(s))
DbgPrint("Input: keyboard wait event create failed st=0x%x\n", s);
}
if (!gInputStateEvent) {
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gInputStateEvent);
if (!KSUCCESS(s))
DbgPrint("Input: state wait event create failed st=0x%x\n", s);
}
int w = 1920, h = 1080;
DISPLAY_MODE mode;
if (KSUCCESS(KeDisplayGetMode(&mode))) {
@@ -252,6 +272,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 1u;
KiInputSetModifier(key, true);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputQueueKey(key, KiInputKeyToChar(key));
KiInputStartRepeat(key, KiInputNowTicks());
} else {
@@ -260,6 +281,7 @@ __declspec(dllexport) void KeInputSubmitKey(const INPUT_KEY_EVENT* event) {
gKeyDown[key] = 0u;
KiInputSetModifier(key, false);
if (gInputStateEvent) KeSignalEvent(gInputStateEvent);
KiInputStopRepeat(key);
}
}
@@ -341,27 +363,60 @@ INPUT_KEY KeInputKbPoll(void) {
return (gKeyHead == gKeyTail) ? INPUT_KEY_NONE : gKeyQueue[gKeyHead].key;
}
// Waits until at least one keyboard event is available.
KSTATUS KeInputWaitKey(uint8_t alertable) {
if (KeInputKbPoll() != INPUT_KEY_NONE)
return STATUS_SUCCESS;
if (!gKeyboardEvent)
return STATUS_FAILURE;
for (;;) {
KSTATUS s = KeWaitForEvent(gKeyboardEvent, alertable);
if (!KSUCCESS(s))
return s;
if (KeInputKbPoll() != INPUT_KEY_NONE)
return STATUS_SUCCESS;
}
}
// Implements the input Is Key Down operation.
bool KeInputIsKeyDown(INPUT_KEY key) {
if (key == INPUT_KEY_NONE || key >= MAX_INPUT_KEYS) return false;
return gKeyDown[key] != 0;
}
INPUT_KEY KeInputReadKey(void) {
// Implements the input Peek Event operation.
bool KeInputPeekEvent(INPUT_KEY_RECORD* record) {
if (!record) return false;
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return INPUT_KEY_NONE;
INPUT_KEY key = gKeyQueue[gKeyHead].key;
if (gKeyHead == gKeyTail) return false;
INPUT_KEY_QUEUE_ENTRY entry = gKeyQueue[gKeyHead];
record->Key = entry.key;
record->Char = entry.ch;
record->Reserved = 0;
record->Modifiers = entry.modifiers;
return true;
}
// Implements the input Read Event operation.
bool KeInputReadEvent(INPUT_KEY_RECORD* record) {
if (!KeInputPeekEvent(record)) return false;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
return key;
return true;
}
// Implements the input Read Key operation.
INPUT_KEY KeInputReadKey(void) {
INPUT_KEY_RECORD record;
return KeInputReadEvent(&record) ? record.Key : INPUT_KEY_NONE;
}
// Implements the input Read Char operation.
char KeInputReadChar(void) {
if (gKeyHead == gKeyTail) KiInputServiceRepeat();
if (gKeyHead == gKeyTail) return 0;
char ch = gKeyQueue[gKeyHead].ch;
gKeyHead = (gKeyHead + 1) & (int)KEY_QUEUE_MASK;
return ch;
INPUT_KEY_RECORD record;
return KeInputReadEvent(&record) ? record.Char : 0;
}
// Implements the input Get Modifiers operation.
@@ -416,3 +471,32 @@ void KeInputMouseSetBounds(int w, int h) {
gMouseBoundsH = h;
KeInputMouseSetPosition(gMouseX, gMouseY);
}
// Destroys an input queue handle.
static void KiInputQueueDestroy(OBJECT* object) {
INPUT_QUEUE* queue = CONTAINING_RECORD(object, INPUT_QUEUE, Header);
if (queue->WaitEvent) {
ObDereferenceObject(&queue->WaitEvent->Header);
queue->WaitEvent = (EVENT*)0;
}
}
// Opens the default keyboard input queue as a waitable object.
KSTATUS KeInputOpenQueue(INPUT_QUEUE** out) {
if (!out) return STATUS_INVALID_PARAMETER;
*out = (INPUT_QUEUE*)0;
if (!gInputStateEvent) return STATUS_FAILURE;
INPUT_QUEUE* queue = (INPUT_QUEUE*)ObAllocateObject(ObjectTypeInputQueue, sizeof(INPUT_QUEUE));
if (!queue) return STATUS_NO_MEMORY;
queue->Header.Destroy = KiInputQueueDestroy;
ObReferenceObject(&gInputStateEvent->Header);
queue->WaitEvent = gInputStateEvent;
*out = queue;
return STATUS_SUCCESS;
}
// Returns the event that wakes when keyboard state changes.
EVENT* KeInputQueueWaitEvent(INPUT_QUEUE* queue) {
return queue ? queue->WaitEvent : (EVENT*)0;
}
code/kernel/ke_irp.c
+1 -1
View File
@@ -138,7 +138,7 @@ static size_t IoIrpSize(uint8_t stackCount) {
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
if (!outIrp) return STATUS_INVALID_PARAMETER;
if (stackCount == 0) stackCount = 1;
IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount));
IRP* irp = (IRP*)MmAllocatePoolTag(IoIrpSize(stackCount), MM_POOL_TAG('i', 'r', 'p', ' '));
if (!irp) return STATUS_NO_MEMORY;
IoInitializeIrp(irp, stackCount);
*outIrp = irp;
code/kernel/ke_mutex.c
+5 -4
View File
@@ -12,7 +12,7 @@
static void KiMutexDestroy(OBJECT* obj) { (void)obj; }
// Implements the mutex Create operation.
KSTATUS KeMutexCreate(MUTEX **out) {
__declspec(dllexport) KSTATUS KeMutexCreate(MUTEX **out) {
if (!out) return STATUS_INVALID_PARAMETER;
MUTEX* m = (MUTEX *)ObAllocateObject(ObjectTypeMutex, sizeof(MUTEX));
if (!m) return STATUS_NO_MEMORY;
@@ -25,7 +25,7 @@ KSTATUS KeMutexCreate(MUTEX **out) {
}
// Implements the acquire Mutex operation.
KSTATUS KeAcquireMutex(MUTEX* mutex) {
__declspec(dllexport) KSTATUS KeAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
if (!KeCanWait()) return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
@@ -45,7 +45,7 @@ KSTATUS KeAcquireMutex(MUTEX* mutex) {
}
// Implements the try Acquire Mutex operation.
KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
__declspec(dllexport) KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
if (!mutex) return STATUS_INVALID_PARAMETER;
THREAD* self = KeGetCurrentThread();
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
@@ -59,13 +59,14 @@ KSTATUS KeTryAcquireMutex(MUTEX* mutex) {
}
// Releases mutex.
void KeReleaseMutex(MUTEX* mutex) {
__declspec(dllexport) void KeReleaseMutex(MUTEX* mutex) {
if (!mutex) return;
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
if (!LIST_IS_EMPTY(&mutex->WaitList)) {
LIST_ENTRY* node = ListPopHead(&mutex->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
// Direct handoff: the selected waiter owns the mutex when its wait
// returns. Do not drop Owner to NULL, or another thread can steal it.
mutex->Owner = waiter;
code/kernel/ke_process.c
+148 -20
View File
@@ -6,7 +6,9 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_event.h>
#include <kernel/ke_system_root.h>
#include <kernel/ke_userenv.h>
#include <kernel/io_file.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_cpu.h>
@@ -140,7 +142,7 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
if (p == system) info->Flags |= SYS_PROCESS_FLAG_SYSTEM;
if (p == current) info->Flags |= SYS_PROCESS_FLAG_CURRENT;
MmQueryProcessUserMemory(p, &info->UserRegionCount,
MmVadQueryProcessMemory(p, &info->VadCount,
&info->UserReservedBytes,
&info->UserCommittedBytes);
@@ -158,27 +160,128 @@ size_t KeCopyProcessInfo(struct SYS_PROCESS_INFO* infoOut, size_t maxCount) {
return count;
}
// Destructor (called by ObDereferenceObject when RefCount hits zero).
static void KiProcessDestroy(OBJECT* obj) {
PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
// Converts a kernel thread state to the public ABI state.
static uint32_t KiThreadStateToSys(THREAD_STATE state) {
switch (state) {
case ThreadReady: return SYS_THREAD_STATE_READY;
case ThreadRunning: return SYS_THREAD_STATE_RUNNING;
case ThreadWaiting: return SYS_THREAD_STATE_WAITING;
case ThreadDead: return SYS_THREAD_STATE_DEAD;
default: return SYS_THREAD_STATE_DEAD;
}
}
if (proc == gSystemProcess)
gSystemProcess = (PROCESS*)0;
// Converts a kernel object type to the public wait-object type.
static uint32_t KiWaitObjectTypeToSys(OBJECT_TYPE type) {
switch (type) {
case ObjectTypeProcess: return SYS_THREAD_WAIT_PROCESS;
case ObjectTypeThread: return SYS_THREAD_WAIT_THREAD;
case ObjectTypeMutex: return SYS_THREAD_WAIT_MUTEX;
case ObjectTypeSemaphore: return SYS_THREAD_WAIT_SEMAPHORE;
case ObjectTypeEvent: return SYS_THREAD_WAIT_EVENT;
case ObjectTypeQueue: return SYS_THREAD_WAIT_QUEUE;
case ObjectTypeFile: return SYS_THREAD_WAIT_FILE;
case ObjectTypeDir: return SYS_THREAD_WAIT_DIR;
case ObjectTypeAsyncIo: return SYS_THREAD_WAIT_ASYNC_IO;
case ObjectTypeSocket: return SYS_THREAD_WAIT_SOCKET;
case ObjectTypeDevice: return SYS_THREAD_WAIT_DEVICE;
case ObjectTypeDriver: return SYS_THREAD_WAIT_DRIVER;
case ObjectTypeUsbDevice: return SYS_THREAD_WAIT_USB_DEVICE;
case ObjectTypeUsbPipe: return SYS_THREAD_WAIT_USB_PIPE;
case ObjectTypeFileMapping: return SYS_THREAD_WAIT_FILE_MAPPING;
case ObjectTypeSurface: return SYS_THREAD_WAIT_SURFACE;
default: return SYS_THREAD_WAIT_UNKNOWN;
}
}
// Release all remaining handles.
for (int i = 1; i < HANDLE_TABLE_SIZE; i++) {
if (proc->Handles[i].Object) {
ObDereferenceObject(proc->Handles[i].Object);
// Copies one process's thread telemetry into caller-provided storage.
KSTATUS KeCopyProcessThreadInfo(uint32_t pid, struct SYS_THREAD_INFO* infoOut, size_t maxCount, size_t* countOut) {
if (!countOut) return STATUS_INVALID_PARAMETER;
if (maxCount && !infoOut) return STATUS_INVALID_PARAMETER;
PROCESS* process = KeGetProcessById(pid);
if (!process) return STATUS_NOT_FOUND;
THREAD* current = KeGetCurrentThread();
size_t count = 0;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
LIST_ENTRY* head = &process->ThreadList;
for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
THREAD* thread = CONTAINING_RECORD(node, THREAD, ProcessLink);
if (infoOut && count < maxCount) {
SYS_THREAD_INFO* info = &infoOut[count];
memset(info, 0, sizeof(*info));
info->Size = sizeof(*info);
info->ProcessId = process->ProcessId;
info->ThreadId = thread->ThreadId;
info->State = KiThreadStateToSys(thread->State);
info->Priority = (uint32_t)thread->Priority;
info->RunningCpu = thread->RunningCpu;
info->LastCpu = thread->LastCpu;
info->QuantumRemaining = thread->QuantumRemaining;
info->QuantumTicks = thread->QuantumTicks;
info->CpuBurstTicks = thread->CpuBurstTicks;
info->LastRunTicks = thread->LastRunTicks;
info->WaitObjectType = thread->WaitObject ? KiWaitObjectTypeToSys(thread->WaitObject->Type) : SYS_THREAD_WAIT_NONE;
info->WaitStatus = (uint32_t)thread->WaitStatus;
info->KernelApcDisable = thread->KernelApcDisable;
if (thread == current) info->Flags |= SYS_THREAD_FLAG_CURRENT;
if (thread->IsIdleThread) info->Flags |= SYS_THREAD_FLAG_IDLE;
if (!thread->Teb) info->Flags |= SYS_THREAD_FLAG_KERNEL;
if (thread->Queued) info->Flags |= SYS_THREAD_FLAG_QUEUED;
if (thread->AlertableWait) info->Flags |= SYS_THREAD_FLAG_ALERTABLE_WAIT;
if (thread->KernelApcPending) info->Flags |= SYS_THREAD_FLAG_KERNEL_APC_PENDING;
if (thread->UserApcPending) info->Flags |= SYS_THREAD_FLAG_USER_APC_PENDING;
}
count++;
}
KeReleaseSpinlock(&process->HandleLock, flags);
*countOut = count;
ObDereferenceObject(&process->Header);
return STATUS_SUCCESS;
}
// Releases all kernel objects still reachable from a process handle table.
// This is part of process exit, not merely object destruction: a parent may keep
// a handle to an exited process for wait/query, but the dead process must not
// keep its own files, mappings, queues, sockets, or synchronization objects alive.
static void KiProcessReleaseAllHandles(PROCESS* proc) {
if (!proc) return;
for (uint32_t i = 1; i < HANDLE_TABLE_SIZE; ++i) {
OBJECT* obj = (OBJECT*)0;
uintptr_t flags = KeAcquireSpinlock(&proc->HandleLock);
obj = proc->Handles[i].Object;
if (obj) {
proc->Handles[i].Object = (OBJECT*)0;
proc->Handles[i].Rights = 0;
proc->Handles[i].Flags = 0;
}
KeReleaseSpinlock(&proc->HandleLock, flags);
if (obj)
ObDereferenceObject(obj);
}
}
// Releases the process-owned address space and per-process kernel resources.
// Idempotent because the normal exit path and the final OBJECT destructor can
// both observe a process whose threads are already gone.
static void KiProcessRundownResources(PROCESS* proc) {
if (!proc) return;
if (_InterlockedExchange(&proc->ResourcesReleased, 1) != 0)
return;
KiProcessReleaseAllHandles(proc);
// Release every user VA region owned by this process. This includes the
// main image, DLLs, TEBs/stacks that survived thread teardown, process
// environment strings, and leaked user allocations.
MmDestroyAllUserRegions(proc);
// environment strings, mmap views, and leaked user allocations.
MmVadDestroyAllForProcess(proc);
proc->ImageBase = 0;
proc->ImageSize = 0;
proc->ModuleCount = 0;
@@ -190,8 +293,21 @@ static void KiProcessDestroy(OBJECT* obj) {
proc->UserCmdlineSize = 0;
proc->UserCwdBase = 0;
proc->UserCwdSize = 0;
}
// Release the exit event.
// Destructor (called by ObDereferenceObject when RefCount hits zero).
static void KiProcessDestroy(OBJECT* obj) {
PROCESS* proc = CONTAINING_RECORD(obj, PROCESS, Header);
if (proc == gSystemProcess)
gSystemProcess = (PROCESS*)0;
// If the process failed during launch, or no normal exit completion path
// ran, make sure its own handles and VADs are still reclaimed.
KiProcessRundownResources(proc);
// Release the exit event only when the process object itself dies. External
// process handles may still wait/query an exited process after rundown.
if (proc->ExitEvent) {
ObDereferenceObject(&proc->ExitEvent->Header);
proc->ExitEvent = (EVENT *)0;
@@ -227,6 +343,7 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->ExitCode = 0;
proc->Terminating = 0;
proc->Exited = 0;
proc->ResourcesReleased = 0;
proc->ImageBase = 0;
proc->ImageSize = 0;
proc->UserPebBase = 0;
@@ -240,10 +357,10 @@ KSTATUS KeCreateProcess(const char* name, PROCESS **out) {
proc->ModuleCount = 0;
proc->CurrentDirectory[0] = '\0';
proc->CurrentDirectoryLength = 0;
proc->CurrentDrive = 0;
proc->CurrentDrive = KeGetSystemRootDrive();
proc->CwdLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&proc->UserRegions);
proc->UserRegionLock = (SPINLOCK)SPINLOCK_INIT;
LIST_INIT(&proc->VadList);
proc->VadLock = (SPINLOCK)SPINLOCK_INIT;
// Slot 0 is the invalid handle - leave it NULL.
memset(proc->Handles, 0, sizeof(proc->Handles));
@@ -427,7 +544,7 @@ fail:
}
// Process exit / termination.
static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
void KeCompleteProcessExitIfLastThread(PROCESS* process) {
if (!process) return;
uintptr_t flags = KeAcquireSpinlock(&process->HandleLock);
@@ -435,6 +552,7 @@ static void KiCompleteProcessExitIfLastThread(PROCESS* process) {
KeReleaseSpinlock(&process->HandleLock, flags);
if (threadCount == 0 && _InterlockedExchange(&process->Exited, 1) == 0) {
KiProcessRundownResources(process);
if (process->ExitEvent) KeSignalEvent(process->ExitEvent);
}
}
@@ -448,7 +566,7 @@ void KeExitProcess(PROCESS* process, int exitCode) {
// The calling thread exits immediately after this in SysExitProcess. Do
// not drop the process object here: each thread owns one process reference,
// and KiThreadDestroy releases that reference after unlinking the thread.
KiCompleteProcessExitIfLastThread(process);
KeCompleteProcessExitIfLastThread(process);
}
// Implements the terminate Process operation.
@@ -469,7 +587,15 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
if (t == self)
continue;
(void)KeAlertThread(t);
// A terminating process must not be held open by worker threads that
// are blocked in non-alertable waits. User-mode DLLs such as Bink can
// leave IO/submit workers parked on semaphores; after the foreground
// thread crashes, those waiters would otherwise keep ThreadCount > 0
// and prevent the process ExitEvent from waking the shell.
if (t->State == ThreadWaiting)
KeAbortThreadWait(t, STATUS_ALERTED);
else
(void)KeAlertThread(t);
if (t->RunningCpu != KE_INVALID_CPU && t->RunningCpu != selfIndex && t->RunningCpu < gCpuCount) {
KCPU* cpu = &gCpus[t->RunningCpu];
@@ -481,7 +607,7 @@ KSTATUS KeTerminateProcess(PROCESS* process, int exitCode) {
}
KeReleaseSpinlock(&process->HandleLock, flags);
KiCompleteProcessExitIfLastThread(process);
KeCompleteProcessExitIfLastThread(process);
return STATUS_SUCCESS;
}
@@ -535,6 +661,8 @@ void KeCloseHandle(PROCESS* process, uint32_t handle) {
process->Handles[handle].Rights = 0;
process->Handles[handle].Flags = 0;
KeReleaseSpinlock(&process->HandleLock, flags);
if (obj->Type == ObjectTypeFile)
(void)IoFlushFile((FILE_OBJECT*)obj);
ObDereferenceObject(obj);
return;
}
code/kernel/ke_sched.c
+133 -7
View File
@@ -17,6 +17,7 @@
#include <kernel/hal/hal_apic.h>
static LIST_ENTRY gRunQueue[PriorityCount];
static LIST_ENTRY gSleepList;
static SPINLOCK gSchedLock;
static uint32_t gAgingCounter = 0;
@@ -305,6 +306,7 @@ static void KiIdleLoop(void) {
// Initializes scheduler Global.
void KiInitializeSchedulerGlobal(void) {
for (int i = 0; i < PriorityCount; i++) LIST_INIT(&gRunQueue[i]);
LIST_INIT(&gSleepList);
gSchedLock = (SPINLOCK)SPINLOCK_INIT;
KSTATUS s = KeCreateProcess("idle", &gIdleProcess);
@@ -349,6 +351,8 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
idle->WaitStatus = STATUS_SUCCESS;
idle->WaitListHead = (LIST_ENTRY*)0;
idle->AlertableWait = 0;
idle->SleepUntilTick = 0;
idle->TimeoutStatus = STATUS_SUCCESS;
idle->UserStackBase = (void*)0;
idle->Teb = (void*)0;
idle->TebSize = 0;
@@ -358,7 +362,7 @@ KSTATUS KiCreateIdleThreadForCpu(KCPU* cpu) {
LIST_INIT(&idle->ProcessLink);
LIST_INIT(&idle->SchedulerLink);
void* kstack = MmAllocatePool(KSTACK_SIZE);
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
ObDereferenceObject(&idle->Header);
return STATUS_NO_MEMORY;
@@ -621,12 +625,10 @@ void KiSchedulerNoteVoluntaryBlock(THREAD* thread) {
}
// Implements the scheduler Wake Thread operation.
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
// Completes a wait while the scheduler lock is held.
static void KiWakeThreadLocked(THREAD* thread, KSTATUS status) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
// Complete the wait while holding the scheduler lock. On SMP a wake can
// race the prepare->commit window on another CPU: the target thread is
// already on the object's wait list and has State=ThreadWaiting, but it is
@@ -634,7 +636,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
// That case must complete in-place, not enqueue the still-running thread.
if (thread->State != ThreadWaiting) {
thread->WaitStatus = status;
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
@@ -642,6 +643,8 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
thread->WaitStatus = status;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
thread->CpuBurstTicks = 0;
thread->QuantaConsumed = 0;
@@ -652,7 +655,6 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
// KeCommitBlockThread() will see State != ThreadWaiting and return the
// completed wait status without ever entering the ready queue.
thread->State = ThreadRunning;
KeReleaseSpinlock(&gSchedLock, flags);
return;
}
@@ -662,10 +664,49 @@ void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
KiInsertReadyLocked(thread);
KiPokeRunnableCpusLocked(thread);
}
// Wakes a thread after first removing its scheduler-link wait membership.
void KeUnblockThreadFromWaitListWithStatus(THREAD* thread, KSTATUS status) {
if (!thread) return;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (thread->State == ThreadWaiting && thread->WaitListHead) {
ListRemove(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
}
KiWakeThreadLocked(thread, status);
KeReleaseSpinlock(&gSchedLock, flags);
}
// Implements the scheduler Wake Thread operation.
void KiSchedulerWakeThread(THREAD* thread, KSTATUS status) {
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
KiWakeThreadLocked(thread, status);
KeReleaseSpinlock(&gSchedLock, flags);
}
// Wakes threads whose scheduler-tick sleep deadline has expired.
static void KiWakeExpiredSleepersLocked(uint64_t nowTick) {
LIST_ENTRY* node = gSleepList.Fwd;
while (node != &gSleepList) {
LIST_ENTRY* next = node->Fwd;
THREAD* thread = CONTAINING_RECORD(node, THREAD, SchedulerLink);
if (thread->State == ThreadWaiting &&
thread->WaitListHead == &gSleepList &&
thread->SleepUntilTick <= nowTick) {
ListRemove(node);
thread->WaitListHead = (LIST_ENTRY*)0;
KiWakeThreadLocked(thread, thread->TimeoutStatus);
}
node = next;
}
}
// Implements the scheduler Tick operation.
void KeSchedulerTick(void) {
KCPU* cpu = KeGetCurrentCpu();
@@ -677,6 +718,7 @@ void KeSchedulerTick(void) {
KiArmPostSwitchUnlock(cur, flags);
gSchedStats.Ticks++;
KiWakeExpiredSleepersLocked(gSchedStats.Ticks);
int requestReschedule = 0;
@@ -745,6 +787,90 @@ void KeSchedulerTick(void) {
KiRequestReschedule();
}
// Arms a timeout for the current thread after a wait primitive has prepared it to block.
KSTATUS KeArmCurrentThreadTimeout(uint64_t milliseconds, KSTATUS timeoutStatus) {
if (milliseconds == 0)
return STATUS_INVALID_PARAMETER;
if (!KeCanWait())
return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (!self || self->IsIdleThread)
return STATUS_INVALID_PARAMETER;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
if (ticks == 0)
ticks = 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (self->State != ThreadWaiting || self->WaitListHead != (LIST_ENTRY*)0 ||
self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
KeReleaseSpinlock(&gSchedLock, flags);
return STATUS_INVALID_PARAMETER;
}
uint64_t now = gSchedStats.Ticks;
uint64_t wakeTick = now + ticks;
if (wakeTick < now)
wakeTick = UINT64_MAX;
self->WaitListHead = &gSleepList;
self->SleepUntilTick = wakeTick;
self->TimeoutStatus = timeoutStatus;
ListInsertTail(&gSleepList, &self->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return STATUS_SUCCESS;
}
// Implements the delay Thread operation.
__declspec(dllexport) KSTATUS KeDelayThread(uint64_t milliseconds) {
if (milliseconds == 0) {
KeYield();
return STATUS_SUCCESS;
}
if (!KeCanWait())
return STATUS_INVALID_IRQL;
THREAD* self = KeGetCurrentThread();
if (!self || self->IsIdleThread)
return STATUS_INVALID_PARAMETER;
uint64_t ticks = milliseconds / SCHED_TICK_MS;
if ((milliseconds % SCHED_TICK_MS) != 0)
ticks++;
if (ticks == 0)
ticks = 1;
uintptr_t flags = KeAcquireSpinlock(&gSchedLock);
if (self->Queued || self->State != ThreadRunning ||
self->RunningCpu != KeGetCurrentCpu()->CpuIndex) {
KeReleaseSpinlock(&gSchedLock, flags);
return STATUS_INVALID_PARAMETER;
}
uint64_t now = gSchedStats.Ticks;
uint64_t wakeTick = now + ticks;
if (wakeTick < now)
wakeTick = UINT64_MAX;
self->State = ThreadWaiting;
self->WaitObject = (OBJECT*)0;
self->WaitStatus = STATUS_PENDING;
self->WaitListHead = &gSleepList;
self->AlertableWait = 0;
self->SleepUntilTick = wakeTick;
self->TimeoutStatus = STATUS_SUCCESS;
ListInsertTail(&gSleepList, &self->SchedulerLink);
KeReleaseSpinlock(&gSchedLock, flags);
return KeCommitBlockThread(self, 0);
}
// Implements the yield operation.
__declspec(dllexport) void KeYield(void) {
KeAssertIrql(PASSIVE_LEVEL);
code/kernel/ke_selftest.c
+443 -3
View File
@@ -14,10 +14,13 @@
#include <kernel/ke_sched.h>
#include <ntlite/status.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_trap.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_swap.h>
#include <kernel/mm_pfn.h>
#include <kernel/ob_object.h>
#include <kernel/ke_smp.h>
@@ -478,11 +481,118 @@ static void KiRunIrpKernelApcSelfTest(const HAL_API* hal) {
// Virtual memory mapping invariants.
// Verifies that the PFN database tracks ownership, references, pins, and contiguous runs.
static void KiRunPfnDatabaseSelfTest(void) {
MM_PFN_STATS before;
MM_PFN_STATS after;
MmPfnQueryStats(&before);
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: failed to allocate kernel frame\n");
MM_PFN* pfn = MmPfnGet(frame);
if (!pfn || pfn->State != MmPfnStateActive || pfn->Usage != MmPfnUseKernel || pfn->RefCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST,
"PFN selftest: allocated frame metadata invalid phys=%016llx\n",
(unsigned long long)frame);
if (!KSUCCESS(MmPfnRetain(frame)) || pfn->RefCount != 2)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: retain failed ref=%u\n", pfn->RefCount);
if (!KSUCCESS(MmPfnRelease(frame)) || pfn->RefCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: release failed ref=%u\n", pfn->RefCount);
if (!KSUCCESS(MmPfnPin(frame)) || pfn->PinCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pin failed pin=%u\n", pfn->PinCount);
if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: pinned frame was freed\n");
if (!KSUCCESS(MmPfnUnpin(frame)) || pfn->PinCount != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: unpin failed pin=%u\n", pfn->PinCount);
if (!KSUCCESS(MmPfnAddMapping(frame)) || pfn->MappedCount != 1)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping add failed maps=%u\n", pfn->MappedCount);
if (MmPfnFree(frame) != STATUS_ACCESS_DENIED)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapped frame was freed\n");
if (!KSUCCESS(MmPfnRemoveMapping(frame)) || pfn->MappedCount != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: mapping remove failed maps=%u\n", pfn->MappedCount);
if (!KSUCCESS(MmPfnFree(frame)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: free failed\n");
PHYSICAL_ADDRESS run = MmPfnAllocateContiguous(2, MmPfnUseDma);
if (!run)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous allocation failed\n");
MM_PFN* p0 = MmPfnGet(run);
MM_PFN* p1 = MmPfnGet(run + PAGE_SIZE);
if (!p0 || !p1 || p0->Usage != MmPfnUseDma || p1->Usage != MmPfnUseDma)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: contiguous metadata invalid\n");
MmPfnFree(run);
MmPfnFree(run + PAGE_SIZE);
// Accounting buckets are part of the VM contract. A missing increment here
// usually means a new allocation path is using an anonymous usage class.
PHYSICAL_ADDRESS clean = MmPfnAllocate(MmPfnUseCleanFileCache);
PHYSICAL_ADDRESS image = MmPfnAllocate(MmPfnUseImage);
PHYSICAL_ADDRESS mapped = MmPfnAllocate(MmPfnUseMappedFile);
PHYSICAL_ADDRESS anon = MmPfnAllocate(MmPfnUseAnonymous);
PHYSICAL_ADDRESS cow = MmPfnAllocate(MmPfnUseCow);
PHYSICAL_ADDRESS surface = MmPfnAllocate(MmPfnUseSurface);
if (!clean || !image || !mapped || !anon || !cow || !surface)
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: bucket allocation failed\n");
MM_PFN_STATS bucketStats;
MmPfnQueryStats(&bucketStats);
if (bucketStats.CleanFileCacheFrames < before.CleanFileCacheFrames + 1 ||
bucketStats.ImageFrames < before.ImageFrames + 1 ||
bucketStats.MappedFileFrames < before.MappedFileFrames + 1 ||
bucketStats.AnonymousFrames < before.AnonymousFrames + 1 ||
bucketStats.CowFrames < before.CowFrames + 1 ||
bucketStats.SurfaceFrames < before.SurfaceFrames + 1) {
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: usage bucket accounting failed\n");
}
// Standby is the clean-page holding state used by later file-cache and image
// reclaim. It must not change the free count until the owning cache drops it.
if (!KSUCCESS(MmPfnMoveToStandby(clean)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby transition failed\n");
if (!KSUCCESS(MmPfnMoveToStandby(image)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby transition failed\n");
MM_PFN_STATS standbyStats;
MmPfnQueryStats(&standbyStats);
if (standbyStats.StandbyCleanFileCacheFrames < bucketStats.StandbyCleanFileCacheFrames + 1 ||
standbyStats.StandbyImageFrames < bucketStats.StandbyImageFrames + 1 ||
standbyStats.ReclaimableFrames < bucketStats.ReclaimableFrames + 2 ||
standbyStats.FreeFrames != bucketStats.FreeFrames) {
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: standby accounting failed\n");
}
if (!KSUCCESS(MmPfnActivate(clean)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: clean standby activation failed\n");
if (!KSUCCESS(MmPfnActivate(image)))
KePanic(KE_PANIC_CODE_SELFTEST, "PFN selftest: image standby activation failed\n");
(void)MmPfnFree(clean);
(void)MmPfnFree(image);
(void)MmPfnFree(mapped);
(void)MmPfnFree(anon);
(void)MmPfnFree(cow);
(void)MmPfnFree(surface);
MmPfnQueryStats(&after);
if (before.FreeFrames != after.FreeFrames)
KePanic(KE_PANIC_CODE_SELFTEST,
"PFN selftest: free count leak before=%llu after=%llu\n",
(unsigned long long)before.FreeFrames,
(unsigned long long)after.FreeFrames);
DbgPrint("PFN selftest: metadata/refcount/pin/buckets/standby/contiguous PASS\n");
}
// Verifies the page-table manager invariants that the higher VMM will depend on.
static void KiRunMemoryMappingSelfTest(void) {
PHYSICAL_ADDRESS phys = 0;
uint64_t flags = 0;
KSTATUS s = MmQueryPage(0, &phys, &flags);
KSTATUS s = MmPteQueryPageFor(MmPteQuerySelfTest, 0, &phys, &flags);
if (s != STATUS_NOT_FOUND) {
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: virtual page zero is mapped phys=%016llx flags=%016llx status=%u\n",
(unsigned long long)phys,
@@ -490,7 +600,329 @@ static void KiRunMemoryMappingSelfTest(void) {
(unsigned)s);
}
DbgPrint("MM selftest: page zero unmapped\n");
uintptr_t testVa = 0xFFFFA00000000000ULL;
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (s != STATUS_NOT_FOUND) {
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA already mapped va=%016llx phys=%016llx flags=%016llx\n",
(unsigned long long)testVa,
(unsigned long long)phys,
(unsigned long long)flags);
}
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseKernel);
if (!frame)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: failed to allocate test frame\n");
s = MmPteMapPage(testVa, frame, PAGE_PRESENT | PAGE_WRITE | PAGE_NX, MmPteMapFailIfPresent);
if (!KSUCCESS(s)) {
MmPfnFree(frame);
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteMapPage failed status=%u\n", s);
}
volatile uint64_t* probe = (volatile uint64_t*)testVa;
*probe = 0x1122334455667788ULL;
if (*probe != 0x1122334455667788ULL)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: mapped test page did not retain data\n");
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (!KSUCCESS(s) || phys != frame || !(flags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: query after map failed status=%u phys=%016llx frame=%016llx flags=%016llx\n",
s,
(unsigned long long)phys,
(unsigned long long)frame,
(unsigned long long)flags);
s = MmPteProtectRange(testVa, PAGE_SIZE, PAGE_PRESENT | PAGE_NX);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteProtectRange failed status=%u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (!KSUCCESS(s) || (flags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: read-only protect failed status=%u flags=%016llx\n",
s,
(unsigned long long)flags);
s = MmPteUnmapPage(testVa);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: MmPteUnmapPage failed status=%u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, testVa, &phys, &flags);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "MM selftest: test VA still mapped after unmap status=%u\n", s);
MmPfnFree(frame);
DbgPrint("MM selftest: PTE map/query/protect/unmap PASS\n");
}
// Verifies sparse VAD commitment and NOACCESS protection metadata.
static void KiRunVadSparseSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: no process context\n");
size_t beforeVadCount = 0;
size_t beforeReserved = 0;
size_t beforeCommitted = 0;
MmVadQueryProcessMemory(process, &beforeVadCount, &beforeReserved, &beforeCommitted);
void* base = (void*)0;
KSTATUS s = MmVadReserve(process,
PAGE_SIZE * 4,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: reserve failed %u\n", s);
uint8_t* p = (uint8_t*)base;
s = MmVadCommit(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first commit failed %u\n", s);
s = MmVadCommit(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX, MM_VAD_ZERO);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse commit failed %u\n", s);
void* queryBase = (void*)0;
size_t querySize = 0;
uint64_t queryFlags = 0;
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (!KSUCCESS(s) || queryBase != p || querySize != PAGE_SIZE || !(queryFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first query failed status=%u base=%p size=%llu flags=%016llx\n",
s, queryBase, (unsigned long long)querySize, (unsigned long long)queryFlags);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: demand-zero page was mapped before access status=%u\n", s);
s = MmVadEnsureResident(process, p, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: first demand-zero materialization failed %u\n", s);
s = MmVadEnsureResident(process, p + PAGE_SIZE * 2, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: sparse demand-zero materialization failed %u\n", s);
s = MmVadQuery(process, p + PAGE_SIZE, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: uncommitted hole queried as committed status=%u\n", s);
p[0] = 0x5A;
p[PAGE_SIZE * 2] = 0xA5;
s = MmVadProtect(process, p, PAGE_SIZE, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS protect failed %u\n", s);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)p, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS page still had a PTE status=%u\n", s);
s = MmVadQuery(process, p, &queryBase, &querySize, &queryFlags, (MM_VAD_TYPE*)0, (uint32_t*)0);
if (!KSUCCESS(s) || queryFlags != 0)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: NOACCESS query failed status=%u flags=%016llx\n",
s, (unsigned long long)queryFlags);
s = MmVadProtect(process, p, PAGE_SIZE, PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: remap after NOACCESS failed %u\n", s);
if (p[0] != 0x5A || p[PAGE_SIZE * 2] != 0xA5)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: protected page contents were lost\n");
size_t afterVadCount = 0;
size_t afterReserved = 0;
size_t afterCommitted = 0;
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
if (afterVadCount != beforeVadCount + 1 || afterCommitted != beforeCommitted + PAGE_SIZE * 2)
KePanic(KE_PANIC_CODE_SELFTEST,
"VAD selftest: accounting mismatch before=%llu after=%llu vads=%llu/%llu\n",
(unsigned long long)beforeCommitted,
(unsigned long long)afterCommitted,
(unsigned long long)beforeVadCount,
(unsigned long long)afterVadCount);
s = MmVadDestroy(process, base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: destroy failed %u\n", s);
MmVadQueryProcessMemory(process, &afterVadCount, &afterReserved, &afterCommitted);
if (afterVadCount != beforeVadCount || afterCommitted != beforeCommitted)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD selftest: accounting leaked after destroy\n");
DbgPrint("VAD selftest: sparse commit/protect/query/demand-zero PASS\n");
}
// Verifies that writable copy-on-write views split before either alias can modify shared state.
static void KiRunVadCopyOnWriteSelfTest(void) {
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: no process context\n");
void* base = (void*)0;
KSTATUS s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source allocation failed %u\n", s);
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source materialization failed %u\n", s);
uint8_t* source = (uint8_t*)base;
source[0] = 0x31;
void* cowBase = (void*)0;
s = MmVadCreateCowView(process, base, PAGE_SIZE, process, &cowBase);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: COW view creation failed %u\n", s);
PHYSICAL_ADDRESS sourcePhys = 0;
PHYSICAL_ADDRESS cowPhys = 0;
uint64_t sourceFlags = 0;
uint64_t cowFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, &sourcePhys, &sourceFlags);
if (!KSUCCESS(s) || (sourceFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: source was not write-protected status=%u flags=%016llx\n",
s, (unsigned long long)sourceFlags);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &cowPhys, &cowFlags);
if (!KSUCCESS(s) || (cowFlags & PAGE_WRITE) || cowPhys != sourcePhys)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: view was not shared read-only status=%u src=%016llx cow=%016llx flags=%016llx\n",
s,
(unsigned long long)sourcePhys,
(unsigned long long)cowPhys,
(unsigned long long)cowFlags);
s = MmVadResolveUserPageFault(process, (uintptr_t)cowBase, 0x3);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: simulated write fault split failed %u\n", s);
uint8_t* copy = (uint8_t*)cowBase;
copy[0] = 0x62;
if (source[0] != 0x31 || copy[0] != 0x62)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: copy isolation failed src=%02x cow=%02x\n",
source[0], copy[0]);
PHYSICAL_ADDRESS splitPhys = 0;
uint64_t splitFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)cowBase, &splitPhys, &splitFlags);
if (!KSUCCESS(s) || splitPhys == sourcePhys || !(splitFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
s,
(unsigned long long)sourcePhys,
(unsigned long long)splitPhys,
(unsigned long long)splitFlags);
(void)MmVadDestroy(process, cowBase);
(void)MmVadDestroy(process, base);
void* zeroBase = (void*)0;
s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&zeroBase);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page VAD creation failed %u\n", s);
s = MmVadEnsureResident(process, zeroBase, PAGE_SIZE, PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page read materialization failed %u\n", s);
PHYSICAL_ADDRESS zeroPhys = 0;
uint64_t zeroFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &zeroPhys, &zeroFlags);
if (!KSUCCESS(s) || (zeroFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero page was not mapped read-only status=%u flags=%016llx\n",
s, (unsigned long long)zeroFlags);
s = MmVadResolveUserPageFault(process, (uintptr_t)zeroBase, 0x3);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split failed %u\n", s);
PHYSICAL_ADDRESS privatePhys = 0;
uint64_t privateFlags = 0;
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)zeroBase, &privatePhys, &privateFlags);
if (!KSUCCESS(s) || privatePhys == zeroPhys || !(privateFlags & PAGE_WRITE))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page split mapping wrong status=%u old=%016llx new=%016llx flags=%016llx\n",
s,
(unsigned long long)zeroPhys,
(unsigned long long)privatePhys,
(unsigned long long)privateFlags);
((uint8_t*)zeroBase)[0] = 0x9C;
if (((uint8_t*)zeroBase)[0] != 0x9C)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD COW selftest: zero-page private write failed\n");
(void)MmVadDestroy(process, zeroBase);
DbgPrint("VAD COW selftest: shared views and zero-page split PASS\n");
}
// Verifies that a private resident VAD page can move to swap and fault back in.
static void KiRunVadSwapSelfTest(void) {
if (!MmSwapIsEnabled()) {
DbgPrint("VAD swap selftest: SKIP, swap is disabled\n");
return;
}
PROCESS* process = KeGetCurrentProcess();
if (!process)
process = KeGetSystemProcess();
if (!process)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: no process context\n");
void* base = (void*)0;
KSTATUS s = MmVadCreateCommitted(process,
PAGE_SIZE,
PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO,
MmVadGeneric,
&base);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: allocation failed %u\n", s);
s = MmVadEnsureResident(process, base, PAGE_SIZE, PAGE_WRITE | PAGE_USER);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: materialization failed %u\n", s);
uint8_t* p = (uint8_t*)base;
p[0] = 0xC1;
p[PAGE_SIZE - 1] = 0x5E;
void* trimmed = (void*)0;
s = MmVadTrimOnePageForTest(process, &trimmed);
if (!KSUCCESS(s) || trimmed != base)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: trim failed status=%u trimmed=%p base=%p\n", s, trimmed, base);
s = MmPteQueryPageFor(MmPteQuerySelfTest, (uintptr_t)base, (PHYSICAL_ADDRESS*)0, (uint64_t*)0);
if (s != STATUS_NOT_FOUND)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: swapped page still had a PTE status=%u\n", s);
s = MmVadResolveUserPageFault(process, (uintptr_t)base, 0);
if (!KSUCCESS(s))
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: page-in fault failed %u\n", s);
if (p[0] != 0xC1 || p[PAGE_SIZE - 1] != 0x5E)
KePanic(KE_PANIC_CODE_SELFTEST, "VAD swap selftest: contents changed after page-in %02x/%02x\n", p[0], p[PAGE_SIZE - 1]);
(void)MmVadDestroy(process, base);
DbgPrint("VAD swap selftest: private pageout/pagein PASS\n");
}
// String-view ABI self-test.
@@ -614,6 +1046,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
&st.Iosb,
st.CompletionEvent,
&st.Apc,
(MM_LOCKED_USER_BUFFER*)0,
(MM_LOCKED_USER_BUFFER*)0,
&readReq);
if (s != STATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: submit returned %u\n", s);
@@ -683,6 +1117,8 @@ static void KiRunAsyncFileIoSelfTest(void) {
&wiosb,
wev,
(KAPC*)0,
(MM_LOCKED_USER_BUFFER*)0,
(MM_LOCKED_USER_BUFFER*)0,
&writeReq);
if (s != STATUS_PENDING)
KePanic(KE_PANIC_CODE_SELFTEST, "FILE/ASYNC selftest: write submit returned %u\n", s);
@@ -832,7 +1268,11 @@ void KiRunEarlyKernelSelfTests(void) {
void KiRunKernelSelfTests(const HAL_API* hal) {
DbgPrint("Kernel selftests: begin\n");
KiRunKernelApcSelfTest();
KiRunPfnDatabaseSelfTest();
KiRunMemoryMappingSelfTest();
KiRunVadSparseSelfTest();
KiRunVadCopyOnWriteSelfTest();
KiRunVadSwapSelfTest();
KiRunStringViewSelfTest();
KiRunIrpKernelApcSelfTest(hal);
KiRunAsyncFileIoSelfTest();
code/kernel/ke_semaphore.c
+1
View File
@@ -68,6 +68,7 @@ __declspec(dllexport) void KeSignalSemaphore(SEMAPHORE* sem) {
// consuming a count later.
LIST_ENTRY* node = ListPopHead(&sem->WaitList);
THREAD* waiter = CONTAINING_RECORD(node, THREAD, SchedulerLink);
KeSetThreadWaitList(waiter, (LIST_ENTRY*)0);
KeReleaseSpinlock(&sem->Lock, flags);
KeUnblockThreadWithStatus(waiter, STATUS_SUCCESS);
return;
code/kernel/ke_smp.c
+9 -7
View File
@@ -20,6 +20,7 @@
#include <kernel/ke_event.h>
#include <kernel/ob_object.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/hal/hal_apic.h>
@@ -141,15 +142,17 @@ static KSTATUS KiBuildApTrampoline(void) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN,
(uint64_t)AP_TRAMPOLINE_MAX_PHYS);
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_PREFERRED_MIN,
AP_TRAMPOLINE_MAX_PHYS);
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_PREFERRED_MIN,
AP_TRAMPOLINE_MAX_PHYS,
MmPfnUseKernel);
}
if (!frame) {
DbgPrint("SMP: trying PMM AP trampoline range 0x%016llX-0x%016llX\n",
(uint64_t)AP_TRAMPOLINE_FALLBACK_MIN,
(uint64_t)AP_TRAMPOLINE_PREFERRED_MIN);
frame = MmAllocateFrameInRange(AP_TRAMPOLINE_FALLBACK_MIN,
AP_TRAMPOLINE_PREFERRED_MIN);
frame = MmPfnAllocateInRange(AP_TRAMPOLINE_FALLBACK_MIN,
AP_TRAMPOLINE_PREFERRED_MIN,
MmPfnUseKernel);
}
if (!frame) {
DbgPrint("SMP: no free PMM frame available below 0x%016llX for AP trampoline\n",
@@ -589,7 +592,7 @@ void KeBuildProcessorTopology(void) {
// Creates ap Stack.
static KSTATUS KiAllocateApStack(KCPU* cpu) {
// AP kernel stacks do not need physically contiguous memory. Requiring
// MmAllocateContiguous(8) made SMP startup depend on a DMA-style run being
// MmPfnAllocateContiguous(8) made SMP startup depend on a DMA-style run being
// available, which is fragile on real machines and unrelated to AP startup.
size_t stackBytes = (size_t)AP_STACK_PAGES * PAGE_SIZE;
void* stack = MmAllocatePool(stackBytes);
@@ -967,12 +970,11 @@ KSTATUS KeRunApplicationProcessorSchedulerSelfTests(void) {
cpuIndex, st);
}
THREAD* thread = (THREAD*)0;
st = KeCreateKernelThreadWithAffinity((uintptr_t)KiSmpApControlledWorker,
(uintptr_t)arg,
PriorityNormal,
(1ULL << cpuIndex),
&thread);
(THREAD**)0);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_SELFTEST,
"SMP scheduler selftest: AP%u worker create failed status=0x%x\n",
code/kernel/ke_syscall.asm
+38 -1
View File
@@ -152,7 +152,7 @@ KeSyscallEntry ENDP
; also shows which register held the null target.
;
; KeFaultDispatchFrame signature (MS x64):
; void KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; KSTATUS KeFaultDispatchFrame(const KFAULT_FRAME* frame) ; rcx
; ---------------------------------------------------------------------------
KFAULT_MAGIC EQU 3146544C5541464Bh ; "KFAULTF1" little-ish
@@ -379,8 +379,45 @@ pf_kernel_frame:
pf_call:
and rsp, -16
sub rsp, FAULT_CALL_SHADOW
mov rbx, rax ; Keep the KFAULT_FRAME pointer across C.
mov rcx, rax
call KeFaultDispatchFrame
test eax, eax
jnz pf_unhandled_return
mov rdx, rbx ; rdx = KFAULT_FRAME*.
mov rcx, [rdx + KF_Rcx]
mov r8, [rdx + KF_R8]
mov r9, [rdx + KF_R9]
mov rsi, [rdx + KF_Rsi]
mov rdi, [rdx + KF_Rdi]
mov rbp, [rdx + KF_Rbp]
mov r12, [rdx + KF_R12]
mov r13, [rdx + KF_R13]
mov r14, [rdx + KF_R14]
mov r15, [rdx + KF_R15]
mov rbx, [rdx + KF_Rbx]
lea rsp, [rdx + KFAULT_SIZE + 24] ; Skip saved RAX/R11 and the CPU error code.
mov rax, [rdx + KF_Rax]
mov r11, [rdx + KF_R11]
mov r10, [rdx + KF_Cs]
test r10, 3
jz pf_resume_no_swapgs
mov r10, [rdx + KF_R10]
mov rdx, [rdx + KF_Rdx]
swapgs
iretq
pf_resume_no_swapgs:
mov r10, [rdx + KF_R10]
mov rdx, [rdx + KF_Rdx]
iretq
pf_unhandled_return:
cli
pf_unhandled_halt:
hlt
jmp pf_unhandled_halt
KeFaultPF ENDP
code/kernel/ke_syscall.c
+1526 -275
View File
File diff suppressed because it is too large. Load diff
code/kernel/ke_system_root.c
+97
View File
@@ -0,0 +1,97 @@
// System root volume selection. This file keeps boot-volume identity in one place so loaders do not guess.
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <kernel/ke_system_root.h>
#include <kernel/ke_debug.h>
#include <kernel/io_file.h>
static uint8_t gSystemRootDrive;
static bool gSystemRootReady;
// Builds a drive-qualified path without depending on any process CWD.
static bool KiBuildDrivePath(uint8_t drive, const char* relative, char* out, size_t outSize) {
if (!relative || !out || drive >= KE_SYSTEM_ROOT_MAX_DRIVES)
return false;
size_t relLen = strlen(relative);
if (relLen + 4 > outSize)
return false;
out[0] = (char)('0' + drive);
out[1] = ':';
out[2] = '/';
memcpy(out + 3, relative, relLen + 1);
return true;
}
// Returns true when the candidate volume looks like the volume that booted the kernel.
static bool KiDriveLooksLikeSystemRoot(uint8_t drive) {
static const char* required[] = {
"system/colonel.exe",
"system/hal.dll",
"drivers/drivers.ini",
};
char path[1024];
for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) {
if (!KiBuildDrivePath(drive, required[i], path, sizeof(path)))
return false;
FILE_OBJECT* file = NULL;
KSTATUS s = IoOpenFile(SysSvFromCstr(path), &file);
if (!KSUCCESS(s))
return false;
IoCloseFile(file);
}
return true;
}
// Selects the mounted volume that contains the kernel/system tree.
KSTATUS KeInitializeSystemRootVolume(void) {
for (uint8_t drive = 0; drive < KE_SYSTEM_ROOT_MAX_DRIVES; drive++) {
if (!IoFilesystemPresent(drive))
continue;
if (!KiDriveLooksLikeSystemRoot(drive))
continue;
gSystemRootDrive = drive;
gSystemRootReady = true;
DbgPrint("System root: drive=%u\n", drive);
return STATUS_SUCCESS;
}
return STATUS_NOT_FOUND;
}
// Returns whether the system root volume has been selected.
bool KeSystemRootReady(void) {
return gSystemRootReady;
}
// Returns the selected system root drive, or zero before discovery.
uint8_t KeGetSystemRootDrive(void) {
return gSystemRootReady ? gSystemRootDrive : 0;
}
// Builds an absolute path rooted at the selected system volume.
bool KeBuildSystemRootPath(const char* relative, char* out, size_t outSize) {
if (!gSystemRootReady)
return false;
return KiBuildDrivePath(gSystemRootDrive, relative, out, outSize);
}
// Opens a file rooted at the selected system volume.
KSTATUS KeOpenSystemRootFile(const char* relative, FILE_OBJECT** out) {
if (!out)
return STATUS_INVALID_PARAMETER;
*out = NULL;
char path[1024];
if (!KeBuildSystemRootPath(relative, path, sizeof(path)))
return STATUS_NOT_FOUND;
return IoOpenFile(SysSvFromCstr(path), out);
}
code/kernel/ke_thread.c
+180 -20
View File
@@ -8,6 +8,8 @@
#include <kernel/ke_cpu.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_event.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_userenv.h>
@@ -36,6 +38,11 @@ extern void KeThreadEntry(void);
static void KiThreadDestroy(OBJECT* obj) {
THREAD* thread = CONTAINING_RECORD(obj, THREAD, Header);
// A wait that uses auxiliary stack wait blocks must be removed before the
// kernel stack is released. A terminated thread may never return to the
// wait syscall to run normal cleanup.
KeRundownThreadWait(thread, STATUS_ALERTED);
KiRundownApcState(thread);
if (thread->ExitEvent) {
@@ -43,16 +50,16 @@ static void KiThreadDestroy(OBJECT* obj) {
thread->ExitEvent = (EVENT*)0;
}
PROCESS* ownerForUserRegions = thread->Owner;
if (ownerForUserRegions) {
PROCESS* ownerForVadList = thread->Owner;
if (ownerForVadList) {
if (thread->Teb && thread->TebSize) {
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->Teb);
MmFreeUserMemoryForProcess(ownerForVadList, thread->Teb);
thread->Teb = (void*)0;
thread->TebSize = 0;
}
if (thread->UserStackBase) {
MmFreeUserMemoryForProcess(ownerForUserRegions, thread->UserStackBase);
MmFreeUserMemoryForProcess(ownerForVadList, thread->UserStackBase);
thread->UserStackBase = (void *)0;
}
}
@@ -66,10 +73,13 @@ static void KiThreadDestroy(OBJECT* obj) {
remaining = owner->ThreadCount;
KeReleaseSpinlock(&owner->HandleLock, flags);
if (remaining == 0 && owner->Terminating &&
// Implements the interlocked Exchange operation.
_InterlockedExchange(&owner->Exited, 1) == 0) {
if (owner->ExitEvent) KeSignalEvent(owner->ExitEvent);
if (remaining == 0) {
// The last thread leaving a process is process exit even if user
// code called ExitThread instead of ExitProcess. KeExitProcess /
// KeTerminateProcess set Terminating earlier; this covers the last
// thread voluntarily exiting without an explicit process-exit call.
_InterlockedExchange(&owner->Terminating, 1);
KeCompleteProcessExitIfLastThread(owner);
}
ObDereferenceObject(&owner->Header);
@@ -111,11 +121,13 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
void* kstack = MmAllocatePool(KSTACK_SIZE);
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
ObDereferenceObject(&thread->Header);
return STATUS_NO_MEMORY;
@@ -127,10 +139,10 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
KSTATUS s;
void* ustack = (void*)0;
s = MmCreateUserRegion(owner, USTACK_SIZE, PAGE_SIZE,
s = MmVadCreateCommitted(owner, USTACK_SIZE, PAGE_SIZE,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
USER_REGION_ZERO | USER_REGION_GUARD_LOW | USER_REGION_GUARD_HIGH,
UserRegionStack,
MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
MmVadStack,
&ustack);
if (!KSUCCESS(s) || !ustack) {
MmFreePool(kstack);
@@ -139,11 +151,19 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
return STATUS_NO_MEMORY;
}
thread->UserStackBase = ustack;
// Ring-3 starts at the image entry via IRET, not CALL, so there is no
// pushed return address. The x64 ABI expects RSP % 16 == 8 at function
// entry; bias the initial stack as if a call had happened so compiler
// generated aligned SSE spills (movaps) are safe.
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8;
// Ring-3 starts at the image entry via IRET, not CALL. The MS x64 ABI
// nevertheless lets callees immediately use the caller-provided 32-byte
// home/shadow area above RSP, and MSVC commonly emits a first instruction
// like "mov [rsp+8], rcx" in unoptimized test code. Bias the synthetic
// entry stack as if a CALL with a normal home area had happened:
//
// RSP % 16 == 8 at function entry
// [RSP + 0] fake return-address slot
// [RSP + 8..39] home/shadow space
//
// Without the 32-byte home area, the first spill lands in the high stack
// guard page and every freshly-created MSVC thread faults immediately.
uintptr_t userStackTop = (uintptr_t)ustack + USTACK_SIZE - 8 - 32;
SYS_TEB* teb = (SYS_TEB*)MmAllocateUserMemoryForProcess(owner, sizeof(SYS_TEB));
if (!teb) {
@@ -211,8 +231,16 @@ __declspec(dllexport) KSTATUS KeCreateThread(PROCESS* owner, uintptr_t entry, ui
KeReleaseSpinlock(&owner->HandleLock, flags);
KeInitThreadFpuContext(thread);
KeReadyThread(thread);
// Hand the creator a short-lived reference before making the thread
// runnable. On SMP a new user thread can run and exit before
// KeCreateThread() returns; without this reference, callers that merely
// read ThreadId for logging or syscall output can race the scheduler's
// dead-thread dereference and touch freed pool.
ObReferenceObject(&thread->Header);
*out = thread;
KeReadyThread(thread);
return STATUS_SUCCESS;
}
@@ -283,6 +311,8 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
thread->WaitStatus = STATUS_SUCCESS;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
thread->SwitchLockCookie = 0x202ULL;
KiInitializeApcState(thread);
thread->TrapFrame = (KTRAP_FRAME*)0;
@@ -290,7 +320,7 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
thread->Teb = (void*)0; // no TEB; GS_BASE will be 0 in kernel ctx
thread->TebSize = 0;
void* kstack = MmAllocatePool(KSTACK_SIZE);
void* kstack = MmAllocatePoolTag(KSTACK_SIZE, MM_POOL_TAG('k', 's', 't', 'k'));
if (!kstack) {
ObDereferenceObject(&thread->Header);
return STATUS_NO_MEMORY;
@@ -320,7 +350,16 @@ static KSTATUS KiCreateKernelThreadWithAffinity(uintptr_t fn, uintptr_t arg,
owner->ThreadCount++;
KeReleaseSpinlock(&owner->HandleLock, flags);
if (out) *out = thread;
// Hand the creator a short-lived reference before making the kernel
// thread runnable. Kernel workers can run, signal their completion event,
// exit, and be reaped on another CPU before KeCreateKernelThread()'s
// caller drops its returned pointer. The live scheduler reference remains
// separate and is released by KeSchedulerPostSwitch() after the dead
// thread has switched away.
if (out) {
ObReferenceObject(&thread->Header);
*out = thread;
}
KeInitThreadFpuContext(thread);
KeReadyThread(thread);
@@ -352,6 +391,8 @@ void KePrepareBlockThread(THREAD* thread, OBJECT* waitObject, uint8_t alertable)
thread->WaitStatus = STATUS_PENDING;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = alertable ? 1 : 0;
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
thread->WaitRundownContext = (void*)0;
}
@@ -362,12 +403,41 @@ void KeSetThreadWaitList(THREAD* thread, LIST_ENTRY* waitListHead) {
thread->WaitListHead = waitListHead;
}
// Registers cleanup for waits with auxiliary wait blocks.
void KeSetThreadWaitRundown(THREAD* thread, KE_WAIT_RUNDOWN_ROUTINE routine, void* context) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
thread->WaitRundownRoutine = routine;
thread->WaitRundownContext = context;
}
// Clears auxiliary wait cleanup after the owning wait has removed its blocks.
void KeClearThreadWaitRundown(THREAD* thread) {
KeAssertIrqlAtOrBelow(DISPATCH_LEVEL);
if (!thread) return;
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
thread->WaitRundownContext = (void*)0;
}
// Runs auxiliary wait cleanup for abort and thread teardown paths.
void KeRundownThreadWait(THREAD* thread, KSTATUS status) {
if (!thread || !thread->WaitRundownRoutine) return;
KE_WAIT_RUNDOWN_ROUTINE routine = thread->WaitRundownRoutine;
void* context = thread->WaitRundownContext;
thread->WaitRundownRoutine = (KE_WAIT_RUNDOWN_ROUTINE)0;
thread->WaitRundownContext = (void*)0;
routine(thread, context, status);
}
// Implements the finish Wait operation.
static void KiFinishWait(THREAD* thread, KSTATUS status) {
thread->WaitObject = (OBJECT*)0;
thread->WaitStatus = status;
thread->WaitListHead = (LIST_ENTRY*)0;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
}
// Implements the commit Block Thread operation.
@@ -385,6 +455,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
if (thread->State != ThreadWaiting) {
KSTATUS status = thread->WaitStatus;
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
return status;
}
@@ -406,6 +478,8 @@ KSTATUS KeCommitBlockThread(THREAD* thread, uint8_t alertable) {
KiDeliverKernelApcs();
thread->AlertableWait = 0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
return thread->WaitStatus;
}
@@ -423,6 +497,92 @@ void KeUnblockThreadWithStatus(THREAD* thread, KSTATUS status) {
KiSchedulerWakeThread(thread, status);
}
// Returns whether the scheduler link still belongs to an object wait list.
static int KiThreadWaitLinkIsLinked(const THREAD* thread) {
if (!thread) return 0;
return thread->SchedulerLink.Fwd != &thread->SchedulerLink ||
thread->SchedulerLink.Bwd != &thread->SchedulerLink;
}
// Removes a thread from one wait list while that wait object's lock is held.
static int KiRemoveThreadFromObjectWaitListLocked(THREAD* thread, LIST_ENTRY* waitListHead) {
if (!thread || !waitListHead) return 0;
if (thread->State != ThreadWaiting || thread->WaitListHead != waitListHead)
return 0;
// The signaling side pops the wait-list entry before it wakes the thread.
// In that handoff window the scheduler link is already self-linked; abort
// must not remove it again or turn a successful mutex/semaphore handoff
// into an aborted wait for a thread that may already own the object.
if (!KiThreadWaitLinkIsLinked(thread))
return 0;
ListRemove(&thread->SchedulerLink);
thread->WaitListHead = (LIST_ENTRY*)0;
thread->SleepUntilTick = 0;
thread->TimeoutStatus = STATUS_SUCCESS;
return 1;
}
// Removes a thread from an object-owned wait queue using that object's lock.
static int KiAbortObjectWait(THREAD* thread) {
if (!thread || !thread->WaitObject || !thread->WaitListHead)
return 0;
OBJECT* object = thread->WaitObject;
switch (object->Type) {
case ObjectTypeEvent: {
EVENT* event = CONTAINING_RECORD(object, EVENT, Header);
uintptr_t flags = KeAcquireSpinlock(&event->Lock);
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &event->WaitList);
KeReleaseSpinlock(&event->Lock, flags);
return removed;
}
case ObjectTypeMutex: {
MUTEX* mutex = CONTAINING_RECORD(object, MUTEX, Header);
uintptr_t flags = KeAcquireSpinlock(&mutex->Lock);
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &mutex->WaitList);
KeReleaseSpinlock(&mutex->Lock, flags);
return removed;
}
case ObjectTypeSemaphore: {
SEMAPHORE* semaphore = CONTAINING_RECORD(object, SEMAPHORE, Header);
uintptr_t flags = KeAcquireSpinlock(&semaphore->Lock);
int removed = KiRemoveThreadFromObjectWaitListLocked(thread, &semaphore->WaitList);
KeReleaseSpinlock(&semaphore->Lock, flags);
return removed;
}
default:
return 0;
}
}
// Aborts a thread's current wait regardless of whether the wait was marked
// alertable. Process teardown uses this for crash/kill paths: a background
// worker blocked forever on a semaphore must not keep the process exit event
// unsignaled after the foreground thread has faulted. Object wait-list removal
// must use the object's own lock; otherwise signal/abort races can leave stale
// scheduler links in an event, mutex, or semaphore wait queue.
void KeAbortThreadWait(THREAD* thread, KSTATUS status) {
if (!thread) return;
if (thread->State != ThreadWaiting) return;
if (thread->WaitRundownRoutine) {
KeRundownThreadWait(thread, status);
return;
}
if (KiAbortObjectWait(thread)) {
KeUnblockThreadWithStatus(thread, status);
return;
}
// Sleep/timeouts and already-dequeued handoffs are owned by the scheduler.
// This path removes from the scheduler sleep list when present and otherwise
// completes the wait in place.
KeUnblockThreadFromWaitListWithStatus(thread, status);
}
// Implements the unblock Thread operation.
void KeUnblockThread(THREAD* thread) {
KeUnblockThreadWithStatus(thread, STATUS_SUCCESS);
code/kernel/ke_work.c
+117
View File
@@ -0,0 +1,117 @@
// Kernel passive-level work queue. This file owns the generic mechanism used
// to move ISR/DPC-originated work to a sleepable system thread context.
#include <stdbool.h>
#include <stddef.h>
#include <intrin.h>
#include <kernel/ke_work.h>
#include <kernel/ke_event.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_debug.h>
static SPINLOCK gWorkQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gWorkQueueList;
static EVENT* gWorkQueueEvent;
static volatile long gWorkQueueListReady;
static volatile long gWorkQueueStartState; // 0=not started, 1=starting, 2=started.
static void KiWorkQueueWorker(void* arg) {
(void)arg;
KeAssertIrql(PASSIVE_LEVEL);
for (;;) {
(void)KeWaitForEvent(gWorkQueueEvent, 0);
for (;;) {
KWORK_ITEM* item = (KWORK_ITEM*)0;
uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
if (!LIST_IS_EMPTY(&gWorkQueueList)) {
LIST_ENTRY* node = ListPopHead(&gWorkQueueList);
item = CONTAINING_RECORD(node, KWORK_ITEM, Link);
}
KeReleaseSpinlock(&gWorkQueueLock, flags);
if (!item)
break;
// Allow the item to be queued again while it is running. This keeps
// level-triggered users such as cursor updates naturally coalesced
// but does not lose an update that arrives during the callback.
_InterlockedExchange(&item->Queued, 0);
if (item->Routine)
item->Routine(item->Context);
}
}
}
void KeInitializeWorkItem(KWORK_ITEM* item, KWORK_ROUTINE routine, void* context) {
if (!item)
return;
LIST_INIT(&item->Link);
item->Routine = routine;
item->Context = context;
item->Queued = 0;
}
bool KeIsWorkQueueStarted(void) {
return _InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 2;
}
bool KeQueueWorkItem(KWORK_ITEM* item) {
if (!item || !item->Routine)
return false;
if (_InterlockedCompareExchange(&item->Queued, 1, 0) != 0)
return false;
if (!_InterlockedCompareExchange(&gWorkQueueListReady, 1, 1)) {
_InterlockedExchange(&item->Queued, 0);
return false;
}
uintptr_t flags = KeAcquireSpinlock(&gWorkQueueLock);
ListInsertTail(&gWorkQueueList, &item->Link);
EVENT* event = gWorkQueueEvent;
KeReleaseSpinlock(&gWorkQueueLock, flags);
if (event)
KeSignalEvent(event);
return true;
}
KSTATUS KeStartWorkQueue(void) {
long state = _InterlockedCompareExchange(&gWorkQueueStartState, 1, 0);
if (state == 2)
return STATUS_SUCCESS;
if (state == 1) {
while (_InterlockedCompareExchange(&gWorkQueueStartState, 2, 2) == 1)
KeYield();
return KeIsWorkQueueStarted() ? STATUS_SUCCESS : STATUS_FAILURE;
}
LIST_INIT(&gWorkQueueList);
_InterlockedExchange(&gWorkQueueListReady, 1);
KSTATUS s = KeEventCreate(EventAutoReset, 0, &gWorkQueueEvent);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
s = KeCreateKernelThread((uintptr_t)KiWorkQueueWorker,
0,
PriorityNormal,
&worker);
// Persistent system worker; keep the creator reference for kernel life.
(void)worker;
}
if (KSUCCESS(s)) {
_InterlockedExchange(&gWorkQueueStartState, 2);
KeSignalEvent(gWorkQueueEvent);
} else {
_InterlockedExchange(&gWorkQueueStartState, 0);
}
return s;
}
code/kernel/kmain.c
+178 -66
View File
@@ -7,7 +7,9 @@
#pragma intrinsic(_enable)
#include <ntlite/boot_info.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_swap.h>
#include <kernel/ob_object.h>
#include <kernel/ke_process.h>
#include <kernel/ke_thread.h>
@@ -28,6 +30,9 @@
#include <kernel/ke_input.h>
#include <kernel/ke_time.h>
#include <kernel/ke_display.h>
#include <kernel/ke_settings.h>
#include <kernel/ke_work.h>
#include <kernel/ke_system_root.h>
#include <common/ini_parser.h>
typedef struct {
@@ -35,6 +40,90 @@ typedef struct {
const HAL_API* hal;
} SYSTEM_INIT_ARGS;
#define KI_SWAP_MIN_MB 4ull
#define KI_SWAP_MAX_MB 8192ull
// Parses a required unsigned decimal integer from settings.ini.
// Configuration errors are treated as boot-time corruption instead of guessed defaults.
static uint64_t KiParseRequiredUInt64Setting(const char* section, const char* key) {
char value[64];
KSTATUS s = KeGetSettingString(section, key, value, sizeof(value));
if (!KSUCCESS(s)) {
KePanic(KE_PANIC_CODE_INIT,
"settings: missing required [%s] %s entry, status=0x%x\n",
section,
key,
s);
}
uint64_t result = 0;
const char* p = value;
if (!*p) {
KePanic(KE_PANIC_CODE_INIT,
"settings: empty numeric value for [%s] %s\n",
section,
key);
}
while (*p) {
if (*p < '0' || *p > '9') {
KePanic(KE_PANIC_CODE_INIT,
"settings: invalid numeric value for [%s] %s: %s\n",
section,
key,
value);
}
uint64_t digit = (uint64_t)(*p - '0');
if (result > (UINT64_MAX - digit) / 10ull) {
KePanic(KE_PANIC_CODE_INIT,
"settings: numeric value for [%s] %s is too large: %s\n",
section,
key,
value);
}
result = result * 10ull + digit;
++p;
}
return result;
}
// Initializes swap according to settings.ini.
// A size of zero is the explicit opt-out. Any missing or invalid size is a broken configuration.
static void KiInitializeSwapFromSettings(void) {
uint64_t swapMb = KiParseRequiredUInt64Setting("vm", "swap_size_mb");
if (swapMb == 0) {
DbgPrint("swap: disabled by settings\n");
return;
}
if (swapMb < KI_SWAP_MIN_MB || swapMb > KI_SWAP_MAX_MB) {
KePanic(KE_PANIC_CODE_INIT,
"settings: [vm] swap_size_mb=%llu is outside the allowed range %llu..%llu MB\n",
(unsigned long long)swapMb,
(unsigned long long)KI_SWAP_MIN_MB,
(unsigned long long)KI_SWAP_MAX_MB);
}
char pagefilePath[1024];
if (!KeBuildSystemRootPath("pagefile.sys", pagefilePath, sizeof(pagefilePath))) {
KePanic(KE_PANIC_CODE_INIT,
"swap: failed to build system-root pagefile path\n");
}
uint64_t swapBytes = swapMb * 1024ull * 1024ull;
KSTATUS s = MmSwapInitialize(SysSvFromCstr(pagefilePath), swapBytes);
if (!KSUCCESS(s)) {
KePanic(KE_PANIC_CODE_INIT,
"swap: failed to initialize %llu MB pagefile at %s, status=0x%x\n",
(unsigned long long)swapMb,
pagefilePath,
s);
}
}
typedef KSTATUS (*DRIVER_ENTRY_FN)(const HAL_API *);
// HAL pointer cached at boot for subsystems that still need platform services.
@@ -68,7 +157,7 @@ static void KiReclaimBootloaderMemory(const BOOT_INFO* bootInfo) {
for (uintptr_t page = base; page < end; page += PAGE_SIZE) {
if (!KiPageInBootloaderRegion(bootInfo, (PHYSICAL_ADDRESS)page)) continue;
MmFreeFrame((PHYSICAL_ADDRESS)page);
MmPfnFree((PHYSICAL_ADDRESS)page);
}
}
}
@@ -86,6 +175,7 @@ static void KiResolveSystemImports(BOOT_INFO* bootInfo) {
}
}
// Phase Helper: Initialize dynamically loaded .sys drivers.
static void KiDriversIniCb(const char* section, const char* key,
const char* val, void* ud) {
@@ -120,7 +210,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
// Map the PE image into kernel space.
uintptr_t mappedBase = 0;
KSTATUS st = LdrLoadImage(rawPeBuffer, mod->ImageSize, &mappedBase, false);
KSTATUS st = LdrLoadKernelImage(rawPeBuffer, mod->ImageSize, &mappedBase);
if (!KSUCCESS(st)) {
DbgPrint("ERROR: Failed to map driver %s (0x%X)\n", mod->Name, st);
continue;
@@ -129,7 +219,9 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
// Update the struct so other modules can find it.
mod->VirtBase = mappedBase;
// Resolve imports (can now see Kernel, HAL, and previous drivers).
// Resolve imports while the image is still writable. Kernel driver images
// stay writable until the loader grows real section-backed mappings; the
// loader still clears NX for the whole image before returning it.
if (!KSUCCESS(LdrResolveImports(mappedBase))) {
DbgPrint("ERROR: Failed to resolve imports for %s\n", mod->Name);
continue;
@@ -152,9 +244,24 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
KSTATUS s;
FILE_OBJECT* f;
s = IoOpenFile(SYS_SV_LITERAL("drivers\\drivers.ini"), &f);
s = KeInitializeSystemRootVolume();
if (s != STATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini (status %d)\n", s);
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to find system root volume (status %d)\n", s);
}
char root[4];
root[0] = (char)('0' + KeGetSystemRootDrive());
root[1] = ':';
root[2] = '/';
root[3] = '\0';
s = IoSetCwd(SysSvFromCstr(root));
if (s != STATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to set system CWD to %s (status %d)\n", root, s);
}
s = KeOpenSystemRootFile("drivers\\drivers.ini", &f);
if (s != STATUS_SUCCESS) {
KePanic(KE_PANIC_CODE_INIT, "KiInitializeDrivers: Failed to open drivers.ini on system root (status %d)\n", s);
}
uint64_t size = IoGetFileSize(f) + 1;
@@ -174,26 +281,27 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
for (uint32_t i = 0; i < arr_len(runtimeKeys); i++) {
DbgPrint("Found runtime driver: %s\n", runtimeKeys[i]);
// Load the driver.
char path[1024];
char relative[512];
size_t len = strlen(runtimeKeys[i]);
char* path = MmAllocatePool(len + 10);
if (!path) {
DbgPrint("KernelMain: Failed to allocate memory for driver path. How?\n");
if (len + 9 > sizeof(relative)) {
DbgPrint("KiInitializeDrivers: runtime driver path too long: %s\n", runtimeKeys[i]);
continue;
}
memcpy(relative, "drivers\\", 8);
memcpy(relative + 8, runtimeKeys[i], len + 1);
// memcpy maneuvers.
memcpy(path, "drivers\\", 8);
memcpy(path + 8, runtimeKeys[i], len + 1);
if (!KeBuildSystemRootPath(relative, path, sizeof(path))) {
DbgPrint("KiInitializeDrivers: failed to build runtime driver path: %s\n", runtimeKeys[i]);
continue;
}
s = IoOpenFile(SysSvFromCstr(path), &f);
if (s != STATUS_SUCCESS) {
DbgPrint("KiInitializeDrivers: Failed to open runtime driver: %s (%s)\n", runtimeKeys[i], path);
continue; // Skip this driver but keep going with others
continue;
}
MmFreePool(path);
uint64_t drvSize = IoGetFileSize(f);
void* drvMem = MmAllocatePool(drvSize);
if (!KSUCCESS(IoReadFile(f, drvMem, drvSize, &got))) {
@@ -204,7 +312,7 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
IoCloseFile(f);
uintptr_t mappedBase = 0;
if (!KSUCCESS(LdrLoadImage(drvMem, drvSize, &mappedBase, false))) { // Load into kernel space
if (!KSUCCESS(LdrLoadKernelImage(drvMem, drvSize, &mappedBase))) { // Load into kernel space
DbgPrint("KiInitializeDrivers: Failed to load runtime driver: %s\n", runtimeKeys[i]);
MmFreePool(drvMem);
continue;
@@ -215,7 +323,6 @@ static void KiInitializeDrivers(BOOT_INFO* bootInfo, const HAL_API* hal) {
DbgPrint("KiInitializeDrivers: Failed to resolve imports for runtime driver: %s\n", runtimeKeys[i]);
continue;
}
// Runtime modules may also export DDIs to drivers loaded later. Boot
// drivers were already registered above; keep the module model uniform.
LdrRegisterModule(runtimeKeys[i], mappedBase);
@@ -250,28 +357,15 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
KSTATUS s;
FILE_OBJECT* f;
// Read the executable file into a raw buffer.
s = IoOpenFile(SysSvFromCstr(filePath), &f);
// Kernel-launched user images are rooted at the selected system volume.
s = KeOpenSystemRootFile(filePath, &f);
if (s != STATUS_SUCCESS) {
DbgPrint("Failed to load file: \"%s\"!\n", filePath);
DbgPrint("Failed to load system file: \"%s\" (status %d)!\n", filePath, s);
return 0;
}
uint64_t size = IoGetFileSize(f);
void* mem = MmAllocatePool(size);
if (!mem) {
IoCloseFile(f);
return 0;
}
size_t got = 0;
s = IoReadFile(f, mem, size, &got);
IoCloseFile(f);
if (!KSUCCESS(s)) {
MmFreePool(mem);
return 0;
}
(void)size;
// Create the process object.
PROCESS* proc;
@@ -280,7 +374,7 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
s = KeCreateProcess(filePath, &proc);
if (!KSUCCESS(s)) {
DbgPrint("Failed to create process. System halted.\n");
MmFreePool(mem);
IoCloseFile(f);
return 0;
}
DbgPrint("Created process (PID %u) with command line: %s\n", proc->ProcessId, (args && args[0]) ? args : filePath);
@@ -291,17 +385,17 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
SysSvFromCstr(proc->CurrentDirectory));
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to build user process environment. Status: %d\n", s);
MmFreePool(mem);
IoCloseFile(f);
ObDereferenceObject(&proc->Header);
return 0;
}
// Parse the raw PE file and map it into executable memory.
// Map the executable through an image section backed by the open file.
uintptr_t imageBase = 0;
s = LdrLoadImageForProcess(proc, mem, size, &imageBase);
MmFreePool(mem);
s = LdrLoadImageFileForProcess(proc, f, &imageBase);
IoCloseFile(f);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: LdrLoadImage failed with status: %d\n", s);
DbgPrint("KiLaunchUserProcess: LdrLoadImageFileForProcess failed with status: %d\n", s);
ObDereferenceObject(&proc->Header);
return 0;
}
@@ -313,6 +407,13 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
return 0;
}
s = LdrProtectImageForProcess(proc, imageBase, true);
if (!KSUCCESS(s)) {
DbgPrint("KiLaunchUserProcess: Failed to protect EXE image for %s (status %d)\n", filePath, s);
ObDereferenceObject(&proc->Header);
return 0;
}
proc->ImageBase = imageBase;
proc->ImageSize = LdrGetMappedImageSize(proc, imageBase);
@@ -327,7 +428,9 @@ static PROCESS* KiLaunchUserProcess(const char* filePath, const char* args) {
ObDereferenceObject(&proc->Header);
return 0;
}
DbgPrint("KiLaunchUserProcess: Thread created and queued!\n");
uint32_t tid = thread->ThreadId;
DbgPrint("KiLaunchUserProcess: Thread created and queued! TID=%u\n", tid);
ObDereferenceObject(&thread->Header);
// Caller owns the reference returned by KeCreateProcess.
return proc;
@@ -344,35 +447,39 @@ void SystemInitThread(void* arg) {
KiInitializeDrivers(args->bootInfo, args->hal);
KeTimeReloadSettings();
KiInitializeSwapFromSettings();
KiResolveSystemImports(args->bootInfo);
KiReclaimBootloaderMemory(args->bootInfo);
if (args->bootInfo->Debug & KE_DEBUG_SELFTEST)
KiRunKernelSelfTests(args->hal);
// Launch User-Mode init.exe and restart it until it exits cleanly.
while (true) {
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
if (!shellProc) {
DbgPrint("SystemInitThread: Failed to start system/init.exe; retrying.\n");
KeYield();
continue;
}
KeWaitForEvent(shellProc->ExitEvent, 0);
int exitCode = shellProc->ExitCode;
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
ObDereferenceObject(&shellProc->Header);
if (exitCode == 0) break;
DbgPrint("SystemInitThread: Restarting init.exe.\n");
// Launch User-Mode init.exe.
PROCESS* shellProc = KiLaunchUserProcess("system/init.exe", args->bootInfo->CommandLine);
if (!shellProc) {
DbgPrint("SystemInitThread: Failed to start system/init.exe.\n");
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: Failed to start system/init.exe.\n");
}
DbgPrint("SystemInitThread: init.exe completed successfully; shutting down.\n");
KeWaitForEvent(shellProc->ExitEvent, 0);
int exitCode = shellProc->ExitCode;
if (exitCode != 0) {
KePanic(KE_PANIC_CODE_INIT, "SystemInitThread: init.exe exited with code %d.\n", exitCode);
}
DbgPrint("SystemInitThread: init.exe exited with code %d.\n", exitCode);
ObDereferenceObject(&shellProc->Header);
if (args->bootInfo->Debug & (KE_DEBUG_CONSOLE | KE_DEBUG_FILE_LOG)) {
KSTATUS logStatus = KeFlushDebugLogToFile(SYS_SV_LITERAL("0:/kernel.log"));
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush to 0:/kernel.log failed st=0x%x\n", logStatus);
char logPath[1024];
KSTATUS logStatus = STATUS_NOT_FOUND;
if (KeBuildSystemRootPath("kernel.log", logPath, sizeof(logPath))) {
logStatus = KeFlushDebugLogToFile(SysSvFromCstr(logPath));
if (!KSUCCESS(logStatus))
DbgPrint("debuglog: shutdown flush to %s failed st=0x%x\n", logPath, logStatus);
} else {
DbgPrint("debuglog: shutdown flush skipped; system root path unavailable\n");
}
}
gKmainHal->Power.Shutdown();
MmFreePool(args);
@@ -459,6 +566,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
KeInitializeScheduler();
KSTATUS workStatus = KeStartWorkQueue();
if (!KSUCCESS(workStatus)) {
KePanic(KE_PANIC_CODE_INIT, "workq: failed to start passive work queue, status=0x%x\n", workStatus);
}
KSTATUS smpSchedStatus = KeAdmitApplicationProcessorsToScheduler();
if (!KSUCCESS(smpSchedStatus)) {
KePanic(KE_PANIC_CODE_INIT, "SMP: failed to admit APs to scheduler, status=0x%x\n", smpSchedStatus);
@@ -471,11 +583,11 @@ extern void KernelMain(BOOT_INFO* bootInfo) {
initArgs->bootInfo = bootInfo;
initArgs->hal = hal;
// Spawn the Initialization Thread.
THREAD* initThread = (THREAD*)0;
KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
(uintptr_t)initArgs,
PriorityNormal, &initThread);
// Spawn the Initialization Thread. No creator reference is needed; the
// scheduler owns the live thread lifetime.
KSTATUS st = KeCreateKernelThread((uintptr_t)SystemInitThread,
(uintptr_t)initArgs,
PriorityNormal, (THREAD**)0);
if (!KSUCCESS(st)) {
KePanic(KE_PANIC_CODE_INIT, "KernelMain: Failed to create SystemInitThread (status 0x%x)\n", st);
code/kernel/ldr_image.c
+164 -75
View File
@@ -3,11 +3,13 @@
#include <stdint.h>
#include <string.h>
#include <kernel/ldr_image.h>
#include <kernel/ldr_section.h>
#include <ntlite/boot_info.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_system_root.h>
#ifndef LDR_DEBUG_VERBOSE
#define LDR_DEBUG_VERBOSE 0
@@ -155,6 +157,8 @@ typedef struct {
#define IMAGE_REL_BASED_DIR64 10
#define IMAGE_SCN_CNT_CODE 0x00000020u
#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040u
#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u
#define IMAGE_SCN_MEM_EXECUTE 0x20000000u
#define IMAGE_SCN_MEM_READ 0x40000000u
#define IMAGE_SCN_MEM_WRITE 0x80000000u
@@ -171,6 +175,11 @@ typedef struct {
static MODULE_ENTRY gModules[MAX_MODULES];
static uint32_t gModuleCount = 0;
// Ensures a user image page is privately writable before the kernel patches it.
static KSTATUS KiEnsureUserImageWrite(PROCESS* process, bool user, uintptr_t address, size_t size) {
if (!user) return STATUS_SUCCESS;
return MmVadMakeImageRangePrivate(process, (void*)address, size);
}
// Implements the canonical Module Name operation.
static const char* KiCanonicalModuleName(const char* name) {
@@ -310,10 +319,10 @@ size_t LdrGetImageSize(uintptr_t base) {
size_t LdrGetMappedImageSize(PROCESS* process, uintptr_t base) {
if (!base) return 0;
if (process) {
size_t regionSize = MmGetUserRegionSize(process, (const void*)base);
size_t regionSize = MmVadGetSize(process, (const void*)base);
if (regionSize) return regionSize;
}
size_t globalRegionSize = MmGetUserRegionSizeByBase((const void*)base);
size_t globalRegionSize = MmVadGetSizeByBase((const void*)base);
if (globalRegionSize) return globalRegionSize;
return LdrGetImageSize(base);
}
@@ -327,13 +336,26 @@ static void KiFreeLoadedImageForProcess(PROCESS* process, uintptr_t imageBase, b
else MmFreePool((void*)imageBase);
}
// Releases loaded Image.
static void KiFreeLoadedImage(uintptr_t imageBase, bool user) {
KiFreeLoadedImageForProcess(KeGetCurrentProcess(), imageBase, user);
// Returns the final page flags for a PE section.
static uint64_t KiSectionPageProtection(const SECTION_HEADER* section, bool user) {
uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
// Some old toolchains mark BSS-like sections as uninitialized data without
// setting MEM_WRITE. Treating those pages as read-only breaks programs that
// legitimately write their zero-fill storage.
if ((section->Characteristics & IMAGE_SCN_MEM_WRITE) ||
(section->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA)) {
prot |= PAGE_WRITE;
}
if (!((section->Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
(section->Characteristics & IMAGE_SCN_CNT_CODE))) {
prot |= PAGE_NX;
}
return prot;
}
// Implements the ldr Protect Image For Process operation.
// Applies final page protections to a mapped PE image.
KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool user) {
if (!imageBase) return STATUS_INVALID_PARAMETER;
if (user && !process) return STATUS_INVALID_PARAMETER;
@@ -353,6 +375,14 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
if (sectionTableOffset > mappedSize || sectionTableSize > mappedSize - sectionTableOffset)
return STATUS_FAILURE;
size_t pageCount = (mappedSize + PAGE_SIZE - 1) / PAGE_SIZE;
uint64_t* pageProt = (uint64_t*)MmAllocatePoolTag(pageCount * sizeof(uint64_t), MM_POOL_TAG('i', 'p', 'r', 't'));
if (!pageProt) return STATUS_NO_MEMORY;
uint64_t headerProt = PAGE_PRESENT | (user ? PAGE_USER : 0) | PAGE_NX;
for (size_t i = 0; i < pageCount; ++i)
pageProt[i] = headerProt;
SECTION_HEADER* sec = (SECTION_HEADER *)(imageBase + sectionTableOffset);
for (int i = 0; i < pe->NumberOfSections; i++) {
size_t protectSize = sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData;
@@ -368,35 +398,56 @@ KSTATUS LdrProtectImageForProcess(PROCESS* process, uintptr_t imageBase, bool us
(unsigned long long)imageBase, (unsigned long long)mappedSize);
continue;
}
if (secEnd < secBase) return STATUS_NO_MEMORY;
if (secEnd < secBase) {
MmFreePool(pageProt);
return STATUS_NO_MEMORY;
}
if (secEnd > mapEnd) {
LdrDbg("LDR: protect clamping section %d end=%016llx to mapEnd=%016llx\n",
i, (unsigned long long)secEnd, (unsigned long long)mapEnd);
protectSize = (size_t)(mapEnd - secBase);
secEnd = mapEnd;
}
uint64_t prot = PAGE_PRESENT | (user ? PAGE_USER : 0);
if (sec[i].Characteristics & IMAGE_SCN_MEM_WRITE)
prot |= PAGE_WRITE;
if (!((sec[i].Characteristics & IMAGE_SCN_MEM_EXECUTE) ||
(sec[i].Characteristics & IMAGE_SCN_CNT_CODE)))
prot |= PAGE_NX;
KSTATUS s;
if (user) {
s = MmProtectUserRegion(process, (void*)secBase, protectSize, prot);
} else {
s = MmSetPageProtect(secBase, protectSize, prot);
}
if (!KSUCCESS(s)) {
LdrDbg("LDR: protect failed section=%d va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
i, (unsigned long long)secBase, (unsigned long long)protectSize,
(unsigned long long)prot, s,
(unsigned long long)imageBase, (unsigned long long)mappedSize);
return s;
size_t firstPage = (size_t)((secBase - imageBase) / PAGE_SIZE);
size_t lastPage = (size_t)((secEnd - 1 - imageBase) / PAGE_SIZE);
uint64_t prot = KiSectionPageProtection(&sec[i], user);
for (size_t page = firstPage; page <= lastPage && page < pageCount; ++page) {
// Protection is page-granular. If sections ever share a page, the
// hardware page must allow the union of their access requirements.
if (prot & PAGE_WRITE)
pageProt[page] |= PAGE_WRITE;
if ((prot & PAGE_NX) == 0)
pageProt[page] &= ~PAGE_NX;
}
}
return STATUS_SUCCESS;
size_t runStart = 0;
KSTATUS s = STATUS_SUCCESS;
while (runStart < pageCount) {
uint64_t prot = pageProt[runStart];
size_t runEnd = runStart + 1;
while (runEnd < pageCount && pageProt[runEnd] == prot)
runEnd++;
uintptr_t va = imageBase + runStart * PAGE_SIZE;
size_t bytes = (runEnd - runStart) * PAGE_SIZE;
if (user)
s = MmVadProtect(process, (void*)va, bytes, prot);
else
s = MmPteProtectRange(va, bytes, prot);
if (!KSUCCESS(s)) {
LdrDbg("LDR: protect failed va=%016llx size=%llx prot=%016llx st=%d image=%016llx mapped=%llx\n",
(unsigned long long)va, (unsigned long long)bytes,
(unsigned long long)prot, s,
(unsigned long long)imageBase, (unsigned long long)mappedSize);
break;
}
runStart = runEnd;
}
MmFreePool(pageProt);
return s;
}
@@ -408,7 +459,7 @@ uintptr_t LdrGetEntryPoint(uintptr_t base) {
}
// Base Relocation.
static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
static KSTATUS LdrApplyRelocationsForProcess(PROCESS* process, uintptr_t base, uintptr_t delta, bool user) {
uint32_t peOffset = *(uint32_t *)(base + 0x3C);
PE_HEADER* pe = (PE_HEADER *)(base + peOffset);
OPT_HEADER64* opt = &pe->OptionalHeader;
@@ -432,6 +483,8 @@ static KSTATUS LdrApplyRelocations(uintptr_t base, uintptr_t delta) {
// For 64-bit PEs, we mostly care about DIR64 (Type 10).
if (type == IMAGE_REL_BASED_DIR64) {
uint64_t* ptr = (uint64_t *)(base + block->VirtualAddress + offset);
KSTATUS ws = KiEnsureUserImageWrite(process, user, (uintptr_t)ptr, sizeof(*ptr));
if (!KSUCCESS(ws)) return ws;
*ptr += delta;
}
}
@@ -534,6 +587,8 @@ static KSTATUS LdrResolveImportsCommon(
return STATUS_NOT_FOUND;
}
KSTATUS ws = KiEnsureUserImageWrite(process, process != (PROCESS*)0, (uintptr_t)thunk, sizeof(*thunk));
if (!KSUCCESS(ws)) return ws;
*thunk = (uint64_t)addr;
thunk++;
@@ -566,6 +621,41 @@ KSTATUS LdrResolveImportsForProcess(PROCESS* process, uintptr_t imageBase) {
);
}
// Loads a user image from an open file through the image-section object model.
KSTATUS LdrLoadImageFileForProcess(PROCESS* process, FILE_OBJECT* file, uintptr_t* baseOut) {
if (!process || !file || !baseOut)
return STATUS_INVALID_PARAMETER;
*baseOut = 0;
IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)0;
KSTATUS s = LdrCreateImageSectionFromFile(file, &section);
if (!KSUCCESS(s))
return s;
void* view = (void*)0;
s = LdrMapImageSection(process, section, &view);
if (!KSUCCESS(s)) {
LdrDereferenceImageSection(section);
return s;
}
const IMAGE_SECTION_METADATA* metadata = LdrImageSectionMetadata(section);
uintptr_t imageBase = (uintptr_t)view;
uintptr_t delta = imageBase - metadata->PreferredBase;
if (delta != 0) {
s = LdrApplyRelocationsForProcess(process, imageBase, delta, true);
if (!KSUCCESS(s)) {
KiFreeLoadedImageForProcess(process, imageBase, true);
LdrDereferenceImageSection(section);
return s;
}
}
LdrDereferenceImageSection(section);
*baseOut = imageBase;
return STATUS_SUCCESS;
}
// Implements the ldr Register Process Module operation.
static KSTATUS LdrRegisterProcessModule(PROCESS* process, const char* name, uintptr_t base) {
if (!process || !name || !base) return STATUS_INVALID_PARAMETER;
@@ -602,36 +692,27 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
FILE_OBJECT* f;
KSTATUS s = IoOpenFile(SysSvFromCstr(filename), &f);
if (!KSUCCESS(s)) {
char path[260];
memcpy(path, "0:/system/", 10);
memcpy(path + 10, filename, strnlen(filename, 248) + 1);
path[259] = '\0';
s = IoOpenFile(SysSvFromCstr(path), &f);
char relative[260];
size_t nameLen = strnlen(filename, sizeof(relative) - 8);
if (nameLen + 8 >= sizeof(relative))
return STATUS_BUFFER_TOO_SMALL;
memcpy(relative, "system/", 7);
memcpy(relative + 7, filename, nameLen);
relative[7 + nameLen] = '\0';
s = KeOpenSystemRootFile(relative, &f);
if (!KSUCCESS(s)) {
DbgPrint("LdrLoadModuleForProcess: Failed to open %s (%d)\n", filename, s);
DbgPrint("LdrLoadModuleForProcess: Failed to open %s on system root (%d)\n", filename, s);
return s;
}
}
uint64_t fileSize = IoGetFileSize(f);
LdrDbg("LDR: load module pid=%u file=%s size=%llu\n", process->ProcessId, filename, fileSize);
void* mem = MmAllocatePool((size_t)fileSize);
if (!mem) {
IoCloseFile(f);
return STATUS_NO_MEMORY;
}
size_t got = 0;
s = IoReadFile(f, mem, (size_t)fileSize, &got);
IoCloseFile(f);
if (!KSUCCESS(s)) {
MmFreePool(mem);
return s;
}
uintptr_t imageBase = 0;
s = LdrLoadImageForProcess(process, mem, (size_t)fileSize, &imageBase);
MmFreePool(mem);
s = LdrLoadImageFileForProcess(process, f, &imageBase);
IoCloseFile(f);
if (!KSUCCESS(s)) {
DbgPrint("LDR: load image for module %s failed st=%d\n", filename, s);
return s;
@@ -665,10 +746,9 @@ KSTATUS LdrLoadModuleForProcess(PROCESS* process, const char* filename, uintptr_
return STATUS_SUCCESS;
}
// LdrLoadImage - load a PE from a raw buffer into virtual space.
static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
// Loads a boot/kernel PE image from a raw memory buffer into kernel space.
static KSTATUS KiLdrLoadKernelImageFromBuffer(const void* buffer, size_t size, uintptr_t* baseOut) {
if (!buffer || !size || !baseOut) return STATUS_INVALID_PARAMETER;
if (user && !process) return STATUS_INVALID_PARAMETER;
if (size < 0x40) return STATUS_FAILURE;
const uint8_t* bytes = (const uint8_t*)buffer;
@@ -687,13 +767,13 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
uint16_t optionalMagic = *(const uint16_t *)(bytes + peOffset + 24);
if (machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
DbgPrint("LdrLoadImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
DbgPrint("LdrLoadKernelImage: hard error: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
if (machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
DbgPrint("LdrLoadImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
DbgPrint("LdrLoadKernelImage: unsupported PE image (machine=0x%X magic=0x%X)\n",
machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
@@ -704,17 +784,16 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
KSTATUS sizeStatus = KiComputeRequiredImageSize(pe, size, &imageSize);
if (!KSUCCESS(sizeStatus)) return sizeStatus;
LdrDbg("LDR: image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u user=%d pid=%u\n",
LdrDbg("LDR: kernel image header sizeOfImage=%x required=%llx preferred=%016llx sections=%u\n",
opt->SizeOfImage,
(unsigned long long)imageSize,
(unsigned long long)opt->ImageBase,
pe->NumberOfSections,
user ? 1 : 0,
process ? process->ProcessId : 0);
void* mem = user ? MmAllocateUserImageForProcess(process, imageSize) : MmAllocatePool(imageSize);
pe->NumberOfSections);
void* mem = MmAllocatePoolTag(imageSize, MM_POOL_TAG('i', 'm', 'g', ' '));
if (!mem) {
DbgPrint("LDR: image allocation failed required=%llx user=%d pid=%u\n",
(unsigned long long)imageSize, user ? 1 : 0, process ? process->ProcessId : 0);
DbgPrint("LDR: kernel image allocation failed required=%llx\n",
(unsigned long long)imageSize);
return STATUS_NO_MEMORY;
}
memset(mem, 0, imageSize);
@@ -730,27 +809,37 @@ static KSTATUS KiLdrLoadImageCommon(PROCESS* process, const void* buffer, size_t
memcpy(dst, src, copySize);
}
uint64_t prot = PAGE_PRESENT | PAGE_WRITE | (user ? PAGE_USER : 0);
MmSetPageProtect((uintptr_t)mem + sec[i].VirtualAddress,
MmPteProtectRange((uintptr_t)mem + sec[i].VirtualAddress,
sec[i].VirtualSize ? sec[i].VirtualSize : sec[i].SizeOfRawData,
prot);
PAGE_PRESENT | PAGE_WRITE);
}
uintptr_t delta = (uintptr_t)mem - opt->ImageBase;
if (delta != 0) {
LdrApplyRelocations((uintptr_t)mem, delta);
KSTATUS rs = LdrApplyRelocationsForProcess((PROCESS*)0, (uintptr_t)mem, delta, false);
if (!KSUCCESS(rs)) {
MmFreePool(mem);
return rs;
}
}
// Kernel drivers are still pool-backed images. Section protections cannot
// safely make pages read-only yet because imports and driver-owned writable
// image data are not separated from callback code by a section object. What
// must be true is simpler: every page in the loaded image must be executable
// enough for driver callbacks, including section-alignment slack near the
// end of the image.
KSTATUS xs = MmPteProtectRange((uintptr_t)mem, imageSize, PAGE_PRESENT | PAGE_WRITE);
if (!KSUCCESS(xs)) {
MmFreePool(mem);
return xs;
}
*baseOut = (uintptr_t)mem;
return STATUS_SUCCESS;
}
// Implements the ldr Load Image For Process operation.
KSTATUS LdrLoadImageForProcess(PROCESS* process, const void* buffer, size_t size, uintptr_t* baseOut) {
return KiLdrLoadImageCommon(process, buffer, size, baseOut, true);
}
// Implements the ldr Load Image operation.
KSTATUS LdrLoadImage(const void* buffer, size_t size, uintptr_t* baseOut, bool user) {
return KiLdrLoadImageCommon(user ? KeGetCurrentProcess() : (PROCESS*)0, buffer, size, baseOut, user);
// Loads a kernel image from a bootloader or driver raw buffer.
KSTATUS LdrLoadKernelImage(const void* buffer, size_t size, uintptr_t* baseOut) {
return KiLdrLoadKernelImageFromBuffer(buffer, size, baseOut);
}
code/kernel/ldr_section.c
+736
View File
@@ -0,0 +1,736 @@
// Image section objects. This file owns parsed PE image identity, sparse prototype pages, and image-section caching.
#include <stdint.h>
#include <stddef.h>
#include <stdbool.h>
#include <string.h>
#include <kernel/ldr_section.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_pte.h>
#include <kernel/ob_object.h>
#include <kernel/io_file.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_debug.h>
#define PE_SIGNATURE 0x00004550UL
#define IMAGE_FILE_MACHINE_AMD64 0x8664
#define IMAGE_FILE_MACHINE_I386 0x014c
#define IMAGE_NT_OPTIONAL_HDR32_MAGIC 0x10b
#define IMAGE_NT_OPTIONAL_HDR64_MAGIC 0x20b
#define IMAGE_SCN_CNT_CODE 0x00000020u
#define IMAGE_SCN_CNT_INITIALIZED_DATA 0x00000040u
#define IMAGE_SCN_CNT_UNINITIALIZED_DATA 0x00000080u
#define IMAGE_SCN_MEM_EXECUTE 0x20000000u
#define IMAGE_SCN_MEM_READ 0x40000000u
#define IMAGE_SCN_MEM_WRITE 0x80000000u
#define LDR_IMAGE_SECTION_CACHE_MAX 32u
#pragma pack(push, 1)
typedef struct LDR_OPT_HEADER64 {
uint16_t Magic;
uint8_t MajorLinkerVersion;
uint8_t MinorLinkerVersion;
uint32_t SizeOfCode;
uint32_t SizeOfInitializedData;
uint32_t SizeOfUninitializedData;
uint32_t AddressOfEntryPoint;
uint32_t BaseOfCode;
uint64_t ImageBase;
uint32_t SectionAlignment;
uint32_t FileAlignment;
uint16_t MajorOSVersion;
uint16_t MinorOSVersion;
uint16_t MajorImageVersion;
uint16_t MinorImageVersion;
uint16_t MajorSubsystemVersion;
uint16_t MinorSubsystemVersion;
uint32_t Win32VersionValue;
uint32_t SizeOfImage;
uint32_t SizeOfHeaders;
uint32_t CheckSum;
uint16_t Subsystem;
uint16_t DllCharacteristics;
uint64_t SizeOfStackReserve;
uint64_t SizeOfStackCommit;
uint64_t SizeOfHeapReserve;
uint64_t SizeOfHeapCommit;
uint32_t LoaderFlags;
uint32_t NumberOfRvaAndSizes;
struct { uint32_t VirtualAddress; uint32_t Size; } DataDirectory[16];
} LDR_OPT_HEADER64;
typedef struct LDR_PE_HEADER {
uint32_t Signature;
uint16_t Machine;
uint16_t NumberOfSections;
uint32_t TimeDateStamp;
uint32_t PointerToSymbolTable;
uint32_t NumberOfSymbols;
uint16_t SizeOfOptionalHeader;
uint16_t Characteristics;
LDR_OPT_HEADER64 OptionalHeader;
} LDR_PE_HEADER;
typedef struct LDR_SECTION_HEADER {
char Name[8];
uint32_t VirtualSize;
uint32_t VirtualAddress;
uint32_t SizeOfRawData;
uint32_t PointerToRawData;
uint32_t PointerToRelocations;
uint32_t PointerToLinenumbers;
uint16_t NumberOfRelocations;
uint16_t NumberOfLinenumbers;
uint32_t Characteristics;
} LDR_SECTION_HEADER;
#pragma pack(pop)
struct IMAGE_SECTION_OBJECT {
OBJECT Header;
IMAGE_SECTION_METADATA Metadata;
IMAGE_SECTION_DESCRIPTOR* Sections;
PHYSICAL_ADDRESS* Frames;
FILE_OBJECT* BackingFile;
size_t HeadersFileSize;
SPINLOCK Lock;
};
static IMAGE_SECTION_OBJECT* gImageSectionCache[LDR_IMAGE_SECTION_CACHE_MAX];
static SPINLOCK gImageSectionCacheLock = SPINLOCK_INIT;
// Aligns a PE image size to the hardware page size.
static size_t KiImageSectionAlignSize(size_t value) {
return (value + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1);
}
// Mixes filesystem identity into a compact cache key.
static uint64_t KiImageSectionIdentityKey(const IO_FILE_IDENTITY* identity) {
if (!identity)
return 0;
uint64_t h = 1469598103934665603ULL;
h ^= identity->VolumeId;
h *= 1099511628211ULL;
h ^= identity->FileId;
h *= 1099511628211ULL;
h ^= identity->Generation;
h *= 1099511628211ULL;
h ^= identity->Size;
h *= 1099511628211ULL;
return h ? h : 1;
}
// Converts PE section characteristics to the final user page protection union for that section.
static uint64_t KiImageSectionProtection(uint32_t characteristics) {
uint64_t prot = PAGE_PRESENT | PAGE_USER | PAGE_NX;
if (characteristics & IMAGE_SCN_MEM_WRITE)
prot |= PAGE_WRITE;
if (characteristics & IMAGE_SCN_MEM_EXECUTE)
prot &= ~PAGE_NX;
return prot;
}
// Computes the required image mapping size after validating raw section extents.
static KSTATUS KiImageSectionComputeSize(const LDR_PE_HEADER* pe, size_t fileSize, size_t* outSize) {
if (!pe || !outSize) return STATUS_INVALID_PARAMETER;
const LDR_OPT_HEADER64* opt = &pe->OptionalHeader;
size_t required = KiImageSectionAlignSize(opt->SizeOfImage);
size_t headers = KiImageSectionAlignSize(opt->SizeOfHeaders);
if (headers > required) required = headers;
const LDR_SECTION_HEADER* sec = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
for (uint32_t i = 0; i < pe->NumberOfSections; i++) {
size_t va = (size_t)sec[i].VirtualAddress;
size_t virt = (size_t)sec[i].VirtualSize;
size_t raw = (size_t)sec[i].SizeOfRawData;
size_t span = virt ? virt : raw;
if (raw > span) span = raw;
if (!span) continue;
if (va > (size_t)-1 - span) return STATUS_NO_MEMORY;
size_t end = KiImageSectionAlignSize(va + span);
if (end > required) required = end;
if (raw) {
size_t rawBase = (size_t)sec[i].PointerToRawData;
if (rawBase > fileSize || raw > fileSize - rawBase)
return STATUS_FAILURE;
}
}
if (!required) return STATUS_FAILURE;
*outSize = required;
return STATUS_SUCCESS;
}
// Parses and validates the PE header that backs an image section object.
static KSTATUS KiImageSectionParseHeader(const void* buffer,
size_t bufferSize,
size_t fileSize,
const LDR_PE_HEADER** peOut,
const LDR_SECTION_HEADER** sectionsOut,
size_t* imageSizeOut) {
if (!buffer || !bufferSize || !peOut || !sectionsOut || !imageSizeOut)
return STATUS_INVALID_PARAMETER;
if (bufferSize < 0x40) return STATUS_FAILURE;
const uint8_t* bytes = (const uint8_t*)buffer;
uint32_t peOffset = *(const uint32_t*)(bytes + 0x3C);
if ((size_t)peOffset > bufferSize || bufferSize - (size_t)peOffset < 24)
return STATUS_FAILURE;
const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)(bytes + peOffset);
if (pe->Signature != PE_SIGNATURE)
return STATUS_FAILURE;
uint16_t optionalSize = pe->SizeOfOptionalHeader;
if (optionalSize < sizeof(LDR_OPT_HEADER64) ||
(size_t)peOffset + 24 + optionalSize > bufferSize) {
return STATUS_FAILURE;
}
uint16_t optionalMagic = pe->OptionalHeader.Magic;
if (pe->Machine == IMAGE_FILE_MACHINE_I386 || optionalMagic == IMAGE_NT_OPTIONAL_HDR32_MAGIC) {
DbgPrint("LdrImageSection: 32-bit PE images are not supported (machine=0x%X magic=0x%X)\n",
pe->Machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
if (pe->Machine != IMAGE_FILE_MACHINE_AMD64 || optionalMagic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) {
DbgPrint("LdrImageSection: unsupported PE image (machine=0x%X magic=0x%X)\n",
pe->Machine, optionalMagic);
return STATUS_NOT_SUPPORTED;
}
if (!pe->NumberOfSections)
return STATUS_FAILURE;
const LDR_SECTION_HEADER* sections = (const LDR_SECTION_HEADER*)((const uint8_t*)&pe->OptionalHeader + pe->SizeOfOptionalHeader);
size_t sectionTableOffset = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader;
size_t sectionTableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER);
if (sectionTableOffset > bufferSize || sectionTableBytes > bufferSize - sectionTableOffset)
return STATUS_FAILURE;
size_t imageSize = 0;
KSTATUS s = KiImageSectionComputeSize(pe, fileSize, &imageSize);
if (!KSUCCESS(s)) return s;
*peOut = pe;
*sectionsOut = sections;
*imageSizeOut = imageSize;
return STATUS_SUCCESS;
}
// Releases all resources owned by an image section object.
static void KiDestroyImageSectionObject(OBJECT* object) {
IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)object;
if (!section) return;
if (section->Frames) {
for (size_t i = 0; i < section->Metadata.PageCount; ++i) {
if (section->Frames[i])
(void)MmPfnFree(section->Frames[i]);
}
MmFreePool(section->Frames);
section->Frames = (PHYSICAL_ADDRESS*)0;
}
if (section->Sections) {
MmFreePool(section->Sections);
section->Sections = (IMAGE_SECTION_DESCRIPTOR*)0;
}
if (section->BackingFile) {
IoCloseFile(section->BackingFile);
section->BackingFile = (FILE_OBJECT*)0;
}
}
// Finds a cached image section while the image-section cache lock is held.
static IMAGE_SECTION_OBJECT* KiFindImageSectionLocked(const IO_FILE_IDENTITY* identity, uint64_t key) {
if (!identity || !key)
return (IMAGE_SECTION_OBJECT*)0;
for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) {
IMAGE_SECTION_OBJECT* section = gImageSectionCache[i];
if (section &&
section->Metadata.ContentKey == key &&
section->Metadata.VolumeId == identity->VolumeId &&
section->Metadata.FileId == identity->FileId &&
section->Metadata.FileGeneration == identity->Generation &&
section->Metadata.RawSize == identity->Size) {
return section;
}
}
return (IMAGE_SECTION_OBJECT*)0;
}
// Verifies that a cached image section is still backed by the same file generation.
static KSTATUS KiValidateImageSectionBackingFile(IMAGE_SECTION_OBJECT* section) {
if (!section || !section->BackingFile)
return STATUS_INVALID_PARAMETER;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(section->BackingFile, &identity);
if (!KSUCCESS(s))
return s;
if (identity.VolumeId != section->Metadata.VolumeId ||
identity.FileId != section->Metadata.FileId ||
identity.Generation != section->Metadata.FileGeneration ||
identity.Size != section->Metadata.RawSize) {
return STATUS_CHECKSUM_MISMATCH;
}
return STATUS_SUCCESS;
}
// Returns true when a virtual image page has initialized bytes in the backing file.
static bool KiImageSectionPageHasFileBytes(const IMAGE_SECTION_OBJECT* section, size_t pageIndex) {
if (!section || pageIndex >= section->Metadata.PageCount)
return false;
size_t pageStart = pageIndex * PAGE_SIZE;
size_t pageEnd = pageStart + PAGE_SIZE;
if (pageStart < section->HeadersFileSize && pageEnd > 0)
return true;
for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) {
const IMAGE_SECTION_DESCRIPTOR* desc = &section->Sections[i];
size_t raw = desc->RawDataSize;
if (desc->VirtualSize && desc->VirtualSize < raw)
raw = desc->VirtualSize;
if (!raw)
continue;
size_t start = desc->VirtualAddress;
size_t end = start + raw;
if (end < start)
continue;
if (pageStart < end && pageEnd > start)
return true;
}
return false;
}
// Copies a file range into an image prototype page, leaving absent bytes zero-filled.
static KSTATUS KiReadImageSectionFileBytes(IMAGE_SECTION_OBJECT* section,
uint64_t fileOffset,
void* dst,
size_t size) {
if (!section || !section->BackingFile || (!dst && size))
return STATUS_INVALID_PARAMETER;
if (!size)
return STATUS_SUCCESS;
size_t got = 0;
KSTATUS s = IoReadFileAt(section->BackingFile, fileOffset, dst, size, &got);
if (!KSUCCESS(s))
return s;
if (got != size)
return STATUS_FAILURE;
return STATUS_SUCCESS;
}
// Fills one prototype frame from the image backing file.
static KSTATUS KiFillImageSectionFrameFromFile(IMAGE_SECTION_OBJECT* section,
size_t pageIndex,
PHYSICAL_ADDRESS frame) {
if (!section || !frame || pageIndex >= section->Metadata.PageCount)
return STATUS_INVALID_PARAMETER;
uint8_t* page = (uint8_t*)MmPfnToVirtual(frame);
if (!page)
return STATUS_FAILURE;
memset(page, 0, PAGE_SIZE);
size_t pageStart = pageIndex * PAGE_SIZE;
size_t pageEnd = pageStart + PAGE_SIZE;
if (pageStart < section->HeadersFileSize) {
size_t copyStart = pageStart;
size_t copyEnd = pageEnd;
if (copyEnd > section->HeadersFileSize)
copyEnd = section->HeadersFileSize;
KSTATUS s = KiReadImageSectionFileBytes(section,
(uint64_t)copyStart,
page,
copyEnd - copyStart);
if (!KSUCCESS(s))
return s;
}
for (uint32_t i = 0; i < section->Metadata.SectionCount; ++i) {
const IMAGE_SECTION_DESCRIPTOR* desc = &section->Sections[i];
size_t raw = desc->RawDataSize;
if (desc->VirtualSize && desc->VirtualSize < raw)
raw = desc->VirtualSize;
if (!raw)
continue;
size_t sectionStart = desc->VirtualAddress;
size_t sectionEnd = sectionStart + raw;
if (sectionEnd < sectionStart)
return STATUS_FAILURE;
if (pageStart >= sectionEnd || pageEnd <= sectionStart)
continue;
size_t copyStart = pageStart > sectionStart ? pageStart : sectionStart;
size_t copyEnd = pageEnd < sectionEnd ? pageEnd : sectionEnd;
size_t dstOffset = copyStart - pageStart;
uint64_t srcOffset = (uint64_t)desc->RawDataOffset + (uint64_t)(copyStart - sectionStart);
KSTATUS s = KiReadImageSectionFileBytes(section,
srcOffset,
page + dstOffset,
copyEnd - copyStart);
if (!KSUCCESS(s))
return s;
}
return STATUS_SUCCESS;
}
// Ensures a prototype page exists for initialized image bytes.
static KSTATUS KiEnsureImageSectionFrame(IMAGE_SECTION_OBJECT* section, size_t pageIndex) {
if (!section || pageIndex >= section->Metadata.PageCount)
return STATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&section->Lock);
PHYSICAL_ADDRESS existing = section->Frames[pageIndex];
KeReleaseSpinlock(&section->Lock, flags);
if (existing)
return STATUS_SUCCESS;
if (section->BackingFile && !KiImageSectionPageHasFileBytes(section, pageIndex))
return STATUS_SUCCESS;
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseImage);
if (!frame)
return STATUS_NO_MEMORY;
void* va = MmPfnToVirtual(frame);
if (!va) {
MmPfnFree(frame);
return STATUS_FAILURE;
}
KSTATUS s = STATUS_SUCCESS;
if (section->BackingFile) {
s = KiValidateImageSectionBackingFile(section);
if (KSUCCESS(s))
s = KiFillImageSectionFrameFromFile(section, pageIndex, frame);
} else {
memset(va, 0, PAGE_SIZE);
}
if (!KSUCCESS(s)) {
MmPfnFree(frame);
return s;
}
flags = KeAcquireSpinlock(&section->Lock);
if (section->Frames[pageIndex]) {
KeReleaseSpinlock(&section->Lock, flags);
MmPfnFree(frame);
return STATUS_SUCCESS;
}
section->Frames[pageIndex] = frame;
KeReleaseSpinlock(&section->Lock, flags);
return STATUS_SUCCESS;
}
// Builds an uncached image section object backed directly by an open file.
static KSTATUS KiBuildImageSectionObjectFromFile(FILE_OBJECT* file,
const void* headerBuffer,
size_t headerBufferSize,
const IO_FILE_IDENTITY* identity,
uint64_t key,
IMAGE_SECTION_OBJECT** sectionOut) {
if (!file || !headerBuffer || !headerBufferSize || !identity || !sectionOut)
return STATUS_INVALID_PARAMETER;
*sectionOut = (IMAGE_SECTION_OBJECT*)0;
size_t fileSize = (size_t)identity->Size;
const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)0;
const LDR_SECTION_HEADER* rawSections = (const LDR_SECTION_HEADER*)0;
size_t imageSize = 0;
KSTATUS s = KiImageSectionParseHeader(headerBuffer,
headerBufferSize,
fileSize,
&pe,
&rawSections,
&imageSize);
if (!KSUCCESS(s))
return s;
size_t pageCount = (imageSize + PAGE_SIZE - 1) / PAGE_SIZE;
if (!pageCount || pageCount > ((size_t)-1 / sizeof(PHYSICAL_ADDRESS)))
return STATUS_FAILURE;
IMAGE_SECTION_OBJECT* section = (IMAGE_SECTION_OBJECT*)ObAllocateObject(ObjectTypeImageSection,
sizeof(IMAGE_SECTION_OBJECT));
if (!section)
return STATUS_NO_MEMORY;
section->Header.Destroy = KiDestroyImageSectionObject;
PHYSICAL_ADDRESS* frames = (PHYSICAL_ADDRESS*)MmAllocatePoolTag(pageCount * sizeof(PHYSICAL_ADDRESS),
MM_POOL_TAG('i', 's', 'f', 'r'));
if (!frames) {
ObDereferenceObject(&section->Header);
return STATUS_NO_MEMORY;
}
memset(frames, 0, pageCount * sizeof(PHYSICAL_ADDRESS));
IMAGE_SECTION_DESCRIPTOR* descriptors = (IMAGE_SECTION_DESCRIPTOR*)MmAllocatePoolTag(
(size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR),
MM_POOL_TAG('i', 's', 'e', 'c'));
if (!descriptors) {
MmFreePool(frames);
ObDereferenceObject(&section->Header);
return STATUS_NO_MEMORY;
}
memset(descriptors, 0, (size_t)pe->NumberOfSections * sizeof(IMAGE_SECTION_DESCRIPTOR));
section->Frames = frames;
section->Sections = descriptors;
section->BackingFile = file;
section->HeadersFileSize = pe->OptionalHeader.SizeOfHeaders;
section->Lock = (SPINLOCK)SPINLOCK_INIT;
ObReferenceObject(&file->Header);
section->Metadata.ContentKey = key;
section->Metadata.VolumeId = identity->VolumeId;
section->Metadata.FileId = identity->FileId;
section->Metadata.FileGeneration = identity->Generation;
section->Metadata.PreferredBase = pe->OptionalHeader.ImageBase;
section->Metadata.EntryRva = pe->OptionalHeader.AddressOfEntryPoint;
section->Metadata.HeaderSize = pe->OptionalHeader.SizeOfHeaders;
section->Metadata.RawSize = fileSize;
section->Metadata.ImageSize = imageSize;
section->Metadata.PageCount = pageCount;
section->Metadata.SectionCount = pe->NumberOfSections;
for (uint32_t i = 0; i < pe->NumberOfSections; ++i) {
IMAGE_SECTION_DESCRIPTOR* dst = &section->Sections[i];
memcpy(dst->Name, rawSections[i].Name, sizeof(rawSections[i].Name));
dst->Name[8] = '\0';
dst->VirtualAddress = rawSections[i].VirtualAddress;
dst->VirtualSize = rawSections[i].VirtualSize;
dst->RawDataOffset = rawSections[i].PointerToRawData;
dst->RawDataSize = rawSections[i].SizeOfRawData;
dst->Characteristics = rawSections[i].Characteristics;
dst->PageProtection = KiImageSectionProtection(rawSections[i].Characteristics);
}
*sectionOut = section;
return STATUS_SUCCESS;
}
// Creates or references an image section object backed by an open file.
KSTATUS LdrCreateImageSectionFromFile(FILE_OBJECT* file,
IMAGE_SECTION_OBJECT** sectionOut) {
if (!file || !sectionOut)
return STATUS_INVALID_PARAMETER;
*sectionOut = (IMAGE_SECTION_OBJECT*)0;
IO_FILE_IDENTITY identity;
KSTATUS s = IoQueryFileIdentity(file, &identity);
if (!KSUCCESS(s))
return s;
size_t fileSize = (size_t)identity.Size;
if (!fileSize)
return STATUS_FAILURE;
size_t firstRead = fileSize < 4096u ? fileSize : 4096u;
uint8_t stackHeader[4096];
size_t got = 0;
s = IoReadFileAt(file, 0, stackHeader, firstRead, &got);
if (!KSUCCESS(s) || got != firstRead)
return KSUCCESS(s) ? STATUS_FAILURE : s;
if (firstRead < 0x40)
return STATUS_FAILURE;
uint32_t peOffset = *(const uint32_t*)(stackHeader + 0x3C);
if ((size_t)peOffset > fileSize || fileSize - (size_t)peOffset < 24)
return STATUS_FAILURE;
size_t minimumHeader = (size_t)peOffset + 24u;
if (minimumHeader > firstRead) {
if (minimumHeader > 65536u || minimumHeader > fileSize)
return STATUS_FAILURE;
void* tmp = MmAllocatePoolTag(minimumHeader, MM_POOL_TAG('i', 's', 'h', 'd'));
if (!tmp)
return STATUS_NO_MEMORY;
s = IoReadFileAt(file, 0, tmp, minimumHeader, &got);
if (!KSUCCESS(s) || got != minimumHeader) {
MmFreePool(tmp);
return KSUCCESS(s) ? STATUS_FAILURE : s;
}
uint16_t optSize = *(const uint16_t*)((const uint8_t*)tmp + peOffset + 20u);
size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize;
MmFreePool(tmp);
if (fullHeader > firstRead) {
firstRead = fullHeader;
}
} else {
uint16_t optSize = *(const uint16_t*)(stackHeader + peOffset + 20u);
size_t fullHeader = (size_t)peOffset + 24u + (size_t)optSize;
if (fullHeader > firstRead)
firstRead = fullHeader;
}
if (firstRead > fileSize || firstRead > 65536u)
return STATUS_FAILURE;
void* header = stackHeader;
bool heapHeader = false;
if (firstRead > sizeof(stackHeader)) {
header = MmAllocatePoolTag(firstRead, MM_POOL_TAG('i', 's', 'h', 'd'));
if (!header)
return STATUS_NO_MEMORY;
heapHeader = true;
s = IoReadFileAt(file, 0, header, firstRead, &got);
if (!KSUCCESS(s) || got != firstRead) {
MmFreePool(header);
return KSUCCESS(s) ? STATUS_FAILURE : s;
}
}
const LDR_PE_HEADER* pe = (const LDR_PE_HEADER*)((const uint8_t*)header + peOffset);
size_t tableBytes = (size_t)pe->NumberOfSections * sizeof(LDR_SECTION_HEADER);
size_t tableEnd = (size_t)peOffset + 24u + (size_t)pe->SizeOfOptionalHeader + tableBytes;
if (tableEnd > firstRead) {
if (tableEnd > fileSize || tableEnd > 65536u) {
if (heapHeader) MmFreePool(header);
return STATUS_FAILURE;
}
void* bigger = MmAllocatePoolTag(tableEnd, MM_POOL_TAG('i', 's', 'h', 'd'));
if (!bigger) {
if (heapHeader) MmFreePool(header);
return STATUS_NO_MEMORY;
}
s = IoReadFileAt(file, 0, bigger, tableEnd, &got);
if (heapHeader) MmFreePool(header);
if (!KSUCCESS(s) || got != tableEnd) {
MmFreePool(bigger);
return KSUCCESS(s) ? STATUS_FAILURE : s;
}
header = bigger;
firstRead = tableEnd;
heapHeader = true;
}
uint64_t key = KiImageSectionIdentityKey(&identity);
uintptr_t flags = KeAcquireSpinlock(&gImageSectionCacheLock);
IMAGE_SECTION_OBJECT* cached = KiFindImageSectionLocked(&identity, key);
if (cached) {
ObReferenceObject(&cached->Header);
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
if (heapHeader) MmFreePool(header);
*sectionOut = cached;
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
IMAGE_SECTION_OBJECT* built = (IMAGE_SECTION_OBJECT*)0;
s = KiBuildImageSectionObjectFromFile(file, header, firstRead, &identity, key, &built);
if (heapHeader) MmFreePool(header);
if (!KSUCCESS(s))
return s;
flags = KeAcquireSpinlock(&gImageSectionCacheLock);
cached = KiFindImageSectionLocked(&identity, key);
if (cached) {
ObReferenceObject(&cached->Header);
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
ObDereferenceObject(&built->Header);
*sectionOut = cached;
return STATUS_SUCCESS;
}
for (uint32_t i = 0; i < LDR_IMAGE_SECTION_CACHE_MAX; ++i) {
if (!gImageSectionCache[i]) {
ObReferenceObject(&built->Header);
gImageSectionCache[i] = built;
break;
}
}
KeReleaseSpinlock(&gImageSectionCacheLock, flags);
*sectionOut = built;
return STATUS_SUCCESS;
}
// Adds a reference to an existing image section object.
void LdrReferenceImageSection(IMAGE_SECTION_OBJECT* section) {
if (!section) return;
ObReferenceObject(&section->Header);
}
// Releases a reference returned by LdrCreateImageSectionFromFile.
void LdrDereferenceImageSection(IMAGE_SECTION_OBJECT* section) {
if (!section) return;
ObDereferenceObject(&section->Header);
}
// Returns immutable metadata for an image section object.
const IMAGE_SECTION_METADATA* LdrImageSectionMetadata(const IMAGE_SECTION_OBJECT* section) {
if (!section) return (const IMAGE_SECTION_METADATA*)0;
return &section->Metadata;
}
// Returns the immutable PE section table owned by the image section object.
const IMAGE_SECTION_DESCRIPTOR* LdrImageSectionDescriptors(const IMAGE_SECTION_OBJECT* section,
uint32_t* countOut) {
if (countOut) *countOut = section ? section->Metadata.SectionCount : 0;
if (!section) return (const IMAGE_SECTION_DESCRIPTOR*)0;
return section->Sections;
}
// Retains one prototype page frame owned by an image section object.
KSTATUS LdrRetainImageSectionPage(IMAGE_SECTION_OBJECT* section,
size_t pageIndex,
PHYSICAL_ADDRESS* frameOut) {
if (!section || !frameOut) return STATUS_INVALID_PARAMETER;
*frameOut = 0;
if (pageIndex >= section->Metadata.PageCount)
return STATUS_INVALID_PARAMETER;
KSTATUS es = KiEnsureImageSectionFrame(section, pageIndex);
if (!KSUCCESS(es))
return es;
uintptr_t flags = KeAcquireSpinlock(&section->Lock);
PHYSICAL_ADDRESS frame = section->Frames[pageIndex];
KeReleaseSpinlock(&section->Lock, flags);
if (!frame)
return STATUS_SUCCESS;
KSTATUS s = MmPfnRetain(frame);
if (!KSUCCESS(s))
return s;
*frameOut = frame;
return STATUS_SUCCESS;
}
// Maps an image section into a process as an image-backed VAD.
KSTATUS LdrMapImageSection(PROCESS* process,
IMAGE_SECTION_OBJECT* section,
void** baseOut) {
if (!process || !section || !baseOut) return STATUS_INVALID_PARAMETER;
*baseOut = (void*)0;
return MmVadCreateImageSectionMapping(process,
section,
PAGE_PRESENT | PAGE_WRITE | PAGE_USER | PAGE_NX,
MM_VAD_ZERO | MM_VAD_GUARD_LOW | MM_VAD_GUARD_HIGH,
baseOut);
}
code/kernel/mm_file_cache.c
+1273
View File
File diff suppressed because it is too large. Load diff
code/kernel/mm_phys.c
+711 -176
View File
File diff suppressed because it is too large. Load diff
code/kernel/mm_pte.c
+575
View File
@@ -0,0 +1,575 @@
// Kernel page-table manager. This file owns x86-64 PTE manipulation, TLB invalidation, and shared-CR3 activation.
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/mm_pte.h>
#include <kernel/mm_pfn.h>
#include <kernel/ke_cpu.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_spinlock.h>
#include <kernel/hal/hal_apic.h>
// Page-table frames are reached through the physical direct map while the kernel still owns one shared CR3.
static uint64_t gPhysBase = PHYS_MAP_BASE;
// Converts a physical address to the kernel physical-window alias used for page-table editing.
static inline void* KiPhysToVirt(PHYSICAL_ADDRESS p) {
return (void *)(p + gPhysBase);
}
// Returns the entry pointer for a page-table frame that is already mapped in the physical window.
static inline uint64_t* KiPtePtr(PHYSICAL_ADDRESS tablePhys, unsigned idx) {
return (uint64_t *)KiPhysToVirt(tablePhys) + idx;
}
// The bootloader builds the initial PML4, and this layer owns all later edits to it.
static PHYSICAL_ADDRESS gPml4Phys = 0;
static SPINLOCK gPageTableLock = SPINLOCK_INIT;
static volatile long gTlbShootdownLock = 0;
static volatile long gTlbShootdownActive = 0;
static volatile long gTlbShootdownAcks = 0;
static uintptr_t gTlbShootdownVirt = 0;
static MM_PTE_STATS gPteStats;
#define TLB_SHOOTDOWN_TIMEOUT_SPINS 20000000u
// These macros extract the hardware page-table index for each level.
#define PML4_IDX(va) (((va) >> 39) & 0x1FF)
#define PDPT_IDX(va) (((va) >> 30) & 0x1FF)
#define PDT_IDX(va) (((va) >> 21) & 0x1FF)
#define PT_IDX(va) (((va) >> 12) & 0x1FF)
#define PAGE_PS (1ULL << 7)
#define PAGE_FLAGS_MASK (0xFFFULL | PAGE_NX)
#define PAGE_FRAME_4K_MASK 0x000FFFFFFFFFF000ULL
#define PAGE_FRAME_2M_MASK 0x000FFFFFFFE00000ULL
#define PAGE_FRAME_1G_MASK 0x000FFFFFC0000000ULL
// Bumps a telemetry counter. These counters are diagnostic only; exactness is
// less important than avoiding extra locks in hot page-table paths.
static inline void KiPteStatInc(uint64_t* counter) {
(*counter)++;
}
// Clamps query attribution so corrupt callers cannot index outside telemetry arrays.
static MM_PTE_QUERY_PURPOSE KiNormalizeQueryPurpose(MM_PTE_QUERY_PURPOSE purpose) {
if ((uint32_t)purpose >= (uint32_t)MmPteQueryPurposeCount)
return MmPteQueryGeneric;
return purpose;
}
// Ensures that one parent entry points to a child page table.
// New child tables are zeroed before publication so no stale mappings become visible.
static PHYSICAL_ADDRESS KiEnsureTable(PHYSICAL_ADDRESS parentPhys,
unsigned index, uint64_t extraFlags) {
uint64_t* entry = KiPtePtr(parentPhys, index);
if (*entry & PAGE_PRESENT) {
*entry |= extraFlags;
return *entry & PAGE_FRAME_4K_MASK;
}
PHYSICAL_ADDRESS childPhys = MmPfnAllocate(MmPfnUsePageTable);
if (!childPhys) return 0;
KiPteStatInc(&gPteStats.PageTablePagesAllocated);
memset(KiPhysToVirt(childPhys), 0, PAGE_SIZE);
*entry = childPhys | PAGE_PRESENT | PAGE_WRITE | extraFlags;
return childPhys;
}
// Initializes the page-table manager around the bootloader-created PML4.
void MmPteInitialize(PHYSICAL_ADDRESS bootPml4Phys) {
gPml4Phys = bootPml4Phys;
gPageTableLock = (SPINLOCK)SPINLOCK_INIT;
gTlbShootdownLock = 0;
gTlbShootdownActive = 0;
gTlbShootdownAcks = 0;
gTlbShootdownVirt = 0;
memset(&gPteStats, 0, sizeof(gPteStats));
}
// Installs the shared PML4 into CR3.
void MmPteActivate(void) {
__writecr3(gPml4Phys);
}
// Flushes one virtual address from the local TLB.
void MmPteFlushLocal(uintptr_t virt) {
KiPteStatInc(&gPteStats.LocalFlushes);
__invlpg((void*)virt);
}
// Acquires the shootdown state lock without disabling interrupts.
static void KiAcquireTlbShootdownLock(void) {
// Do not use the normal spinlock here: it masks interrupts while spinning.
// A CPU spinning with IF=0 on the shootdown lock could not acknowledge a
// concurrent fixed-vector TLB IPI, creating the exact deadlock this lock is
// meant to avoid.
while (_InterlockedCompareExchange(&gTlbShootdownLock, 1, 0) != 0)
_mm_pause();
_ReadWriteBarrier();
}
// Releases the shootdown state lock after all shared fields are stable.
static void KiReleaseTlbShootdownLock(void) {
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownLock, 0);
}
// Flushes one kernel mapping on every online CPU that shares this PML4.
void MmPteFlushKernelAddress(uintptr_t virt) {
KiPteStatInc(&gPteStats.GlobalFlushes);
MmPteFlushLocal(virt);
if (gCpuCount <= 1) {
KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets);
return;
}
KCPU* self = KeGetCurrentCpu();
uint32_t selfIndex = self ? self->CpuIndex : KE_INVALID_CPU;
KiAcquireTlbShootdownLock();
long targets = 0;
gTlbShootdownVirt = virt;
_InterlockedExchange(&gTlbShootdownAcks, 0);
_ReadWriteBarrier();
_InterlockedExchange(&gTlbShootdownActive, 1);
_ReadWriteBarrier();
for (uint32_t i = 0; i < gCpuCount; i++) {
KCPU* cpu = &gCpus[i];
if (i == selfIndex)
continue;
if (!cpu->Present || !cpu->Started || !cpu->Online)
continue;
targets++;
KiPteStatInc(&gPteStats.ShootdownTargetCpus);
HalApicSendTlbShootdownIpi(cpu->LapicId);
}
if (!targets) {
KiPteStatInc(&gPteStats.ShootdownSkippedNoTargets);
_InterlockedExchange(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
return;
}
KiPteStatInc(&gPteStats.ShootdownRequests);
for (uint32_t spin = 0; targets > 0 && spin < TLB_SHOOTDOWN_TIMEOUT_SPINS; spin++) {
if (_InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0) >= targets)
break;
_mm_pause();
}
long acks = _InterlockedCompareExchange(&gTlbShootdownAcks, 0, 0);
_InterlockedExchange(&gTlbShootdownActive, 0);
_ReadWriteBarrier();
KiReleaseTlbShootdownLock();
gPteStats.ShootdownAcks += (uint64_t)acks;
if (acks < targets) {
KiPteStatInc(&gPteStats.ShootdownTimeouts);
KePanic(KE_PANIC_CODE_GENERIC,
"TLB shootdown timeout: va=0x%016llX targets=%ld acks=%ld selfCpu=%u\n",
(uint64_t)virt, targets, acks, selfIndex);
}
}
// Handles an incoming shootdown request on the current CPU.
static int KiHandleTlbShootdown(void) {
if (_InterlockedCompareExchange(&gTlbShootdownActive, 0, 0) == 0)
return 0;
uintptr_t va = gTlbShootdownVirt;
_ReadWriteBarrier();
MmPteFlushLocal(va);
_ReadWriteBarrier();
_InterlockedIncrement(&gTlbShootdownAcks);
return 1;
}
// Handles the NMI vector reserved for TLB shootdowns.
int MmPteHandleShootdownNmi(void) {
return KiHandleTlbShootdown();
}
// Maps or replaces one 4 KiB leaf PTE in the shared page tables.
__declspec(dllexport) KSTATUS MmPteMapPage(uintptr_t virt, PHYSICAL_ADDRESS phys, uint64_t flags, MM_PTE_MAP_MODE mode) {
KiPteStatInc(&gPteStats.MapCalls);
unsigned userFlag = (flags & PAGE_USER) ? PAGE_USER : 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
PHYSICAL_ADDRESS pdpt = KiEnsureTable(gPml4Phys, PML4_IDX(virt), userFlag);
if (!pdpt) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pdt = KiEnsureTable(pdpt, PDPT_IDX(virt), userFlag);
if (!pdt) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
PHYSICAL_ADDRESS pt = KiEnsureTable(pdt, PDT_IDX(virt), userFlag);
if (!pt) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NO_MEMORY;
}
uint64_t* pte = KiPtePtr(pt, PT_IDX(virt));
if ((*pte & PAGE_PRESENT) && mode == MmPteMapFailIfPresent) {
KiPteStatInc(&gPteStats.MapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_ALREADY_EXISTS;
}
int wasPresent = ((*pte & PAGE_PRESENT) != 0);
*pte = phys | flags;
if (!wasPresent) {
KiPteStatInc(&gPteStats.CurrentLeafMappings);
KiPteStatInc(&gPteStats.NewMappingNoFlushes);
}
KeReleaseSpinlock(&gPageTableLock, lockFlags);
if (wasPresent) {
KiPteStatInc(&gPteStats.ReplacedMappingFlushes);
MmPteFlushKernelAddress(virt);
}
return STATUS_SUCCESS;
}
// Removes one 4 KiB leaf PTE from the shared page tables.
__declspec(dllexport) KSTATUS MmPteUnmapPage(uintptr_t virt) {
KiPteStatInc(&gPteStats.UnmapCalls);
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(virt));
if (!(*e4 & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(virt));
if (!(*e3 & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(virt));
if (!(*e2 & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(virt));
if (!(*pte & PAGE_PRESENT)) {
KiPteStatInc(&gPteStats.UnmapFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return STATUS_NOT_FOUND;
}
*pte = 0;
if (gPteStats.CurrentLeafMappings)
gPteStats.CurrentLeafMappings--;
KeReleaseSpinlock(&gPageTableLock, lockFlags);
MmPteFlushKernelAddress(virt);
return STATUS_SUCCESS;
}
// Replaces leaf flags for a 4 KiB range and refuses huge pages instead of silently corrupting them.
__declspec(dllexport) KSTATUS MmPteProtectRange(uintptr_t virt, size_t size, uint64_t flags) {
KiPteStatInc(&gPteStats.ProtectCalls);
if (!size) return STATUS_SUCCESS;
// The protection operation is page-granular even when callers pass an interior address.
uintptr_t base = virt & PAGE_MASK;
uintptr_t end = (virt + size + PAGE_SIZE - 1) & PAGE_MASK;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
KSTATUS status = STATUS_SUCCESS;
uintptr_t flushEnd = base;
for (uintptr_t va = base; va < end; va += PAGE_SIZE) {
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
if (*e3 & PAGE_PS) {
status = STATUS_NOT_SUPPORTED;
break;
}
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
if (*e2 & PAGE_PS) {
status = STATUS_NOT_SUPPORTED;
break;
}
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT)) {
status = STATUS_NOT_FOUND;
break;
}
// The physical frame stays attached while access and cache attributes change.
PHYSICAL_ADDRESS phys = *pte & PAGE_FRAME_4K_MASK;
*pte = phys | flags;
flushEnd = va + PAGE_SIZE;
}
if (!KSUCCESS(status))
KiPteStatInc(&gPteStats.ProtectFailures);
KeReleaseSpinlock(&gPageTableLock, lockFlags);
for (uintptr_t va = base; va < flushEnd; va += PAGE_SIZE)
MmPteFlushKernelAddress(va);
return status;
}
// Tests and clears the hardware accessed bit on a mapped 4 KiB leaf. This is
// used by the pressure trimmer as a second-chance test for clean file-backed
// pages. If the page was accessed recently, reclaim skips it once and flushes
// the TLB entry so future accesses can set PAGE_ACCESSED again.
__declspec(dllexport) int MmPteTestAndClearAccessed(uintptr_t virt) {
if (!gPml4Phys)
return 0;
uintptr_t va = virt & PAGE_MASK;
int wasAccessed = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT))
goto out;
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS))
goto out;
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS))
goto out;
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT))
goto out;
if (*pte & PAGE_ACCESSED) {
*pte &= ~PAGE_ACCESSED;
wasAccessed = 1;
}
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
if (wasAccessed)
MmPteFlushKernelAddress(va);
return wasAccessed;
}
// Tests and clears the hardware dirty bit on a mapped 4 KiB leaf.
__declspec(dllexport) int MmPteTestAndClearDirty(uintptr_t virt) {
if (!gPml4Phys)
return 0;
uintptr_t va = virt & PAGE_MASK;
int wasDirty = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* e4 = KiPtePtr(gPml4Phys, PML4_IDX(va));
if (!(*e4 & PAGE_PRESENT))
goto out;
uint64_t* e3 = KiPtePtr(*e4 & PAGE_FRAME_4K_MASK, PDPT_IDX(va));
if (!(*e3 & PAGE_PRESENT) || (*e3 & PAGE_PS))
goto out;
uint64_t* e2 = KiPtePtr(*e3 & PAGE_FRAME_4K_MASK, PDT_IDX(va));
if (!(*e2 & PAGE_PRESENT) || (*e2 & PAGE_PS))
goto out;
uint64_t* pte = KiPtePtr(*e2 & PAGE_FRAME_4K_MASK, PT_IDX(va));
if (!(*pte & PAGE_PRESENT))
goto out;
if (*pte & PAGE_DIRTY) {
*pte &= ~PAGE_DIRTY;
wasDirty = 1;
}
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
if (wasDirty)
MmPteFlushKernelAddress(va);
return wasDirty;
}
// Translates a mapped virtual address to a physical address.
__declspec(dllexport) PHYSICAL_ADDRESS MmPteTranslate(uintptr_t virt) {
KiPteStatInc(&gPteStats.TranslateCalls);
if (!gPml4Phys) return 0;
PHYSICAL_ADDRESS result = 0;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t pml4_idx = (virt >> 39) & 0x1FF;
uint64_t pdpt_idx = (virt >> 30) & 0x1FF;
uint64_t pd_idx = (virt >> 21) & 0x1FF;
uint64_t pt_idx = (virt >> 12) & 0x1FF;
uint64_t* pml4 = KiPhysToVirt(gPml4Phys);
if (!(pml4[pml4_idx] & PAGE_PRESENT)) goto out;
uint64_t* pdpt = KiPhysToVirt(pml4[pml4_idx] & PAGE_FRAME_4K_MASK);
if (!(pdpt[pdpt_idx] & PAGE_PRESENT)) goto out;
// A present PDPT leaf maps a 1 GiB huge page.
if (pdpt[pdpt_idx] & PAGE_PS) {
result = (pdpt[pdpt_idx] & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFF);
goto out;
}
uint64_t* pd = KiPhysToVirt(pdpt[pdpt_idx] & PAGE_FRAME_4K_MASK);
if (!(pd[pd_idx] & PAGE_PRESENT)) goto out;
// A present PDT leaf maps a 2 MiB huge page.
if (pd[pd_idx] & PAGE_PS) {
result = (pd[pd_idx] & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFF);
goto out;
}
uint64_t* pt = KiPhysToVirt(pd[pd_idx] & PAGE_FRAME_4K_MASK);
if (!(pt[pt_idx] & PAGE_PRESENT)) goto out;
// A normal PT leaf maps a 4 KiB page.
result = (pt[pt_idx] & PAGE_FRAME_4K_MASK) + (virt & 0xFFF);
out:
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return result;
}
// Queries one virtual address through the shared page-table hierarchy.
KSTATUS MmPteQueryPageFor(MM_PTE_QUERY_PURPOSE purpose,
uintptr_t virt,
PHYSICAL_ADDRESS* physOut,
uint64_t* flagsOut) {
purpose = KiNormalizeQueryPurpose(purpose);
KiPteStatInc(&gPteStats.QueryCalls);
KiPteStatInc(&gPteStats.QueryByPurpose[purpose]);
if (!gPml4Phys) {
KiPteStatInc(&gPteStats.QueryFailures);
KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]);
return STATUS_NOT_FOUND;
}
KSTATUS status = STATUS_NOT_FOUND;
uintptr_t lockFlags = KeAcquireSpinlock(&gPageTableLock);
uint64_t* pml4 = (uint64_t*)KiPhysToVirt(gPml4Phys);
uint64_t e4 = pml4[PML4_IDX(virt)];
if (!(e4 & PAGE_PRESENT)) goto out;
uint64_t effectiveWrite = e4 & PAGE_WRITE;
uint64_t effectiveUser = e4 & PAGE_USER;
uint64_t effectiveNx = e4 & PAGE_NX;
uint64_t* pdpt = (uint64_t*)KiPhysToVirt(e4 & PAGE_FRAME_4K_MASK);
uint64_t e3 = pdpt[PDPT_IDX(virt)];
if (!(e3 & PAGE_PRESENT)) goto out;
effectiveWrite &= e3 & PAGE_WRITE;
effectiveUser &= e3 & PAGE_USER;
effectiveNx |= e3 & PAGE_NX;
if (e3 & PAGE_PS) {
if (physOut)
*physOut = (e3 & PAGE_FRAME_1G_MASK) + (virt & 0x3FFFFFFFULL);
if (flagsOut) {
uint64_t flags = (e3 & PAGE_FLAGS_MASK) | effectiveNx;
if (!effectiveWrite) flags &= ~PAGE_WRITE;
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pd = (uint64_t*)KiPhysToVirt(e3 & PAGE_FRAME_4K_MASK);
uint64_t e2 = pd[PDT_IDX(virt)];
if (!(e2 & PAGE_PRESENT)) goto out;
effectiveWrite &= e2 & PAGE_WRITE;
effectiveUser &= e2 & PAGE_USER;
effectiveNx |= e2 & PAGE_NX;
if (e2 & PAGE_PS) {
if (physOut)
*physOut = (e2 & PAGE_FRAME_2M_MASK) + (virt & 0x1FFFFFULL);
if (flagsOut) {
uint64_t flags = (e2 & PAGE_FLAGS_MASK) | effectiveNx;
if (!effectiveWrite) flags &= ~PAGE_WRITE;
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
status = STATUS_SUCCESS;
goto out;
}
uint64_t* pt = (uint64_t*)KiPhysToVirt(e2 & PAGE_FRAME_4K_MASK);
uint64_t e1 = pt[PT_IDX(virt)];
if (!(e1 & PAGE_PRESENT)) goto out;
effectiveWrite &= e1 & PAGE_WRITE;
effectiveUser &= e1 & PAGE_USER;
effectiveNx |= e1 & PAGE_NX;
if (physOut)
*physOut = (e1 & PAGE_FRAME_4K_MASK) + (virt & 0xFFFULL);
if (flagsOut) {
uint64_t flags = (e1 & PAGE_FLAGS_MASK) | effectiveNx;
if (!effectiveWrite) flags &= ~PAGE_WRITE;
if (!effectiveUser) flags &= ~PAGE_USER;
*flagsOut = flags;
}
status = STATUS_SUCCESS;
out:
if (!KSUCCESS(status)) {
KiPteStatInc(&gPteStats.QueryFailures);
KiPteStatInc(&gPteStats.QueryFailuresByPurpose[purpose]);
}
KeReleaseSpinlock(&gPageTableLock, lockFlags);
return status;
}
// Queries one virtual address without caller-specific attribution.
__declspec(dllexport) KSTATUS MmPteQueryPage(uintptr_t virt,
PHYSICAL_ADDRESS* physOut,
uint64_t* flagsOut) {
return MmPteQueryPageFor(MmPteQueryGeneric, virt, physOut, flagsOut);
}
// Returns page-table telemetry for diagnostics.
void MmPteQueryStats(MM_PTE_STATS* out) {
if (!out)
return;
*out = gPteStats;
}
code/kernel/mm_swap.c
+243
View File
@@ -0,0 +1,243 @@
// Kernel swap manager.
// The first backend is a fixed-size pagefile accessed through VFS offset I/O.
// Kernel memory and filesystem code remain non-pageable, so this layer is safe
// to use before the VMM has direct block-extents for pageout.
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <kernel/mm_swap.h>
#include <kernel/mm_virt.h>
#include <kernel/mm_pfn.h>
#include <kernel/io_file.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_debug.h>
typedef struct MM_SWAP_STATE {
SPINLOCK Lock;
FILE_OBJECT* File;
uint8_t* Bitmap;
uint32_t SlotCount;
uint32_t UsedSlots;
uint32_t HighWaterSlots;
int Enabled;
MM_SWAP_STATS Stats;
} MM_SWAP_STATE;
static MM_SWAP_STATE gSwap = { .Lock = SPINLOCK_INIT };
// Returns whether a swap slot is marked in use.
static int KiSwapTestSlot(uint32_t slot) {
return (gSwap.Bitmap[slot >> 3] & (uint8_t)(1u << (slot & 7))) != 0;
}
// Marks a swap slot as allocated or free.
static void KiSwapSetSlot(uint32_t slot, int used) {
uint8_t mask = (uint8_t)(1u << (slot & 7));
if (used)
gSwap.Bitmap[slot >> 3] |= mask;
else
gSwap.Bitmap[slot >> 3] &= (uint8_t)~mask;
}
// Allocates one pagefile slot.
static KSTATUS KiSwapAllocateSlot(uint32_t* slotOut) {
if (!slotOut) return STATUS_INVALID_PARAMETER;
*slotOut = 0;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
if (!gSwap.Enabled || !gSwap.Bitmap || !gSwap.SlotCount) {
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_NOT_SUPPORTED;
}
for (uint32_t slot = 0; slot < gSwap.SlotCount; ++slot) {
if (!KiSwapTestSlot(slot)) {
KiSwapSetSlot(slot, 1);
gSwap.UsedSlots++;
if (gSwap.UsedSlots > gSwap.HighWaterSlots)
gSwap.HighWaterSlots = gSwap.UsedSlots;
gSwap.Stats.SlotAllocations++;
*slotOut = slot;
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_SUCCESS;
}
}
gSwap.Stats.SlotAllocationFailures++;
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_NO_MEMORY;
}
// Initializes the fixed pagefile backend.
KSTATUS MmSwapInitialize(SYS_STRING_VIEW path, uint64_t sizeBytes) {
if (!path.Data || !path.Length || sizeBytes < PAGE_SIZE)
return STATUS_INVALID_PARAMETER;
if (sizeBytes & (PAGE_SIZE - 1))
sizeBytes = (sizeBytes + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1);
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
if (gSwap.Enabled) {
KeReleaseSpinlock(&gSwap.Lock, flags);
return STATUS_SUCCESS;
}
KeReleaseSpinlock(&gSwap.Lock, flags);
FILE_OBJECT* file = (FILE_OBJECT*)0;
KSTATUS s = IoOpenFile(path, &file);
if (!KSUCCESS(s)) {
s = IoCreateFile(path, &file);
if (!KSUCCESS(s))
return s;
}
// Swap slots are never read before MmSwapPageOut overwrites the whole
// page, so the filesystem may allocate backing clusters without zeroing
// them. Normal files still get zero-on-extend semantics.
file->Flags |= IO_FILE_FLAG_NO_CACHE | IO_FILE_FLAG_SKIP_ZERO_EXTEND;
s = IoTruncateFile(file, sizeBytes);
if (!KSUCCESS(s)) {
IoCloseFile(file);
return s;
}
uint64_t slots64 = sizeBytes / PAGE_SIZE;
if (!slots64 || slots64 > 0xFFFFFFFFull) {
IoCloseFile(file);
return STATUS_INVALID_PARAMETER;
}
uint32_t slotCount = (uint32_t)slots64;
size_t bitmapBytes = (slotCount + 7u) / 8u;
uint8_t* bitmap = (uint8_t*)MmAllocatePoolTag(bitmapBytes, MM_POOL_TAG('s', 'w', 'a', 'p'));
if (!bitmap) {
IoCloseFile(file);
return STATUS_NO_MEMORY;
}
memset(bitmap, 0, bitmapBytes);
flags = KeAcquireSpinlock(&gSwap.Lock);
if (gSwap.Enabled) {
KeReleaseSpinlock(&gSwap.Lock, flags);
MmFreePool(bitmap);
IoCloseFile(file);
return STATUS_SUCCESS;
}
gSwap.File = file;
gSwap.Bitmap = bitmap;
gSwap.SlotCount = slotCount;
gSwap.UsedSlots = 0;
gSwap.HighWaterSlots = 0;
gSwap.Enabled = 1;
memset(&gSwap.Stats, 0, sizeof(gSwap.Stats));
gSwap.Stats.Enabled = 1;
gSwap.Stats.TotalSlots = slotCount;
gSwap.Stats.FreeSlots = slotCount;
KeReleaseSpinlock(&gSwap.Lock, flags);
DbgPrint("swap: initialized %u slots (%llu KiB) at %v\n",
slotCount,
(unsigned long long)(sizeBytes / 1024),
path);
return STATUS_SUCCESS;
}
// Returns whether swap is available for pageout.
int MmSwapIsEnabled(void) {
return gSwap.Enabled;
}
// Writes one physical page to a newly allocated swap slot.
KSTATUS MmSwapPageOut(PHYSICAL_ADDRESS frame, uint32_t* slotOut) {
if (!slotOut || !frame)
return STATUS_INVALID_PARAMETER;
*slotOut = 0;
if (!gSwap.Enabled || !gSwap.File)
return STATUS_NOT_SUPPORTED;
uint32_t slot = 0;
KSTATUS s = KiSwapAllocateSlot(&slot);
if (!KSUCCESS(s)) {
gSwap.Stats.PageOutFailures++;
return s;
}
void* page = MmPfnToVirtual(frame);
if (!page) {
MmSwapFreeSlot(slot);
gSwap.Stats.PageOutFailures++;
return STATUS_FAILURE;
}
size_t written = 0;
s = IoWriteFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &written);
if (!KSUCCESS(s) || written != PAGE_SIZE) {
MmSwapFreeSlot(slot);
gSwap.Stats.PageOutFailures++;
return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s;
}
gSwap.Stats.PageOuts++;
*slotOut = slot;
return STATUS_SUCCESS;
}
// Reads one swap slot into an already allocated physical page.
KSTATUS MmSwapPageIn(uint32_t slot, PHYSICAL_ADDRESS frame) {
if (!frame)
return STATUS_INVALID_PARAMETER;
if (!gSwap.Enabled || !gSwap.File)
return STATUS_NOT_SUPPORTED;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
int valid = slot < gSwap.SlotCount && KiSwapTestSlot(slot);
KeReleaseSpinlock(&gSwap.Lock, flags);
if (!valid) {
gSwap.Stats.PageInFailures++;
return STATUS_NOT_FOUND;
}
void* page = MmPfnToVirtual(frame);
if (!page) {
gSwap.Stats.PageInFailures++;
return STATUS_FAILURE;
}
size_t read = 0;
KSTATUS s = IoReadFileAt(gSwap.File, (uint64_t)slot * PAGE_SIZE, page, PAGE_SIZE, &read);
if (!KSUCCESS(s) || read != PAGE_SIZE) {
gSwap.Stats.PageInFailures++;
return KSUCCESS(s) ? STATUS_DEVICE_ERROR : s;
}
gSwap.Stats.PageIns++;
return STATUS_SUCCESS;
}
// Releases a swap slot after the owning VAD page no longer references it.
void MmSwapFreeSlot(uint32_t slot) {
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
if (gSwap.Enabled && slot < gSwap.SlotCount && KiSwapTestSlot(slot)) {
KiSwapSetSlot(slot, 0);
if (gSwap.UsedSlots)
gSwap.UsedSlots--;
gSwap.Stats.SlotFrees++;
}
KeReleaseSpinlock(&gSwap.Lock, flags);
}
// Copies swap telemetry for user diagnostics.
void MmSwapQueryStats(MM_SWAP_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gSwap.Lock);
MM_SWAP_STATS stats = gSwap.Stats;
stats.Enabled = gSwap.Enabled ? 1 : 0;
stats.TotalSlots = gSwap.SlotCount;
stats.UsedSlots = gSwap.UsedSlots;
stats.FreeSlots = gSwap.SlotCount >= gSwap.UsedSlots ? gSwap.SlotCount - gSwap.UsedSlots : 0;
stats.HighWaterSlots = gSwap.HighWaterSlots;
KeReleaseSpinlock(&gSwap.Lock, flags);
*out = stats;
}
code/kernel/mm_virt.c
+2723 -889
View File
File diff suppressed because it is too large. Load diff
code/kernel/ob_object.c
+27 -2
View File
@@ -5,11 +5,36 @@
#include <kernel/ob_object.h>
#include <kernel/mm_virt.h>
// Creates object.
// Returns a diagnostic pool tag for the object type.
static uint32_t KiObjectPoolTag(OBJECT_TYPE type) {
switch (type) {
case ObjectTypeProcess: return MM_POOL_TAG('p', 'r', 'o', 'c');
case ObjectTypeThread: return MM_POOL_TAG('t', 'h', 'r', 'd');
case ObjectTypeMutex: return MM_POOL_TAG('m', 't', 'x', ' ');
case ObjectTypeSemaphore: return MM_POOL_TAG('s', 'e', 'm', ' ');
case ObjectTypeEvent: return MM_POOL_TAG('e', 'v', 'n', 't');
case ObjectTypeQueue: return MM_POOL_TAG('q', 'u', 'e', 'u');
case ObjectTypeFile: return MM_POOL_TAG('f', 'i', 'l', 'e');
case ObjectTypeDir: return MM_POOL_TAG('d', 'i', 'r', ' ');
case ObjectTypeAsyncIo: return MM_POOL_TAG('a', 'i', 'o', ' ');
case ObjectTypeSocket: return MM_POOL_TAG('s', 'o', 'c', 'k');
case ObjectTypeDevice: return MM_POOL_TAG('d', 'e', 'v', ' ');
case ObjectTypeDriver: return MM_POOL_TAG('d', 'r', 'v', 'r');
case ObjectTypeUsbDevice: return MM_POOL_TAG('u', 's', 'b', 'd');
case ObjectTypeUsbPipe: return MM_POOL_TAG('u', 's', 'b', 'p');
case ObjectTypeFileMapping: return MM_POOL_TAG('m', 'a', 'p', 'f');
case ObjectTypeSurface: return MM_POOL_TAG('s', 'u', 'r', 'f');
case ObjectTypeInputQueue: return MM_POOL_TAG('i', 'n', 'p', 'q');
case ObjectTypeImageSection: return MM_POOL_TAG('i', 'm', 's', 'e');
default: return MM_POOL_TAG('o', 'b', 'j', ' ');
}
}
// Creates a zeroed, tagged object with one initial reference.
__declspec(dllexport) OBJECT* ObAllocateObject(OBJECT_TYPE type, size_t size) {
if (size < sizeof(OBJECT)) size = sizeof(OBJECT);
OBJECT* obj = (OBJECT *)MmAllocatePool(size);
OBJECT* obj = (OBJECT *)MmAllocatePoolTag(size, KiObjectPoolTag(type));
if (!obj) return (OBJECT *)0;
memset(obj, 0, size);