maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
d36f19509c72ba98336e2d83d22e7fce55c1e759
nt-lite/code/kernel/mm_file_cache.c
T

1015 lines
35 KiB
C
Raw Blame History

// Coherent file page cache.
//
// The cache owns one resident page per file/offset. Reads, buffered writes, and
// mapped-file faults converge on this page so writeback has a single source of
// truth. Dirty pages are flushed explicitly, by close, or by the background
// writeback worker under pressure.
#include <stdbool.h>
#include <stdint.h>
#include <stddef.h>
#include <string.h>
#include <intrin.h>
#include <kernel/mm_file_cache.h>
#include <kernel/io_file.h>
#include <kernel/io_volume.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_spinlock.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#define MM_FILE_CACHE_BUCKETS 64u
#define MM_FILE_CACHE_SOFT_MAX_PAGES 512u
#define MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES 128u
#define MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES 192u
#define MM_FILE_CACHE_WRITEBACK_BATCH_PAGES 16u
typedef struct MM_FILE_CACHE_ENTRY {
struct MM_FILE_CACHE_ENTRY* Next;
FILE_OBJECT* File;
uint64_t Offset;
PHYSICAL_ADDRESS Frame;
uint32_t ValidBytes;
uint8_t Referenced;
uint8_t Standby;
uint8_t Loading;
bool Invalidated;
uint8_t Dirty;
uint8_t Writeback;
uint64_t DirtySequence;
} MM_FILE_CACHE_ENTRY;
static SPINLOCK gFileCacheLock = SPINLOCK_INIT;
static MM_FILE_CACHE_ENTRY* gFileCacheBuckets[MM_FILE_CACHE_BUCKETS];
static uint64_t gFileCacheResidentPages;
static uint64_t gFileCacheActivePages;
static uint64_t gFileCacheStandbyPages;
static uint64_t gFileCacheDirtyPages;
static uint64_t gFileCacheMaxDirtyPages;
static uint64_t gFileCacheMaxResidentPages;
static MM_FILE_CACHE_STATS gFileCacheStats;
static bool KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry);
static SPINLOCK gWritebackLock = SPINLOCK_INIT;
static SEMAPHORE* gWritebackSemaphore;
static int gWritebackStartState;
static uint32_t KiFileCacheHash(FILE_OBJECT* file, uint64_t offset) {
uintptr_t f = (uintptr_t)file >> 4;
uint64_t p = offset >> 12;
return (uint32_t)((f ^ p ^ (p >> 11)) & (MM_FILE_CACHE_BUCKETS - 1u));
}
static MM_FILE_CACHE_ENTRY* KiFileCacheFindLocked(FILE_OBJECT* file, uint64_t offset, uint32_t bucket) {
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[bucket]; e; e = e->Next) {
if (e->File == file && e->Offset == offset)
return e;
}
return (MM_FILE_CACHE_ENTRY*)0;
}
static void KiFileCacheBumpResidentLocked(int delta, int standby) {
if (delta > 0) {
gFileCacheResidentPages += (uint64_t)delta;
if (standby) gFileCacheStandbyPages += (uint64_t)delta;
else gFileCacheActivePages += (uint64_t)delta;
if (gFileCacheResidentPages > gFileCacheMaxResidentPages)
gFileCacheMaxResidentPages = gFileCacheResidentPages;
} else if (delta < 0) {
uint64_t dec = (uint64_t)(-delta);
gFileCacheResidentPages = gFileCacheResidentPages > dec ? gFileCacheResidentPages - dec : 0;
if (standby) gFileCacheStandbyPages = gFileCacheStandbyPages > dec ? gFileCacheStandbyPages - dec : 0;
else gFileCacheActivePages = gFileCacheActivePages > dec ? gFileCacheActivePages - dec : 0;
}
}
// Updates dirty-page accounting while the cache lock is held.
static void KiFileCacheSetDirtyLocked(MM_FILE_CACHE_ENTRY* e, int dirty) {
if (!e)
return;
if (dirty) {
if (!e->Dirty) {
e->Dirty = 1;
gFileCacheDirtyPages++;
gFileCacheStats.DirtyPages = gFileCacheDirtyPages;
if (gFileCacheDirtyPages > gFileCacheMaxDirtyPages) {
gFileCacheMaxDirtyPages = gFileCacheDirtyPages;
gFileCacheStats.MaxDirtyPages = gFileCacheMaxDirtyPages;
}
}
e->DirtySequence++;
gFileCacheStats.DirtyMarks++;
} else {
if (e->Dirty) {
e->Dirty = 0;
if (gFileCacheDirtyPages)
gFileCacheDirtyPages--;
gFileCacheStats.DirtyPages = gFileCacheDirtyPages;
}
}
}
static void KiFileCacheMoveEntryToStandbyLocked(MM_FILE_CACHE_ENTRY* e) {
if (!e || e->Standby)
return;
e->Standby = 1;
if (gFileCacheActivePages) gFileCacheActivePages--;
gFileCacheStandbyPages++;
gFileCacheStats.StandbyTransitions++;
}
static void KiFileCacheActivateEntryLocked(MM_FILE_CACHE_ENTRY* e) {
if (!e || !e->Standby)
return;
e->Standby = 0;
if (gFileCacheStandbyPages) gFileCacheStandbyPages--;
gFileCacheActivePages++;
gFileCacheStats.StandbyHits++;
}
static void KiFileCacheFreeEntry(MM_FILE_CACHE_ENTRY* e) {
if (!e) return;
if (e->File) ObDereferenceObject(&e->File->Header);
if (e->Frame) (void)MmPfnRelease(e->Frame);
MmFreePool(e);
}
static void KiFileCacheRemoveEntryLocked(uint32_t bucket, MM_FILE_CACHE_ENTRY* prev, MM_FILE_CACHE_ENTRY* e) {
if (!e) return;
if (prev) prev->Next = e->Next;
else gFileCacheBuckets[bucket] = e->Next;
if (e->Frame)
KiFileCacheBumpResidentLocked(-1, e->Standby);
if (e->Dirty)
KiFileCacheSetDirtyLocked(e, 0);
}
static bool KiFileCacheIsReclaimCandidateLocked(MM_FILE_CACHE_ENTRY* e, FILE_OBJECT* skipFile, uint64_t skipOffset) {
if (!e) return false;
if (skipFile && e->File == skipFile && e->Offset == skipOffset)
return false;
if (e->Loading || e->Invalidated || !e->Frame || e->Dirty || e->Writeback)
return false;
MM_PFN* pfn = MmPfnGet(e->Frame);
return pfn && pfn->PinCount == 0 && pfn->MappedCount == 0 && pfn->RefCount == 1;
}
static KSTATUS KiFileCacheDemoteOnePage(FILE_OBJECT* skipFile, uint64_t skipOffset) {
// A cold clean page first moves to PFN standby so the cache can reactivate it
// without disk I/O if it becomes hot again before final pressure reclaim.
for (int pass = 0; pass < 2; ++pass) {
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
MM_FILE_CACHE_ENTRY* demote = (MM_FILE_CACHE_ENTRY*)0;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; e = e->Next) {
if (e->Standby || !KiFileCacheIsReclaimCandidateLocked(e, skipFile, skipOffset))
continue;
if (e->Referenced && pass == 0) {
e->Referenced = 0;
continue;
}
demote = e;
break;
}
if (demote) {
KSTATUS s = MmPfnMoveToStandby(demote->Frame);
if (KSUCCESS(s)) {
KiFileCacheMoveEntryToStandbyLocked(demote);
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_SUCCESS;
}
}
KeReleaseSpinlock(&gFileCacheLock, flags);
}
}
return NTLSTATUS_NOT_FOUND;
}
static KSTATUS KiFileCacheDropOneStandbyPage(FILE_OBJECT* skipFile, uint64_t skipOffset) {
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) {
if (!e->Standby || !KiFileCacheIsReclaimCandidateLocked(e, skipFile, skipOffset))
continue;
victim = e;
KiFileCacheRemoveEntryLocked(b, prev, e);
gFileCacheStats.Drops++;
break;
}
KeReleaseSpinlock(&gFileCacheLock, flags);
if (victim) {
KiFileCacheFreeEntry(victim);
return NTLSTATUS_SUCCESS;
}
}
return NTLSTATUS_NOT_FOUND;
}
static KSTATUS KiFileCacheReclaimOnePage(FILE_OBJECT* skipFile, uint64_t skipOffset) {
KSTATUS s = KiFileCacheDropOneStandbyPage(skipFile, skipOffset);
if (KSUCCESS(s))
return NTLSTATUS_SUCCESS;
s = KiFileCacheDemoteOnePage(skipFile, skipOffset);
if (KSUCCESS(s)) {
s = KiFileCacheDropOneStandbyPage(skipFile, skipOffset);
if (KSUCCESS(s))
return NTLSTATUS_SUCCESS;
}
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
gFileCacheStats.DropFailures++;
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_NOT_FOUND;
}
// Adds to one file-cache statistic under the cache lock.
static void KiFileCacheAddStat(uint64_t* field, uint64_t value) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
*field += value;
KeReleaseSpinlock(&gFileCacheLock, flags);
}
// Wakes the writeback worker after dirty pressure crosses the high watermark.
static void KiFileCacheSignalWriteback(void) {
if (gWritebackSemaphore)
KeSignalSemaphore(gWritebackSemaphore);
}
// Finds the lowest dirty page for a file, or globally when file is NULL.
static MM_FILE_CACHE_ENTRY* KiFileCacheFindLowestDirtyLocked(FILE_OBJECT* file) {
MM_FILE_CACHE_ENTRY* best = (MM_FILE_CACHE_ENTRY*)0;
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; e = e->Next) {
if (!e->Dirty || e->Writeback || e->Loading || e->Invalidated || !e->Frame)
continue;
if (file && e->File != file)
continue;
if (!best || e->Offset < best->Offset)
best = e;
}
}
return best;
}
// Flushes the next dirty cache page for file, or globally when file is NULL.
static KSTATUS KiFileCacheFlushOneDirtyPage(FILE_OBJECT* file, int* didWork) {
if (didWork) *didWork = 0;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* e = KiFileCacheFindLowestDirtyLocked(file);
if (!e) {
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_NOT_FOUND;
}
KSTATUS rs = MmPfnRetain(e->Frame);
if (!KSUCCESS(rs)) {
gFileCacheStats.WritebackFailures++;
KeReleaseSpinlock(&gFileCacheLock, flags);
return rs;
}
FILE_OBJECT* target = e->File;
ObReferenceObject(&target->Header);
PHYSICAL_ADDRESS frame = e->Frame;
uint64_t offset = e->Offset;
uint32_t validBytes = e->ValidBytes ? e->ValidBytes : PAGE_SIZE;
uint64_t sequence = e->DirtySequence;
e->Writeback = 1;
gFileCacheStats.WritebackRequests++;
KeReleaseSpinlock(&gFileCacheLock, flags);
void* page = MmPfnToVirtual(frame);
size_t written = 0;
KSTATUS s = NTLSTATUS_FAILURE;
if (page)
s = IoVolumeWrite(target->Volume, target->FileCtx, offset, page, validBytes, &written);
if (KSUCCESS(s) && written != validBytes)
s = NTLSTATUS_FAILURE;
MM_FILE_CACHE_ENTRY* invalidatedVictim = (MM_FILE_CACHE_ENTRY*)0;
flags = KeAcquireSpinlock(&gFileCacheLock);
uint32_t bucket = KiFileCacheHash(target, offset);
MM_FILE_CACHE_ENTRY* cur = KiFileCacheFindLocked(target, offset, bucket);
if (cur && cur->Frame == frame) {
cur->Writeback = 0;
if (cur->Invalidated) {
if (cur->Dirty)
KiFileCacheSetDirtyLocked(cur, 0);
if (KiFileCacheUnlinkEntryLocked(cur))
invalidatedVictim = cur;
} else if (KSUCCESS(s) && cur->Dirty && cur->DirtySequence == sequence) {
KiFileCacheSetDirtyLocked(cur, 0);
gFileCacheStats.WritebackPages++;
gFileCacheStats.WritebackBytes += validBytes;
} else if (!KSUCCESS(s)) {
gFileCacheStats.WritebackFailures++;
}
}
KeReleaseSpinlock(&gFileCacheLock, flags);
(void)MmPfnRelease(frame);
if (invalidatedVictim)
KiFileCacheFreeEntry(invalidatedVictim);
ObDereferenceObject(&target->Header);
if (didWork) *didWork = 1;
return s;
}
// Background writeback worker drains dirty cache pressure best-effort.
static void KiFileCacheWritebackWorker(void* arg) {
(void)arg;
KeAssertIrql(PASSIVE_LEVEL);
while (true) {
KSTATUS ws = KeWaitSemaphore(gWritebackSemaphore, KE_WAIT_INFINITE);
if (!KSUCCESS(ws)) {
KeYield();
continue;
}
for (uint32_t i = 0; i < MM_FILE_CACHE_WRITEBACK_BATCH_PAGES; ++i) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
int overHigh = gFileCacheDirtyPages >= MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (!overHigh)
break;
int didWork = 0;
KSTATUS s = KiFileCacheFlushOneDirtyPage((FILE_OBJECT*)0, &didWork);
if (!didWork || !KSUCCESS(s))
break;
}
}
}
// Starts the writeback worker once.
static KSTATUS KiFileCacheEnsureWritebackStarted(void) {
uintptr_t flags = KeAcquireSpinlock(&gWritebackLock);
if (gWritebackStartState == 2) {
KeReleaseSpinlock(&gWritebackLock, flags);
return NTLSTATUS_SUCCESS;
}
if (gWritebackStartState == 1) {
KeReleaseSpinlock(&gWritebackLock, flags);
while (gWritebackStartState == 1)
KeYield();
return gWritebackStartState == 2 ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
}
gWritebackStartState = 1;
KeReleaseSpinlock(&gWritebackLock, flags);
KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gWritebackSemaphore);
if (KSUCCESS(s)) {
THREAD* worker = (THREAD*)0;
const KE_THREAD_CREATE_PARAMETERS parameters =
KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "mm.writeback");
s = KeCreateKernelThread((uintptr_t)KiFileCacheWritebackWorker,
0,
&parameters,
&worker);
(void)worker;
}
flags = KeAcquireSpinlock(&gWritebackLock);
gWritebackStartState = KSUCCESS(s) ? 2 : 0;
KeReleaseSpinlock(&gWritebackLock, flags);
return s;
}
static KSTATUS KiFileCacheTryHit(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_PAGE* outPage, int waitForLoading) {
if (outPage)
memset(outPage, 0, sizeof(*outPage));
uint32_t bucket = KiFileCacheHash(file, aligned);
int counted = 0;
while (true) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
if (!counted) {
gFileCacheStats.Lookups++;
counted = 1;
}
MM_FILE_CACHE_ENTRY* hit = KiFileCacheFindLocked(file, aligned, bucket);
if (!hit) {
gFileCacheStats.Misses++;
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_NOT_FOUND;
}
if (hit->Loading) {
if (!waitForLoading) {
gFileCacheStats.Misses++;
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_PENDING;
}
gFileCacheStats.InflightWaits++;
KeReleaseSpinlock(&gFileCacheLock, flags);
KeYield();
continue;
}
if (hit->Invalidated || !hit->Frame) {
gFileCacheStats.Misses++;
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_NOT_FOUND;
}
if (hit->Standby) {
KSTATUS as = MmPfnActivate(hit->Frame);
if (KSUCCESS(as))
KiFileCacheActivateEntryLocked(hit);
}
KSTATUS rs = NTLSTATUS_SUCCESS;
if (outPage)
rs = MmPfnRetain(hit->Frame);
if (KSUCCESS(rs)) {
hit->Referenced = 1;
gFileCacheStats.Hits++;
if (outPage) {
outPage->File = file;
outPage->Offset = aligned;
outPage->Frame = hit->Frame;
outPage->ValidBytes = hit->ValidBytes;
}
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_SUCCESS;
}
// A retained cache entry should never point at an invalid frame, but do not
// return corrupt data if the PFN layer rejects it.
gFileCacheStats.Misses++;
KeReleaseSpinlock(&gFileCacheLock, flags);
return rs;
}
}
static void KiFileCacheKeepBounded(FILE_OBJECT* skipFile, uint64_t skipOffset) {
while (true) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
int over = gFileCacheResidentPages > MM_FILE_CACHE_SOFT_MAX_PAGES;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (!over) break;
if (!KSUCCESS(KiFileCacheReclaimOnePage(skipFile, skipOffset))) break;
}
}
static MM_FILE_CACHE_ENTRY* KiFileCacheAllocateLoadingEntry(FILE_OBJECT* file, uint64_t aligned) {
MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)MmAllocatePoolTag(sizeof(MM_FILE_CACHE_ENTRY), MM_POOL_TAG('p', 'c', 'a', 'c'));
if (!entry)
return (MM_FILE_CACHE_ENTRY*)0;
memset(entry, 0, sizeof(*entry));
entry->File = file;
entry->Offset = aligned;
entry->Referenced = 1;
entry->Loading = 1;
ObReferenceObject(&file->Header);
return entry;
}
static void KiFileCacheFreeLoadingEntry(MM_FILE_CACHE_ENTRY* entry) {
if (!entry)
return;
entry->Frame = 0;
entry->Loading = 0;
KiFileCacheFreeEntry(entry);
}
static bool KiFileCacheUnlinkEntryLocked(MM_FILE_CACHE_ENTRY* entry) {
if (!entry)
return false;
uint32_t bucket = KiFileCacheHash(entry->File, entry->Offset);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[bucket]; e; prev = e, e = e->Next) {
if (e == entry) {
KiFileCacheRemoveEntryLocked(bucket, prev, e);
return true;
}
}
return false;
}
static KSTATUS KiFileCacheReserveLoadingPage(FILE_OBJECT* file, uint64_t aligned, MM_FILE_CACHE_ENTRY** outEntry) {
if (outEntry)
*outEntry = (MM_FILE_CACHE_ENTRY*)0;
MM_FILE_CACHE_ENTRY* entry = KiFileCacheAllocateLoadingEntry(file, aligned);
if (!entry)
return NTLSTATUS_NO_MEMORY;
uint32_t bucket = KiFileCacheHash(file, aligned);
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
if (KiFileCacheFindLocked(file, aligned, bucket)) {
KeReleaseSpinlock(&gFileCacheLock, flags);
KiFileCacheFreeLoadingEntry(entry);
return NTLSTATUS_ALREADY_EXISTS;
}
entry->Next = gFileCacheBuckets[bucket];
gFileCacheBuckets[bucket] = entry;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (outEntry)
*outEntry = entry;
return NTLSTATUS_SUCCESS;
}
static void KiFileCacheCancelLoadingPage(MM_FILE_CACHE_ENTRY* entry) {
if (!entry)
return;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
bool unlinked = KiFileCacheUnlinkEntryLocked(entry);
KeReleaseSpinlock(&gFileCacheLock, flags);
if (unlinked)
KiFileCacheFreeLoadingEntry(entry);
}
static KSTATUS KiFileCacheCompleteLoadingPage(MM_FILE_CACHE_ENTRY* entry,
PHYSICAL_ADDRESS frame,
uint32_t validBytes,
MM_FILE_CACHE_PAGE* requestedPage,
uint64_t* publishedCount) {
if (!entry || !frame)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
if (entry->Invalidated) {
(void)KiFileCacheUnlinkEntryLocked(entry);
KeReleaseSpinlock(&gFileCacheLock, flags);
(void)MmPfnRelease(frame);
KiFileCacheFreeLoadingEntry(entry);
return NTLSTATUS_NOT_FOUND;
}
entry->Frame = frame;
entry->ValidBytes = validBytes;
entry->Referenced = 1;
entry->Standby = 0;
entry->Loading = 0;
gFileCacheStats.Loads++;
KiFileCacheBumpResidentLocked(1, 0);
if (publishedCount)
(*publishedCount)++;
if (requestedPage) {
KSTATUS rs = MmPfnRetain(frame);
if (!KSUCCESS(rs)) {
(void)KiFileCacheUnlinkEntryLocked(entry);
gFileCacheStats.LoadFailures++;
KeReleaseSpinlock(&gFileCacheLock, flags);
KiFileCacheFreeEntry(entry);
return rs;
}
requestedPage->File = entry->File;
requestedPage->Offset = entry->Offset;
requestedPage->Frame = frame;
requestedPage->ValidBytes = validBytes;
}
KeReleaseSpinlock(&gFileCacheLock, flags);
return NTLSTATUS_SUCCESS;
}
// Loads exactly one demanded file page into the coherent cache.
static KSTATUS KiFileCacheLoadPage(FILE_OBJECT* file,
uint64_t aligned,
MM_FILE_CACHE_PAGE* outPage) {
if (!file || !outPage)
return NTLSTATUS_INVALID_PARAMETER;
uintptr_t fileFlags = KeAcquireSpinlock(&file->Lock);
uint64_t fileSize = file->Size;
KeReleaseSpinlock(&file->Lock, fileFlags);
if (aligned >= fileSize)
return NTLSTATUS_SUCCESS;
MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)0;
KSTATUS s = KiFileCacheReserveLoadingPage(file, aligned, &entry);
if (s == NTLSTATUS_ALREADY_EXISTS)
return KiFileCacheTryHit(file, aligned, outPage, 1);
if (!KSUCCESS(s))
return s;
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseCleanFileCache);
if (!frame) {
KiFileCacheCancelLoadingPage(entry);
KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
return NTLSTATUS_NO_MEMORY;
}
void* page = MmPfnToVirtual(frame);
if (!page) {
(void)MmPfnRelease(frame);
KiFileCacheCancelLoadingPage(entry);
KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
return NTLSTATUS_FAILURE;
}
memset(page, 0, PAGE_SIZE);
size_t toRead = PAGE_SIZE;
uint64_t remaining = fileSize - aligned;
if (remaining < PAGE_SIZE)
toRead = (size_t)remaining;
size_t got = 0;
s = IoVolumeRead(file->Volume, file->FileCtx, aligned, page, toRead, &got);
if (!KSUCCESS(s)) {
(void)MmPfnRelease(frame);
KiFileCacheCancelLoadingPage(entry);
KiFileCacheAddStat(&gFileCacheStats.LoadFailures, 1);
return s;
}
if (got > toRead)
got = toRead;
if (got < PAGE_SIZE)
memset((uint8_t*)page + got, 0, PAGE_SIZE - got);
s = KiFileCacheCompleteLoadingPage(entry, frame, (uint32_t)got, outPage, (uint64_t*)0);
if (!KSUCCESS(s))
return s;
KiFileCacheKeepBounded(file, aligned);
return NTLSTATUS_SUCCESS;
}
// Returns a retained cache page suitable for in-place buffered writes.
static KSTATUS KiFileCacheGetWritablePage(FILE_OBJECT* file,
uint64_t aligned,
uint64_t oldFileSize,
bool needExistingBytes,
MM_FILE_CACHE_PAGE* outPage) {
if (!file || !outPage)
return NTLSTATUS_INVALID_PARAMETER;
memset(outPage, 0, sizeof(*outPage));
KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1);
if (KSUCCESS(s))
return NTLSTATUS_SUCCESS;
if (needExistingBytes && aligned < oldFileSize) {
s = KiFileCacheLoadPage(file, aligned, outPage);
if (KSUCCESS(s))
return NTLSTATUS_SUCCESS;
if (s != NTLSTATUS_NOT_FOUND)
return s;
}
MM_FILE_CACHE_ENTRY* entry = (MM_FILE_CACHE_ENTRY*)0;
s = KiFileCacheReserveLoadingPage(file, aligned, &entry);
if (s == NTLSTATUS_ALREADY_EXISTS)
return KiFileCacheTryHit(file, aligned, outPage, 1);
if (!KSUCCESS(s))
return s;
PHYSICAL_ADDRESS frame = MmPfnAllocate(MmPfnUseCleanFileCache);
if (!frame) {
KiFileCacheCancelLoadingPage(entry);
return NTLSTATUS_NO_MEMORY;
}
void* page = MmPfnToVirtual(frame);
if (!page) {
(void)MmPfnRelease(frame);
KiFileCacheCancelLoadingPage(entry);
return NTLSTATUS_FAILURE;
}
memset(page, 0, PAGE_SIZE);
uint32_t validBytes = 0;
if (oldFileSize > aligned) {
uint64_t oldBytes = oldFileSize - aligned;
validBytes = oldBytes < PAGE_SIZE ? (uint32_t)oldBytes : PAGE_SIZE;
}
s = KiFileCacheCompleteLoadingPage(entry, frame, validBytes, outPage, (uint64_t*)0);
if (!KSUCCESS(s))
return s;
return NTLSTATUS_SUCCESS;
}
// Marks one cache page dirty and updates its valid byte range.
static void KiFileCacheMarkPageDirty(FILE_OBJECT* file, uint64_t aligned, uint32_t validBytes) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
uint32_t bucket = KiFileCacheHash(file, aligned);
MM_FILE_CACHE_ENTRY* e = KiFileCacheFindLocked(file, aligned, bucket);
if (e) {
if (validBytes > e->ValidBytes)
e->ValidBytes = validBytes;
if (e->Standby)
KiFileCacheActivateEntryLocked(e);
KiFileCacheSetDirtyLocked(e, 1);
}
uint64_t dirty = gFileCacheDirtyPages;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (dirty >= MM_FILE_CACHE_DIRTY_HIGH_WATER_PAGES) {
if (KSUCCESS(KiFileCacheEnsureWritebackStarted()))
KiFileCacheSignalWriteback();
}
}
KSTATUS MmFileCacheGetWritablePage(FILE_OBJECT* file, uint64_t offset,
uint64_t fileSizeBefore, bool needExistingBytes,
MM_FILE_CACHE_PAGE* outPage) {
if (!outPage)
return NTLSTATUS_INVALID_PARAMETER;
uint64_t aligned = offset & ~(uint64_t)(PAGE_SIZE - 1);
return KiFileCacheGetWritablePage(file, aligned, fileSizeBefore, needExistingBytes, outPage);
}
void MmFileCacheMarkDirty(FILE_OBJECT* file, uint64_t offset, uint32_t validBytes) {
uint64_t aligned = offset & ~(uint64_t)(PAGE_SIZE - 1);
if (validBytes > PAGE_SIZE)
validBytes = PAGE_SIZE;
if (validBytes == 0)
validBytes = PAGE_SIZE;
KiFileCacheMarkPageDirty(file, aligned, validBytes);
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
gFileCacheStats.MappedDirtySyncs++;
gFileCacheStats.MappedDirtyPages++;
KeReleaseSpinlock(&gFileCacheLock, flags);
}
KSTATUS MmFileCacheWrite(FILE_OBJECT* file, uint64_t offset, const void* buf,
size_t size, uint64_t fileSizeBefore, size_t* writtenOut) {
if (writtenOut) *writtenOut = 0;
if (!file || (!buf && size) || !writtenOut)
return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait())
return NTLSTATUS_INVALID_IRQL;
if (!IoVolumeIsMounted(file->Volume))
return NTLSTATUS_OBJECT_DEAD;
if (size == 0)
return NTLSTATUS_SUCCESS;
size_t total = 0;
const uint8_t* src = (const uint8_t*)buf;
while (total < size) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
uint64_t dirty = gFileCacheDirtyPages;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (dirty >= MM_FILE_CACHE_DIRTY_HARD_LIMIT_PAGES) {
KSTATUS fs = MmFileCacheFlushFile(file);
if (!KSUCCESS(fs) && total == 0)
return fs;
}
uint64_t absolute = offset + total;
if (absolute < offset)
break;
uint64_t aligned = absolute & ~(uint64_t)(PAGE_SIZE - 1);
size_t inPage = (size_t)(absolute - aligned);
size_t chunk = PAGE_SIZE - inPage;
if (chunk > size - total)
chunk = size - total;
int coversFullPage = (inPage == 0 && chunk == PAGE_SIZE);
int needExisting = !coversFullPage;
MM_FILE_CACHE_PAGE page;
KSTATUS s = KiFileCacheGetWritablePage(file, aligned, fileSizeBefore, needExisting, &page);
if (!KSUCCESS(s))
return total ? NTLSTATUS_SUCCESS : s;
void* dst = MmPfnToVirtual(page.Frame);
if (!dst) {
MmFileCacheReleasePage(&page);
return total ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
}
memcpy((uint8_t*)dst + inPage, src + total, chunk);
uint32_t valid = (uint32_t)(inPage + chunk);
uint64_t pageEnd = aligned + valid;
uint64_t newEnd = offset + total + chunk;
if (pageEnd > newEnd)
valid = (uint32_t)(newEnd - aligned);
KiFileCacheMarkPageDirty(file, aligned, valid);
MmFileCacheReleasePage(&page);
total += chunk;
}
*writtenOut = total;
return NTLSTATUS_SUCCESS;
}
KSTATUS MmFileCacheFlushAll(void) {
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
for (uint32_t i = 0; i < 65536u; ++i) {
int didWork = 0;
KSTATUS st = KiFileCacheFlushOneDirtyPage((FILE_OBJECT*)0, &didWork);
if (st == NTLSTATUS_NOT_FOUND && !didWork) {
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
bool clean = gFileCacheDirtyPages == 0;
KeReleaseSpinlock(&gFileCacheLock, flags);
if (clean) return NTLSTATUS_SUCCESS;
KeYield();
continue;
}
if (!KSUCCESS(st)) return st;
if (!didWork) return NTLSTATUS_SUCCESS;
}
return NTLSTATUS_LIMIT_REACHED;
}
KSTATUS MmFileCacheFlushFile(FILE_OBJECT* file) {
if (!file)
return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait())
return NTLSTATUS_INVALID_IRQL;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
gFileCacheStats.FlushFileCalls++;
KeReleaseSpinlock(&gFileCacheLock, flags);
KSTATUS firstError = NTLSTATUS_SUCCESS;
while (true) {
int didWork = 0;
KSTATUS s = KiFileCacheFlushOneDirtyPage(file, &didWork);
if (!didWork)
break;
if (!KSUCCESS(s) && KSUCCESS(firstError))
firstError = s;
}
if (!KSUCCESS(firstError)) {
flags = KeAcquireSpinlock(&gFileCacheLock);
gFileCacheStats.FlushFileFailures++;
KeReleaseSpinlock(&gFileCacheLock, flags);
}
return firstError;
}
KSTATUS MmFileCacheGetPage(FILE_OBJECT* file, uint64_t offset, MM_FILE_CACHE_PAGE* outPage) {
if (!outPage) return NTLSTATUS_INVALID_PARAMETER;
memset(outPage, 0, sizeof(*outPage));
if (!file) return NTLSTATUS_INVALID_PARAMETER;
if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
uint64_t aligned = offset & ~(uint64_t)(PAGE_SIZE - 1);
KSTATUS s = KiFileCacheTryHit(file, aligned, outPage, 1);
if (KSUCCESS(s))
return NTLSTATUS_SUCCESS;
return KiFileCacheLoadPage(file, aligned, outPage);
}
void MmFileCacheReleasePage(MM_FILE_CACHE_PAGE* page) {
if (!page || !page->Frame) return;
(void)MmPfnRelease(page->Frame);
memset(page, 0, sizeof(*page));
}
void MmFileCacheInvalidate(FILE_OBJECT* file, uint64_t offset, size_t size) {
if (!file) return;
if (size == 0) return;
uint64_t start = offset & ~(uint64_t)(PAGE_SIZE - 1);
uint64_t end = offset + size;
if (end < offset || size == (size_t)-1) {
end = UINT64_MAX;
} else {
end = (end + PAGE_SIZE - 1) & ~(uint64_t)(PAGE_SIZE - 1);
}
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
while (true) {
MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) {
uint64_t pageEnd = e->Offset + PAGE_SIZE;
if (e->File == file && e->Offset < end && pageEnd > start) {
if (e->Loading) {
if (!e->Invalidated) {
e->Invalidated = 1;
gFileCacheStats.Invalidations++;
}
continue;
}
if (e->Dirty || e->Writeback) {
e->Invalidated = 1;
continue;
}
victim = e;
KiFileCacheRemoveEntryLocked(b, prev, e);
gFileCacheStats.Invalidations++;
break;
}
}
KeReleaseSpinlock(&gFileCacheLock, flags);
if (!victim) break;
KiFileCacheFreeEntry(victim);
}
}
}
void MmFileCacheInvalidateAll(FILE_OBJECT* file) {
if (!file) return;
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
while (true) {
MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) {
if (e->File == file) {
if (e->Loading) {
if (!e->Invalidated) {
e->Invalidated = 1;
gFileCacheStats.Invalidations++;
}
continue;
}
if (e->Dirty || e->Writeback) {
e->Invalidated = 1;
continue;
}
victim = e;
KiFileCacheRemoveEntryLocked(b, prev, e);
gFileCacheStats.Invalidations++;
break;
}
}
KeReleaseSpinlock(&gFileCacheLock, flags);
if (!victim) break;
KiFileCacheFreeEntry(victim);
}
}
}
// Discards all cache state owned by one dead volume without attempting writeback.
void MmFileCacheDiscardVolume(IO_VOLUME* volume) {
if (!volume) return;
for (uint32_t b = 0; b < MM_FILE_CACHE_BUCKETS; ++b) {
while (true) {
MM_FILE_CACHE_ENTRY* victim = (MM_FILE_CACHE_ENTRY*)0;
bool pending = false;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_ENTRY* prev = (MM_FILE_CACHE_ENTRY*)0;
for (MM_FILE_CACHE_ENTRY* e = gFileCacheBuckets[b]; e; prev = e, e = e->Next) {
if (!e->File || e->File->Volume != volume)
continue;
if (e->Loading || e->Writeback) {
e->Invalidated = 1;
if (e->Dirty)
KiFileCacheSetDirtyLocked(e, 0);
pending = true;
continue;
}
victim = e;
KiFileCacheRemoveEntryLocked(b, prev, e);
gFileCacheStats.Invalidations++;
break;
}
KeReleaseSpinlock(&gFileCacheLock, flags);
if (victim) {
KiFileCacheFreeEntry(victim);
continue;
}
if (!pending)
break;
// Loading/writeback entries unlink themselves when their in-flight operation completes.
break;
}
}
}
KSTATUS MmFileCacheDropOnePageForPressure(void) {
return KiFileCacheReclaimOnePage((FILE_OBJECT*)0, 0);
}
void MmFileCacheQueryStats(MM_FILE_CACHE_STATS* out) {
if (!out) return;
uintptr_t flags = KeAcquireSpinlock(&gFileCacheLock);
MM_FILE_CACHE_STATS st = gFileCacheStats;
st.ResidentPages = gFileCacheResidentPages;
st.ActivePages = gFileCacheActivePages;
st.StandbyPages = gFileCacheStandbyPages;
st.DirtyPages = gFileCacheDirtyPages;
st.MaxDirtyPages = gFileCacheMaxDirtyPages;
st.MaxResidentPages = gFileCacheMaxResidentPages;
KeReleaseSpinlock(&gFileCacheLock, flags);
*out = st;
}
Reference in new issue View Git Blame Copy Permalink