add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

This commit is contained in:
maksw committed 2026-06-20 13:17:39 +02:00
1 parent 61ec2442eb
commit 2c4a68a75f
114 files changed
+18069 -2620

No files matched your search

+563 -79
View File
@@ -128,12 +128,79 @@ typedef struct {
static FAT_CACHE_ENTRY gFatCache[FAT_CACHE_SLOTS];
#define FAT32_PATH_CACHE_SLOTS 64u
#define FAT32_PATH_CACHE_MAX 512u
#define FAT32_CHAIN_CACHE_SLOTS 16u
#define FAT32_WRITE_RUN_MAX_SECTORS 128u
#define FAT32_READ_RUN_MAX_SECTORS 512u
typedef struct FAT32_PATH_CACHE_ENTRY {
bool Valid;
uint64_t Generation;
uint32_t Hash;
size_t PathLength;
char Path[FAT32_PATH_CACHE_MAX];
uint32_t Cluster;
uint32_t Size;
uint8_t Attr;
uint64_t DirentLba;
uint32_t DirentOffset;
} FAT32_PATH_CACHE_ENTRY;
typedef struct FAT32_CHAIN_CACHE_ENTRY {
bool Valid;
uint32_t LogicalStart;
uint32_t PhysicalStart;
uint32_t Length;
uint32_t NextCluster;
} FAT32_CHAIN_CACHE_ENTRY;
// The FAT sector cache is global and direct-mapped. mmap-backed video can fault
// file pages from multiple Bink worker threads at the same time; without
// serialization one thread can evict/reload a cache slot while another thread is
// still reading a FAT entry from that slot, corrupting cluster-chain traversal.
// Use a sleepable mutex rather than a spinlock because cache misses perform disk
// I/O. The static mutex avoids allocating an object from the paging path.
static MUTEX gFatCacheMutex;
static SPINLOCK gFatCacheMutexInitLock = SPINLOCK_INIT;
static int gFatCacheMutexReady;
static void FatCacheEnsureMutex(void) {
if (gFatCacheMutexReady)
return;
uintptr_t flags = KeAcquireSpinlock(&gFatCacheMutexInitLock);
if (!gFatCacheMutexReady) {
memset(&gFatCacheMutex, 0, sizeof(gFatCacheMutex));
gFatCacheMutex.Header.Type = ObjectTypeMutex;
gFatCacheMutex.Header.RefCount = 1;
gFatCacheMutex.Header.Destroy = (void*)0;
gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
gFatCacheMutex.Owner = (struct THREAD*)0;
LIST_INIT(&gFatCacheMutex.WaitList);
gFatCacheMutexReady = 1;
}
KeReleaseSpinlock(&gFatCacheMutexInitLock, flags);
}
static int FatCacheAcquire(void) {
if (!KeCanWait())
return 0;
FatCacheEnsureMutex();
return KSUCCESS(KeAcquireMutex(&gFatCacheMutex));
}
static void FatCacheRelease(int held) {
if (held)
KeReleaseMutex(&gFatCacheMutex);
}
// Raw (non-cached) sector I/O — forward declarations.
static KSTATUS ReadSectorsRaw (FAT32_VOL *v, uint64_t lba, uint32_t count, void *buf);
static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, const void *buf);
// Read one FAT sector through the cache.
// Returns a pointer into the cache buffer (valid until the next eviction).
// Caller must hold the FAT cache mutex while using the returned pointer.
static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) {
uint32_t slot = (uint32_t)(lba % FAT_CACHE_SLOTS);
FAT_CACHE_ENTRY *e = &gFatCache[slot];
@@ -170,11 +237,13 @@ static void FatCacheMarkDirty(FAT32_VOL *v, uint64_t lba) {
// Implements the fat Cache Invalidate operation.
static void FatCacheInvalidate(void) {
int held = FatCacheAcquire();
for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
gFatCache[i].vol = (FAT32_VOL*)0;
gFatCache[i].valid = false;
gFatCache[i].dirty = false;
}
FatCacheRelease(held);
}
// FlushFatCache — write every dirty slot to all FAT copies, clear dirty flag.
@@ -196,6 +265,10 @@ struct FAT32_VOL_S {
uint64_t DataStart; // absolute LBA of cluster 2
uint32_t TotalClusters; // usable data clusters
uint32_t NextFreeHint; // fix #3: start cluster for next free-scan
SPINLOCK PathCacheLock;
uint64_t PathCacheGeneration;
IO_FILESYSTEM_STATS Stats;
FAT32_PATH_CACHE_ENTRY PathCache[FAT32_PATH_CACHE_SLOTS];
};
typedef struct {
uint32_t StartCluster;
@@ -203,6 +276,9 @@ typedef struct {
FAT32_VOL *Vol;
uint64_t DirentLba;
uint32_t DirentOffset;
SPINLOCK ChainCacheLock;
uint32_t ChainCacheClock;
FAT32_CHAIN_CACHE_ENTRY ChainCache[FAT32_CHAIN_CACHE_SLOTS];
} FAT32_FILE;
// Raw sector I/O (bypasses FAT cache; used for file data and cache eviction).
@@ -231,6 +307,7 @@ static uint64_t ClusterToLba(FAT32_VOL *v, uint32_t cluster) {
// FlushFatCache — needs full FAT32_VOL definition.
static void FlushFatCache(FAT32_VOL *v) {
uint64_t fat0Start = (uint64_t)v->PartitionLba + v->ReservedSectors;
int held = FatCacheAcquire();
for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
FAT_CACHE_ENTRY *e = &gFatCache[i];
@@ -246,6 +323,8 @@ static void FlushFatCache(FAT32_VOL *v) {
}
e->dirty = false;
}
FatCacheRelease(held);
}
// FAT read / write (through cache).
@@ -256,9 +335,14 @@ static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) {
+ fatOffset / v->BytesPerSector;
uint32_t offInSec = fatOffset % v->BytesPerSector;
int held = FatCacheAcquire();
uint8_t *sec;
if (!KSUCCESS(FatCacheRead(v, sector, &sec))) return 0;
return *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
KSTATUS s = FatCacheRead(v, sector, &sec);
uint32_t value = 0;
if (KSUCCESS(s))
value = *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
FatCacheRelease(held);
return value;
}
// Updates fat Entry.
@@ -268,19 +352,24 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) {
+ fatOffset / v->BytesPerSector;
uint32_t offInSec = fatOffset % v->BytesPerSector;
int held = FatCacheAcquire();
uint8_t *sec;
KSTATUS s = FatCacheRead(v, sector, &sec);
if (!KSUCCESS(s)) return s;
if (!KSUCCESS(s)) {
FatCacheRelease(held);
return s;
}
uint32_t existing = *(uint32_t *)(sec + offInSec);
*(uint32_t *)(sec + offInSec) = (existing & 0xF0000000u)
| (value & 0x0FFFFFFFu);
FatCacheMarkDirty(v, sector);
FatCacheRelease(held);
return STATUS_SUCCESS;
}
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan).
static uint32_t AllocCluster(FAT32_VOL *v) {
static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) {
if (v->TotalClusters == 0) return 0;
uint32_t total = v->TotalClusters;
@@ -295,12 +384,18 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
if (!KSUCCESS(WriteFatEntry(v, c, FAT32_EOC_MARK))) return 0;
v->NextFreeHint = c + 1;
// Zero the cluster (required for directory correctness).
uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (zero) {
memset(zero, 0, v->BytesPerCluster);
WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero);
MmFreePool(zero);
// Normal file growth zeroes newly allocated clusters so old
// disk contents are never exposed. Swap/pagefile creation is
// the one exception: every allocated slot is overwritten before
// it can ever be read back, so skipping these writes avoids a
// slow boot-time full-pagefile zero pass.
if (zeroCluster) {
uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (zero) {
memset(zero, 0, v->BytesPerCluster);
WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero);
MmFreePool(zero);
}
}
return c;
}
@@ -309,14 +404,23 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
return 0;
}
// Appends cluster.
static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
uint32_t newCluster = AllocCluster(v);
static uint32_t AllocCluster(FAT32_VOL *v) {
return AllocClusterEx(v, 1);
}
static uint32_t AppendClusterEx(FAT32_VOL *v, uint32_t lastCluster, int zeroCluster) {
uint32_t newCluster = AllocClusterEx(v, zeroCluster);
if (!newCluster) return 0;
if (!KSUCCESS(WriteFatEntry(v, lastCluster, newCluster))) return 0;
return newCluster;
}
// Appends cluster.
static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
return AppendClusterEx(v, lastCluster, 1);
}
// Returns chain End.
static uint32_t GetChainEnd(FAT32_VOL *v, uint32_t start, uint32_t *outCount) {
uint32_t cluster = start, prev = start, count = 0;
@@ -973,6 +1077,302 @@ static int MatchName83V(const char *dirName, const char *dirExt,
return 1;
}
// Returns the path cache canonical byte used for hashes and comparisons.
static char Fat32PathCacheChar(char c) {
if (c == '/') return '\\';
if (c >= 'A' && c <= 'Z') return (char)(c + 32);
return c;
}
// Hashes a drive-relative path using the same case/separator rules as lookup.
static uint32_t Fat32PathCacheHash(SYS_STRING_VIEW path) {
uint32_t h = 2166136261u;
for (size_t i = 0; i < path.Length; i++) {
h ^= (uint8_t)Fat32PathCacheChar(path.Data[i]);
h *= 16777619u;
}
return h ? h : 1u;
}
// Tests whether the cached path names the same FAT path as the caller's view.
static bool Fat32PathCachePathEquals(const FAT32_PATH_CACHE_ENTRY* e, SYS_STRING_VIEW path) {
if (!e || e->PathLength != path.Length) return false;
for (size_t i = 0; i < path.Length; i++) {
if (Fat32PathCacheChar(e->Path[i]) != Fat32PathCacheChar(path.Data[i]))
return false;
}
return true;
}
// Adds directory search counters without holding the lock during disk I/O.
static void Fat32PathCacheAddDirectoryStats(FAT32_VOL* v, uint64_t clusters, uint64_t entries) {
if (!v) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.DirectorySearches++;
v->Stats.DirectoryClustersRead += clusters;
v->Stats.DirectoryEntriesScanned += entries;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Looks up a positive path-cache entry for a full drive-relative path.
static bool Fat32PathCacheLookup(FAT32_VOL* v, SYS_STRING_VIEW path,
uint32_t* outCluster, uint32_t* outSize,
uint8_t* outAttr, uint64_t* outDirentLba,
uint32_t* outDirentOffset) {
if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
return false;
uint32_t hash = Fat32PathCacheHash(path);
uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
bool hit = false;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.PathLookups++;
FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
if (e->Valid && e->Generation == v->PathCacheGeneration &&
e->Hash == hash && Fat32PathCachePathEquals(e, path)) {
if (outCluster) *outCluster = e->Cluster;
if (outSize) *outSize = e->Size;
if (outAttr) *outAttr = e->Attr;
if (outDirentLba) *outDirentLba = e->DirentLba;
if (outDirentOffset) *outDirentOffset = e->DirentOffset;
v->Stats.PathCacheHits++;
hit = true;
} else {
v->Stats.PathCacheMisses++;
}
KeReleaseSpinlock(&v->PathCacheLock, flags);
return hit;
}
// Stores a positive path lookup in the volume-local path cache.
static void Fat32PathCacheStore(FAT32_VOL* v, SYS_STRING_VIEW path,
uint32_t cluster, uint32_t size, uint8_t attr,
uint64_t direntLba, uint32_t direntOffset) {
if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
return;
uint32_t hash = Fat32PathCacheHash(path);
uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
memset(e, 0, sizeof(*e));
e->Valid = true;
e->Generation = v->PathCacheGeneration;
e->Hash = hash;
e->PathLength = path.Length;
memcpy(e->Path, path.Data, path.Length);
e->Path[path.Length] = '\0';
e->Cluster = cluster;
e->Size = size;
e->Attr = attr;
e->DirentLba = direntLba;
e->DirentOffset = direntOffset;
v->Stats.PathCacheStores++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Invalidates every positive path-cache entry after a directory or size mutation.
static void Fat32PathCacheInvalidate(FAT32_VOL* v) {
if (!v) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->PathCacheGeneration++;
if (v->PathCacheGeneration == 0)
v->PathCacheGeneration = 1;
v->Stats.PathCacheInvalidations++;
v->Stats.PathCacheGeneration = v->PathCacheGeneration;
for (uint32_t i = 0; i < FAT32_PATH_CACHE_SLOTS; i++)
v->PathCache[i].Valid = false;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Returns the current filesystem cache and traversal counters.
void Fat32QueryStats(IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out) {
if (!fs || !out) return;
FAT32_VOL* v = (FAT32_VOL*)fs->DriverCtx;
if (!v) { memset(out, 0, sizeof(*out)); return; }
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
*out = v->Stats;
out->PathCacheGeneration = v->PathCacheGeneration;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Adds FAT-chain traversal counters shared by the read path helpers.
static void Fat32StatsAddChainWalk(FAT32_VOL* v, uint64_t steps) {
if (!v || !steps) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainWalkSteps += steps;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Records one data read span and its physically contiguous cluster merges.
static void Fat32StatsAddReadRun(FAT32_VOL* v, uint32_t sectors, uint32_t merges) {
if (!v || !sectors) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.DataReadRuns++;
v->Stats.DataReadSectors += sectors;
v->Stats.DataReadRunMerges += merges;
if (v->Stats.MaxDataReadSpanSectors < sectors)
v->Stats.MaxDataReadSpanSectors = sectors;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Initializes the per-open cluster-chain cache for a FAT file object.
static void Fat32FileInitChainCache(FAT32_FILE* f) {
if (!f) return;
f->ChainCacheLock = (SPINLOCK)SPINLOCK_INIT;
f->ChainCacheClock = 0;
memset(f->ChainCache, 0, sizeof(f->ChainCache));
}
// Invalidates cached FAT extents after the file's cluster chain changes.
static void Fat32FileInvalidateChainCache(FAT32_FILE* f) {
if (!f) return;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
memset(f->ChainCache, 0, sizeof(f->ChainCache));
f->ChainCacheClock = 0;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheInvalidations++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Locates the best cached starting point for a logical cluster index.
static bool Fat32ChainCacheLookupStart(FAT32_FILE* f, uint32_t logicalIndex,
uint32_t* outLogical, uint32_t* outCluster) {
if (!f || !outLogical || !outCluster)
return false;
bool hit = false;
uint32_t bestLogical = 0;
uint32_t bestCluster = 0;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
for (uint32_t i = 0; i < FAT32_CHAIN_CACHE_SLOTS; i++) {
FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[i];
if (!e->Valid || e->Length == 0)
continue;
uint32_t end = e->LogicalStart + e->Length;
if (logicalIndex >= e->LogicalStart && logicalIndex < end) {
*outLogical = logicalIndex;
*outCluster = e->PhysicalStart + (logicalIndex - e->LogicalStart);
hit = true;
break;
}
if (end <= logicalIndex && e->NextCluster >= 2 && e->NextCluster < FAT32_EOC) {
if (!hit || end > bestLogical) {
bestLogical = end;
bestCluster = e->NextCluster;
hit = true;
}
}
}
if (hit && bestCluster) {
*outLogical = bestLogical;
*outCluster = bestCluster;
}
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (v) {
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheLookups++;
if (hit) v->Stats.FatChainCacheHits++;
else v->Stats.FatChainCacheMisses++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
return hit;
}
// Stores a physically contiguous FAT extent in the per-open chain cache.
static void Fat32ChainCacheStore(FAT32_FILE* f, uint32_t logicalStart,
uint32_t physicalStart, uint32_t length,
uint32_t nextCluster) {
if (!f || physicalStart < 2 || physicalStart >= FAT32_EOC || length == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
uint32_t slot = f->ChainCacheClock++ % FAT32_CHAIN_CACHE_SLOTS;
FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[slot];
e->Valid = true;
e->LogicalStart = logicalStart;
e->PhysicalStart = physicalStart;
e->Length = length;
e->NextCluster = nextCluster;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheStores++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Resolves a logical file cluster through the cached FAT-chain extent map.
static uint32_t Fat32ResolveFileCluster(FAT32_FILE* f, uint32_t logicalIndex) {
if (!f || f->StartCluster < 2 || f->StartCluster >= FAT32_EOC)
return 0;
if (logicalIndex == 0)
return f->StartCluster;
uint32_t currentLogical = 0;
uint32_t cluster = f->StartCluster;
if (!Fat32ChainCacheLookupStart(f, logicalIndex, &currentLogical, &cluster)) {
currentLogical = 0;
cluster = f->StartCluster;
}
uint64_t steps = 0;
while (currentLogical < logicalIndex && cluster >= 2 && cluster < FAT32_EOC) {
cluster = FatEntry(f->Vol, cluster);
currentLogical++;
steps++;
}
Fat32StatsAddChainWalk(f->Vol, steps);
return cluster;
}
// Builds and caches one physically contiguous read run from a starting cluster.
static bool Fat32BuildReadRun(FAT32_FILE* f, uint32_t logicalIndex,
uint32_t startCluster, uint32_t maxSectors,
uint32_t* outRunSectors, uint32_t* outNextCluster) {
if (!f || !outRunSectors || !outNextCluster)
return false;
FAT32_VOL* v = f->Vol;
if (!v || startCluster < 2 || startCluster >= FAT32_EOC)
return false;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t runClusters = 1;
uint32_t last = startCluster;
uint32_t next = FatEntry(v, last);
uint64_t steps = 1;
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= maxSectors) {
last = next;
runClusters++;
runSectors += v->SectorsPerCluster;
next = FatEntry(v, last);
steps++;
}
Fat32StatsAddChainWalk(v, steps);
Fat32ChainCacheStore(f, logicalIndex, startCluster, runClusters, next);
*outRunSectors = runSectors;
*outNextCluster = next;
return true;
}
static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
uint32_t *outCluster, uint32_t *outSize, uint8_t *outAttr,
uint64_t *outDirentLba, uint32_t *outDirentOffset) {
@@ -986,15 +1386,20 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
int allowShortMatch = FatNameIsStructurallyShort83V(name);
Make83NameV(name, wantName83, wantExt83);
uint32_t cluster = dirCluster;
uint64_t clustersRead = 0;
uint64_t entriesScanned = 0;
int found = 0;
while (cluster >= 2 && cluster < FAT32_EOC) {
uint64_t clusterLba = ClusterToLba(v, cluster);
if (!KSUCCESS(ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf))) break;
clustersRead++;
uint32_t entries = v->BytesPerCluster / sizeof(DIRENT);
DIRENT *dir = (DIRENT *)clusterBuf;
for (uint32_t i = 0; i < entries; i++) {
entriesScanned++;
if (dir[i].Name[0] == 0x00) goto done;
if ((uint8_t)dir[i].Name[0] == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; }
if (dir[i].Attr == ATTR_LFN) { LfnAppend(lfn, (LFN_ENTRY *)&dir[i]); hasLfn = 1; continue; }
@@ -1016,15 +1421,16 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
if (outDirentLba) *outDirentLba = clusterLba + eb / v->BytesPerSector;
if (outDirentOffset) *outDirentOffset = (uint32_t)(eb % v->BytesPerSector);
}
MmFreePool(clusterBuf);
return 1;
found = 1;
goto done;
}
}
cluster = FatEntry(v, cluster);
}
done:
Fat32PathCacheAddDirectoryStats(v, clustersRead, entriesScanned);
MmFreePool(clusterBuf);
return 0;
return found;
}
static int Fat32PathNextComponent(SYS_STRING_VIEW path, size_t* cursor,
@@ -1058,6 +1464,10 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
if (outDirentLba) *outDirentLba = 0;
if (outDirentOffset) *outDirentOffset = 0;
if (Fat32PathCacheLookup(v, path, outCluster, outSize, &attr,
outDirentLba, outDirentOffset))
return STATUS_SUCCESS;
size_t cursor = 0;
SYS_STRING_VIEW component;
while (Fat32PathNextComponent(path, &cursor, &component)) {
@@ -1078,6 +1488,9 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
*outCluster = cluster;
*outSize = size;
Fat32PathCacheStore(v, path, cluster, size, attr,
outDirentLba ? *outDirentLba : 0,
outDirentOffset ? *outDirentOffset : 0);
return STATUS_SUCCESS;
}
@@ -1220,6 +1633,7 @@ KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = cluster; f->FileSize = size; f->Vol = v;
f->DirentLba = direntLba; f->DirentOffset = direntOffset;
Fat32FileInitChainCache(f);
*fileCtx = f;
return STATUS_SUCCESS;
}
@@ -1233,15 +1647,9 @@ void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
}
// Read (fix #4: contiguous-run coalescing)
// We peek ahead in the FAT chain: if cluster N+1 == cluster N physically,
// extend the current run instead of issuing a new ReadSectors call.
// Run length is capped at MAX_RUN_SECTORS to stay within the AHCI bounce
// buffer (128 sectors = 64 KB).
// FatEntry() calls in the peek loop are nearly free because they hit the
// FAT sector cache (fix #1).
#define MAX_RUN_SECTORS 128u
// Cluster-chain lookup is cached per open file, and data spans are allowed to
// grow beyond the AHCI bounce size because the device layer now splits large
// sector requests for the concrete controller.
KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
uint64_t offset, void *buf,
@@ -1257,14 +1665,12 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
if (size > (size_t)(f->FileSize - offset)) size = (size_t)(f->FileSize - offset);
size_t wanted = size;
// Allocate only as much bounce space as this read can actually use.
// The old code allocated 64 KiB and could read a 64 KiB contiguous run even
// for tiny random reads; Doom does lots of seek/read pairs for small lumps.
uint32_t initialOffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
uint64_t initialBytesCovered = (uint64_t)initialOffsetInCluster + (uint64_t)size;
uint32_t maxUsefulSectors = (uint32_t)((initialBytesCovered + 511u) / 512u);
if (maxUsefulSectors < v->SectorsPerCluster) maxUsefulSectors = v->SectorsPerCluster;
if (maxUsefulSectors > MAX_RUN_SECTORS) maxUsefulSectors = MAX_RUN_SECTORS;
if (maxUsefulSectors > FAT32_READ_RUN_MAX_SECTORS)
maxUsefulSectors = FAT32_READ_RUN_MAX_SECTORS;
uint8_t *runBuf = (uint8_t *)MmAllocatePool((size_t)maxUsefulSectors * 512u);
if (!runBuf) return STATUS_NO_MEMORY;
@@ -1272,35 +1678,27 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
size_t totalRead = 0;
uint8_t *out = (uint8_t *)buf;
KSTATUS readStatus = STATUS_SUCCESS;
uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster);
uint32_t logicalCluster = (uint32_t)(offset / v->BytesPerCluster);
uint32_t offsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
uint32_t cluster = f->StartCluster;
for (uint32_t i = 0; i < skipClusters && cluster < FAT32_EOC; i++)
cluster = FatEntry(v, cluster);
uint32_t cluster = Fat32ResolveFileCluster(f, logicalCluster);
while (size > 0 && cluster >= 2 && cluster < FAT32_EOC) {
// Build the longest physically-contiguous run that can contribute to
// this caller's remaining read. Do not prefetch an unrelated 64 KiB run
// for a tiny random read.
uint64_t bytesCovered = (uint64_t)offsetInCluster + (uint64_t)size;
uint32_t usefulSectors = (uint32_t)((bytesCovered + 511u) / 512u);
if (usefulSectors < v->SectorsPerCluster) usefulSectors = v->SectorsPerCluster;
if (usefulSectors > MAX_RUN_SECTORS) usefulSectors = MAX_RUN_SECTORS;
if (usefulSectors > FAT32_READ_RUN_MAX_SECTORS)
usefulSectors = FAT32_READ_RUN_MAX_SECTORS;
if (usefulSectors > maxUsefulSectors) usefulSectors = maxUsefulSectors;
uint32_t runStart = cluster;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t next = FatEntry(v, cluster);
while (next == cluster + 1 && next < FAT32_EOC
&& runSectors + v->SectorsPerCluster <= usefulSectors) {
cluster = next;
next = FatEntry(v, cluster);
runSectors += v->SectorsPerCluster;
uint32_t runSectors = 0;
uint32_t next = 0;
if (!Fat32BuildReadRun(f, logicalCluster, cluster, usefulSectors,
&runSectors, &next)) {
readStatus = STATUS_DEVICE_ERROR;
break;
}
KSTATUS rs = ReadSectors(v, ClusterToLba(v, runStart), runSectors, runBuf);
KSTATUS rs = ReadSectors(v, ClusterToLba(v, cluster), runSectors, runBuf);
if (!KSUCCESS(rs)) {
readStatus = rs;
break;
@@ -1309,12 +1707,16 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
uint32_t runBytes = runSectors * 512u;
size_t avail = runBytes - offsetInCluster;
size_t copy = avail < size ? avail : size;
uint32_t runClusters = runSectors / v->SectorsPerCluster;
uint32_t merges = runClusters ? runClusters - 1u : 0;
Fat32StatsAddReadRun(v, runSectors, merges);
memcpy(out, runBuf + offsetInCluster, copy);
out += copy;
totalRead += copy;
size -= copy;
offsetInCluster = 0;
logicalCluster += runClusters;
cluster = next;
}
@@ -1345,6 +1747,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
const uint8_t *src = (const uint8_t *)buf;
Fat32FileInvalidateChainCache(f);
// Ensure at least one cluster exists.
if (f->StartCluster < 2) {
@@ -1368,7 +1771,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
cluster = next;
}
uint8_t *runBuf = (uint8_t *)MmAllocatePool(MAX_RUN_SECTORS * 512u);
uint8_t *runBuf = (uint8_t *)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
if (!runBuf) { *bytesWritten = 0; return STATUS_NO_MEMORY; }
size_t written = 0, remaining = size;
@@ -1382,7 +1785,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
if (offsetInCluster == 0) {
while (next == cluster + 1 && next < FAT32_EOC
&& runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
&& runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
cluster = next;
next = FatEntry(v, cluster);
runSectors += v->SectorsPerCluster;
@@ -1439,7 +1842,9 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
u.Flags |= FAT32_DIRENT_UPDATE_SIZE;
u.Size = f->FileSize;
}
DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
KSTATUS ms = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
if (KSUCCESS(ms))
Fat32PathCacheInvalidate(v);
}
*bytesWritten = written;
@@ -1447,8 +1852,6 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
}
// Create.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
// Creates the requested object.
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
@@ -1483,7 +1886,7 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
// Create means overwrite by default for regular files. Reuse the
// existing directory entry, drop the old cluster chain, and return
// an open zero-length file instead of allocating a duplicate name.
KSTATUS ts = Fat32Truncate(fs, f, 0);
KSTATUS ts = Fat32Truncate(fs, f, 0, 0);
if (!KSUCCESS(ts)) { MmFreePool(f); return ts; }
*fileCtx = f;
return STATUS_SUCCESS;
@@ -1558,21 +1961,27 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = 0; f->FileSize = 0; f->Vol = v;
f->DirentLba = direntLba; f->DirentOffset = direntOffset;
Fat32FileInitChainCache(f);
Fat32PathCacheInvalidate(v);
*fileCtx = f;
return STATUS_SUCCESS;
}
// Truncate.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags) {
(void)fs;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
Fat32FileInvalidateChainCache(f);
int zeroNewClusters = (fileFlags & IO_FILE_FLAG_SKIP_ZERO_EXTEND) ? 0 : 1;
if ((uint32_t)newSize == f->FileSize) {
FAT32_DIRENT_UPDATE u;
memset(&u, 0, sizeof(u));
FatQueryEncodedTime(&u.Timestamp);
u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME;
return DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
KSTATUS s = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s;
}
if (newSize == 0) {
@@ -1584,6 +1993,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
f->FileSize = 0;
KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, 0);
FlushFatCache(v);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s;
}
@@ -1604,7 +2014,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
? (f->FileSize + v->BytesPerCluster - 1) / v->BytesPerCluster : 0;
uint32_t lastCluster = 0;
if (f->StartCluster < 2) {
uint32_t first = AllocCluster(v);
uint32_t first = AllocClusterEx(v, zeroNewClusters);
if (!first) return STATUS_NO_MEMORY;
f->StartCluster = first;
DirentUpdateFirstCluster(v, f->DirentLba, f->DirentOffset, first);
@@ -1613,7 +2023,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
lastCluster = GetChainEnd(v, f->StartCluster, NULL);
}
for (uint32_t i = curClusters; i < newClusters; i++) {
uint32_t next = AppendCluster(v, lastCluster);
uint32_t next = AppendClusterEx(v, lastCluster, zeroNewClusters);
if (!next) return STATUS_NO_MEMORY;
lastCluster = next;
}
@@ -1623,6 +2033,7 @@ update_size:
f->FileSize = (uint32_t)newSize;
{ KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, (uint32_t)newSize);
FlushFatCache(v);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s; }
}
@@ -1646,6 +2057,7 @@ KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (cluster >= 2) FreeClusterChain(v, cluster);
FlushFatCache(v);
Fat32PathCacheInvalidate(v);
return STATUS_SUCCESS;
}
@@ -1654,6 +2066,75 @@ uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs; return ((FAT32_FILE *)fileCtx)->FileSize;
}
// Mixes FAT directory metadata into a per-file content generation.
static uint64_t Fat32DirentGeneration(const DIRENT *d) {
if (!d)
return 0;
uint32_t firstCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
uint64_t generation = ((uint64_t)d->WrtDate << 48) |
((uint64_t)d->WrtTime << 32) |
((uint64_t)d->LastAccDate << 16) |
(uint64_t)d->Attr;
generation ^= ((uint64_t)d->CrtDate << 40);
generation ^= ((uint64_t)d->CrtTime << 24);
generation ^= ((uint64_t)d->CrtTimeTenth << 16);
generation ^= ((uint64_t)firstCluster << 1);
generation ^= ((uint64_t)d->FileSize << 17);
return generation ? generation : 1;
}
// Returns a stable FAT file identity for cross-open image-section caches.
KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out) {
(void)fs;
if (!fileCtx || !out)
return STATUS_INVALID_PARAMETER;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
if (!v)
return STATUS_INVALID_PARAMETER;
uint32_t startCluster = f->StartCluster;
uint32_t fileSize = f->FileSize;
uint64_t generation = ((uint64_t)startCluster << 1) ^
((uint64_t)fileSize << 17);
if (f->DirentLba != 0) {
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
if (!sec)
return STATUS_NO_MEMORY;
KSTATUS s = ReadSectors(v, f->DirentLba, 1, sec);
if (!KSUCCESS(s)) {
MmFreePool(sec);
return s;
}
DIRENT *d = (DIRENT *)(sec + f->DirentOffset);
if ((uint8_t)d->Name[0] == 0x00 || (uint8_t)d->Name[0] == 0xE5) {
MmFreePool(sec);
return STATUS_NOT_FOUND;
}
startCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
fileSize = d->FileSize;
generation = Fat32DirentGeneration(d);
MmFreePool(sec);
}
out->VolumeId = ((uint64_t)v->DriveIndex << 56) ^
((uint64_t)v->PartitionLba << 8) ^
((uint64_t)v->RootCluster << 1) ^
0x4641543332494445ULL;
out->FileId = f->DirentLba ^
((uint64_t)f->DirentOffset << 32) ^
((uint64_t)startCluster << 1);
out->Generation = generation ? generation : 1;
out->Size = fileSize;
return STATUS_SUCCESS;
}
// Directory enumeration (fix #2: cluster buffer cached in context)
// Previously: allocate + ReadSectors + return 1 entry + free, every call.
// Now: buffer lives in FAT32_DIR_CTX; ReadSectors fires only on cluster
@@ -1757,8 +2238,7 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
}
ctx->CurCluster = FatEntry(v, ctx->CurCluster);
ctx->EntryIndex = 0; ctx->HasLfn = 0;
memset(ctx->LfnBuf, 0, sizeof(ctx->LfnBuf));
ctx->EntryIndex = 0;
}
return STATUS_NO_MORE_FILES;
@@ -1962,6 +2442,7 @@ typedef struct FAT32_ASYNC_RW {
uint64_t Offset;
uint32_t OffsetInCluster;
uint32_t Cluster;
uint32_t LogicalCluster;
size_t Remaining;
size_t Done;
size_t Requested;
@@ -1972,6 +2453,7 @@ typedef struct FAT32_ASYNC_RW {
uint32_t CurrentNext;
uint32_t CurrentLast;
uint32_t CurrentClusters;
uint32_t CurrentOffset;
size_t CurrentCopy;
@@ -2088,7 +2570,13 @@ static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
ctx->Done += ctx->CurrentCopy;
ctx->Remaining -= ctx->CurrentCopy;
ctx->OffsetInCluster = 0;
ctx->LogicalCluster += ctx->CurrentClusters;
ctx->Cluster = ctx->CurrentNext;
if (ctx->CurrentClusters) {
uint32_t sectors = ctx->CurrentClusters * ctx->Vol->SectorsPerCluster;
uint32_t merges = ctx->CurrentClusters - 1u;
Fat32StatsAddReadRun(ctx->Vol, sectors, merges);
}
Fat32AsyncQueueContinuation(ctx);
}
@@ -2109,15 +2597,12 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
FAT32_VOL* v = ctx->Vol;
uint32_t runStart = ctx->Cluster;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t last = ctx->Cluster;
uint32_t next = FatEntry(v, last);
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
last = next;
next = FatEntry(v, last);
runSectors += v->SectorsPerCluster;
uint32_t runSectors = 0;
uint32_t next = 0;
if (!Fat32BuildReadRun(ctx->File, ctx->LogicalCluster, ctx->Cluster,
FAT32_READ_RUN_MAX_SECTORS, &runSectors, &next)) {
Fat32AsyncFinish(ctx, STATUS_DEVICE_ERROR, ctx->Done);
return;
}
uint32_t runBytes = runSectors * 512u;
@@ -2125,7 +2610,8 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
size_t copy = avail < ctx->Remaining ? avail : ctx->Remaining;
ctx->CurrentNext = next;
ctx->CurrentLast = last;
ctx->CurrentLast = runStart + (runSectors / v->SectorsPerCluster) - 1u;
ctx->CurrentClusters = runSectors / v->SectorsPerCluster;
ctx->CurrentOffset = ctx->OffsetInCluster;
ctx->CurrentCopy = copy;
@@ -2184,7 +2670,7 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
memset(ctx, 0, sizeof(*ctx));
LIST_INIT(&ctx->Link);
ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_READ_RUN_MAX_SECTORS * 512u);
if (!ctx->RunBuf) {
MmFreePool(ctx);
complete(completeContext, STATUS_NO_MEMORY, 0);
@@ -2196,17 +2682,14 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
ctx->IsWrite = 0;
ctx->Offset = offset;
ctx->OffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
ctx->Cluster = f->StartCluster;
ctx->LogicalCluster = (uint32_t)(offset / v->BytesPerCluster);
ctx->Cluster = Fat32ResolveFileCluster(f, ctx->LogicalCluster);
ctx->Remaining = size;
ctx->Requested = size;
ctx->Dst = (uint8_t*)buf;
ctx->Complete = complete;
ctx->CompleteContext = completeContext;
uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster);
for (uint32_t i = 0; i < skipClusters && ctx->Cluster < FAT32_EOC; i++)
ctx->Cluster = FatEntry(v, ctx->Cluster);
Fat32AsyncQueueContinuation(ctx);
return STATUS_PENDING;
}
@@ -2276,7 +2759,7 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
if (ctx->OffsetInCluster == 0) {
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
last = next;
next = FatEntry(v, last);
runSectors += v->SectorsPerCluster;
@@ -2347,6 +2830,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
FAT32_FILE* f = (FAT32_FILE*)fileCtx;
FAT32_VOL* v = f->Vol;
Fat32FileInvalidateChainCache(f);
if (f->StartCluster < 2) {
uint32_t first = AllocCluster(v);
@@ -2380,7 +2864,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
memset(ctx, 0, sizeof(*ctx));
LIST_INIT(&ctx->Link);
ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
if (!ctx->RunBuf) {
MmFreePool(ctx);
complete(completeContext, STATUS_NO_MEMORY, 0);