maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

Browse Source
This commit is contained in:
maksw committed 2026-06-20 13:17:39 +02:00
1 parent 61ec2442eb
commit 2c4a68a75f
114 files changed
+18069 -2620
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/drivers/bus/pci/pci.c
+24 -1
View File
@@ -13,6 +13,7 @@
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_irp.h>
#include <kernel/ke_irq.h>
#include <kernel/hal/hal_apic.h>
int _fltused; // MSVC floating-point helper anchor required by printf.obj
@@ -540,7 +541,7 @@ __declspec(dllexport) KSTATUS PciMapBar(DEVICE_OBJECT* pdo, uint8_t index, uint6
return st;
for (uint64_t off = 0; off < mapLength; off += PAGE_SIZE) {
st = MmMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmMapFailIfPresent);
st = MmPteMapPage(virtBase + (uintptr_t)off, physBase + off, flags, MmPteMapFailIfPresent);
if (!KSUCCESS(st))
return st;
}
@@ -659,6 +660,28 @@ __declspec(dllexport) KSTATUS PciEnableMsi(DEVICE_OBJECT* pdo, uint8_t irqSlot)
return STATUS_SUCCESS;
}
// Enables a single MSI vector, allocating the nt-lite IRQ slot internally.
__declspec(dllexport) KSTATUS PciEnableMsiAuto(DEVICE_OBJECT* pdo, uint8_t* outIrqSlot) {
PCI_PDO_EXTENSION* ext;
if (!outIrqSlot || !PciIsPdo(pdo, &ext))
return STATUS_INVALID_PARAMETER;
if (!PciFindCapability(ext, PCI_CAP_ID_MSI))
return STATUS_NOT_SUPPORTED;
uint8_t irq = 0xFFu;
KSTATUS st = KeIrqAllocateMsiVector(&irq);
if (!KSUCCESS(st))
return st;
st = PciEnableMsi(pdo, irq);
if (!KSUCCESS(st))
return st;
*outIrqSlot = irq;
return STATUS_SUCCESS;
}
// Implements the driver Entry operation.
KSTATUS DriverEntry(const HAL_API* hal) {
if (!hal) return STATUS_INVALID_PARAMETER;
code/drivers/bus/usb/usb.h
+1
View File
@@ -21,6 +21,7 @@
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#ifndef USB_DEBUG_VERBOSE
code/drivers/bus/usb/usb_xhci.c
+36 -20
View File
@@ -95,9 +95,9 @@ void* UsbDmaAllocPages(XHCI_CONTROLLER* xhci, size_t pages,
// hid this because it had simple low memory; laptop firmware does not.
PHYSICAL_ADDRESS phys = 0;
if (pages == 1) {
phys = MmAllocateFrame();
phys = MmPfnAllocate(MmPfnUseDma);
} else {
phys = MmAllocateContiguous(pages);
phys = MmPfnAllocateContiguous(pages, MmPfnUseDma);
}
if (!phys) {
@@ -1930,21 +1930,31 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return STATUS_DEVICE_ERROR;
}
st = PciGetInterrupt(xhci->Pdo, &xhci->Irq);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI PciGetInterrupt failed st=0x%x\n", st);
return st;
}
if (xhci->Irq == 0xFFu) {
DbgPrint("usb: xHCI PCI %02x:%02x.%x has no INTx IRQ; MSI is not in pci.sys yet\n",
xhci->Bus, xhci->Dev, xhci->Fn);
return STATUS_NOT_SUPPORTED;
xhci->Irq = 0xFFu;
xhci->UsingMsi = false;
st = PciEnableMsiAuto(xhci->Pdo, &xhci->Irq);
if (KSUCCESS(st)) {
xhci->UsingMsi = true;
DbgPrint("usb: xHCI using MSI IRQ slot %u\n", xhci->Irq);
} else {
KSTATUS intxStatus = PciGetInterrupt(xhci->Pdo, &xhci->Irq);
if (KSUCCESS(intxStatus) && xhci->Irq != 0xFFu) {
DbgPrint("usb: xHCI using legacy INTx IRQ %u\n", xhci->Irq);
} else {
xhci->Irq = 0xFFu;
DbgPrint("usb: xHCI no MSI/INTx interrupt available; using polled event-ring mode (msi=0x%x intx=0x%x)\n",
st, intxStatus);
}
}
st = PciEnableDevice(xhci->Pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX);
uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER;
if (xhci->UsingMsi || xhci->Irq == 0xFFu)
enableFlags |= PCI_DISABLE_INTX;
else
enableFlags |= PCI_ENABLE_INTX;
st = PciEnableDevice(xhci->Pdo, enableFlags);
if (!KSUCCESS(st)) {
DbgPrint("usb: xHCI PciEnableDevice failed st=0x%x\n", st);
return st;
@@ -2056,12 +2066,18 @@ KSTATUS XhciStartController(XHCI_CONTROLLER* xhci) {
return st;
}
KeIrqRegister(xhci->Irq, XhciIsr);
XhciEnableInterrupts(xhci);
xhci->InterruptsEnabled = true;
if (xhci->Irq != 0xFFu) {
KeIrqRegister(xhci->Irq, XhciIsr);
XhciEnableInterrupts(xhci);
xhci->InterruptsEnabled = true;
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
DbgPrint("usb: xHCI ISR registered on IRQ %u (%s); event ring handled by DPC\n",
xhci->Irq, xhci->UsingMsi ? "MSI" : "INTx");
} else {
xhci->InterruptsEnabled = false;
DbgPrint("usb: xHCI started in polling mode; synchronous waits drain event ring\n");
}
xhci->Fdo->PnpState = DevicePnpStarted;
KeInsertQueueDpc(&xhci->EventDpc, (void*)0, (void*)0);
DbgPrint("usb: xHCI ISR registered on IRQ %u; event ring handled by DPC\n", xhci->Irq);
st = UsbHubScheduleRootScan(&xhci->Hcd);
if (!KSUCCESS(st))
code/drivers/display/vmwaresvga/vmwaresvga.c
+1 -7
View File
@@ -890,19 +890,13 @@ static KSTATUS VmwSvgaDisplayPresent(DISPLAY_DEVICE* device, const DISPLAY_RECT*
result = syncSt;
}
if (v->HardwareCursor && v->CursorVisible)
VmwSvgaCursorCommit(v);
return result;
}
// Implements the display Sync operation.
static KSTATUS VmwSvgaDisplaySync(DISPLAY_DEVICE* device) {
VMWSVGA_DEVICE* v = VmwSvgaFromDisplay(device);
KSTATUS st = VmwSvgaSyncDevice(v);
if (v && v->HardwareCursor && v->CursorVisible)
VmwSvgaCursorCommit(v);
return st;
return VmwSvgaSyncDevice(v);
}
// Implements the display Set Cursor Image operation.
code/drivers/drivers.ini
+1
View File
@@ -1,6 +1,7 @@
[boot]
pci.sys
ps2.sys
serial.sys
ahci.sys
ata.sys
usb.sys
code/drivers/fs/fat32/fat32.c
+563 -79
View File
@@ -128,12 +128,79 @@ typedef struct {
static FAT_CACHE_ENTRY gFatCache[FAT_CACHE_SLOTS];
#define FAT32_PATH_CACHE_SLOTS 64u
#define FAT32_PATH_CACHE_MAX 512u
#define FAT32_CHAIN_CACHE_SLOTS 16u
#define FAT32_WRITE_RUN_MAX_SECTORS 128u
#define FAT32_READ_RUN_MAX_SECTORS 512u
typedef struct FAT32_PATH_CACHE_ENTRY {
bool Valid;
uint64_t Generation;
uint32_t Hash;
size_t PathLength;
char Path[FAT32_PATH_CACHE_MAX];
uint32_t Cluster;
uint32_t Size;
uint8_t Attr;
uint64_t DirentLba;
uint32_t DirentOffset;
} FAT32_PATH_CACHE_ENTRY;
typedef struct FAT32_CHAIN_CACHE_ENTRY {
bool Valid;
uint32_t LogicalStart;
uint32_t PhysicalStart;
uint32_t Length;
uint32_t NextCluster;
} FAT32_CHAIN_CACHE_ENTRY;
// The FAT sector cache is global and direct-mapped. mmap-backed video can fault
// file pages from multiple Bink worker threads at the same time; without
// serialization one thread can evict/reload a cache slot while another thread is
// still reading a FAT entry from that slot, corrupting cluster-chain traversal.
// Use a sleepable mutex rather than a spinlock because cache misses perform disk
// I/O. The static mutex avoids allocating an object from the paging path.
static MUTEX gFatCacheMutex;
static SPINLOCK gFatCacheMutexInitLock = SPINLOCK_INIT;
static int gFatCacheMutexReady;
static void FatCacheEnsureMutex(void) {
if (gFatCacheMutexReady)
return;
uintptr_t flags = KeAcquireSpinlock(&gFatCacheMutexInitLock);
if (!gFatCacheMutexReady) {
memset(&gFatCacheMutex, 0, sizeof(gFatCacheMutex));
gFatCacheMutex.Header.Type = ObjectTypeMutex;
gFatCacheMutex.Header.RefCount = 1;
gFatCacheMutex.Header.Destroy = (void*)0;
gFatCacheMutex.Lock = (SPINLOCK)SPINLOCK_INIT;
gFatCacheMutex.Owner = (struct THREAD*)0;
LIST_INIT(&gFatCacheMutex.WaitList);
gFatCacheMutexReady = 1;
}
KeReleaseSpinlock(&gFatCacheMutexInitLock, flags);
}
static int FatCacheAcquire(void) {
if (!KeCanWait())
return 0;
FatCacheEnsureMutex();
return KSUCCESS(KeAcquireMutex(&gFatCacheMutex));
}
static void FatCacheRelease(int held) {
if (held)
KeReleaseMutex(&gFatCacheMutex);
}
// Raw (non-cached) sector I/O — forward declarations.
static KSTATUS ReadSectorsRaw (FAT32_VOL *v, uint64_t lba, uint32_t count, void *buf);
static KSTATUS WriteSectorsRaw(FAT32_VOL *v, uint64_t lba, uint32_t count, const void *buf);
// Read one FAT sector through the cache.
// Returns a pointer into the cache buffer (valid until the next eviction).
// Caller must hold the FAT cache mutex while using the returned pointer.
static KSTATUS FatCacheRead(FAT32_VOL *v, uint64_t lba, uint8_t **out) {
uint32_t slot = (uint32_t)(lba % FAT_CACHE_SLOTS);
FAT_CACHE_ENTRY *e = &gFatCache[slot];
@@ -170,11 +237,13 @@ static void FatCacheMarkDirty(FAT32_VOL *v, uint64_t lba) {
// Implements the fat Cache Invalidate operation.
static void FatCacheInvalidate(void) {
int held = FatCacheAcquire();
for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
gFatCache[i].vol = (FAT32_VOL*)0;
gFatCache[i].valid = false;
gFatCache[i].dirty = false;
}
FatCacheRelease(held);
}
// FlushFatCache — write every dirty slot to all FAT copies, clear dirty flag.
@@ -196,6 +265,10 @@ struct FAT32_VOL_S {
uint64_t DataStart; // absolute LBA of cluster 2
uint32_t TotalClusters; // usable data clusters
uint32_t NextFreeHint; // fix #3: start cluster for next free-scan
SPINLOCK PathCacheLock;
uint64_t PathCacheGeneration;
IO_FILESYSTEM_STATS Stats;
FAT32_PATH_CACHE_ENTRY PathCache[FAT32_PATH_CACHE_SLOTS];
};
typedef struct {
uint32_t StartCluster;
@@ -203,6 +276,9 @@ typedef struct {
FAT32_VOL *Vol;
uint64_t DirentLba;
uint32_t DirentOffset;
SPINLOCK ChainCacheLock;
uint32_t ChainCacheClock;
FAT32_CHAIN_CACHE_ENTRY ChainCache[FAT32_CHAIN_CACHE_SLOTS];
} FAT32_FILE;
// Raw sector I/O (bypasses FAT cache; used for file data and cache eviction).
@@ -231,6 +307,7 @@ static uint64_t ClusterToLba(FAT32_VOL *v, uint32_t cluster) {
// FlushFatCache — needs full FAT32_VOL definition.
static void FlushFatCache(FAT32_VOL *v) {
uint64_t fat0Start = (uint64_t)v->PartitionLba + v->ReservedSectors;
int held = FatCacheAcquire();
for (int i = 0; i < FAT_CACHE_SLOTS; i++) {
FAT_CACHE_ENTRY *e = &gFatCache[i];
@@ -246,6 +323,8 @@ static void FlushFatCache(FAT32_VOL *v) {
}
e->dirty = false;
}
FatCacheRelease(held);
}
// FAT read / write (through cache).
@@ -256,9 +335,14 @@ static uint32_t FatEntry(FAT32_VOL *v, uint32_t cluster) {
+ fatOffset / v->BytesPerSector;
uint32_t offInSec = fatOffset % v->BytesPerSector;
int held = FatCacheAcquire();
uint8_t *sec;
if (!KSUCCESS(FatCacheRead(v, sector, &sec))) return 0;
return *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
KSTATUS s = FatCacheRead(v, sector, &sec);
uint32_t value = 0;
if (KSUCCESS(s))
value = *(uint32_t *)(sec + offInSec) & 0x0FFFFFFF;
FatCacheRelease(held);
return value;
}
// Updates fat Entry.
@@ -268,19 +352,24 @@ static KSTATUS WriteFatEntry(FAT32_VOL *v, uint32_t cluster, uint32_t value) {
+ fatOffset / v->BytesPerSector;
uint32_t offInSec = fatOffset % v->BytesPerSector;
int held = FatCacheAcquire();
uint8_t *sec;
KSTATUS s = FatCacheRead(v, sector, &sec);
if (!KSUCCESS(s)) return s;
if (!KSUCCESS(s)) {
FatCacheRelease(held);
return s;
}
uint32_t existing = *(uint32_t *)(sec + offInSec);
*(uint32_t *)(sec + offInSec) = (existing & 0xF0000000u)
| (value & 0x0FFFFFFFu);
FatCacheMarkDirty(v, sector);
FatCacheRelease(held);
return STATUS_SUCCESS;
}
// Cluster allocation (fix #3: next-free hint + two-phase wrap-around scan).
static uint32_t AllocCluster(FAT32_VOL *v) {
static uint32_t AllocClusterEx(FAT32_VOL *v, int zeroCluster) {
if (v->TotalClusters == 0) return 0;
uint32_t total = v->TotalClusters;
@@ -295,12 +384,18 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
if (!KSUCCESS(WriteFatEntry(v, c, FAT32_EOC_MARK))) return 0;
v->NextFreeHint = c + 1;
// Zero the cluster (required for directory correctness).
uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (zero) {
memset(zero, 0, v->BytesPerCluster);
WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero);
MmFreePool(zero);
// Normal file growth zeroes newly allocated clusters so old
// disk contents are never exposed. Swap/pagefile creation is
// the one exception: every allocated slot is overwritten before
// it can ever be read back, so skipping these writes avoids a
// slow boot-time full-pagefile zero pass.
if (zeroCluster) {
uint8_t *zero = (uint8_t *)MmAllocatePool(v->BytesPerCluster);
if (zero) {
memset(zero, 0, v->BytesPerCluster);
WriteSectors(v, ClusterToLba(v, c), v->SectorsPerCluster, zero);
MmFreePool(zero);
}
}
return c;
}
@@ -309,14 +404,23 @@ static uint32_t AllocCluster(FAT32_VOL *v) {
return 0;
}
// Appends cluster.
static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
uint32_t newCluster = AllocCluster(v);
static uint32_t AllocCluster(FAT32_VOL *v) {
return AllocClusterEx(v, 1);
}
static uint32_t AppendClusterEx(FAT32_VOL *v, uint32_t lastCluster, int zeroCluster) {
uint32_t newCluster = AllocClusterEx(v, zeroCluster);
if (!newCluster) return 0;
if (!KSUCCESS(WriteFatEntry(v, lastCluster, newCluster))) return 0;
return newCluster;
}
// Appends cluster.
static uint32_t AppendCluster(FAT32_VOL *v, uint32_t lastCluster) {
return AppendClusterEx(v, lastCluster, 1);
}
// Returns chain End.
static uint32_t GetChainEnd(FAT32_VOL *v, uint32_t start, uint32_t *outCount) {
uint32_t cluster = start, prev = start, count = 0;
@@ -973,6 +1077,302 @@ static int MatchName83V(const char *dirName, const char *dirExt,
return 1;
}
// Returns the path cache canonical byte used for hashes and comparisons.
static char Fat32PathCacheChar(char c) {
if (c == '/') return '\\';
if (c >= 'A' && c <= 'Z') return (char)(c + 32);
return c;
}
// Hashes a drive-relative path using the same case/separator rules as lookup.
static uint32_t Fat32PathCacheHash(SYS_STRING_VIEW path) {
uint32_t h = 2166136261u;
for (size_t i = 0; i < path.Length; i++) {
h ^= (uint8_t)Fat32PathCacheChar(path.Data[i]);
h *= 16777619u;
}
return h ? h : 1u;
}
// Tests whether the cached path names the same FAT path as the caller's view.
static bool Fat32PathCachePathEquals(const FAT32_PATH_CACHE_ENTRY* e, SYS_STRING_VIEW path) {
if (!e || e->PathLength != path.Length) return false;
for (size_t i = 0; i < path.Length; i++) {
if (Fat32PathCacheChar(e->Path[i]) != Fat32PathCacheChar(path.Data[i]))
return false;
}
return true;
}
// Adds directory search counters without holding the lock during disk I/O.
static void Fat32PathCacheAddDirectoryStats(FAT32_VOL* v, uint64_t clusters, uint64_t entries) {
if (!v) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.DirectorySearches++;
v->Stats.DirectoryClustersRead += clusters;
v->Stats.DirectoryEntriesScanned += entries;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Looks up a positive path-cache entry for a full drive-relative path.
static bool Fat32PathCacheLookup(FAT32_VOL* v, SYS_STRING_VIEW path,
uint32_t* outCluster, uint32_t* outSize,
uint8_t* outAttr, uint64_t* outDirentLba,
uint32_t* outDirentOffset) {
if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
return false;
uint32_t hash = Fat32PathCacheHash(path);
uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
bool hit = false;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.PathLookups++;
FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
if (e->Valid && e->Generation == v->PathCacheGeneration &&
e->Hash == hash && Fat32PathCachePathEquals(e, path)) {
if (outCluster) *outCluster = e->Cluster;
if (outSize) *outSize = e->Size;
if (outAttr) *outAttr = e->Attr;
if (outDirentLba) *outDirentLba = e->DirentLba;
if (outDirentOffset) *outDirentOffset = e->DirentOffset;
v->Stats.PathCacheHits++;
hit = true;
} else {
v->Stats.PathCacheMisses++;
}
KeReleaseSpinlock(&v->PathCacheLock, flags);
return hit;
}
// Stores a positive path lookup in the volume-local path cache.
static void Fat32PathCacheStore(FAT32_VOL* v, SYS_STRING_VIEW path,
uint32_t cluster, uint32_t size, uint8_t attr,
uint64_t direntLba, uint32_t direntOffset) {
if (!v || path.Length == 0 || path.Length >= FAT32_PATH_CACHE_MAX)
return;
uint32_t hash = Fat32PathCacheHash(path);
uint32_t slot = hash % FAT32_PATH_CACHE_SLOTS;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
FAT32_PATH_CACHE_ENTRY* e = &v->PathCache[slot];
memset(e, 0, sizeof(*e));
e->Valid = true;
e->Generation = v->PathCacheGeneration;
e->Hash = hash;
e->PathLength = path.Length;
memcpy(e->Path, path.Data, path.Length);
e->Path[path.Length] = '\0';
e->Cluster = cluster;
e->Size = size;
e->Attr = attr;
e->DirentLba = direntLba;
e->DirentOffset = direntOffset;
v->Stats.PathCacheStores++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Invalidates every positive path-cache entry after a directory or size mutation.
static void Fat32PathCacheInvalidate(FAT32_VOL* v) {
if (!v) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->PathCacheGeneration++;
if (v->PathCacheGeneration == 0)
v->PathCacheGeneration = 1;
v->Stats.PathCacheInvalidations++;
v->Stats.PathCacheGeneration = v->PathCacheGeneration;
for (uint32_t i = 0; i < FAT32_PATH_CACHE_SLOTS; i++)
v->PathCache[i].Valid = false;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Returns the current filesystem cache and traversal counters.
void Fat32QueryStats(IO_FILESYSTEM* fs, IO_FILESYSTEM_STATS* out) {
if (!fs || !out) return;
FAT32_VOL* v = (FAT32_VOL*)fs->DriverCtx;
if (!v) { memset(out, 0, sizeof(*out)); return; }
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
*out = v->Stats;
out->PathCacheGeneration = v->PathCacheGeneration;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Adds FAT-chain traversal counters shared by the read path helpers.
static void Fat32StatsAddChainWalk(FAT32_VOL* v, uint64_t steps) {
if (!v || !steps) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainWalkSteps += steps;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Records one data read span and its physically contiguous cluster merges.
static void Fat32StatsAddReadRun(FAT32_VOL* v, uint32_t sectors, uint32_t merges) {
if (!v || !sectors) return;
uintptr_t flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.DataReadRuns++;
v->Stats.DataReadSectors += sectors;
v->Stats.DataReadRunMerges += merges;
if (v->Stats.MaxDataReadSpanSectors < sectors)
v->Stats.MaxDataReadSpanSectors = sectors;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Initializes the per-open cluster-chain cache for a FAT file object.
static void Fat32FileInitChainCache(FAT32_FILE* f) {
if (!f) return;
f->ChainCacheLock = (SPINLOCK)SPINLOCK_INIT;
f->ChainCacheClock = 0;
memset(f->ChainCache, 0, sizeof(f->ChainCache));
}
// Invalidates cached FAT extents after the file's cluster chain changes.
static void Fat32FileInvalidateChainCache(FAT32_FILE* f) {
if (!f) return;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
memset(f->ChainCache, 0, sizeof(f->ChainCache));
f->ChainCacheClock = 0;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheInvalidations++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Locates the best cached starting point for a logical cluster index.
static bool Fat32ChainCacheLookupStart(FAT32_FILE* f, uint32_t logicalIndex,
uint32_t* outLogical, uint32_t* outCluster) {
if (!f || !outLogical || !outCluster)
return false;
bool hit = false;
uint32_t bestLogical = 0;
uint32_t bestCluster = 0;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
for (uint32_t i = 0; i < FAT32_CHAIN_CACHE_SLOTS; i++) {
FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[i];
if (!e->Valid || e->Length == 0)
continue;
uint32_t end = e->LogicalStart + e->Length;
if (logicalIndex >= e->LogicalStart && logicalIndex < end) {
*outLogical = logicalIndex;
*outCluster = e->PhysicalStart + (logicalIndex - e->LogicalStart);
hit = true;
break;
}
if (end <= logicalIndex && e->NextCluster >= 2 && e->NextCluster < FAT32_EOC) {
if (!hit || end > bestLogical) {
bestLogical = end;
bestCluster = e->NextCluster;
hit = true;
}
}
}
if (hit && bestCluster) {
*outLogical = bestLogical;
*outCluster = bestCluster;
}
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (v) {
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheLookups++;
if (hit) v->Stats.FatChainCacheHits++;
else v->Stats.FatChainCacheMisses++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
return hit;
}
// Stores a physically contiguous FAT extent in the per-open chain cache.
static void Fat32ChainCacheStore(FAT32_FILE* f, uint32_t logicalStart,
uint32_t physicalStart, uint32_t length,
uint32_t nextCluster) {
if (!f || physicalStart < 2 || physicalStart >= FAT32_EOC || length == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&f->ChainCacheLock);
uint32_t slot = f->ChainCacheClock++ % FAT32_CHAIN_CACHE_SLOTS;
FAT32_CHAIN_CACHE_ENTRY* e = &f->ChainCache[slot];
e->Valid = true;
e->LogicalStart = logicalStart;
e->PhysicalStart = physicalStart;
e->Length = length;
e->NextCluster = nextCluster;
KeReleaseSpinlock(&f->ChainCacheLock, flags);
FAT32_VOL* v = f->Vol;
if (!v) return;
flags = KeAcquireSpinlock(&v->PathCacheLock);
v->Stats.FatChainCacheStores++;
KeReleaseSpinlock(&v->PathCacheLock, flags);
}
// Resolves a logical file cluster through the cached FAT-chain extent map.
static uint32_t Fat32ResolveFileCluster(FAT32_FILE* f, uint32_t logicalIndex) {
if (!f || f->StartCluster < 2 || f->StartCluster >= FAT32_EOC)
return 0;
if (logicalIndex == 0)
return f->StartCluster;
uint32_t currentLogical = 0;
uint32_t cluster = f->StartCluster;
if (!Fat32ChainCacheLookupStart(f, logicalIndex, &currentLogical, &cluster)) {
currentLogical = 0;
cluster = f->StartCluster;
}
uint64_t steps = 0;
while (currentLogical < logicalIndex && cluster >= 2 && cluster < FAT32_EOC) {
cluster = FatEntry(f->Vol, cluster);
currentLogical++;
steps++;
}
Fat32StatsAddChainWalk(f->Vol, steps);
return cluster;
}
// Builds and caches one physically contiguous read run from a starting cluster.
static bool Fat32BuildReadRun(FAT32_FILE* f, uint32_t logicalIndex,
uint32_t startCluster, uint32_t maxSectors,
uint32_t* outRunSectors, uint32_t* outNextCluster) {
if (!f || !outRunSectors || !outNextCluster)
return false;
FAT32_VOL* v = f->Vol;
if (!v || startCluster < 2 || startCluster >= FAT32_EOC)
return false;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t runClusters = 1;
uint32_t last = startCluster;
uint32_t next = FatEntry(v, last);
uint64_t steps = 1;
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= maxSectors) {
last = next;
runClusters++;
runSectors += v->SectorsPerCluster;
next = FatEntry(v, last);
steps++;
}
Fat32StatsAddChainWalk(v, steps);
Fat32ChainCacheStore(f, logicalIndex, startCluster, runClusters, next);
*outRunSectors = runSectors;
*outNextCluster = next;
return true;
}
static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
uint32_t *outCluster, uint32_t *outSize, uint8_t *outAttr,
uint64_t *outDirentLba, uint32_t *outDirentOffset) {
@@ -986,15 +1386,20 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
int allowShortMatch = FatNameIsStructurallyShort83V(name);
Make83NameV(name, wantName83, wantExt83);
uint32_t cluster = dirCluster;
uint64_t clustersRead = 0;
uint64_t entriesScanned = 0;
int found = 0;
while (cluster >= 2 && cluster < FAT32_EOC) {
uint64_t clusterLba = ClusterToLba(v, cluster);
if (!KSUCCESS(ReadSectors(v, clusterLba, v->SectorsPerCluster, clusterBuf))) break;
clustersRead++;
uint32_t entries = v->BytesPerCluster / sizeof(DIRENT);
DIRENT *dir = (DIRENT *)clusterBuf;
for (uint32_t i = 0; i < entries; i++) {
entriesScanned++;
if (dir[i].Name[0] == 0x00) goto done;
if ((uint8_t)dir[i].Name[0] == 0xE5) { hasLfn = 0; memset(lfn, 0, sizeof(lfn)); continue; }
if (dir[i].Attr == ATTR_LFN) { LfnAppend(lfn, (LFN_ENTRY *)&dir[i]); hasLfn = 1; continue; }
@@ -1016,15 +1421,16 @@ static int SearchDirV(FAT32_VOL *v, uint32_t dirCluster, SYS_STRING_VIEW name,
if (outDirentLba) *outDirentLba = clusterLba + eb / v->BytesPerSector;
if (outDirentOffset) *outDirentOffset = (uint32_t)(eb % v->BytesPerSector);
}
MmFreePool(clusterBuf);
return 1;
found = 1;
goto done;
}
}
cluster = FatEntry(v, cluster);
}
done:
Fat32PathCacheAddDirectoryStats(v, clustersRead, entriesScanned);
MmFreePool(clusterBuf);
return 0;
return found;
}
static int Fat32PathNextComponent(SYS_STRING_VIEW path, size_t* cursor,
@@ -1058,6 +1464,10 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
if (outDirentLba) *outDirentLba = 0;
if (outDirentOffset) *outDirentOffset = 0;
if (Fat32PathCacheLookup(v, path, outCluster, outSize, &attr,
outDirentLba, outDirentOffset))
return STATUS_SUCCESS;
size_t cursor = 0;
SYS_STRING_VIEW component;
while (Fat32PathNextComponent(path, &cursor, &component)) {
@@ -1078,6 +1488,9 @@ static KSTATUS Fat32LookupPathV(FAT32_VOL *v, SYS_STRING_VIEW path,
*outCluster = cluster;
*outSize = size;
Fat32PathCacheStore(v, path, cluster, size, attr,
outDirentLba ? *outDirentLba : 0,
outDirentOffset ? *outDirentOffset : 0);
return STATUS_SUCCESS;
}
@@ -1220,6 +1633,7 @@ KSTATUS Fat32Open(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = cluster; f->FileSize = size; f->Vol = v;
f->DirentLba = direntLba; f->DirentOffset = direntOffset;
Fat32FileInitChainCache(f);
*fileCtx = f;
return STATUS_SUCCESS;
}
@@ -1233,15 +1647,9 @@ void Fat32Close(IO_FILESYSTEM *fs, void *fileCtx) {
}
// Read (fix #4: contiguous-run coalescing)
// We peek ahead in the FAT chain: if cluster N+1 == cluster N physically,
// extend the current run instead of issuing a new ReadSectors call.
// Run length is capped at MAX_RUN_SECTORS to stay within the AHCI bounce
// buffer (128 sectors = 64 KB).
// FatEntry() calls in the peek loop are nearly free because they hit the
// FAT sector cache (fix #1).
#define MAX_RUN_SECTORS 128u
// Cluster-chain lookup is cached per open file, and data spans are allowed to
// grow beyond the AHCI bounce size because the device layer now splits large
// sector requests for the concrete controller.
KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
uint64_t offset, void *buf,
@@ -1257,14 +1665,12 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
if (size > (size_t)(f->FileSize - offset)) size = (size_t)(f->FileSize - offset);
size_t wanted = size;
// Allocate only as much bounce space as this read can actually use.
// The old code allocated 64 KiB and could read a 64 KiB contiguous run even
// for tiny random reads; Doom does lots of seek/read pairs for small lumps.
uint32_t initialOffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
uint64_t initialBytesCovered = (uint64_t)initialOffsetInCluster + (uint64_t)size;
uint32_t maxUsefulSectors = (uint32_t)((initialBytesCovered + 511u) / 512u);
if (maxUsefulSectors < v->SectorsPerCluster) maxUsefulSectors = v->SectorsPerCluster;
if (maxUsefulSectors > MAX_RUN_SECTORS) maxUsefulSectors = MAX_RUN_SECTORS;
if (maxUsefulSectors > FAT32_READ_RUN_MAX_SECTORS)
maxUsefulSectors = FAT32_READ_RUN_MAX_SECTORS;
uint8_t *runBuf = (uint8_t *)MmAllocatePool((size_t)maxUsefulSectors * 512u);
if (!runBuf) return STATUS_NO_MEMORY;
@@ -1272,35 +1678,27 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
size_t totalRead = 0;
uint8_t *out = (uint8_t *)buf;
KSTATUS readStatus = STATUS_SUCCESS;
uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster);
uint32_t logicalCluster = (uint32_t)(offset / v->BytesPerCluster);
uint32_t offsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
uint32_t cluster = f->StartCluster;
for (uint32_t i = 0; i < skipClusters && cluster < FAT32_EOC; i++)
cluster = FatEntry(v, cluster);
uint32_t cluster = Fat32ResolveFileCluster(f, logicalCluster);
while (size > 0 && cluster >= 2 && cluster < FAT32_EOC) {
// Build the longest physically-contiguous run that can contribute to
// this caller's remaining read. Do not prefetch an unrelated 64 KiB run
// for a tiny random read.
uint64_t bytesCovered = (uint64_t)offsetInCluster + (uint64_t)size;
uint32_t usefulSectors = (uint32_t)((bytesCovered + 511u) / 512u);
if (usefulSectors < v->SectorsPerCluster) usefulSectors = v->SectorsPerCluster;
if (usefulSectors > MAX_RUN_SECTORS) usefulSectors = MAX_RUN_SECTORS;
if (usefulSectors > FAT32_READ_RUN_MAX_SECTORS)
usefulSectors = FAT32_READ_RUN_MAX_SECTORS;
if (usefulSectors > maxUsefulSectors) usefulSectors = maxUsefulSectors;
uint32_t runStart = cluster;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t next = FatEntry(v, cluster);
while (next == cluster + 1 && next < FAT32_EOC
&& runSectors + v->SectorsPerCluster <= usefulSectors) {
cluster = next;
next = FatEntry(v, cluster);
runSectors += v->SectorsPerCluster;
uint32_t runSectors = 0;
uint32_t next = 0;
if (!Fat32BuildReadRun(f, logicalCluster, cluster, usefulSectors,
&runSectors, &next)) {
readStatus = STATUS_DEVICE_ERROR;
break;
}
KSTATUS rs = ReadSectors(v, ClusterToLba(v, runStart), runSectors, runBuf);
KSTATUS rs = ReadSectors(v, ClusterToLba(v, cluster), runSectors, runBuf);
if (!KSUCCESS(rs)) {
readStatus = rs;
break;
@@ -1309,12 +1707,16 @@ KSTATUS Fat32Read(IO_FILESYSTEM *fs, void *fileCtx,
uint32_t runBytes = runSectors * 512u;
size_t avail = runBytes - offsetInCluster;
size_t copy = avail < size ? avail : size;
uint32_t runClusters = runSectors / v->SectorsPerCluster;
uint32_t merges = runClusters ? runClusters - 1u : 0;
Fat32StatsAddReadRun(v, runSectors, merges);
memcpy(out, runBuf + offsetInCluster, copy);
out += copy;
totalRead += copy;
size -= copy;
offsetInCluster = 0;
logicalCluster += runClusters;
cluster = next;
}
@@ -1345,6 +1747,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
const uint8_t *src = (const uint8_t *)buf;
Fat32FileInvalidateChainCache(f);
// Ensure at least one cluster exists.
if (f->StartCluster < 2) {
@@ -1368,7 +1771,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
cluster = next;
}
uint8_t *runBuf = (uint8_t *)MmAllocatePool(MAX_RUN_SECTORS * 512u);
uint8_t *runBuf = (uint8_t *)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
if (!runBuf) { *bytesWritten = 0; return STATUS_NO_MEMORY; }
size_t written = 0, remaining = size;
@@ -1382,7 +1785,7 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
if (offsetInCluster == 0) {
while (next == cluster + 1 && next < FAT32_EOC
&& runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
&& runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
cluster = next;
next = FatEntry(v, cluster);
runSectors += v->SectorsPerCluster;
@@ -1439,7 +1842,9 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
u.Flags |= FAT32_DIRENT_UPDATE_SIZE;
u.Size = f->FileSize;
}
DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
KSTATUS ms = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
if (KSUCCESS(ms))
Fat32PathCacheInvalidate(v);
}
*bytesWritten = written;
@@ -1447,8 +1852,6 @@ KSTATUS Fat32Write(IO_FILESYSTEM *fs, void *fileCtx,
}
// Create.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
// Creates the requested object.
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!fs || !fileCtx || !SysSvIsValid(path) || path.Length == 0)
@@ -1483,7 +1886,7 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
// Create means overwrite by default for regular files. Reuse the
// existing directory entry, drop the old cluster chain, and return
// an open zero-length file instead of allocating a duplicate name.
KSTATUS ts = Fat32Truncate(fs, f, 0);
KSTATUS ts = Fat32Truncate(fs, f, 0, 0);
if (!KSUCCESS(ts)) { MmFreePool(f); return ts; }
*fileCtx = f;
return STATUS_SUCCESS;
@@ -1558,21 +1961,27 @@ KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx) {
if (!f) return STATUS_NO_MEMORY;
f->StartCluster = 0; f->FileSize = 0; f->Vol = v;
f->DirentLba = direntLba; f->DirentOffset = direntOffset;
Fat32FileInitChainCache(f);
Fat32PathCacheInvalidate(v);
*fileCtx = f;
return STATUS_SUCCESS;
}
// Truncate.
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags) {
(void)fs;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
Fat32FileInvalidateChainCache(f);
int zeroNewClusters = (fileFlags & IO_FILE_FLAG_SKIP_ZERO_EXTEND) ? 0 : 1;
if ((uint32_t)newSize == f->FileSize) {
FAT32_DIRENT_UPDATE u;
memset(&u, 0, sizeof(u));
FatQueryEncodedTime(&u.Timestamp);
u.Flags = FAT32_DIRENT_UPDATE_WRITE_TIME;
return DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
KSTATUS s = DirentUpdateMetadata(v, f->DirentLba, f->DirentOffset, &u);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s;
}
if (newSize == 0) {
@@ -1584,6 +1993,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
f->FileSize = 0;
KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, 0);
FlushFatCache(v);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s;
}
@@ -1604,7 +2014,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
? (f->FileSize + v->BytesPerCluster - 1) / v->BytesPerCluster : 0;
uint32_t lastCluster = 0;
if (f->StartCluster < 2) {
uint32_t first = AllocCluster(v);
uint32_t first = AllocClusterEx(v, zeroNewClusters);
if (!first) return STATUS_NO_MEMORY;
f->StartCluster = first;
DirentUpdateFirstCluster(v, f->DirentLba, f->DirentOffset, first);
@@ -1613,7 +2023,7 @@ KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize) {
lastCluster = GetChainEnd(v, f->StartCluster, NULL);
}
for (uint32_t i = curClusters; i < newClusters; i++) {
uint32_t next = AppendCluster(v, lastCluster);
uint32_t next = AppendClusterEx(v, lastCluster, zeroNewClusters);
if (!next) return STATUS_NO_MEMORY;
lastCluster = next;
}
@@ -1623,6 +2033,7 @@ update_size:
f->FileSize = (uint32_t)newSize;
{ KSTATUS s = DirentUpdateSize(v, f->DirentLba, f->DirentOffset, (uint32_t)newSize);
FlushFatCache(v);
if (KSUCCESS(s)) Fat32PathCacheInvalidate(v);
return s; }
}
@@ -1646,6 +2057,7 @@ KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path) {
if (cluster >= 2) FreeClusterChain(v, cluster);
FlushFatCache(v);
Fat32PathCacheInvalidate(v);
return STATUS_SUCCESS;
}
@@ -1654,6 +2066,75 @@ uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx) {
(void)fs; return ((FAT32_FILE *)fileCtx)->FileSize;
}
// Mixes FAT directory metadata into a per-file content generation.
static uint64_t Fat32DirentGeneration(const DIRENT *d) {
if (!d)
return 0;
uint32_t firstCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
uint64_t generation = ((uint64_t)d->WrtDate << 48) |
((uint64_t)d->WrtTime << 32) |
((uint64_t)d->LastAccDate << 16) |
(uint64_t)d->Attr;
generation ^= ((uint64_t)d->CrtDate << 40);
generation ^= ((uint64_t)d->CrtTime << 24);
generation ^= ((uint64_t)d->CrtTimeTenth << 16);
generation ^= ((uint64_t)firstCluster << 1);
generation ^= ((uint64_t)d->FileSize << 17);
return generation ? generation : 1;
}
// Returns a stable FAT file identity for cross-open image-section caches.
KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out) {
(void)fs;
if (!fileCtx || !out)
return STATUS_INVALID_PARAMETER;
FAT32_FILE *f = (FAT32_FILE *)fileCtx;
FAT32_VOL *v = f->Vol;
if (!v)
return STATUS_INVALID_PARAMETER;
uint32_t startCluster = f->StartCluster;
uint32_t fileSize = f->FileSize;
uint64_t generation = ((uint64_t)startCluster << 1) ^
((uint64_t)fileSize << 17);
if (f->DirentLba != 0) {
uint8_t *sec = (uint8_t *)MmAllocatePool(v->BytesPerSector);
if (!sec)
return STATUS_NO_MEMORY;
KSTATUS s = ReadSectors(v, f->DirentLba, 1, sec);
if (!KSUCCESS(s)) {
MmFreePool(sec);
return s;
}
DIRENT *d = (DIRENT *)(sec + f->DirentOffset);
if ((uint8_t)d->Name[0] == 0x00 || (uint8_t)d->Name[0] == 0xE5) {
MmFreePool(sec);
return STATUS_NOT_FOUND;
}
startCluster = ((uint32_t)d->FirstClusterHi << 16) | d->FirstClusterLo;
fileSize = d->FileSize;
generation = Fat32DirentGeneration(d);
MmFreePool(sec);
}
out->VolumeId = ((uint64_t)v->DriveIndex << 56) ^
((uint64_t)v->PartitionLba << 8) ^
((uint64_t)v->RootCluster << 1) ^
0x4641543332494445ULL;
out->FileId = f->DirentLba ^
((uint64_t)f->DirentOffset << 32) ^
((uint64_t)startCluster << 1);
out->Generation = generation ? generation : 1;
out->Size = fileSize;
return STATUS_SUCCESS;
}
// Directory enumeration (fix #2: cluster buffer cached in context)
// Previously: allocate + ReadSectors + return 1 entry + free, every call.
// Now: buffer lives in FAT32_DIR_CTX; ReadSectors fires only on cluster
@@ -1757,8 +2238,7 @@ KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out) {
}
ctx->CurCluster = FatEntry(v, ctx->CurCluster);
ctx->EntryIndex = 0; ctx->HasLfn = 0;
memset(ctx->LfnBuf, 0, sizeof(ctx->LfnBuf));
ctx->EntryIndex = 0;
}
return STATUS_NO_MORE_FILES;
@@ -1962,6 +2442,7 @@ typedef struct FAT32_ASYNC_RW {
uint64_t Offset;
uint32_t OffsetInCluster;
uint32_t Cluster;
uint32_t LogicalCluster;
size_t Remaining;
size_t Done;
size_t Requested;
@@ -1972,6 +2453,7 @@ typedef struct FAT32_ASYNC_RW {
uint32_t CurrentNext;
uint32_t CurrentLast;
uint32_t CurrentClusters;
uint32_t CurrentOffset;
size_t CurrentCopy;
@@ -2088,7 +2570,13 @@ static void Fat32AsyncReadIrpComplete(IRP* irp, void* completeCtx) {
ctx->Done += ctx->CurrentCopy;
ctx->Remaining -= ctx->CurrentCopy;
ctx->OffsetInCluster = 0;
ctx->LogicalCluster += ctx->CurrentClusters;
ctx->Cluster = ctx->CurrentNext;
if (ctx->CurrentClusters) {
uint32_t sectors = ctx->CurrentClusters * ctx->Vol->SectorsPerCluster;
uint32_t merges = ctx->CurrentClusters - 1u;
Fat32StatsAddReadRun(ctx->Vol, sectors, merges);
}
Fat32AsyncQueueContinuation(ctx);
}
@@ -2109,15 +2597,12 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
FAT32_VOL* v = ctx->Vol;
uint32_t runStart = ctx->Cluster;
uint32_t runSectors = v->SectorsPerCluster;
uint32_t last = ctx->Cluster;
uint32_t next = FatEntry(v, last);
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
last = next;
next = FatEntry(v, last);
runSectors += v->SectorsPerCluster;
uint32_t runSectors = 0;
uint32_t next = 0;
if (!Fat32BuildReadRun(ctx->File, ctx->LogicalCluster, ctx->Cluster,
FAT32_READ_RUN_MAX_SECTORS, &runSectors, &next)) {
Fat32AsyncFinish(ctx, STATUS_DEVICE_ERROR, ctx->Done);
return;
}
uint32_t runBytes = runSectors * 512u;
@@ -2125,7 +2610,8 @@ static void Fat32AsyncReadSubmitNext(FAT32_ASYNC_RW* ctx) {
size_t copy = avail < ctx->Remaining ? avail : ctx->Remaining;
ctx->CurrentNext = next;
ctx->CurrentLast = last;
ctx->CurrentLast = runStart + (runSectors / v->SectorsPerCluster) - 1u;
ctx->CurrentClusters = runSectors / v->SectorsPerCluster;
ctx->CurrentOffset = ctx->OffsetInCluster;
ctx->CurrentCopy = copy;
@@ -2184,7 +2670,7 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
memset(ctx, 0, sizeof(*ctx));
LIST_INIT(&ctx->Link);
ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_READ_RUN_MAX_SECTORS * 512u);
if (!ctx->RunBuf) {
MmFreePool(ctx);
complete(completeContext, STATUS_NO_MEMORY, 0);
@@ -2196,17 +2682,14 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx,
ctx->IsWrite = 0;
ctx->Offset = offset;
ctx->OffsetInCluster = (uint32_t)(offset % v->BytesPerCluster);
ctx->Cluster = f->StartCluster;
ctx->LogicalCluster = (uint32_t)(offset / v->BytesPerCluster);
ctx->Cluster = Fat32ResolveFileCluster(f, ctx->LogicalCluster);
ctx->Remaining = size;
ctx->Requested = size;
ctx->Dst = (uint8_t*)buf;
ctx->Complete = complete;
ctx->CompleteContext = completeContext;
uint32_t skipClusters = (uint32_t)(offset / v->BytesPerCluster);
for (uint32_t i = 0; i < skipClusters && ctx->Cluster < FAT32_EOC; i++)
ctx->Cluster = FatEntry(v, ctx->Cluster);
Fat32AsyncQueueContinuation(ctx);
return STATUS_PENDING;
}
@@ -2276,7 +2759,7 @@ static void Fat32AsyncWriteSubmitNext(FAT32_ASYNC_RW* ctx) {
if (ctx->OffsetInCluster == 0) {
while (next == last + 1 && next < FAT32_EOC &&
runSectors + v->SectorsPerCluster <= MAX_RUN_SECTORS) {
runSectors + v->SectorsPerCluster <= FAT32_WRITE_RUN_MAX_SECTORS) {
last = next;
next = FatEntry(v, last);
runSectors += v->SectorsPerCluster;
@@ -2347,6 +2830,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
FAT32_FILE* f = (FAT32_FILE*)fileCtx;
FAT32_VOL* v = f->Vol;
Fat32FileInvalidateChainCache(f);
if (f->StartCluster < 2) {
uint32_t first = AllocCluster(v);
@@ -2380,7 +2864,7 @@ KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx,
memset(ctx, 0, sizeof(*ctx));
LIST_INIT(&ctx->Link);
ctx->RunBuf = (uint8_t*)MmAllocatePool(MAX_RUN_SECTORS * 512u);
ctx->RunBuf = (uint8_t*)MmAllocatePool(FAT32_WRITE_RUN_MAX_SECTORS * 512u);
if (!ctx->RunBuf) {
MmFreePool(ctx);
complete(completeContext, STATUS_NO_MEMORY, 0);
code/drivers/fs/fat32/fat32.h
+5 -1
View File
@@ -16,6 +16,8 @@
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_time.h>
#include <kernel/ke_mutex.h>
#include <kernel/ke_irql.h>
typedef struct FAT32_VOL_S FAT32_VOL;
@@ -27,10 +29,12 @@ KSTATUS Fat32ReadAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, void *
KSTATUS Fat32WriteAsync(IO_FILESYSTEM *fs, void *fileCtx, uint64_t offset, const void *buf, size_t size, IO_FS_ASYNC_COMPLETE complete, void *completeContext);
KSTATUS Fat32Create(IO_FILESYSTEM *fs, SYS_STRING_VIEW path, void **fileCtx);
KSTATUS Fat32Delete(IO_FILESYSTEM *fs, SYS_STRING_VIEW path);
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize);
KSTATUS Fat32Truncate(IO_FILESYSTEM *fs, void *fileCtx, uint64_t newSize, uint32_t fileFlags);
uint64_t Fat32GetSize(IO_FILESYSTEM *fs, void *fileCtx);
KSTATUS Fat32QueryIdentity(IO_FILESYSTEM *fs, void *fileCtx, IO_FILE_IDENTITY *out);
KSTATUS Fat32OpenDir(IO_FILESYSTEM *fs, SYS_STRING_VIEW dirPath, void **dirCtx);
KSTATUS Fat32ReadDir(IO_FILESYSTEM *fs, void *dirCtx, KDIR_ENTRY *out);
void Fat32CloseDir(IO_FILESYSTEM *fs, void *dirCtx);
void Fat32QueryStats(IO_FILESYSTEM *fs, IO_FILESYSTEM_STATS *out);
void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol);
code/drivers/fs/fat32/fat32_mount.c
+2
View File
@@ -14,9 +14,11 @@ void Fat32InitVtable(IO_FILESYSTEM* fs, FAT32_VOL* vol) {
fs->Delete = Fat32Delete;
fs->Truncate = Fat32Truncate;
fs->GetSize = Fat32GetSize;
fs->QueryIdentity = Fat32QueryIdentity;
fs->Close = Fat32Close;
fs->DriverCtx = vol;
fs->OpenDir = Fat32OpenDir;
fs->ReadDir = Fat32ReadDir;
fs->CloseDir = Fat32CloseDir;
fs->QueryStats = Fat32QueryStats;
}
code/drivers/input/serial/serial.c
+301
View File
@@ -0,0 +1,301 @@
// 16550-compatible serial input driver. This file routes COM1 receive bytes into the canonical input core.
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#include <intrin.h>
#include <ntlite/status.h>
#include <ntlite/ddi/hal_api.h>
#include <ntlite/ddi/input_device.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_irq.h>
#define SERIAL_COM1_BASE 0x3F8u
#define SERIAL_COM1_IRQ 4u
#define SERIAL_REG_DATA 0u
#define SERIAL_REG_IER 1u
#define SERIAL_REG_IIR 2u
#define SERIAL_REG_FCR 2u
#define SERIAL_REG_LCR 3u
#define SERIAL_REG_MCR 4u
#define SERIAL_REG_LSR 5u
#define SERIAL_LSR_DATA_READY 0x01u
#define SERIAL_LSR_TX_EMPTY 0x20u
#define SERIAL_LCR_DLAB 0x80u
#define SERIAL_MCR_DTR 0x01u
#define SERIAL_MCR_RTS 0x02u
#define SERIAL_MCR_OUT2 0x08u
#define SERIAL_IER_RX_READY 0x01u
#define SERIAL_FCR_ENABLE 0x01u
#define SERIAL_FCR_CLEAR_RX 0x02u
#define SERIAL_FCR_CLEAR_TX 0x04u
#define SERIAL_FCR_TRIGGER_14 0xC0u
typedef enum SERIAL_ESCAPE_STATE {
SerialEscapeNone = 0,
SerialEscapeEsc,
SerialEscapeCsi,
SerialEscapeCsiNumber,
} SERIAL_ESCAPE_STATE;
static volatile SERIAL_ESCAPE_STATE gSerialEscapeState;
static volatile uint8_t gSerialCsiNumber;
static volatile bool gSerialDropNextLf;
// Reads one byte from a COM register.
static uint8_t SerialRead(uint16_t reg) {
return __inbyte((unsigned short)(SERIAL_COM1_BASE + reg));
}
// Writes one byte to a COM register.
static void SerialWrite(uint16_t reg, uint8_t value) {
__outbyte((unsigned short)(SERIAL_COM1_BASE + reg), value);
}
// Submits one input key transition.
static void SerialSubmitTransition(INPUT_KEY key, bool pressed) {
INPUT_KEY_EVENT event;
event.Key = key;
event.Pressed = pressed;
KeInputSubmitKey(&event);
}
// Submits one logical key press and release.
static void SerialSubmitKey(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return;
SerialSubmitTransition(key, true);
SerialSubmitTransition(key, false);
}
// Submits one shifted key press and release.
static void SerialSubmitShiftedKey(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return;
SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, true);
SerialSubmitKey(key);
SerialSubmitTransition(INPUT_KEY_LEFT_SHIFT, false);
}
// Submits one control-modified key press and release.
static void SerialSubmitCtrlKey(INPUT_KEY key) {
if (key == INPUT_KEY_NONE) return;
SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, true);
SerialSubmitKey(key);
SerialSubmitTransition(INPUT_KEY_LEFT_CTRL, false);
}
// Converts an ASCII printable byte into input events.
static void SerialSubmitPrintable(uint8_t ch) {
if (ch >= 'a' && ch <= 'z') {
SerialSubmitKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'a')));
return;
}
if (ch >= 'A' && ch <= 'Z') {
SerialSubmitShiftedKey((INPUT_KEY)(INPUT_KEY_A + (ch - 'A')));
return;
}
if (ch >= '1' && ch <= '9') {
SerialSubmitKey((INPUT_KEY)(INPUT_KEY_1 + (ch - '1')));
return;
}
if (ch == '0') {
SerialSubmitKey(INPUT_KEY_0);
return;
}
switch (ch) {
case ' ': SerialSubmitKey(INPUT_KEY_SPACE); return;
case '-': SerialSubmitKey(INPUT_KEY_MINUS); return;
case '=': SerialSubmitKey(INPUT_KEY_EQUAL); return;
case '[': SerialSubmitKey(INPUT_KEY_LEFT_BRACKET); return;
case ']': SerialSubmitKey(INPUT_KEY_RIGHT_BRACKET); return;
case '\\': SerialSubmitKey(INPUT_KEY_BACKSLASH); return;
case ';': SerialSubmitKey(INPUT_KEY_SEMICOLON); return;
case '\'': SerialSubmitKey(INPUT_KEY_APOSTROPHE); return;
case '`': SerialSubmitKey(INPUT_KEY_GRAVE); return;
case ',': SerialSubmitKey(INPUT_KEY_COMMA); return;
case '.': SerialSubmitKey(INPUT_KEY_PERIOD); return;
case '/': SerialSubmitKey(INPUT_KEY_SLASH); return;
case '!': SerialSubmitShiftedKey(INPUT_KEY_1); return;
case '@': SerialSubmitShiftedKey(INPUT_KEY_2); return;
case '#': SerialSubmitShiftedKey(INPUT_KEY_3); return;
case '$': SerialSubmitShiftedKey(INPUT_KEY_4); return;
case '%': SerialSubmitShiftedKey(INPUT_KEY_5); return;
case '^': SerialSubmitShiftedKey(INPUT_KEY_6); return;
case '&': SerialSubmitShiftedKey(INPUT_KEY_7); return;
case '*': SerialSubmitShiftedKey(INPUT_KEY_8); return;
case '(': SerialSubmitShiftedKey(INPUT_KEY_9); return;
case ')': SerialSubmitShiftedKey(INPUT_KEY_0); return;
case '_': SerialSubmitShiftedKey(INPUT_KEY_MINUS); return;
case '+': SerialSubmitShiftedKey(INPUT_KEY_EQUAL); return;
case '{': SerialSubmitShiftedKey(INPUT_KEY_LEFT_BRACKET); return;
case '}': SerialSubmitShiftedKey(INPUT_KEY_RIGHT_BRACKET); return;
case '|': SerialSubmitShiftedKey(INPUT_KEY_BACKSLASH); return;
case ':': SerialSubmitShiftedKey(INPUT_KEY_SEMICOLON); return;
case '"': SerialSubmitShiftedKey(INPUT_KEY_APOSTROPHE); return;
case '~': SerialSubmitShiftedKey(INPUT_KEY_GRAVE); return;
case '<': SerialSubmitShiftedKey(INPUT_KEY_COMMA); return;
case '>': SerialSubmitShiftedKey(INPUT_KEY_PERIOD); return;
case '?': SerialSubmitShiftedKey(INPUT_KEY_SLASH); return;
default: return;
}
}
// Submits a completed CSI escape sequence.
static void SerialSubmitCsi(uint8_t finalByte) {
uint8_t number = gSerialCsiNumber;
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
switch (finalByte) {
case 'A': SerialSubmitKey(INPUT_KEY_UP); return;
case 'B': SerialSubmitKey(INPUT_KEY_DOWN); return;
case 'C': SerialSubmitKey(INPUT_KEY_RIGHT); return;
case 'D': SerialSubmitKey(INPUT_KEY_LEFT); return;
case 'H': SerialSubmitKey(INPUT_KEY_HOME); return;
case 'F': SerialSubmitKey(INPUT_KEY_END); return;
case '~':
switch (number) {
case 1: SerialSubmitKey(INPUT_KEY_HOME); return;
case 3: SerialSubmitKey(INPUT_KEY_DELETE); return;
case 4: SerialSubmitKey(INPUT_KEY_END); return;
case 5: SerialSubmitKey(INPUT_KEY_PAGE_UP); return;
case 6: SerialSubmitKey(INPUT_KEY_PAGE_DOWN); return;
default: return;
}
default: return;
}
}
// Handles one byte after an ESC prefix.
static bool SerialHandleEscapeByte(uint8_t ch) {
if (gSerialEscapeState == SerialEscapeNone) return false;
if (gSerialEscapeState == SerialEscapeEsc) {
if (ch == '[') {
gSerialEscapeState = SerialEscapeCsi;
gSerialCsiNumber = 0;
return true;
}
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
SerialSubmitKey(INPUT_KEY_ESCAPE);
return false;
}
if (gSerialEscapeState == SerialEscapeCsi || gSerialEscapeState == SerialEscapeCsiNumber) {
if (ch >= '0' && ch <= '9') {
gSerialEscapeState = SerialEscapeCsiNumber;
if (gSerialCsiNumber < 25)
gSerialCsiNumber = (uint8_t)(gSerialCsiNumber * 10u + (ch - '0'));
return true;
}
if (ch == ';')
return true;
SerialSubmitCsi(ch);
return true;
}
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
return false;
}
// Routes one received serial byte into the input core.
static void SerialReceiveByte(uint8_t ch) {
if (SerialHandleEscapeByte(ch)) return;
if (ch == 0x1Bu) {
gSerialEscapeState = SerialEscapeEsc;
gSerialCsiNumber = 0;
return;
}
if (ch == '\r') {
SerialSubmitKey(INPUT_KEY_ENTER);
gSerialDropNextLf = true;
return;
}
if (ch == '\n') {
if (gSerialDropNextLf) {
gSerialDropNextLf = false;
return;
}
SerialSubmitKey(INPUT_KEY_ENTER);
return;
}
gSerialDropNextLf = false;
if (ch == '\t') {
SerialSubmitKey(INPUT_KEY_TAB);
return;
}
if (ch == '\b' || ch == 0x7Fu) {
SerialSubmitKey(INPUT_KEY_BACKSPACE);
return;
}
if (ch == 0x03u) {
SerialSubmitCtrlKey(INPUT_KEY_C);
return;
}
if (ch >= 0x20u && ch <= 0x7Eu)
SerialSubmitPrintable(ch);
}
// Handles receive interrupts for COM1.
static void SerialIsr(void) {
for (uint32_t i = 0; i < 256u; i++) {
if ((SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY) == 0)
break;
SerialReceiveByte(SerialRead(SERIAL_REG_DATA));
}
}
// Initializes COM1 as an 8N1 115200 baud input device.
static void SerialInitializeCom1(void) {
SerialWrite(SERIAL_REG_IER, 0x00u);
SerialWrite(SERIAL_REG_LCR, SERIAL_LCR_DLAB);
SerialWrite(SERIAL_REG_DATA, 0x01u);
SerialWrite(SERIAL_REG_IER, 0x00u);
SerialWrite(SERIAL_REG_LCR, 0x03u);
SerialWrite(SERIAL_REG_FCR, SERIAL_FCR_ENABLE | SERIAL_FCR_CLEAR_RX |
SERIAL_FCR_CLEAR_TX | SERIAL_FCR_TRIGGER_14);
SerialWrite(SERIAL_REG_MCR, SERIAL_MCR_DTR | SERIAL_MCR_RTS | SERIAL_MCR_OUT2);
while (SerialRead(SERIAL_REG_LSR) & SERIAL_LSR_DATA_READY)
(void)SerialRead(SERIAL_REG_DATA);
SerialWrite(SERIAL_REG_IER, SERIAL_IER_RX_READY);
}
// DriverEntry registers COM1 as an input source.
KSTATUS DriverEntry(const HAL_API* hal) {
(void)hal;
uint32_t deviceId = 0;
KSTATUS st = KeInputRegisterDevice("serial-com1", 0, &deviceId);
if (!KSUCCESS(st))
return st;
gSerialEscapeState = SerialEscapeNone;
gSerialCsiNumber = 0;
gSerialDropNextLf = false;
SerialInitializeCom1();
KeIrqRegister(SERIAL_COM1_IRQ, SerialIsr);
DbgPrint("serial: initialized COM1 input device=%u irq=%u\n", deviceId, SERIAL_COM1_IRQ);
return STATUS_SUCCESS;
}
code/drivers/net/rtl8139/rtl8139.c
+2 -2
View File
@@ -88,8 +88,8 @@ static void Rtl8139ReadMac(RTL8139_DEVICE* r) {
// Creates dma.
static KSTATUS Rtl8139AllocateDma(RTL8139_DEVICE* r) {
r->RxPhys = MmAllocateContiguous(RTL8139_RX_PAGES);
r->TxPhys = MmAllocateContiguous(RTL8139_TX_PAGES);
r->RxPhys = MmPfnAllocateContiguous(RTL8139_RX_PAGES, MmPfnUseDma);
r->TxPhys = MmPfnAllocateContiguous(RTL8139_TX_PAGES, MmPfnUseDma);
if (!r->RxPhys || !r->TxPhys)
return STATUS_NO_MEMORY;
code/drivers/net/rtl8139/rtl8139.h
+1
View File
@@ -17,6 +17,7 @@
#include <kernel/ke_dpc.h>
#include <kernel/ke_spinlock.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <ntlite/ddi/net_device.h>
code/drivers/net/rtl816x/rtl816x.c
+4 -4
View File
@@ -85,10 +85,10 @@ static void RtlReadMac(RTL816X_DEVICE* r) {
// Implements the rtl Allocate Dma operation.
static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
r->RxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
r->TxRingPhys = MmAllocateContiguous(RTL_RING_PAGES);
r->RxBuffersPhys = MmAllocateContiguous(RTL_RXBUF_PAGES);
r->TxBuffersPhys = MmAllocateContiguous(RTL_TXBUF_PAGES);
r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma);
r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma);
if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys)
return STATUS_NO_MEMORY;
code/drivers/net/rtl816x/rtl816x.h
+1
View File
@@ -17,6 +17,7 @@
#include <kernel/ke_dpc.h>
#include <kernel/ke_spinlock.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <ntlite/ddi/net_device.h>
code/drivers/storage/ahci/ahci.c
+40 -16
View File
@@ -58,20 +58,20 @@ static inline uint64_t cmd_tbl_phys(AHCI_DRIVE *drv, int slot) {
static KSTATUS port_init_memory(AHCI_DRIVE *drv) {
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
drv->structPhys = MmAllocateContiguous(PORT_PAGES_STRUCT);
drv->structPhys = MmPfnAllocateContiguous(PORT_PAGES_STRUCT, MmPfnUseDma);
if (!drv->structPhys) return STATUS_NO_MEMORY;
drv->structVirt = g_hal->PhysToVirt(drv->structPhys);
// Force NOCACHE on DMA descriptor memory.
MmSetPageProtect((uintptr_t)drv->structVirt,
MmPteProtectRange((uintptr_t)drv->structVirt,
PORT_PAGES_STRUCT * 4096, flags);
memset(drv->structVirt, 0, PORT_PAGES_STRUCT * 4096);
drv->bouncePhys = MmAllocateContiguous(BOUNCE_PAGES);
drv->bouncePhys = MmPfnAllocateContiguous(BOUNCE_PAGES, MmPfnUseDma);
if (!drv->bouncePhys) return STATUS_NO_MEMORY;
drv->bounceVirt = g_hal->PhysToVirt(drv->bouncePhys);
// Bounce buffer: NOCACHE to ensure CPU writes reach physical memory
// before the HBA's DMA engine reads them.
MmSetPageProtect((uintptr_t)drv->bounceVirt,
MmPteProtectRange((uintptr_t)drv->bounceVirt,
BOUNCE_PAGES * 4096, flags);
memset(drv->bounceVirt, 0, BOUNCE_BYTES);
@@ -203,6 +203,9 @@ static void AhciIsr(void) {
// Blocks the current thread until the ISR signals completionEvent.
static KSTATUS port_issue_cmd(AHCI_DRIVE *drv) {
if (drv->controller && drv->controller->pollingMode)
return port_issue_cmd_poll(drv);
HBA_PORT *port = drv->port;
// Clear stale IS and snapshot before issuing so we don't mis-read a
@@ -647,12 +650,25 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
ctl->pciInfo.VendorId,
ctl->pciInfo.DeviceId);
st = PciGetInterrupt(ctl->pdo, &ctl->irq);
if (!KSUCCESS(st) || ctl->irq == 0xFF) {
DbgPrint("AHCI: no PCI IRQ wired - cannot use interrupt-driven mode\n");
return KSUCCESS(st) ? STATUS_NOT_SUPPORTED : st;
ctl->irq = 0xFFu;
ctl->usingMsi = false;
ctl->pollingMode = false;
st = PciEnableMsiAuto(ctl->pdo, &ctl->irq);
if (KSUCCESS(st)) {
ctl->usingMsi = true;
DbgPrint("AHCI: using MSI IRQ slot %u\n", (unsigned)ctl->irq);
} else {
KSTATUS intxStatus = PciGetInterrupt(ctl->pdo, &ctl->irq);
if (KSUCCESS(intxStatus) && ctl->irq != 0xFFu) {
DbgPrint("AHCI: using legacy INTx IRQ %u\n", (unsigned)ctl->irq);
} else {
ctl->irq = 0xFFu;
ctl->pollingMode = true;
DbgPrint("AHCI: no MSI/INTx interrupt available; using polled command completion (msi=0x%x intx=0x%x)\n",
st, intxStatus);
}
}
DbgPrint("AHCI: PCI IRQ line = %u\n", (unsigned)ctl->irq);
st = PciGetBar(ctl->pdo, AHCI_PCI_BAR_ABAR, &ctl->abar);
if (!KSUCCESS(st)) {
@@ -674,10 +690,13 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
return st;
}
st = PciEnableDevice(ctl->pdo,
PCI_ENABLE_MEMORY |
PCI_ENABLE_BUSMASTER |
PCI_ENABLE_INTX);
uint32_t enableFlags = PCI_ENABLE_MEMORY | PCI_ENABLE_BUSMASTER;
if (ctl->usingMsi || ctl->pollingMode)
enableFlags |= PCI_DISABLE_INTX;
else
enableFlags |= PCI_ENABLE_INTX;
st = PciEnableDevice(ctl->pdo, enableFlags);
if (!KSUCCESS(st)) {
DbgPrint("AHCI: PciEnableDevice failed: 0x%x\n", st);
return st;
@@ -831,11 +850,16 @@ static KSTATUS AhciStartController(AHCI_CONTROLLER* ctl) {
ctl->pciInfo.Address.Device,
ctl->pciInfo.Address.Function);
KeIrqRegister(ctl->irq, AhciIsr);
if (!ctl->pollingMode) {
KeIrqRegister(ctl->irq, AhciIsr);
DbgPrint("AHCI: ISR registered on IRQ %u (%s); controller drives=%d\n",
(unsigned)ctl->irq, ctl->usingMsi ? "MSI" : "INTx", ctl->driveCount);
} else {
DbgPrint("AHCI: controller started in polling mode; controller drives=%d\n",
ctl->driveCount);
}
ctl->started = true;
ctl->fdo->PnpState = DevicePnpStarted;
DbgPrint("AHCI: ISR registered on IRQ %u; controller drives=%d\n",
(unsigned)ctl->irq, ctl->driveCount);
return STATUS_SUCCESS;
}
code/drivers/storage/ahci/ahci.h
+3
View File
@@ -8,6 +8,7 @@
#include <ntlite/ddi/io_driver.h>
#include <ntlite/ddi/pci.h>
#include <kernel/mm_phys.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ke_event.h>
@@ -367,6 +368,8 @@ struct AHCI_CONTROLLER {
HBA_MEM* hba;
uint64_t abarLength;
uint8_t irq;
bool usingMsi;
bool pollingMode;
AHCI_DRIVE drives[MAX_AHCI_DRIVES];
int driveCount;
code/drivers/storage/ahci/ahci_port.c
+4 -4
View File
@@ -12,15 +12,15 @@ KSTATUS ahci_map_mmio(uint64_t physBase, uint64_t length) {
for (uint64_t phys = pageBase; phys < pageEnd; phys += PAGE_SIZE, pageIndex++) {
uintptr_t virt = (uintptr_t)g_hal->PhysToVirt(phys);
uint64_t flags = PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE;
KSTATUS st = MmMapPage(virt, phys, flags, MmMapFailIfPresent);
KSTATUS st = MmPteMapPage(virt, phys, flags, MmPteMapFailIfPresent);
// Ignore STATUS_ALREADY_EXISTS - a prior mapping with different flags
// is possible for the first page if the bootloader was overly generous.
// Use MmSetPageProtect to force NOCACHE in that case.
// Use MmPteProtectRange to force NOCACHE in that case.
if (st == STATUS_ALREADY_EXISTS) {
st = MmSetPageProtect(virt, 4096, flags);
st = MmPteProtectRange(virt, 4096, flags);
}
if (!KSUCCESS(st)) {
DbgPrint("AHCI: MmMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
DbgPrint("AHCI: MmPteMapPage failed for page %d (phys=0x%016llX, st=0x%x)\n",
pageIndex, (unsigned long long)phys, st);
return st;
}