maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add page cache with read-ahead and dirty writeback, file-backed and anonymous mmap with CoW, VAD-based virtual memory replacing the old user region model, PFN database with eviction and reclaim, file identity for cross-open section sharing, FAT32 path cache and per-file cluster chain cache, serial.sys for COM1 keyboard input, xHCI MSI with INTx and polled fallback, PciEnableMsiAuto, waitable input queue, KeWaitForAnyEvent, per-process thread info syscall, expanded kernel stats (PFN/PTE/VAD/file-cache/disk), pool allocation tags, async I/O user buffer locking, file flush on close, add docs.md.

Browse Source
This commit is contained in:
maksw committed 2026-06-20 13:17:39 +02:00
1 parent 61ec2442eb
commit 2c4a68a75f
114 files changed
+18069 -2620
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/io_device.c
+336 -6
View File
@@ -7,6 +7,7 @@
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
#include <kernel/ke_sched.h>
#define MAX_DRIVER_OBJECTS 64
#define MAX_PDO_OBJECTS 128
@@ -443,6 +444,170 @@ static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
#define IO_DISK_READAHEAD_MAX_SECTORS 128u
#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u
#define IO_DISK_READAHEAD_MIN_SECTORS 8u
#define IO_DISK_SECTOR_BYTES 512u
typedef struct IO_DISK_READAHEAD_STATE {
SPINLOCK Lock;
uint8_t* Buffer;
uint8_t Valid;
uint8_t Loading;
uint64_t Generation;
uint64_t BaseLba;
uint32_t Count;
uint64_t NextLba;
uint32_t SequentialScore;
IO_DISK_STATS Stats;
} IO_DISK_READAHEAD_STATE;
static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES];
// Updates a maximum statistic while holding the disk read-ahead lock.
static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) {
if (field && *field < value) *field = value;
}
// Prepares the per-disk read-ahead buffer.
static void IoDiskReadAheadPrepare(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
state->Lock = (SPINLOCK)SPINLOCK_INIT;
state->Valid = 0;
state->Loading = 0;
state->Generation = 0;
state->BaseLba = 0;
state->Count = 0;
state->NextLba = 0;
state->SequentialScore = 0;
memset(&state->Stats, 0, sizeof(state->Stats));
if (!state->Buffer) {
state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES,
MM_POOL_TAG('d', 'r', 'a', 'h'));
if (!state->Buffer)
state->Stats.ReadaheadAllocFailures++;
}
}
// Releases the per-disk read-ahead buffer.
static void IoDiskReadAheadRelease(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uint8_t* buffer = (uint8_t*)0;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (!state->Loading) {
buffer = state->Buffer;
state->Buffer = (uint8_t*)0;
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->SequentialScore = 0;
KeReleaseSpinlock(&state->Lock, flags);
break;
}
KeReleaseSpinlock(&state->Lock, flags);
KeYield();
}
if (buffer)
MmFreePool(buffer);
}
// Invalidates cached disk read-ahead data for one disk.
static void IoDiskReadAheadInvalidate(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Returns true when one range fully contains another range.
static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount,
uint64_t innerBase, uint32_t innerCount) {
uint64_t outerEnd = outerBase + outerCount;
uint64_t innerEnd = innerBase + innerCount;
if (outerEnd < outerBase || innerEnd < innerBase)
return false;
return innerBase >= outerBase && innerEnd <= outerEnd;
}
// Attempts to satisfy a disk read from the per-disk read-ahead buffer.
static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) {
if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0)
return false;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool hit = state->Buffer && state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, lba, count);
if (hit) {
size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES;
size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES;
memcpy(buffer, state->Buffer + offset, bytes);
state->Stats.ReadaheadHits++;
state->Stats.ReadaheadBytesServed += bytes;
} else {
state->Stats.ReadaheadMisses++;
}
KeReleaseSpinlock(&state->Lock, flags);
return hit;
}
// Records request size and sequential access state for one disk read.
static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES || count == 0)
return 0;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadRequests++;
state->Stats.ReadSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count);
if (lba == state->NextLba) {
if (state->SequentialScore < 32)
state->SequentialScore++;
} else {
state->SequentialScore = 1;
}
state->NextLba = lba + count;
uint32_t score = state->SequentialScore;
if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE)
state->Stats.ReadaheadSequentialDetections++;
KeReleaseSpinlock(&state->Lock, flags);
return score;
}
// Records one disk write and invalidates stale read-ahead data.
static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.WriteRequests++;
state->Stats.WriteSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Implements the notify Disk Change operation.
static void IoNotifyDiskChange(uint8_t driveIndex, DEVICE_OBJECT* device, IO_DEVICE_CHANGE change) {
for (uint32_t i = 0; i < gDiskChangeCallbackCount; i++) {
@@ -477,6 +642,8 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
if (pnpStatus == STATUS_NOT_SUPPORTED)
device->PnpState = DevicePnpStarted;
IoDiskReadAheadPrepare(driveIndex);
IoReferenceDeviceObject(device);
gDiskDevices[driveIndex] = device;
IoNotifyDiskChange(driveIndex, device, IoDeviceArrived);
@@ -490,6 +657,8 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return STATUS_NOT_FOUND;
IoDiskReadAheadInvalidate(driveIndex);
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
@@ -518,6 +687,7 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoDiskReadAheadRelease(driveIndex);
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
@@ -632,6 +802,112 @@ __declspec(dllexport) KSTATUS IoCallDriver(DEVICE_OBJECT* device, IRP* irp) {
return st;
}
// Validates that a sector range fits the disk and IRP byte length.
static KSTATUS IoValidateDiskSectorRange(DEVICE_OBJECT* dev, uint64_t lba, uint32_t count) {
if (!dev || count == 0) return STATUS_INVALID_PARAMETER;
if (count > (0xffffffffu / IO_DISK_SECTOR_BYTES)) return STATUS_INVALID_PARAMETER;
if (lba > dev->u.Disk.SectorCount) return STATUS_INVALID_PARAMETER;
if ((uint64_t)count > dev->u.Disk.SectorCount - lba) return STATUS_INVALID_PARAMETER;
return STATUS_SUCCESS;
}
// Submits one raw disk read IRP.
static KSTATUS IoReadDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
uint32_t count, void* buffer) {
if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
return IoSubmitIrpSync(dev, &irp);
}
// Submits one raw disk write IRP.
static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
uint32_t count, void* buffer) {
if (!dev || !buffer || count == 0) return STATUS_INVALID_PARAMETER;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, buffer);
return IoSubmitIrpSync(dev, &irp);
}
// Chooses a conservative read-ahead span from the current sequential score.
static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) {
uint32_t sectors = currentCount;
if (sectors < IO_DISK_READAHEAD_MIN_SECTORS)
sectors = IO_DISK_READAHEAD_MIN_SECTORS;
if (score >= 8 && sectors < 128u)
sectors = 128u;
else if (score >= 4 && sectors < 64u)
sectors = 64u;
else if (score >= 2 && sectors < 32u)
sectors = 32u;
if (sectors > IO_DISK_READAHEAD_MAX_SECTORS)
sectors = IO_DISK_READAHEAD_MAX_SECTORS;
return sectors;
}
// Loads the next sequential disk range into the read-ahead buffer.
static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev,
uint64_t nextLba, uint32_t currentCount, uint32_t score) {
if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
if (!state->Buffer) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score);
uint64_t diskSectors = dev->u.Disk.SectorCount;
if (nextLba >= diskSectors)
return;
if ((uint64_t)sectors > diskSectors - nextLba)
sectors = (uint32_t)(diskSectors - nextLba);
if (sectors == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool covered = state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors);
if (covered) {
KeReleaseSpinlock(&state->Lock, flags);
return;
}
if (state->Loading) {
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint64_t generation = state->Generation;
state->Loading = 1;
state->Valid = 0;
state->Count = 0;
KeReleaseSpinlock(&state->Lock, flags);
KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer);
flags = KeAcquireSpinlock(&state->Lock);
state->Loading = 0;
if (KSUCCESS(st) && generation == state->Generation) {
state->BaseLba = nextLba;
state->Count = sectors;
state->Valid = 1;
state->Stats.ReadaheadStores++;
state->Stats.ReadaheadSectorsLoaded += sectors;
IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors);
} else {
state->Valid = 0;
state->Count = 0;
if (!KSUCCESS(st))
state->Stats.ReadaheadReadFailures++;
}
KeReleaseSpinlock(&state->Lock, flags);
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
@@ -641,9 +917,23 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_READ, lba * 512ull, count * 512u, buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
if (!KSUCCESS(rangeStatus)) {
IoDereferenceDeviceObject(dev);
return rangeStatus;
}
if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) {
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return STATUS_SUCCESS;
}
KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
if (KSUCCESS(st))
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return st;
@@ -658,10 +948,50 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return STATUS_NOT_FOUND;
IRP irp;
IoBuildReadWriteIrp(&irp, IRP_MJ_WRITE, lba * 512ull, count * 512u, (void*)buffer);
KSTATUS st = IoSubmitIrpSync(dev, &irp);
KSTATUS rangeStatus = IoValidateDiskSectorRange(dev, lba, count);
if (!KSUCCESS(rangeStatus)) {
IoDereferenceDeviceObject(dev);
return rangeStatus;
}
KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
if (KSUCCESS(st))
IoDiskReadAheadNoteWrite(driveIndex, count);
IoDereferenceDeviceObject(dev);
return st;
}
// Queries aggregate disk I/O statistics.
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (gDiskDevices[i])
out->RegisteredDisks++;
out->ReadRequests += state->Stats.ReadRequests;
out->ReadSectors += state->Stats.ReadSectors;
out->WriteRequests += state->Stats.WriteRequests;
out->WriteSectors += state->Stats.WriteSectors;
if (out->MaxReadSectors < state->Stats.MaxReadSectors)
out->MaxReadSectors = state->Stats.MaxReadSectors;
if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
out->MaxWriteSectors = state->Stats.MaxWriteSectors;
out->ReadaheadHits += state->Stats.ReadaheadHits;
out->ReadaheadMisses += state->Stats.ReadaheadMisses;
out->ReadaheadStores += state->Stats.ReadaheadStores;
out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations;
out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed;
out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded;
out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections;
out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed;
out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures;
out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures;
if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors)
out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors;
KeReleaseSpinlock(&state->Lock, flags);
}
}