maksw/nt-lite
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

Browse Source
This commit is contained in:
maksw committed 2026-08-11 18:38:36 +02:00
1 parent 8b254f84f4
commit 4add6789ba
177 files changed
+22565 -3070
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

code/kernel/io_device.c
+51 -240
View File
@@ -4,6 +4,7 @@
#include <intrin.h>
#include <ntlite/ddi/io_driver.h>
#include <kernel/ke_irp.h>
#include <kernel/io_dma.h>
#include <kernel/mm_virt.h>
#include <kernel/ke_debug.h>
#include <kernel/ob_object.h>
@@ -107,6 +108,7 @@ __declspec(dllexport) KSTATUS IoRegisterDriverObject(DRIVER_OBJECT* driver) {
static void IoDeviceDestroy(OBJECT* object) {
DEVICE_OBJECT* dev = (DEVICE_OBJECT*)object;
IoDmaDetachDevice(dev);
for (uint32_t i = 0; i < gPhysicalDeviceCount; i++) {
if (gPhysicalDevices[i] == dev) {
@@ -316,12 +318,19 @@ static void IoUnlinkDeviceFromParent(DEVICE_OBJECT* device) {
// Releases device.
__declspec(dllexport) void IoDeleteDevice(DEVICE_OBJECT* device) {
if (!device) return;
if (device->DeletePending)
if (!device)
return;
uintptr_t flags = KeAcquireSpinlock(&device->Lock);
if (device->DeletePending) {
KeReleaseSpinlock(&device->Lock, flags);
return;
}
device->DeletePending = true;
device->PnpState = DevicePnpDeleted;
KeReleaseSpinlock(&device->Lock, flags);
IoDmaDetachDevice(device);
IoUnlinkDeviceFromParent(device);
IoRemovePhysicalDeviceRegistryEntry(device);
IoDereferenceDeviceObject(device);
@@ -441,167 +450,50 @@ static uint32_t gDiskChangeCallbackCount;
static IO_DISK_ARRIVAL_CALLBACK_ENTRY gDiskArrivalCallbacks[MAX_DISK_CALLBACKS];
static uint32_t gDiskArrivalCallbackCount;
#define IO_DISK_READAHEAD_MAX_SECTORS 128u
#define IO_DISK_READAHEAD_TRIGGER_SCORE 2u
#define IO_DISK_READAHEAD_MIN_SECTORS 8u
#define IO_DISK_SECTOR_BYTES 512u
typedef struct IO_DISK_READAHEAD_STATE {
typedef struct IO_DISK_STATS_STATE {
SPINLOCK Lock;
uint8_t* Buffer;
uint8_t Valid;
uint8_t Loading;
uint64_t Generation;
uint64_t BaseLba;
uint32_t Count;
uint64_t NextLba;
uint32_t SequentialScore;
IO_DISK_STATS Stats;
} IO_DISK_READAHEAD_STATE;
} IO_DISK_STATS_STATE;
static IO_DISK_READAHEAD_STATE gDiskReadAhead[MAX_DISK_DEVICES];
static IO_DISK_STATS_STATE gDiskStats[MAX_DISK_DEVICES];
// Updates a maximum statistic while holding the disk read-ahead lock.
static void IoDiskStatMaxLocked(uint64_t* field, uint64_t value) {
if (field && *field < value) *field = value;
}
// Prepares the per-disk read-ahead buffer.
static void IoDiskReadAheadPrepare(uint8_t driveIndex) {
// Initializes per-disk I/O statistics.
static void IoDiskStatsPrepare(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
state->Lock = (SPINLOCK)SPINLOCK_INIT;
state->Valid = 0;
state->Loading = 0;
state->Generation = 0;
state->BaseLba = 0;
state->Count = 0;
state->NextLba = 0;
state->SequentialScore = 0;
memset(&state->Stats, 0, sizeof(state->Stats));
if (!state->Buffer) {
state->Buffer = (uint8_t*)MmAllocatePoolTag(IO_DISK_READAHEAD_MAX_SECTORS * IO_DISK_SECTOR_BYTES,
MM_POOL_TAG('d', 'r', 'a', 'h'));
if (!state->Buffer)
state->Stats.ReadaheadAllocFailures++;
}
}
// Releases the per-disk read-ahead buffer.
static void IoDiskReadAheadRelease(uint8_t driveIndex) {
// Records one disk read request.
static void IoDiskStatsNoteRead(uint8_t driveIndex, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uint8_t* buffer = (uint8_t*)0;
for (;;) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (!state->Loading) {
buffer = state->Buffer;
state->Buffer = (uint8_t*)0;
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->SequentialScore = 0;
KeReleaseSpinlock(&state->Lock, flags);
break;
}
KeReleaseSpinlock(&state->Lock, flags);
KeYield();
}
if (buffer)
MmFreePool(buffer);
}
// Invalidates cached disk read-ahead data for one disk.
static void IoDiskReadAheadInvalidate(uint8_t driveIndex) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
KeReleaseSpinlock(&state->Lock, flags);
}
// Returns true when one range fully contains another range.
static bool IoDiskRangeContains(uint64_t outerBase, uint32_t outerCount,
uint64_t innerBase, uint32_t innerCount) {
uint64_t outerEnd = outerBase + outerCount;
uint64_t innerEnd = innerBase + innerCount;
if (outerEnd < outerBase || innerEnd < innerBase)
return false;
return innerBase >= outerBase && innerEnd <= outerEnd;
}
// Attempts to satisfy a disk read from the per-disk read-ahead buffer.
static bool IoDiskReadAheadTryHit(uint8_t driveIndex, uint64_t lba, uint32_t count, void* buffer) {
if (driveIndex >= MAX_DISK_DEVICES || !buffer || count == 0)
return false;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool hit = state->Buffer && state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, lba, count);
if (hit) {
size_t offset = (size_t)(lba - state->BaseLba) * IO_DISK_SECTOR_BYTES;
size_t bytes = (size_t)count * IO_DISK_SECTOR_BYTES;
memcpy(buffer, state->Buffer + offset, bytes);
state->Stats.ReadaheadHits++;
state->Stats.ReadaheadBytesServed += bytes;
} else {
state->Stats.ReadaheadMisses++;
}
KeReleaseSpinlock(&state->Lock, flags);
return hit;
}
// Records request size and sequential access state for one disk read.
static uint32_t IoDiskReadAheadNoteRead(uint8_t driveIndex, uint64_t lba, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES || count == 0)
return 0;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadRequests++;
state->Stats.ReadSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxReadSectors, count);
if (lba == state->NextLba) {
if (state->SequentialScore < 32)
state->SequentialScore++;
} else {
state->SequentialScore = 1;
}
state->NextLba = lba + count;
uint32_t score = state->SequentialScore;
if (score >= IO_DISK_READAHEAD_TRIGGER_SCORE)
state->Stats.ReadaheadSequentialDetections++;
if (state->Stats.MaxReadSectors < count)
state->Stats.MaxReadSectors = count;
KeReleaseSpinlock(&state->Lock, flags);
return score;
}
// Records one disk write and invalidates stale read-ahead data.
static void IoDiskReadAheadNoteWrite(uint8_t driveIndex, uint32_t count) {
// Records one successful disk write request.
static void IoDiskStatsNoteWrite(uint8_t driveIndex, uint32_t count) {
if (driveIndex >= MAX_DISK_DEVICES)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
IO_DISK_STATS_STATE* state = &gDiskStats[driveIndex];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.WriteRequests++;
state->Stats.WriteSectors += count;
IoDiskStatMaxLocked(&state->Stats.MaxWriteSectors, count);
state->Valid = 0;
state->Generation++;
state->Count = 0;
state->Stats.ReadaheadInvalidations++;
if (state->Stats.MaxWriteSectors < count)
state->Stats.MaxWriteSectors = count;
KeReleaseSpinlock(&state->Lock, flags);
}
@@ -653,7 +545,7 @@ __declspec(dllexport) KSTATUS IoRegisterDiskDevice(uint8_t driveIndex,
if (pnpStatus == NTLSTATUS_NOT_SUPPORTED)
device->PnpState = DevicePnpStarted;
IoDiskReadAheadPrepare(driveIndex);
IoDiskStatsPrepare(driveIndex);
IoReferenceDeviceObject(device);
gDiskDevices[driveIndex] = device;
@@ -668,8 +560,6 @@ __declspec(dllexport) KSTATUS IoNotifyDiskSurpriseRemoved(uint8_t driveIndex) {
DEVICE_OBJECT* dev = gDiskDevices[driveIndex];
if (!dev) return NTLSTATUS_NOT_FOUND;
IoDiskReadAheadInvalidate(driveIndex);
dev->PnpState = DevicePnpSurpriseRemoved;
IoNotifyDiskChange(driveIndex, dev, IoDeviceSurpriseRemoved);
(void)IoSendPnpIrpSync(dev, IRP_MN_SURPRISE_REMOVAL);
@@ -698,7 +588,6 @@ __declspec(dllexport) KSTATUS IoUnregisterDiskDevice(uint8_t driveIndex) {
(void)IoSendPnpIrpSync(dev, IRP_MN_REMOVE_DEVICE);
gDiskDevices[driveIndex] = (DEVICE_OBJECT*)0;
IoDiskReadAheadRelease(driveIndex);
IoNotifyDiskChange(driveIndex, dev, IoDeviceRemoved);
IoDereferenceDeviceObject(dev);
return NTLSTATUS_SUCCESS;
@@ -846,83 +735,6 @@ static KSTATUS IoWriteDiskSectorsRaw(DEVICE_OBJECT* dev, uint64_t lba,
return IoSubmitIrpSync(dev, &irp);
}
// Chooses a conservative read-ahead span from the current sequential score.
static uint32_t IoDiskChooseReadAheadSectors(uint32_t currentCount, uint32_t score) {
uint32_t sectors = currentCount;
if (sectors < IO_DISK_READAHEAD_MIN_SECTORS)
sectors = IO_DISK_READAHEAD_MIN_SECTORS;
if (score >= 8 && sectors < 128u)
sectors = 128u;
else if (score >= 4 && sectors < 64u)
sectors = 64u;
else if (score >= 2 && sectors < 32u)
sectors = 32u;
if (sectors > IO_DISK_READAHEAD_MAX_SECTORS)
sectors = IO_DISK_READAHEAD_MAX_SECTORS;
return sectors;
}
// Loads the next sequential disk range into the read-ahead buffer.
static void IoDiskMaybeReadAhead(uint8_t driveIndex, DEVICE_OBJECT* dev,
uint64_t nextLba, uint32_t currentCount, uint32_t score) {
if (driveIndex >= MAX_DISK_DEVICES || !dev || score < IO_DISK_READAHEAD_TRIGGER_SCORE)
return;
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[driveIndex];
if (!state->Buffer) {
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint32_t sectors = IoDiskChooseReadAheadSectors(currentCount, score);
uint64_t diskSectors = dev->u.Disk.SectorCount;
if (nextLba >= diskSectors)
return;
if ((uint64_t)sectors > diskSectors - nextLba)
sectors = (uint32_t)(diskSectors - nextLba);
if (sectors == 0)
return;
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
bool covered = state->Valid && !state->Loading &&
IoDiskRangeContains(state->BaseLba, state->Count, nextLba, sectors);
if (covered) {
KeReleaseSpinlock(&state->Lock, flags);
return;
}
if (state->Loading) {
state->Stats.ReadaheadSuppressed++;
KeReleaseSpinlock(&state->Lock, flags);
return;
}
uint64_t generation = state->Generation;
state->Loading = 1;
state->Valid = 0;
state->Count = 0;
KeReleaseSpinlock(&state->Lock, flags);
KSTATUS st = IoReadDiskSectorsRaw(dev, nextLba, sectors, state->Buffer);
flags = KeAcquireSpinlock(&state->Lock);
state->Loading = 0;
if (KSUCCESS(st) && generation == state->Generation) {
state->BaseLba = nextLba;
state->Count = sectors;
state->Valid = 1;
state->Stats.ReadaheadStores++;
state->Stats.ReadaheadSectorsLoaded += sectors;
IoDiskStatMaxLocked(&state->Stats.MaxReadaheadSectors, sectors);
} else {
state->Valid = 0;
state->Count = 0;
if (!KSUCCESS(st))
state->Stats.ReadaheadReadFailures++;
}
KeReleaseSpinlock(&state->Lock, flags);
}
__declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
uint64_t lba,
uint32_t count,
@@ -938,17 +750,8 @@ __declspec(dllexport) KSTATUS IoReadDiskSectors(uint8_t driveIndex,
return rangeStatus;
}
if (IoDiskReadAheadTryHit(driveIndex, lba, count, buffer)) {
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDereferenceDeviceObject(dev);
return NTLSTATUS_SUCCESS;
}
KSTATUS st = IoReadDiskSectorsRaw(dev, lba, count, buffer);
uint32_t score = IoDiskReadAheadNoteRead(driveIndex, lba, count);
if (KSUCCESS(st))
IoDiskMaybeReadAhead(driveIndex, dev, lba + count, count, score);
IoDiskStatsNoteRead(driveIndex, count);
IoDereferenceDeviceObject(dev);
return st;
@@ -971,19 +774,39 @@ __declspec(dllexport) KSTATUS IoWriteDiskSectors(uint8_t driveIndex,
KSTATUS st = IoWriteDiskSectorsRaw(dev, lba, count, (void*)buffer);
if (KSUCCESS(st))
IoDiskReadAheadNoteWrite(driveIndex, count);
IoDiskStatsNoteWrite(driveIndex, count);
IoDereferenceDeviceObject(dev);
return st;
}
// Flushes volatile write caches for one disk device.
__declspec(dllexport) KSTATUS IoFlushDisk(uint8_t driveIndex) {
DEVICE_OBJECT* dev = IoReferenceDiskDevice(driveIndex);
if (!dev) return NTLSTATUS_NOT_FOUND;
IRP irp;
IoInitializeIrp(&irp, 1);
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(&irp);
if (!ios) {
IoDereferenceDeviceObject(dev);
return NTLSTATUS_FAILURE;
}
ios->MajorFunction = IRP_MJ_FLUSH_BUFFERS;
ios->MinorFunction = 0;
KSTATUS st = IoSubmitIrpSync(dev, &irp);
IoDereferenceDeviceObject(dev);
return st;
}
// Queries aggregate disk I/O statistics.
__declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
if (!out) return;
memset(out, 0, sizeof(*out));
for (uint8_t i = 0; i < MAX_DISK_DEVICES; i++) {
IO_DISK_READAHEAD_STATE* state = &gDiskReadAhead[i];
IO_DISK_STATS_STATE* state = &gDiskStats[i];
uintptr_t flags = KeAcquireSpinlock(&state->Lock);
if (gDiskDevices[i])
out->RegisteredDisks++;
@@ -995,18 +818,6 @@ __declspec(dllexport) void IoQueryDiskStats(IO_DISK_STATS* out) {
out->MaxReadSectors = state->Stats.MaxReadSectors;
if (out->MaxWriteSectors < state->Stats.MaxWriteSectors)
out->MaxWriteSectors = state->Stats.MaxWriteSectors;
out->ReadaheadHits += state->Stats.ReadaheadHits;
out->ReadaheadMisses += state->Stats.ReadaheadMisses;
out->ReadaheadStores += state->Stats.ReadaheadStores;
out->ReadaheadInvalidations += state->Stats.ReadaheadInvalidations;
out->ReadaheadBytesServed += state->Stats.ReadaheadBytesServed;
out->ReadaheadSectorsLoaded += state->Stats.ReadaheadSectorsLoaded;
out->ReadaheadSequentialDetections += state->Stats.ReadaheadSequentialDetections;
out->ReadaheadSuppressed += state->Stats.ReadaheadSuppressed;
out->ReadaheadAllocFailures += state->Stats.ReadaheadAllocFailures;
out->ReadaheadReadFailures += state->Stats.ReadaheadReadFailures;
if (out->MaxReadaheadSectors < state->Stats.MaxReadaheadSectors)
out->MaxReadaheadSectors = state->Stats.MaxReadaheadSectors;
KeReleaseSpinlock(&state->Lock, flags);
}
}