nt-lite/code/kernel/io_file.c

// Kernel file I/O layer. This file owns file handles, filesystem dispatch, and asynchronous request plumbing.

#include <string.h>
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/mm_file_cache.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_userenv.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_event.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <ntlite/syscall.h>

#define MAX_DRIVES  8

typedef struct {
    IO_FILESYSTEM   *Fs;
    IO_VOLUME_PARAMS Vol;
} VOLUME_ENTRY;

static VOLUME_ENTRY gVolumes[MAX_DRIVES];

// Async file I/O worker.
typedef enum IO_ASYNC_FILE_OP {
    IoAsyncFileRead  = 0,
    IoAsyncFileWrite = 1,
} IO_ASYNC_FILE_OP;

typedef enum IO_ASYNC_FILE_STATE {
    IoAsyncStateQueued    = 0,
    IoAsyncStateRunning   = 1,
    IoAsyncStateCompleted = 2,
    IoAsyncStateCancelled = 3,
} IO_ASYNC_FILE_STATE;

struct IO_ASYNC_FILE_REQUEST_OBJECT {
    OBJECT Header;                    // ObjectTypeAsyncIo
    LIST_ENTRY Link;                  // protected by gAsyncFileQueueLock

    IO_ASYNC_FILE_OP Op;
    IO_ASYNC_FILE_STATE State;
    uint8_t CancelRequested;

    FILE_OBJECT* File;                // referenced until object destruction
    EVENT* WaitEvent;                 // internal completion event
    EVENT* CompletionEvent;           // optional, referenced until object destruction
    KAPC* CompletionApc;              // caller-owned APC storage
    IO_STATUS_BLOCK* Iosb;            // caller/user-owned completion block
    IO_STATUS_BLOCK IosbSnapshot;
    MM_LOCKED_USER_BUFFER* LockedBuffer;
    MM_LOCKED_USER_BUFFER* LockedIosb;

    void* Buffer;                     // caller-owned; user buffers must be locked until completion/cancel
    size_t Size;
    uint64_t Offset;
};

static SPINLOCK   gAsyncFileInitLock = SPINLOCK_INIT;
static int        gAsyncFileInitState;     // 0=off, 1=starting, 2=ready
static SPINLOCK   gAsyncFileQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gAsyncFileQueue;
static SEMAPHORE* gAsyncFileSemaphore;

// Implements the io Async File Request Destroy operation.
static void KiIoAsyncFileRequestDestroy(OBJECT* obj) {
    IO_ASYNC_FILE_REQUEST_OBJECT* req =
        CONTAINING_RECORD(obj, IO_ASYNC_FILE_REQUEST_OBJECT, Header);

    if (req->WaitEvent) {
        ObDereferenceObject(&req->WaitEvent->Header);
        req->WaitEvent = (EVENT*)0;
    }
    if (req->CompletionEvent) {
        ObDereferenceObject(&req->CompletionEvent->Header);
        req->CompletionEvent = (EVENT*)0;
    }
    if (req->File) {
        ObDereferenceObject(&req->File->Header);
        req->File = (FILE_OBJECT*)0;
    }
    if (req->LockedBuffer) {
        MmUnlockUserBuffer(req->LockedBuffer);
        req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
    }
    if (req->LockedIosb) {
        MmUnlockUserBuffer(req->LockedIosb);
        req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
    }
}

static void KiIoCompleteAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
                                       KSTATUS status,
                                       size_t information) {
    if (!req) return;

    req->IosbSnapshot.Status = status;
    req->IosbSnapshot.Information = information;

    uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
    req->State = (status == STATUS_CANCELLED) ? IoAsyncStateCancelled : IoAsyncStateCompleted;
    KeReleaseSpinlock(&gAsyncFileQueueLock, flags);

    if (req->Iosb) {
        req->Iosb->Status = status;
        req->Iosb->Information = information;
    }

    // Match IRP completion ordering: queue APC first, then signal the event.
    if (req->CompletionApc) {
        if (!KeInsertQueueApc(req->CompletionApc, req->Iosb, (void*)(uintptr_t)status)) {
            if (req->CompletionApc->RundownRoutine)
                req->CompletionApc->RundownRoutine(req->CompletionApc);
        }
    }

    if (req->CompletionEvent)
        KeSignalEvent(req->CompletionEvent);

    if (req->WaitEvent)
        KeSignalEvent(req->WaitEvent);

    if (req->LockedBuffer) {
        MmUnlockUserBuffer(req->LockedBuffer);
        req->LockedBuffer = (MM_LOCKED_USER_BUFFER*)0;
    }
    if (req->LockedIosb) {
        MmUnlockUserBuffer(req->LockedIosb);
        req->LockedIosb = (MM_LOCKED_USER_BUFFER*)0;
    }
}

// Implements the io Async File Dequeue operation.
static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
    uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
    LIST_ENTRY* node = ListPopHead(&gAsyncFileQueue);
    if (!node) {
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
        return (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
    }

    IO_ASYNC_FILE_REQUEST_OBJECT* req =
        CONTAINING_RECORD(node, IO_ASYNC_FILE_REQUEST_OBJECT, Link);

    if (req->CancelRequested) {
        req->State = IoAsyncStateCancelled;
    } else {
        req->State = IoAsyncStateRunning;
    }
    KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
    return req;
}

// Implements the io Async File Worker operation.
static void KiIoAsyncFileWorker(void* arg) {
    (void)arg;

    for (;;) {
        KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore);
        if (!KSUCCESS(s)) {
            KeYield();
            continue;
        }

        IO_ASYNC_FILE_REQUEST_OBJECT* req = KiIoAsyncFileDequeue();
        if (!req) continue;

        if (req->CancelRequested || req->State == IoAsyncStateCancelled) {
            KiIoCompleteAsyncFileRequest(req, STATUS_CANCELLED, 0);
            ObDereferenceObject(&req->Header); // queue/worker reference
            continue;
        }

        size_t done = 0;
        if (req->Op == IoAsyncFileRead) {
            s = IoReadFileAt(req->File, req->Offset, req->Buffer,
                             req->Size, &done);
        } else {
            s = IoWriteFileAt(req->File, req->Offset, req->Buffer,
                              req->Size, &done);
        }

        if (req->CancelRequested) {
            // The operation may already have reached the filesystem. The
            // completion contract still reports cancellation so waiters/APCs do
            // not mistake a cancelled request for useful data.
            s = STATUS_CANCELLED;
            done = 0;
        }

        KiIoCompleteAsyncFileRequest(req, s, done);
        ObDereferenceObject(&req->Header); // queue/worker reference
    }
}

// Implements the io Ensure Async File Io Started operation.
static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
    uintptr_t flags = KeAcquireSpinlock(&gAsyncFileInitLock);
    if (gAsyncFileInitState == 2) {
        KeReleaseSpinlock(&gAsyncFileInitLock, flags);
        return STATUS_SUCCESS;
    }
    if (gAsyncFileInitState == 1) {
        KeReleaseSpinlock(&gAsyncFileInitLock, flags);
        while (gAsyncFileInitState == 1)
            KeYield();
        return (gAsyncFileInitState == 2) ? STATUS_SUCCESS : STATUS_FAILURE;
    }

    gAsyncFileInitState = 1;
    KeReleaseSpinlock(&gAsyncFileInitLock, flags);

    LIST_INIT(&gAsyncFileQueue);

    KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore);
    if (KSUCCESS(s)) {
        THREAD* worker = (THREAD*)0;
        s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker,
                                 0,
                                 PriorityNormal,
                                 &worker);
        // Keep the creator reference: this worker is intentionally persistent
        // for the lifetime of the kernel.
        (void)worker;
    }

    flags = KeAcquireSpinlock(&gAsyncFileInitLock);
    gAsyncFileInitState = KSUCCESS(s) ? 2 : 0;
    KeReleaseSpinlock(&gAsyncFileInitLock, flags);

    return s;
}

static void KiIoNativeFsAsyncComplete(void* completeContext,
                                      KSTATUS status,
                                      size_t information) {
    IO_ASYNC_FILE_REQUEST_OBJECT* req =
        (IO_ASYNC_FILE_REQUEST_OBJECT*)completeContext;
    if (!req) return;

    if (req->CancelRequested) {
        status = STATUS_CANCELLED;
        information = 0;
    }

    KiIoCompleteAsyncFileRequest(req, status, information);
    ObDereferenceObject(&req->Header); // native filesystem provider reference
}

static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
                                         FILE_OBJECT* file,
                                         void* buffer,
                                         size_t size,
                                         uint64_t offset,
                                         IO_STATUS_BLOCK* iosb,
                                         EVENT* completionEvent,
                                         KAPC* completionApc,
                                         MM_LOCKED_USER_BUFFER* lockedBuffer,
                                         MM_LOCKED_USER_BUFFER* lockedIosb,
                                         IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
    if (!file || !iosb) {
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return STATUS_INVALID_PARAMETER;
    }
    if (!buffer && size) {
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return STATUS_INVALID_PARAMETER;
    }
    if (!KeCanWait()) {
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return STATUS_INVALID_IRQL;
    }

    iosb->Status = STATUS_PENDING;
    iosb->Information = 0;

    IO_ASYNC_FILE_REQUEST_OBJECT* req =
        (IO_ASYNC_FILE_REQUEST_OBJECT*)ObAllocateObject(ObjectTypeAsyncIo,
                                                       sizeof(IO_ASYNC_FILE_REQUEST_OBJECT));
    if (!req) {
        iosb->Status = STATUS_NO_MEMORY;
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return STATUS_NO_MEMORY;
    }

    req->Header.Destroy = KiIoAsyncFileRequestDestroy;
    LIST_INIT(&req->Link);
    req->Op = op;
    req->State = IoAsyncStateQueued;
    req->CancelRequested = 0;
    KSTATUS waitStatus = KeEventCreate(EventManualReset, 0, &req->WaitEvent);
    if (!KSUCCESS(waitStatus)) {
        iosb->Status = waitStatus;
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        ObDereferenceObject(&req->Header);
        return waitStatus;
    }

    req->File = file;
    req->CompletionEvent = completionEvent;
    req->CompletionApc = completionApc;
    req->Iosb = iosb;
    req->IosbSnapshot.Status = STATUS_PENDING;
    req->IosbSnapshot.Information = 0;
    req->LockedBuffer = lockedBuffer;
    req->LockedIosb = lockedIosb;
    req->Buffer = buffer;
    req->Size = size;
    req->Offset = offset;

    ObReferenceObject(&file->Header);
    if (completionEvent)
        ObReferenceObject(&completionEvent->Header);

    IO_FS_ASYNC_COMPLETE complete = KiIoNativeFsAsyncComplete;
    KSTATUS ns = STATUS_NOT_SUPPORTED;
    bool hasNative = (op == IoAsyncFileRead) ? (file->Fs->ReadAsync != 0)
                                             : (file->Fs->WriteAsync != 0);

    if (hasNative) {
        uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
        req->State = IoAsyncStateRunning;
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);

        ObReferenceObject(&req->Header); // native filesystem provider reference
        if (op == IoAsyncFileRead) {
            ns = file->Fs->ReadAsync(file->Fs, file->FileCtx, offset, buffer,
                                     size, complete, req);
        } else {
            ns = file->Fs->WriteAsync(file->Fs, file->FileCtx, offset, buffer,
                                      size, complete, req);
        }
        if (ns == STATUS_PENDING) {
            if (requestOut) {
                *requestOut = req;
            } else {
                ObDereferenceObject(&req->Header);
            }
            return STATUS_PENDING;
        }

        ObDereferenceObject(&req->Header); // provider rejected synchronously
        iosb->Status = ns;
        req->IosbSnapshot.Status = ns;
        req->IosbSnapshot.Information = 0;
        ObDereferenceObject(&req->Header); // creator reference
        return ns;
    }

    KSTATUS s = KiIoEnsureAsyncFileIoStarted();
    if (!KSUCCESS(s)) {
        iosb->Status = s;
        req->IosbSnapshot.Status = s;
        req->IosbSnapshot.Information = 0;
        ObDereferenceObject(&req->Header);
        return s;
    }

    // Queue ownership keeps the request alive until the worker completes or a
    // successful queued cancellation completes it from IoCancelAsyncFileRequest.
    ObReferenceObject(&req->Header);

    uintptr_t qflags = KeAcquireSpinlock(&gAsyncFileQueueLock);
    ListInsertTail(&gAsyncFileQueue, &req->Link);
    KeReleaseSpinlock(&gAsyncFileQueueLock, qflags);

    KeSignalSemaphore(gAsyncFileSemaphore);

    if (requestOut) {
        *requestOut = req;       // creator reference returned to caller
    } else {
        ObDereferenceObject(&req->Header);
    }

    return STATUS_PENDING;
}

__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf,
                                                    size_t size, uint64_t offset,
                                                    IO_STATUS_BLOCK* iosb,
                                                    EVENT* completionEvent,
                                                    KAPC* completionApc,
                                                    MM_LOCKED_USER_BUFFER* lockedBuffer,
                                                    MM_LOCKED_USER_BUFFER* lockedIosb,
                                                    IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
    return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset,
                                    iosb, completionEvent, completionApc,
                                    lockedBuffer, lockedIosb, requestOut);
}

__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
                                                     size_t size, uint64_t offset,
                                                     IO_STATUS_BLOCK* iosb,
                                                     EVENT* completionEvent,
                                                     KAPC* completionApc,
                                                     MM_LOCKED_USER_BUFFER* lockedBuffer,
                                                     MM_LOCKED_USER_BUFFER* lockedIosb,
                                                     IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
    return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset,
                                    iosb, completionEvent, completionApc,
                                    lockedBuffer, lockedIosb, requestOut);
}

// Implements the async File Request Object operation.
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
    return req ? &req->Header : (OBJECT*)0;
}


// Returns the wait event owned by an async file request.
__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
    return req ? req->WaitEvent : (EVENT*)0;
}

// Waits for an async file request to complete according to wait flags.
__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, uint32_t flags) {
    if (!req || !req->WaitEvent) return STATUS_INVALID_PARAMETER;
    if (flags & ~SYS_WAIT_FLAG_ALERTABLE) return STATUS_INVALID_PARAMETER;
    return KeWaitForEvent(req->WaitEvent, (flags & SYS_WAIT_FLAG_ALERTABLE) ? 1 : 0);
}

// Cancels async File Request.
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
    if (!req) return STATUS_INVALID_PARAMETER;

    IO_ASYNC_FILE_STATE state;
    uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
    state = req->State;

    if (state == IoAsyncStateQueued) {
        ListRemove(&req->Link);
        req->CancelRequested = 1;
        req->State = IoAsyncStateCancelled;
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);

        KiIoCompleteAsyncFileRequest(req, STATUS_CANCELLED, 0);
        ObDereferenceObject(&req->Header); // queue reference removed above
        return STATUS_SUCCESS;
    }

    if (state == IoAsyncStateRunning) {
        req->CancelRequested = 1;
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
        return STATUS_PENDING;
    }

    KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
    return STATUS_OBJECT_DEAD;
}

__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
                                                      IO_STATUS_BLOCK* iosbOut) {
    if (!req || !iosbOut) return STATUS_INVALID_PARAMETER;
    iosbOut->Status = req->IosbSnapshot.Status;
    iosbOut->Information = req->IosbSnapshot.Information;
    return STATUS_SUCCESS;
}

// Current Working Directory
// Stored as (CurrentDrive, drive-relative path with no trailing backslash).
// Exported as native "N:/path" so personalities can round-trip volumes.
#define CWD_MAX PROCESS_CWD_MAX

typedef struct KRESOLVED_PATH {
    uint8_t Drive;
    char Storage[CWD_MAX];
    SYS_STRING_VIEW Path;
} KRESOLVED_PATH;

// Implements the format Cwd operation.
static size_t KiFormatCwd(uint8_t drive, SYS_STRING_VIEW rel, char* buf, size_t size) {
    if (!buf || size == 0 || !SysSvIsValid(rel)) return 0;
    if (drive > 7) drive = 0;

    size_t needed = 3 + rel.Length + 1;
    if (needed > size) return needed;

    buf[0] = (char)('0' + drive);
    buf[1] = ':';
    buf[2] = '/';
    if (rel.Length) memcpy(buf + 3, rel.Data, rel.Length);
    buf[3 + rel.Length] = '\0';
    return needed;
}

// Returns cwd.
__declspec(dllexport) KSTATUS IoGetCwd(char* buf, size_t size) {
    if (!buf || size == 0) return STATUS_INVALID_PARAMETER;

    PROCESS* proc = KeGetCurrentProcess();
    if (!proc) return STATUS_ACCESS_DENIED;

    uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
    size_t needed = KiFormatCwd(proc->CurrentDrive, SysSvFromParts(proc->CurrentDirectory, proc->CurrentDirectoryLength), buf, size);
    KeReleaseSpinlock(&proc->CwdLock, flags);

    return needed && needed <= size ? STATUS_SUCCESS : STATUS_BUFFER_TOO_SMALL;
}

// Implements the sv Has Drive Root operation.
static int KiSvHasDriveRoot(SYS_STRING_VIEW path) {
    return path.Length >= 3 &&
           path.Data[0] >= '0' && path.Data[0] <= '7' &&
           path.Data[1] == ':' &&
           (path.Data[2] == '\\' || path.Data[2] == '/');
}

static int KiAppendBytes(char* work, size_t* workLen, size_t cap,
                         const char* data, size_t len) {
    if (!work || !workLen || (!data && len)) return 0;
    if (*workLen + len >= cap) return 0;
    if (len) memcpy(work + *workLen, data, len);
    *workLen += len;
    work[*workLen] = '\0';
    return 1;
}

static int KiAppendComponent(char* work, size_t* workLen, size_t cap,
                             const char* data, size_t len) {
    if (!work || !workLen || (!data && len)) return 0;
    if (*workLen && work[*workLen - 1] != '\\' && work[*workLen - 1] != '/') {
        if (!KiAppendBytes(work, workLen, cap, "\\", 1)) return 0;
    }
    return KiAppendBytes(work, workLen, cap, data, len);
}

// Implements the pop Component operation.
static void KiPopComponent(char* work, size_t* workLen) {
    if (!work || !workLen || *workLen == 0) return;
    size_t i = *workLen;
    while (i > 0 && work[i - 1] != '\\' && work[i - 1] != '/') i--;
    if (i > 0) i--;
    *workLen = i;
    work[i] = '\0';
}

// Resolve a possibly-relative string view against the CWD.
// The returned view is backed by out->Storage and remains valid while 'out' lives.
static KSTATUS KiResolvePathV(SYS_STRING_VIEW input, KRESOLVED_PATH* out) {
    if (!out) return STATUS_INVALID_PARAMETER;
    memset(out, 0, sizeof(*out));

    if (!SysSvIsValid(input)) return STATUS_INVALID_PARAMETER;
    if (SysSvContainsByte(input, '\0')) return STATUS_INVALID_PARAMETER;
    if (input.Length >= CWD_MAX) return STATUS_BUFFER_TOO_SMALL;

    uint8_t drive = 0;
    PROCESS* proc = KeGetCurrentProcess();
    if (proc) {
        uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
        drive = proc->CurrentDrive;
        KeReleaseSpinlock(&proc->CwdLock, flags);
    }

    size_t pos = 0;
    bool fromDriveRoot = false;
    if (KiSvHasDriveRoot(input)) {
        drive = (uint8_t)(input.Data[0] - '0');
        pos = 3;
        fromDriveRoot = true;
    }

    char work[CWD_MAX * 2];
    size_t workLen = 0;
    work[0] = '\0';

    bool absolute = fromDriveRoot ||
                    (pos < input.Length && (input.Data[pos] == '\\' || input.Data[pos] == '/'));
    if (!absolute && proc) {
        uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
        size_t cwdLen = proc->CurrentDirectoryLength;
        if (cwdLen >= sizeof(work)) cwdLen = sizeof(work) - 1;
        memcpy(work, proc->CurrentDirectory, cwdLen);
        work[cwdLen] = '\0';
        workLen = cwdLen;
        KeReleaseSpinlock(&proc->CwdLock, flags);
    }

    while (pos < input.Length) {
        while (pos < input.Length && (input.Data[pos] == '\\' || input.Data[pos] == '/')) pos++;
        if (pos >= input.Length) break;

        size_t compStart = pos;
        while (pos < input.Length && input.Data[pos] != '\\' && input.Data[pos] != '/') pos++;
        size_t compLen = pos - compStart;
        if (compLen == 0) continue;

        if (compLen == 1 && input.Data[compStart] == '.') {
            continue;
        }
        if (compLen == 2 && input.Data[compStart] == '.' && input.Data[compStart + 1] == '.') {
            KiPopComponent(work, &workLen);
            continue;
        }
        if (compLen >= 256) return STATUS_BUFFER_TOO_SMALL;
        if (!KiAppendComponent(work, &workLen, sizeof(work), input.Data + compStart, compLen))
            return STATUS_BUFFER_TOO_SMALL;
    }

    if (workLen + 1 > sizeof(out->Storage)) return STATUS_BUFFER_TOO_SMALL;
    memcpy(out->Storage, work, workLen);
    out->Storage[workLen] = '\0';
    out->Drive = drive;
    out->Path = SysSvFromParts(out->Storage, workLen);
    return STATUS_SUCCESS;
}

// Updates cwd.
__declspec(dllexport) KSTATUS IoSetCwd(SYS_STRING_VIEW path) {
    if (path.Length && !path.Data) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    PROCESS* proc = KeGetCurrentProcess();
    if (!proc) return STATUS_ACCESS_DENIED;

    KRESOLVED_PATH resolved;
    KSTATUS s = KiResolvePathV(path, &resolved);
    if (!KSUCCESS(s)) return s;

    IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
    if (!fs) return STATUS_NOT_FOUND;
    if (!fs->OpenDir) return STATUS_NOT_SUPPORTED;

    void* dirCtx = NULL;
    s = fs->OpenDir(fs, resolved.Path, &dirCtx);
    if (!KSUCCESS(s)) return s;
    fs->CloseDir(fs, dirCtx);

    if (resolved.Path.Length + 1 > sizeof(proc->CurrentDirectory)) return STATUS_BUFFER_TOO_SMALL;

    uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
    proc->CurrentDrive = resolved.Drive;
    proc->CurrentDirectoryLength = resolved.Path.Length;
    memcpy(proc->CurrentDirectory, resolved.Path.Data, resolved.Path.Length);
    proc->CurrentDirectory[resolved.Path.Length] = '\0';

    if (proc->UserCwdBase && proc->UserCwdSize) {
        size_t formatted = KiFormatCwd(resolved.Drive, resolved.Path, (char*)proc->UserCwdBase, proc->UserCwdSize);
        size_t copyLen = (formatted && formatted <= proc->UserCwdSize) ? formatted - 1 : proc->UserCwdSize - 1;

        SYS_PEB* peb = (SYS_PEB*)proc->UserPebBase;
        if (peb && peb->Magic == SYS_PEB_MAGIC) {
            peb->CurrentDirectory = SysSvFromParts((const char*)proc->UserCwdBase, copyLen);
        }
    }

    KeReleaseSpinlock(&proc->CwdLock, flags);
    return STATUS_SUCCESS;
}

// Registration.
__declspec(dllexport) KSTATUS IoRegisterFilesystem(uint8_t volumeIndex,
                              IO_FILESYSTEM *fs,
                              const IO_VOLUME_PARAMS *vol) {
    if (volumeIndex >= MAX_DRIVES || !fs || !vol)
        return STATUS_INVALID_PARAMETER;
    if (gVolumes[volumeIndex].Fs)
        return STATUS_ALREADY_EXISTS;
    gVolumes[volumeIndex].Fs  = fs;
    gVolumes[volumeIndex].Vol = *vol;
    return STATUS_SUCCESS;
}

// Implements the unregister Filesystem operation.
__declspec(dllexport) KSTATUS IoUnregisterFilesystem(uint8_t volumeIndex) {
    if (volumeIndex >= MAX_DRIVES)
        return STATUS_INVALID_PARAMETER;
    if (!gVolumes[volumeIndex].Fs)
        return STATUS_NOT_FOUND;

    memset(&gVolumes[volumeIndex], 0, sizeof(gVolumes[volumeIndex]));
    return STATUS_SUCCESS;
}

// Implements the unregister Filesystems For Disk operation.
__declspec(dllexport) KSTATUS IoUnregisterFilesystemsForDisk(uint8_t diskIndex) {
    KSTATUS last = STATUS_NOT_FOUND;
    for (uint8_t i = 0; i < MAX_DRIVES; i++) {
        if (gVolumes[i].Fs && gVolumes[i].Vol.DiskIndex == diskIndex) {
            memset(&gVolumes[i], 0, sizeof(gVolumes[i]));
            last = STATUS_SUCCESS;
        }
    }
    return last;
}

// Implements the filesystem Present operation.
__declspec(dllexport) bool IoFilesystemPresent(uint8_t volumeIndex) {
    if (volumeIndex >= MAX_DRIVES)
        return false;
    return gVolumes[volumeIndex].Fs != (IO_FILESYSTEM*)0;
}

// Aggregates filesystem traversal statistics across mounted volumes.
__declspec(dllexport) void IoQueryFilesystemStats(IO_FILESYSTEM_STATS* out) {
    if (!out) return;
    memset(out, 0, sizeof(*out));

    for (uint8_t i = 0; i < MAX_DRIVES; i++) {
        IO_FILESYSTEM* fs = gVolumes[i].Fs;
        if (!fs || !fs->QueryStats) continue;

        IO_FILESYSTEM_STATS one;
        memset(&one, 0, sizeof(one));
        fs->QueryStats(fs, &one);

        out->PathLookups += one.PathLookups;
        out->PathCacheHits += one.PathCacheHits;
        out->PathCacheMisses += one.PathCacheMisses;
        out->PathCacheStores += one.PathCacheStores;
        out->PathCacheInvalidations += one.PathCacheInvalidations;
        if (one.PathCacheGeneration > out->PathCacheGeneration)
            out->PathCacheGeneration = one.PathCacheGeneration;
        out->DirectorySearches += one.DirectorySearches;
        out->DirectoryClustersRead += one.DirectoryClustersRead;
        out->DirectoryEntriesScanned += one.DirectoryEntriesScanned;
        out->FatChainCacheLookups += one.FatChainCacheLookups;
        out->FatChainCacheHits += one.FatChainCacheHits;
        out->FatChainCacheMisses += one.FatChainCacheMisses;
        out->FatChainCacheStores += one.FatChainCacheStores;
        out->FatChainCacheInvalidations += one.FatChainCacheInvalidations;
        out->FatChainWalkSteps += one.FatChainWalkSteps;
        out->DataReadRuns += one.DataReadRuns;
        out->DataReadSectors += one.DataReadSectors;
        out->DataReadRunMerges += one.DataReadRunMerges;
        if (one.MaxDataReadSpanSectors > out->MaxDataReadSpanSectors)
            out->MaxDataReadSpanSectors = one.MaxDataReadSpanSectors;
    }
}

// Destructor.
static void KiFileDestroy(OBJECT* obj) {
    FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
    if (f->Fs && f->FileCtx) {
        (void)IoFlushFile(f);
        MmFileCacheInvalidateAll(f);
        f->Fs->Close(f->Fs, f->FileCtx);
    }
}

// IoOpenFile.
__declspec(dllexport) KSTATUS IoOpenFile(SYS_STRING_VIEW path, FILE_OBJECT **out) {
    if ((!path.Data && path.Length) || !out) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;
    if (path.Length == 0) return STATUS_INVALID_PARAMETER;

    KRESOLVED_PATH resolved;
    KSTATUS s = KiResolvePathV(path, &resolved);
    if (!KSUCCESS(s)) return s;

    IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
    if (!fs) return STATUS_NOT_FOUND;

    void* fileCtx;
    s = fs->Open(fs, resolved.Path, &fileCtx);
    if (!KSUCCESS(s)) return s;

    // File objects reuse the generic object slot because the handle table only needs the shared header contract here.
    FILE_OBJECT* f = (FILE_OBJECT *)ObAllocateObject(ObjectTypeFile,
                                                      sizeof(FILE_OBJECT));
    if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; }
    f->Header.Destroy = KiFileDestroy;
    f->Fs      = fs;
    f->FileCtx = fileCtx;
    f->Lock    = (SPINLOCK)SPINLOCK_INIT;
    f->Size    = fs->GetSize(fs, fileCtx);
    f->Offset  = 0;
    f->Flags   = 0;
    f->ReadaheadScore = 0;
    f->ReadaheadNextOffset = 0;
    f->ReadaheadIssuedUntil = 0;
    f->ReadaheadSuppressedUntil = 0;

    *out = f;
    return STATUS_SUCCESS;
}


static uint64_t KiAlignUpPage64(uint64_t value) {
    return (value + PAGE_SIZE - 1u) & ~(uint64_t)(PAGE_SIZE - 1u);
}

static void KiIoMaybeScheduleReadAhead(FILE_OBJECT* file, uint64_t offset, size_t bytesRead, uint64_t fileSize) {
    if (!file || bytesRead < 512)
        return;
    if ((file->Flags & IO_FILE_FLAG_NO_CACHE) != 0)
        return;

    uint64_t end = offset + bytesRead;
    uint64_t start = 0;
    uint32_t pages = 0;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    if (offset == file->ReadaheadNextOffset) {
        if (file->ReadaheadScore < 32)
            file->ReadaheadScore++;
    } else {
        file->ReadaheadScore = 1;
        file->ReadaheadIssuedUntil = end;
    }
    file->ReadaheadNextOffset = end;

    if (end < file->ReadaheadSuppressedUntil) {
        pages = 0;
    } else if (file->ReadaheadScore >= 2 || bytesRead >= PAGE_SIZE) {
        pages = file->ReadaheadScore < 4 ? 8u : (file->ReadaheadScore < 8 ? 16u : 32u);
        start = KiAlignUpPage64(end);
        if (start < file->ReadaheadIssuedUntil)
            start = KiAlignUpPage64(file->ReadaheadIssuedUntil);
        if (start >= fileSize) {
            pages = 0;
        } else {
            uint64_t maxEnd = start + ((uint64_t)pages * PAGE_SIZE);
            if (maxEnd > fileSize)
                pages = (uint32_t)((KiAlignUpPage64(fileSize) - start) / PAGE_SIZE);
            file->ReadaheadIssuedUntil = start + ((uint64_t)pages * PAGE_SIZE);
        }
    }
    KeReleaseSpinlock(&file->Lock, flags);

    if (pages) {
        KSTATUS ras = MmFileCacheScheduleReadAhead(file, start, pages);
        if (ras == STATUS_QUEUE_FULL) {
            flags = KeAcquireSpinlock(&file->Lock);
            uint64_t suppressUntil = end + (1024u * 1024u);
            if (suppressUntil < end || suppressUntil > fileSize)
                suppressUntil = fileSize;
            if (file->ReadaheadSuppressedUntil < suppressUntil)
                file->ReadaheadSuppressedUntil = suppressUntil;
            KeReleaseSpinlock(&file->Lock, flags);
        }
    }
}

// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
                                           void* buf, size_t size,
                                           size_t* bytesRead) {
    if (bytesRead) *bytesRead = 0;
    if (!file || !buf || !bytesRead) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;
    if (size == 0) return STATUS_SUCCESS;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t fileSize = file->Size;
    KeReleaseSpinlock(&file->Lock, flags);

    if (offset >= fileSize) return STATUS_SUCCESS;

    size_t remaining = (size_t)(fileSize - offset);
    if (size > remaining) size = remaining;

    if ((file->Flags & IO_FILE_FLAG_NO_CACHE) || size < 512)
        return file->Fs->Read(file->Fs, file->FileCtx, offset, buf, size, bytesRead);

    size_t total = 0;
    uint8_t* dst = (uint8_t*)buf;
    while (total < size) {
        uint64_t absolute = offset + total;
        uint64_t pageOffset = absolute & ~(uint64_t)(PAGE_SIZE - 1);
        size_t inPage = (size_t)(absolute - pageOffset);
        size_t chunk = PAGE_SIZE - inPage;
        if (chunk > size - total) chunk = size - total;

        MM_FILE_CACHE_PAGE page;
        KSTATUS s = MmFileCacheGetPage(file, pageOffset, &page);
        if (!KSUCCESS(s)) {
            if (total) {
                *bytesRead = total;
                return STATUS_SUCCESS;
            }
            return s;
        }

        if (inPage >= page.ValidBytes) {
            MmFileCacheReleasePage(&page);
            break;
        }
        size_t available = page.ValidBytes - inPage;
        if (chunk > available) chunk = available;

        void* srcPage = MmPfnToVirtual(page.Frame);
        if (!srcPage) {
            MmFileCacheReleasePage(&page);
            return total ? STATUS_SUCCESS : STATUS_FAILURE;
        }
        memcpy(dst + total, (const uint8_t*)srcPage + inPage, chunk);
        MmFileCacheReleasePage(&page);

        total += chunk;
        if (chunk == 0) break;
    }

    *bytesRead = total;
    KiIoMaybeScheduleReadAhead(file, offset, total, fileSize);
    return STATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
                                            const void* buf, size_t size,
                                            size_t* bytesWritten) {
    if (bytesWritten) *bytesWritten = 0;
    if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;
    if (!file->Fs->Write) return STATUS_NOT_SUPPORTED;
    if (size == 0) return STATUS_SUCCESS;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t oldSize = file->Size;
    KeReleaseSpinlock(&file->Lock, flags);

    size_t written = 0;
    KSTATUS s;
    if (file->Flags & IO_FILE_FLAG_NO_CACHE) {
        s = file->Fs->Write(file->Fs, file->FileCtx, offset, buf, size, &written);
        if (KSUCCESS(s) && written)
            MmFileCacheInvalidate(file, offset, written);
    } else {
        s = MmFileCacheWrite(file, offset, buf, size, oldSize, &written);
    }
    if (KSUCCESS(s)) {
        flags = KeAcquireSpinlock(&file->Lock);
        uint64_t end = offset + written;
        if (end > file->Size) file->Size = end;
        KeReleaseSpinlock(&file->Lock, flags);
    }

    *bytesWritten = written;
    return s;
}

// Flushes dirty mapped and cached pages for one file object.
__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) {
    if (!file) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    KSTATUS sync = MmVadSyncFileMappings(file);
    KSTATUS flush = MmFileCacheFlushFile(file);
    return KSUCCESS(sync) ? flush : sync;
}

// IoReadFile - sequential read at current offset.
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
                                         size_t size, size_t* bytesRead) {
    if (bytesRead) *bytesRead = 0;
    if (!file || !buf || !bytesRead) return STATUS_INVALID_PARAMETER;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t offset = file->Offset;
    KeReleaseSpinlock(&file->Lock, flags);

    KSTATUS s = IoReadFileAt(file, offset, buf, size, bytesRead);
    if (KSUCCESS(s)) {
        flags = KeAcquireSpinlock(&file->Lock);
        file->Offset = offset + *bytesRead;
        KeReleaseSpinlock(&file->Lock, flags);
    }
    return s;
}

// IoWriteFile - sequential write at current offset.
__declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
                                          size_t size, size_t* bytesWritten) {
    if (bytesWritten) *bytesWritten = 0;
    if (!file || !buf || !bytesWritten) return STATUS_INVALID_PARAMETER;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t offset = file->Offset;
    KeReleaseSpinlock(&file->Lock, flags);

    KSTATUS s = IoWriteFileAt(file, offset, buf, size, bytesWritten);
    if (KSUCCESS(s)) {
        flags = KeAcquireSpinlock(&file->Lock);
        file->Offset = offset + *bytesWritten;
        KeReleaseSpinlock(&file->Lock, flags);
    }
    return s;
}

// IoCreateFile - create a new file and return it open for writing.
__declspec(dllexport) KSTATUS IoCreateFile(SYS_STRING_VIEW path, FILE_OBJECT** out) {
    if ((!path.Data && path.Length) || !out || path.Length == 0) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    KRESOLVED_PATH resolved;
    KSTATUS s = KiResolvePathV(path, &resolved);
    if (!KSUCCESS(s)) return s;

    IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
    if (!fs)          return STATUS_NOT_FOUND;
    if (!fs->Create)  return STATUS_NOT_SUPPORTED;

    void* fileCtx;
    s = fs->Create(fs, resolved.Path, &fileCtx);
    if (!KSUCCESS(s)) return s;

    FILE_OBJECT* f = (FILE_OBJECT*)ObAllocateObject(ObjectTypeFile,
                                                     sizeof(FILE_OBJECT));
    if (!f) { fs->Close(fs, fileCtx); return STATUS_NO_MEMORY; }
    f->Header.Destroy = KiFileDestroy;
    f->Fs      = fs;
    f->FileCtx = fileCtx;
    f->Lock    = (SPINLOCK)SPINLOCK_INIT;
    f->Size    = 0;
    f->Offset  = 0;
    f->Flags   = 0;
    f->ReadaheadScore = 0;
    f->ReadaheadNextOffset = 0;
    f->ReadaheadIssuedUntil = 0;
    f->ReadaheadSuppressedUntil = 0;

    *out = f;
    return STATUS_SUCCESS;
}

// IoDeleteFile - delete a file by path (does not require an open handle).
__declspec(dllexport) KSTATUS IoDeleteFile(SYS_STRING_VIEW path) {
    if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    KRESOLVED_PATH resolved;
    KSTATUS s = KiResolvePathV(path, &resolved);
    if (!KSUCCESS(s)) return s;

    IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
    if (!fs)         return STATUS_NOT_FOUND;
    if (!fs->Delete) return STATUS_NOT_SUPPORTED;

    return fs->Delete(fs, resolved.Path);
}

// Implements the rename File operation.
__declspec(dllexport) KSTATUS IoRenameFile(SYS_STRING_VIEW oldPath, SYS_STRING_VIEW newPath) {
    if ((!oldPath.Data && oldPath.Length) || (!newPath.Data && newPath.Length) ||
        oldPath.Length == 0 || newPath.Length == 0) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    KRESOLVED_PATH oldResolved;
    KRESOLVED_PATH newResolved;
    KSTATUS s = KiResolvePathV(oldPath, &oldResolved);
    if (!KSUCCESS(s)) return s;
    s = KiResolvePathV(newPath, &newResolved);
    if (!KSUCCESS(s)) return s;
    if (oldResolved.Drive != newResolved.Drive) return STATUS_NOT_SUPPORTED;

    IO_FILESYSTEM* fs = gVolumes[oldResolved.Drive].Fs;
    if (!fs)          return STATUS_NOT_FOUND;
    if (!fs->Rename)  return STATUS_NOT_SUPPORTED;

    return fs->Rename(fs, oldResolved.Path, newResolved.Path);
}

// Creates dir.
__declspec(dllexport) KSTATUS IoCreateDir(SYS_STRING_VIEW path) {
    if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    KRESOLVED_PATH resolved;
    KSTATUS s = KiResolvePathV(path, &resolved);
    if (!KSUCCESS(s)) return s;

    IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
    if (!fs)           return STATUS_NOT_FOUND;
    if (!fs->CreateDir) return STATUS_NOT_SUPPORTED;

    return fs->CreateDir(fs, resolved.Path);
}

// Releases dir.
__declspec(dllexport) KSTATUS IoRemoveDir(SYS_STRING_VIEW path) {
    if ((!path.Data && path.Length) || path.Length == 0) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    KRESOLVED_PATH resolved;
    KSTATUS s = KiResolvePathV(path, &resolved);
    if (!KSUCCESS(s)) return s;

    IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
    if (!fs)           return STATUS_NOT_FOUND;
    if (!fs->RemoveDir) return STATUS_NOT_SUPPORTED;

    return fs->RemoveDir(fs, resolved.Path);
}

// IoTruncateFile - resize the file; updates the cached Size on success.
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize) {
    if (!file) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;
    if (!file->Fs->Truncate) return STATUS_NOT_SUPPORTED;

    KSTATUS flush = IoFlushFile(file);
    if (!KSUCCESS(flush)) return flush;

    KSTATUS s = file->Fs->Truncate(file->Fs, file->FileCtx, newSize, file->Flags);
    if (KSUCCESS(s)) {
        MmFileCacheInvalidate(file, newSize, (size_t)-1);
        file->Size = newSize;
        // Pull the file pointer back if it now sits past EOF.
        if (file->Offset > file->Size)
            file->Offset = file->Size;
    }
    return s;
}

__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) {
    if (!file) return STATUS_INVALID_PARAMETER;
    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    file->Offset = offset;
    KeReleaseSpinlock(&file->Lock, flags);
    return STATUS_SUCCESS;
}

// Returns file Size.
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
    if (!file) return 0;
    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t size = file->Size;
    KeReleaseSpinlock(&file->Lock, flags);
    return size;
}

// Queries the filesystem identity for a file object.
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) {
    if (!file || !out)
        return STATUS_INVALID_PARAMETER;
    memset(out, 0, sizeof(*out));
    if (!file->Fs || !file->Fs->QueryIdentity)
        return STATUS_NOT_SUPPORTED;
    return file->Fs->QueryIdentity(file->Fs, file->FileCtx, out);
}

// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
    if (file) ObDereferenceObject(&file->Header);
}

// Directory enumeration.

static void KiDirDestroy(OBJECT* obj) {
    DIR_OBJECT* d = (DIR_OBJECT*)obj;  // Header is first member
    if (d->Fs && d->DirCtx) d->Fs->CloseDir(d->Fs, d->DirCtx);
}

// Opens dir.
__declspec(dllexport) KSTATUS IoOpenDir(SYS_STRING_VIEW dirPath, DIR_OBJECT** out) {
    if (!out) return STATUS_INVALID_PARAMETER;
    if (dirPath.Length && !dirPath.Data) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;

    KRESOLVED_PATH resolved;
    KSTATUS s;

    if (dirPath.Length == 0) {
        PROCESS* proc = KeGetCurrentProcess();
        if (!proc) return STATUS_ACCESS_DENIED;

        uintptr_t flags = KeAcquireSpinlock(&proc->CwdLock);
        size_t cwdLen = proc->CurrentDirectoryLength;
        if (cwdLen + 1 > sizeof(resolved.Storage)) {
            KeReleaseSpinlock(&proc->CwdLock, flags);
            return STATUS_BUFFER_TOO_SMALL;
        }
        memcpy(resolved.Storage, proc->CurrentDirectory, cwdLen + 1);
        resolved.Drive = proc->CurrentDrive;
        resolved.Path = SysSvFromParts(resolved.Storage, cwdLen);
        KeReleaseSpinlock(&proc->CwdLock, flags);
    } else {
        s = KiResolvePathV(dirPath, &resolved);
        if (!KSUCCESS(s)) return s;
    }

    IO_FILESYSTEM* fs = gVolumes[resolved.Drive].Fs;
    if (!fs)          return STATUS_NOT_FOUND;
    if (!fs->OpenDir) return STATUS_NOT_SUPPORTED;

    void* dirCtx = NULL;
    s = fs->OpenDir(fs, resolved.Path, &dirCtx);
    if (!KSUCCESS(s)) return s;

    DIR_OBJECT* d = (DIR_OBJECT*)ObAllocateObject(ObjectTypeDir, sizeof(DIR_OBJECT));
    if (!d) { fs->CloseDir(fs, dirCtx); return STATUS_NO_MEMORY; }

    d->Header.Destroy = KiDirDestroy;
    d->Fs     = fs;
    d->DirCtx = dirCtx;

    *out = d;
    return STATUS_SUCCESS;
}

// Returns dir.
__declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) {
    if (!dir || !entryOut) return STATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return STATUS_INVALID_IRQL;
    if (!dir->Fs->ReadDir) return STATUS_NOT_SUPPORTED;
    return dir->Fs->ReadDir(dir->Fs, dir->DirCtx, entryOut);
}

// Releases dir.
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir) {
    if (dir) ObDereferenceObject(&dir->Header);
}