nt-lite/code/kernel/io_file.c

// Kernel file I/O layer. This file owns file handles, filesystem dispatch, and asynchronous request plumbing.

#include <stdbool.h>
#include <string.h>
#include <kernel/ke_list.h>
#include <kernel/ke_process.h>
#include <kernel/io_file.h>
#include <kernel/io_volume.h>
#include <kernel/mm_file_cache.h>
#include <kernel/mm_pfn.h>
#include <kernel/mm_virt.h>
#include <kernel/ob_object.h>
#include <kernel/ke_irql.h>
#include <kernel/ke_event.h>
#include <kernel/ke_apc.h>
#include <kernel/ke_semaphore.h>
#include <kernel/ke_thread.h>
#include <kernel/ke_sched.h>
#include <kernel/ke_time.h>
#include <ntlite/syscall.h>

// Async file I/O worker.
typedef enum IO_ASYNC_FILE_OP {
    IoAsyncFileRead  = 0,
    IoAsyncFileWrite = 1,
} IO_ASYNC_FILE_OP;

struct IO_ASYNC_FILE_REQUEST_OBJECT {
    IO_ASYNC_REQUEST_OBJECT Async;

    IO_ASYNC_FILE_OP Op;
    FILE_OBJECT* File;
    void* Buffer;
    size_t Size;
    uint64_t Offset;
};

static SPINLOCK   gAsyncFileInitLock = SPINLOCK_INIT;
static int        gAsyncFileInitState;     // 0=off, 1=starting, 2=ready
static SPINLOCK   gAsyncFileQueueLock = SPINLOCK_INIT;
static LIST_ENTRY gAsyncFileQueue;
static SEMAPHORE* gAsyncFileSemaphore;

typedef struct IO_FILE_LOCK_ENTRY {
    LIST_ENTRY Link;
    IO_FILE_IDENTITY Identity;
    PROCESS* Owner;
    uint32_t OwnerProcessId;
    uint64_t Offset;
    uint64_t Length;
    uint32_t Mode;
} IO_FILE_LOCK_ENTRY;

typedef struct IO_FILE_LOCK_WAITER {
    LIST_ENTRY Link;
    IO_FILE_IDENTITY Identity;
    PROCESS* Owner;
    uint64_t Offset;
    uint64_t Length;
    uint32_t Mode;
    EVENT* Event;
    bool Armed;
} IO_FILE_LOCK_WAITER;

static SPINLOCK   gFileLockLock = SPINLOCK_INIT;
static LIST_ENTRY gFileLocks;
static LIST_ENTRY gFileLockWaiters;
static bool       gFileLocksInitialized;

// Initializes the global advisory file-lock lists on first use.
static void KiIoEnsureFileLockState(void) {
    uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
    if (!gFileLocksInitialized) {
        LIST_INIT(&gFileLocks);
        LIST_INIT(&gFileLockWaiters);
        gFileLocksInitialized = 1;
    }
    KeReleaseSpinlock(&gFileLockLock, flags);
}

// Compares the stable filesystem identity fields used for lock ownership.
static bool KiIoSameFileIdentity(const IO_FILE_IDENTITY* a, const IO_FILE_IDENTITY* b) {
    if (!a || !b) return false;
    return a->VolumeId == b->VolumeId &&
           a->FileId == b->FileId &&
           a->Generation == b->Generation;
}

// Returns the exclusive end offset for a public lock range.
static uint64_t KiIoFileLockEnd(uint64_t offset, uint64_t length) {
    if (length == SYS_FILE_LOCK_TO_EOF)
        return UINT64_MAX;
    uint64_t end = offset + length;
    if (end < offset)
        return UINT64_MAX;
    return end;
}

// Tests whether two byte ranges overlap.
static bool KiIoFileLockRangesOverlap(uint64_t aOffset, uint64_t aLength,
                                      uint64_t bOffset, uint64_t bLength) {
    uint64_t aEnd = KiIoFileLockEnd(aOffset, aLength);
    uint64_t bEnd = KiIoFileLockEnd(bOffset, bLength);
    return aOffset < bEnd && bOffset < aEnd;
}

// Converts an exclusive range end back to the public lock-length representation.
static uint64_t KiIoFileLockLengthFromEnd(uint64_t offset, uint64_t end) {
    if (end == UINT64_MAX)
        return SYS_FILE_LOCK_TO_EOF;
    return end - offset;
}

// Initializes one file-lock entry from an existing owner's identity.
static void KiIoInitializeFileLockEntry(IO_FILE_LOCK_ENTRY* entry,
                                        const IO_FILE_IDENTITY* identity,
                                        PROCESS* owner,
                                        uint32_t mode,
                                        uint64_t offset,
                                        uint64_t length) {
    memset(entry, 0, sizeof(*entry));
    LIST_INIT(&entry->Link);
    entry->Identity = *identity;
    entry->Owner = owner;
    entry->OwnerProcessId = owner->ProcessId;
    entry->Offset = offset;
    entry->Length = length;
    entry->Mode = mode;
}

// Frees every detached file-lock entry linked on a private list.
static void KiIoFreeDetachedFileLocks(LIST_ENTRY* head) {
    if (!head)
        return;
    while (head->Fwd != head) {
        LIST_ENTRY* node = ListPopHead(head);
        IO_FILE_LOCK_ENTRY* entry = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
        MmFreePool(entry);
    }
}

// Tests whether one existing lock conflicts with a requested lock.
static bool KiIoFileLockConflicts(const IO_FILE_LOCK_ENTRY* lock,
                                  const IO_FILE_IDENTITY* identity,
                                  PROCESS* owner,
                                  uint32_t mode,
                                  uint64_t offset,
                                  uint64_t length) {
    if (!lock || !identity || !owner)
        return false;
    if (lock->Owner == owner)
        return false;
    if (!KiIoSameFileIdentity(&lock->Identity, identity))
        return false;
    if (!KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length))
        return false;
    if (lock->Mode == SYS_FILE_LOCK_EXCLUSIVE || mode == SYS_FILE_LOCK_EXCLUSIVE)
        return true;
    return false;
}

// Tests whether the requested lock is currently blocked by another process.
static bool KiIoFileLockHasConflictLocked(const IO_FILE_IDENTITY* identity,
                                          PROCESS* owner,
                                          uint32_t mode,
                                          uint64_t offset,
                                          uint64_t length) {
    LIST_ENTRY* head = &gFileLocks;
    for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
        IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
        if (KiIoFileLockConflicts(lock, identity, owner, mode, offset, length))
            return true;
    }
    return false;
}

// Computes an absolute monotonic deadline for a finite file-lock timeout.
static uint64_t KiIoFileLockTimeoutDeadline(uint64_t timeoutMilliseconds) {
    return KeMonotonicDeadlineAfterMilliseconds(timeoutMilliseconds);
}

// Computes the remaining wait time for a finite file-lock timeout deadline.
static uint64_t KiIoFileLockRemainingMilliseconds(uint64_t deadlineMicroseconds) {
    return KeMonotonicMillisecondsUntil(deadlineMicroseconds);
}

// Waits for a file-lock conflict to change.
static KSTATUS KiIoWaitForFileLockChange(EVENT* event, uint64_t timeoutMilliseconds, uint64_t deadlineMicroseconds) {
    if (!event)
        return NTLSTATUS_INVALID_PARAMETER;

    uint64_t waitMilliseconds = timeoutMilliseconds;
    if (timeoutMilliseconds != SYS_WAIT_TIMEOUT_INFINITE) {
        waitMilliseconds = KiIoFileLockRemainingMilliseconds(deadlineMicroseconds);
        if (waitMilliseconds == 0)
            return NTLSTATUS_TIMEOUT;
    }

    EVENT* events[1] = { event };
    return KeWaitForMultipleEvents(events, 1, 0, waitMilliseconds, (uint32_t*)0);
}

// Wakes waiters whose requested file identity may have become available.
static void KiIoSignalFileLockWaitersLocked(const IO_FILE_IDENTITY* identity) {
    if (!identity || !gFileLocksInitialized)
        return;

    LIST_ENTRY* head = &gFileLockWaiters;
    for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
        IO_FILE_LOCK_WAITER* waiter = CONTAINING_RECORD(node, IO_FILE_LOCK_WAITER, Link);
        if (waiter->Armed && waiter->Event && KiIoSameFileIdentity(&waiter->Identity, identity))
            KeSignalEvent(waiter->Event);
    }
}

// Releases file-specific references owned by an async file request.
static void KiIoAsyncFileRequestCleanup(IO_ASYNC_REQUEST_OBJECT* async, void* context) {
    (void)context;
    IO_ASYNC_FILE_REQUEST_OBJECT* req =
        (IO_ASYNC_FILE_REQUEST_OBJECT*)async;

    if (req->File) {
        ObDereferenceObject(&req->File->Header);
        req->File = (FILE_OBJECT*)0;
    }
}

// Cancels an async file request from the common async request surface.
static KSTATUS KiIoCancelAsyncFileRequest(IO_ASYNC_REQUEST_OBJECT* async, void* context) {
    (void)context;
    if (!async) return NTLSTATUS_INVALID_PARAMETER;

    uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
    IO_ASYNC_REQUEST_STATE state = IoAsyncRequestGetState(async);

    if (state == IoAsyncRequestQueued) {
        ListRemove(&async->Link);
        IoAsyncRequestRequestCancel(async);
        IoAsyncRequestSetState(async, IoAsyncRequestCancelled);
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);

        IoCompleteAsyncRequest(async, NTLSTATUS_CANCELLED, 0);
        ObDereferenceObject(&async->Header);
        return NTLSTATUS_SUCCESS;
    }

    if (state == IoAsyncRequestRunning) {
        IoAsyncRequestRequestCancel(async);
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
        return NTLSTATUS_PENDING;
    }

    KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
    return NTLSTATUS_OBJECT_DEAD;
}

// Implements the io Async File Dequeue operation.
static IO_ASYNC_FILE_REQUEST_OBJECT* KiIoAsyncFileDequeue(void) {
    uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
    LIST_ENTRY* node = ListPopHead(&gAsyncFileQueue);
    if (!node) {
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
        return (IO_ASYNC_FILE_REQUEST_OBJECT*)0;
    }

    IO_ASYNC_FILE_REQUEST_OBJECT* req =
        (IO_ASYNC_FILE_REQUEST_OBJECT*)CONTAINING_RECORD(node, IO_ASYNC_REQUEST_OBJECT, Link);

    if (IoAsyncRequestIsCancelRequested(&req->Async)) {
        IoAsyncRequestSetState(&req->Async, IoAsyncRequestCancelled);
    } else {
        IoAsyncRequestSetState(&req->Async, IoAsyncRequestRunning);
    }
    KeReleaseSpinlock(&gAsyncFileQueueLock, flags);
    return req;
}

// Implements the io Async File Worker operation.
static void KiIoAsyncFileWorker(void* arg) {
    (void)arg;

    while (true) {
        KSTATUS s = KeWaitSemaphore(gAsyncFileSemaphore, KE_WAIT_INFINITE);
        if (!KSUCCESS(s)) {
            KeYield();
            continue;
        }

        IO_ASYNC_FILE_REQUEST_OBJECT* req = KiIoAsyncFileDequeue();
        if (!req) continue;

        if (IoAsyncRequestIsCancelRequested(&req->Async) || IoAsyncRequestGetState(&req->Async) == IoAsyncRequestCancelled) {
            IoCompleteAsyncRequest(&req->Async, NTLSTATUS_CANCELLED, 0);
            ObDereferenceObject(&req->Async.Header); // queue/worker reference
            continue;
        }

        size_t done = 0;
        if (req->Op == IoAsyncFileRead) {
            s = IoReadFileAt(req->File, req->Offset, req->Buffer,
                             req->Size, &done);
        } else {
            s = IoWriteFileAt(req->File, req->Offset, req->Buffer,
                              req->Size, &done);
        }

        if (IoAsyncRequestIsCancelRequested(&req->Async)) {
            // The operation may already have reached the filesystem. The
            // completion contract still reports cancellation so waiters/APCs do
            // not mistake a cancelled request for useful data.
            s = NTLSTATUS_CANCELLED;
            done = 0;
        }

        IoCompleteAsyncRequest(&req->Async, s, done);
        ObDereferenceObject(&req->Async.Header); // queue/worker reference
    }
}

// Implements the io Ensure Async File Io Started operation.
static KSTATUS KiIoEnsureAsyncFileIoStarted(void) {
    uintptr_t flags = KeAcquireSpinlock(&gAsyncFileInitLock);
    if (gAsyncFileInitState == 2) {
        KeReleaseSpinlock(&gAsyncFileInitLock, flags);
        return NTLSTATUS_SUCCESS;
    }
    if (gAsyncFileInitState == 1) {
        KeReleaseSpinlock(&gAsyncFileInitLock, flags);
        while (gAsyncFileInitState == 1)
            KeYield();
        return (gAsyncFileInitState == 2) ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
    }

    gAsyncFileInitState = 1;
    KeReleaseSpinlock(&gAsyncFileInitLock, flags);

    LIST_INIT(&gAsyncFileQueue);

    KSTATUS s = KeSemaphoreCreate(0, 0x7fffffff, &gAsyncFileSemaphore);
    if (KSUCCESS(s)) {
        THREAD* worker = (THREAD*)0;
        const KE_THREAD_CREATE_PARAMETERS parameters =
            KE_THREAD_CREATE_NAMED_NORMAL(PriorityNormal, "io.file.async");
        s = KeCreateKernelThread((uintptr_t)KiIoAsyncFileWorker,
                                 0,
                                 &parameters,
                                 &worker);
        // Keep the creator reference: this worker is intentionally persistent
        // for the lifetime of the kernel.
        (void)worker;
    }

    flags = KeAcquireSpinlock(&gAsyncFileInitLock);
    gAsyncFileInitState = KSUCCESS(s) ? 2 : 0;
    KeReleaseSpinlock(&gAsyncFileInitLock, flags);

    return s;
}

static void KiIoNativeFsAsyncComplete(void* completeContext,
                                      KSTATUS status,
                                      size_t information) {
    IO_ASYNC_FILE_REQUEST_OBJECT* req =
        (IO_ASYNC_FILE_REQUEST_OBJECT*)completeContext;
    if (!req) return;

    if (IoAsyncRequestIsCancelRequested(&req->Async)) {
        status = NTLSTATUS_CANCELLED;
        information = 0;
    }

    IoCompleteAsyncRequest(&req->Async, status, information);
    ObDereferenceObject(&req->Async.Header); // native filesystem provider reference
}

static KSTATUS KiIoSubmitAsyncFileRequest(IO_ASYNC_FILE_OP op,
                                         FILE_OBJECT* file,
                                         void* buffer,
                                         size_t size,
                                         uint64_t offset,
                                         IO_STATUS_BLOCK* iosb,
                                         EVENT* completionEvent,
                                         KAPC* completionApc,
                                         MM_LOCKED_USER_BUFFER* lockedBuffer,
                                         MM_LOCKED_USER_BUFFER* lockedIosb,
                                         IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
    if (!file || !iosb) {
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    if (!buffer && size) {
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    if (!KeCanWait()) {
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return NTLSTATUS_INVALID_IRQL;
    }

    iosb->Status = NTLSTATUS_PENDING;
    iosb->Information = 0;

    IO_ASYNC_REQUEST_OBJECT* async = (IO_ASYNC_REQUEST_OBJECT*)0;
    KSTATUS createStatus = IoCreateAsyncRequest(sizeof(IO_ASYNC_FILE_REQUEST_OBJECT),
                                                KiIoAsyncFileRequestCleanup,
                                                (void*)0,
                                                KiIoCancelAsyncFileRequest,
                                                (void*)0,
                                                &async);
    if (!KSUCCESS(createStatus)) {
        iosb->Status = createStatus;
        MmUnlockUserBuffer(lockedBuffer);
        MmUnlockUserBuffer(lockedIosb);
        return createStatus;
    }

    IO_ASYNC_FILE_REQUEST_OBJECT* req = (IO_ASYNC_FILE_REQUEST_OBJECT*)async;
    req->Op = op;
    req->File = file;
    req->Buffer = buffer;
    req->Size = size;
    req->Offset = offset;

    IoInitializeAsyncRequestCompletion(&req->Async, iosb, completionEvent,
                                       completionApc, lockedBuffer, lockedIosb);
    ObReferenceObject(&file->Header);

    IO_FS_ASYNC_COMPLETE complete = KiIoNativeFsAsyncComplete;
    KSTATUS ns = NTLSTATUS_NOT_SUPPORTED;
    bool hasNative = (op == IoAsyncFileRead) ? IoVolumeSupportsReadAsync(file->Volume)
                                             : IoVolumeSupportsWriteAsync(file->Volume);

    if (hasNative) {
        uintptr_t flags = KeAcquireSpinlock(&gAsyncFileQueueLock);
        IoAsyncRequestSetState(&req->Async, IoAsyncRequestRunning);
        KeReleaseSpinlock(&gAsyncFileQueueLock, flags);

        ObReferenceObject(&req->Async.Header); // native filesystem provider reference
        if (op == IoAsyncFileRead) {
            ns = IoVolumeReadAsync(file->Volume, file->FileCtx, offset, buffer,
                                   size, complete, req);
        } else {
            ns = IoVolumeWriteAsync(file->Volume, file->FileCtx, offset, buffer,
                                    size, complete, req);
        }
        if (ns == NTLSTATUS_PENDING) {
            if (requestOut) {
                *requestOut = req;
            } else {
                ObDereferenceObject(&req->Async.Header);
            }
            return NTLSTATUS_PENDING;
        }

        ObDereferenceObject(&req->Async.Header); // provider rejected synchronously
        iosb->Status = ns;
        req->Async.IosbSnapshot.Status = ns;
        req->Async.IosbSnapshot.Information = 0;
        ObDereferenceObject(&req->Async.Header); // creator reference
        return ns;
    }

    KSTATUS s = KiIoEnsureAsyncFileIoStarted();
    if (!KSUCCESS(s)) {
        iosb->Status = s;
        req->Async.IosbSnapshot.Status = s;
        req->Async.IosbSnapshot.Information = 0;
        ObDereferenceObject(&req->Async.Header);
        return s;
    }

    // Queue ownership keeps the request alive until the worker completes or a
    // successful queued cancellation completes it from IoCancelAsyncFileRequest.
    ObReferenceObject(&req->Async.Header);

    uintptr_t qflags = KeAcquireSpinlock(&gAsyncFileQueueLock);
    ListInsertTail(&gAsyncFileQueue, &req->Async.Link);
    KeReleaseSpinlock(&gAsyncFileQueueLock, qflags);

    KeSignalSemaphore(gAsyncFileSemaphore);

    if (requestOut) {
        *requestOut = req;       // creator reference returned to caller
    } else {
        ObDereferenceObject(&req->Async.Header);
    }

    return NTLSTATUS_PENDING;
}

__declspec(dllexport) KSTATUS IoReadFileAsyncKernel(FILE_OBJECT* file, void* buf,
                                                    size_t size, uint64_t offset,
                                                    IO_STATUS_BLOCK* iosb,
                                                    EVENT* completionEvent,
                                                    KAPC* completionApc,
                                                    MM_LOCKED_USER_BUFFER* lockedBuffer,
                                                    MM_LOCKED_USER_BUFFER* lockedIosb,
                                                    IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
    return KiIoSubmitAsyncFileRequest(IoAsyncFileRead, file, buf, size, offset,
                                    iosb, completionEvent, completionApc,
                                    lockedBuffer, lockedIosb, requestOut);
}

__declspec(dllexport) KSTATUS IoWriteFileAsyncKernel(FILE_OBJECT* file, const void* buf,
                                                     size_t size, uint64_t offset,
                                                     IO_STATUS_BLOCK* iosb,
                                                     EVENT* completionEvent,
                                                     KAPC* completionApc,
                                                     MM_LOCKED_USER_BUFFER* lockedBuffer,
                                                     MM_LOCKED_USER_BUFFER* lockedIosb,
                                                     IO_ASYNC_FILE_REQUEST_OBJECT** requestOut) {
    return KiIoSubmitAsyncFileRequest(IoAsyncFileWrite, file, (void*)buf, size, offset,
                                    iosb, completionEvent, completionApc,
                                    lockedBuffer, lockedIosb, requestOut);
}

// Implements the async File Request Object operation.
__declspec(dllexport) OBJECT* IoAsyncFileRequestObject(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
    return req ? &req->Async.Header : (OBJECT*)0;
}

// Returns the wait event owned by an async file request.
__declspec(dllexport) EVENT* IoGetAsyncFileRequestWaitEvent(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
    return req ? IoGetAsyncRequestWaitEvent(&req->Async) : (EVENT*)0;
}

// Waits for an async file request to complete according to wait flags.
__declspec(dllexport) KSTATUS IoWaitAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req, uint32_t flags) {
    return IoWaitAsyncRequest(req ? &req->Async : (IO_ASYNC_REQUEST_OBJECT*)0, flags);
}

// Cancels an async file request through the common async I/O object.
__declspec(dllexport) KSTATUS IoCancelAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req) {
    return IoCancelAsyncRequest(req ? &req->Async : (IO_ASYNC_REQUEST_OBJECT*)0);
}

__declspec(dllexport) KSTATUS IoQueryAsyncFileRequest(IO_ASYNC_FILE_REQUEST_OBJECT* req,
                                                      IO_STATUS_BLOCK* iosbOut) {
    return IoQueryAsyncRequest(req ? &req->Async : (IO_ASYNC_REQUEST_OBJECT*)0,
                               iosbOut);
}

// Validates one canonical native path relative to an already-authoritative root capability.
// Native relative paths use '/' only and never carry drive/CWD or parent-traversal semantics.
static KSTATUS KiValidateNativeRelativePath(SYS_STRING_VIEW path, bool allowEmpty) {
    if (!SysSvIsUtf8Text(path)) return NTLSTATUS_INVALID_PARAMETER;
    if (SysSvContainsByte(path, '\\'))
        return NTLSTATUS_INVALID_PARAMETER;
    if (path.Length == 0)
        return allowEmpty ? NTLSTATUS_SUCCESS : NTLSTATUS_INVALID_PARAMETER;
    if (path.Length >= SYS_PATH_MAX)
        return NTLSTATUS_BUFFER_TOO_SMALL;
    if (path.Data[0] == '/' || path.Data[path.Length - 1] == '/')
        return NTLSTATUS_INVALID_PARAMETER;

    size_t componentStart = 0;
    for (size_t i = 0; i <= path.Length; ++i) {
        if (i != path.Length && path.Data[i] != '/')
            continue;

        size_t componentLength = i - componentStart;
        if (componentLength == 0 || componentLength >= 256)
            return componentLength >= 256 ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_INVALID_PARAMETER;
        if ((componentLength == 1 && path.Data[componentStart] == '.') ||
            (componentLength == 2 && path.Data[componentStart] == '.' &&
                                     path.Data[componentStart + 1] == '.'))
            return NTLSTATUS_INVALID_PARAMETER;
        componentStart = i + 1;
    }
    return NTLSTATUS_SUCCESS;
}

// Composes a canonical path beneath one already-open directory capability.
static KSTATUS KiComposeDirectoryRelativePath(const DIR_OBJECT* directory,
                                               SYS_STRING_VIEW child,
                                               char* storage,
                                               size_t storageSize,
                                               SYS_STRING_VIEW* pathOut) {
    if (!directory || !storage || !pathOut)
        return NTLSTATUS_INVALID_PARAMETER;

    KSTATUS status = KiValidateNativeRelativePath(child, true);
    if (!KSUCCESS(status))
        return status;

    size_t total = directory->PathLength;
    if (directory->PathLength && child.Length) {
        if (total == SIZE_MAX)
            return NTLSTATUS_BUFFER_TOO_SMALL;
        total++;
    }
    if (child.Length > SIZE_MAX - total)
        return NTLSTATUS_BUFFER_TOO_SMALL;
    total += child.Length;
    if (total >= storageSize)
        return NTLSTATUS_BUFFER_TOO_SMALL;

    size_t offset = 0;
    if (directory->PathLength) {
        memcpy(storage, directory->Path, directory->PathLength);
        offset = directory->PathLength;
    }
    if (directory->PathLength && child.Length)
        storage[offset++] = '/';
    if (child.Length) {
        memcpy(storage + offset, child.Data, child.Length);
        offset += child.Length;
    }
    storage[offset] = '\0';
    *pathOut = SysSvFromParts(storage, offset);
    return NTLSTATUS_SUCCESS;
}

// Destructor.
static void KiFileDestroy(OBJECT* obj) {
    FILE_OBJECT* f = CONTAINING_RECORD(obj, FILE_OBJECT, Header);
    if (f->Volume && f->FileCtx) {
        // Dead volumes must never attempt writeback. The provider context remains
        // alive only so file-specific state can be closed safely.
        if (IoVolumeIsMounted(f->Volume)) {
            (void)MmVadSyncFileMappings(f);
            (void)MmFileCacheFlushFile(f);
        }
        MmFileCacheInvalidateAll(f);
        IoVolumeClose(f->Volume, f->FileCtx);
        ObDereferenceObject(&f->Volume->Header);
        f->Volume = (IO_VOLUME*)0;
    }
}

// Opens or creates a file using one already-owned mounted-volume reference.
// The reference is consumed on every path and becomes FILE_OBJECT ownership on success.
static KSTATUS KiOpenFileWithVolumeReference(IO_VOLUME* volume,
                                             SYS_STRING_VIEW path,
                                             bool create,
                                             FILE_OBJECT** out) {
    if (!volume || !out) {
        if (volume) ObDereferenceObject(&volume->Header);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    *out = (FILE_OBJECT*)0;

    void* fileCtx = (void*)0;
    KSTATUS status = create
                   ? IoVolumeCreate(volume, path, &fileCtx)
                   : IoVolumeOpen(volume, path, &fileCtx);
    if (!KSUCCESS(status)) {
        ObDereferenceObject(&volume->Header);
        return status;
    }

    uint64_t fileSize = 0;
    if (!create) {
        status = IoVolumeGetSize(volume, fileCtx, &fileSize);
        if (!KSUCCESS(status)) {
            IoVolumeClose(volume, fileCtx);
            ObDereferenceObject(&volume->Header);
            return status;
        }
    }

    FILE_OBJECT* file = (FILE_OBJECT*)ObAllocateObject(ObjectTypeFile, sizeof(FILE_OBJECT));
    if (!file) {
        IoVolumeClose(volume, fileCtx);
        ObDereferenceObject(&volume->Header);
        return NTLSTATUS_NO_MEMORY;
    }

    file->Header.Destroy = KiFileDestroy;
    file->Volume = volume;
    file->FileCtx = fileCtx;
    file->Lock = (SPINLOCK)SPINLOCK_INIT;
    file->Size = fileSize;
    file->Offset = 0;
    file->Flags = 0;
    *out = file;
    return NTLSTATUS_SUCCESS;
}

// Opens one canonical native path relative to a mounted volume capability.
__declspec(dllexport) KSTATUS IoOpenFileAtVolume(IO_VOLUME* volume,
                                                SYS_STRING_VIEW path,
                                                FILE_OBJECT** out) {
    if (!volume || !out) return NTLSTATUS_INVALID_PARAMETER;
    KSTATUS status = KiValidateNativeRelativePath(path, false);
    if (!KSUCCESS(status)) return status;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
    if (!IoVolumeTryReferenceMounted(volume)) return NTLSTATUS_OBJECT_DEAD;
    return KiOpenFileWithVolumeReference(volume, path, false, out);
}

// Creates one canonical native path relative to a mounted volume capability.
__declspec(dllexport) KSTATUS IoCreateFileAtVolume(IO_VOLUME* volume,
                                                  SYS_STRING_VIEW path,
                                                  FILE_OBJECT** out) {
    if (!volume || !out) return NTLSTATUS_INVALID_PARAMETER;
    KSTATUS status = KiValidateNativeRelativePath(path, false);
    if (!KSUCCESS(status)) return status;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
    if (!IoVolumeTryReferenceMounted(volume)) return NTLSTATUS_OBJECT_DEAD;
    return KiOpenFileWithVolumeReference(volume, path, true, out);
}

// Explicit-offset file I/O. Async I/O is built on these routines so it does
// not race the sequential FILE_OBJECT.Offset used by legacy read/write calls.
__declspec(dllexport) KSTATUS IoReadFileAt(FILE_OBJECT* file, uint64_t offset,
                                           void* buf, size_t size,
                                           size_t* bytesRead) {
    if (bytesRead) *bytesRead = 0;
    if (!file || !buf || !bytesRead) return NTLSTATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
    if (size == 0) return IoVolumeIsMounted(file->Volume) ? NTLSTATUS_SUCCESS : NTLSTATUS_OBJECT_DEAD;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t fileSize = file->Size;
    KeReleaseSpinlock(&file->Lock, flags);

    if (offset >= fileSize)
        return IoVolumeIsMounted(file->Volume) ? NTLSTATUS_SUCCESS : NTLSTATUS_OBJECT_DEAD;

    size_t remaining = (size_t)(fileSize - offset);
    if (size > remaining) size = remaining;

    if (file->Flags & IO_FILE_FLAG_NO_CACHE)
        return IoVolumeRead(file->Volume, file->FileCtx, offset, buf, size, bytesRead);

    size_t total = 0;
    uint8_t* dst = (uint8_t*)buf;
    while (total < size) {
        uint64_t absolute = offset + total;
        uint64_t pageOffset = absolute & ~(uint64_t)(PAGE_SIZE - 1);
        size_t inPage = (size_t)(absolute - pageOffset);
        size_t chunk = PAGE_SIZE - inPage;
        if (chunk > size - total) chunk = size - total;

        MM_FILE_CACHE_PAGE page;
        KSTATUS readStatus = MmFileCacheGetPage(file, pageOffset, &page);
        if (!KSUCCESS(readStatus)) {
            if (total) {
                *bytesRead = total;
                return NTLSTATUS_SUCCESS;
            }
            return readStatus;
        }

        if (inPage >= page.ValidBytes) {
            MmFileCacheReleasePage(&page);
            break;
        }
        size_t available = page.ValidBytes - inPage;
        if (chunk > available) chunk = available;

        void* srcPage = MmPfnToVirtual(page.Frame);
        if (!srcPage) {
            MmFileCacheReleasePage(&page);
            return total ? NTLSTATUS_SUCCESS : NTLSTATUS_FAILURE;
        }
        memcpy(dst + total, (const uint8_t*)srcPage + inPage, chunk);
        MmFileCacheReleasePage(&page);

        total += chunk;
        if (chunk == 0) break;
    }

    *bytesRead = total;
    return NTLSTATUS_SUCCESS;
}

__declspec(dllexport) KSTATUS IoWriteFileAt(FILE_OBJECT* file, uint64_t offset,
                                            const void* buf, size_t size,
                                            size_t* bytesWritten) {
    if (bytesWritten) *bytesWritten = 0;
    if (!file || !buf || !bytesWritten) return NTLSTATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
    if (size == 0) return IoVolumeIsMounted(file->Volume) ? NTLSTATUS_SUCCESS : NTLSTATUS_OBJECT_DEAD;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t oldSize = file->Size;
    KeReleaseSpinlock(&file->Lock, flags);

    size_t written = 0;
    KSTATUS s;
    if (file->Flags & IO_FILE_FLAG_NO_CACHE) {
        s = IoVolumeWrite(file->Volume, file->FileCtx, offset, buf, size, &written);
        if (KSUCCESS(s) && written)
            MmFileCacheInvalidate(file, offset, written);
    } else {
        if (!IoVolumeIsMounted(file->Volume)) return NTLSTATUS_OBJECT_DEAD;
        s = MmFileCacheWrite(file, offset, buf, size, oldSize, &written);
    }
    if (KSUCCESS(s)) {
        flags = KeAcquireSpinlock(&file->Lock);
        uint64_t endOffset = offset + written;
        if (endOffset > file->Size) file->Size = endOffset;
        KeReleaseSpinlock(&file->Lock, flags);
    }

    *bytesWritten = written;
    return s;
}

// Flushes dirty mapped and cached pages for one file object.
__declspec(dllexport) KSTATUS IoFlushFile(FILE_OBJECT* file) {
    if (!file) return NTLSTATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
    if (!IoVolumeIsMounted(file->Volume)) return NTLSTATUS_OBJECT_DEAD;

    KSTATUS sync = MmVadSyncFileMappings(file);
    KSTATUS cache = MmFileCacheFlushFile(file);
    KSTATUS media = NTLSTATUS_SUCCESS;
    if (KSUCCESS(sync) && KSUCCESS(cache))
        media = IoVolumeFlush(file->Volume, file->FileCtx);
    if (!KSUCCESS(sync)) return sync;
    if (!KSUCCESS(cache)) return cache;
    return media;
}

// IoReadFile - sequential read at current offset.
__declspec(dllexport) KSTATUS IoReadFile(FILE_OBJECT* file, void* buf,
                                         size_t size, size_t* bytesRead) {
    if (bytesRead) *bytesRead = 0;
    if (!file || !buf || !bytesRead) return NTLSTATUS_INVALID_PARAMETER;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t offset = file->Offset;
    KeReleaseSpinlock(&file->Lock, flags);

    KSTATUS s = IoReadFileAt(file, offset, buf, size, bytesRead);
    if (KSUCCESS(s)) {
        flags = KeAcquireSpinlock(&file->Lock);
        file->Offset = offset + *bytesRead;
        KeReleaseSpinlock(&file->Lock, flags);
    }
    return s;
}

// IoWriteFile - sequential write at current offset.
__declspec(dllexport) KSTATUS IoWriteFile(FILE_OBJECT* file, const void* buf,
                                          size_t size, size_t* bytesWritten) {
    if (bytesWritten) *bytesWritten = 0;
    if (!file || !buf || !bytesWritten) return NTLSTATUS_INVALID_PARAMETER;

    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t offset = (file->Flags & IO_FILE_FLAG_APPEND) ? file->Size : file->Offset;
    KeReleaseSpinlock(&file->Lock, flags);

    KSTATUS s = IoWriteFileAt(file, offset, buf, size, bytesWritten);
    if (KSUCCESS(s)) {
        flags = KeAcquireSpinlock(&file->Lock);
        file->Offset = offset + *bytesWritten;
        KeReleaseSpinlock(&file->Lock, flags);
    }
    return s;
}

// IoTruncateFile - resize the file; updates the cached Size on success.
__declspec(dllexport) KSTATUS IoTruncateFile(FILE_OBJECT* file, uint64_t newSize) {
    if (!file) return NTLSTATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;

    KSTATUS flush = IoFlushFile(file);
    if (!KSUCCESS(flush)) return flush;

    KSTATUS s = IoVolumeTruncate(file->Volume, file->FileCtx, newSize, file->Flags);
    if (KSUCCESS(s)) {
        MmFileCacheInvalidate(file, newSize, (size_t)-1);
        file->Size = newSize;
        if (file->Offset > file->Size)
            file->Offset = file->Size;
    }
    return s;
}

__declspec(dllexport) KSTATUS IoSeekFile(FILE_OBJECT* file, uint64_t offset) {
    if (!file) return NTLSTATUS_INVALID_PARAMETER;
    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    file->Offset = offset;
    KeReleaseSpinlock(&file->Lock, flags);
    return NTLSTATUS_SUCCESS;
}

// Returns file Size.
__declspec(dllexport) uint64_t IoGetFileSize(FILE_OBJECT* file) {
    if (!file) return 0;
    uintptr_t flags = KeAcquireSpinlock(&file->Lock);
    uint64_t size = file->Size;
    KeReleaseSpinlock(&file->Lock, flags);
    return size;
}

// Queries metadata for an open file object.
__declspec(dllexport) KSTATUS IoGetFileStat(FILE_OBJECT* file, SYS_FILE_STAT* out) {
    if (!file || !out) return NTLSTATUS_INVALID_PARAMETER;
    memset(out, 0, sizeof(*out));
    out->Size = sizeof(*out);
    out->Type = SYS_FILE_TYPE_REGULAR;
    out->LinkCount = 1;
    out->FileSize = IoGetFileSize(file);

    KSTATUS s = IoVolumeQueryInfo(file->Volume, file->FileCtx, out);
    if (s == NTLSTATUS_NOT_SUPPORTED)
        return NTLSTATUS_SUCCESS;
    if (KSUCCESS(s)) {
        uintptr_t flags = KeAcquireSpinlock(&file->Lock);
        file->Size = out->FileSize;
        KeReleaseSpinlock(&file->Lock, flags);
    }
    return s;
}

// Queries the filesystem identity for a file object. VolumeId is always the IO_VOLUME runtime InstanceId.
__declspec(dllexport) KSTATUS IoQueryFileIdentity(FILE_OBJECT* file, IO_FILE_IDENTITY* out) {
    if (!file || !out)
        return NTLSTATUS_INVALID_PARAMETER;
    memset(out, 0, sizeof(*out));
    return IoVolumeQueryIdentity(file->Volume, file->FileCtx, out);
}

// Queries the first conflicting advisory lock held by another process.
__declspec(dllexport) KSTATUS IoQueryFileLock(FILE_OBJECT* file, PROCESS* owner, uint32_t mode,
                                              uint64_t offset, uint64_t length,
                                              SYS_FILE_LOCK_INFO* infoOut) {
    if (!file || !owner || !infoOut)
        return NTLSTATUS_INVALID_PARAMETER;
    if (mode != SYS_FILE_LOCK_SHARED && mode != SYS_FILE_LOCK_EXCLUSIVE)
        return NTLSTATUS_INVALID_PARAMETER;
    if (length == 0)
        return NTLSTATUS_INVALID_PARAMETER;

    IO_FILE_IDENTITY identity;
    KSTATUS status = IoQueryFileIdentity(file, &identity);
    if (!KSUCCESS(status))
        return status;

    KiIoEnsureFileLockState();
    memset(infoOut, 0, sizeof(*infoOut));
    infoOut->Size = sizeof(*infoOut);
    infoOut->Mode = SYS_FILE_LOCK_NONE;

    uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
    LIST_ENTRY* head = &gFileLocks;
    for (LIST_ENTRY* node = head->Fwd; node != head; node = node->Fwd) {
        IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
        if (!KiIoFileLockConflicts(lock, &identity, owner, mode, offset, length))
            continue;
        infoOut->Mode = lock->Mode;
        infoOut->OwnerProcessId = lock->OwnerProcessId;
        infoOut->Offset = lock->Offset;
        infoOut->Length = lock->Length;
        break;
    }
    KeReleaseSpinlock(&gFileLockLock, flags);
    return NTLSTATUS_SUCCESS;
}

// Changes mutable file-object status flags without changing handle authority.
__declspec(dllexport) KSTATUS IoSetFileStatus(FILE_OBJECT* file, uint32_t mask, uint32_t flags) {
    if (!file)
        return NTLSTATUS_INVALID_PARAMETER;
    if (mask & ~SYS_FILE_STATUS_APPEND)
        return NTLSTATUS_INVALID_PARAMETER;
    if (flags & ~mask)
        return NTLSTATUS_INVALID_PARAMETER;

    uintptr_t lockFlags = KeAcquireSpinlock(&file->Lock);
    if (mask & SYS_FILE_STATUS_APPEND) {
        if (flags & SYS_FILE_STATUS_APPEND)
            file->Flags |= IO_FILE_FLAG_APPEND;
        else
            file->Flags &= ~IO_FILE_FLAG_APPEND;
    }
    KeReleaseSpinlock(&file->Lock, lockFlags);
    return NTLSTATUS_SUCCESS;
}

// Acquires an advisory process-owned byte-range lock for one file identity.
__declspec(dllexport) KSTATUS IoLockFile(FILE_OBJECT* file, PROCESS* owner, uint32_t mode,
                                         uint64_t offset, uint64_t length,
                                         uint64_t timeoutMilliseconds) {
    if (!file || !owner) return NTLSTATUS_INVALID_PARAMETER;
    if (mode != SYS_FILE_LOCK_SHARED && mode != SYS_FILE_LOCK_EXCLUSIVE) return NTLSTATUS_INVALID_PARAMETER;
    if (length == 0) return NTLSTATUS_INVALID_PARAMETER;
    if (timeoutMilliseconds != 0 && !KeCanWait()) return NTLSTATUS_INVALID_IRQL;

    IO_FILE_IDENTITY identity;
    KSTATUS s = IoQueryFileIdentity(file, &identity);
    if (!KSUCCESS(s)) return s;

    KiIoEnsureFileLockState();

    uint64_t deadlineMicroseconds = 0;
    if (timeoutMilliseconds != 0 && timeoutMilliseconds != SYS_WAIT_TIMEOUT_INFINITE)
        deadlineMicroseconds = KiIoFileLockTimeoutDeadline(timeoutMilliseconds);

    EVENT* waitEvent = (EVENT*)0;
    IO_FILE_LOCK_WAITER waiter;
    memset(&waiter, 0, sizeof(waiter));

    while (true) {
        IO_FILE_LOCK_ENTRY* entry =
            (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*entry), MM_POOL_TAG('f', 'l', 'c', 'k'));
        IO_FILE_LOCK_ENTRY* split =
            (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*split), MM_POOL_TAG('f', 'l', 'c', 'k'));
        if (!entry || !split) {
            if (entry) MmFreePool(entry);
            if (split) MmFreePool(split);
            if (waitEvent) ObDereferenceObject(&waitEvent->Header);
            return NTLSTATUS_NO_MEMORY;
        }

        LIST_ENTRY detached;
        LIST_INIT(&detached);
        bool changedOwnedRange = false;
        bool splitUsed = false;
        uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
        if (!KiIoFileLockHasConflictLocked(&identity, owner, mode, offset, length)) {
            uint64_t requestedEnd = KiIoFileLockEnd(offset, length);
            LIST_ENTRY* head = &gFileLocks;
            LIST_ENTRY* node = head->Fwd;
            while (node != head) {
                LIST_ENTRY* next = node->Fwd;
                IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
                if (lock->Owner != owner || !KiIoSameFileIdentity(&lock->Identity, &identity) ||
                    !KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length)) {
                    node = next;
                    continue;
                }

                uint64_t lockStart = lock->Offset;
                uint64_t lockEnd = KiIoFileLockEnd(lock->Offset, lock->Length);
                changedOwnedRange = true;

                if (lockStart < offset && requestedEnd < lockEnd) {
                    lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
                    KiIoInitializeFileLockEntry(split, &identity, owner, lock->Mode,
                                                requestedEnd,
                                                KiIoFileLockLengthFromEnd(requestedEnd, lockEnd));
                    ListInsertTail(&gFileLocks, &split->Link);
                    splitUsed = true;
                } else if (lockStart < offset) {
                    lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
                } else if (requestedEnd < lockEnd) {
                    lock->Offset = requestedEnd;
                    lock->Length = KiIoFileLockLengthFromEnd(requestedEnd, lockEnd);
                } else {
                    ListRemove(&lock->Link);
                    ListInsertTail(&detached, &lock->Link);
                }
                node = next;
            }

            KiIoInitializeFileLockEntry(entry, &identity, owner, mode, offset, length);
            ListInsertTail(&gFileLocks, &entry->Link);
            if (changedOwnedRange)
                KiIoSignalFileLockWaitersLocked(&identity);
            KeReleaseSpinlock(&gFileLockLock, flags);

            if (!splitUsed)
                MmFreePool(split);
            KiIoFreeDetachedFileLocks(&detached);
            if (waitEvent) ObDereferenceObject(&waitEvent->Header);
            return NTLSTATUS_SUCCESS;
        }
        KeReleaseSpinlock(&gFileLockLock, flags);
        MmFreePool(entry);
        MmFreePool(split);

        if (timeoutMilliseconds == 0) {
            if (waitEvent) ObDereferenceObject(&waitEvent->Header);
            return NTLSTATUS_WOULD_BLOCK;
        }

        if (!waitEvent) {
            s = KeEventCreate(EventAutoReset, 0, &waitEvent);
            if (!KSUCCESS(s)) return s;
            waiter.Identity = identity;
            waiter.Owner = owner;
            waiter.Offset = offset;
            waiter.Length = length;
            waiter.Mode = mode;
            waiter.Event = waitEvent;
            LIST_INIT(&waiter.Link);
        } else {
            KeClearEvent(waitEvent);
        }

        flags = KeAcquireSpinlock(&gFileLockLock);
        if (!KiIoFileLockHasConflictLocked(&identity, owner, mode, offset, length)) {
            KeReleaseSpinlock(&gFileLockLock, flags);
            continue;
        }
        waiter.Armed = 1;
        ListInsertTail(&gFileLockWaiters, &waiter.Link);
        KeReleaseSpinlock(&gFileLockLock, flags);

        s = KiIoWaitForFileLockChange(waitEvent, timeoutMilliseconds, deadlineMicroseconds);

        flags = KeAcquireSpinlock(&gFileLockLock);
        if (waiter.Armed) {
            ListRemove(&waiter.Link);
            waiter.Armed = 0;
        }
        KeReleaseSpinlock(&gFileLockLock, flags);

        if (!KSUCCESS(s)) {
            ObDereferenceObject(&waitEvent->Header);
            return s;
        }
    }
}

// Releases this process's requested advisory byte range while preserving outside fragments.
__declspec(dllexport) KSTATUS IoUnlockFile(FILE_OBJECT* file, PROCESS* owner, uint64_t offset, uint64_t length) {
    if (!file || !owner || length == 0) return NTLSTATUS_INVALID_PARAMETER;

    IO_FILE_IDENTITY identity;
    KSTATUS s = IoQueryFileIdentity(file, &identity);
    if (!KSUCCESS(s)) return s;

    KiIoEnsureFileLockState();

    IO_FILE_LOCK_ENTRY* split =
        (IO_FILE_LOCK_ENTRY*)MmAllocatePoolTag(sizeof(*split), MM_POOL_TAG('f', 'l', 'c', 'k'));
    LIST_ENTRY detached;
    LIST_INIT(&detached);
    bool changed = false;
    bool splitUsed = false;
    uint64_t requestedEnd = KiIoFileLockEnd(offset, length);

    uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
    LIST_ENTRY* head = &gFileLocks;
    LIST_ENTRY* node = head->Fwd;
    while (node != head) {
        LIST_ENTRY* next = node->Fwd;
        IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
        if (lock->Owner != owner || !KiIoSameFileIdentity(&lock->Identity, &identity) ||
            !KiIoFileLockRangesOverlap(lock->Offset, lock->Length, offset, length)) {
            node = next;
            continue;
        }

        uint64_t lockStart = lock->Offset;
        uint64_t lockEnd = KiIoFileLockEnd(lock->Offset, lock->Length);
        if (lockStart < offset && requestedEnd < lockEnd) {
            if (!split) {
                KeReleaseSpinlock(&gFileLockLock, flags);
                KiIoFreeDetachedFileLocks(&detached);
                return NTLSTATUS_NO_MEMORY;
            }
            lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
            KiIoInitializeFileLockEntry(split, &identity, owner, lock->Mode,
                                        requestedEnd,
                                        KiIoFileLockLengthFromEnd(requestedEnd, lockEnd));
            ListInsertTail(&gFileLocks, &split->Link);
            splitUsed = true;
        } else if (lockStart < offset) {
            lock->Length = KiIoFileLockLengthFromEnd(lockStart, offset);
        } else if (requestedEnd < lockEnd) {
            lock->Offset = requestedEnd;
            lock->Length = KiIoFileLockLengthFromEnd(requestedEnd, lockEnd);
        } else {
            ListRemove(&lock->Link);
            ListInsertTail(&detached, &lock->Link);
        }
        changed = true;
        node = next;
    }

    if (changed)
        KiIoSignalFileLockWaitersLocked(&identity);
    KeReleaseSpinlock(&gFileLockLock, flags);

    if (split && !splitUsed)
        MmFreePool(split);
    KiIoFreeDetachedFileLocks(&detached);
    return changed ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND;
}

// Releases every advisory file lock owned by one exiting process.
__declspec(dllexport) void IoReleaseFileLocksForProcess(PROCESS* owner) {
    if (!owner) return;

    KiIoEnsureFileLockState();

    bool released = false;
    uintptr_t flags = KeAcquireSpinlock(&gFileLockLock);
    LIST_ENTRY* head = &gFileLocks;
    LIST_ENTRY* node = head->Fwd;
    while (node != head) {
        LIST_ENTRY* next = node->Fwd;
        IO_FILE_LOCK_ENTRY* lock = CONTAINING_RECORD(node, IO_FILE_LOCK_ENTRY, Link);
        if (lock->Owner == owner) {
            ListRemove(&lock->Link);
            MmFreePool(lock);
            released = true;
        }
        node = next;
    }

    if (released) {
        LIST_ENTRY* waiterHead = &gFileLockWaiters;
        for (LIST_ENTRY* waiterNode = waiterHead->Fwd; waiterNode != waiterHead; waiterNode = waiterNode->Fwd) {
            IO_FILE_LOCK_WAITER* waiter = CONTAINING_RECORD(waiterNode, IO_FILE_LOCK_WAITER, Link);
            if (waiter->Armed && waiter->Event)
                KeSignalEvent(waiter->Event);
        }
    }
    KeReleaseSpinlock(&gFileLockLock, flags);
}

// Releases file.
__declspec(dllexport) void IoCloseFile(FILE_OBJECT* file) {
    if (file) ObDereferenceObject(&file->Header);
}

// Directory enumeration.

static void KiDirDestroy(OBJECT* obj) {
    DIR_OBJECT* dir = (DIR_OBJECT*)obj; // Header is first member.
    if (dir->Volume && dir->DirCtx) {
        IoVolumeCloseDir(dir->Volume, dir->DirCtx);
        ObDereferenceObject(&dir->Volume->Header);
        dir->Volume = (IO_VOLUME*)0;
    }
}

// Opens a provider directory using one already-owned mounted-volume reference.
// The reference is consumed on every path and becomes DIR_OBJECT ownership on success.
static KSTATUS KiOpenDirWithVolumeReference(IO_VOLUME* volume,
                                            SYS_STRING_VIEW path,
                                            DIR_OBJECT** out) {
    if (!volume || !out) {
        if (volume) ObDereferenceObject(&volume->Header);
        return NTLSTATUS_INVALID_PARAMETER;
    }
    *out = (DIR_OBJECT*)0;

    void* dirCtx = (void*)0;
    KSTATUS status = IoVolumeOpenDir(volume, path, &dirCtx);
    if (!KSUCCESS(status)) {
        ObDereferenceObject(&volume->Header);
        return status;
    }

    if (path.Length > SIZE_MAX - sizeof(DIR_OBJECT) - 1) {
        IoVolumeCloseDir(volume, dirCtx);
        ObDereferenceObject(&volume->Header);
        return NTLSTATUS_BUFFER_TOO_SMALL;
    }
    size_t allocationSize = sizeof(DIR_OBJECT) + path.Length + 1;
    DIR_OBJECT* dir = (DIR_OBJECT*)ObAllocateObject(ObjectTypeDir, allocationSize);
    if (!dir) {
        IoVolumeCloseDir(volume, dirCtx);
        ObDereferenceObject(&volume->Header);
        return NTLSTATUS_NO_MEMORY;
    }

    dir->Header.Destroy = KiDirDestroy;
    dir->Volume = volume;
    dir->DirCtx = dirCtx;
    dir->Path = (char*)(dir + 1);
    dir->PathLength = path.Length;
    for (size_t i = 0; i < path.Length; ++i)
        dir->Path[i] = path.Data[i] == '\\' ? '/' : path.Data[i];
    dir->Path[path.Length] = '\0';
    *out = dir;
    return NTLSTATUS_SUCCESS;
}

// Opens one canonical native directory path relative to a mounted volume capability.
__declspec(dllexport) KSTATUS IoOpenDirAtVolume(IO_VOLUME* volume,
                                               SYS_STRING_VIEW path,
                                               DIR_OBJECT** out) {
    if (!volume || !out) return NTLSTATUS_INVALID_PARAMETER;
    KSTATUS status = KiValidateNativeRelativePath(path, true);
    if (!KSUCCESS(status)) return status;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
    if (!IoVolumeTryReferenceMounted(volume)) return NTLSTATUS_OBJECT_DEAD;
    return KiOpenDirWithVolumeReference(volume, path, out);
}

// Opens one file beneath an already-open directory capability.
KSTATUS IoOpenFileAtDirectory(DIR_OBJECT* directory, SYS_STRING_VIEW path, FILE_OBJECT** out) {
    if (!directory || !out) return NTLSTATUS_INVALID_PARAMETER;
    char storage[SYS_PATH_MAX];
    SYS_STRING_VIEW combined;
    KSTATUS status = KiComposeDirectoryRelativePath(directory, path, storage, sizeof(storage), &combined);
    if (!KSUCCESS(status)) return status;
    if (combined.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
    return IoOpenFileAtVolume(directory->Volume, combined, out);
}

// Creates one file beneath an already-open directory capability.
KSTATUS IoCreateFileAtDirectory(DIR_OBJECT* directory, SYS_STRING_VIEW path, FILE_OBJECT** out) {
    if (!directory || !out) return NTLSTATUS_INVALID_PARAMETER;
    char storage[SYS_PATH_MAX];
    SYS_STRING_VIEW combined;
    KSTATUS status = KiComposeDirectoryRelativePath(directory, path, storage, sizeof(storage), &combined);
    if (!KSUCCESS(status)) return status;
    if (combined.Length == 0) return NTLSTATUS_INVALID_PARAMETER;
    return IoCreateFileAtVolume(directory->Volume, combined, out);
}

// Opens one directory beneath an already-open directory capability.
KSTATUS IoOpenDirAtDirectory(DIR_OBJECT* directory, SYS_STRING_VIEW path, DIR_OBJECT** out) {
    if (!directory || !out) return NTLSTATUS_INVALID_PARAMETER;
    char storage[SYS_PATH_MAX];
    SYS_STRING_VIEW combined;
    KSTATUS status = KiComposeDirectoryRelativePath(directory, path, storage, sizeof(storage), &combined);
    if (!KSUCCESS(status)) return status;
    return IoOpenDirAtVolume(directory->Volume, combined, out);
}

// Resolves one capability-relative namespace path to its owning volume and canonical provider path.
static KSTATUS KiResolveAtRootPath(OBJECT* root,
                                   SYS_STRING_VIEW path,
                                   bool allowEmpty,
                                   char* storage,
                                   size_t storageSize,
                                   IO_VOLUME** volumeOut,
                                   SYS_STRING_VIEW* pathOut) {
    if (!root || !storage || !volumeOut || !pathOut)
        return NTLSTATUS_INVALID_PARAMETER;
    *volumeOut = (IO_VOLUME*)0;
    *pathOut = SysSvFromParts((const char*)0, 0);

    if (root->Type == ObjectTypeVolume) {
        KSTATUS status = KiValidateNativeRelativePath(path, allowEmpty);
        if (!KSUCCESS(status)) return status;
        *volumeOut = (IO_VOLUME*)root;
        *pathOut = path;
        return NTLSTATUS_SUCCESS;
    }
    if (root->Type == ObjectTypeDir) {
        DIR_OBJECT* directory = (DIR_OBJECT*)root;
        KSTATUS status = KiComposeDirectoryRelativePath(directory, path, storage, storageSize, pathOut);
        if (!KSUCCESS(status)) return status;
        if (!allowEmpty && pathOut->Length == 0) return NTLSTATUS_INVALID_PARAMETER;
        *volumeOut = directory->Volume;
        return NTLSTATUS_SUCCESS;
    }
    return NTLSTATUS_INVALID_HANDLE;
}

// Queries metadata beneath one mounted-volume or directory capability.
KSTATUS IoStatAtRoot(OBJECT* root, SYS_STRING_VIEW path, SYS_FILE_STAT* out) {
    if (!out) return NTLSTATUS_INVALID_PARAMETER;
    char storage[SYS_PATH_MAX];
    IO_VOLUME* volume = (IO_VOLUME*)0;
    SYS_STRING_VIEW resolved;
    KSTATUS status = KiResolveAtRootPath(root, path, true, storage, sizeof(storage), &volume, &resolved);
    if (!KSUCCESS(status)) return status;
    memset(out, 0, sizeof(*out));
    out->Size = sizeof(*out);
    return IoVolumeStat(volume, resolved, out);
}

// Deletes one file beneath one mounted-volume or directory capability.
KSTATUS IoDeleteFileAtRoot(OBJECT* root, SYS_STRING_VIEW path) {
    char storage[SYS_PATH_MAX];
    IO_VOLUME* volume = (IO_VOLUME*)0;
    SYS_STRING_VIEW resolved;
    KSTATUS status = KiResolveAtRootPath(root, path, false, storage, sizeof(storage), &volume, &resolved);
    if (!KSUCCESS(status)) return status;
    return IoVolumeDelete(volume, resolved);
}

// Renames one path between capability roots on the same mounted volume.
KSTATUS IoRenameFileAtRoots(OBJECT* oldRoot, SYS_STRING_VIEW oldPath,
                            OBJECT* newRoot, SYS_STRING_VIEW newPath) {
    char oldStorage[SYS_PATH_MAX];
    char newStorage[SYS_PATH_MAX];
    IO_VOLUME* oldVolume = (IO_VOLUME*)0;
    IO_VOLUME* newVolume = (IO_VOLUME*)0;
    SYS_STRING_VIEW oldResolved;
    SYS_STRING_VIEW newResolved;
    KSTATUS status = KiResolveAtRootPath(oldRoot, oldPath, false, oldStorage, sizeof(oldStorage),
                                         &oldVolume, &oldResolved);
    if (!KSUCCESS(status)) return status;
    status = KiResolveAtRootPath(newRoot, newPath, false, newStorage, sizeof(newStorage),
                                 &newVolume, &newResolved);
    if (!KSUCCESS(status)) return status;
    if (oldVolume != newVolume) return NTLSTATUS_NOT_SUPPORTED;
    return IoVolumeRename(oldVolume, oldResolved, newResolved);
}

// Creates one directory beneath one mounted-volume or directory capability.
KSTATUS IoCreateDirAtRoot(OBJECT* root, SYS_STRING_VIEW path) {
    char storage[SYS_PATH_MAX];
    IO_VOLUME* volume = (IO_VOLUME*)0;
    SYS_STRING_VIEW resolved;
    KSTATUS status = KiResolveAtRootPath(root, path, false, storage, sizeof(storage), &volume, &resolved);
    if (!KSUCCESS(status)) return status;
    return IoVolumeCreateDir(volume, resolved);
}

// Removes one directory beneath one mounted-volume or directory capability.
KSTATUS IoRemoveDirAtRoot(OBJECT* root, SYS_STRING_VIEW path) {
    char storage[SYS_PATH_MAX];
    IO_VOLUME* volume = (IO_VOLUME*)0;
    SYS_STRING_VIEW resolved;
    KSTATUS status = KiResolveAtRootPath(root, path, false, storage, sizeof(storage), &volume, &resolved);
    if (!KSUCCESS(status)) return status;
    return IoVolumeRemoveDir(volume, resolved);
}

// Returns dir.
__declspec(dllexport) KSTATUS IoReadDir(DIR_OBJECT* dir, KDIR_ENTRY* entryOut) {
    if (!dir || !entryOut) return NTLSTATUS_INVALID_PARAMETER;
    if (!KeCanWait()) return NTLSTATUS_INVALID_IRQL;
    return IoVolumeReadDir(dir->Volume, dir->DirCtx, entryOut);
}

// Releases dir.
__declspec(dllexport) void IoCloseDir(DIR_OBJECT* dir) {
    if (dir) ObDereferenceObject(&dir->Header);
}