usb hubs, hotplug support, unplug support, saving log files to disk, smt
This commit is contained in:
maksw committed
1 parent
b838f8d83d
commit
27b31f62b6
39 files changed
+6294
-735
No files matched your search
+259
-68
@@ -1,130 +1,321 @@
|
||||
// ke_irp.c - I/O Request Packet dispatch and queue helpers
|
||||
//
|
||||
// This file lives in the kernel layer and is the ONLY place that bridges the
|
||||
// IRP abstraction to kernel sync objects. All calls to KeEventCreate,
|
||||
// KeSignalEvent, KeWaitForEvent, KeSignalSemaphore, and ObDereferenceObject
|
||||
// are confined here; hal_disk.c and io_driver.h remain free of those symbols.
|
||||
//
|
||||
// Dependency graph (→ = includes / calls):
|
||||
// ke_irp.c → ke_irp.h (IRP type, IRP_QUEUE, forward decls)
|
||||
// → ke_event.h (KeEventCreate, KeSignalEvent, KeWaitForEvent)
|
||||
// → ke_semaphore.h (KeSignalSemaphore)
|
||||
// → ob_object.h (ObDereferenceObject)
|
||||
// → io_driver.h (DEVICE_OBJECT / IoCallDriver)
|
||||
// ke_irp.c - I/O Request Packet dispatch, queue, cancellation, remove-locks
|
||||
|
||||
#include <string.h>
|
||||
#include <intrin.h>
|
||||
#include <kernel/ke_irp.h>
|
||||
#include <kernel/ke_event.h>
|
||||
#include <kernel/ke_semaphore.h>
|
||||
#include <kernel/ke_apc.h>
|
||||
#include <kernel/ob_object.h>
|
||||
#include <kernel/mm_virt.h>
|
||||
#include <ntlite/ddi/io_driver.h>
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// IRP_QUEUE operations
|
||||
// -------------------------------------------------------------------------
|
||||
// These are defined here (not as static inlines in ke_irp.h) so that
|
||||
// ke_irp.h can avoid including ke_semaphore.h, keeping the header safe
|
||||
// for inclusion from the HAL layer.
|
||||
|
||||
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE *q, struct SEMAPHORE *sem) {
|
||||
__declspec(dllexport) void IrpQueueInit(IRP_QUEUE* q, struct SEMAPHORE* sem) {
|
||||
q->Lock = (SPINLOCK)SPINLOCK_INIT;
|
||||
q->Semaphore = sem;
|
||||
LIST_INIT(&q->Head);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE *q, IRP *irp) {
|
||||
__declspec(dllexport) void IrpQueueEnqueue(IRP_QUEUE* q, IRP* irp) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
|
||||
if (irp->Flags & IRP_FLAG_PRIORITY)
|
||||
ListInsertHead(&q->Head, &irp->Link);
|
||||
else
|
||||
ListInsertTail(&q->Head, &irp->Link);
|
||||
KeReleaseSpinlock(&q->Lock, flags);
|
||||
// Signal outside the spinlock - safe, and avoids priority-inversion risk.
|
||||
KeSignalSemaphore((SEMAPHORE *)q->Semaphore);
|
||||
KeSignalSemaphore((SEMAPHORE*)q->Semaphore);
|
||||
}
|
||||
|
||||
__declspec(dllexport) IRP *IrpQueueDequeue(IRP_QUEUE *q) {
|
||||
__declspec(dllexport) IRP* IrpQueueDequeue(IRP_QUEUE* q) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
|
||||
LIST_ENTRY *node = ListPopHead(&q->Head);
|
||||
LIST_ENTRY* node = ListPopHead(&q->Head);
|
||||
KeReleaseSpinlock(&q->Lock, flags);
|
||||
if (!node) return (IRP *)0;
|
||||
if (!node) return (IRP*)0;
|
||||
return CONTAINING_RECORD(node, IRP, Link);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// IoCompleteRequest
|
||||
// -------------------------------------------------------------------------
|
||||
__declspec(dllexport) void IrpQueueCancelAllEx(IRP_QUEUE* q,
|
||||
KSTATUS status,
|
||||
IRP_QUEUE_CANCEL_CALLBACK callback,
|
||||
void* context) {
|
||||
if (!q) return;
|
||||
for (;;) {
|
||||
uintptr_t flags = KeAcquireSpinlock(&q->Lock);
|
||||
LIST_ENTRY* node = ListPopHead(&q->Head);
|
||||
KeReleaseSpinlock(&q->Lock, flags);
|
||||
if (!node) break;
|
||||
|
||||
static size_t IoIrpTransferBytes(const IRP *irp, KSTATUS status) {
|
||||
if (!irp || !KSUCCESS(status)) return 0;
|
||||
return (size_t)irp->SectorCount * 512u;
|
||||
IRP* irp = CONTAINING_RECORD(node, IRP, Link);
|
||||
irp->Cancel = true;
|
||||
irp->Flags |= IRP_FLAG_CANCELLED;
|
||||
if (callback)
|
||||
callback(irp, context);
|
||||
IoCompleteRequest(irp, status, 0);
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoCompleteRequest(IRP *irp, KSTATUS status, size_t information) {
|
||||
__declspec(dllexport) void IrpQueueCancelAll(IRP_QUEUE* q, KSTATUS status) {
|
||||
IrpQueueCancelAllEx(q, status, (IRP_QUEUE_CANCEL_CALLBACK)0, (void*)0);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Remove locks
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
__declspec(dllexport) KSTATUS IoInitializeRemoveLock(IO_REMOVE_LOCK* lock) {
|
||||
if (!lock) return STATUS_INVALID_PARAMETER;
|
||||
memset(lock, 0, sizeof(*lock));
|
||||
lock->Lock = (SPINLOCK)SPINLOCK_INIT;
|
||||
lock->IoCount = 1; // sentinel reference released by IoReleaseRemoveLockAndWait
|
||||
KSTATUS st = KeEventCreate(EventManualReset, 0, &lock->RemovedEvent);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoDestroyRemoveLock(IO_REMOVE_LOCK* lock) {
|
||||
if (!lock) return;
|
||||
if (lock->RemovedEvent) {
|
||||
ObDereferenceObject(&lock->RemovedEvent->Header);
|
||||
lock->RemovedEvent = (EVENT*)0;
|
||||
}
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoAcquireRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
|
||||
(void)irp;
|
||||
if (!lock) return STATUS_INVALID_PARAMETER;
|
||||
uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
|
||||
if (lock->Removing) {
|
||||
KeReleaseSpinlock(&lock->Lock, flags);
|
||||
return STATUS_OBJECT_DEAD;
|
||||
}
|
||||
_InterlockedIncrement(&lock->IoCount);
|
||||
KeReleaseSpinlock(&lock->Lock, flags);
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoReleaseRemoveLock(IO_REMOVE_LOCK* lock, IRP* irp) {
|
||||
(void)irp;
|
||||
if (!lock) return;
|
||||
long left = _InterlockedDecrement(&lock->IoCount);
|
||||
if (left == 0 && lock->Removing && lock->RemovedEvent)
|
||||
KeSignalEvent(lock->RemovedEvent);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoReleaseRemoveLockAndWait(IO_REMOVE_LOCK* lock, IRP* removeIrp) {
|
||||
(void)removeIrp;
|
||||
if (!lock) return;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&lock->Lock);
|
||||
lock->Removing = true;
|
||||
long left = _InterlockedDecrement(&lock->IoCount); // release sentinel
|
||||
bool done = (left == 0);
|
||||
KeReleaseSpinlock(&lock->Lock, flags);
|
||||
|
||||
if (done) {
|
||||
if (lock->RemovedEvent) KeSignalEvent(lock->RemovedEvent);
|
||||
return;
|
||||
}
|
||||
|
||||
if (lock->RemovedEvent)
|
||||
KeWaitForEvent(lock->RemovedEvent, 0);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// IRP helpers
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
static size_t IoIrpSize(uint8_t stackCount) {
|
||||
if (stackCount == 0) stackCount = 1;
|
||||
return sizeof(IRP) + ((size_t)stackCount - 1u) * sizeof(IO_STACK_LOCATION);
|
||||
}
|
||||
|
||||
__declspec(dllexport) KSTATUS IoAllocateIrp(uint8_t stackCount, IRP** outIrp) {
|
||||
if (!outIrp) return STATUS_INVALID_PARAMETER;
|
||||
if (stackCount == 0) stackCount = 1;
|
||||
IRP* irp = (IRP*)MmAllocatePool(IoIrpSize(stackCount));
|
||||
if (!irp) return STATUS_NO_MEMORY;
|
||||
IoInitializeIrp(irp, stackCount);
|
||||
*outIrp = irp;
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoFreeIrp(IRP* irp) {
|
||||
if (irp) MmFreePool(irp);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoInitializeIrp(IRP* irp, uint8_t stackCount) {
|
||||
if (!irp) return;
|
||||
if (stackCount == 0) stackCount = 1;
|
||||
memset(irp, 0, IoIrpSize(stackCount));
|
||||
LIST_INIT(&irp->Link);
|
||||
irp->Status = STATUS_FAILURE;
|
||||
irp->Information = 0;
|
||||
irp->CancelLock = (SPINLOCK)SPINLOCK_INIT;
|
||||
irp->StackCount = stackCount;
|
||||
irp->CurrentLocation = 0;
|
||||
}
|
||||
|
||||
__declspec(dllexport) IO_STACK_LOCATION* IoGetCurrentIrpStackLocation(IRP* irp) {
|
||||
if (!irp || irp->CurrentLocation >= irp->StackCount) return (IO_STACK_LOCATION*)0;
|
||||
return &irp->Stack[irp->CurrentLocation];
|
||||
}
|
||||
|
||||
__declspec(dllexport) IO_STACK_LOCATION* IoGetNextIrpStackLocation(IRP* irp) {
|
||||
if (!irp || irp->CurrentLocation + 1u >= irp->StackCount) return (IO_STACK_LOCATION*)0;
|
||||
return &irp->Stack[irp->CurrentLocation + 1u];
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoSkipCurrentIrpStackLocation(IRP* irp) {
|
||||
if (!irp) return;
|
||||
if (irp->CurrentLocation + 1u < irp->StackCount)
|
||||
irp->CurrentLocation++;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoCopyCurrentIrpStackLocationToNext(IRP* irp) {
|
||||
IO_STACK_LOCATION* cur = IoGetCurrentIrpStackLocation(irp);
|
||||
IO_STACK_LOCATION* next = IoGetNextIrpStackLocation(irp);
|
||||
if (!cur || !next) return;
|
||||
memcpy(next, cur, sizeof(*next));
|
||||
irp->CurrentLocation++;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoSetCompletionRoutine(IRP* irp, IRP_COMPLETION_FN fn, void* ctx) {
|
||||
if (!irp) return;
|
||||
irp->CompletionFn = fn;
|
||||
irp->CompletionCtx = ctx;
|
||||
}
|
||||
|
||||
__declspec(dllexport) IRP_CANCEL_ROUTINE IoSetCancelRoutine(IRP* irp, IRP_CANCEL_ROUTINE routine) {
|
||||
if (!irp) return (IRP_CANCEL_ROUTINE)0;
|
||||
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
|
||||
IRP_CANCEL_ROUTINE old = irp->CancelRoutine;
|
||||
irp->CancelRoutine = routine;
|
||||
KeReleaseSpinlock(&irp->CancelLock, flags);
|
||||
return old;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoCancelIrp(IRP* irp) {
|
||||
if (!irp) return;
|
||||
|
||||
uintptr_t flags = KeAcquireSpinlock(&irp->CancelLock);
|
||||
irp->Cancel = true;
|
||||
irp->Flags |= IRP_FLAG_CANCELLED;
|
||||
IRP_CANCEL_ROUTINE routine = irp->CancelRoutine;
|
||||
irp->CancelRoutine = (IRP_CANCEL_ROUTINE)0;
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
|
||||
KeReleaseSpinlock(&irp->CancelLock, flags);
|
||||
|
||||
if (routine)
|
||||
routine(dev, irp);
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoBuildReadWriteIrp(IRP* irp,
|
||||
IO_MAJOR_FUNCTION major,
|
||||
uint64_t byteOffset,
|
||||
uint32_t length,
|
||||
void* buffer) {
|
||||
if (!irp) return;
|
||||
IoInitializeIrp(irp, 1);
|
||||
irp->UserBuffer = buffer;
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios) return;
|
||||
ios->MajorFunction = major;
|
||||
ios->MinorFunction = 0;
|
||||
ios->Parameters.ReadWrite.ByteOffset = byteOffset;
|
||||
ios->Parameters.ReadWrite.Length = length;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoBuildPnpIrp(IRP* irp,
|
||||
IO_PNP_MINOR_FUNCTION minor) {
|
||||
if (!irp) return;
|
||||
uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
|
||||
IoInitializeIrp(irp, stackCount);
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios) return;
|
||||
ios->MajorFunction = IRP_MJ_PNP;
|
||||
ios->MinorFunction = (uint8_t)minor;
|
||||
}
|
||||
|
||||
__declspec(dllexport) void IoBuildDeviceIoControlIrp(IRP* irp,
|
||||
IO_MAJOR_FUNCTION major,
|
||||
uint32_t ioControlCode,
|
||||
void* inputBuffer,
|
||||
uint32_t inputLength,
|
||||
void* outputBuffer,
|
||||
uint32_t outputLength) {
|
||||
if (!irp) return;
|
||||
uint8_t stackCount = irp->StackCount ? irp->StackCount : 1;
|
||||
IoInitializeIrp(irp, stackCount);
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
if (!ios) return;
|
||||
ios->MajorFunction = major;
|
||||
ios->MinorFunction = 0;
|
||||
ios->Parameters.DeviceIoControl.IoControlCode = ioControlCode;
|
||||
ios->Parameters.DeviceIoControl.InputBuffer = inputBuffer;
|
||||
ios->Parameters.DeviceIoControl.InputLength = inputLength;
|
||||
ios->Parameters.DeviceIoControl.OutputBuffer = outputBuffer;
|
||||
ios->Parameters.DeviceIoControl.OutputLength = outputLength;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// Completion/submission
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
__declspec(dllexport) KSTATUS IoCompleteRequest(IRP* irp, KSTATUS status, size_t information) {
|
||||
if (!irp) return STATUS_INVALID_PARAMETER;
|
||||
if (irp->Flags & IRP_FLAG_COMPLETED)
|
||||
return STATUS_ALREADY_EXISTS;
|
||||
|
||||
irp->Flags |= IRP_FLAG_COMPLETED;
|
||||
irp->Status = status;
|
||||
irp->Information = information;
|
||||
|
||||
// Queue the APC before waking event waiters so a thread that waits on the
|
||||
// completion event will see and deliver the APC on its wait-return path.
|
||||
IO_STACK_LOCATION* ios = IoGetCurrentIrpStackLocation(irp);
|
||||
DEVICE_OBJECT* dev = ios ? ios->DeviceObject : (DEVICE_OBJECT*)0;
|
||||
if (dev) {
|
||||
_InterlockedDecrement(&dev->OutstandingIrps);
|
||||
IoDereferenceDeviceObject(dev);
|
||||
ios->DeviceObject = (DEVICE_OBJECT*)0;
|
||||
}
|
||||
|
||||
if (irp->CompletionApc)
|
||||
KeInsertQueueApc((KAPC *)irp->CompletionApc, irp, (void *)(uintptr_t)status);
|
||||
KeInsertQueueApc((KAPC*)irp->CompletionApc, irp, (void*)(uintptr_t)status);
|
||||
if (irp->CompletionEvent)
|
||||
KeSignalEvent((EVENT *)irp->CompletionEvent);
|
||||
KeSignalEvent((EVENT*)irp->CompletionEvent);
|
||||
if (irp->CompletionFn)
|
||||
irp->CompletionFn(irp, irp->CompletionCtx);
|
||||
|
||||
return STATUS_SUCCESS;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// IoSubmitIrp
|
||||
// -------------------------------------------------------------------------
|
||||
// Dispatches through the registered DEVICE_OBJECT.
|
||||
// Accepted IRPs are always completed by the target driver via IoCompleteRequest().
|
||||
|
||||
__declspec(dllexport) KSTATUS IoSubmitIrp(IRP *irp) {
|
||||
if (!irp) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
__declspec(dllexport) KSTATUS IoSubmitIrp(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp) return STATUS_INVALID_PARAMETER;
|
||||
irp->Status = STATUS_FAILURE;
|
||||
irp->Information = 0;
|
||||
|
||||
DEVICE_OBJECT *device = IoGetDiskDevice(irp->DriveIndex);
|
||||
if (!device) return STATUS_NOT_FOUND;
|
||||
|
||||
KSTATUS st = IoCallDriver(device, irp);
|
||||
|
||||
// Definitive driver contract:
|
||||
// STATUS_PENDING means the IRP was accepted and will be completed via
|
||||
// IoCompleteRequest(), possibly inline before IoCallDriver returns.
|
||||
// Any other status means the IRP was rejected and no completion occurs.
|
||||
return st;
|
||||
irp->Flags &= ~IRP_FLAG_COMPLETED;
|
||||
return IoCallDriver(device, irp);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// IoSubmitIrpSync
|
||||
// -------------------------------------------------------------------------
|
||||
// Creates a stack-allocated auto-reset event, attaches it to the IRP,
|
||||
// submits, and blocks. Callers that already provided a CompletionEvent
|
||||
// should use IoSubmitIrp() directly and wait on their own event instead.
|
||||
__declspec(dllexport) KSTATUS IoSubmitIrpSync(DEVICE_OBJECT* device, IRP* irp) {
|
||||
if (!device || !irp) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
__declspec(dllexport) KSTATUS IoSubmitIrpSync(IRP *irp) {
|
||||
if (!irp) return STATUS_INVALID_PARAMETER;
|
||||
|
||||
EVENT *ev = (EVENT *)0;
|
||||
EVENT* ev = (EVENT*)0;
|
||||
KSTATUS st = KeEventCreate(EventAutoReset, 0, &ev);
|
||||
if (!KSUCCESS(st)) return st;
|
||||
|
||||
// Attach our event; preserve any APC/callback the caller may have set.
|
||||
irp->CompletionEvent = ev;
|
||||
|
||||
st = IoSubmitIrp(irp);
|
||||
st = IoSubmitIrp(device, irp);
|
||||
if (st == STATUS_PENDING) {
|
||||
KeWaitForEvent(ev, 0);
|
||||
st = irp->Status;
|
||||
}
|
||||
|
||||
// Release our reference to the temporary event object.
|
||||
ObDereferenceObject(&ev->Header);
|
||||
|
||||
return st;
|
||||
}
|
||||
Reference in new issue
Block a user