add a realtime scheduler class and priority-inheritance mutexes with timeouts and abandonment, add an HDA audio driver and a SYS_AUDIO_* syscall family, flesh out kernel32 with Win32 threads, events, mutexes, critical sections, and condition variables, add SYS_RESUME_THREAD for suspended thread creation, add KeInitializeMemoryTypes to reprogram MTRRs and install a canonical PAT for WB/WC/UC control, rework the display pipeline with a pitched present path and console output flush, bring back disk.vhd with mtools since vvfat fucking sucks, harden AHCI with serialized cache flush, USB mass storage with SCSI sense-data decoding, and xHCI with cleaner DMA teardown, fix the last stray for(;;) in HalStopProcessor to while(true), pile on smoke tests for audio, win32 sync primitives, and thread suspend, implement AHCI timeout, kill readahead since it was desperetaly trying to murder me, fix doom sound x2.

code/drivers/net/rtl816x/rtl816x.c

@@ -83,48 +83,117 @@ static void RtlReadMac(RTL816X_DEVICE* r) {
     r->NetDevice.HardwareAddressLength = 6;
 }
 
-// Implements the rtl Allocate Dma operation.
+// Stops bus mastering before DMA memory can be released.
+static void RtlStopDma(RTL816X_DEVICE* r) {
+    if (!r || !r->Started)
+        return;
+    r->Started = false;
+    if (r->SurpriseRemoved)
+        return;
+    RtlWrite16(r, RTL_REG_INTR_MASK, 0);
+    RtlWrite8(r, RTL_REG_CHIP_CMD, 0);
+    KSTATUS status = RtlReset(r);
+    if (!KSUCCESS(status))
+        KePanic(KE_PANIC_CODE_DRIVER,
+                "RTL816X: failed to stop DMA before release status=0x%x\n",
+                status);
+}
+
+// Releases the common buffers and adapter owned by one RTL816x device.
+static void RtlReleaseDma(RTL816X_DEVICE* r) {
+    if (!r || !gRtl816xDdk || !gRtl816xDdk->Dma)
+        return;
+    gRtl816xDdk->Dma->FreeCommonBuffer(r->TxBuffersDma);
+    gRtl816xDdk->Dma->FreeCommonBuffer(r->RxBuffersDma);
+    gRtl816xDdk->Dma->FreeCommonBuffer(r->TxRingDma);
+    gRtl816xDdk->Dma->FreeCommonBuffer(r->RxRingDma);
+    r->TxBuffersDma = (IO_DMA_BUFFER*)0;
+    r->RxBuffersDma = (IO_DMA_BUFFER*)0;
+    r->TxRingDma = (IO_DMA_BUFFER*)0;
+    r->RxRingDma = (IO_DMA_BUFFER*)0;
+    r->TxBuffers = (uint8_t*)0;
+    r->RxBuffers = (uint8_t*)0;
+    r->TxRing = (RTL_DESC*)0;
+    r->RxRing = (RTL_DESC*)0;
+    r->TxBuffersDeviceAddress = 0;
+    r->RxBuffersDeviceAddress = 0;
+    r->TxRingDeviceAddress = 0;
+    r->RxRingDeviceAddress = 0;
+    gRtl816xDdk->Dma->PutAdapter(r->DmaAdapter);
+    r->DmaAdapter = (IO_DMA_ADAPTER*)0;
+}
+
+// Allocates descriptor rings and packet buffers through the device DMA adapter.
 static KSTATUS RtlAllocateDma(RTL816X_DEVICE* r) {
-    r->RxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
-    r->TxRingPhys = MmPfnAllocateContiguous(RTL_RING_PAGES, MmPfnUseDma);
-    r->RxBuffersPhys = MmPfnAllocateContiguous(RTL_RXBUF_PAGES, MmPfnUseDma);
-    r->TxBuffersPhys = MmPfnAllocateContiguous(RTL_TXBUF_PAGES, MmPfnUseDma);
+    IO_DMA_REQUIREMENTS requirements;
+    memset(&requirements, 0, sizeof(requirements));
+    requirements.MaximumDeviceAddress = (uint64_t)-1;
+    requirements.Alignment = 1;
+    requirements.MaximumSegmentLength = RTL_BUF_SIZE;
+    requirements.MaximumSegments = RTL_TX_COUNT;
 
-    if (!r->RxRingPhys || !r->TxRingPhys || !r->RxBuffersPhys || !r->TxBuffersPhys)
-        return NTLSTATUS_NO_MEMORY;
+    KSTATUS status = gRtl816xDdk->Dma->GetAdapter(r->Fdo,
+                                                  &requirements,
+                                                  &r->DmaAdapter);
+    if (!KSUCCESS(status))
+        return status;
 
-    r->RxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->RxRingPhys);
-    r->TxRing = (RTL_DESC*)gRtl816xDdk->Memory->PhysToVirt(r->TxRingPhys);
-    r->RxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->RxBuffersPhys);
-    r->TxBuffers = (uint8_t*)gRtl816xDdk->Memory->PhysToVirt(r->TxBuffersPhys);
+    status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
+                                                    PAGE_SIZE * RTL_RING_PAGES,
+                                                    PAGE_SIZE,
+                                                    &r->RxRingDma);
+    if (!KSUCCESS(status)) goto fail;
+    status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
+                                                    PAGE_SIZE * RTL_RING_PAGES,
+                                                    PAGE_SIZE,
+                                                    &r->TxRingDma);
+    if (!KSUCCESS(status)) goto fail;
+    status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
+                                                    PAGE_SIZE * RTL_RXBUF_PAGES,
+                                                    PAGE_SIZE,
+                                                    &r->RxBuffersDma);
+    if (!KSUCCESS(status)) goto fail;
+    status = gRtl816xDdk->Dma->AllocateCommonBuffer(r->DmaAdapter,
+                                                    PAGE_SIZE * RTL_TXBUF_PAGES,
+                                                    PAGE_SIZE,
+                                                    &r->TxBuffersDma);
+    if (!KSUCCESS(status)) goto fail;
 
-    memset(r->RxRing, 0, PAGE_SIZE * RTL_RING_PAGES);
-    memset(r->TxRing, 0, PAGE_SIZE * RTL_RING_PAGES);
-    memset(r->RxBuffers, 0, PAGE_SIZE * RTL_RXBUF_PAGES);
-    memset(r->TxBuffers, 0, PAGE_SIZE * RTL_TXBUF_PAGES);
+    r->RxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxRingDma);
+    r->TxRing = (RTL_DESC*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxRingDma);
+    r->RxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->RxBuffersDma);
+    r->TxBuffers = (uint8_t*)gRtl816xDdk->Dma->GetCommonBufferAddress(r->TxBuffersDma);
+    r->RxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxRingDma);
+    r->TxRingDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxRingDma);
+    r->RxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->RxBuffersDma);
+    r->TxBuffersDeviceAddress = gRtl816xDdk->Dma->GetCommonBufferDeviceAddress(r->TxBuffersDma);
 
     DbgPrint("RTL816X: rxRing=0x%016llX txRing=0x%016llX rxBuf=0x%016llX txBuf=0x%016llX\n",
-             (unsigned long long)r->RxRingPhys,
-             (unsigned long long)r->TxRingPhys,
-             (unsigned long long)r->RxBuffersPhys,
-             (unsigned long long)r->TxBuffersPhys);
+             (unsigned long long)r->RxRingDeviceAddress,
+             (unsigned long long)r->TxRingDeviceAddress,
+             (unsigned long long)r->RxBuffersDeviceAddress,
+             (unsigned long long)r->TxBuffersDeviceAddress);
     return NTLSTATUS_SUCCESS;
+
+fail:
+    RtlReleaseDma(r);
+    return status;
 }
 
 // Implements the rtl Init Descriptors operation.
 static void RtlInitDescriptors(RTL816X_DEVICE* r) {
     for (uint32_t i = 0; i < RTL_RX_COUNT; i++) {
-        PHYSICAL_ADDRESS p = r->RxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE;
-        r->RxRing[i].addrLo = (uint32_t)p;
-        r->RxRing[i].addrHi = (uint32_t)(p >> 32);
+        uint64_t address = r->RxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE;
+        r->RxRing[i].addrLo = (uint32_t)address;
+        r->RxRing[i].addrHi = (uint32_t)(address >> 32);
         r->RxRing[i].opts2 = 0;
         r->RxRing[i].opts1 = RTL_DESC_OWN | RTL_BUF_SIZE | ((i == RTL_RX_COUNT - 1) ? RTL_DESC_EOR : 0);
     }
 
     for (uint32_t i = 0; i < RTL_TX_COUNT; i++) {
-        PHYSICAL_ADDRESS p = r->TxBuffersPhys + (PHYSICAL_ADDRESS)i * RTL_BUF_SIZE;
-        r->TxRing[i].addrLo = (uint32_t)p;
-        r->TxRing[i].addrHi = (uint32_t)(p >> 32);
+        uint64_t address = r->TxBuffersDeviceAddress + (uint64_t)i * RTL_BUF_SIZE;
+        r->TxRing[i].addrLo = (uint32_t)address;
+        r->TxRing[i].addrHi = (uint32_t)(address >> 32);
         r->TxRing[i].opts2 = 0;
         r->TxRing[i].opts1 = (i == RTL_TX_COUNT - 1) ? RTL_DESC_EOR : 0;
     }
@@ -141,12 +210,12 @@ static void RtlProgramHardware(RTL816X_DEVICE* r) {
     RtlWrite16(r, RTL_REG_INTR_MASK, 0);
     RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
 
-    RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingPhys);
-    RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingPhys >> 32));
-    RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingPhys);
-    RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingPhys >> 32));
-    RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingPhys);
-    RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingPhys >> 32));
+    RtlWrite32(r, RTL_REG_TNPDS_LO, (uint32_t)r->TxRingDeviceAddress);
+    RtlWrite32(r, RTL_REG_TNPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32));
+    RtlWrite32(r, RTL_REG_THPDS_LO, (uint32_t)r->TxRingDeviceAddress);
+    RtlWrite32(r, RTL_REG_THPDS_HI, (uint32_t)(r->TxRingDeviceAddress >> 32));
+    RtlWrite32(r, RTL_REG_RDSAR_LO, (uint32_t)r->RxRingDeviceAddress);
+    RtlWrite32(r, RTL_REG_RDSAR_HI, (uint32_t)(r->RxRingDeviceAddress >> 32));
 
     RtlWrite16(r, RTL_REG_CPLUS_CMD, RtlRead16(r, RTL_REG_CPLUS_CMD) & ~(uint16_t)0x00E0u);
     RtlWrite16(r, RTL_REG_RX_MAXSIZE, RTL_BUF_SIZE);
@@ -366,7 +435,7 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
     if (best.Type == PciBarMemory32 || best.Type == PciBarMemory64) {
         KSTATUS st = PciMapBar(r->Pdo,
                                best.Index,
-                               PAGE_PRESENT | PAGE_WRITE | PAGE_NOCACHE,
+                               PAGE_PRESENT | PAGE_WRITE | PAGE_CACHE_UC,
                                (void**)&r->Mmio,
                                &r->RegisterBytes);
         if (!KSUCCESS(st))
@@ -400,6 +469,41 @@ static KSTATUS RtlSelectRegisterBar(RTL816X_DEVICE* r) {
     return NTLSTATUS_NOT_FOUND;
 }
 
+// Configures a dedicated MSI vector and falls back to a routed INTx line.
+static KSTATUS RtlConfigureInterrupt(RTL816X_DEVICE* r) {
+    if (!r || !r->Pdo)
+        return NTLSTATUS_INVALID_PARAMETER;
+
+    r->Irq = 0xFFu;
+    r->UsingMsi = false;
+
+    KSTATUS msiStatus = PciEnableMsiAuto(r->Pdo, &r->Irq);
+    if (KSUCCESS(msiStatus)) {
+        r->UsingMsi = true;
+        DbgPrint("RTL816X: using MSI IRQ slot %u\n", r->Irq);
+        return NTLSTATUS_SUCCESS;
+    }
+
+    DbgPrint("RTL816X: MSI setup failed status=0x%x; attempting explicit INTx fallback\n",
+             msiStatus);
+
+    KSTATUS intxStatus = PciGetInterrupt(r->Pdo, &r->Irq);
+    if (!KSUCCESS(intxStatus))
+        return intxStatus;
+    if (r->Irq == 0xFFu || r->Irq >= 24u)
+        return NTLSTATUS_NOT_SUPPORTED;
+
+    intxStatus = PciEnableDevice(r->Pdo,
+                                PCI_ENABLE_BUSMASTER |
+                                PCI_ENABLE_INTX |
+                                (r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
+    if (!KSUCCESS(intxStatus))
+        return intxStatus;
+
+    DbgPrint("RTL816X: using legacy INTx IRQ %u\n", r->Irq);
+    return NTLSTATUS_SUCCESS;
+}
+
 // Implements the rtl Start Device operation.
 static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
     if (!r || r->Magic != RTL816X_DEVICE_MAGIC || !r->Pdo || !r->Fdo)
@@ -428,18 +532,15 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
 
     st = PciEnableDevice(r->Pdo,
                          PCI_ENABLE_BUSMASTER |
-                         PCI_ENABLE_INTX |
+                         PCI_DISABLE_INTX |
                          (r->UseMmio ? PCI_ENABLE_MEMORY : PCI_ENABLE_IO));
     if (!KSUCCESS(st)) {
         DbgPrint("RTL816X: PciEnableDevice failed: 0x%x\n", st);
         return st;
     }
 
-    st = PciGetInterrupt(r->Pdo, &r->Irq);
-    if (!KSUCCESS(st) || r->Irq == 0xFFu) {
-        DbgPrint("RTL816X: no legacy PCI IRQ line wired\n");
-        return KSUCCESS(st) ? NTLSTATUS_NOT_SUPPORTED : st;
-    }
+    RtlWrite16(r, RTL_REG_INTR_MASK, 0);
+    RtlWrite16(r, RTL_REG_INTR_STAT, 0xFFFFu);
 
     st = RtlReset(r);
     if (!KSUCCESS(st)) {
@@ -447,12 +548,18 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
         return st;
     }
 
+    st = RtlConfigureInterrupt(r);
+    if (!KSUCCESS(st)) {
+        DbgPrint("RTL816X: no usable MSI or INTx interrupt status=0x%x\n", st);
+        return st;
+    }
+
     RtlReadMac(r);
-    DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u\n",
+    DbgPrint("RTL816X: MAC %02x:%02x:%02x:%02x:%02x:%02x irq=%u mode=%s\n",
              r->NetDevice.HardwareAddress[0], r->NetDevice.HardwareAddress[1],
              r->NetDevice.HardwareAddress[2], r->NetDevice.HardwareAddress[3],
              r->NetDevice.HardwareAddress[4], r->NetDevice.HardwareAddress[5],
-             r->Irq);
+             r->Irq, r->UsingMsi ? "MSI" : "INTx");
 
     st = RtlAllocateDma(r);
     if (!KSUCCESS(st))
@@ -466,8 +573,10 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
     r->NetDevice.SendFrame = RtlSendFrame;
 
     st = RtlRegisterDevice(r);
-    if (!KSUCCESS(st))
+    if (!KSUCCESS(st)) {
+        RtlReleaseDma(r);
         return st;
+    }
 
     KeIrqRegister(r->Irq, RtlIsr);
     r->Started = true;
@@ -476,13 +585,14 @@ static KSTATUS RtlStartDevice(RTL816X_DEVICE* r) {
 
     st = NetRegisterDevice(&r->NetDevice);
     if (!KSUCCESS(st)) {
-        RtlWrite16(r, RTL_REG_INTR_MASK, 0);
-        r->Started = false;
+        RtlStopDma(r);
         RtlUnregisterDevice(r);
+        RtlReleaseDma(r);
         return st;
     }
 
-    DbgPrint("RTL816X: initialized; RX/TX enabled on IRQ %u\n", r->Irq);
+    DbgPrint("RTL816X: initialized; RX/TX enabled on %s IRQ %u\n",
+             r->UsingMsi ? "MSI" : "INTx", r->Irq);
     return NTLSTATUS_SUCCESS;
 }
 
@@ -509,17 +619,16 @@ KSTATUS Rtl816xDispatchPnp(DEVICE_OBJECT* device, IRP* irp) {
 
         case IRP_MN_SURPRISE_REMOVAL:
             r->SurpriseRemoved = true;
+            r->Started = false;
             device->PnpState = DevicePnpSurpriseRemoved;
-            if (r->Started)
-                RtlWrite16(r, RTL_REG_INTR_MASK, 0);
             return NTLSTATUS_SUCCESS;
 
         case IRP_MN_REMOVE_DEVICE:
             r->Removed = true;
             device->PnpState = DevicePnpDeleted;
-            if (r->Started)
-                RtlWrite16(r, RTL_REG_INTR_MASK, 0);
+            RtlStopDma(r);
             RtlUnregisterDevice(r);
+            RtlReleaseDma(r);
             IoDetachDevice(device);
             IoDeleteDevice(device);
             return NTLSTATUS_SUCCESS;
@@ -561,7 +670,7 @@ KSTATUS Rtl816xAddDevice(DRIVER_OBJECT* driver, DEVICE_OBJECT* physicalDeviceObj
 // Implements the driver Entry operation.
 KSTATUS DriverEntry(const NTL_DDK* ddk, NTL_DRIVER_INIT* init) {
     (void)init;
-    if (!ddk || !ddk->Memory || !ddk->Memory->PhysToVirt || !ddk->Timer)
+    if (!ddk || !ddk->Dma || !ddk->Timer)
         return NTLSTATUS_INVALID_PARAMETER;
 
     DbgPrint("RTL816X: driver loading...\n");