// Intel High Definition Audio controller transport, command rings, output DMA, and recovery. #include #include #include "hda.h" #include #include // Orders CPU stores before a controller ownership transition. static __forceinline void HdaMemoryBarrier(void) { _ReadWriteBarrier(); _mm_mfence(); _ReadWriteBarrier(); } // Returns the controller's monotonic microsecond counter. static uint64_t HdaQueryTicks(void) { if (!gHdaDdk || !gHdaDdk->Timer || !gHdaDdk->Timer->QueryTicks) return 0; return gHdaDdk->Timer->QueryTicks(); } // Delays for a bounded controller settling interval. static void HdaStall(uint32_t microseconds) { if (gHdaDdk && gHdaDdk->Timer && gHdaDdk->Timer->Stall) gHdaDdk->Timer->Stall(microseconds); } // Returns one byte from the controller register aperture. uint8_t HdaRead8(const HDA_CONTROLLER* controller, uint32_t offset) { return *(volatile const uint8_t*)(controller->Registers + offset); } // Returns one word from the controller register aperture. uint16_t HdaRead16(const HDA_CONTROLLER* controller, uint32_t offset) { return *(volatile const uint16_t*)(controller->Registers + offset); } // Returns one dword from the controller register aperture. uint32_t HdaRead32(const HDA_CONTROLLER* controller, uint32_t offset) { return *(volatile const uint32_t*)(controller->Registers + offset); } // Writes one byte to the controller register aperture. void HdaWrite8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t value) { *(volatile uint8_t*)(controller->Registers + offset) = value; } // Writes one word to the controller register aperture. void HdaWrite16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t value) { *(volatile uint16_t*)(controller->Registers + offset) = value; } // Writes one dword to the controller register aperture. void HdaWrite32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t value) { *(volatile uint32_t*)(controller->Registers + offset) = value; } // Returns the register offset of the controller's one output stream descriptor. uint32_t HdaOutputStreamBase(const HDA_CONTROLLER* controller) { return HDA_REG_STREAM_BASE + (uint32_t)controller->Output.DescriptorIndex * HDA_REG_STREAM_STRIDE; } // Reads the 24-bit stream control register without crossing into status. static uint32_t HdaReadStreamControl(const HDA_CONTROLLER* controller) { uint32_t base = HdaOutputStreamBase(controller); return (uint32_t)HdaRead16(controller, base + HDA_STREAM_REG_CTL) | ((uint32_t)HdaRead8(controller, base + HDA_STREAM_REG_CTL + 2u) << 16); } // Writes the 24-bit stream control register without modifying status. static void HdaWriteStreamControl(HDA_CONTROLLER* controller, uint32_t value) { uint32_t base = HdaOutputStreamBase(controller); HdaWrite16(controller, base + HDA_STREAM_REG_CTL, (uint16_t)value); HdaWrite8(controller, base + HDA_STREAM_REG_CTL + 2u, (uint8_t)(value >> 16)); } // Waits until selected bits in one dword register match the requested state. static KSTATUS HdaWait32(HDA_CONTROLLER* controller, uint32_t offset, uint32_t mask, uint32_t expected, uint32_t timeoutMicroseconds) { for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; if ((HdaRead32(controller, offset) & mask) == expected) return NTLSTATUS_SUCCESS; HdaStall(1); } return NTLSTATUS_TIMEOUT; } // Waits until selected bits in one byte register match the requested state. static KSTATUS HdaWait8(HDA_CONTROLLER* controller, uint32_t offset, uint8_t mask, uint8_t expected, uint32_t timeoutMicroseconds) { for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; if ((HdaRead8(controller, offset) & mask) == expected) return NTLSTATUS_SUCCESS; HdaStall(1); } return NTLSTATUS_TIMEOUT; } // Waits until selected bits in one word register match the requested state. static KSTATUS HdaWait16(HDA_CONTROLLER* controller, uint32_t offset, uint16_t mask, uint16_t expected, uint32_t timeoutMicroseconds) { for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; if ((HdaRead16(controller, offset) & mask) == expected) return NTLSTATUS_SUCCESS; HdaStall(1); } return NTLSTATUS_TIMEOUT; } // Waits until selected bits in the output stream control register match. static KSTATUS HdaWaitStreamControl(HDA_CONTROLLER* controller, uint32_t mask, uint32_t expected, uint32_t timeoutMicroseconds) { for (uint32_t elapsed = 0; elapsed < timeoutMicroseconds; elapsed++) { if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; if ((HdaReadStreamControl(controller) & mask) == expected) return NTLSTATUS_SUCCESS; HdaStall(1); } return NTLSTATUS_TIMEOUT; } // Selects the largest command-ring size advertised by the controller. static KSTATUS HdaSelectRingSize(uint8_t capability, uint16_t* countOut) { if (!countOut) return NTLSTATUS_INVALID_PARAMETER; if (capability & HDA_RING_SIZE_CAP_256) { *countOut = 256u; return NTLSTATUS_SUCCESS; } if (capability & HDA_RING_SIZE_CAP_16) { *countOut = 16u; return NTLSTATUS_SUCCESS; } if (capability & HDA_RING_SIZE_CAP_2) { *countOut = 2u; return NTLSTATUS_SUCCESS; } return NTLSTATUS_NOT_SUPPORTED; } // Returns the hardware selector encoding for a supported ring entry count. static uint8_t HdaRingSizeSelector(uint16_t count) { if (count == 256u) return HDA_RING_SIZE_SELECT_256; if (count == 16u) return HDA_RING_SIZE_SELECT_16; return HDA_RING_SIZE_SELECT_2; } // Releases all DMA and synchronization resources owned by one controller. void HdaReleaseDma(HDA_CONTROLLER* controller) { if (!controller || !gHdaDdk || !gHdaDdk->Dma) return; if (controller->StreamMutex) { ObDereferenceObject(&controller->StreamMutex->Header); controller->StreamMutex = (MUTEX*)0; } if (controller->CommandMutex) { ObDereferenceObject(&controller->CommandMutex->Header); controller->CommandMutex = (MUTEX*)0; } if (controller->RecoverySemaphore) { ObDereferenceObject(&controller->RecoverySemaphore->Header); controller->RecoverySemaphore = (SEMAPHORE*)0; } if (controller->RecoveryFinished) { ObDereferenceObject(&controller->RecoveryFinished->Header); controller->RecoveryFinished = (EVENT*)0; } gHdaDdk->Dma->FreeCommonBuffer(controller->Output.BdlDma); for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) gHdaDdk->Dma->FreeCommonBuffer(controller->Output.Rings[ringIndex].Dma); gHdaDdk->Dma->FreeCommonBuffer(controller->PositionDma); gHdaDdk->Dma->FreeCommonBuffer(controller->RirbDma); gHdaDdk->Dma->FreeCommonBuffer(controller->CorbDma); controller->Output.BdlDma = (IO_DMA_BUFFER*)0; for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { controller->Output.Rings[ringIndex].Dma = (IO_DMA_BUFFER*)0; controller->Output.Rings[ringIndex].Buffer = (uint8_t*)0; controller->Output.Rings[ringIndex].DeviceAddress = 0; } controller->PositionDma = (IO_DMA_BUFFER*)0; controller->RirbDma = (IO_DMA_BUFFER*)0; controller->CorbDma = (IO_DMA_BUFFER*)0; controller->Output.Bdl = (HDA_BDL_ENTRY*)0; controller->PositionBuffer = (volatile uint64_t*)0; controller->Rirb = (HDA_RIRB_ENTRY*)0; controller->Corb = (uint32_t*)0; if (controller->DmaAdapter) { gHdaDdk->Dma->PutAdapter(controller->DmaAdapter); controller->DmaAdapter = (IO_DMA_ADAPTER*)0; } } // Allocates the transport and output DMA resources for one controller. KSTATUS HdaAllocateDma(HDA_CONTROLLER* controller) { if (!controller || !controller->Fdo || !gHdaDdk || !gHdaDdk->Dma) return NTLSTATUS_INVALID_PARAMETER; IO_DMA_REQUIREMENTS requirements; memset(&requirements, 0, sizeof(requirements)); requirements.MaximumDeviceAddress = (controller->Gcap & HDA_GCAP_64OK) ? UINT64_MAX : 0xFFFFFFFFull; requirements.Alignment = 128u; requirements.MaximumSegments = 1u; KSTATUS status = gHdaDdk->Dma->GetAdapter(controller->Fdo, &requirements, &controller->DmaAdapter); if (!KSUCCESS(status)) return status; status = HdaSelectRingSize(HdaRead8(controller, HDA_REG_CORBSIZE), &controller->CorbEntryCount); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } status = HdaSelectRingSize(HdaRead8(controller, HDA_REG_RIRBSIZE), &controller->RirbEntryCount); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, (size_t)controller->CorbEntryCount * sizeof(uint32_t), 128u, &controller->CorbDma); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, (size_t)controller->RirbEntryCount * sizeof(HDA_RIRB_ENTRY), 128u, &controller->RirbDma); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } // The position buffer is optional because LPIB is the supported fallback. status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, (size_t)controller->TotalStreamCount * sizeof(uint64_t), 128u, &controller->PositionDma); if (!KSUCCESS(status)) controller->PositionDma = (IO_DMA_BUFFER*)0; for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { status = gHdaDdk->Dma->AllocateCommonBuffer( controller->DmaAdapter, HDA_OUTPUT_BUFFER_BYTES, 128u, &controller->Output.Rings[ringIndex].Dma); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } } status = gHdaDdk->Dma->AllocateCommonBuffer(controller->DmaAdapter, HDA_OUTPUT_PERIOD_COUNT * sizeof(HDA_BDL_ENTRY), 128u, &controller->Output.BdlDma); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } controller->Corb = (uint32_t*)gHdaDdk->Dma->GetCommonBufferAddress(controller->CorbDma); controller->Rirb = (HDA_RIRB_ENTRY*)gHdaDdk->Dma->GetCommonBufferAddress(controller->RirbDma); controller->CorbDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->CorbDma); controller->RirbDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->RirbDma); if (controller->PositionDma) { controller->PositionBuffer = (volatile uint64_t*) gHdaDdk->Dma->GetCommonBufferAddress(controller->PositionDma); controller->PositionDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->PositionDma); } for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) { controller->Output.Rings[ringIndex].Buffer = (uint8_t*) gHdaDdk->Dma->GetCommonBufferAddress(controller->Output.Rings[ringIndex].Dma); controller->Output.Rings[ringIndex].DeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress( controller->Output.Rings[ringIndex].Dma); } controller->Output.Bdl = (HDA_BDL_ENTRY*) gHdaDdk->Dma->GetCommonBufferAddress(controller->Output.BdlDma); controller->Output.BdlDeviceAddress = gHdaDdk->Dma->GetCommonBufferDeviceAddress(controller->Output.BdlDma); memset(controller->Corb, 0, (size_t)controller->CorbEntryCount * sizeof(uint32_t)); memset(controller->Rirb, 0, (size_t)controller->RirbEntryCount * sizeof(HDA_RIRB_ENTRY)); if (controller->PositionBuffer) memset((void*)controller->PositionBuffer, 0, (size_t)controller->TotalStreamCount * sizeof(uint64_t)); for (uint32_t ringIndex = 0; ringIndex < HDA_MAX_LOGICAL_ENDPOINTS; ringIndex++) memset(controller->Output.Rings[ringIndex].Buffer, 0, HDA_OUTPUT_BUFFER_BYTES); memset(controller->Output.Bdl, 0, HDA_OUTPUT_PERIOD_COUNT * sizeof(HDA_BDL_ENTRY)); status = KeMutexCreate(&controller->CommandMutex); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } status = KeMutexCreate(&controller->StreamMutex); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } status = KeSemaphoreCreate(0, 1, &controller->RecoverySemaphore); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } status = KeEventCreate(EventManualReset, 0, &controller->RecoveryFinished); if (!KSUCCESS(status)) { HdaReleaseDma(controller); return status; } return NTLSTATUS_SUCCESS; } // Stops the output stream and waits for the run bit to clear. static KSTATUS HdaStopOutputHardware(HDA_CONTROLLER* controller) { uint32_t control = HdaReadStreamControl(controller); control &= ~(HDA_STREAM_CTL_RUN | HDA_STREAM_CTL_INTERRUPT_MASK); HdaWriteStreamControl(controller, control); return HdaWaitStreamControl(controller, HDA_STREAM_CTL_RUN, 0, HDA_STREAM_RESET_TIMEOUT_US); } // Resets the one output stream descriptor to its architected idle state. static KSTATUS HdaResetOutputDescriptor(HDA_CONTROLLER* controller) { KSTATUS status = HdaStopOutputHardware(controller); if (!KSUCCESS(status)) return status; uint32_t control = HdaReadStreamControl(controller) | HDA_STREAM_CTL_RESET; HdaWriteStreamControl(controller, control); status = HdaWaitStreamControl(controller, HDA_STREAM_CTL_RESET, HDA_STREAM_CTL_RESET, HDA_STREAM_RESET_TIMEOUT_US); if (!KSUCCESS(status)) return status; control &= ~HDA_STREAM_CTL_RESET; HdaWriteStreamControl(controller, control); status = HdaWaitStreamControl(controller, HDA_STREAM_CTL_RESET, 0, HDA_STREAM_RESET_TIMEOUT_US); if (!KSUCCESS(status)) return status; HdaWrite8(controller, HdaOutputStreamBase(controller) + HDA_STREAM_REG_STS, HDA_STREAM_STS_INTERRUPT_MASK); return NTLSTATUS_SUCCESS; } // Builds a four-entry BDL beginning with the active ring period owned by software. static void HdaBuildOutputBdl(HDA_CONTROLLER* controller) { HDA_OUTPUT_RING* ring = &controller->Output.Rings[controller->Output.ActiveRingIndex]; for (uint32_t bdlIndex = 0; bdlIndex < HDA_OUTPUT_PERIOD_COUNT; bdlIndex++) { uint8_t period = (uint8_t)((ring->NextLogicalPeriod + bdlIndex) % HDA_OUTPUT_PERIOD_COUNT); controller->Output.BdlPeriodMap[bdlIndex] = period; controller->Output.Bdl[bdlIndex].Address = ring->DeviceAddress + (uint64_t)period * HDA_OUTPUT_PERIOD_BYTES; controller->Output.Bdl[bdlIndex].Length = HDA_OUTPUT_PERIOD_BYTES; controller->Output.Bdl[bdlIndex].Flags = HDA_BDL_FLAG_INTERRUPT_ON_COMPLETION; } controller->Output.NextCompletionBdl = 0; HdaMemoryBarrier(); } // Programs the fixed-format output descriptor without starting DMA. static KSTATUS HdaProgramOutputDescriptor(HDA_CONTROLLER* controller) { KSTATUS status = HdaResetOutputDescriptor(controller); if (!KSUCCESS(status)) return status; if (controller->PositionBuffer) controller->PositionBuffer[controller->Output.DescriptorIndex] = 0; HdaBuildOutputBdl(controller); uint32_t base = HdaOutputStreamBase(controller); HdaWrite32(controller, base + HDA_STREAM_REG_CBL, HDA_OUTPUT_BUFFER_BYTES); HdaWrite16(controller, base + HDA_STREAM_REG_LVI, (uint16_t)(HDA_OUTPUT_PERIOD_COUNT - 1u)); HdaWrite16(controller, base + HDA_STREAM_REG_FORMAT, HDA_OUTPUT_STREAM_FORMAT); HdaWrite32(controller, base + HDA_STREAM_REG_BDLPL, (uint32_t)controller->Output.BdlDeviceAddress); HdaWrite32(controller, base + HDA_STREAM_REG_BDLPU, (uint32_t)(controller->Output.BdlDeviceAddress >> 32)); uint32_t control = ((uint32_t)controller->Output.StreamTag << HDA_STREAM_CTL_TAG_SHIFT) | HDA_STREAM_CTL_INTERRUPT_COMPLETE | HDA_STREAM_CTL_INTERRUPT_FIFO_ERROR | HDA_STREAM_CTL_INTERRUPT_DESCRIPTOR_ERROR; HdaWriteStreamControl(controller, control); controller->Output.Prepared = true; return NTLSTATUS_SUCCESS; } // Resets the controller global state after all DMA engines are stopped. static KSTATUS HdaResetControllerHardware(HDA_CONTROLLER* controller) { HdaWrite32(controller, HDA_REG_INTCTL, 0); (void)HdaStopOutputHardware(controller); HdaWrite8(controller, HDA_REG_CORBCTL, 0); HdaWrite8(controller, HDA_REG_RIRBCTL, 0); uint32_t gctl = HdaRead32(controller, HDA_REG_GCTL); HdaWrite32(controller, HDA_REG_GCTL, gctl & ~HDA_GCTL_RESET); KSTATUS status = HdaWait32(controller, HDA_REG_GCTL, HDA_GCTL_RESET, 0, HDA_CONTROLLER_RESET_TIMEOUT_US); if (!KSUCCESS(status)) return status; HdaStall(100); HdaWrite32(controller, HDA_REG_GCTL, HDA_GCTL_RESET); status = HdaWait32(controller, HDA_REG_GCTL, HDA_GCTL_RESET, HDA_GCTL_RESET, HDA_CONTROLLER_RESET_TIMEOUT_US); if (!KSUCCESS(status)) return status; HdaStall(HDA_CODEC_RESET_SETTLE_US); return NTLSTATUS_SUCCESS; } // Resets the CORB read pointer while accepting controllers that self-clear the reset bit. static KSTATUS HdaResetCorbReadPointer(HDA_CONTROLLER* controller) { HdaWrite16(controller, HDA_REG_CORBRP, HDA_CORBRP_RESET); for (uint32_t elapsed = 0; elapsed < HDA_RING_RESET_TIMEOUT_US; elapsed++) { if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; uint16_t value = HdaRead16(controller, HDA_REG_CORBRP); if ((value & HDA_CORBRP_RESET) != 0) { HdaWrite16(controller, HDA_REG_CORBRP, 0); return HdaWait16(controller, HDA_REG_CORBRP, HDA_CORBRP_RESET, 0, HDA_RING_RESET_TIMEOUT_US); } // Some controllers complete the reset before software can observe the // reset bit. A zero read pointer proves that the requested state was // reached, so requiring an intermediate asserted state is incorrect. if ((value & HDA_CORBRP_POINTER_MASK) == 0) { HdaWrite16(controller, HDA_REG_CORBRP, 0); return NTLSTATUS_SUCCESS; } HdaStall(1); } return NTLSTATUS_TIMEOUT; } // Reports the register state that owns a controller initialization failure. static KSTATUS HdaReportInitializationFailure(HDA_CONTROLLER* controller, const char* stage, KSTATUS status) { gHdaDdk->Debug->Print( "HDA: init stage=%s failed status=0x%x GCTL=0x%08x STATESTS=0x%04x " "CORB=%02x/%02x/%04x/%04x RIRB=%02x/%02x/%04x SDCTL=0x%06x\n", stage, status, HdaRead32(controller, HDA_REG_GCTL), HdaRead16(controller, HDA_REG_STATESTS), HdaRead8(controller, HDA_REG_CORBCTL), HdaRead8(controller, HDA_REG_CORBSTS), HdaRead16(controller, HDA_REG_CORBWP), HdaRead16(controller, HDA_REG_CORBRP), HdaRead8(controller, HDA_REG_RIRBCTL), HdaRead8(controller, HDA_REG_RIRBSTS), HdaRead16(controller, HDA_REG_RIRBWP), HdaReadStreamControl(controller)); return status; } // Initializes CORB and RIRB with software pointers synchronized to hardware. static KSTATUS HdaInitializeCommandRings(HDA_CONTROLLER* controller) { HdaWrite8(controller, HDA_REG_CORBCTL, 0); HdaWrite8(controller, HDA_REG_RIRBCTL, 0); KSTATUS status = HdaWait8(controller, HDA_REG_CORBCTL, HDA_CORBCTL_RUN, 0, HDA_RING_RESET_TIMEOUT_US); if (!KSUCCESS(status)) return status; status = HdaWait8(controller, HDA_REG_RIRBCTL, HDA_RIRBCTL_RUN, 0, HDA_RING_RESET_TIMEOUT_US); if (!KSUCCESS(status)) return status; HdaWrite8(controller, HDA_REG_CORBSTS, HDA_CORBSTS_MEMORY_ERROR); HdaWrite8(controller, HDA_REG_RIRBSTS, HDA_RIRBSTS_RESPONSE | HDA_RIRBSTS_OVERRUN); HdaWrite32(controller, HDA_REG_CORBLBASE, (uint32_t)controller->CorbDeviceAddress); HdaWrite32(controller, HDA_REG_CORBUBASE, (uint32_t)(controller->CorbDeviceAddress >> 32)); HdaWrite8(controller, HDA_REG_CORBSIZE, HdaRingSizeSelector(controller->CorbEntryCount)); HdaWrite16(controller, HDA_REG_CORBWP, 0); status = HdaResetCorbReadPointer(controller); if (!KSUCCESS(status)) return status; HdaWrite32(controller, HDA_REG_RIRBLBASE, (uint32_t)controller->RirbDeviceAddress); HdaWrite32(controller, HDA_REG_RIRBUBASE, (uint32_t)(controller->RirbDeviceAddress >> 32)); HdaWrite8(controller, HDA_REG_RIRBSIZE, HdaRingSizeSelector(controller->RirbEntryCount)); HdaWrite16(controller, HDA_REG_RIRBWP, HDA_RIRBWP_RESET); HdaWrite16(controller, HDA_REG_RINTCNT, 1); controller->CorbWritePointer = 0; controller->RirbReadPointer = 0; controller->CommandWaiting = false; controller->CommandResponseReady = false; HdaMemoryBarrier(); HdaWrite8(controller, HDA_REG_CORBCTL, HDA_CORBCTL_MEMORY_ERROR_INTERRUPT | HDA_CORBCTL_RUN); HdaWrite8(controller, HDA_REG_RIRBCTL, HDA_RIRBCTL_RESPONSE_INTERRUPT | HDA_RIRBCTL_OVERRUN_INTERRUPT | HDA_RIRBCTL_RUN); status = HdaWait8(controller, HDA_REG_CORBCTL, HDA_CORBCTL_RUN, HDA_CORBCTL_RUN, HDA_RING_RESET_TIMEOUT_US); if (!KSUCCESS(status)) return status; return HdaWait8(controller, HDA_REG_RIRBCTL, HDA_RIRBCTL_RUN, HDA_RIRBCTL_RUN, HDA_RING_RESET_TIMEOUT_US); } // Queues one unsolicited response for DPC-level topology handling. static void HdaQueueUnsolicitedLocked(HDA_CONTROLLER* controller, uint32_t response, uint32_t responseExtra) { if (controller->UnsolicitedCount == HDA_MAX_UNSOLICITED_RESPONSES) { controller->UnsolicitedRead = (uint8_t)((controller->UnsolicitedRead + 1u) % HDA_MAX_UNSOLICITED_RESPONSES); controller->UnsolicitedCount--; } controller->Unsolicited[controller->UnsolicitedWrite].Response = response; controller->Unsolicited[controller->UnsolicitedWrite].ResponseExtra = responseExtra; controller->UnsolicitedWrite = (uint8_t)((controller->UnsolicitedWrite + 1u) % HDA_MAX_UNSOLICITED_RESPONSES); controller->UnsolicitedCount++; controller->RirbUnsolicitedCount++; } // Drains every hardware-owned RIRB entry while the controller lock is held. static void HdaDrainRirbLocked(HDA_CONTROLLER* controller) { uint16_t hardwareWritePointer = (uint16_t)(HdaRead16(controller, HDA_REG_RIRBWP) & 0x00FFu); while (controller->RirbReadPointer != hardwareWritePointer) { controller->RirbReadPointer = (uint16_t)((controller->RirbReadPointer + 1u) % controller->RirbEntryCount); HDA_RIRB_ENTRY entry = controller->Rirb[controller->RirbReadPointer]; controller->RirbResponseCount++; if (entry.ResponseExtra & HDA_RIRB_RESPONSE_UNSOLICITED) { HdaQueueUnsolicitedLocked(controller, entry.Response, entry.ResponseExtra); continue; } uint8_t codecAddress = (uint8_t)(entry.ResponseExtra & HDA_RIRB_RESPONSE_CODEC_MASK); if (controller->CommandWaiting && codecAddress == controller->CommandCodecAddress && !controller->CommandResponseReady) { controller->CommandResponse = entry.Response; controller->CommandResponseExtra = entry.ResponseExtra; controller->CommandResponseReady = true; } } } // Drains RIRB responses and acknowledges response or overrun status. static uint8_t HdaServiceRirbLocked(HDA_CONTROLLER* controller) { uint8_t status = HdaRead8(controller, HDA_REG_RIRBSTS); if (status & HDA_RIRBSTS_RESPONSE) HdaDrainRirbLocked(controller); if (status & HDA_RIRBSTS_OVERRUN) { controller->RirbOverrunCount++; if (controller->Started) controller->PendingRecovery = HDA_RECOVERY_CONTROLLER; } if (status) HdaWrite8(controller, HDA_REG_RIRBSTS, status); return status; } // Sends one codec verb attempt while the command mutex is held. static KSTATUS HdaCodecVerbAttemptLocked(HDA_CONTROLLER* controller, uint8_t codecAddress, uint8_t nodeId, uint32_t verbPayload, uint32_t* responseOut) { uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); (void)HdaServiceRirbLocked(controller); uint64_t startingOverruns = controller->RirbOverrunCount; uint16_t next = (uint16_t)((controller->CorbWritePointer + 1u) % controller->CorbEntryCount); uint16_t hardwareReadPointer = (uint16_t)(HdaRead16(controller, HDA_REG_CORBRP) & 0x00FFu); if (next == hardwareReadPointer) { KeReleaseSpinlock(&controller->StateLock, flags); return NTLSTATUS_WOULD_BLOCK; } controller->CommandWaiting = true; controller->CommandResponseReady = false; controller->CommandCodecAddress = codecAddress; controller->Corb[next] = ((uint32_t)codecAddress << 28) | ((uint32_t)nodeId << 20) | (verbPayload & 0x000FFFFFu); HdaMemoryBarrier(); controller->CorbWritePointer = next; HdaWrite16(controller, HDA_REG_CORBWP, next); KeReleaseSpinlock(&controller->StateLock, flags); for (uint32_t elapsed = 0; elapsed < HDA_CODEC_COMMAND_TIMEOUT_US; elapsed += 10u) { flags = KeAcquireSpinlock(&controller->StateLock); (void)HdaServiceRirbLocked(controller); if (controller->CommandResponseReady) { uint32_t response = controller->CommandResponse; controller->CommandWaiting = false; controller->CommandResponseReady = false; KeReleaseSpinlock(&controller->StateLock, flags); if (responseOut) *responseOut = response; return NTLSTATUS_SUCCESS; } bool overrun = controller->RirbOverrunCount != startingOverruns; bool removed = controller->Removed || controller->SurpriseRemoved; KeReleaseSpinlock(&controller->StateLock, flags); if (overrun || removed) break; HdaStall(10); } flags = KeAcquireSpinlock(&controller->StateLock); controller->CommandWaiting = false; controller->CommandResponseReady = false; KeReleaseSpinlock(&controller->StateLock, flags); return NTLSTATUS_TIMEOUT; } // Sends one codec verb with the required retry while the command mutex is held. static KSTATUS HdaCodecVerbSerialized(HDA_CONTROLLER* controller, uint8_t codecAddress, uint8_t nodeId, uint32_t verbPayload, uint32_t* responseOut) { KSTATUS finalStatus = NTLSTATUS_TIMEOUT; for (uint32_t attempt = 0; attempt < HDA_CODEC_COMMAND_ATTEMPTS; attempt++) { controller->CodecCommandCount++; finalStatus = HdaCodecVerbAttemptLocked(controller, codecAddress, nodeId, verbPayload, responseOut); if (KSUCCESS(finalStatus)) break; if (attempt + 1u < HDA_CODEC_COMMAND_ATTEMPTS) { controller->CodecCommandRetryCount++; HdaStall(100); } } if (!KSUCCESS(finalStatus)) controller->CodecCommandFailureCount++; return finalStatus; } // Sends one 20-bit codec verb payload through CORB/RIRB with one retry. KSTATUS HdaCodecVerb(HDA_CONTROLLER* controller, uint8_t codecAddress, uint8_t nodeId, uint32_t verbPayload, uint32_t* responseOut) { if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || codecAddress >= HDA_MAX_CODECS || (verbPayload & 0xFFF00000u) != 0 || !controller->CommandMutex) return NTLSTATUS_INVALID_PARAMETER; if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return status; status = HdaCodecVerbSerialized(controller, codecAddress, nodeId, verbPayload, responseOut); KeReleaseMutex(controller->CommandMutex); bool queueDpc = false; uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); queueDpc = controller->Started && !controller->RecoveryInProgress && (controller->UnsolicitedCount != 0 || controller->PendingRecovery != HDA_RECOVERY_NONE); KeReleaseSpinlock(&controller->StateLock, flags); if (queueDpc) (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); return status; } // Verifies responding codec addresses through the architectural root node. static KSTATUS HdaProbeCodecs(HDA_CONTROLLER* controller) { uint16_t present = controller->CodecMask; controller->CodecMask = 0; for (uint8_t codec = 0; codec < HDA_MAX_CODECS; codec++) { if ((present & (uint16_t)(1u << codec)) == 0) continue; uint32_t vendorId = 0; KSTATUS status = HdaCodecVerbSerialized( controller, codec, 0, HDA_VERB_GET_PARAMETER(HDA_PARAMETER_VENDOR_ID), &vendorId); if (!KSUCCESS(status)) continue; controller->CodecMask |= (uint16_t)(1u << codec); } return controller->CodecMask ? NTLSTATUS_SUCCESS : NTLSTATUS_NOT_FOUND; } // Initializes the controller while no other command producer is active. static KSTATUS HdaInitializeControllerLocked(HDA_CONTROLLER* controller) { KSTATUS status = HdaResetControllerHardware(controller); if (!KSUCCESS(status)) return HdaReportInitializationFailure(controller, "controller-reset", status); uint16_t state = (uint16_t)(HdaRead16(controller, HDA_REG_STATESTS) & 0x7FFFu); controller->CodecMask = state; if (state) HdaWrite16(controller, HDA_REG_STATESTS, state); status = HdaInitializeCommandRings(controller); if (!KSUCCESS(status)) return HdaReportInitializationFailure(controller, "command-rings", status); if (controller->PositionBuffer) { HdaWrite32(controller, HDA_REG_DPLBASE, (uint32_t)controller->PositionDeviceAddress | HDA_DPLBASE_ENABLE); HdaWrite32(controller, HDA_REG_DPUBASE, (uint32_t)(controller->PositionDeviceAddress >> 32)); } else { HdaWrite32(controller, HDA_REG_DPLBASE, 0); HdaWrite32(controller, HDA_REG_DPUBASE, 0); } status = HdaProgramOutputDescriptor(controller); if (!KSUCCESS(status)) return HdaReportInitializationFailure(controller, "output-descriptor", status); status = HdaProbeCodecs(controller); if (!KSUCCESS(status)) return HdaReportInitializationFailure(controller, "codec-probe", status); HdaWrite32(controller, HDA_REG_GCTL, HdaRead32(controller, HDA_REG_GCTL) | HDA_GCTL_RESET | HDA_GCTL_UNSOLICITED_ENABLE); HdaWrite32(controller, HDA_REG_INTCTL, 0); return NTLSTATUS_SUCCESS; } // Resets and initializes the controller, command rings, position buffer, and output descriptor. KSTATUS HdaInitializeController(HDA_CONTROLLER* controller) { if (!controller || controller->Magic != HDA_CONTROLLER_MAGIC || !controller->Registers || !controller->CommandMutex) return NTLSTATUS_INVALID_PARAMETER; KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return status; status = HdaInitializeControllerLocked(controller); KeReleaseMutex(controller->CommandMutex); return status; } // Arms controller and output-stream interrupts after ISR registration. void HdaArmInterrupts(HDA_CONTROLLER* controller) { if (!controller || !controller->Registers || !controller->Started || controller->Removed || controller->SurpriseRemoved) return; uint32_t streamInterrupt = 1u << controller->Output.DescriptorIndex; HdaWrite32(controller, HDA_REG_INTCTL, HDA_INT_GLOBAL_ENABLE | HDA_INT_CONTROLLER_ENABLE | streamInterrupt); } // Stops hardware access without releasing software resources. void HdaQuiesceController(HDA_CONTROLLER* controller) { if (!controller || !controller->Registers || controller->SurpriseRemoved) return; bool streamLocked = false; bool commandLocked = false; if (controller->StreamMutex) { KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return; streamLocked = true; } if (controller->CommandMutex) { KSTATUS status = KeAcquireMutex(controller->CommandMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) { if (streamLocked) KeReleaseMutex(controller->StreamMutex); return; } commandLocked = true; } HdaWrite32(controller, HDA_REG_INTCTL, 0); (void)HdaStopOutputHardware(controller); HdaWrite8(controller, HDA_REG_CORBCTL, 0); HdaWrite8(controller, HDA_REG_RIRBCTL, 0); uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); controller->Output.Running = false; KeReleaseSpinlock(&controller->StateLock, flags); if (commandLocked) KeReleaseMutex(controller->CommandMutex); if (streamLocked) KeReleaseMutex(controller->StreamMutex); } // Copies one logical endpoint ring contract for codec and endpoint setup. KSTATUS HdaDescribeOutput(HDA_CONTROLLER* controller, uint8_t ringIndex, HDA_OUTPUT_DESCRIPTION* descriptionOut) { if (!controller || !descriptionOut || ringIndex >= HDA_MAX_LOGICAL_ENDPOINTS || !controller->Output.Rings[ringIndex].Buffer) return NTLSTATUS_INVALID_PARAMETER; descriptionOut->PeriodBuffer = controller->Output.Rings[ringIndex].Buffer; descriptionOut->PeriodBufferBytes = HDA_OUTPUT_BUFFER_BYTES; descriptionOut->PeriodCount = HDA_OUTPUT_PERIOD_COUNT; descriptionOut->FramesPerPeriod = HDA_OUTPUT_PERIOD_FRAMES; descriptionOut->PeriodStrideBytes = HDA_OUTPUT_PERIOD_BYTES; descriptionOut->HardwareFormat.SampleRate = HDA_OUTPUT_SAMPLE_RATE; descriptionOut->HardwareFormat.ChannelCount = HDA_OUTPUT_CHANNELS; descriptionOut->HardwareFormat.SampleFormat = IoAudioSampleS16; descriptionOut->StreamTag = controller->Output.StreamTag; return NTLSTATUS_SUCCESS; } // Selects one logical endpoint ring while the stream is stopped. KSTATUS HdaSelectOutputRing(HDA_CONTROLLER* controller, uint8_t ringIndex) { if (!controller || !controller->StreamMutex || ringIndex >= HDA_MAX_LOGICAL_ENDPOINTS || !controller->Output.Rings[ringIndex].Buffer) return NTLSTATUS_INVALID_PARAMETER; KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return status; if (controller->Output.Running) { KeReleaseMutex(controller->StreamMutex); return NTLSTATUS_WOULD_BLOCK; } controller->Output.ActiveRingIndex = ringIndex; status = HdaProgramOutputDescriptor(controller); KeReleaseMutex(controller->StreamMutex); return status; } // Replaces the transport callbacks while the output stream is stopped. KSTATUS HdaSetTransportCallbacks(HDA_CONTROLLER* controller, const HDA_TRANSPORT_CALLBACKS* callbacks, void* context) { if (!controller || !controller->StreamMutex) return NTLSTATUS_INVALID_PARAMETER; KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return status; if (controller->Output.Running) { KeReleaseMutex(controller->StreamMutex); return NTLSTATUS_INVALID_PARAMETER; } uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); if (callbacks) controller->Callbacks = *callbacks; else memset(&controller->Callbacks, 0, sizeof(controller->Callbacks)); controller->CallbackContext = context; KeReleaseSpinlock(&controller->StateLock, flags); KeReleaseMutex(controller->StreamMutex); return NTLSTATUS_SUCCESS; } // Starts one already programmed output descriptor without rebuilding its BDL. static KSTATUS HdaRunPreparedOutput(HDA_CONTROLLER* controller) { uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); controller->Output.PositionRunBaseFrames = controller->Output.LastPositionFrames; controller->Output.PositionWrapFrames = 0; controller->Output.LastHardwarePositionBytes = 0; controller->Output.PositionInitialized = false; KeReleaseSpinlock(&controller->StateLock, flags); HdaMemoryBarrier(); uint32_t control = HdaReadStreamControl(controller) | HDA_STREAM_CTL_RUN; HdaWriteStreamControl(controller, control); KSTATUS status = HdaWaitStreamControl(controller, HDA_STREAM_CTL_RUN, HDA_STREAM_CTL_RUN, HDA_STREAM_RESET_TIMEOUT_US); if (KSUCCESS(status)) { flags = KeAcquireSpinlock(&controller->StateLock); controller->Output.Running = true; KeReleaseSpinlock(&controller->StateLock, flags); } return status; } // Starts the prepared fixed-format output stream. KSTATUS HdaStartOutput(HDA_CONTROLLER* controller) { if (!controller || !controller->StreamMutex) return NTLSTATUS_INVALID_PARAMETER; KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return status; if (!controller->Started || controller->Removed || controller->SurpriseRemoved || !controller->Output.Prepared) { KeReleaseMutex(controller->StreamMutex); return NTLSTATUS_INVALID_PARAMETER; } if (controller->Output.Running) { KeReleaseMutex(controller->StreamMutex); return NTLSTATUS_SUCCESS; } status = HdaProgramOutputDescriptor(controller); if (KSUCCESS(status)) { uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); controller->Output.PendingPeriodInterrupts = 0; controller->Output.PendingDmaErrors = 0; KeReleaseSpinlock(&controller->StateLock, flags); status = HdaRunPreparedOutput(controller); } KeReleaseMutex(controller->StreamMutex); return status; } // Stops the output stream while retaining its DMA ring. void HdaStopOutput(HDA_CONTROLLER* controller) { if (!controller || !controller->Registers || controller->SurpriseRemoved || !controller->StreamMutex) return; KSTATUS status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); if (!KE_MUTEX_ACQUIRE_SUCCEEDED(status)) return; (void)HdaQueryOutputPositionFrames(controller); (void)HdaStopOutputHardware(controller); uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); controller->Output.Running = false; KeReleaseSpinlock(&controller->StateLock, flags); KeReleaseMutex(controller->StreamMutex); } // Returns the current byte position from the position buffer or LPIB. static uint32_t HdaQueryOutputPositionBytesLocked(HDA_CONTROLLER* controller) { uint32_t position; if (controller->PositionBuffer) { position = (uint32_t)controller->PositionBuffer[controller->Output.DescriptorIndex]; controller->Output.PositionBufferReadCount++; } else { position = HdaRead32(controller, HdaOutputStreamBase(controller) + HDA_STREAM_REG_LPIB); controller->Output.LpibReadCount++; } if (position > HDA_OUTPUT_BUFFER_BYTES) { controller->Output.PositionDiscontinuityCount++; position %= HDA_OUTPUT_BUFFER_BYTES; } else if (position == HDA_OUTPUT_BUFFER_BYTES) { position = 0; } return position; } // Advances the monotonic fine position while the state lock is held. static uint64_t HdaQueryOutputPositionFramesLocked(HDA_CONTROLLER* controller) { if (!controller->Output.Running) return controller->Output.LastPositionFrames; uint32_t positionBytes = HdaQueryOutputPositionBytesLocked(controller); if (controller->Output.PositionInitialized && positionBytes < controller->Output.LastHardwarePositionBytes) { controller->Output.PositionWrapFrames += HDA_OUTPUT_PERIOD_COUNT * HDA_OUTPUT_PERIOD_FRAMES; } controller->Output.PositionInitialized = true; controller->Output.LastHardwarePositionBytes = positionBytes; uint64_t candidate = controller->Output.PositionRunBaseFrames + controller->Output.PositionWrapFrames + positionBytes / HDA_OUTPUT_BYTES_PER_FRAME; if (candidate < controller->Output.LastPositionFrames) { controller->Output.PositionDiscontinuityCount++; candidate = controller->Output.LastPositionFrames; } controller->Output.LastPositionFrames = candidate; return candidate; } // Returns the monotonic fine position derived from the position buffer or LPIB. uint64_t HdaQueryOutputPositionFrames(HDA_CONTROLLER* controller) { if (!controller || !controller->Registers) return 0; uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); uint64_t position = HdaQueryOutputPositionFramesLocked(controller); KeReleaseSpinlock(&controller->StateLock, flags); return position; } // Captures one controller interrupt and queues deferred processing. void HdaHandleInterrupt(HDA_CONTROLLER* controller) { if (!controller) return; NtlAtomicIncrement32(&controller->IsrActive); if (!controller->Started || controller->Removed || controller->SurpriseRemoved) goto finished; uint32_t interruptStatus = HdaRead32(controller, HDA_REG_INTSTS); if (interruptStatus == 0 || interruptStatus == UINT32_MAX) goto finished; bool queueDpc = false; uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); bool recovering = controller->RecoveryInProgress; controller->InterruptCount++; uint32_t streamBit = 1u << controller->Output.DescriptorIndex; if (interruptStatus & streamBit) { uint32_t base = HdaOutputStreamBase(controller); uint8_t streamStatus = HdaRead8(controller, base + HDA_STREAM_REG_STS); streamStatus &= HDA_STREAM_STS_INTERRUPT_MASK; if (streamStatus) { HdaWrite8(controller, base + HDA_STREAM_REG_STS, streamStatus); controller->Output.InterruptCount++; if ((streamStatus & HDA_STREAM_STS_COMPLETE) && !recovering) { controller->Output.PendingPeriodInterrupts++; queueDpc = true; } if (streamStatus & HDA_STREAM_STS_ERROR_MASK) { controller->Output.DmaErrorCount++; if (!recovering) { controller->Output.PendingDmaErrors++; queueDpc = true; } } } } if (interruptStatus & HDA_INT_CONTROLLER_STATUS) { controller->ControllerInterruptCount++; (void)HdaServiceRirbLocked(controller); if (!recovering) { queueDpc = queueDpc || controller->UnsolicitedCount != 0 || controller->PendingRecovery != HDA_RECOVERY_NONE; } uint8_t corbStatus = HdaRead8(controller, HDA_REG_CORBSTS); if (corbStatus & HDA_CORBSTS_MEMORY_ERROR) { controller->CorbMemoryErrorCount++; HdaWrite8(controller, HDA_REG_CORBSTS, corbStatus); controller->PendingRecovery = HDA_RECOVERY_CONTROLLER; if (!recovering) queueDpc = true; } } KeReleaseSpinlock(&controller->StateLock, flags); if (queueDpc) (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); finished: NtlAtomicDecrement32(&controller->IsrActive); } // Promotes an error to stream or controller recovery using the one-second threshold. static void HdaRecordDmaErrorsLocked(HDA_CONTROLLER* controller, uint32_t errorCount) { uint64_t now = HdaQueryTicks(); if (controller->StreamErrorsInWindow == 0 || now - controller->StreamErrorWindowStart > HDA_STREAM_ERROR_WINDOW_US) { controller->StreamErrorWindowStart = now; controller->StreamErrorsInWindow = 0; } controller->StreamErrorsInWindow += errorCount; uint8_t requested = controller->StreamErrorsInWindow >= HDA_STREAM_ERRORS_BEFORE_CONTROLLER_RESET ? HDA_RECOVERY_CONTROLLER : HDA_RECOVERY_STREAM; if (requested > controller->PendingRecovery) controller->PendingRecovery = requested; } // Runs period delivery, unsolicited callbacks, and recovery scheduling at DPC level. void HdaDpcRoutine(KDPC* dpc, void* context, void* systemArgument1, void* systemArgument2) { (void)dpc; (void)systemArgument1; (void)systemArgument2; HDA_CONTROLLER* controller = (HDA_CONTROLLER*)context; if (!controller) return; NtlAtomicIncrement32(&controller->DpcActive); if (controller->Removed || controller->SurpriseRemoved) goto finished; while (true) { uint32_t periodCount = 0; uint32_t errorCount = 0; HDA_UNSOLICITED_RESPONSE unsolicited; bool haveUnsolicited = false; HDA_TRANSPORT_CALLBACKS callbacks; void* callbackContext; uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); if (controller->Output.PendingPeriodInterrupts > 0) { periodCount = (uint32_t)controller->Output.PendingPeriodInterrupts; controller->Output.PendingPeriodInterrupts = 0; } if (controller->Output.PendingDmaErrors > 0) { errorCount = (uint32_t)controller->Output.PendingDmaErrors; controller->Output.PendingDmaErrors = 0; } if (controller->UnsolicitedCount) { unsolicited = controller->Unsolicited[controller->UnsolicitedRead]; controller->UnsolicitedRead = (uint8_t)((controller->UnsolicitedRead + 1u) % HDA_MAX_UNSOLICITED_RESPONSES); controller->UnsolicitedCount--; haveUnsolicited = true; } callbacks = controller->Callbacks; callbackContext = controller->CallbackContext; KeReleaseSpinlock(&controller->StateLock, flags); for (uint32_t index = 0; index < periodCount; index++) { flags = KeAcquireSpinlock(&controller->StateLock); uint8_t bdlIndex = controller->Output.NextCompletionBdl; uint32_t completedPeriod = controller->Output.BdlPeriodMap[bdlIndex]; controller->Output.NextCompletionBdl = (uint8_t)((bdlIndex + 1u) % HDA_OUTPUT_PERIOD_COUNT); HDA_OUTPUT_RING* ring = &controller->Output.Rings[controller->Output.ActiveRingIndex]; ring->NextLogicalPeriod = (uint8_t)((completedPeriod + 1u) % HDA_OUTPUT_PERIOD_COUNT); ring->PresentedFrames += HDA_OUTPUT_PERIOD_FRAMES; controller->Output.PeriodCompletionCount++; uint64_t presentedFrames = ring->PresentedFrames; uint64_t finePosition = HdaQueryOutputPositionFramesLocked(controller); uint64_t interruptCount = controller->Output.InterruptCount; KeReleaseSpinlock(&controller->StateLock, flags); uint64_t timestamp = HdaQueryTicks(); if (callbacks.PeriodComplete) { callbacks.PeriodComplete(callbackContext, completedPeriod, presentedFrames, finePosition, interruptCount, timestamp); } } if (haveUnsolicited && callbacks.UnsolicitedResponse) { callbacks.UnsolicitedResponse( callbackContext, (uint8_t)(unsolicited.ResponseExtra & HDA_RIRB_RESPONSE_CODEC_MASK), unsolicited.Response, unsolicited.ResponseExtra); } if (errorCount) { flags = KeAcquireSpinlock(&controller->StateLock); HdaRecordDmaErrorsLocked(controller, errorCount); KeReleaseSpinlock(&controller->StateLock, flags); } bool signalRecovery = false; flags = KeAcquireSpinlock(&controller->StateLock); signalRecovery = controller->PendingRecovery != HDA_RECOVERY_NONE; KeReleaseSpinlock(&controller->StateLock, flags); if (signalRecovery && controller->RecoverySemaphore) KeSignalSemaphore(controller->RecoverySemaphore); if (periodCount == 0 && errorCount == 0 && !haveUnsolicited) break; } finished: NtlAtomicDecrement32(&controller->DpcActive); } // Recovers one isolated stream error without disturbing the codec topology. static KSTATUS HdaRecoverOutputStream(HDA_CONTROLLER* controller) { if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; (void)HdaQueryOutputPositionFrames(controller); uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); bool restart = controller->Output.Running; controller->Output.Running = false; KeReleaseSpinlock(&controller->StateLock, flags); KSTATUS status = HdaProgramOutputDescriptor(controller); if (!KSUCCESS(status)) return status; controller->Output.StreamResetCount++; HdaArmInterrupts(controller); if (restart) status = HdaRunPreparedOutput(controller); return status; } // Recovers command and DMA engines, then asks the codec layer to restore routing. static KSTATUS HdaRecoverController(HDA_CONTROLLER* controller) { if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; (void)HdaQueryOutputPositionFrames(controller); uintptr_t stateFlags = KeAcquireSpinlock(&controller->StateLock); bool restart = controller->Output.Running; controller->Output.Running = false; KeReleaseSpinlock(&controller->StateLock, stateFlags); KSTATUS status = HdaInitializeController(controller); if (!KSUCCESS(status)) return status; controller->ControllerResetCount++; controller->StreamErrorWindowStart = 0; controller->StreamErrorsInWindow = 0; HDA_TRANSPORT_CALLBACKS callbacks; void* callbackContext; uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); callbacks = controller->Callbacks; callbackContext = controller->CallbackContext; KeReleaseSpinlock(&controller->StateLock, flags); if (callbacks.RestoreAfterControllerReset) { status = callbacks.RestoreAfterControllerReset(callbackContext); if (!KSUCCESS(status)) return status; } HdaArmInterrupts(controller); if (restart) { status = HdaRunPreparedOutput(controller); if (KSUCCESS(status) && callbacks.ResumeAfterControllerReset) status = callbacks.ResumeAfterControllerReset(callbackContext); if (!KSUCCESS(status)) { (void)HdaStopOutputHardware(controller); uintptr_t stopFlags = KeAcquireSpinlock(&controller->StateLock); controller->Output.Running = false; KeReleaseSpinlock(&controller->StateLock, stopFlags); } } return status; } // Waits until every pre-recovery ISR and DPC has left the controller object. static KSTATUS HdaWaitForInterruptWorkIdle(HDA_CONTROLLER* controller) { for (uint32_t elapsed = 0; elapsed < 100000u; elapsed += 10u) { if (controller->IsrActive == 0 && controller->Dpc.Inserted == 0 && controller->DpcActive == 0) return NTLSTATUS_SUCCESS; if (controller->Removed || controller->SurpriseRemoved) return NTLSTATUS_DEVICE_REMOVED; HdaStall(10); } return NTLSTATUS_TIMEOUT; } // Runs passive-level reset work that cannot safely execute in an ISR or DPC. static void HdaRecoveryWorker(uintptr_t argument) { HDA_CONTROLLER* controller = (HDA_CONTROLLER*)argument; while (controller->RecoveryWorkerRunning) { KSTATUS waitStatus = KeWaitSemaphore(controller->RecoverySemaphore, KE_WAIT_INFINITE); if (!KSUCCESS(waitStatus) || !controller->RecoveryWorkerRunning) break; uint8_t recovery; uintptr_t flags = KeAcquireSpinlock(&controller->StateLock); recovery = controller->PendingRecovery; controller->PendingRecovery = HDA_RECOVERY_NONE; controller->RecoveryInProgress = recovery != HDA_RECOVERY_NONE; KeReleaseSpinlock(&controller->StateLock, flags); if (recovery == HDA_RECOVERY_NONE) continue; if (!controller->Removed && !controller->SurpriseRemoved) HdaWrite32(controller, HDA_REG_INTCTL, 0); KSTATUS status = HdaWaitForInterruptWorkIdle(controller); bool streamLocked = false; if (KSUCCESS(status)) { status = KeAcquireMutex(controller->StreamMutex, KE_WAIT_INFINITE); streamLocked = KE_MUTEX_ACQUIRE_SUCCEEDED(status); } if (streamLocked) { status = recovery == HDA_RECOVERY_CONTROLLER ? HdaRecoverController(controller) : HdaRecoverOutputStream(controller); KeReleaseMutex(controller->StreamMutex); } HDA_TRANSPORT_CALLBACKS callbacks; void* callbackContext; bool queueDpc; bool repeatRecovery; flags = KeAcquireSpinlock(&controller->StateLock); controller->RecoveryInProgress = false; callbacks = controller->Callbacks; callbackContext = controller->CallbackContext; queueDpc = controller->UnsolicitedCount != 0 || controller->Output.PendingPeriodInterrupts != 0 || controller->Output.PendingDmaErrors != 0; repeatRecovery = controller->PendingRecovery != HDA_RECOVERY_NONE; KeReleaseSpinlock(&controller->StateLock, flags); if (!KSUCCESS(status) && status != NTLSTATUS_DEVICE_REMOVED) { if (callbacks.TransportFailure) callbacks.TransportFailure(callbackContext, status); gHdaDdk->Debug->Print("HDA: recovery failed status=0x%x\n", status); } if (queueDpc) (void)KeInsertQueueDpc(&controller->Dpc, (void*)0, (void*)0); if (repeatRecovery) KeSignalSemaphore(controller->RecoverySemaphore); } if (controller->RecoveryFinished) KeSignalEvent(controller->RecoveryFinished); } // Starts the passive recovery worker for one controller. KSTATUS HdaStartRecoveryWorker(HDA_CONTROLLER* controller) { if (!controller || !controller->RecoverySemaphore || !controller->RecoveryFinished) return NTLSTATUS_INVALID_PARAMETER; controller->RecoveryWorkerRunning = true; KE_THREAD_CREATE_PARAMETERS parameters = KE_THREAD_CREATE_NAMED_NORMAL(PriorityHigh, "hda.recovery"); KSTATUS status = KeCreateKernelThread((uintptr_t)HdaRecoveryWorker, (uintptr_t)controller, ¶meters, &controller->RecoveryThread); if (!KSUCCESS(status)) controller->RecoveryWorkerRunning = false; return status; } // Stops and joins the passive recovery worker for one controller. void HdaStopRecoveryWorker(HDA_CONTROLLER* controller) { if (!controller) return; controller->RecoveryWorkerRunning = false; if (controller->RecoverySemaphore) KeSignalSemaphore(controller->RecoverySemaphore); if (controller->RecoveryThread && controller->RecoveryFinished) (void)KeWaitForEvent(controller->RecoveryFinished, 0); if (controller->RecoveryThread) { ObDereferenceObject((OBJECT*)controller->RecoveryThread); controller->RecoveryThread = (THREAD*)0; } }