// Kernel audio core. This file owns endpoint identity, logical streams, mixing, and provider timing handoff. #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define KE_AUDIO_ENDPOINT_TAG MM_POOL_TAG('a', 'e', 'p', 't') #define KE_AUDIO_RING_TAG MM_POOL_TAG('a', 'r', 'n', 'g') #define KE_AUDIO_PERIOD_STATE_FREE 0u #define KE_AUDIO_PERIOD_STATE_FILLING 1u #define KE_AUDIO_PERIOD_STATE_READY 2u #define KE_AUDIO_PERIOD_STATE_COMPLETED 3u #define KE_AUDIO_PERIOD_STATE_REAPING 4u #define KE_AUDIO_TICKS_PER_MILLISECOND 1000ull typedef struct KE_AUDIO_PERIOD_CONTRIBUTION { KE_AUDIO_STREAM* Stream; uint64_t StreamGeneration; uint32_t SourceFrames; } KE_AUDIO_PERIOD_CONTRIBUTION; typedef struct KE_AUDIO_PERIOD_STATE { uint64_t Generation; uint32_t ContributionCount; uint8_t State; KE_AUDIO_PERIOD_CONTRIBUTION Contributions[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; } KE_AUDIO_PERIOD_STATE; struct KE_AUDIO_STREAM { OBJECT Header; MUTEX* Lock; EVENT* WaitEvent; LIST_ENTRY EndpointLink; IO_AUDIO_ENDPOINT* Endpoint; KE_AUDIO_STREAM_STATE State; volatile int32_t ProcessHandleCount; volatile int32_t CloseRequested; IO_AUDIO_FORMAT Format; uint8_t* Ring; uint32_t FrameBytes; uint32_t ReadFrame; uint32_t WriteFrame; uint32_t BufferedFrames; uint64_t ResamplePositionQ32; uint64_t ResampleStepQ32; uint64_t FramesSubmitted; uint64_t FramesConsumed; uint64_t FramesPresented; uint64_t FramesDiscarded; uint64_t OutputFramesMixed; uint64_t SilenceFramesInserted; uint64_t Underruns; uint64_t ConsecutiveUnderrunPeriods; uint64_t DrainedGeneration; uint64_t DiscontinuityGeneration; uint64_t BufferGeneration; KSTATUS FailureStatus; uint32_t WaitMask; uint32_t WaitWritableFrames; uint64_t WaitConsumedFrames; uint64_t WaitPresentedFrames; uint64_t WaitDrainedGeneration; uint64_t WaitDiscontinuityGeneration; float Volume; float Pan; bool Muted; float CurrentLeftGain; float CurrentRightGain; float TargetLeftGain; float TargetRightGain; float LeftGainStep; float RightGainStep; uint32_t GainRampRemaining; }; struct IO_AUDIO_ENDPOINT { OBJECT Header; LIST_ENTRY RegistryLink; LIST_ENTRY StreamList; SPINLOCK StreamListLock; MUTEX* ControlLock; EVENT* WorkEvent; EVENT* MixerExitEvent; THREAD* MixerThread; SPINLOCK NotifyLock; uint32_t EndpointId; IO_AUDIO_ENDPOINT_KIND Kind; KE_AUDIO_ENDPOINT_STATE State; uint32_t Capabilities; bool Available; bool Registered; bool HardwareRunning; bool SkipProviderStop; IO_AUDIO_REMOVAL_MODE RemovalMode; DEVICE_OBJECT* Device; void* ProviderContext; IO_AUDIO_PROVIDER_OPERATIONS Operations; uint8_t* PeriodBuffer; size_t PeriodBufferBytes; uint32_t PeriodCount; uint32_t FramesPerPeriod; size_t PeriodStrideBytes; IO_AUDIO_FORMAT HardwareFormat; int32_t HardwareGainMinimumMilliDb; int32_t HardwareGainMaximumMilliDb; uint32_t HardwareGainStepMilliDb; KE_AUDIO_PERIOD_STATE Periods[KE_AUDIO_OUTPUT_PERIOD_COUNT]; uint32_t ActiveStreamCount; uint64_t IdleDeadlineTicks; float MasterCurrentGain; float MasterTargetGain; float MasterRequestedVolume; int32_t MasterHardwareGainMilliDb; float MasterGainStep; uint32_t MasterRampRemaining; uint64_t MasterGainGeneration; bool MasterMuted; KSTATUS FailureStatus; uint64_t PeriodsMixed; uint64_t PeriodsCompleted; uint64_t FramesPresented; uint64_t HardwarePositionFrames; uint64_t InterruptCount; uint64_t LastPresentationTimestampTicks; uint64_t SilenceFramesInserted; uint64_t Underruns; uint64_t DiscontinuityGeneration; uint64_t HardwareStarts; uint64_t HardwareStops; uint64_t SpuriousNotifications; uint64_t PendingDiscontinuities; }; typedef struct KE_AUDIO_CORE_STATS { volatile int64_t EndpointsRegistered; volatile int64_t EndpointsUnregistered; volatile int64_t StreamsCreated; volatile int64_t StreamsDestroyed; volatile int64_t Starts; volatile int64_t Stops; volatile int64_t Pauses; volatile int64_t Resumes; volatile int64_t Drains; volatile int64_t Resets; } KE_AUDIO_CORE_STATS; typedef struct KE_AUDIO_LIFECYCLE_SNAPSHOT { KE_AUDIO_ENDPOINT_STATE State; KSTATUS FailureStatus; IO_AUDIO_REMOVAL_MODE RemovalMode; bool SkipProviderStop; } KE_AUDIO_LIFECYCLE_SNAPSHOT; static SPINLOCK gAudioRegistryLock = SPINLOCK_INIT; static LIST_ENTRY gAudioEndpointList; static uint32_t gAudioEndpointCount; static KE_ID_ALLOCATOR gAudioEndpointIds = KE_ID_ALLOCATOR_INIT(1u); static uint32_t gAudioDefaultEndpointId; static bool gAudioInitialized; static KE_AUDIO_CORE_STATS gAudioCoreStats; // Stops on one violated internal audio invariant. static void KiAudioBug(const char* where, const char* reason, uint64_t value) { KePanic(KE_PANIC_CODE_GENERIC, "audio invariant failed: %s: %s value=0x%016llX\n", where ? where : "?", reason ? reason : "?", (unsigned long long)value); } // Acquires an internal mutex and rejects abandoned ownership as kernel corruption. static void KiAudioAcquireMutex(MUTEX* mutex, const char* where) { KSTATUS status = KeAcquireMutex(mutex, KE_WAIT_INFINITE); if (status != NTLSTATUS_SUCCESS) KiAudioBug(where, "internal mutex acquisition failed", status); } // Releases one internal mutex. static void KiAudioReleaseMutex(MUTEX* mutex) { KeReleaseMutex(mutex); } // Returns the provider timer in microseconds when available. static uint64_t KiAudioQueryTicks(void) { return KeQueryMonotonicTicks(); } // Returns whether one float is finite without depending on a runtime math library. static bool KiAudioFloatIsFinite(float value) { union { float Float; uint32_t Bits; } bits; bits.Float = value; return (bits.Bits & 0x7F800000u) != 0x7F800000u; } // Clamps one mixer sample to the signed unit interval. static float KiAudioClampSample(float value) { const float maximum = 0.99999988079071044921875f; if (value > maximum) return maximum; if (value < -1.0f) return -1.0f; return value; } // Converts one normalized mixer sample to signed 16-bit PCM. static int16_t KiAudioFloatToOutputSample(float value) { float scaled = KiAudioClampSample(value) * 32768.0f; int32_t sample = scaled >= 0.0f ? (int32_t)(scaled + 0.5f) : (int32_t)(scaled - 0.5f); if (sample > 32767) sample = 32767; if (sample < -32768) sample = -32768; return (int16_t)sample; } // Approximates one linear gain as milli-decibels without a runtime math dependency. static int32_t KiAudioMilliDbFromLinearGain(float gain) { if (gain <= 0.0f) return INT32_MIN; union { float Float; uint32_t Bits; } bits; bits.Float = gain; uint32_t encodedExponent = (bits.Bits >> 23) & 0xFFu; uint32_t fraction = bits.Bits & 0x007FFFFFu; int32_t exponent = 0; if (encodedExponent) { exponent = (int32_t)encodedExponent - 127; } else { exponent = -126; while ((fraction & 0x00800000u) == 0) { fraction <<= 1; exponent--; } fraction &= 0x007FFFFFu; } bits.Bits = fraction | 0x3F800000u; float mantissa = bits.Float; float y = (mantissa - 1.0f) / (mantissa + 1.0f); float y2 = y * y; float logarithm = 2.0f * y * (1.0f + y2 * (0.3333333333333333f + y2 * (0.2f + y2 * (0.1428571428571429f + y2 * 0.1111111111111111f)))); float log2Gain = (float)exponent + logarithm * 1.4426950408889634f; float milliDb = log2Gain * 6020.599913279624f; return milliDb >= 0.0f ? (int32_t)(milliDb + 0.5f) : (int32_t)(milliDb - 0.5f); } // Approximates one nonpositive milli-decibel residual as a linear gain. static float KiAudioLinearGainFromMilliDb(int32_t milliDb) { if (milliDb >= 0) return 1.0f; float exponent = (float)milliDb * (1.0f / 6020.599913279624f); int32_t whole = (int32_t)exponent; if ((float)whole > exponent) whole--; float fraction = exponent - (float)whole; float z = fraction * 0.6931471805599453f; float z2 = z * z; float result = 1.0f + z + z2 * 0.5f + z2 * z * (1.0f / 6.0f) + z2 * z2 * (1.0f / 24.0f) + z2 * z2 * z * (1.0f / 120.0f); while (whole < 0) { result *= 0.5f; whole++; } while (whole > 0) { result *= 2.0f; whole--; } return result; } // Chooses the highest supported nonpositive hardware gain. static int32_t KiAudioHighestSafeHardwareGain(const IO_AUDIO_ENDPOINT* endpoint) { if (!(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) || !endpoint->Operations.SetGain) return 0; int32_t minimum = endpoint->HardwareGainMinimumMilliDb; int32_t maximum = endpoint->HardwareGainMaximumMilliDb; int64_t step = endpoint->HardwareGainStepMilliDb; if (maximum <= 0) return maximum; int64_t steps = ((int64_t)0 - minimum) / step; return (int32_t)((int64_t)minimum + steps * step); } // Converts one requested master volume into software gain for the stable hardware gain. static float KiAudioSoftwareGainForVolume(const IO_AUDIO_ENDPOINT* endpoint, float requestedVolume) { if (requestedVolume <= 0.0f) return 0.0f; if (!(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) || !endpoint->Operations.SetGain) return requestedVolume; int32_t requestedMilliDb = KiAudioMilliDbFromLinearGain(requestedVolume); int64_t residualMilliDb = (int64_t)requestedMilliDb - endpoint->MasterHardwareGainMilliDb; if (residualMilliDb > 0) residualMilliDb = 0; if (residualMilliDb < -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB) return 0.0f; return KiAudioLinearGainFromMilliDb((int32_t)residualMilliDb); } // Returns the byte width of one source frame. static uint32_t KiAudioFrameBytes(const IO_AUDIO_FORMAT* format) { if (!format) return 0; if (format->SampleFormat != IoAudioSampleS16) return 0; return 2u * format->ChannelCount; } // Validates one logical source format accepted by the first mixer contract. static bool KiAudioValidateSourceFormat(const IO_AUDIO_FORMAT* format) { if (!format) return false; if (format->ChannelCount != 1 && format->ChannelCount != 2) return false; if (format->SampleFormat != IoAudioSampleS16) return false; return format->SampleRate == 11025u || format->SampleRate == 22050u || format->SampleRate == 44100u || format->SampleRate == 48000u; } // Validates the fixed endpoint ring contract used by the HDA provider series. static KSTATUS KiAudioValidateRegistration( const IO_AUDIO_ENDPOINT_REGISTRATION* registration) { if (!registration || !registration->PeriodBuffer) return NTLSTATUS_INVALID_PARAMETER; if (!registration->Device) return NTLSTATUS_INVALID_PARAMETER; if (registration->Kind < IoAudioEndpointLineOut || registration->Kind > IoAudioEndpointOtherAnalog || (registration->Capabilities & ~IO_AUDIO_ENDPOINT_CAP_VALID_MASK) != 0) return NTLSTATUS_INVALID_PARAMETER; if (!registration->Operations.Start || !registration->Operations.Stop) return NTLSTATUS_INVALID_PARAMETER; if (registration->HardwareFormat.SampleRate != KE_AUDIO_OUTPUT_SAMPLE_RATE || registration->HardwareFormat.ChannelCount != KE_AUDIO_OUTPUT_CHANNELS || registration->HardwareFormat.SampleFormat != IoAudioSampleS16) return NTLSTATUS_NOT_SUPPORTED; if (registration->PeriodCount != KE_AUDIO_OUTPUT_PERIOD_COUNT || registration->FramesPerPeriod != KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD || registration->PeriodStrideBytes != KE_AUDIO_OUTPUT_PERIOD_BYTES) return NTLSTATUS_NOT_SUPPORTED; if (registration->PeriodBufferBytes < registration->PeriodStrideBytes * registration->PeriodCount) return NTLSTATUS_BUFFER_TOO_SMALL; if (((uintptr_t)registration->PeriodBuffer & (sizeof(uint32_t) - 1u)) != 0) return NTLSTATUS_INVALID_PARAMETER; if ((registration->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) != 0) { if (!registration->Operations.SetGain || !registration->HardwareGainStepMilliDb || registration->HardwareGainStepMilliDb > INT32_MAX || registration->HardwareGainMinimumMilliDb > registration->HardwareGainMaximumMilliDb || registration->HardwareGainMinimumMilliDb > 0 || registration->HardwareGainMinimumMilliDb < -IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB || registration->HardwareGainMaximumMilliDb > IO_AUDIO_HARDWARE_GAIN_LIMIT_MILLIDB || ((int64_t)registration->HardwareGainMaximumMilliDb - registration->HardwareGainMinimumMilliDb) % registration->HardwareGainStepMilliDb != 0) return NTLSTATUS_INVALID_PARAMETER; } else if (registration->Operations.SetGain || registration->HardwareGainMinimumMilliDb || registration->HardwareGainMaximumMilliDb || registration->HardwareGainStepMilliDb) { return NTLSTATUS_INVALID_PARAMETER; } return NTLSTATUS_SUCCESS; } // Returns whether one endpoint lifecycle transition is valid. static bool KiAudioEndpointTransitionAllowed(KE_AUDIO_ENDPOINT_STATE oldState, KE_AUDIO_ENDPOINT_STATE newState) { if (oldState == newState) return true; switch (oldState) { case KeAudioEndpointStarting: return newState == KeAudioEndpointReady || newState == KeAudioEndpointStopping; case KeAudioEndpointReady: return newState == KeAudioEndpointStopping || newState == KeAudioEndpointFailed; case KeAudioEndpointFailed: return newState == KeAudioEndpointStopping; case KeAudioEndpointStopping: return newState == KeAudioEndpointRemoved; case KeAudioEndpointRemoved: return false; default: return false; } } // Changes one endpoint state while the registry lock serializes lifecycle ownership. static void KiAudioSetEndpointStateLocked(IO_AUDIO_ENDPOINT* endpoint, KE_AUDIO_ENDPOINT_STATE newState) { if (!KiAudioEndpointTransitionAllowed(endpoint->State, newState)) KiAudioBug("KiAudioSetEndpointStateLocked", "invalid endpoint transition", ((uint64_t)endpoint->State << 32) | (uint32_t)newState); endpoint->State = newState; } // Returns the fixed endpoint preference rank used until policy moves to a service. static uint32_t KiAudioEndpointRank(IO_AUDIO_ENDPOINT_KIND kind) { if (kind == IoAudioEndpointLineOut) return 0; if (kind == IoAudioEndpointSpeakerHeadphone) return 1; return 2; } // Recomputes the default endpoint while the registry lock is held. static void KiAudioSelectDefaultEndpointLocked(void) { uint32_t selectedId = 0; uint32_t selectedRank = UINT32_MAX; for (LIST_ENTRY* link = gAudioEndpointList.Fwd; link != &gAudioEndpointList; link = link->Fwd) { IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); if (endpoint->State != KeAudioEndpointReady || !endpoint->Available) continue; uint32_t rank = KiAudioEndpointRank(endpoint->Kind); if (!selectedId || rank < selectedRank || (rank == selectedRank && endpoint->EndpointId < selectedId)) { selectedId = endpoint->EndpointId; selectedRank = rank; } } gAudioDefaultEndpointId = selectedId; } // References one ready endpoint by stable identifier. static IO_AUDIO_ENDPOINT* KiAudioReferenceEndpointById(uint32_t endpointId) { IO_AUDIO_ENDPOINT* result = (IO_AUDIO_ENDPOINT*)0; uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); for (LIST_ENTRY* link = gAudioEndpointList.Fwd; link != &gAudioEndpointList; link = link->Fwd) { IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); if (endpoint->EndpointId != endpointId) continue; if (endpoint->State == KeAudioEndpointReady && ObTryReferenceObject(&endpoint->Header)) result = endpoint; break; } KeReleaseSpinlock(&gAudioRegistryLock, flags); return result; } // Signals one stream when an armed progress condition or terminal state is present. static void KiAudioRefreshStreamEventLocked(KE_AUDIO_STREAM* stream) { uint32_t writableFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; bool signaled = stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed; if ((stream->WaitMask & KE_AUDIO_WAIT_WRITABLE) && writableFrames >= stream->WaitWritableFrames) signaled = 1; if ((stream->WaitMask & KE_AUDIO_WAIT_CONSUMED) && stream->FramesConsumed >= stream->WaitConsumedFrames) signaled = 1; if ((stream->WaitMask & KE_AUDIO_WAIT_PRESENTED) && stream->FramesPresented >= stream->WaitPresentedFrames) signaled = 1; if ((stream->WaitMask & KE_AUDIO_WAIT_STOPPED) && stream->State == KeAudioStreamStopped) signaled = 1; if ((stream->WaitMask & KE_AUDIO_WAIT_DRAINED) && stream->DrainedGeneration != stream->WaitDrainedGeneration) signaled = 1; if ((stream->WaitMask & KE_AUDIO_WAIT_FAILED) && stream->State == KeAudioStreamFailed) signaled = 1; if ((stream->WaitMask & KE_AUDIO_WAIT_DISCONTINUITY) && stream->DiscontinuityGeneration != stream->WaitDiscontinuityGeneration) signaled = 1; if (signaled) KeSignalEvent(stream->WaitEvent); else KeClearEvent(stream->WaitEvent); } // Updates active-stream timing under the endpoint notification lock. static void KiAudioAdjustActiveStreams(IO_AUDIO_ENDPOINT* endpoint, int delta) { uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); uint32_t oldCount = endpoint->ActiveStreamCount; if (delta > 0) { endpoint->ActiveStreamCount += (uint32_t)delta; endpoint->IdleDeadlineTicks = 0; } else { uint32_t amount = (uint32_t)(-delta); if (oldCount < amount) { KeReleaseSpinlock(&endpoint->NotifyLock, flags); KiAudioBug("KiAudioAdjustActiveStreams", "active stream count underflow", oldCount); } endpoint->ActiveStreamCount -= amount; if (endpoint->ActiveStreamCount == 0) { uint64_t now = KiAudioQueryTicks(); endpoint->IdleDeadlineTicks = now + (uint64_t)KE_AUDIO_IDLE_STOP_MILLISECONDS * KE_AUDIO_TICKS_PER_MILLISECOND; } } KeReleaseSpinlock(&endpoint->NotifyLock, flags); KeSignalEvent(endpoint->WorkEvent); } // Returns whether one stream state contributes to the active hardware clock. static bool KiAudioStreamStateIsActive(KE_AUDIO_STREAM_STATE state) { return state == KeAudioStreamRunning || state == KeAudioStreamDraining; } // Returns whether every frame targeted by one drain has reached a terminal disposition. static bool KiAudioDrainIsCompleteLocked(const KE_AUDIO_STREAM* stream) { return stream->BufferedFrames == 0 && stream->FramesPresented + stream->FramesDiscarded >= stream->FramesSubmitted; } // Returns whether one logical stream transition is valid. static bool KiAudioStreamTransitionAllowed(KE_AUDIO_STREAM_STATE oldState, KE_AUDIO_STREAM_STATE newState) { if (oldState == newState) return true; if (newState == KeAudioStreamFailed || newState == KeAudioStreamClosed) return oldState != KeAudioStreamClosed; switch (oldState) { case KeAudioStreamCreated: return newState == KeAudioStreamRunning || newState == KeAudioStreamStopped; case KeAudioStreamRunning: return newState == KeAudioStreamPaused || newState == KeAudioStreamDraining || newState == KeAudioStreamStopped; case KeAudioStreamPaused: return newState == KeAudioStreamRunning || newState == KeAudioStreamDraining || newState == KeAudioStreamStopped; case KeAudioStreamDraining: return newState == KeAudioStreamStopped; case KeAudioStreamStopped: return newState == KeAudioStreamRunning; case KeAudioStreamFailed: case KeAudioStreamClosed: default: return false; } } // Changes one stream state and updates endpoint activity in the same locked transaction. static void KiAudioSetStreamStateLocked(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_STATE newState) { if (!KiAudioStreamTransitionAllowed(stream->State, newState)) KiAudioBug("KiAudioSetStreamStateLocked", "invalid stream transition", ((uint64_t)stream->State << 32) | (uint32_t)newState); bool oldActive = KiAudioStreamStateIsActive(stream->State); bool newActive = KiAudioStreamStateIsActive(newState); stream->State = newState; if (oldActive != newActive) KiAudioAdjustActiveStreams(stream->Endpoint, newActive ? 1 : -1); } // Records one completed drain before publishing the stopped state. static void KiAudioCompleteDrainLocked(KE_AUDIO_STREAM* stream) { if (stream->State != KeAudioStreamDraining || !KiAudioDrainIsCompleteLocked(stream)) KiAudioBug("KiAudioCompleteDrainLocked", "incomplete drain reached completion", stream->State); stream->DrainedGeneration++; KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); } // Computes the target left and right gains for one volume, pan, and mute state. static void KiAudioComputeStreamTargets(float volume, float pan, uint8_t muted, float* leftOut, float* rightOut) { float left = volume; float right = volume; if (pan > 0.0f) left *= 1.0f - pan; else if (pan < 0.0f) right *= 1.0f + pan; if (muted) { left = 0.0f; right = 0.0f; } *leftOut = left; *rightOut = right; } // Starts a ten-millisecond linear gain ramp while the stream mutex is held. static void KiAudioBeginStreamGainRampLocked(KE_AUDIO_STREAM* stream) { KiAudioComputeStreamTargets(stream->Volume, stream->Pan, stream->Muted, &stream->TargetLeftGain, &stream->TargetRightGain); stream->GainRampRemaining = KE_AUDIO_GAIN_RAMP_FRAMES; stream->LeftGainStep = (stream->TargetLeftGain - stream->CurrentLeftGain) / (float)KE_AUDIO_GAIN_RAMP_FRAMES; stream->RightGainStep = (stream->TargetRightGain - stream->CurrentRightGain) / (float)KE_AUDIO_GAIN_RAMP_FRAMES; } // Advances one stream gain ramp by one output frame. static void KiAudioAdvanceStreamGainLocked(KE_AUDIO_STREAM* stream) { if (!stream->GainRampRemaining) return; stream->CurrentLeftGain += stream->LeftGainStep; stream->CurrentRightGain += stream->RightGainStep; stream->GainRampRemaining--; if (!stream->GainRampRemaining) { stream->CurrentLeftGain = stream->TargetLeftGain; stream->CurrentRightGain = stream->TargetRightGain; } } // Decodes one interleaved source frame into normalized stereo samples. static void KiAudioDecodeSourceFrame(const KE_AUDIO_STREAM* stream, uint32_t offset, float* leftOut, float* rightOut) { uint32_t index = (stream->ReadFrame + offset) % KE_AUDIO_STREAM_RING_FRAMES; const uint8_t* frame = stream->Ring + (size_t)index * stream->FrameBytes; float left = 0.0f; float right = 0.0f; const int16_t* samples = (const int16_t*)frame; left = (float)samples[0] * (1.0f / 32768.0f); right = stream->Format.ChannelCount == 2 ? (float)samples[1] * (1.0f / 32768.0f) : left; *leftOut = left; *rightOut = right; } // Applies one final-handle close request while the stream mutex serializes state. static bool KiAudioApplyCloseRequestLocked(KE_AUDIO_STREAM* stream) { if (!NtlAtomicLoad32(&stream->CloseRequested) || stream->State == KeAudioStreamClosed) return stream->State == KeAudioStreamClosed; uint64_t terminalFrames = stream->FramesPresented + stream->FramesDiscarded; if (terminalFrames < stream->FramesSubmitted) stream->FramesDiscarded += stream->FramesSubmitted - terminalFrames; stream->BufferGeneration++; KiAudioSetStreamStateLocked(stream, KeAudioStreamClosed); stream->ReadFrame = 0; stream->WriteFrame = 0; stream->BufferedFrames = 0; stream->ResamplePositionQ32 = 0; stream->ConsecutiveUnderrunPeriods = 0; stream->DiscontinuityGeneration++; KiAudioRefreshStreamEventLocked(stream); return true; } // Mixes one logical stream into one endpoint period. static uint32_t KiAudioMixStreamPeriod(KE_AUDIO_STREAM* stream, float* mix, uint32_t frames, uint64_t* streamGenerationOut) { KiAudioAcquireMutex(stream->Lock, "KiAudioMixStreamPeriod"); (void)KiAudioApplyCloseRequestLocked(stream); if (!KiAudioStreamStateIsActive(stream->State)) { KiAudioReleaseMutex(stream->Lock); return 0; } uint64_t position = stream->ResamplePositionQ32; uint32_t silentFrames = 0; for (uint32_t outputFrame = 0; outputFrame < frames; outputFrame++) { uint32_t sourceOffset = (uint32_t)(position >> 32); if (sourceOffset >= stream->BufferedFrames) { silentFrames += frames - outputFrame; for (; outputFrame < frames; outputFrame++) KiAudioAdvanceStreamGainLocked(stream); break; } float left0 = 0.0f; float right0 = 0.0f; float left1 = 0.0f; float right1 = 0.0f; KiAudioDecodeSourceFrame(stream, sourceOffset, &left0, &right0); if (sourceOffset + 1u < stream->BufferedFrames) KiAudioDecodeSourceFrame(stream, sourceOffset + 1u, &left1, &right1); else { left1 = left0; right1 = right0; } float fraction = (float)(uint32_t)position * (1.0f / 4294967296.0f); float left = left0 + (left1 - left0) * fraction; float right = right0 + (right1 - right0) * fraction; mix[outputFrame * 2u] += left * stream->CurrentLeftGain; mix[outputFrame * 2u + 1u] += right * stream->CurrentRightGain; KiAudioAdvanceStreamGainLocked(stream); position += stream->ResampleStepQ32; } uint32_t consumed = (uint32_t)(position >> 32); if (consumed > stream->BufferedFrames) consumed = stream->BufferedFrames; stream->ReadFrame = (stream->ReadFrame + consumed) % KE_AUDIO_STREAM_RING_FRAMES; stream->BufferedFrames -= consumed; stream->FramesConsumed += consumed; stream->ResamplePositionQ32 = position - ((uint64_t)consumed << 32); stream->OutputFramesMixed += frames - silentFrames; if (silentFrames && stream->State == KeAudioStreamRunning) { stream->SilenceFramesInserted += silentFrames; stream->Underruns++; stream->ConsecutiveUnderrunPeriods++; } else if (!silentFrames && stream->ConsecutiveUnderrunPeriods) { stream->ConsecutiveUnderrunPeriods = 0; } *streamGenerationOut = stream->BufferGeneration; KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); return consumed; } // Snapshots referenced streams under the nonblocking membership lock. static uint32_t KiAudioSnapshotStreams(IO_AUDIO_ENDPOINT* endpoint, KE_AUDIO_STREAM** streams) { uint32_t count = 0; uintptr_t flags = KeAcquireSpinlock(&endpoint->StreamListLock); for (LIST_ENTRY* link = endpoint->StreamList.Fwd; link != &endpoint->StreamList && count < KE_AUDIO_MAX_STREAMS_PER_ENDPOINT; link = link->Fwd) { KE_AUDIO_STREAM* stream = CONTAINING_RECORD(link, KE_AUDIO_STREAM, EndpointLink); if (ObTryReferenceObject(&stream->Header)) streams[count++] = stream; } KeReleaseSpinlock(&endpoint->StreamListLock, flags); return count; } // Applies the endpoint master ramp and writes one interleaved HDA period. static void KiAudioWriteOutputPeriod(IO_AUDIO_ENDPOINT* endpoint, uint32_t periodIndex, float* mix) { KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioWriteOutputPeriod"); float current = 1.0f; float target = 1.0f; float step = 0.0f; uint32_t remaining = 0; uint64_t generation = 0; uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); current = endpoint->MasterCurrentGain; target = endpoint->MasterTargetGain; step = endpoint->MasterGainStep; remaining = endpoint->MasterRampRemaining; generation = endpoint->MasterGainGeneration; KeReleaseSpinlock(&endpoint->NotifyLock, flags); int16_t* output = (int16_t*)(endpoint->PeriodBuffer + (size_t)periodIndex * endpoint->PeriodStrideBytes); for (uint32_t frame = 0; frame < endpoint->FramesPerPeriod; frame++) { if (remaining) { current += step; remaining--; if (!remaining) current = target; } output[frame * 2u] = KiAudioFloatToOutputSample(mix[frame * 2u] * current); output[frame * 2u + 1u] = KiAudioFloatToOutputSample(mix[frame * 2u + 1u] * current); } flags = KeAcquireSpinlock(&endpoint->NotifyLock); if (endpoint->MasterGainGeneration == generation) { endpoint->MasterCurrentGain = current; endpoint->MasterRampRemaining = remaining; if (!remaining) endpoint->MasterGainStep = 0.0f; } KeReleaseSpinlock(&endpoint->NotifyLock, flags); KiAudioReleaseMutex(endpoint->ControlLock); } // Reserves one free hardware period for passive-level mixing. static bool KiAudioReservePeriod(IO_AUDIO_ENDPOINT* endpoint, uint32_t* periodIndexOut, uint64_t* generationOut) { uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { if (endpoint->Periods[i].State != KE_AUDIO_PERIOD_STATE_FREE) continue; if (endpoint->Periods[i].ContributionCount) KiAudioBug("KiAudioReservePeriod", "free period retains contributions", i); endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FILLING; *periodIndexOut = i; *generationOut = endpoint->Periods[i].Generation; KeReleaseSpinlock(&endpoint->NotifyLock, flags); return true; } KeReleaseSpinlock(&endpoint->NotifyLock, flags); return false; } // Discards one contribution array after its mixed source frames cannot be presented. static void KiAudioDiscardContributions(KE_AUDIO_PERIOD_CONTRIBUTION* contributions, uint32_t contributionCount) { for (uint32_t i = 0; i < contributionCount; i++) { KE_AUDIO_PERIOD_CONTRIBUTION* contribution = &contributions[i]; KE_AUDIO_STREAM* stream = contribution->Stream; KiAudioAcquireMutex(stream->Lock, "KiAudioDiscardContributions"); if (stream->BufferGeneration == contribution->StreamGeneration) { stream->FramesDiscarded += contribution->SourceFrames; if (stream->State == KeAudioStreamDraining && KiAudioDrainIsCompleteLocked(stream)) KiAudioCompleteDrainLocked(stream); KiAudioRefreshStreamEventLocked(stream); } KiAudioReleaseMutex(stream->Lock); ObDereferenceObject(&stream->Header); } } // Commits one mixed period only if no missed deadline invalidated the reservation. static int KiAudioCommitPeriod(IO_AUDIO_ENDPOINT* endpoint, uint32_t periodIndex, uint64_t generation, KE_AUDIO_PERIOD_CONTRIBUTION* contributions, uint32_t contributionCount) { uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); KE_AUDIO_PERIOD_STATE* period = &endpoint->Periods[periodIndex]; int committed = period->State == KE_AUDIO_PERIOD_STATE_FILLING && period->Generation == generation; if (committed) { if (period->ContributionCount) KiAudioBug("KiAudioCommitPeriod", "stale period contributions", periodIndex); for (uint32_t i = 0; i < contributionCount; i++) period->Contributions[i] = contributions[i]; period->ContributionCount = contributionCount; period->State = KE_AUDIO_PERIOD_STATE_READY; endpoint->PeriodsMixed++; } KeReleaseSpinlock(&endpoint->NotifyLock, flags); if (!committed) { memset(endpoint->PeriodBuffer + (size_t)periodIndex * endpoint->PeriodStrideBytes, 0, endpoint->PeriodStrideBytes); KiAudioDiscardContributions(contributions, contributionCount); flags = KeAcquireSpinlock(&endpoint->NotifyLock); if (period->State == KE_AUDIO_PERIOD_STATE_FILLING || period->State == KE_AUDIO_PERIOD_STATE_COMPLETED) period->State = KE_AUDIO_PERIOD_STATE_FREE; KeReleaseSpinlock(&endpoint->NotifyLock, flags); } return committed; } // Mixes one free endpoint period from all currently referenced streams. static int KiAudioMixOnePeriod(IO_AUDIO_ENDPOINT* endpoint) { uint32_t periodIndex = 0; uint64_t generation = 0; if (!KiAudioReservePeriod(endpoint, &periodIndex, &generation)) return 0; float mix[KE_AUDIO_OUTPUT_FRAMES_PER_PERIOD * KE_AUDIO_OUTPUT_CHANNELS]; memset(mix, 0, sizeof(mix)); KE_AUDIO_PERIOD_CONTRIBUTION contributions[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; uint32_t contributionCount = 0; KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); for (uint32_t i = 0; i < streamCount; i++) { uint64_t streamGeneration = 0; uint32_t consumed = KiAudioMixStreamPeriod(streams[i], mix, endpoint->FramesPerPeriod, &streamGeneration); if (consumed) { contributions[contributionCount].Stream = streams[i]; contributions[contributionCount].StreamGeneration = streamGeneration; contributions[contributionCount].SourceFrames = consumed; contributionCount++; } else { ObDereferenceObject(&streams[i]->Header); } } KiAudioWriteOutputPeriod(endpoint, periodIndex, mix); return KiAudioCommitPeriod(endpoint, periodIndex, generation, contributions, contributionCount); } // Applies one hardware-completed period to per-stream presentation counters. static bool KiAudioReapOneCompletedPeriod(IO_AUDIO_ENDPOINT* endpoint) { uint32_t periodIndex = 0; KE_AUDIO_PERIOD_STATE* period = (KE_AUDIO_PERIOD_STATE*)0; uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { if (endpoint->Periods[i].State == KE_AUDIO_PERIOD_STATE_COMPLETED) { periodIndex = i; period = &endpoint->Periods[i]; period->State = KE_AUDIO_PERIOD_STATE_REAPING; break; } } KeReleaseSpinlock(&endpoint->NotifyLock, flags); if (!period) return false; for (uint32_t i = 0; i < period->ContributionCount; i++) { KE_AUDIO_PERIOD_CONTRIBUTION* contribution = &period->Contributions[i]; KE_AUDIO_STREAM* stream = contribution->Stream; KiAudioAcquireMutex(stream->Lock, "KiAudioReapOneCompletedPeriod"); if (stream->BufferGeneration == contribution->StreamGeneration) { stream->FramesPresented += contribution->SourceFrames; if (stream->State == KeAudioStreamDraining && KiAudioDrainIsCompleteLocked(stream)) KiAudioCompleteDrainLocked(stream); KiAudioRefreshStreamEventLocked(stream); } KiAudioReleaseMutex(stream->Lock); ObDereferenceObject(&stream->Header); } memset(endpoint->PeriodBuffer + (size_t)periodIndex * endpoint->PeriodStrideBytes, 0, endpoint->PeriodStrideBytes); flags = KeAcquireSpinlock(&endpoint->NotifyLock); period->ContributionCount = 0; period->State = KE_AUDIO_PERIOD_STATE_FREE; KeReleaseSpinlock(&endpoint->NotifyLock, flags); return true; } // Counts periods ready for hardware consumption. static uint32_t KiAudioReadyPeriodCount(IO_AUDIO_ENDPOINT* endpoint) { uint32_t ready = 0; uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { if (endpoint->Periods[i].State == KE_AUDIO_PERIOD_STATE_READY) ready++; } KeReleaseSpinlock(&endpoint->NotifyLock, flags); return ready; } // Starts provider DMA after at least two periods are prepared. static KSTATUS KiAudioStartHardware(IO_AUDIO_ENDPOINT* endpoint) { KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioStartHardware"); uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); if (endpoint->State != KeAudioEndpointReady) { KSTATUS status = KFAILED(endpoint->FailureStatus) ? endpoint->FailureStatus : NTLSTATUS_DEVICE_REMOVED; KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); KiAudioReleaseMutex(endpoint->ControlLock); return status; } KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); if (endpoint->HardwareRunning || endpoint->ActiveStreamCount == 0) { KeReleaseSpinlock(&endpoint->NotifyLock, flags); KiAudioReleaseMutex(endpoint->ControlLock); return NTLSTATUS_SUCCESS; } endpoint->HardwareRunning = 1; endpoint->IdleDeadlineTicks = 0; KeReleaseSpinlock(&endpoint->NotifyLock, flags); KSTATUS status = NTLSTATUS_SUCCESS; int startAttempted = 0; if (endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) { status = endpoint->Operations.SetGain(endpoint->ProviderContext, endpoint->MasterHardwareGainMilliDb, 0); } if (KSUCCESS(status)) { startAttempted = 1; status = endpoint->Operations.Start(endpoint->ProviderContext); } if (!KSUCCESS(status) && startAttempted) endpoint->Operations.Stop(endpoint->ProviderContext); if (!KSUCCESS(status)) { flags = KeAcquireSpinlock(&endpoint->NotifyLock); endpoint->HardwareRunning = 0; KeReleaseSpinlock(&endpoint->NotifyLock, flags); KiAudioReleaseMutex(endpoint->ControlLock); return status; } flags = KeAcquireSpinlock(&endpoint->NotifyLock); endpoint->HardwareStarts++; KeReleaseSpinlock(&endpoint->NotifyLock, flags); KiAudioReleaseMutex(endpoint->ControlLock); return NTLSTATUS_SUCCESS; } // Discards one period's unpresented stream contributions during hardware teardown. static void KiAudioDiscardPeriodContributions(KE_AUDIO_PERIOD_STATE* period) { KiAudioDiscardContributions(period->Contributions, period->ContributionCount); period->ContributionCount = 0; } // Stops provider DMA and returns every period to a zeroed free state. static void KiAudioStopHardware(IO_AUDIO_ENDPOINT* endpoint, int hardwareAccessible) { KiAudioAcquireMutex(endpoint->ControlLock, "KiAudioStopHardware"); uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); bool wasRunning = endpoint->HardwareRunning != 0; endpoint->HardwareRunning = 0; endpoint->IdleDeadlineTicks = 0; KeReleaseSpinlock(&endpoint->NotifyLock, flags); if (wasRunning && hardwareAccessible) { if (endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_HARDWARE_GAIN) { KSTATUS muteStatus = endpoint->Operations.SetGain( endpoint->ProviderContext, endpoint->MasterHardwareGainMilliDb, 1); if (!KSUCCESS(muteStatus)) { DbgPrint("audio: endpoint %llu hardware mute failed status=0x%x\n", (unsigned long long)endpoint->EndpointId, muteStatus); } } endpoint->Operations.Stop(endpoint->ProviderContext); } while (KiAudioReapOneCompletedPeriod(endpoint)) { } for (uint32_t i = 0; i < endpoint->PeriodCount; i++) KiAudioDiscardPeriodContributions(&endpoint->Periods[i]); memset(endpoint->PeriodBuffer, 0, endpoint->PeriodBufferBytes); flags = KeAcquireSpinlock(&endpoint->NotifyLock); for (uint32_t i = 0; i < endpoint->PeriodCount; i++) { endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FREE; endpoint->Periods[i].Generation++; } endpoint->MasterCurrentGain = endpoint->MasterTargetGain; endpoint->MasterGainStep = 0.0f; endpoint->MasterRampRemaining = 0; endpoint->MasterGainGeneration++; if (wasRunning) endpoint->HardwareStops++; KeReleaseSpinlock(&endpoint->NotifyLock, flags); KiAudioReleaseMutex(endpoint->ControlLock); } // Fails every logical stream while preserving final counters for later queries. static void KiAudioFailAllStreams(IO_AUDIO_ENDPOINT* endpoint, KSTATUS status) { KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); for (uint32_t i = 0; i < streamCount; i++) { KE_AUDIO_STREAM* stream = streams[i]; KiAudioAcquireMutex(stream->Lock, "KiAudioFailAllStreams"); if (stream->State != KeAudioStreamClosed) { stream->FramesDiscarded += stream->BufferedFrames; stream->BufferedFrames = 0; stream->ReadFrame = 0; stream->WriteFrame = 0; stream->ResamplePositionQ32 = 0; KiAudioSetStreamStateLocked(stream, KeAudioStreamFailed); stream->FailureStatus = status; stream->BufferGeneration++; stream->DiscontinuityGeneration++; KiAudioRefreshStreamEventLocked(stream); } KiAudioReleaseMutex(stream->Lock); ObDereferenceObject(&stream->Header); } uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); if (endpoint->ActiveStreamCount != 0) { uint32_t active = endpoint->ActiveStreamCount; KeReleaseSpinlock(&endpoint->NotifyLock, flags); KiAudioBug("KiAudioFailAllStreams", "active streams remained after failure", active); } endpoint->IdleDeadlineTicks = 0; KeReleaseSpinlock(&endpoint->NotifyLock, flags); } // Applies hardware deadline discontinuities to every live logical stream at passive level. static void KiAudioApplyPendingDiscontinuities(IO_AUDIO_ENDPOINT* endpoint) { uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); uint64_t pending = endpoint->PendingDiscontinuities; endpoint->PendingDiscontinuities = 0; KeReleaseSpinlock(&endpoint->NotifyLock, flags); if (!pending) return; KE_AUDIO_STREAM* streams[KE_AUDIO_MAX_STREAMS_PER_ENDPOINT]; uint32_t streamCount = KiAudioSnapshotStreams(endpoint, streams); for (uint32_t i = 0; i < streamCount; i++) { KE_AUDIO_STREAM* stream = streams[i]; KiAudioAcquireMutex(stream->Lock, "KiAudioApplyPendingDiscontinuities"); if (KiAudioStreamStateIsActive(stream->State)) { stream->Underruns += pending; stream->SilenceFramesInserted += pending * endpoint->FramesPerPeriod; stream->DiscontinuityGeneration += pending; KiAudioRefreshStreamEventLocked(stream); } KiAudioReleaseMutex(stream->Lock); ObDereferenceObject(&stream->Header); } } // Returns one coherent endpoint lifecycle snapshot. static void KiAudioSnapshotLifecycle(IO_AUDIO_ENDPOINT* endpoint, KE_AUDIO_LIFECYCLE_SNAPSHOT* snapshot) { uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); snapshot->State = endpoint->State; snapshot->FailureStatus = endpoint->FailureStatus; snapshot->RemovalMode = endpoint->RemovalMode; snapshot->SkipProviderStop = endpoint->SkipProviderStop; KeReleaseSpinlock(&gAudioRegistryLock, flags); } // Processes all currently available period and lifecycle work. static bool KiAudioProcessEndpoint(IO_AUDIO_ENDPOINT* endpoint, uint64_t* waitMillisecondsOut) { *waitMillisecondsOut = (uint64_t)-1; while (KiAudioReapOneCompletedPeriod(endpoint)) { } KiAudioApplyPendingDiscontinuities(endpoint); KE_AUDIO_LIFECYCLE_SNAPSHOT lifecycle; KiAudioSnapshotLifecycle(endpoint, &lifecycle); KSTATUS failure = lifecycle.FailureStatus; if (lifecycle.State == KeAudioEndpointStopping || lifecycle.State == KeAudioEndpointFailed || lifecycle.State == KeAudioEndpointRemoved) { bool accessible = !lifecycle.SkipProviderStop && lifecycle.RemovalMode != IoAudioSurpriseRemoval; KiAudioStopHardware(endpoint, accessible); if (lifecycle.State == KeAudioEndpointRemoved && KSUCCESS(failure)) failure = NTLSTATUS_DEVICE_REMOVED; if (KSUCCESS(failure)) failure = NTLSTATUS_DEVICE_ERROR; KiAudioFailAllStreams(endpoint, failure); return false; } uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); uint32_t active = endpoint->ActiveStreamCount; bool running = endpoint->HardwareRunning != 0; uint64_t idleDeadline = endpoint->IdleDeadlineTicks; KeReleaseSpinlock(&endpoint->NotifyLock, flags); if (active && !running) { while (KiAudioReadyPeriodCount(endpoint) < 2u && KiAudioMixOnePeriod(endpoint)) { } if (KiAudioReadyPeriodCount(endpoint) >= 2u) { KSTATUS start = KiAudioStartHardware(endpoint); if (!KSUCCESS(start)) { uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointFailed); endpoint->FailureStatus = start; KiAudioSelectDefaultEndpointLocked(); KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); KeSignalEvent(endpoint->WorkEvent); return true; } } } flags = KeAcquireSpinlock(&endpoint->NotifyLock); active = endpoint->ActiveStreamCount; running = endpoint->HardwareRunning; KeReleaseSpinlock(&endpoint->NotifyLock, flags); if (active || running) { while (KiAudioMixOnePeriod(endpoint)) { } } flags = KeAcquireSpinlock(&endpoint->NotifyLock); active = endpoint->ActiveStreamCount; running = endpoint->HardwareRunning; idleDeadline = endpoint->IdleDeadlineTicks; KeReleaseSpinlock(&endpoint->NotifyLock, flags); if (!active && running) { uint64_t now = KiAudioQueryTicks(); if (!idleDeadline) { flags = KeAcquireSpinlock(&endpoint->NotifyLock); endpoint->IdleDeadlineTicks = now + (uint64_t)KE_AUDIO_IDLE_STOP_MILLISECONDS * KE_AUDIO_TICKS_PER_MILLISECOND; idleDeadline = endpoint->IdleDeadlineTicks; KeReleaseSpinlock(&endpoint->NotifyLock, flags); } if (now >= idleDeadline) { KiAudioStopHardware(endpoint, 1); } else { uint64_t remainingTicks = idleDeadline - now; *waitMillisecondsOut = (remainingTicks + KE_AUDIO_TICKS_PER_MILLISECOND - 1u) / KE_AUDIO_TICKS_PER_MILLISECOND; if (!*waitMillisecondsOut) *waitMillisecondsOut = 1; } } return true; } // Runs one endpoint's real-time mixer and provider lifecycle loop. static void KiAudioMixerThread(void* argument) { IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)argument; while (true) { KeClearEvent(endpoint->WorkEvent); uint64_t waitMilliseconds = (uint64_t)-1; if (!KiAudioProcessEndpoint(endpoint, &waitMilliseconds)) break; KSTATUS waitStatus = NTLSTATUS_SUCCESS; if (waitMilliseconds == (uint64_t)-1) { waitStatus = KeWaitForEvent(endpoint->WorkEvent, 0); } else { EVENT* events[1] = { endpoint->WorkEvent }; uint32_t selected = 0; waitStatus = KeWaitForMultipleEvents(events, 1, 0, waitMilliseconds, &selected); } if (waitStatus != NTLSTATUS_SUCCESS && waitStatus != NTLSTATUS_TIMEOUT) KiAudioBug("KiAudioMixerThread", "work wait failed", waitStatus); } KeSignalEvent(endpoint->MixerExitEvent); ObDereferenceObject(&endpoint->Header); } // Destroys one fully unregistered endpoint after its mixer and streams leave. static void KiAudioEndpointDestroy(OBJECT* object) { IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(object, IO_AUDIO_ENDPOINT, Header); if (endpoint->Registered || endpoint->MixerThread || !LIST_IS_EMPTY(&endpoint->StreamList) || endpoint->HardwareRunning || endpoint->ActiveStreamCount) KiAudioBug("KiAudioEndpointDestroy", "live endpoint reached destruction", endpoint->EndpointId); if (endpoint->Device) IoDereferenceDeviceObject(endpoint->Device); if (endpoint->ControlLock) ObDereferenceObject(&endpoint->ControlLock->Header); if (endpoint->WorkEvent) ObDereferenceObject(&endpoint->WorkEvent->Header); if (endpoint->MixerExitEvent) ObDereferenceObject(&endpoint->MixerExitEvent->Header); } // Records one process handle that owns the stream. void KeAudioStreamHandleOpened(KE_AUDIO_STREAM* stream) { if (!stream) return; int32_t handleCount = NtlAtomicIncrement32(&stream->ProcessHandleCount); if (handleCount <= 0) KiAudioBug("KeAudioStreamHandleOpened", "invalid process handle count", handleCount); } // Requests passive stream closure when the final process handle disappears. void KeAudioStreamHandleClosed(KE_AUDIO_STREAM* stream) { if (!stream) return; int32_t handleCount = NtlAtomicDecrement32(&stream->ProcessHandleCount); if (handleCount < 0) KiAudioBug("KeAudioStreamHandleClosed", "process handle underflow", handleCount); if (handleCount != 0) return; NtlAtomicExchange32(&stream->CloseRequested, 1); KeSignalEvent(stream->WaitEvent); if (stream->Endpoint) KeSignalEvent(stream->Endpoint->WorkEvent); } // Destroys one exclusively owned stream without entering blocking primitives. static void KiAudioStreamDestroy(OBJECT* object) { KE_AUDIO_STREAM* stream = CONTAINING_RECORD(object, KE_AUDIO_STREAM, Header); IO_AUDIO_ENDPOINT* endpoint = stream->Endpoint; if (endpoint) { uintptr_t flags = KeAcquireSpinlock(&endpoint->StreamListLock); if (stream->EndpointLink.Fwd != &stream->EndpointLink || stream->EndpointLink.Bwd != &stream->EndpointLink) KeListRemoveChecked(&stream->EndpointLink); KeReleaseSpinlock(&endpoint->StreamListLock, flags); } if (stream->Lock && (stream->Lock->Owner || !LIST_IS_EMPTY(&stream->Lock->WaitList))) KiAudioBug("KiAudioStreamDestroy", "live stream mutex reached destruction", (uint64_t)(uintptr_t)stream->Lock->Owner); if (!KiAudioStreamTransitionAllowed(stream->State, KeAudioStreamClosed)) KiAudioBug("KiAudioStreamDestroy", "invalid terminal stream state", stream->State); bool wasActive = KiAudioStreamStateIsActive(stream->State); stream->State = KeAudioStreamClosed; if (wasActive) KiAudioAdjustActiveStreams(endpoint, -1); if (stream->Ring) MmFreePool(stream->Ring); if (stream->WaitEvent) ObDereferenceObject(&stream->WaitEvent->Header); if (stream->Lock) ObDereferenceObject(&stream->Lock->Header); if (endpoint) ObDereferenceObject(&endpoint->Header); NtlAtomicIncrement64(&gAudioCoreStats.StreamsDestroyed); } // Registers one hardware-provider endpoint. static KSTATUS KiAudioRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, IO_AUDIO_ENDPOINT** outEndpoint) { if (!outEndpoint) return NTLSTATUS_INVALID_PARAMETER; *outEndpoint = (IO_AUDIO_ENDPOINT*)0; KSTATUS status = KiAudioValidateRegistration(registration); if (!KSUCCESS(status)) return status; IO_AUDIO_ENDPOINT* endpoint = (IO_AUDIO_ENDPOINT*)ObAllocateObject( ObjectTypeAudioEndpoint, sizeof(IO_AUDIO_ENDPOINT)); if (!endpoint) return NTLSTATUS_NO_MEMORY; endpoint->Header.Destroy = KiAudioEndpointDestroy; LIST_INIT(&endpoint->RegistryLink); LIST_INIT(&endpoint->StreamList); endpoint->StreamListLock = (SPINLOCK)SPINLOCK_INIT; endpoint->NotifyLock = (SPINLOCK)SPINLOCK_INIT; endpoint->Kind = registration->Kind; endpoint->State = KeAudioEndpointStarting; endpoint->Capabilities = registration->Capabilities; endpoint->Available = registration->InitiallyAvailable ? 1u : 0u; endpoint->Device = registration->Device; endpoint->ProviderContext = registration->ProviderContext; endpoint->Operations = registration->Operations; endpoint->PeriodBuffer = (uint8_t*)registration->PeriodBuffer; endpoint->PeriodBufferBytes = registration->PeriodBufferBytes; endpoint->PeriodCount = registration->PeriodCount; endpoint->FramesPerPeriod = registration->FramesPerPeriod; endpoint->PeriodStrideBytes = registration->PeriodStrideBytes; endpoint->HardwareFormat = registration->HardwareFormat; endpoint->HardwareGainMinimumMilliDb = registration->HardwareGainMinimumMilliDb; endpoint->HardwareGainMaximumMilliDb = registration->HardwareGainMaximumMilliDb; endpoint->HardwareGainStepMilliDb = registration->HardwareGainStepMilliDb; endpoint->MasterRequestedVolume = 1.0f; endpoint->MasterHardwareGainMilliDb = KiAudioHighestSafeHardwareGain(endpoint); endpoint->MasterCurrentGain = KiAudioSoftwareGainForVolume(endpoint, 1.0f); endpoint->MasterTargetGain = endpoint->MasterCurrentGain; endpoint->RemovalMode = IoAudioOrderlyRemoval; memset(endpoint->PeriodBuffer, 0, endpoint->PeriodBufferBytes); for (uint32_t i = 0; i < endpoint->PeriodCount; i++) endpoint->Periods[i].State = KE_AUDIO_PERIOD_STATE_FREE; if (endpoint->Device) IoReferenceDeviceObject(endpoint->Device); status = KeMutexCreate(&endpoint->ControlLock); if (!KSUCCESS(status)) { ObDereferenceObject(&endpoint->Header); return status; } status = KeEventCreate(EventManualReset, 0, &endpoint->WorkEvent); if (!KSUCCESS(status)) { ObDereferenceObject(&endpoint->Header); return status; } status = KeEventCreate(EventManualReset, 0, &endpoint->MixerExitEvent); if (!KSUCCESS(status)) { ObDereferenceObject(&endpoint->Header); return status; } ObReferenceObject(&endpoint->Header); const KE_THREAD_CREATE_PARAMETERS mixerParameters = KE_THREAD_CREATE_NAMED_REALTIME(KE_AUDIO_MIXER_REALTIME_PRIORITY, "audio.mixer"); status = KeCreateKernelThread((uintptr_t)KiAudioMixerThread, (uintptr_t)endpoint, &mixerParameters, &endpoint->MixerThread); if (!KSUCCESS(status)) { ObDereferenceObject(&endpoint->Header); ObDereferenceObject(&endpoint->Header); return status; } uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); if (gAudioEndpointCount >= KE_AUDIO_MAX_ENDPOINTS) { KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping); endpoint->FailureStatus = NTLSTATUS_NO_MEMORY; KeReleaseSpinlock(&gAudioRegistryLock, flags); KeSignalEvent(endpoint->WorkEvent); (void)KeWaitForEvent(endpoint->MixerExitEvent, 0); ObDereferenceObject(&endpoint->MixerThread->Header); endpoint->MixerThread = (THREAD*)0; ObDereferenceObject(&endpoint->Header); return NTLSTATUS_NO_MEMORY; } KSTATUS idStatus = KeAllocateId32(&gAudioEndpointIds, &endpoint->EndpointId); if (!KSUCCESS(idStatus)) { KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping); endpoint->FailureStatus = idStatus; KeReleaseSpinlock(&gAudioRegistryLock, flags); KeSignalEvent(endpoint->WorkEvent); (void)KeWaitForEvent(endpoint->MixerExitEvent, 0); ObDereferenceObject(&endpoint->MixerThread->Header); endpoint->MixerThread = (THREAD*)0; ObDereferenceObject(&endpoint->Header); return idStatus; } KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointReady); endpoint->Registered = 1; ObReferenceObject(&endpoint->Header); KeListInsertTailChecked(&gAudioEndpointList, &endpoint->RegistryLink); gAudioEndpointCount++; KiAudioSelectDefaultEndpointLocked(); KeReleaseSpinlock(&gAudioRegistryLock, flags); NtlAtomicIncrement64(&gAudioCoreStats.EndpointsRegistered); *outEndpoint = endpoint; return NTLSTATUS_SUCCESS; } // Implements the provider-facing endpoint registration operation. static KSTATUS KiAudioDdkRegisterEndpoint(const IO_AUDIO_ENDPOINT_REGISTRATION* registration, IO_AUDIO_ENDPOINT** outEndpoint) { return KiAudioRegisterEndpoint(registration, outEndpoint); } // Removes one endpoint and waits until no provider operation can run again. static void KiAudioDdkUnregisterEndpoint(IO_AUDIO_ENDPOINT* endpoint, IO_AUDIO_REMOVAL_MODE mode) { if (!endpoint || (mode != IoAudioOrderlyRemoval && mode != IoAudioSurpriseRemoval)) KiAudioBug("KiAudioDdkUnregisterEndpoint", "invalid unregister request", mode); uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); if (!endpoint->Registered) { KeReleaseSpinlock(&gAudioRegistryLock, flags); KiAudioBug("KiAudioDdkUnregisterEndpoint", "duplicate unregister", endpoint->EndpointId); } KeListRemoveChecked(&endpoint->RegistryLink); endpoint->Registered = 0; KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointStopping); if (KSUCCESS(endpoint->FailureStatus)) endpoint->FailureStatus = NTLSTATUS_DEVICE_REMOVED; endpoint->RemovalMode = mode; endpoint->SkipProviderStop = mode == IoAudioSurpriseRemoval ? 1u : 0u; gAudioEndpointCount--; KiAudioSelectDefaultEndpointLocked(); KeReleaseSpinlock(&gAudioRegistryLock, flags); KeSignalEvent(endpoint->WorkEvent); (void)KeWaitForEvent(endpoint->MixerExitEvent, 0); flags = KeAcquireSpinlock(&gAudioRegistryLock); KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointRemoved); KeReleaseSpinlock(&gAudioRegistryLock, flags); if (!endpoint->MixerThread) KiAudioBug("KiAudioDdkUnregisterEndpoint", "missing mixer thread", endpoint->EndpointId); ObDereferenceObject(&endpoint->MixerThread->Header); endpoint->MixerThread = (THREAD*)0; NtlAtomicIncrement64(&gAudioCoreStats.EndpointsUnregistered); ObDereferenceObject(&endpoint->Header); ObDereferenceObject(&endpoint->Header); } // Advances one provider ring period and wakes the real-time mixer. static void KiAudioDdkNotifyPeriod(IO_AUDIO_ENDPOINT* endpoint, uint32_t completedPeriodIndex, uint64_t presentedFrames, uint64_t hardwarePositionFrames, uint64_t interruptCount, uint64_t timestampTicks) { if (!endpoint || completedPeriodIndex >= endpoint->PeriodCount) KiAudioBug("KiAudioDdkNotifyPeriod", "invalid period notification", completedPeriodIndex); uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); KE_AUDIO_PERIOD_STATE* period = &endpoint->Periods[completedPeriodIndex]; if (!endpoint->HardwareRunning) { endpoint->SpuriousNotifications++; KeReleaseSpinlock(&endpoint->NotifyLock, flags); return; } int missedDeadline = period->State != KE_AUDIO_PERIOD_STATE_READY; if (missedDeadline) { endpoint->Underruns++; endpoint->DiscontinuityGeneration++; endpoint->SilenceFramesInserted += endpoint->FramesPerPeriod; endpoint->PendingDiscontinuities++; } if (period->State == KE_AUDIO_PERIOD_STATE_COMPLETED || period->State == KE_AUDIO_PERIOD_STATE_REAPING) { endpoint->SpuriousNotifications++; } else { period->Generation++; period->State = KE_AUDIO_PERIOD_STATE_COMPLETED; } endpoint->PeriodsCompleted++; if (presentedFrames < endpoint->FramesPresented) { endpoint->DiscontinuityGeneration++; endpoint->PendingDiscontinuities++; } else { endpoint->FramesPresented = presentedFrames; } if (hardwarePositionFrames < endpoint->HardwarePositionFrames || interruptCount < endpoint->InterruptCount) { endpoint->DiscontinuityGeneration++; endpoint->PendingDiscontinuities++; } else { endpoint->HardwarePositionFrames = hardwarePositionFrames; endpoint->InterruptCount = interruptCount; } endpoint->LastPresentationTimestampTicks = timestampTicks; KeReleaseSpinlock(&endpoint->NotifyLock, flags); KeSignalEvent(endpoint->WorkEvent); } // Updates one endpoint's jack-derived availability and default selection. static void KiAudioDdkSetEndpointAvailable(IO_AUDIO_ENDPOINT* endpoint, bool available) { if (!endpoint || !(endpoint->Capabilities & IO_AUDIO_ENDPOINT_CAP_JACK_SENSE)) KiAudioBug("KiAudioDdkSetEndpointAvailable", "invalid availability update", 0); uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); if (endpoint->State == KeAudioEndpointReady) { endpoint->Available = available ? 1u : 0u; KiAudioSelectDefaultEndpointLocked(); } KeReleaseSpinlock(&gAudioRegistryLock, flags); } // Fails one endpoint from provider completion or recovery code. static void KiAudioDdkReportEndpointFailure(IO_AUDIO_ENDPOINT* endpoint, KSTATUS status, uint32_t flags) { if (!endpoint || !KFAILED(status) || (flags & ~IO_AUDIO_FAILURE_VALID_MASK) != 0) KiAudioBug("KiAudioDdkReportEndpointFailure", "invalid failure report", status); uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); if (flags & IO_AUDIO_FAILURE_HARDWARE_INACCESSIBLE) endpoint->SkipProviderStop = 1; if (endpoint->State == KeAudioEndpointReady) { KiAudioSetEndpointStateLocked(endpoint, KeAudioEndpointFailed); endpoint->FailureStatus = status; endpoint->Available = 0; KiAudioSelectDefaultEndpointLocked(); } KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); DbgPrint("audio: endpoint %llu failed status=0x%x flags=0x%x\n", (unsigned long long)endpoint->EndpointId, status, flags); KeSignalEvent(endpoint->WorkEvent); } static const NTL_DDK_AUDIO gAudioDdk = { .RegisterEndpoint = KiAudioDdkRegisterEndpoint, .UnregisterEndpoint = KiAudioDdkUnregisterEndpoint, .NotifyPeriod = KiAudioDdkNotifyPeriod, .SetEndpointAvailable = KiAudioDdkSetEndpointAvailable, .ReportEndpointFailure = KiAudioDdkReportEndpointFailure, }; // Initializes the one kernel audio registry. void KeAudioInitialize(void) { if (gAudioInitialized) KiAudioBug("KeAudioInitialize", "duplicate initialization", 0); LIST_INIT(&gAudioEndpointList); gAudioInitialized = 1; } // Returns the immutable provider-facing audio table. const NTL_DDK_AUDIO* KeAudioGetDdk(void) { return &gAudioDdk; } // Returns the current default endpoint identifier. uint32_t KeAudioGetDefaultEndpointId(void) { uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); uint32_t endpointId = gAudioDefaultEndpointId; KeReleaseSpinlock(&gAudioRegistryLock, flags); return endpointId; } // Copies endpoint snapshots without retaining registry locks across passive operations. KSTATUS KeAudioQueryEndpoints(KE_AUDIO_ENDPOINT_INFO* endpoints, uint32_t capacity, uint32_t* countOut) { if (!countOut || (capacity && !endpoints)) return NTLSTATUS_INVALID_PARAMETER; IO_AUDIO_ENDPOINT* references[KE_AUDIO_MAX_ENDPOINTS]; uint32_t count = 0; uint32_t defaultId = 0; uintptr_t flags = KeAcquireSpinlock(&gAudioRegistryLock); defaultId = gAudioDefaultEndpointId; for (LIST_ENTRY* link = gAudioEndpointList.Fwd; link != &gAudioEndpointList && count < KE_AUDIO_MAX_ENDPOINTS; link = link->Fwd) { IO_AUDIO_ENDPOINT* endpoint = CONTAINING_RECORD(link, IO_AUDIO_ENDPOINT, RegistryLink); if (ObTryReferenceObject(&endpoint->Header)) references[count++] = endpoint; } KeReleaseSpinlock(&gAudioRegistryLock, flags); *countOut = count; uint32_t copyCount = capacity < count ? capacity : count; for (uint32_t i = 0; i < count; i++) { IO_AUDIO_ENDPOINT* endpoint = references[i]; if (i < copyCount) { KE_AUDIO_ENDPOINT_INFO* info = &endpoints[i]; memset(info, 0, sizeof(*info)); flags = KeAcquireSpinlock(&gAudioRegistryLock); info->EndpointId = endpoint->EndpointId; info->Kind = endpoint->Kind; info->State = endpoint->State; info->Capabilities = endpoint->Capabilities; info->HardwareFormat = endpoint->HardwareFormat; info->PeriodCount = endpoint->PeriodCount; info->FramesPerPeriod = endpoint->FramesPerPeriod; info->Available = endpoint->Available; info->IsDefault = endpoint->EndpointId == defaultId; info->FailureStatus = endpoint->FailureStatus; KeReleaseSpinlock(&gAudioRegistryLock, flags); uintptr_t streamFlags = KeAcquireSpinlock(&endpoint->StreamListLock); for (LIST_ENTRY* link = endpoint->StreamList.Fwd; link != &endpoint->StreamList; link = link->Fwd) info->StreamCount++; KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); flags = KeAcquireSpinlock(&endpoint->NotifyLock); info->ActiveStreamCount = endpoint->ActiveStreamCount; info->HardwareRunning = endpoint->HardwareRunning; info->MasterMuted = endpoint->MasterMuted; info->MasterHardwareGainMilliDb = endpoint->MasterHardwareGainMilliDb; info->MasterRequestedVolume = endpoint->MasterRequestedVolume; info->PeriodsMixed = endpoint->PeriodsMixed; info->PeriodsCompleted = endpoint->PeriodsCompleted; info->FramesPresented = endpoint->FramesPresented; info->HardwarePositionFrames = endpoint->HardwarePositionFrames; info->InterruptCount = endpoint->InterruptCount; info->LastPresentationTimestampTicks = endpoint->LastPresentationTimestampTicks; info->SilenceFramesInserted = endpoint->SilenceFramesInserted; info->Underruns = endpoint->Underruns; info->DiscontinuityGeneration = endpoint->DiscontinuityGeneration; KeReleaseSpinlock(&endpoint->NotifyLock, flags); } ObDereferenceObject(&endpoint->Header); } return capacity < count ? NTLSTATUS_BUFFER_TOO_SMALL : NTLSTATUS_SUCCESS; } // Creates one logical render stream with a fixed-capacity source ring. KSTATUS KeAudioCreateStream(uint32_t endpointId, const IO_AUDIO_FORMAT* format, KE_AUDIO_STREAM** outStream) { if (!outStream || !KiAudioValidateSourceFormat(format)) return NTLSTATUS_INVALID_PARAMETER; *outStream = (KE_AUDIO_STREAM*)0; IO_AUDIO_ENDPOINT* endpoint = KiAudioReferenceEndpointById(endpointId); if (!endpoint) return NTLSTATUS_NOT_FOUND; KE_AUDIO_STREAM* stream = (KE_AUDIO_STREAM*)ObAllocateObject( ObjectTypeAudioStream, sizeof(KE_AUDIO_STREAM)); if (!stream) { ObDereferenceObject(&endpoint->Header); return NTLSTATUS_NO_MEMORY; } stream->Header.Destroy = KiAudioStreamDestroy; LIST_INIT(&stream->EndpointLink); stream->Endpoint = endpoint; stream->State = KeAudioStreamCreated; stream->Format = *format; stream->FrameBytes = KiAudioFrameBytes(format); stream->ResampleStepQ32 = ((uint64_t)format->SampleRate << 32) / KE_AUDIO_OUTPUT_SAMPLE_RATE; stream->WaitMask = KE_AUDIO_WAIT_WRITABLE; stream->WaitWritableFrames = 1; stream->Volume = 1.0f; stream->CurrentLeftGain = 1.0f; stream->CurrentRightGain = 1.0f; stream->TargetLeftGain = 1.0f; stream->TargetRightGain = 1.0f; KSTATUS status = KeMutexCreate(&stream->Lock); if (!KSUCCESS(status)) { ObDereferenceObject(&stream->Header); return status; } status = KeEventCreate(EventManualReset, 1, &stream->WaitEvent); if (!KSUCCESS(status)) { ObDereferenceObject(&stream->Header); return status; } stream->Ring = (uint8_t*)MmAllocatePoolTag( (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes, KE_AUDIO_RING_TAG); if (!stream->Ring) { ObDereferenceObject(&stream->Header); return NTLSTATUS_NO_MEMORY; } memset(stream->Ring, 0, (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes); uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); int endpointReady = endpoint->Registered && endpoint->State == KeAudioEndpointReady; uint32_t streamCount = 0; uintptr_t streamFlags = KeAcquireSpinlock(&endpoint->StreamListLock); if (endpointReady) { for (LIST_ENTRY* link = endpoint->StreamList.Fwd; link != &endpoint->StreamList; link = link->Fwd) streamCount++; } if (!endpointReady || streamCount >= KE_AUDIO_MAX_STREAMS_PER_ENDPOINT) { KSTATUS createStatus = !endpointReady ? (KFAILED(endpoint->FailureStatus) ? endpoint->FailureStatus : NTLSTATUS_DEVICE_REMOVED) : NTLSTATUS_NO_MEMORY; KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); ObDereferenceObject(&stream->Header); return createStatus; } KeListInsertTailChecked(&endpoint->StreamList, &stream->EndpointLink); KeReleaseSpinlock(&endpoint->StreamListLock, streamFlags); KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); NtlAtomicIncrement64(&gAudioCoreStats.StreamsCreated); *outStream = stream; return NTLSTATUS_SUCCESS; } // Returns the stream event used by future waitable handles. EVENT* KeAudioStreamWaitEvent(KE_AUDIO_STREAM* stream) { return stream ? stream->WaitEvent : (EVENT*)0; } // Arms one wait condition while holding the stream state that drives its event. KSTATUS KeAudioArmStreamWait(KE_AUDIO_STREAM* stream, const KE_AUDIO_STREAM_WAIT_CONDITION* condition) { if (!stream || !condition || !condition->Mask || (condition->Mask & ~KE_AUDIO_WAIT_VALID_MASK) != 0 || ((condition->Mask & KE_AUDIO_WAIT_WRITABLE) && (!condition->WritableFrames || condition->WritableFrames > KE_AUDIO_STREAM_RING_FRAMES))) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioArmStreamWait"); if (stream->State == KeAudioStreamClosed) { KiAudioReleaseMutex(stream->Lock); return NTLSTATUS_OBJECT_DEAD; } stream->WaitMask = condition->Mask; stream->WaitWritableFrames = condition->WritableFrames; stream->WaitConsumedFrames = condition->ConsumedFrames; stream->WaitPresentedFrames = condition->PresentedFrames; stream->WaitDrainedGeneration = condition->DrainedGeneration; stream->WaitDiscontinuityGeneration = condition->DiscontinuityGeneration; KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); return NTLSTATUS_SUCCESS; } // Copies a bounded source-frame run into one stream ring. KSTATUS KeAudioWriteStream(KE_AUDIO_STREAM* stream, const void* frames, uint32_t requestedFrames, uint32_t* writtenFramesOut) { if (!stream || !writtenFramesOut || (requestedFrames && !frames)) return NTLSTATUS_INVALID_PARAMETER; *writtenFramesOut = 0; if (!requestedFrames) return NTLSTATUS_SUCCESS; KiAudioAcquireMutex(stream->Lock, "KeAudioWriteStream"); if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; KiAudioReleaseMutex(stream->Lock); return status; } if (stream->State == KeAudioStreamDraining) { KiAudioReleaseMutex(stream->Lock); return NTLSTATUS_INVALID_PARAMETER; } uint32_t writable = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; if (!writable) { KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); return NTLSTATUS_WOULD_BLOCK; } uint32_t toWrite = requestedFrames < writable ? requestedFrames : writable; uint32_t firstFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->WriteFrame; if (firstFrames > toWrite) firstFrames = toWrite; size_t firstBytes = (size_t)firstFrames * stream->FrameBytes; memcpy(stream->Ring + (size_t)stream->WriteFrame * stream->FrameBytes, frames, firstBytes); uint32_t secondFrames = toWrite - firstFrames; if (secondFrames) { memcpy(stream->Ring, (const uint8_t*)frames + firstBytes, (size_t)secondFrames * stream->FrameBytes); } stream->WriteFrame = (stream->WriteFrame + toWrite) % KE_AUDIO_STREAM_RING_FRAMES; stream->BufferedFrames += toWrite; stream->FramesSubmitted += toWrite; *writtenFramesOut = toWrite; KiAudioRefreshStreamEventLocked(stream); bool active = KiAudioStreamStateIsActive(stream->State); KiAudioReleaseMutex(stream->Lock); if (active) KeSignalEvent(stream->Endpoint->WorkEvent); return NTLSTATUS_SUCCESS; } // Starts one created or stopped stream. KSTATUS KeAudioStartStream(KE_AUDIO_STREAM* stream) { if (!stream) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioStartStream"); if (stream->State != KeAudioStreamCreated && stream->State != KeAudioStreamStopped) { KSTATUS status = stream->State == KeAudioStreamFailed ? stream->FailureStatus : NTLSTATUS_INVALID_PARAMETER; KiAudioReleaseMutex(stream->Lock); return status; } KiAudioSetStreamStateLocked(stream, KeAudioStreamRunning); stream->FailureStatus = NTLSTATUS_SUCCESS; KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); NtlAtomicIncrement64(&gAudioCoreStats.Starts); return NTLSTATUS_SUCCESS; } // Pauses one running stream without discarding source frames. KSTATUS KeAudioPauseStream(KE_AUDIO_STREAM* stream) { if (!stream) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioPauseStream"); if (stream->State != KeAudioStreamRunning) { KSTATUS status = stream->State == KeAudioStreamFailed ? stream->FailureStatus : NTLSTATUS_INVALID_PARAMETER; KiAudioReleaseMutex(stream->Lock); return status; } KiAudioSetStreamStateLocked(stream, KeAudioStreamPaused); KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); NtlAtomicIncrement64(&gAudioCoreStats.Pauses); return NTLSTATUS_SUCCESS; } // Resumes one paused stream. KSTATUS KeAudioResumeStream(KE_AUDIO_STREAM* stream) { if (!stream) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioResumeStream"); if (stream->State != KeAudioStreamPaused) { KSTATUS status = stream->State == KeAudioStreamFailed ? stream->FailureStatus : NTLSTATUS_INVALID_PARAMETER; KiAudioReleaseMutex(stream->Lock); return status; } KiAudioSetStreamStateLocked(stream, KeAudioStreamRunning); KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); NtlAtomicIncrement64(&gAudioCoreStats.Resumes); return NTLSTATUS_SUCCESS; } // Stops accepting writes and plays all currently buffered frames. KSTATUS KeAudioDrainStream(KE_AUDIO_STREAM* stream) { if (!stream) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioDrainStream"); if (stream->State == KeAudioStreamRunning || stream->State == KeAudioStreamPaused) { KiAudioSetStreamStateLocked(stream, KeAudioStreamDraining); if (KiAudioDrainIsCompleteLocked(stream)) KiAudioCompleteDrainLocked(stream); } else { KSTATUS status = stream->State == KeAudioStreamFailed ? stream->FailureStatus : NTLSTATUS_INVALID_PARAMETER; KiAudioReleaseMutex(stream->Lock); return status; } KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); KeSignalEvent(stream->Endpoint->WorkEvent); NtlAtomicIncrement64(&gAudioCoreStats.Drains); return NTLSTATUS_SUCCESS; } // Stops future mixing and discards source frames not yet committed to endpoint periods. KSTATUS KeAudioStopStream(KE_AUDIO_STREAM* stream) { if (!stream) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioStopStream"); if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; KiAudioReleaseMutex(stream->Lock); return status; } stream->FramesDiscarded += stream->BufferedFrames; KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); stream->ReadFrame = 0; stream->WriteFrame = 0; stream->BufferedFrames = 0; stream->ResamplePositionQ32 = 0; stream->ConsecutiveUnderrunPeriods = 0; stream->DiscontinuityGeneration++; KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); KeSignalEvent(stream->Endpoint->WorkEvent); NtlAtomicIncrement64(&gAudioCoreStats.Stops); return NTLSTATUS_SUCCESS; } // Resets one live stream and invalidates presentation accounting from earlier periods. KSTATUS KeAudioResetStream(KE_AUDIO_STREAM* stream) { if (!stream) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioResetStream"); if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; KiAudioReleaseMutex(stream->Lock); return status; } stream->BufferGeneration++; KiAudioSetStreamStateLocked(stream, KeAudioStreamStopped); stream->ReadFrame = 0; stream->WriteFrame = 0; stream->BufferedFrames = 0; stream->ResamplePositionQ32 = 0; stream->FramesSubmitted = 0; stream->FramesConsumed = 0; stream->FramesPresented = 0; stream->FramesDiscarded = 0; stream->OutputFramesMixed = 0; stream->SilenceFramesInserted = 0; stream->Underruns = 0; stream->ConsecutiveUnderrunPeriods = 0; stream->DiscontinuityGeneration++; memset(stream->Ring, 0, (size_t)KE_AUDIO_STREAM_RING_FRAMES * stream->FrameBytes); KiAudioRefreshStreamEventLocked(stream); KiAudioReleaseMutex(stream->Lock); KeSignalEvent(stream->Endpoint->WorkEvent); NtlAtomicIncrement64(&gAudioCoreStats.Resets); return NTLSTATUS_SUCCESS; } // Updates per-stream volume, linear balance pan, and mute targets. KSTATUS KeAudioSetStreamVolume(KE_AUDIO_STREAM* stream, float volume, float pan, bool muted) { if (!stream || !KiAudioFloatIsFinite(volume) || !KiAudioFloatIsFinite(pan) || volume < 0.0f || volume > 1.0f || pan < -1.0f || pan > 1.0f) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioSetStreamVolume"); if (stream->State == KeAudioStreamFailed || stream->State == KeAudioStreamClosed) { KSTATUS status = stream->FailureStatus ? stream->FailureStatus : NTLSTATUS_OBJECT_DEAD; KiAudioReleaseMutex(stream->Lock); return status; } stream->Volume = volume; stream->Pan = pan; stream->Muted = muted ? 1u : 0u; if (stream->State == KeAudioStreamCreated || stream->State == KeAudioStreamStopped) { KiAudioComputeStreamTargets(stream->Volume, stream->Pan, stream->Muted, &stream->CurrentLeftGain, &stream->CurrentRightGain); stream->TargetLeftGain = stream->CurrentLeftGain; stream->TargetRightGain = stream->CurrentRightGain; stream->GainRampRemaining = 0; stream->LeftGainStep = 0.0f; stream->RightGainStep = 0.0f; } else { KiAudioBeginStreamGainRampLocked(stream); } KiAudioReleaseMutex(stream->Lock); KeSignalEvent(stream->Endpoint->WorkEvent); return NTLSTATUS_SUCCESS; } // Updates endpoint master software gain and separate logical mute state. KSTATUS KeAudioSetMasterVolume(uint32_t endpointId, float volume, bool muted) { if (!KiAudioFloatIsFinite(volume) || volume < 0.0f || volume > 1.0f) return NTLSTATUS_INVALID_PARAMETER; IO_AUDIO_ENDPOINT* endpoint = KiAudioReferenceEndpointById(endpointId); if (!endpoint) return NTLSTATUS_NOT_FOUND; DbgPrint("audiodiag: master endpoint=%u waiting control lock\n", endpointId); KiAudioAcquireMutex(endpoint->ControlLock, "KeAudioSetMasterVolume"); DbgPrint("audiodiag: master endpoint=%u acquired control lock\n", endpointId); float softwareGain = muted ? 0.0f : KiAudioSoftwareGainForVolume(endpoint, volume); uintptr_t registryFlags = KeAcquireSpinlock(&gAudioRegistryLock); if (endpoint->State != KeAudioEndpointReady) { KSTATUS failure = endpoint->FailureStatus; KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); KiAudioReleaseMutex(endpoint->ControlLock); ObDereferenceObject(&endpoint->Header); return KFAILED(failure) ? failure : NTLSTATUS_OBJECT_DEAD; } uintptr_t flags = KeAcquireSpinlock(&endpoint->NotifyLock); endpoint->MasterRequestedVolume = volume; endpoint->MasterTargetGain = softwareGain; if (endpoint->HardwareRunning) { endpoint->MasterGainStep = (endpoint->MasterTargetGain - endpoint->MasterCurrentGain) / (float)KE_AUDIO_GAIN_RAMP_FRAMES; endpoint->MasterRampRemaining = KE_AUDIO_GAIN_RAMP_FRAMES; } else { endpoint->MasterCurrentGain = endpoint->MasterTargetGain; endpoint->MasterGainStep = 0.0f; endpoint->MasterRampRemaining = 0; } endpoint->MasterGainGeneration++; endpoint->MasterMuted = muted ? 1u : 0u; KeReleaseSpinlock(&endpoint->NotifyLock, flags); KeReleaseSpinlock(&gAudioRegistryLock, registryFlags); KiAudioReleaseMutex(endpoint->ControlLock); DbgPrint("audiodiag: master endpoint=%u update committed\n", endpointId); KeSignalEvent(endpoint->WorkEvent); ObDereferenceObject(&endpoint->Header); return NTLSTATUS_SUCCESS; } // Copies one stream snapshot under its priority-inheriting mutex. KSTATUS KeAudioQueryStream(KE_AUDIO_STREAM* stream, KE_AUDIO_STREAM_INFO* infoOut) { if (!stream || !infoOut) return NTLSTATUS_INVALID_PARAMETER; KiAudioAcquireMutex(stream->Lock, "KeAudioQueryStream"); memset(infoOut, 0, sizeof(*infoOut)); infoOut->State = stream->State; infoOut->Format = stream->Format; infoOut->CapacityFrames = KE_AUDIO_STREAM_RING_FRAMES; infoOut->BufferedFrames = stream->BufferedFrames; infoOut->WritableFrames = KE_AUDIO_STREAM_RING_FRAMES - stream->BufferedFrames; infoOut->FramesSubmitted = stream->FramesSubmitted; infoOut->FramesConsumed = stream->FramesConsumed; infoOut->FramesPresented = stream->FramesPresented; infoOut->FramesDiscarded = stream->FramesDiscarded; infoOut->OutputFramesMixed = stream->OutputFramesMixed; infoOut->SilenceFramesInserted = stream->SilenceFramesInserted; infoOut->Underruns = stream->Underruns; infoOut->DrainedGeneration = stream->DrainedGeneration; infoOut->DiscontinuityGeneration = stream->DiscontinuityGeneration; infoOut->FailureStatus = stream->FailureStatus; infoOut->Volume = stream->Volume; infoOut->Pan = stream->Pan; infoOut->Muted = stream->Muted; KiAudioReleaseMutex(stream->Lock); return NTLSTATUS_SUCCESS; }