From 47207b69e69eb5e7d3abe1b0882586180bd75044 Mon Sep 17 00:00:00 2001 From: Zaggy1024 Date: Wed, 3 Jun 2026 03:35:48 -0500 Subject: [PATCH] LibMedia: Remap audio time based on block durations in media time This will allow the media time to advance faster or slower on a per- block basis, so that playback rate doesn't lose timing. --- Libraries/LibMedia/AudioBlockTiming.h | 23 ++++ Libraries/LibMedia/AudioBlockTimingRing.h | 106 ++++++++++++++++++ .../LibMedia/Sinks/AudioPlaybackSink.cpp | 48 ++++++-- Libraries/LibMedia/Sinks/AudioPlaybackSink.h | 5 +- 4 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 Libraries/LibMedia/AudioBlockTimingRing.h diff --git a/Libraries/LibMedia/AudioBlockTiming.h b/Libraries/LibMedia/AudioBlockTiming.h index ca5557119e..cc13bdb17e 100644 --- a/Libraries/LibMedia/AudioBlockTiming.h +++ b/Libraries/LibMedia/AudioBlockTiming.h @@ -22,6 +22,29 @@ public: i64 frame_count() const { return m_frame_count; } i64 end_frame_index() const { return saturating_add(m_first_frame_index, m_frame_count); } + bool contains_frame_index(i64 frame_index) const + { + return frame_index >= first_frame_index() && frame_index < end_frame_index(); + } + + AK::Duration media_time_at_frame_index(i64 frame_index) const + { + VERIFY(frame_count() > 0); + if (frame_index < first_frame_index()) { + auto frame_offset = AK::clamp_to(first_frame_index() - frame_index); + auto block_frame_count = AK::clamp_to(frame_count()); + return media_time_start() - media_time_duration().scaled_by(frame_offset, block_frame_count); + } + if (frame_index == first_frame_index()) + return media_time_start(); + if (frame_index >= end_frame_index()) + return media_time_end(); + + auto frame_offset = AK::clamp_to(frame_index - first_frame_index()); + auto block_frame_count = AK::clamp_to(frame_count()); + return media_time_start() + media_time_duration().scaled_by(frame_offset, block_frame_count); + } + void clear() { m_media_time_start = {}; diff --git a/Libraries/LibMedia/AudioBlockTimingRing.h b/Libraries/LibMedia/AudioBlockTimingRing.h new file mode 100644 index 0000000000..aba0742eb5 --- /dev/null +++ b/Libraries/LibMedia/AudioBlockTimingRing.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2026-present, the Ladybird developers. + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace Media { + +class AudioBlockTimingRing { +public: + static constexpr size_t capacity = 32; + + void clear() + { + m_first_valid_sequence = m_latest_sequence.load() + 1; + } + + void enqueue(AudioBlockTiming timing) + { + auto sequence = m_latest_sequence.load() + 1; + auto& slot = m_slots[sequence % capacity]; + + auto version = slot.version.load(); + slot.version.store(version + 1); + slot.sequence = sequence; + slot.timing = timing; + slot.version.store(version + 2); + + m_latest_sequence.store(sequence); + } + + Optional find_timing_for_frame_index(i64 frame_index) const + { + auto latest_sequence = m_latest_sequence.load(); + auto first_valid_sequence = m_first_valid_sequence.load(); + if (latest_sequence < first_valid_sequence) + return {}; + + Optional oldest_timing; + auto sequence_count = min(latest_sequence - first_valid_sequence + 1, capacity); + for (u64 i = 0; i < sequence_count; ++i) { + auto sequence = latest_sequence - i; + if (sequence < first_valid_sequence) + break; + + auto record = read_record(sequence); + if (!record.has_value()) + continue; + + auto const& timing = record->timing; + if (timing.contains_frame_index(frame_index)) + return timing; + if (frame_index >= timing.end_frame_index()) + return timing; + oldest_timing = timing; + } + + return oldest_timing; + } + +private: + struct Record { + u64 sequence { 0 }; + AudioBlockTiming timing; + }; + + struct Slot { + Atomic version { 0 }; + u64 sequence { 0 }; + AudioBlockTiming timing; + }; + + Optional read_record(u64 expected_sequence) const + { + auto const& slot = m_slots[expected_sequence % capacity]; + + auto version_before = slot.version.load(); + if (version_before % 2 != 0) + return {}; + + Record record { + .sequence = slot.sequence, + .timing = slot.timing, + }; + + auto version_after = slot.version.load(); + if (version_before != version_after || version_after % 2 != 0) + return {}; + if (record.sequence != expected_sequence) + return {}; + return record; + } + + Atomic m_latest_sequence { 0 }; + Atomic m_first_valid_sequence { 1 }; + Array m_slots; +}; + +} diff --git a/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp b/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp index be08e034b7..962aaf612d 100644 --- a/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp +++ b/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -46,6 +48,7 @@ public: size_t m_block_tail { 0 }; size_t m_block_count { 0 }; i64 m_next_frame_to_play { 0 }; + AudioBlockTimingRing m_block_timings; PipelineStateChangeHandler m_on_state_changed; PipelineStatus m_last_pull_status { PipelineStatus::Pending }; @@ -82,6 +85,7 @@ ErrorOr> AudioPlaybackSink::try_create(Pipeline output_thread_data->m_block_head = 0; output_thread_data->m_block_tail = 0; output_thread_data->m_block_count = 0; + output_thread_data->m_block_timings.clear(); output_thread_data->m_last_real_data_end_in_frames = output_thread_data->m_next_frame_to_play; output_thread_data->m_waiting_for_upstream_data = false; output_thread_data->m_output_condition.broadcast(); @@ -131,6 +135,7 @@ ErrorOr> AudioPlaybackSink::try_create(Pipeline VERIFY(can_carry_data(status)); output_thread_data->m_block_tail = (output_thread_data->m_block_tail + 1) % OUTPUT_BLOCK_QUEUE_CAPACITY; output_thread_data->m_block_count++; + output_thread_data->m_block_timings.enqueue(output_block.timing()); if (output_thread_data->m_playback_stream) output_thread_data->m_playback_stream->notify_data_available(); @@ -341,11 +346,22 @@ AK::Duration AudioPlaybackSink::current_time() const { if (m_temporary_time.has_value()) return m_temporary_time.value(); - if (!m_output_thread_data->m_playback_stream) - return m_last_media_time; + if (!m_output_thread_data->m_playback_stream || !m_sample_specification.is_valid()) + return m_minimum_media_time; auto stream_time = m_output_thread_data->m_playback_stream->total_time_played(); - return m_last_media_time + (stream_time - m_last_stream_time); + auto stream_delta = stream_time - m_anchor_stream_time; + auto frames_played = stream_delta.to_time_units(1, m_sample_specification.sample_rate()); + auto current_output_frame_index = m_anchor_output_frame_index + frames_played; + + auto maybe_timing = m_output_thread_data->m_block_timings.find_timing_for_frame_index(current_output_frame_index); + if (!maybe_timing.has_value()) + return m_minimum_media_time; + + auto time = maybe_timing->media_time_at_frame_index(current_output_frame_index); + time = max(time, m_minimum_media_time); + m_minimum_media_time = time; + return time; } void AudioPlaybackSink::resume() @@ -359,11 +375,9 @@ void AudioPlaybackSink::resume() if (!m_output_thread_data->m_playback_stream) return; m_output_thread_data->m_playback_stream->resume() - ->when_resolved([self = NonnullRefPtr(*this)](auto new_stream_time) { - self->m_main_thread_event_loop.deferred_invoke([self, new_stream_time]() { - auto new_media_time = self->m_last_media_time + (new_stream_time - self->m_last_stream_time); - self->m_last_stream_time = new_stream_time; - self->m_last_media_time = new_media_time; + ->when_resolved([self = NonnullRefPtr(*this)](auto new_device_time) { + self->m_main_thread_event_loop.deferred_invoke([self, new_device_time]() { + self->m_anchor_stream_time = new_device_time; }); }) .when_rejected([](auto&& error) { @@ -378,7 +392,15 @@ void AudioPlaybackSink::pause() if (!m_output_thread_data->m_playback_stream) return; m_output_thread_data->m_playback_stream->drain_buffer_and_suspend() - ->when_resolved([]() { + ->when_resolved([self = NonnullRefPtr(*this)]() { + auto new_stream_time = self->m_output_thread_data->m_playback_stream->total_time_played(); + + self->m_main_thread_event_loop.deferred_invoke([self, new_stream_time]() { + auto stream_delta = new_stream_time - self->m_anchor_stream_time; + auto frames_played = stream_delta.to_time_units(1, self->m_sample_specification.sample_rate()); + self->m_anchor_output_frame_index += frames_played; + self->m_anchor_stream_time = new_stream_time; + }); }) .when_rejected([](auto&& error) { warnln("Unexpected error while pausing AudioPlaybackSink: {}", error.string_literal()); @@ -389,6 +411,7 @@ void AudioPlaybackSink::seek(AK::Duration time) { bool already_draining_for_seek = m_temporary_time.has_value(); m_temporary_time = time; + m_minimum_media_time = time; if (!m_output_thread_data->m_playback_stream) return; @@ -404,6 +427,7 @@ void AudioPlaybackSink::seek(AK::Duration time) m_output_thread_data->m_last_dispatched_status = PipelineStatus::Pending; m_output_thread_data->m_waiting_for_upstream_data = true; + m_output_thread_data->m_block_timings.clear(); } if (m_output_thread_data->m_input != nullptr) @@ -417,8 +441,10 @@ void AudioPlaybackSink::seek(AK::Duration time) auto new_stream_time = self->m_output_thread_data->m_playback_stream->total_time_played(); self->m_main_thread_event_loop.deferred_invoke([self, new_stream_time]() { - self->m_last_stream_time = new_stream_time; - self->m_last_media_time = self->m_temporary_time.release_value(); + self->m_anchor_stream_time = new_stream_time; + auto seek_target = self->m_temporary_time.release_value(); + self->m_anchor_output_frame_index = seek_target.to_time_units(1, self->m_sample_specification.sample_rate()); + self->m_minimum_media_time = seek_target; if (self->m_playing) self->resume(); diff --git a/Libraries/LibMedia/Sinks/AudioPlaybackSink.h b/Libraries/LibMedia/Sinks/AudioPlaybackSink.h index c918596040..4438e9dd13 100644 --- a/Libraries/LibMedia/Sinks/AudioPlaybackSink.h +++ b/Libraries/LibMedia/Sinks/AudioPlaybackSink.h @@ -55,9 +55,10 @@ private: bool m_playing { false }; double m_volume { 1 }; - AK::Duration m_last_stream_time; - AK::Duration m_last_media_time; + AK::Duration m_anchor_stream_time; + i64 m_anchor_output_frame_index { 0 }; Optional m_temporary_time; + mutable AK::Duration m_minimum_media_time; NonnullRefPtr m_output_thread_data; };