diff --git a/Libraries/LibMedia/AudioBlock.h b/Libraries/LibMedia/AudioBlock.h index e70bcacd0a..ee9a3f1f0f 100644 --- a/Libraries/LibMedia/AudioBlock.h +++ b/Libraries/LibMedia/AudioBlock.h @@ -6,53 +6,93 @@ #pragma once +#include +#include #include +#include #include #include -#include #include namespace Media { class AudioBlock { public: - using Data = Vector; - Audio::SampleSpecification const& sample_specification() const { return m_sample_specification; } AK::Duration timestamp() const { return m_timestamp; } i64 timestamp_in_frames() const { return m_timestamp_in_frames; } i64 end_timestamp_in_frames() const { return saturating_add(m_timestamp_in_frames, AK::clamp_to(frame_count())); } AK::Duration end_timestamp() const { return AK::Duration::from_time_units(end_timestamp_in_frames(), 1, sample_rate()); } - Span data() { return m_data; } - ReadonlySpan data() const { return m_data; } + Span channel_data(size_t channel) + { + VERIFY(channel < channel_count()); + return m_data.span().slice(channel * frame_capacity(), m_frame_count); + } + ReadonlySpan channel_data(size_t channel) const + { + VERIFY(channel < channel_count()); + return m_data.span().slice(channel * frame_capacity(), m_frame_count); + } + float sample(size_t channel, size_t frame) const + { + return channel_data(channel)[frame]; + } + void set_sample(size_t channel, size_t frame, float sample) + { + channel_data(channel)[frame] = sample; + } void clear() { m_sample_specification = {}; + m_timestamp = {}; m_timestamp_in_frames = 0; - m_data.clear_with_capacity(); + m_frame_count = 0; } - template - void emplace(Audio::SampleSpecification sample_specification, AK::Duration timestamp, Callback data_callback) + void initialize(Audio::SampleSpecification sample_specification, AK::Duration timestamp, size_t frame_count) { VERIFY(sample_specification.is_valid()); + VERIFY(frame_count <= NumericLimits::max()); + VERIFY(!Checked::multiplication_would_overflow(frame_count, sample_specification.channel_count())); m_sample_specification = sample_specification; m_timestamp = timestamp; m_timestamp_in_frames = timestamp.to_time_units(1, sample_rate()); - data_callback(m_data); + m_frame_count = frame_count; + ensure_frame_capacity(frame_count); } - template - void emplace(Audio::SampleSpecification sample_specification, i64 timestamp_in_frames, Callback data_callback) + void initialize(Audio::SampleSpecification sample_specification, i64 timestamp_in_frames, size_t frame_count) { VERIFY(sample_specification.is_valid()); + VERIFY(frame_count <= NumericLimits::max()); + VERIFY(!Checked::multiplication_would_overflow(frame_count, sample_specification.channel_count())); m_sample_specification = sample_specification; m_timestamp_in_frames = timestamp_in_frames; m_timestamp = AK::Duration::from_time_units(timestamp_in_frames, 1, sample_rate()); - data_callback(m_data); + m_frame_count = frame_count; + ensure_frame_capacity(frame_count); } void trim(size_t frame_count) { - m_data.resize_and_keep_capacity(frame_count * channel_count()); + VERIFY(frame_count <= m_frame_count); + m_frame_count = frame_count; + } + size_t copy_to_interleaved(Span destination, size_t source_frame_offset = 0) const + { + VERIFY(!is_empty()); + auto channels = channel_count(); + VERIFY(destination.size() % channels == 0); + + auto available_frames = frame_count(); + if (source_frame_offset >= available_frames) + return 0; + + auto frames_to_copy = min(destination.size() / channels, available_frames - source_frame_offset); + for (size_t channel = 0; channel < channels; channel++) { + auto source_channel = channel_data(channel).slice(source_frame_offset, frames_to_copy); + for (size_t frame = 0; frame < frames_to_copy; frame++) + destination[(frame * channels) + channel] = source_channel[frame]; + } + return frames_to_copy * channels; } u32 sample_rate() const { @@ -70,7 +110,7 @@ public: } size_t sample_count() const { - return data().size(); + return frame_count() * channel_count(); } u8 channel_count() const { @@ -78,14 +118,30 @@ public: } size_t frame_count() const { - return sample_count() / channel_count(); + return m_frame_count; } private: + size_t frame_capacity() const + { + if (!is_empty()) + return m_data.size() / channel_count(); + return 0; + } + + void ensure_frame_capacity(size_t frame_count) + { + if (frame_capacity() >= frame_count) + return; + VERIFY(!Checked::multiplication_would_overflow(frame_count, channel_count())); + m_data = MUST(FixedArray::create(frame_count * channel_count())); + } + Audio::SampleSpecification m_sample_specification; AK::Duration m_timestamp; i64 m_timestamp_in_frames { 0 }; - Data m_data; + size_t m_frame_count { 0 }; + FixedArray m_data; }; } diff --git a/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.cpp b/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.cpp index 84e3b3549b..91e7b20b56 100644 --- a/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.cpp +++ b/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.cpp @@ -62,8 +62,8 @@ ErrorOr FFmpegAudioConverter::set_sample_specifications(Audio::SampleSpeci auto output_sample_rate = static_cast(output.sample_rate()); auto allocation_result = swr_alloc_set_opts2(&m_context, - &output_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLT, output_sample_rate, - &input_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLT, input_sample_rate, + &output_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLTP, output_sample_rate, + &input_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLTP, input_sample_rate, 0, nullptr); if (allocation_result < 0) return Error::from_string_view(av_error_code_to_string(allocation_result)); @@ -78,12 +78,15 @@ ErrorOr FFmpegAudioConverter::set_sample_specifications(Audio::SampleSpeci void FFmpegAudioConverter::free_output_buffer() { - if (m_output_buffer == nullptr) { + if (m_output_buffers == nullptr) { VERIFY(m_output_buffer_frame_count == 0); return; } - av_freep(static_cast(&m_output_buffer)); - VERIFY(m_output_buffer == nullptr); + // The output buffers is a pointer to an array of pointers to the same allocation, so we only want to free the + // at the first index, then free the array of pointers. + av_freep(static_cast(&m_output_buffers[0])); + av_freep(static_cast(&m_output_buffers)); + VERIFY(m_output_buffers == nullptr); m_output_buffer_frame_count = 0; } @@ -114,40 +117,41 @@ ErrorOr FFmpegAudioConverter::convert(AudioBlock& input) VERIFY(m_input_sample_specification.is_valid()); VERIFY(m_output_sample_specification.is_valid()); - auto input_data = input.data(); - auto output_channel_count = m_output_sample_specification.channel_count(); - auto output_frame_count = TRY(get_maximum_output_frames(input_data.size())); + auto output_frame_count = TRY(get_maximum_output_frames(input.sample_count())); if (output_frame_count > m_output_buffer_frame_count) { free_output_buffer(); - auto alloc_samples_result = av_samples_alloc(&m_output_buffer, nullptr, output_channel_count, output_frame_count, AVSampleFormat::AV_SAMPLE_FMT_FLT, 0); + auto alloc_samples_result = av_samples_alloc_array_and_samples(&m_output_buffers, nullptr, output_channel_count, output_frame_count, AVSampleFormat::AV_SAMPLE_FMT_FLTP, 0); if (alloc_samples_result < 0) return Error::from_string_view(av_error_code_to_string(alloc_samples_result)); - VERIFY(m_output_buffer != nullptr); + VERIFY(m_output_buffers != nullptr); m_output_buffer_frame_count = output_frame_count; } - auto const* input_buffer = input_data.reinterpret().data(); // The input buffer size should already be safe to cast to int here. - auto input_frame_count = static_cast(input_data.size() / m_input_sample_specification.channel_count()); + auto input_frame_count = static_cast(input.frame_count()); VERIFY(input_frame_count >= 0); - auto converted_frames_result = swr_convert(m_context, &m_output_buffer, m_output_buffer_frame_count, &input_buffer, input_frame_count); + Array input_buffers; + for (size_t channel = 0; channel < input.channel_count(); channel++) + input_buffers[channel] = input.channel_data(channel).reinterpret().data(); + + auto converted_frames_result = swr_convert(m_context, m_output_buffers, m_output_buffer_frame_count, input_buffers.data(), input_frame_count); if (converted_frames_result < 0) return Error::from_string_view(av_error_code_to_string(converted_frames_result)); VERIFY(converted_frames_result <= m_output_buffer_frame_count); auto converted_frames = static_cast(converted_frames_result); - input.emplace(m_output_sample_specification, input.timestamp(), [&](AudioBlock::Data& data) { - data.resize_and_keep_capacity(converted_frames * output_channel_count); - AK::TypedTransfer::copy(data.data(), reinterpret_cast(m_output_buffer), data.size()); - }); + input.initialize(m_output_sample_specification, input.timestamp(), converted_frames); + for (size_t channel = 0; channel < output_channel_count; channel++) + AK::TypedTransfer::copy(input.channel_data(channel).data(), reinterpret_cast(m_output_buffers[channel]), converted_frames); return {}; } FFmpegAudioConverter::~FFmpegAudioConverter() { + free_output_buffer(); swr_free(&m_context); } diff --git a/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.h b/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.h index 0c6eaa4125..043facfb70 100644 --- a/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.h +++ b/Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.h @@ -35,7 +35,7 @@ private: Audio::SampleSpecification m_input_sample_specification; Audio::SampleSpecification m_output_sample_specification; SwrContext* m_context { nullptr }; - u8* m_output_buffer { nullptr }; + u8** m_output_buffers { nullptr }; int m_output_buffer_frame_count { 0 }; }; diff --git a/Libraries/LibMedia/FFmpeg/FFmpegAudioDecoder.cpp b/Libraries/LibMedia/FFmpeg/FFmpegAudioDecoder.cpp index c9860fdeac..be9c434a51 100644 --- a/Libraries/LibMedia/FFmpeg/FFmpegAudioDecoder.cpp +++ b/Libraries/LibMedia/FFmpeg/FFmpegAudioDecoder.cpp @@ -169,48 +169,49 @@ DecoderErrorOr FFmpegAudioDecoder::write_next_block(AudioBlock& block) auto channel_map = channel_map_result.release_value(); auto sample_specification = Audio::SampleSpecification(m_frame->sample_rate, channel_map); - block.emplace(sample_specification, timestamp, [&](AudioBlock::Data& data) { - auto format = static_cast(m_frame->format); - auto is_planar = av_sample_fmt_is_planar(format) != 0; - auto planar_format = av_get_planar_sample_fmt(format); + auto format = static_cast(m_frame->format); + auto is_planar = av_sample_fmt_is_planar(format) != 0; + auto planar_format = av_get_planar_sample_fmt(format); - VERIFY(m_frame->nb_samples >= 0); - auto frame_count = static_cast(m_frame->nb_samples); - auto channel_count = static_cast(m_frame->ch_layout.nb_channels); - auto sample_count = frame_count * channel_count; - data.resize_and_keep_capacity(sample_count); + VERIFY(m_frame->nb_samples >= 0); + auto frame_count = static_cast(m_frame->nb_samples); + auto channel_count = static_cast(m_frame->ch_layout.nb_channels); + auto sample_count = frame_count * channel_count; + block.initialize(sample_specification, timestamp, frame_count); - auto sample_size = [&] { - switch (planar_format) { - case AV_SAMPLE_FMT_U8P: - return sizeof(u8); - case AV_SAMPLE_FMT_S16P: - return sizeof(i16); - case AV_SAMPLE_FMT_S32P: - return sizeof(i32); - case AV_SAMPLE_FMT_FLTP: - return sizeof(float); - case AV_SAMPLE_FMT_DBLP: - return sizeof(double); - case AV_SAMPLE_FMT_S64P: - return sizeof(i64); - default: - VERIFY_NOT_REACHED(); - } - }(); + auto sample_size = [&] { + switch (planar_format) { + case AV_SAMPLE_FMT_U8P: + return sizeof(u8); + case AV_SAMPLE_FMT_S16P: + return sizeof(i16); + case AV_SAMPLE_FMT_S32P: + return sizeof(i32); + case AV_SAMPLE_FMT_FLTP: + return sizeof(float); + case AV_SAMPLE_FMT_DBLP: + return sizeof(double); + case AV_SAMPLE_FMT_S64P: + return sizeof(i64); + default: + VERIFY_NOT_REACHED(); + } + }(); - VERIFY(m_frame->linesize[0] > 0); - if (is_planar) - VERIFY(static_cast(m_frame->linesize[0]) >= frame_count * sample_size); - else - VERIFY(static_cast(m_frame->linesize[0]) >= sample_count * sample_size); + VERIFY(m_frame->linesize[0] > 0); + if (is_planar) + VERIFY(static_cast(m_frame->linesize[0]) >= frame_count * sample_size); + else + VERIFY(static_cast(m_frame->linesize[0]) >= sample_count * sample_size); - for (size_t i = 0; i < sample_count; i++) { + for (size_t channel = 0; channel < channel_count; ++channel) { + auto channel_data = block.channel_data(channel); + for (size_t frame = 0; frame < frame_count; ++frame) { size_t plane = 0; - size_t index_in_plane = i; + size_t index_in_plane = (frame * channel_count) + channel; if (is_planar) { - plane = i % channel_count; - index_in_plane = i / channel_count; + plane = channel; + index_in_plane = frame; } auto float_sample = [&] { @@ -231,9 +232,9 @@ DecoderErrorOr FFmpegAudioDecoder::write_next_block(AudioBlock& block) VERIFY_NOT_REACHED(); } }(); - data[i] = float_sample; + channel_data[frame] = float_sample; } - }); + } return {}; } diff --git a/Libraries/LibMedia/Processors/AudioMixer.cpp b/Libraries/LibMedia/Processors/AudioMixer.cpp index 708cacc7d0..d2582c5910 100644 --- a/Libraries/LibMedia/Processors/AudioMixer.cpp +++ b/Libraries/LibMedia/Processors/AudioMixer.cpp @@ -227,7 +227,6 @@ void AudioMixer::pull(AudioBlock& into) auto buffer_start_frame = m_next_frame_to_write; auto frames_end_cap = buffer_start_frame + static_cast(max_frame_count); - auto write_size = max_frame_count * channel_count; auto combined_status_after_mix = PipelineStatus::EndOfStream; i64 latest_mixed_frame = frames_end_cap; @@ -235,96 +234,97 @@ void AudioMixer::pull(AudioBlock& into) for (auto& [input, input_data] : m_inputs) input_data.next_frame = buffer_start_frame; - into.emplace(m_sample_specification, buffer_start_frame, [&](AudioBlock::Data& data) { - data.resize_and_keep_capacity(write_size); - for (size_t i = 0; i < write_size; i++) - data[i] = 0.0f; + into.initialize(m_sample_specification, buffer_start_frame, max_frame_count); + for (size_t channel = 0; channel < channel_count; ++channel) + into.channel_data(channel).fill(0.0f); - while (true) { - struct MixTarget { - AudioProducer& input; - InputMixingData& input_data; - }; - auto mix_target = [&] { - Optional result; - for (auto& [input, input_data] : m_inputs) { - if (input_data.next_frame >= frames_end_cap) - continue; - if (!result.has_value() || input_data.next_frame < result->input_data.next_frame) - result = { input, input_data }; - } - return result; - }(); - if (!mix_target.has_value()) - break; - auto [input, input_data] = mix_target.release_value(); - - auto& current_block = input_data.current_block; - input_data.last_status = input.status(); - while (input_data.last_status == PipelineStatus::MovedPosition) { - input.pull(current_block); - VERIFY(current_block.is_empty()); - input_data.last_status = input.status(); - } - - auto current_block_is_usable = [&] { - if (current_block.is_empty()) - return false; - if (current_block.sample_specification() != m_sample_specification) - return false; - if (current_block.end_timestamp_in_frames() <= input_data.next_frame) - return false; - return true; - }(); - - if (!current_block_is_usable) { - current_block.clear(); - if (input_data.last_status == PipelineStatus::EndOfStream) { - input_data.next_frame = frames_end_cap; + while (true) { + struct MixTarget { + AudioProducer& input; + InputMixingData& input_data; + }; + auto mix_target = [&] { + Optional result; + for (auto& [input, input_data] : m_inputs) { + if (input_data.next_frame >= frames_end_cap) continue; - } - if (input_data.last_status != PipelineStatus::HaveData) - break; - input.pull(current_block); - VERIFY(!current_block.is_empty()); - continue; + if (!result.has_value() || input_data.next_frame < result->input_data.next_frame) + result = { input, input_data }; } + return result; + }(); + if (!mix_target.has_value()) + break; + auto [input, input_data] = mix_target.release_value(); - auto first_frame_offset = current_block.timestamp_in_frames(); - if (first_frame_offset >= frames_end_cap) { + auto& current_block = input_data.current_block; + input_data.last_status = input.status(); + while (input_data.last_status == PipelineStatus::MovedPosition) { + input.pull(current_block); + VERIFY(current_block.is_empty()); + input_data.last_status = input.status(); + } + + auto current_block_is_usable = [&] { + if (current_block.is_empty()) + return false; + if (current_block.sample_specification() != m_sample_specification) + return false; + if (current_block.end_timestamp_in_frames() <= input_data.next_frame) + return false; + return true; + }(); + + if (!current_block_is_usable) { + current_block.clear(); + if (input_data.last_status == PipelineStatus::EndOfStream) { input_data.next_frame = frames_end_cap; continue; } - - auto next_frame = max(input_data.next_frame, first_frame_offset); - - VERIFY(next_frame >= first_frame_offset); - auto index_in_block = static_cast((next_frame - first_frame_offset) * channel_count); - VERIFY(index_in_block < current_block.sample_count()); - - VERIFY(next_frame >= buffer_start_frame); - auto index_in_buffer = static_cast((next_frame - buffer_start_frame) * channel_count); - VERIFY(index_in_buffer < write_size); - - VERIFY(current_block.sample_count() >= index_in_block); - auto write_count = current_block.sample_count() - index_in_block; - write_count = min(write_count, write_size - index_in_buffer); - VERIFY(write_count > 0); - VERIFY(index_in_buffer + write_count <= write_size); - VERIFY(write_count % channel_count == 0); - - for (size_t i = 0; i < write_count; i++) - data[index_in_buffer + i] += current_block.data()[index_in_block + i]; - - input_data.next_frame = next_frame + static_cast(write_count / channel_count); + if (input_data.last_status != PipelineStatus::HaveData) + break; + input.pull(current_block); + VERIFY(!current_block.is_empty()); + continue; } - for (auto& [input, input_data] : m_inputs) { - VERIFY(input_data.last_status != PipelineStatus::MovedPosition); - latest_mixed_frame = min(latest_mixed_frame, input_data.next_frame); - combined_status_after_mix = select_combined_pipeline_status(combined_status_after_mix, input_data.last_status); + auto first_frame_offset = current_block.timestamp_in_frames(); + if (first_frame_offset >= frames_end_cap) { + input_data.next_frame = frames_end_cap; + continue; } - }); + + auto next_frame = max(input_data.next_frame, first_frame_offset); + + VERIFY(next_frame >= first_frame_offset); + auto frame_index_in_block = static_cast(next_frame - first_frame_offset); + VERIFY(frame_index_in_block < current_block.frame_count()); + + VERIFY(next_frame >= buffer_start_frame); + auto frame_index_in_buffer = static_cast(next_frame - buffer_start_frame); + VERIFY(frame_index_in_buffer < max_frame_count); + + VERIFY(current_block.frame_count() >= frame_index_in_block); + auto frames_to_write = current_block.frame_count() - frame_index_in_block; + frames_to_write = min(frames_to_write, max_frame_count - frame_index_in_buffer); + VERIFY(frames_to_write > 0); + VERIFY(frame_index_in_buffer + frames_to_write <= max_frame_count); + + for (size_t channel = 0; channel < channel_count; ++channel) { + auto input_channel = current_block.channel_data(channel).slice(frame_index_in_block, frames_to_write); + auto output_channel = into.channel_data(channel).slice(frame_index_in_buffer, frames_to_write); + for (size_t frame = 0; frame < frames_to_write; ++frame) + output_channel[frame] += input_channel[frame]; + } + + input_data.next_frame = next_frame + static_cast(frames_to_write); + } + + for (auto& [input, input_data] : m_inputs) { + VERIFY(input_data.last_status != PipelineStatus::MovedPosition); + latest_mixed_frame = min(latest_mixed_frame, input_data.next_frame); + combined_status_after_mix = select_combined_pipeline_status(combined_status_after_mix, input_data.last_status); + } VERIFY(latest_mixed_frame >= buffer_start_frame); auto frame_count = static_cast(latest_mixed_frame - buffer_start_frame); diff --git a/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp b/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp index 97ab37cf9f..e78c2f63e6 100644 --- a/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp +++ b/Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp @@ -298,17 +298,13 @@ ReadonlySpan AudioPlaybackSink::OutputThreadData::move_output_to_playback continue; } - auto offset_in_head_samples = static_cast(m_next_frame_to_play - block_start_frame) * channel_count; - auto samples_remaining_in_head = head_block.sample_count() - offset_in_head_samples; - auto samples_to_copy = min(samples_remaining_in_head, buffer.size() - samples_written); - - for (size_t i = 0; i < samples_to_copy; i++) - buffer[samples_written + i] = head_block.data()[offset_in_head_samples + i]; + auto offset_in_head_frames = static_cast(m_next_frame_to_play - block_start_frame); + auto samples_to_copy = head_block.copy_to_interleaved(buffer.slice(samples_written), offset_in_head_frames); samples_written += samples_to_copy; m_next_frame_to_play += static_cast(samples_to_copy / channel_count); - if (offset_in_head_samples + samples_to_copy == head_block.sample_count()) { + if ((offset_in_head_frames * channel_count) + samples_to_copy == head_block.sample_count()) { m_block_head = (m_block_head + 1) % OUTPUT_BLOCK_QUEUE_CAPACITY; m_block_count--; } diff --git a/Tests/LibMedia/TestFFmpegAudioNormalization.cpp b/Tests/LibMedia/TestFFmpegAudioNormalization.cpp index a6ee4f8400..bc9883fa0a 100644 --- a/Tests/LibMedia/TestFFmpegAudioNormalization.cpp +++ b/Tests/LibMedia/TestFFmpegAudioNormalization.cpp @@ -99,16 +99,18 @@ static void decode_and_expect() producer->pull(block); if (status == Media::PipelineStatus::HaveData) { EXPECT(!block.is_empty()); - for (float sample : block.data()) { - EXPECT(sample >= -1.0f); - if constexpr (sizeof(Sample) >= sizeof(i32)) - EXPECT(sample <= 1.0f); - else - EXPECT(sample < 1.0f); - if (sample == -1.0f) - saw_negative_full_scale_sample = true; - if (sample > 0.0f) - saw_positive_peak_sample = true; + for (size_t channel = 0; channel < block.channel_count(); ++channel) { + for (float sample : block.channel_data(channel)) { + EXPECT(sample >= -1.0f); + if constexpr (sizeof(Sample) >= sizeof(i32)) + EXPECT(sample <= 1.0f); + else + EXPECT(sample < 1.0f); + if (sample == -1.0f) + saw_negative_full_scale_sample = true; + if (sample > 0.0f) + saw_positive_peak_sample = true; + } } decoded_frame_count += block.frame_count(); } else if (status == Media::PipelineStatus::EndOfStream) {