dmaax/ladybird
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

LibMedia: Store raw audio data as planar

Browse source 
This avoids strided loads for vectorized audio data processing loops.
This commit is contained in:
Zaggy1024 2026-06-02 15:19:36 -05:00 committed by Gregory Bertilson
parent ea15484700
commit c8c64e4819
7 changed files with 229 additions and 170 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
Libraries/LibMedia/AudioBlock.h
View file

@ -6,53 +6,93 @@
#pragma once
#include <AK/Checked.h>
#include <AK/FixedArray.h>
#include <AK/Math.h>
#include <AK/NumericLimits.h>
#include <AK/SaturatingMath.h>
#include <AK/Time.h>
#include <AK/Vector.h>
#include <LibMedia/Audio/SampleSpecification.h>
namespace Media {
class AudioBlock {
public:
using Data = Vector<float>;
Audio::SampleSpecification const& sample_specification() const { return m_sample_specification; }
AK::Duration timestamp() const { return m_timestamp; }
i64 timestamp_in_frames() const { return m_timestamp_in_frames; }
i64 end_timestamp_in_frames() const { return saturating_add(m_timestamp_in_frames, AK::clamp_to<i64>(frame_count())); }
AK::Duration end_timestamp() const { return AK::Duration::from_time_units(end_timestamp_in_frames(), 1, sample_rate()); }
Span<float> data() { return m_data; }
ReadonlySpan<float> data() const { return m_data; }
Span<float> channel_data(size_t channel)
{
VERIFY(channel < channel_count());
return m_data.span().slice(channel * frame_capacity(), m_frame_count);
}
ReadonlySpan<float> channel_data(size_t channel) const
{
VERIFY(channel < channel_count());
return m_data.span().slice(channel * frame_capacity(), m_frame_count);
}
float sample(size_t channel, size_t frame) const
{
return channel_data(channel)[frame];
}
void set_sample(size_t channel, size_t frame, float sample)
{
channel_data(channel)[frame] = sample;
}
void clear()
{
m_sample_specification = {};
m_timestamp = {};
m_timestamp_in_frames = 0;
m_data.clear_with_capacity();
m_frame_count = 0;
}
template<typename Callback>
void emplace(Audio::SampleSpecification sample_specification, AK::Duration timestamp, Callback data_callback)
void initialize(Audio::SampleSpecification sample_specification, AK::Duration timestamp, size_t frame_count)
{
VERIFY(sample_specification.is_valid());
VERIFY(frame_count <= NumericLimits<i64>::max());
VERIFY(!Checked<size_t>::multiplication_would_overflow(frame_count, sample_specification.channel_count()));
m_sample_specification = sample_specification;
m_timestamp = timestamp;
m_timestamp_in_frames = timestamp.to_time_units(1, sample_rate());
data_callback(m_data);
m_frame_count = frame_count;
ensure_frame_capacity(frame_count);
}
template<typename Callback>
void emplace(Audio::SampleSpecification sample_specification, i64 timestamp_in_frames, Callback data_callback)
void initialize(Audio::SampleSpecification sample_specification, i64 timestamp_in_frames, size_t frame_count)
{
VERIFY(sample_specification.is_valid());
VERIFY(frame_count <= NumericLimits<i64>::max());
VERIFY(!Checked<size_t>::multiplication_would_overflow(frame_count, sample_specification.channel_count()));
m_sample_specification = sample_specification;
m_timestamp_in_frames = timestamp_in_frames;
m_timestamp = AK::Duration::from_time_units(timestamp_in_frames, 1, sample_rate());
data_callback(m_data);
m_frame_count = frame_count;
ensure_frame_capacity(frame_count);
}
void trim(size_t frame_count)
{
m_data.resize_and_keep_capacity(frame_count * channel_count());
VERIFY(frame_count <= m_frame_count);
m_frame_count = frame_count;
}
size_t copy_to_interleaved(Span<float> destination, size_t source_frame_offset = 0) const
{
VERIFY(!is_empty());
auto channels = channel_count();
VERIFY(destination.size() % channels == 0);
auto available_frames = frame_count();
if (source_frame_offset >= available_frames)
return 0;
auto frames_to_copy = min(destination.size() / channels, available_frames - source_frame_offset);
for (size_t channel = 0; channel < channels; channel++) {
auto source_channel = channel_data(channel).slice(source_frame_offset, frames_to_copy);
for (size_t frame = 0; frame < frames_to_copy; frame++)
destination[(frame * channels) + channel] = source_channel[frame];
}
return frames_to_copy * channels;
}
u32 sample_rate() const
{
@ -70,7 +110,7 @@ public:
}
size_t sample_count() const
{
return data().size();
return frame_count() * channel_count();
}
u8 channel_count() const
{
@ -78,14 +118,30 @@ public:
}
size_t frame_count() const
{
return sample_count() / channel_count();
return m_frame_count;
}
private:
size_t frame_capacity() const
{
if (!is_empty())
return m_data.size() / channel_count();
return 0;
}
void ensure_frame_capacity(size_t frame_count)
{
if (frame_capacity() >= frame_count)
return;
VERIFY(!Checked<size_t>::multiplication_would_overflow(frame_count, channel_count()));
m_data = MUST(FixedArray<float>::create(frame_count * channel_count()));
}
Audio::SampleSpecification m_sample_specification;
AK::Duration m_timestamp;
i64 m_timestamp_in_frames { 0 };
Data m_data;
size_t m_frame_count { 0 };
FixedArray<float> m_data;
};
}

38
Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.cpp
View file

@ -62,8 +62,8 @@ ErrorOr<void> FFmpegAudioConverter::set_sample_specifications(Audio::SampleSpeci
auto output_sample_rate = static_cast<int>(output.sample_rate());
auto allocation_result = swr_alloc_set_opts2(&m_context,
&output_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLT, output_sample_rate,
&input_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLT, input_sample_rate,
&output_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLTP, output_sample_rate,
&input_channel_layout, AVSampleFormat::AV_SAMPLE_FMT_FLTP, input_sample_rate,
0, nullptr);
if (allocation_result < 0)
return Error::from_string_view(av_error_code_to_string(allocation_result));
@ -78,12 +78,15 @@ ErrorOr<void> FFmpegAudioConverter::set_sample_specifications(Audio::SampleSpeci
void FFmpegAudioConverter::free_output_buffer()
{
if (m_output_buffer == nullptr) {
if (m_output_buffers == nullptr) {
VERIFY(m_output_buffer_frame_count == 0);
return;
}
av_freep(static_cast<void*>(&m_output_buffer));
VERIFY(m_output_buffer == nullptr);
// The output buffers is a pointer to an array of pointers to the same allocation, so we only want to free the
// at the first index, then free the array of pointers.
av_freep(static_cast<void*>(&m_output_buffers[0]));
av_freep(static_cast<void*>(&m_output_buffers));
VERIFY(m_output_buffers == nullptr);
m_output_buffer_frame_count = 0;
}
@ -114,40 +117,41 @@ ErrorOr<void> FFmpegAudioConverter::convert(AudioBlock& input)
VERIFY(m_input_sample_specification.is_valid());
VERIFY(m_output_sample_specification.is_valid());
auto input_data = input.data();
auto output_channel_count = m_output_sample_specification.channel_count();
auto output_frame_count = TRY(get_maximum_output_frames(input_data.size()));
auto output_frame_count = TRY(get_maximum_output_frames(input.sample_count()));
if (output_frame_count > m_output_buffer_frame_count) {
free_output_buffer();
auto alloc_samples_result = av_samples_alloc(&m_output_buffer, nullptr, output_channel_count, output_frame_count, AVSampleFormat::AV_SAMPLE_FMT_FLT, 0);
auto alloc_samples_result = av_samples_alloc_array_and_samples(&m_output_buffers, nullptr, output_channel_count, output_frame_count, AVSampleFormat::AV_SAMPLE_FMT_FLTP, 0);
if (alloc_samples_result < 0)
return Error::from_string_view(av_error_code_to_string(alloc_samples_result));
VERIFY(m_output_buffer != nullptr);
VERIFY(m_output_buffers != nullptr);
m_output_buffer_frame_count = output_frame_count;
}
auto const* input_buffer = input_data.reinterpret<u8 const>().data();
// The input buffer size should already be safe to cast to int here.
auto input_frame_count = static_cast<int>(input_data.size() / m_input_sample_specification.channel_count());
auto input_frame_count = static_cast<int>(input.frame_count());
VERIFY(input_frame_count >= 0);
auto converted_frames_result = swr_convert(m_context, &m_output_buffer, m_output_buffer_frame_count, &input_buffer, input_frame_count);
Array<u8 const*, Audio::ChannelMap::capacity()> input_buffers;
for (size_t channel = 0; channel < input.channel_count(); channel++)
input_buffers[channel] = input.channel_data(channel).reinterpret<u8 const>().data();
auto converted_frames_result = swr_convert(m_context, m_output_buffers, m_output_buffer_frame_count, input_buffers.data(), input_frame_count);
if (converted_frames_result < 0)
return Error::from_string_view(av_error_code_to_string(converted_frames_result));
VERIFY(converted_frames_result <= m_output_buffer_frame_count);
auto converted_frames = static_cast<size_t>(converted_frames_result);
input.emplace(m_output_sample_specification, input.timestamp(), [&](AudioBlock::Data& data) {
data.resize_and_keep_capacity(converted_frames * output_channel_count);
AK::TypedTransfer<float>::copy(data.data(), reinterpret_cast<float*>(m_output_buffer), data.size());
});
input.initialize(m_output_sample_specification, input.timestamp(), converted_frames);
for (size_t channel = 0; channel < output_channel_count; channel++)
AK::TypedTransfer<float>::copy(input.channel_data(channel).data(), reinterpret_cast<float*>(m_output_buffers[channel]), converted_frames);
return {};
}
FFmpegAudioConverter::~FFmpegAudioConverter()
{
free_output_buffer();
swr_free(&m_context);
}

2
Libraries/LibMedia/FFmpeg/FFmpegAudioConverter.h
View file

@ -35,7 +35,7 @@ private:
Audio::SampleSpecification m_input_sample_specification;
Audio::SampleSpecification m_output_sample_specification;
SwrContext* m_context { nullptr };
u8* m_output_buffer { nullptr };
u8** m_output_buffers { nullptr };
int m_output_buffer_frame_count { 0 };
};

77
Libraries/LibMedia/FFmpeg/FFmpegAudioDecoder.cpp
View file

@ -169,48 +169,49 @@ DecoderErrorOr<void> FFmpegAudioDecoder::write_next_block(AudioBlock& block)
auto channel_map = channel_map_result.release_value();
auto sample_specification = Audio::SampleSpecification(m_frame->sample_rate, channel_map);
block.emplace(sample_specification, timestamp, [&](AudioBlock::Data& data) {
auto format = static_cast<AVSampleFormat>(m_frame->format);
auto is_planar = av_sample_fmt_is_planar(format) != 0;
auto planar_format = av_get_planar_sample_fmt(format);
auto format = static_cast<AVSampleFormat>(m_frame->format);
auto is_planar = av_sample_fmt_is_planar(format) != 0;
auto planar_format = av_get_planar_sample_fmt(format);
VERIFY(m_frame->nb_samples >= 0);
auto frame_count = static_cast<size_t>(m_frame->nb_samples);
auto channel_count = static_cast<size_t>(m_frame->ch_layout.nb_channels);
auto sample_count = frame_count * channel_count;
data.resize_and_keep_capacity(sample_count);
VERIFY(m_frame->nb_samples >= 0);
auto frame_count = static_cast<size_t>(m_frame->nb_samples);
auto channel_count = static_cast<size_t>(m_frame->ch_layout.nb_channels);
auto sample_count = frame_count * channel_count;
block.initialize(sample_specification, timestamp, frame_count);
auto sample_size = [&] {
switch (planar_format) {
case AV_SAMPLE_FMT_U8P:
return sizeof(u8);
case AV_SAMPLE_FMT_S16P:
return sizeof(i16);
case AV_SAMPLE_FMT_S32P:
return sizeof(i32);
case AV_SAMPLE_FMT_FLTP:
return sizeof(float);
case AV_SAMPLE_FMT_DBLP:
return sizeof(double);
case AV_SAMPLE_FMT_S64P:
return sizeof(i64);
default:
VERIFY_NOT_REACHED();
}
}();
auto sample_size = [&] {
switch (planar_format) {
case AV_SAMPLE_FMT_U8P:
return sizeof(u8);
case AV_SAMPLE_FMT_S16P:
return sizeof(i16);
case AV_SAMPLE_FMT_S32P:
return sizeof(i32);
case AV_SAMPLE_FMT_FLTP:
return sizeof(float);
case AV_SAMPLE_FMT_DBLP:
return sizeof(double);
case AV_SAMPLE_FMT_S64P:
return sizeof(i64);
default:
VERIFY_NOT_REACHED();
}
}();
VERIFY(m_frame->linesize[0] > 0);
if (is_planar)
VERIFY(static_cast<size_t>(m_frame->linesize[0]) >= frame_count * sample_size);
else
VERIFY(static_cast<size_t>(m_frame->linesize[0]) >= sample_count * sample_size);
VERIFY(m_frame->linesize[0] > 0);
if (is_planar)
VERIFY(static_cast<size_t>(m_frame->linesize[0]) >= frame_count * sample_size);
else
VERIFY(static_cast<size_t>(m_frame->linesize[0]) >= sample_count * sample_size);
for (size_t i = 0; i < sample_count; i++) {
for (size_t channel = 0; channel < channel_count; ++channel) {
auto channel_data = block.channel_data(channel);
for (size_t frame = 0; frame < frame_count; ++frame) {
size_t plane = 0;
size_t index_in_plane = i;
size_t index_in_plane = (frame * channel_count) + channel;
if (is_planar) {
plane = i % channel_count;
index_in_plane = i / channel_count;
plane = channel;
index_in_plane = frame;
}
auto float_sample = [&] {
@ -231,9 +232,9 @@ DecoderErrorOr<void> FFmpegAudioDecoder::write_next_block(AudioBlock& block)
VERIFY_NOT_REACHED();
}
}();
data[i] = float_sample;
channel_data[frame] = float_sample;
}
});
}
return {};
}

162
Libraries/LibMedia/Processors/AudioMixer.cpp
View file

@ -227,7 +227,6 @@ void AudioMixer::pull(AudioBlock& into)
auto buffer_start_frame = m_next_frame_to_write;
auto frames_end_cap = buffer_start_frame + static_cast<i64>(max_frame_count);
auto write_size = max_frame_count * channel_count;
auto combined_status_after_mix = PipelineStatus::EndOfStream;
i64 latest_mixed_frame = frames_end_cap;
@ -235,96 +234,97 @@ void AudioMixer::pull(AudioBlock& into)
for (auto& [input, input_data] : m_inputs)
input_data.next_frame = buffer_start_frame;
into.emplace(m_sample_specification, buffer_start_frame, [&](AudioBlock::Data& data) {
data.resize_and_keep_capacity(write_size);
for (size_t i = 0; i < write_size; i++)
data[i] = 0.0f;
into.initialize(m_sample_specification, buffer_start_frame, max_frame_count);
for (size_t channel = 0; channel < channel_count; ++channel)
into.channel_data(channel).fill(0.0f);
while (true) {
struct MixTarget {
AudioProducer& input;
InputMixingData& input_data;
};
auto mix_target = [&] {
Optional<MixTarget> result;
for (auto& [input, input_data] : m_inputs) {
if (input_data.next_frame >= frames_end_cap)
continue;
if (!result.has_value() || input_data.next_frame < result->input_data.next_frame)
result = { input, input_data };
}
return result;
}();
if (!mix_target.has_value())
break;
auto [input, input_data] = mix_target.release_value();
auto& current_block = input_data.current_block;
input_data.last_status = input.status();
while (input_data.last_status == PipelineStatus::MovedPosition) {
input.pull(current_block);
VERIFY(current_block.is_empty());
input_data.last_status = input.status();
}
auto current_block_is_usable = [&] {
if (current_block.is_empty())
return false;
if (current_block.sample_specification() != m_sample_specification)
return false;
if (current_block.end_timestamp_in_frames() <= input_data.next_frame)
return false;
return true;
}();
if (!current_block_is_usable) {
current_block.clear();
if (input_data.last_status == PipelineStatus::EndOfStream) {
input_data.next_frame = frames_end_cap;
while (true) {
struct MixTarget {
AudioProducer& input;
InputMixingData& input_data;
};
auto mix_target = [&] {
Optional<MixTarget> result;
for (auto& [input, input_data] : m_inputs) {
if (input_data.next_frame >= frames_end_cap)
continue;
}
if (input_data.last_status != PipelineStatus::HaveData)
break;
input.pull(current_block);
VERIFY(!current_block.is_empty());
continue;
if (!result.has_value() || input_data.next_frame < result->input_data.next_frame)
result = { input, input_data };
}
return result;
}();
if (!mix_target.has_value())
break;
auto [input, input_data] = mix_target.release_value();
auto first_frame_offset = current_block.timestamp_in_frames();
if (first_frame_offset >= frames_end_cap) {
auto& current_block = input_data.current_block;
input_data.last_status = input.status();
while (input_data.last_status == PipelineStatus::MovedPosition) {
input.pull(current_block);
VERIFY(current_block.is_empty());
input_data.last_status = input.status();
}
auto current_block_is_usable = [&] {
if (current_block.is_empty())
return false;
if (current_block.sample_specification() != m_sample_specification)
return false;
if (current_block.end_timestamp_in_frames() <= input_data.next_frame)
return false;
return true;
}();
if (!current_block_is_usable) {
current_block.clear();
if (input_data.last_status == PipelineStatus::EndOfStream) {
input_data.next_frame = frames_end_cap;
continue;
}
auto next_frame = max(input_data.next_frame, first_frame_offset);
VERIFY(next_frame >= first_frame_offset);
auto index_in_block = static_cast<size_t>((next_frame - first_frame_offset) * channel_count);
VERIFY(index_in_block < current_block.sample_count());
VERIFY(next_frame >= buffer_start_frame);
auto index_in_buffer = static_cast<size_t>((next_frame - buffer_start_frame) * channel_count);
VERIFY(index_in_buffer < write_size);
VERIFY(current_block.sample_count() >= index_in_block);
auto write_count = current_block.sample_count() - index_in_block;
write_count = min(write_count, write_size - index_in_buffer);
VERIFY(write_count > 0);
VERIFY(index_in_buffer + write_count <= write_size);
VERIFY(write_count % channel_count == 0);
for (size_t i = 0; i < write_count; i++)
data[index_in_buffer + i] += current_block.data()[index_in_block + i];
input_data.next_frame = next_frame + static_cast<i64>(write_count / channel_count);
if (input_data.last_status != PipelineStatus::HaveData)
break;
input.pull(current_block);
VERIFY(!current_block.is_empty());
continue;
}
for (auto& [input, input_data] : m_inputs) {
VERIFY(input_data.last_status != PipelineStatus::MovedPosition);
latest_mixed_frame = min(latest_mixed_frame, input_data.next_frame);
combined_status_after_mix = select_combined_pipeline_status(combined_status_after_mix, input_data.last_status);
auto first_frame_offset = current_block.timestamp_in_frames();
if (first_frame_offset >= frames_end_cap) {
input_data.next_frame = frames_end_cap;
continue;
}
});
auto next_frame = max(input_data.next_frame, first_frame_offset);
VERIFY(next_frame >= first_frame_offset);
auto frame_index_in_block = static_cast<size_t>(next_frame - first_frame_offset);
VERIFY(frame_index_in_block < current_block.frame_count());
VERIFY(next_frame >= buffer_start_frame);
auto frame_index_in_buffer = static_cast<size_t>(next_frame - buffer_start_frame);
VERIFY(frame_index_in_buffer < max_frame_count);
VERIFY(current_block.frame_count() >= frame_index_in_block);
auto frames_to_write = current_block.frame_count() - frame_index_in_block;
frames_to_write = min(frames_to_write, max_frame_count - frame_index_in_buffer);
VERIFY(frames_to_write > 0);
VERIFY(frame_index_in_buffer + frames_to_write <= max_frame_count);
for (size_t channel = 0; channel < channel_count; ++channel) {
auto input_channel = current_block.channel_data(channel).slice(frame_index_in_block, frames_to_write);
auto output_channel = into.channel_data(channel).slice(frame_index_in_buffer, frames_to_write);
for (size_t frame = 0; frame < frames_to_write; ++frame)
output_channel[frame] += input_channel[frame];
}
input_data.next_frame = next_frame + static_cast<i64>(frames_to_write);
}
for (auto& [input, input_data] : m_inputs) {
VERIFY(input_data.last_status != PipelineStatus::MovedPosition);
latest_mixed_frame = min(latest_mixed_frame, input_data.next_frame);
combined_status_after_mix = select_combined_pipeline_status(combined_status_after_mix, input_data.last_status);
}
VERIFY(latest_mixed_frame >= buffer_start_frame);
auto frame_count = static_cast<size_t>(latest_mixed_frame - buffer_start_frame);

10
Libraries/LibMedia/Sinks/AudioPlaybackSink.cpp
View file

@ -298,17 +298,13 @@ ReadonlySpan<float> AudioPlaybackSink::OutputThreadData::move_output_to_playback
continue;
}
auto offset_in_head_samples = static_cast<size_t>(m_next_frame_to_play - block_start_frame) * channel_count;
auto samples_remaining_in_head = head_block.sample_count() - offset_in_head_samples;
auto samples_to_copy = min(samples_remaining_in_head, buffer.size() - samples_written);
for (size_t i = 0; i < samples_to_copy; i++)
buffer[samples_written + i] = head_block.data()[offset_in_head_samples + i];
auto offset_in_head_frames = static_cast<size_t>(m_next_frame_to_play - block_start_frame);
auto samples_to_copy = head_block.copy_to_interleaved(buffer.slice(samples_written), offset_in_head_frames);
samples_written += samples_to_copy;
m_next_frame_to_play += static_cast<i64>(samples_to_copy / channel_count);
if (offset_in_head_samples + samples_to_copy == head_block.sample_count()) {
if ((offset_in_head_frames * channel_count) + samples_to_copy == head_block.sample_count()) {
m_block_head = (m_block_head + 1) % OUTPUT_BLOCK_QUEUE_CAPACITY;
m_block_count--;
}

22
Tests/LibMedia/TestFFmpegAudioNormalization.cpp
View file

@ -99,16 +99,18 @@ static void decode_and_expect()
producer->pull(block);
if (status == Media::PipelineStatus::HaveData) {
EXPECT(!block.is_empty());
for (float sample : block.data()) {
EXPECT(sample >= -1.0f);
if constexpr (sizeof(Sample) >= sizeof(i32))
EXPECT(sample <= 1.0f);
else
EXPECT(sample < 1.0f);
if (sample == -1.0f)
saw_negative_full_scale_sample = true;
if (sample > 0.0f)
saw_positive_peak_sample = true;
for (size_t channel = 0; channel < block.channel_count(); ++channel) {
for (float sample : block.channel_data(channel)) {
EXPECT(sample >= -1.0f);
if constexpr (sizeof(Sample) >= sizeof(i32))
EXPECT(sample <= 1.0f);
else
EXPECT(sample < 1.0f);
if (sample == -1.0f)
saw_negative_full_scale_sample = true;
if (sample > 0.0f)
saw_positive_peak_sample = true;
}
}
decoded_frame_count += block.frame_count();
} else if (status == Media::PipelineStatus::EndOfStream) {