LibMedia: Implement audio time stretching with WSOLA
The algorithm is ported over from Chromium's, which produces very good results for speech, while being not objectionable for music, especially in the background. Other algorithms were tested. Phase vocoders: - Bungee - Signalsmith Stretch - pvdoneright All three of these exhibited the usual phase shifting artifacts, causing speech to sound slightly shifted into the high end. Speech is the main thing we want to optimize for, so these aren't ideal. Sonic (TD-PSOLA) performs better than WSOLA for speech, especially at rates higher than 2x, but makes background sounds/music garbled and unpleasant. It is still worth considering for speech clarity, and could be added as an optional feature.
This commit is contained in:
Zaggy1024
Gregory Bertilson
parent
543db2b828
commit
28f670f09f
16 changed files with 1459 additions and 1 deletions
149
Libraries/LibMedia/Audio/AudioBuffer.cpp
Normal file
149
Libraries/LibMedia/Audio/AudioBuffer.cpp
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/Math.h>
|
||||
#include <AK/TypedTransfer.h>
|
||||
#include <LibMedia/Audio/AudioBuffer.h>
|
||||
|
||||
namespace Audio {
|
||||
|
||||
AudioBuffer::AudioBuffer(SampleSpecification sample_specification)
|
||||
: m_sample_specification(sample_specification)
|
||||
{
|
||||
}
|
||||
|
||||
void AudioBuffer::clear()
|
||||
{
|
||||
m_frame_count = 0;
|
||||
m_start_offset = 0;
|
||||
}
|
||||
|
||||
void AudioBuffer::ensure_capacity(size_t required_frame_capacity)
|
||||
{
|
||||
if (m_frame_capacity >= required_frame_capacity)
|
||||
return;
|
||||
|
||||
auto new_frame_capacity = max(required_frame_capacity, max<size_t>(m_frame_capacity * 2, 1));
|
||||
auto new_data = MUST(FixedArray<float>::create(new_frame_capacity * m_sample_specification.channel_count()));
|
||||
for (size_t channel = 0; channel < m_sample_specification.channel_count(); channel++) {
|
||||
auto new_channel = new_data.span().slice(channel * new_frame_capacity, m_frame_count);
|
||||
copy_channel_from_buffer(channel, 0, new_channel);
|
||||
}
|
||||
|
||||
m_data = move(new_data);
|
||||
m_frame_capacity = new_frame_capacity;
|
||||
m_start_offset = 0;
|
||||
}
|
||||
|
||||
void AudioBuffer::copy_channel_to_buffer(ReadonlySpan<float> source, size_t channel, size_t destination_offset)
|
||||
{
|
||||
VERIFY(channel < m_sample_specification.channel_count());
|
||||
VERIFY(destination_offset <= m_frame_capacity);
|
||||
VERIFY(source.size() <= m_frame_capacity - destination_offset);
|
||||
|
||||
auto write_offset = (m_start_offset + destination_offset) % m_frame_capacity;
|
||||
auto first_chunk_size = min(source.size(), m_frame_capacity - write_offset);
|
||||
auto channel_data = m_data.span().slice(channel * m_frame_capacity, m_frame_capacity);
|
||||
AK::TypedTransfer<float>::copy(channel_data.slice(write_offset).data(), source.data(), first_chunk_size);
|
||||
|
||||
auto second_chunk_size = source.size() - first_chunk_size;
|
||||
if (second_chunk_size > 0)
|
||||
AK::TypedTransfer<float>::copy(channel_data.data(), source.slice(first_chunk_size).data(), second_chunk_size);
|
||||
}
|
||||
|
||||
void AudioBuffer::copy_channel_from_buffer(size_t channel, size_t source_offset, Span<float> destination) const
|
||||
{
|
||||
VERIFY(channel < m_sample_specification.channel_count());
|
||||
VERIFY(source_offset <= m_frame_count);
|
||||
VERIFY(destination.size() <= m_frame_count - source_offset);
|
||||
|
||||
if (destination.is_empty())
|
||||
return;
|
||||
|
||||
auto read_offset = (m_start_offset + source_offset) % m_frame_capacity;
|
||||
auto first_chunk_size = min(destination.size(), m_frame_capacity - read_offset);
|
||||
auto channel_data = m_data.span().slice(channel * m_frame_capacity, m_frame_capacity);
|
||||
AK::TypedTransfer<float>::copy(destination.data(), channel_data.slice(read_offset).data(), first_chunk_size);
|
||||
|
||||
auto second_chunk_size = destination.size() - first_chunk_size;
|
||||
if (second_chunk_size > 0)
|
||||
AK::TypedTransfer<float>::copy(destination.slice(first_chunk_size).data(), channel_data.data(), second_chunk_size);
|
||||
}
|
||||
|
||||
void AudioBuffer::zero_channel(size_t channel, size_t destination_offset, size_t frame_count)
|
||||
{
|
||||
VERIFY(channel < m_sample_specification.channel_count());
|
||||
VERIFY(destination_offset <= m_frame_capacity);
|
||||
VERIFY(frame_count <= m_frame_capacity - destination_offset);
|
||||
|
||||
auto write_offset = (m_start_offset + destination_offset) % m_frame_capacity;
|
||||
auto first_chunk_size = min(frame_count, m_frame_capacity - write_offset);
|
||||
auto channel_data = m_data.span().slice(channel * m_frame_capacity, m_frame_capacity);
|
||||
for (auto& sample : channel_data.slice(write_offset, first_chunk_size))
|
||||
sample = 0.0f;
|
||||
|
||||
auto second_chunk_size = frame_count - first_chunk_size;
|
||||
if (second_chunk_size > 0) {
|
||||
for (auto& sample : channel_data.slice(0, second_chunk_size))
|
||||
sample = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
void AudioBuffer::append(Media::AudioBlock const& block)
|
||||
{
|
||||
VERIFY(block.sample_specification() == m_sample_specification);
|
||||
if (block.frame_count() == 0)
|
||||
return;
|
||||
|
||||
auto old_frame_count = m_frame_count;
|
||||
ensure_capacity(m_frame_count + block.frame_count());
|
||||
for (size_t channel = 0; channel < block.channel_count(); channel++)
|
||||
copy_channel_to_buffer(block.channel_data(channel), channel, old_frame_count);
|
||||
m_frame_count += block.frame_count();
|
||||
}
|
||||
|
||||
void AudioBuffer::append_silence(size_t frame_count)
|
||||
{
|
||||
if (frame_count == 0)
|
||||
return;
|
||||
|
||||
auto old_frame_count = m_frame_count;
|
||||
ensure_capacity(m_frame_count + frame_count);
|
||||
for (size_t channel = 0; channel < m_sample_specification.channel_count(); channel++)
|
||||
zero_channel(channel, old_frame_count, frame_count);
|
||||
m_frame_count += frame_count;
|
||||
}
|
||||
|
||||
void AudioBuffer::drop_front(size_t frame_count)
|
||||
{
|
||||
VERIFY(frame_count <= m_frame_count);
|
||||
if (frame_count == 0)
|
||||
return;
|
||||
|
||||
m_frame_count -= frame_count;
|
||||
if (m_frame_count == 0) {
|
||||
m_start_offset = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
m_start_offset = (m_start_offset + frame_count) % m_frame_capacity;
|
||||
}
|
||||
|
||||
void AudioBuffer::copy_frames_to(size_t source_offset, size_t frame_count, size_t destination_offset, Media::AudioBlock& destination) const
|
||||
{
|
||||
VERIFY(destination.sample_specification() == m_sample_specification);
|
||||
VERIFY(source_offset <= m_frame_count);
|
||||
VERIFY(frame_count <= m_frame_count - source_offset);
|
||||
VERIFY(destination_offset <= destination.frame_count());
|
||||
VERIFY(frame_count <= destination.frame_count() - destination_offset);
|
||||
|
||||
for (size_t channel = 0; channel < destination.channel_count(); channel++) {
|
||||
auto destination_channel = destination.channel_data(channel).slice(destination_offset, frame_count);
|
||||
copy_channel_from_buffer(channel, source_offset, destination_channel);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
40
Libraries/LibMedia/Audio/AudioBuffer.h
Normal file
40
Libraries/LibMedia/Audio/AudioBuffer.h
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/FixedArray.h>
|
||||
#include <LibMedia/Audio/SampleSpecification.h>
|
||||
#include <LibMedia/AudioBlock.h>
|
||||
|
||||
namespace Audio {
|
||||
|
||||
class AudioBuffer {
|
||||
public:
|
||||
explicit AudioBuffer(SampleSpecification);
|
||||
|
||||
size_t frame_count() const { return m_frame_count; }
|
||||
|
||||
void clear();
|
||||
void append(Media::AudioBlock const&);
|
||||
void append_silence(size_t frame_count);
|
||||
void drop_front(size_t frame_count);
|
||||
void copy_frames_to(size_t source_offset, size_t frame_count, size_t destination_offset, Media::AudioBlock&) const;
|
||||
|
||||
private:
|
||||
void ensure_capacity(size_t required_frame_capacity);
|
||||
void copy_channel_to_buffer(ReadonlySpan<float>, size_t channel, size_t destination_offset);
|
||||
void copy_channel_from_buffer(size_t channel, size_t source_offset, Span<float>) const;
|
||||
void zero_channel(size_t channel, size_t destination_offset, size_t frame_count);
|
||||
|
||||
SampleSpecification const m_sample_specification;
|
||||
FixedArray<float> m_data;
|
||||
size_t m_frame_capacity { 0 };
|
||||
size_t m_frame_count { 0 };
|
||||
size_t m_start_offset { 0 };
|
||||
};
|
||||
|
||||
}
|
||||
31
Libraries/LibMedia/Audio/TimeStretcher.h
Normal file
31
Libraries/LibMedia/Audio/TimeStretcher.h
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Error.h>
|
||||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <AK/Time.h>
|
||||
#include <LibMedia/AudioBlock.h>
|
||||
#include <LibMedia/DecoderError.h>
|
||||
|
||||
namespace Audio {
|
||||
|
||||
class TimeStretcher {
|
||||
public:
|
||||
virtual ~TimeStretcher() = default;
|
||||
|
||||
virtual i64 preroll_frame_count() const = 0;
|
||||
virtual void flush(AK::Duration media_start_timestamp, i64 output_start_frame_index) = 0;
|
||||
|
||||
virtual void set_rate(float) = 0;
|
||||
|
||||
virtual void push_block(Media::AudioBlock const&) = 0;
|
||||
virtual Media::DecoderErrorOr<Media::AudioBlock> retrieve_block() = 0;
|
||||
virtual void signal_end_of_stream() = 0;
|
||||
};
|
||||
|
||||
}
|
||||
277
Libraries/LibMedia/Audio/WSOLAAlgorithm.cpp
Normal file
277
Libraries/LibMedia/Audio/WSOLAAlgorithm.cpp
Normal file
|
|
@ -0,0 +1,277 @@
|
|||
/*
|
||||
* Copyright 2012 The Chromium Authors
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
// Source: chromium/media/filters/audio_renderer_algorithm.{h,cc}.
|
||||
|
||||
#include <AK/Math.h>
|
||||
#include <AK/TypedTransfer.h>
|
||||
#include <LibMedia/Audio/WSOLAAlgorithm.h>
|
||||
#include <LibMedia/Audio/WSOLAInternals.h>
|
||||
|
||||
namespace Audio {
|
||||
|
||||
namespace {
|
||||
|
||||
constexpr u64 OLA_WINDOW_SIZE_MS = 20;
|
||||
constexpr u64 WSOLA_SEARCH_INTERVAL_MS = 30;
|
||||
|
||||
size_t window_size(u32 sample_rate)
|
||||
{
|
||||
auto const frame_count = (static_cast<u64>(sample_rate) * OLA_WINDOW_SIZE_MS + 500) / 1000;
|
||||
return max(frame_count + (frame_count & 1), 2);
|
||||
}
|
||||
|
||||
size_t search_interval(u32 sample_rate)
|
||||
{
|
||||
auto const frame_count = (static_cast<u64>(sample_rate) * WSOLA_SEARCH_INTERVAL_MS + 500) / 1000;
|
||||
return max(frame_count, 1);
|
||||
}
|
||||
|
||||
void zero_block(Media::AudioBlock& block)
|
||||
{
|
||||
for (size_t channel = 0; channel < block.channel_count(); channel++) {
|
||||
auto channel_data = block.channel_data(channel);
|
||||
for (auto& sample : channel_data)
|
||||
sample = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
void zero_frames(Media::AudioBlock& block, size_t starting_frame, size_t frame_count)
|
||||
{
|
||||
VERIFY(starting_frame <= block.frame_count());
|
||||
VERIFY(frame_count <= block.frame_count() - starting_frame);
|
||||
for (size_t channel = 0; channel < block.channel_count(); channel++) {
|
||||
auto channel_data = block.channel_data(channel);
|
||||
for (size_t frame = 0; frame < frame_count; frame++)
|
||||
channel_data[starting_frame + frame] = 0.0f;
|
||||
}
|
||||
}
|
||||
|
||||
void copy_partial_frames(Media::AudioBlock const& source, size_t source_offset, size_t frame_count, size_t destination_offset, Media::AudioBlock& destination)
|
||||
{
|
||||
VERIFY(source.sample_specification() == destination.sample_specification());
|
||||
VERIFY(source_offset <= source.frame_count());
|
||||
VERIFY(frame_count <= source.frame_count() - source_offset);
|
||||
VERIFY(destination_offset <= destination.frame_count());
|
||||
VERIFY(frame_count <= destination.frame_count() - destination_offset);
|
||||
|
||||
for (size_t channel = 0; channel < source.channel_count(); channel++) {
|
||||
auto source_channel = source.channel_data(channel).slice(source_offset, frame_count);
|
||||
auto destination_channel = destination.channel_data(channel).slice(destination_offset, frame_count);
|
||||
AK::TypedTransfer<float>::copy(destination_channel.data(), source_channel.data(), frame_count);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
WSOLAAlgorithm::WSOLAAlgorithm(SampleSpecification sample_specification)
|
||||
: m_sample_specification(sample_specification)
|
||||
, m_audio_buffer(sample_specification)
|
||||
, m_ola_window_size(window_size(sample_rate()))
|
||||
, m_ola_hop_size(m_ola_window_size / 2)
|
||||
, m_num_candidate_blocks(search_interval(sample_rate()))
|
||||
, m_search_block_center_offset((m_num_candidate_blocks / 2) + ((m_ola_window_size / 2) - 1))
|
||||
{
|
||||
VERIFY(m_sample_specification.is_valid());
|
||||
|
||||
m_ola_window.resize(m_ola_window_size);
|
||||
WSOLAInternals::get_periodic_hanning_window(m_ola_window.span());
|
||||
|
||||
m_transition_window.resize(m_ola_window_size * 2);
|
||||
WSOLAInternals::get_periodic_hanning_window(m_transition_window.span());
|
||||
|
||||
m_wsola_output = create_block(m_ola_window_size + m_ola_hop_size);
|
||||
zero_block(m_wsola_output);
|
||||
|
||||
m_optimal_block = create_block(m_ola_window_size);
|
||||
m_search_block = create_block(m_num_candidate_blocks + (m_ola_window_size - 1));
|
||||
m_target_block = create_block(m_ola_window_size);
|
||||
}
|
||||
|
||||
WSOLAAlgorithm::~WSOLAAlgorithm() = default;
|
||||
|
||||
Media::AudioBlock WSOLAAlgorithm::create_block(size_t frame_count) const
|
||||
{
|
||||
Media::AudioBlock block;
|
||||
block.initialize(m_sample_specification, 0, frame_count);
|
||||
return block;
|
||||
}
|
||||
|
||||
void WSOLAAlgorithm::flush_buffers()
|
||||
{
|
||||
m_audio_buffer.clear();
|
||||
|
||||
m_input_position_remainder = 0.0f;
|
||||
m_search_block_center_index = 0;
|
||||
m_search_block_index = 0;
|
||||
m_target_block_index = 0;
|
||||
m_num_complete_frames = 0;
|
||||
|
||||
zero_block(m_wsola_output);
|
||||
}
|
||||
|
||||
void WSOLAAlgorithm::enqueue_buffer(Media::AudioBlock const& buffer_in)
|
||||
{
|
||||
VERIFY(buffer_in.sample_specification() == m_sample_specification);
|
||||
m_audio_buffer.append(buffer_in);
|
||||
}
|
||||
|
||||
void WSOLAAlgorithm::mark_end_of_stream(float playback_rate)
|
||||
{
|
||||
VERIFY(playback_rate > 0.0f);
|
||||
auto const search_block_size = m_num_candidate_blocks + (m_ola_window_size - 1);
|
||||
auto const padding = search_block_size + m_ola_window_size + m_ola_hop_size;
|
||||
m_audio_buffer.append_silence(padding);
|
||||
}
|
||||
|
||||
size_t WSOLAAlgorithm::fill_buffer(Media::AudioBlock& destination, size_t destination_offset, size_t requested_frames, float playback_rate)
|
||||
{
|
||||
if (playback_rate == 0.0f)
|
||||
return 0;
|
||||
VERIFY(playback_rate > 0.0f);
|
||||
VERIFY(destination.sample_specification() == m_sample_specification);
|
||||
|
||||
size_t rendered_frames = 0;
|
||||
do {
|
||||
rendered_frames += write_completed_frames_to(
|
||||
requested_frames - rendered_frames,
|
||||
destination_offset + rendered_frames, destination);
|
||||
} while (rendered_frames < requested_frames && run_one_wsola_iteration(playback_rate));
|
||||
return rendered_frames;
|
||||
}
|
||||
|
||||
bool WSOLAAlgorithm::can_perform_wsola() const
|
||||
{
|
||||
auto const search_block_size = m_num_candidate_blocks + (m_ola_window_size - 1);
|
||||
auto const frames = m_audio_buffer.frame_count();
|
||||
return m_target_block_index + static_cast<i64>(m_ola_window_size) <= static_cast<i64>(frames)
|
||||
&& m_search_block_index + static_cast<i64>(search_block_size) <= static_cast<i64>(frames);
|
||||
}
|
||||
|
||||
bool WSOLAAlgorithm::target_is_within_search_region() const
|
||||
{
|
||||
auto const search_block_size = m_num_candidate_blocks + (m_ola_window_size - 1);
|
||||
return m_target_block_index >= m_search_block_index
|
||||
&& m_target_block_index + static_cast<i64>(m_ola_window_size) <= m_search_block_index + static_cast<i64>(search_block_size);
|
||||
}
|
||||
|
||||
void WSOLAAlgorithm::peek_audio_with_zero_prepend(i64 read_offset_frames, Media::AudioBlock& destination)
|
||||
{
|
||||
auto const destination_frame_count = destination.frame_count();
|
||||
VERIFY(read_offset_frames <= static_cast<i64>(m_audio_buffer.frame_count()) - static_cast<i64>(destination_frame_count));
|
||||
|
||||
size_t write_offset = 0;
|
||||
auto frames_to_read = destination_frame_count;
|
||||
if (read_offset_frames < 0) {
|
||||
auto num_zero_frames_appended = min(-static_cast<size_t>(read_offset_frames), frames_to_read);
|
||||
read_offset_frames = 0;
|
||||
frames_to_read -= num_zero_frames_appended;
|
||||
write_offset = num_zero_frames_appended;
|
||||
zero_frames(destination, 0, num_zero_frames_appended);
|
||||
}
|
||||
if (frames_to_read > 0)
|
||||
m_audio_buffer.copy_frames_to(static_cast<size_t>(read_offset_frames), frames_to_read, write_offset, destination);
|
||||
}
|
||||
|
||||
void WSOLAAlgorithm::get_optimal_block()
|
||||
{
|
||||
i64 optimal_index = 0;
|
||||
|
||||
constexpr i64 exclude_interval_length_frames = 160;
|
||||
|
||||
if (target_is_within_search_region()) {
|
||||
optimal_index = m_target_block_index;
|
||||
peek_audio_with_zero_prepend(optimal_index, m_optimal_block);
|
||||
} else {
|
||||
peek_audio_with_zero_prepend(m_target_block_index, m_target_block);
|
||||
peek_audio_with_zero_prepend(m_search_block_index, m_search_block);
|
||||
auto last_optimal = m_target_block_index - static_cast<i64>(m_ola_hop_size) - m_search_block_index;
|
||||
auto exclude_interval_low = last_optimal - (exclude_interval_length_frames / 2);
|
||||
auto exclude_interval_high = last_optimal + (exclude_interval_length_frames / 2);
|
||||
WSOLAInternals::Interval exclude_interval {
|
||||
exclude_interval_low <= 0 ? 0 : static_cast<size_t>(exclude_interval_low),
|
||||
exclude_interval_high <= 0 ? 0 : static_cast<size_t>(exclude_interval_high),
|
||||
};
|
||||
|
||||
optimal_index = static_cast<i64>(WSOLAInternals::optimal_index(m_search_block, m_target_block, exclude_interval));
|
||||
|
||||
optimal_index += m_search_block_index;
|
||||
peek_audio_with_zero_prepend(optimal_index, m_optimal_block);
|
||||
|
||||
for (size_t channel_index = 0; channel_index < m_sample_specification.channel_count(); channel_index++) {
|
||||
auto optimal_channel = m_optimal_block.channel_data(channel_index);
|
||||
auto target_channel = m_target_block.channel_data(channel_index);
|
||||
for (size_t n = 0; n < m_ola_window_size; n++) {
|
||||
optimal_channel[n] = (optimal_channel[n] * m_transition_window[n])
|
||||
+ (target_channel[n] * m_transition_window[m_ola_window_size + n]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m_target_block_index = optimal_index + static_cast<i64>(m_ola_hop_size);
|
||||
}
|
||||
|
||||
bool WSOLAAlgorithm::run_one_wsola_iteration(float playback_rate)
|
||||
{
|
||||
if (!can_perform_wsola())
|
||||
return false;
|
||||
|
||||
get_optimal_block();
|
||||
|
||||
for (size_t channel_index = 0; channel_index < m_sample_specification.channel_count(); channel_index++) {
|
||||
auto optimal_channel = m_optimal_block.channel_data(channel_index);
|
||||
auto output_channel = m_wsola_output.channel_data(channel_index).slice(m_num_complete_frames);
|
||||
|
||||
for (size_t n = 0; n < m_ola_hop_size; n++) {
|
||||
output_channel[n] = output_channel[n] * m_ola_window[m_ola_hop_size + n];
|
||||
output_channel[n] += optimal_channel[n] * m_ola_window[n];
|
||||
}
|
||||
|
||||
for (size_t n = 0; n < m_ola_hop_size; n++)
|
||||
output_channel[m_ola_hop_size + n] = optimal_channel[m_ola_hop_size + n];
|
||||
}
|
||||
|
||||
m_num_complete_frames += m_ola_hop_size;
|
||||
auto input_position_advance = (static_cast<float>(m_ola_hop_size) * playback_rate) + m_input_position_remainder;
|
||||
auto whole_input_frames = AK::round_to<i64>(input_position_advance);
|
||||
m_input_position_remainder = input_position_advance - static_cast<float>(whole_input_frames);
|
||||
m_search_block_center_index += whole_input_frames;
|
||||
m_search_block_index = m_search_block_center_index - static_cast<i64>(m_search_block_center_offset);
|
||||
remove_old_input_frames();
|
||||
return true;
|
||||
}
|
||||
|
||||
void WSOLAAlgorithm::remove_old_input_frames()
|
||||
{
|
||||
auto const earliest_used_index = min(m_target_block_index, m_search_block_index);
|
||||
if (earliest_used_index <= 0)
|
||||
return;
|
||||
|
||||
m_audio_buffer.drop_front(static_cast<size_t>(earliest_used_index));
|
||||
m_target_block_index -= earliest_used_index;
|
||||
m_search_block_center_index -= earliest_used_index;
|
||||
m_search_block_index -= earliest_used_index;
|
||||
}
|
||||
|
||||
size_t WSOLAAlgorithm::write_completed_frames_to(size_t requested_frames, size_t destination_offset, Media::AudioBlock& destination)
|
||||
{
|
||||
auto const rendered_frames = min(m_num_complete_frames, requested_frames);
|
||||
if (rendered_frames == 0)
|
||||
return 0;
|
||||
|
||||
copy_partial_frames(m_wsola_output, 0, rendered_frames, destination_offset, destination);
|
||||
|
||||
auto const frames_to_move = m_wsola_output.frame_count() - rendered_frames;
|
||||
for (size_t channel_index = 0; channel_index < m_sample_specification.channel_count(); channel_index++) {
|
||||
auto channel_data = m_wsola_output.channel_data(channel_index);
|
||||
for (size_t i = 0; i < frames_to_move; i++)
|
||||
channel_data[i] = channel_data[i + rendered_frames];
|
||||
}
|
||||
m_num_complete_frames -= rendered_frames;
|
||||
return rendered_frames;
|
||||
}
|
||||
|
||||
}
|
||||
85
Libraries/LibMedia/Audio/WSOLAAlgorithm.h
Normal file
85
Libraries/LibMedia/Audio/WSOLAAlgorithm.h
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
/*
|
||||
* Copyright 2012 The Chromium Authors
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
// Source: chromium/media/filters/audio_renderer_algorithm.{h,cc}.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Vector.h>
|
||||
#include <LibMedia/Audio/AudioBuffer.h>
|
||||
#include <LibMedia/Audio/SampleSpecification.h>
|
||||
#include <LibMedia/AudioBlock.h>
|
||||
|
||||
namespace Audio {
|
||||
|
||||
class WSOLAAlgorithm {
|
||||
public:
|
||||
explicit WSOLAAlgorithm(SampleSpecification);
|
||||
~WSOLAAlgorithm();
|
||||
|
||||
WSOLAAlgorithm(WSOLAAlgorithm const&) = delete;
|
||||
WSOLAAlgorithm& operator=(WSOLAAlgorithm const&) = delete;
|
||||
|
||||
size_t fill_buffer(Media::AudioBlock& destination, size_t destination_offset, size_t requested_frames, float playback_rate);
|
||||
|
||||
void enqueue_buffer(Media::AudioBlock const&);
|
||||
|
||||
void flush_buffers();
|
||||
|
||||
void mark_end_of_stream(float playback_rate);
|
||||
|
||||
size_t buffered_frames() const { return m_audio_buffer.frame_count(); }
|
||||
|
||||
SampleSpecification const& sample_specification() const { return m_sample_specification; }
|
||||
u32 sample_rate() const { return m_sample_specification.sample_rate(); }
|
||||
u8 channel_count() const { return m_sample_specification.channel_count(); }
|
||||
size_t ola_window_size() const { return m_ola_window_size; }
|
||||
size_t ola_hop_size() const { return m_ola_hop_size; }
|
||||
|
||||
private:
|
||||
bool run_one_wsola_iteration(float playback_rate);
|
||||
|
||||
size_t write_completed_frames_to(size_t requested_frames, size_t destination_offset, Media::AudioBlock&);
|
||||
|
||||
void peek_audio_with_zero_prepend(i64 read_offset_frames, Media::AudioBlock& destination);
|
||||
|
||||
void get_optimal_block();
|
||||
|
||||
bool can_perform_wsola() const;
|
||||
|
||||
bool target_is_within_search_region() const;
|
||||
|
||||
void remove_old_input_frames();
|
||||
|
||||
Media::AudioBlock create_block(size_t frame_count) const;
|
||||
|
||||
SampleSpecification const m_sample_specification;
|
||||
|
||||
AudioBuffer m_audio_buffer;
|
||||
|
||||
size_t const m_ola_window_size;
|
||||
size_t const m_ola_hop_size;
|
||||
size_t const m_num_candidate_blocks;
|
||||
size_t const m_search_block_center_offset;
|
||||
|
||||
float m_input_position_remainder { 0.0f };
|
||||
i64 m_search_block_index { 0 };
|
||||
i64 m_search_block_center_index { 0 };
|
||||
i64 m_target_block_index { 0 };
|
||||
|
||||
size_t m_num_complete_frames { 0 };
|
||||
|
||||
Media::AudioBlock m_wsola_output;
|
||||
|
||||
Vector<float> m_ola_window;
|
||||
Vector<float> m_transition_window;
|
||||
|
||||
Media::AudioBlock m_optimal_block;
|
||||
Media::AudioBlock m_search_block;
|
||||
Media::AudioBlock m_target_block;
|
||||
};
|
||||
|
||||
}
|
||||
232
Libraries/LibMedia/Audio/WSOLAInternals.cpp
Normal file
232
Libraries/LibMedia/Audio/WSOLAInternals.cpp
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
/*
|
||||
* Copyright 2013 The Chromium Authors
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
// Source: chromium/media/filters/wsola_internals.{h,cc}.
|
||||
|
||||
#include <AK/Math.h>
|
||||
#include <AK/Vector.h>
|
||||
#include <LibMedia/Audio/WSOLAInternals.h>
|
||||
#include <LibMedia/AudioBlock.h>
|
||||
|
||||
namespace Audio::WSOLAInternals {
|
||||
|
||||
namespace {
|
||||
|
||||
bool in_interval(size_t n, Interval interval)
|
||||
{
|
||||
return n >= interval.low && n <= interval.high;
|
||||
}
|
||||
|
||||
float multi_channel_similarity_measure(ReadonlySpan<float> dot_prod_a_b,
|
||||
ReadonlySpan<float> energy_a, ReadonlySpan<float> energy_b)
|
||||
{
|
||||
VERIFY(dot_prod_a_b.size() == energy_a.size());
|
||||
VERIFY(energy_a.size() == energy_b.size());
|
||||
constexpr float epsilon = 1e-12f;
|
||||
auto similarity_measure = 0.0f;
|
||||
for (size_t n = 0; n < dot_prod_a_b.size(); n++)
|
||||
similarity_measure += dot_prod_a_b[n] / AK::sqrt((energy_a[n] * energy_b[n]) + epsilon);
|
||||
return similarity_measure;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void multi_channel_dot_product(Media::AudioBlock const& a, size_t frame_offset_a,
|
||||
Media::AudioBlock const& b, size_t frame_offset_b,
|
||||
size_t num_frames, Span<float> dot_product)
|
||||
{
|
||||
VERIFY(a.channel_count() == b.channel_count());
|
||||
VERIFY(dot_product.size() == a.channel_count());
|
||||
VERIFY(frame_offset_a <= a.frame_count());
|
||||
VERIFY(num_frames <= a.frame_count() - frame_offset_a);
|
||||
VERIFY(frame_offset_b <= b.frame_count());
|
||||
VERIFY(num_frames <= b.frame_count() - frame_offset_b);
|
||||
|
||||
for (size_t channel_index = 0; channel_index < a.channel_count(); channel_index++) {
|
||||
auto channel_a = a.channel_data(channel_index).slice(frame_offset_a, num_frames);
|
||||
auto channel_b = b.channel_data(channel_index).slice(frame_offset_b, num_frames);
|
||||
auto sum = 0.0f;
|
||||
for (size_t i = 0; i < num_frames; i++)
|
||||
sum += channel_a[i] * channel_b[i];
|
||||
dot_product[channel_index] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
void multi_channel_moving_block_energies(Media::AudioBlock const& input,
|
||||
size_t frames_per_window, Span<float> energy)
|
||||
{
|
||||
auto num_blocks = input.frame_count() - (frames_per_window - 1);
|
||||
auto channel_count = static_cast<size_t>(input.channel_count());
|
||||
VERIFY(energy.size() == num_blocks * channel_count);
|
||||
|
||||
for (size_t channel_index = 0; channel_index < input.channel_count(); channel_index++) {
|
||||
auto input_channel = input.channel_data(channel_index);
|
||||
|
||||
auto first_block_energy = 0.0f;
|
||||
for (size_t i = 0; i < frames_per_window; i++)
|
||||
first_block_energy += input_channel[i] * input_channel[i];
|
||||
energy[channel_index] = first_block_energy;
|
||||
|
||||
for (size_t block_index = 1; block_index < num_blocks; block_index++) {
|
||||
auto leaving_sample = input_channel[block_index - 1];
|
||||
auto entering_sample = input_channel[block_index + frames_per_window - 1];
|
||||
energy[channel_index + (block_index * channel_count)] = energy[channel_index + ((block_index - 1) * channel_count)]
|
||||
- (leaving_sample * leaving_sample)
|
||||
+ (entering_sample * entering_sample);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void quadratic_interpolation(ReadonlySpan<float> y_values, float& extremum, float& extremum_value)
|
||||
{
|
||||
VERIFY(y_values.size() == 3);
|
||||
auto const a = (0.5f * (y_values[2] + y_values[0])) - y_values[1];
|
||||
auto const b = 0.5f * (y_values[2] - y_values[0]);
|
||||
auto const c = y_values[1];
|
||||
|
||||
if (a == 0.0f) {
|
||||
extremum = 0.0f;
|
||||
extremum_value = y_values[1];
|
||||
} else {
|
||||
auto const ext = -b / (2.0f * a);
|
||||
extremum = ext;
|
||||
extremum_value = (a * ext * ext) + (b * ext) + c;
|
||||
}
|
||||
}
|
||||
|
||||
size_t decimated_search(size_t decimation, Interval exclude_interval,
|
||||
Media::AudioBlock const& target_block, Media::AudioBlock const& search_segment,
|
||||
ReadonlySpan<float> energy_target_block,
|
||||
ReadonlySpan<float> energy_candidate_blocks)
|
||||
{
|
||||
auto channel_count = static_cast<size_t>(search_segment.channel_count());
|
||||
auto block_size = target_block.frame_count();
|
||||
auto num_candidate_blocks = search_segment.frame_count() - (block_size - 1);
|
||||
Vector<float> dot_product;
|
||||
dot_product.resize(channel_count);
|
||||
float similarity[3];
|
||||
|
||||
size_t n = 0;
|
||||
multi_channel_dot_product(target_block, 0, search_segment, n, block_size, dot_product.span());
|
||||
similarity[0] = multi_channel_similarity_measure(
|
||||
dot_product.span(), energy_target_block,
|
||||
energy_candidate_blocks.slice(n * channel_count, channel_count));
|
||||
|
||||
auto best_similarity = similarity[0];
|
||||
size_t optimal_block_index = 0;
|
||||
|
||||
n += decimation;
|
||||
if (n >= num_candidate_blocks)
|
||||
return 0;
|
||||
|
||||
multi_channel_dot_product(target_block, 0, search_segment, n, block_size, dot_product.span());
|
||||
similarity[1] = multi_channel_similarity_measure(
|
||||
dot_product.span(), energy_target_block,
|
||||
energy_candidate_blocks.slice(n * channel_count, channel_count));
|
||||
|
||||
n += decimation;
|
||||
if (n >= num_candidate_blocks)
|
||||
return similarity[1] > similarity[0] ? decimation : 0;
|
||||
|
||||
for (; n < num_candidate_blocks; n += decimation) {
|
||||
multi_channel_dot_product(target_block, 0, search_segment, n, block_size, dot_product.span());
|
||||
similarity[2] = multi_channel_similarity_measure(
|
||||
dot_product.span(), energy_target_block,
|
||||
energy_candidate_blocks.slice(n * channel_count, channel_count));
|
||||
|
||||
bool is_local_max = (similarity[1] > similarity[0] && similarity[1] >= similarity[2])
|
||||
|| (similarity[1] >= similarity[0] && similarity[1] > similarity[2]);
|
||||
|
||||
if (is_local_max) {
|
||||
float normalized_candidate_index;
|
||||
float candidate_similarity;
|
||||
quadratic_interpolation({ similarity, 3 }, normalized_candidate_index, candidate_similarity);
|
||||
|
||||
auto candidate_index = static_cast<i64>(n - decimation)
|
||||
+ AK::round_to<i64>(normalized_candidate_index * static_cast<float>(decimation));
|
||||
if (candidate_similarity > best_similarity
|
||||
&& candidate_index >= 0
|
||||
&& !in_interval(static_cast<size_t>(candidate_index), exclude_interval)) {
|
||||
optimal_block_index = static_cast<size_t>(candidate_index);
|
||||
best_similarity = candidate_similarity;
|
||||
}
|
||||
} else if (n + decimation >= num_candidate_blocks
|
||||
&& similarity[2] > best_similarity
|
||||
&& !in_interval(n, exclude_interval)) {
|
||||
optimal_block_index = n;
|
||||
best_similarity = similarity[2];
|
||||
}
|
||||
|
||||
similarity[0] = similarity[1];
|
||||
similarity[1] = similarity[2];
|
||||
}
|
||||
return optimal_block_index;
|
||||
}
|
||||
|
||||
size_t full_search(size_t low_limit, size_t high_limit, Interval exclude_interval,
|
||||
Media::AudioBlock const& target_block, Media::AudioBlock const& search_block,
|
||||
ReadonlySpan<float> energy_target_block,
|
||||
ReadonlySpan<float> energy_candidate_blocks)
|
||||
{
|
||||
auto channel_count = static_cast<size_t>(search_block.channel_count());
|
||||
auto block_size = target_block.frame_count();
|
||||
Vector<float> dot_product;
|
||||
dot_product.resize(channel_count);
|
||||
|
||||
auto best_similarity = -AK::Infinity<float>;
|
||||
size_t optimal_block_index = 0;
|
||||
|
||||
for (size_t n = low_limit; n <= high_limit; n++) {
|
||||
if (in_interval(n, exclude_interval))
|
||||
continue;
|
||||
multi_channel_dot_product(target_block, 0, search_block, n, block_size, dot_product.span());
|
||||
auto similarity = multi_channel_similarity_measure(
|
||||
dot_product.span(), energy_target_block,
|
||||
energy_candidate_blocks.slice(n * channel_count, channel_count));
|
||||
if (similarity > best_similarity) {
|
||||
best_similarity = similarity;
|
||||
optimal_block_index = n;
|
||||
}
|
||||
}
|
||||
return optimal_block_index;
|
||||
}
|
||||
|
||||
size_t optimal_index(Media::AudioBlock const& search_block, Media::AudioBlock const& target_block,
|
||||
Interval exclude_interval)
|
||||
{
|
||||
VERIFY(search_block.channel_count() == target_block.channel_count());
|
||||
auto channel_count = static_cast<size_t>(search_block.channel_count());
|
||||
auto target_size = target_block.frame_count();
|
||||
auto num_candidate_blocks = search_block.frame_count() - (target_size - 1);
|
||||
|
||||
constexpr size_t search_decimation = 5;
|
||||
|
||||
Vector<float> energy_target_block;
|
||||
energy_target_block.resize(channel_count);
|
||||
Vector<float> energy_candidate_blocks;
|
||||
energy_candidate_blocks.resize(channel_count * num_candidate_blocks);
|
||||
|
||||
multi_channel_moving_block_energies(search_block, target_size, energy_candidate_blocks.span());
|
||||
multi_channel_dot_product(target_block, 0, target_block, 0, target_size, energy_target_block.span());
|
||||
|
||||
auto coarse_index = decimated_search(
|
||||
search_decimation, exclude_interval, target_block, search_block,
|
||||
energy_target_block.span(), energy_candidate_blocks.span());
|
||||
|
||||
size_t low_limit = coarse_index < search_decimation ? 0 : coarse_index - search_decimation;
|
||||
auto high_limit = min(num_candidate_blocks - 1, coarse_index + search_decimation);
|
||||
return full_search(low_limit, high_limit, exclude_interval, target_block, search_block,
|
||||
energy_target_block.span(), energy_candidate_blocks.span());
|
||||
}
|
||||
|
||||
void get_periodic_hanning_window(Span<float> window)
|
||||
{
|
||||
auto const scale = 2.0f * AK::Pi<float> / static_cast<float>(window.size());
|
||||
for (size_t n = 0; n < window.size(); n++)
|
||||
window[n] = 0.5f * (1.0f - AK::cos(static_cast<float>(n) * scale));
|
||||
}
|
||||
|
||||
}
|
||||
52
Libraries/LibMedia/Audio/WSOLAInternals.h
Normal file
52
Libraries/LibMedia/Audio/WSOLAInternals.h
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
/*
|
||||
* Copyright 2013 The Chromium Authors
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*/
|
||||
// Source: chromium/media/filters/wsola_internals.{h,cc}.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Span.h>
|
||||
#include <AK/Types.h>
|
||||
|
||||
namespace Media {
|
||||
|
||||
class AudioBlock;
|
||||
|
||||
}
|
||||
|
||||
namespace Audio::WSOLAInternals {
|
||||
|
||||
struct Interval {
|
||||
size_t low;
|
||||
size_t high;
|
||||
};
|
||||
|
||||
void multi_channel_dot_product(Media::AudioBlock const& a, size_t frame_offset_a,
|
||||
Media::AudioBlock const& b, size_t frame_offset_b,
|
||||
size_t num_frames, Span<float> dot_product);
|
||||
|
||||
void multi_channel_moving_block_energies(Media::AudioBlock const& input,
|
||||
size_t frames_per_window, Span<float> energy);
|
||||
|
||||
void quadratic_interpolation(ReadonlySpan<float> y_values,
|
||||
float& extremum, float& extremum_value);
|
||||
|
||||
size_t decimated_search(size_t decimation, Interval exclude_interval,
|
||||
Media::AudioBlock const& target_block, Media::AudioBlock const& search_segment,
|
||||
ReadonlySpan<float> energy_target_block,
|
||||
ReadonlySpan<float> energy_candidate_blocks);
|
||||
|
||||
size_t full_search(size_t low_limit, size_t high_limit, Interval exclude_interval,
|
||||
Media::AudioBlock const& target_block, Media::AudioBlock const& search_block,
|
||||
ReadonlySpan<float> energy_target_block,
|
||||
ReadonlySpan<float> energy_candidate_blocks);
|
||||
|
||||
size_t optimal_index(Media::AudioBlock const& search_block, Media::AudioBlock const& target_block,
|
||||
Interval exclude_interval);
|
||||
|
||||
void get_periodic_hanning_window(Span<float> window);
|
||||
|
||||
}
|
||||
187
Libraries/LibMedia/Audio/WSOLATimeStretcher.cpp
Normal file
187
Libraries/LibMedia/Audio/WSOLATimeStretcher.cpp
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/Math.h>
|
||||
#include <AK/OwnPtr.h>
|
||||
#include <AK/SaturatingMath.h>
|
||||
#include <LibMedia/Audio/WSOLAAlgorithm.h>
|
||||
#include <LibMedia/Audio/WSOLATimeStretcher.h>
|
||||
|
||||
namespace Audio {
|
||||
|
||||
struct WSOLATimeStretcher::Impl {
|
||||
SampleSpecification sample_specification;
|
||||
|
||||
NonnullOwnPtr<WSOLAAlgorithm> algorithm;
|
||||
|
||||
float rate { 1.0f };
|
||||
|
||||
i64 first_input_frame_after_flush { 0 };
|
||||
i64 next_output_frame_index { 0 };
|
||||
i64 expected_next_input_media_frame { 0 };
|
||||
i64 input_frames_consumed_since_flush { 0 };
|
||||
i64 input_end_frame_at_eos { 0 };
|
||||
float media_frames_remainder { 0.0f };
|
||||
|
||||
bool eos_signalled { false };
|
||||
|
||||
size_t output_chunk_frames() const
|
||||
{
|
||||
return max<size_t>(1, AK::round_to<size_t>(sample_specification.sample_rate() * 0.03));
|
||||
}
|
||||
|
||||
Impl(SampleSpecification sample_specification)
|
||||
: sample_specification(sample_specification)
|
||||
, algorithm(make<WSOLAAlgorithm>(sample_specification))
|
||||
{
|
||||
}
|
||||
};
|
||||
|
||||
ErrorOr<NonnullOwnPtr<TimeStretcher>> WSOLATimeStretcher::create(SampleSpecification sample_specification)
|
||||
{
|
||||
if (!sample_specification.is_valid())
|
||||
return Error::from_string_literal("Invalid sample specification");
|
||||
|
||||
auto impl = adopt_own(*new Impl(sample_specification));
|
||||
return adopt_own(*new WSOLATimeStretcher(move(impl)));
|
||||
}
|
||||
|
||||
WSOLATimeStretcher::WSOLATimeStretcher(NonnullOwnPtr<Impl> impl)
|
||||
: m_impl(move(impl))
|
||||
{
|
||||
}
|
||||
|
||||
WSOLATimeStretcher::~WSOLATimeStretcher() = default;
|
||||
|
||||
i64 WSOLATimeStretcher::preroll_frame_count() const
|
||||
{
|
||||
auto const& impl = *m_impl;
|
||||
return static_cast<i64>(AK::ceil(static_cast<float>(impl.algorithm->ola_window_size()) * impl.rate));
|
||||
}
|
||||
|
||||
void WSOLATimeStretcher::set_rate(float rate)
|
||||
{
|
||||
VERIFY(isfinite(rate));
|
||||
VERIFY(rate > 0.0f);
|
||||
m_impl->rate = rate;
|
||||
}
|
||||
|
||||
void WSOLATimeStretcher::flush(AK::Duration media_start_timestamp, i64 output_start_frame_index)
|
||||
{
|
||||
m_impl->algorithm->flush_buffers();
|
||||
m_impl->input_frames_consumed_since_flush = 0;
|
||||
m_impl->input_end_frame_at_eos = 0;
|
||||
m_impl->media_frames_remainder = 0.0f;
|
||||
m_impl->eos_signalled = false;
|
||||
|
||||
auto media_start_in_frames = media_start_timestamp.to_time_units(1, m_impl->sample_specification.sample_rate());
|
||||
m_impl->first_input_frame_after_flush = media_start_in_frames;
|
||||
m_impl->next_output_frame_index = output_start_frame_index;
|
||||
m_impl->expected_next_input_media_frame = media_start_in_frames;
|
||||
}
|
||||
|
||||
void WSOLATimeStretcher::push_block(Media::AudioBlock const& input)
|
||||
{
|
||||
VERIFY(!input.is_empty());
|
||||
auto& impl = *m_impl;
|
||||
VERIFY(input.sample_specification() == impl.sample_specification);
|
||||
|
||||
auto block_start = input.first_frame_index();
|
||||
auto frame_count = input.frame_count();
|
||||
size_t frames_to_skip = 0;
|
||||
|
||||
auto gap = saturating_sub(block_start, impl.expected_next_input_media_frame);
|
||||
if (gap > 0) {
|
||||
constexpr i64 max_silence_chunk = 4096;
|
||||
while (gap > 0) {
|
||||
auto chunk_frame_count = static_cast<size_t>(min<i64>(gap, max_silence_chunk));
|
||||
Media::AudioBlock silence;
|
||||
silence.initialize(impl.sample_specification, impl.expected_next_input_media_frame, chunk_frame_count);
|
||||
for (size_t channel_index = 0; channel_index < silence.channel_count(); channel_index++) {
|
||||
auto channel = silence.channel_data(channel_index);
|
||||
for (auto& sample : channel)
|
||||
sample = 0.0f;
|
||||
}
|
||||
impl.algorithm->enqueue_buffer(silence);
|
||||
impl.expected_next_input_media_frame = saturating_add(impl.expected_next_input_media_frame, AK::clamp_to<i64>(chunk_frame_count));
|
||||
gap -= static_cast<i64>(chunk_frame_count);
|
||||
}
|
||||
} else if (gap < 0) {
|
||||
frames_to_skip = min(-static_cast<size_t>(gap), frame_count);
|
||||
if (frames_to_skip == frame_count) {
|
||||
impl.expected_next_input_media_frame = max(
|
||||
impl.expected_next_input_media_frame, saturating_add(block_start, AK::clamp_to<i64>(frame_count)));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
auto const frames_to_append = frame_count - frames_to_skip;
|
||||
Media::AudioBlock block_to_append;
|
||||
block_to_append.initialize(impl.sample_specification, saturating_add(block_start, AK::clamp_to<i64>(frames_to_skip)), frames_to_append);
|
||||
for (size_t channel_index = 0; channel_index < input.channel_count(); channel_index++) {
|
||||
auto input_channel = input.channel_data(channel_index).slice(frames_to_skip, frames_to_append);
|
||||
auto output_channel = block_to_append.channel_data(channel_index);
|
||||
AK::TypedTransfer<float>::copy(output_channel.data(), input_channel.data(), frames_to_append);
|
||||
}
|
||||
|
||||
impl.algorithm->enqueue_buffer(block_to_append);
|
||||
impl.expected_next_input_media_frame = saturating_add(block_start, AK::clamp_to<i64>(frame_count));
|
||||
}
|
||||
|
||||
void WSOLATimeStretcher::signal_end_of_stream()
|
||||
{
|
||||
if (m_impl->eos_signalled)
|
||||
return;
|
||||
m_impl->input_end_frame_at_eos = m_impl->expected_next_input_media_frame;
|
||||
m_impl->eos_signalled = true;
|
||||
m_impl->algorithm->mark_end_of_stream(m_impl->rate);
|
||||
}
|
||||
|
||||
Media::DecoderErrorOr<Media::AudioBlock> WSOLATimeStretcher::retrieve_block()
|
||||
{
|
||||
auto const target_output_count = m_impl->output_chunk_frames();
|
||||
|
||||
Media::AudioBlock output_block;
|
||||
output_block.initialize(m_impl->sample_specification, m_impl->next_output_frame_index, target_output_count);
|
||||
auto const rendered = m_impl->algorithm->fill_buffer(output_block, 0, target_output_count, m_impl->rate);
|
||||
|
||||
if (rendered == 0) {
|
||||
if (m_impl->eos_signalled)
|
||||
return Media::DecoderError::with_description(Media::DecoderErrorCategory::EndOfStream, "End of stream"sv);
|
||||
return Media::DecoderError::with_description(Media::DecoderErrorCategory::NeedsMoreInput, "Need more input"sv);
|
||||
}
|
||||
|
||||
output_block.trim(rendered);
|
||||
|
||||
auto media_start_in_frames = saturating_add(m_impl->first_input_frame_after_flush, m_impl->input_frames_consumed_since_flush);
|
||||
auto media_duration_in_frames_fractional = (static_cast<float>(rendered) * m_impl->rate) + m_impl->media_frames_remainder;
|
||||
if (m_impl->eos_signalled) {
|
||||
if (m_impl->input_end_frame_at_eos < media_start_in_frames)
|
||||
return Media::DecoderError::with_description(Media::DecoderErrorCategory::EndOfStream, "End of stream"sv);
|
||||
auto remaining_media_frames = static_cast<float>(m_impl->input_end_frame_at_eos - media_start_in_frames);
|
||||
if (media_duration_in_frames_fractional > remaining_media_frames) {
|
||||
auto remaining_output_frames_from_this_block = remaining_media_frames / m_impl->rate;
|
||||
auto frames_to_keep = min(rendered, static_cast<size_t>(remaining_output_frames_from_this_block));
|
||||
if (frames_to_keep == 0)
|
||||
return Media::DecoderError::with_description(Media::DecoderErrorCategory::EndOfStream, "End of stream"sv);
|
||||
output_block.trim(frames_to_keep);
|
||||
media_duration_in_frames_fractional = static_cast<float>(frames_to_keep) * m_impl->rate;
|
||||
}
|
||||
}
|
||||
|
||||
auto media_duration_in_frames = static_cast<i64>(media_duration_in_frames_fractional);
|
||||
|
||||
output_block.set_media_time_start(AK::Duration::from_time_units(media_start_in_frames, 1, m_impl->sample_specification.sample_rate()));
|
||||
output_block.set_media_time_duration(AK::Duration::from_time_units(media_duration_in_frames, 1, m_impl->sample_specification.sample_rate()));
|
||||
|
||||
m_impl->next_output_frame_index = saturating_add(m_impl->next_output_frame_index, AK::clamp_to<i64>(output_block.frame_count()));
|
||||
m_impl->input_frames_consumed_since_flush = saturating_add(m_impl->input_frames_consumed_since_flush, media_duration_in_frames);
|
||||
m_impl->media_frames_remainder = media_duration_in_frames_fractional - static_cast<float>(media_duration_in_frames);
|
||||
|
||||
return output_block;
|
||||
}
|
||||
|
||||
}
|
||||
34
Libraries/LibMedia/Audio/WSOLATimeStretcher.h
Normal file
34
Libraries/LibMedia/Audio/WSOLATimeStretcher.h
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Error.h>
|
||||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <LibMedia/Audio/TimeStretcher.h>
|
||||
|
||||
namespace Audio {
|
||||
|
||||
class WSOLATimeStretcher final : public TimeStretcher {
|
||||
public:
|
||||
static ErrorOr<NonnullOwnPtr<TimeStretcher>> create(SampleSpecification);
|
||||
virtual ~WSOLATimeStretcher() override;
|
||||
|
||||
virtual i64 preroll_frame_count() const override;
|
||||
virtual void flush(AK::Duration media_start_timestamp, i64 output_start_frame_index) override;
|
||||
virtual void set_rate(float) override;
|
||||
virtual void push_block(Media::AudioBlock const&) override;
|
||||
virtual Media::DecoderErrorOr<Media::AudioBlock> retrieve_block() override;
|
||||
virtual void signal_end_of_stream() override;
|
||||
|
||||
private:
|
||||
struct Impl;
|
||||
|
||||
explicit WSOLATimeStretcher(NonnullOwnPtr<Impl>);
|
||||
NonnullOwnPtr<Impl> m_impl;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,7 +1,11 @@
|
|||
include(audio)
|
||||
|
||||
set(SOURCES
|
||||
Audio/AudioBuffer.cpp
|
||||
Audio/AudioDevices.cpp
|
||||
Audio/WSOLAAlgorithm.cpp
|
||||
Audio/WSOLAInternals.cpp
|
||||
Audio/WSOLATimeStretcher.cpp
|
||||
Codecs/FLAC.cpp
|
||||
Codecs/Opus.cpp
|
||||
Codecs/Vorbis.cpp
|
||||
|
|
@ -22,6 +26,7 @@ set(SOURCES
|
|||
PlaybackStates/PlaybackStateHandler.cpp
|
||||
PlaybackStates/ResumingStateHandler.cpp
|
||||
Processors/AudioMixer.cpp
|
||||
Processors/AudioTimeStretchProcessor.cpp
|
||||
Producers/DecodedAudioProducer.cpp
|
||||
Producers/DecodedVideoProducer.cpp
|
||||
Sinks/AudioPlaybackSink.cpp
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ class MediaStreamCursor;
|
|||
class MediaTimeProvider;
|
||||
class PlaybackManager;
|
||||
class ReadonlyBytesCursor;
|
||||
class AudioTimeStretchProcessor;
|
||||
class Track;
|
||||
class VideoDecoder;
|
||||
class VideoFrame;
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
#include <LibMedia/GenericTimeProvider.h>
|
||||
#include <LibMedia/PlaybackStates/StartingStateHandler.h>
|
||||
#include <LibMedia/Processors/AudioMixer.h>
|
||||
#include <LibMedia/Processors/AudioTimeStretchProcessor.h>
|
||||
#include <LibMedia/Producers/DecodedAudioProducer.h>
|
||||
#include <LibMedia/Producers/DecodedVideoProducer.h>
|
||||
#include <LibMedia/Sinks/AudioPlaybackSink.h>
|
||||
|
|
@ -116,13 +117,15 @@ DecoderErrorOr<void> PlaybackManager::prepare_playback_from_demuxer(WeakPlayback
|
|||
|
||||
if (!self->m_audio_output_disabled && !self->m_audio_sink && !self->m_audio_tracks.is_empty()) {
|
||||
self->m_audio_mixer = MUST(AudioMixer::try_create());
|
||||
self->m_audio_time_stretch_processor = MUST(AudioTimeStretchProcessor::try_create());
|
||||
self->m_audio_sink = MUST(AudioPlaybackSink::try_create(
|
||||
[self](PipelineStatus status) {
|
||||
if (!self)
|
||||
return;
|
||||
self->on_audio_sink_state_changed(status);
|
||||
}));
|
||||
MUST(self->m_audio_sink->connect_input(*self->m_audio_mixer));
|
||||
MUST(self->m_audio_time_stretch_processor->connect_input(*self->m_audio_mixer));
|
||||
MUST(self->m_audio_sink->connect_input(*self->m_audio_time_stretch_processor));
|
||||
self->set_time_provider(*self->m_audio_sink);
|
||||
self->m_audio_sink->on_audio_output_error = [self](Error&& error) {
|
||||
if (!self)
|
||||
|
|
@ -310,6 +313,7 @@ void PlaybackManager::disable_audio()
|
|||
{
|
||||
m_audio_buffering = false;
|
||||
m_audio_mixer = nullptr;
|
||||
m_audio_time_stretch_processor = nullptr;
|
||||
m_audio_sink = nullptr;
|
||||
set_time_provider(make_ref_counted<GenericTimeProvider>());
|
||||
on_audio_sink_state_changed(PipelineStatus::EndOfStream);
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ private:
|
|||
VideoTrackDatas m_video_track_datas;
|
||||
|
||||
RefPtr<AudioMixer> m_audio_mixer;
|
||||
RefPtr<AudioTimeStretchProcessor> m_audio_time_stretch_processor;
|
||||
RefPtr<AudioPlaybackSink> m_audio_sink;
|
||||
AudioTracks m_audio_tracks;
|
||||
AudioTrackDatas m_audio_track_datas;
|
||||
|
|
|
|||
289
Libraries/LibMedia/Processors/AudioTimeStretchProcessor.cpp
Normal file
289
Libraries/LibMedia/Processors/AudioTimeStretchProcessor.cpp
Normal file
|
|
@ -0,0 +1,289 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/Math.h>
|
||||
#include <AK/TypedTransfer.h>
|
||||
#include <LibMedia/Audio/WSOLATimeStretcher.h>
|
||||
#include <LibMedia/Processors/AudioTimeStretchProcessor.h>
|
||||
|
||||
namespace Media {
|
||||
|
||||
ErrorOr<NonnullRefPtr<AudioTimeStretchProcessor>> AudioTimeStretchProcessor::try_create()
|
||||
{
|
||||
return adopt_nonnull_ref_or_enomem(new (nothrow) AudioTimeStretchProcessor);
|
||||
}
|
||||
|
||||
AudioTimeStretchProcessor::AudioTimeStretchProcessor() = default;
|
||||
|
||||
AudioTimeStretchProcessor::~AudioTimeStretchProcessor()
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
if (m_input != nullptr)
|
||||
m_input->set_wake_handler(nullptr);
|
||||
}
|
||||
|
||||
ErrorOr<void> AudioTimeStretchProcessor::connect_input(NonnullRefPtr<AudioProducer> const& input)
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
VERIFY(m_input == nullptr);
|
||||
m_input = input;
|
||||
input->set_wake_handler([this] {
|
||||
bool should_wake_downstream;
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
auto status = PipelineStatus::HaveData;
|
||||
if (m_pending_block.is_empty())
|
||||
status = produce_block_while_locked(m_pending_block);
|
||||
if (!m_pending_block.is_empty())
|
||||
status = PipelineStatus::HaveData;
|
||||
should_wake_downstream = m_downstream_needs_wake && resolves_seek(status);
|
||||
}
|
||||
if (should_wake_downstream)
|
||||
dispatch_wake();
|
||||
});
|
||||
|
||||
if (m_sample_specification.is_valid()) {
|
||||
if (auto result = input->set_output_sample_specification(m_sample_specification); result.is_error()) {
|
||||
input->set_wake_handler(nullptr);
|
||||
m_input = nullptr;
|
||||
return result.release_error();
|
||||
}
|
||||
if (m_started)
|
||||
input->start();
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::disconnect_input(NonnullRefPtr<AudioProducer> const& input)
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
VERIFY(m_input == input);
|
||||
input->set_wake_handler(nullptr);
|
||||
m_input = nullptr;
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::seek(AK::Duration timestamp)
|
||||
{
|
||||
RefPtr<AudioProducer> input;
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
VERIFY(m_sample_specification.is_valid());
|
||||
auto sample_rate = m_sample_specification.sample_rate();
|
||||
auto target_frame = timestamp.to_time_units(1, sample_rate);
|
||||
auto output_frame = target_frame;
|
||||
|
||||
ensure_stretcher_while_locked();
|
||||
auto prerolled_target_frame = max(target_frame - m_stretcher->preroll_frame_count(), 0);
|
||||
auto actual_preroll_delta = target_frame - prerolled_target_frame;
|
||||
target_frame = prerolled_target_frame;
|
||||
output_frame = output_frame - AK::round_to<i64>(static_cast<float>(actual_preroll_delta) / m_playback_rate);
|
||||
|
||||
m_next_emit_media_time = AK::Duration::from_time_units(target_frame, 1, sample_rate);
|
||||
m_next_output_frame = output_frame;
|
||||
|
||||
m_stretcher->flush(m_next_emit_media_time, m_next_output_frame);
|
||||
|
||||
m_moved_position_pending = true;
|
||||
m_pending_block.clear();
|
||||
m_downstream_needs_wake = true;
|
||||
m_stretcher_reached_eos = false;
|
||||
|
||||
input = m_input;
|
||||
timestamp = m_next_emit_media_time;
|
||||
}
|
||||
|
||||
if (input != nullptr) {
|
||||
input->seek(timestamp);
|
||||
return;
|
||||
}
|
||||
|
||||
dispatch_wake();
|
||||
}
|
||||
|
||||
ErrorOr<void> AudioTimeStretchProcessor::set_output_sample_specification(Audio::SampleSpecification sample_specification)
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
if (m_sample_specification == sample_specification)
|
||||
return {};
|
||||
m_sample_specification = sample_specification;
|
||||
m_stretcher = nullptr;
|
||||
m_pending_block.clear();
|
||||
m_stretcher_reached_eos = false;
|
||||
if (m_input != nullptr)
|
||||
TRY(m_input->set_output_sample_specification(sample_specification));
|
||||
return {};
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::start()
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
m_started = true;
|
||||
if (m_input != nullptr)
|
||||
m_input->start();
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::set_wake_handler(PipelineWakeHandler handler)
|
||||
{
|
||||
m_wake_handler = move(handler);
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::dispatch_wake()
|
||||
{
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
m_downstream_needs_wake = false;
|
||||
}
|
||||
if (m_wake_handler)
|
||||
m_wake_handler();
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::set_playback_rate(float rate)
|
||||
{
|
||||
VERIFY(isfinite(rate));
|
||||
VERIFY(rate > 0.0f);
|
||||
|
||||
bool should_wake_downstream = false;
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
if (m_playback_rate == rate)
|
||||
return;
|
||||
m_playback_rate = rate;
|
||||
should_wake_downstream = m_downstream_needs_wake;
|
||||
}
|
||||
|
||||
if (should_wake_downstream)
|
||||
dispatch_wake();
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::ensure_stretcher_while_locked() const
|
||||
{
|
||||
if (m_stretcher) {
|
||||
m_stretcher->set_rate(m_playback_rate);
|
||||
return;
|
||||
}
|
||||
VERIFY(m_sample_specification.is_valid());
|
||||
m_stretcher = MUST(Audio::WSOLATimeStretcher::create(m_sample_specification));
|
||||
m_stretcher->set_rate(m_playback_rate);
|
||||
m_stretcher->flush(m_next_emit_media_time, m_next_output_frame);
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::maybe_recover_from_stale_upstream_eos_while_locked() const
|
||||
{
|
||||
if (!m_stretcher_reached_eos)
|
||||
return;
|
||||
|
||||
auto status = m_input->status();
|
||||
while (status == PipelineStatus::MovedPosition) {
|
||||
m_input->pull(m_input_block);
|
||||
VERIFY(m_input_block.is_empty());
|
||||
status = m_input->status();
|
||||
}
|
||||
if (is_terminal(status))
|
||||
return;
|
||||
|
||||
m_stretcher->flush(m_next_emit_media_time, m_next_output_frame);
|
||||
m_input_block.clear();
|
||||
m_stretcher_reached_eos = false;
|
||||
}
|
||||
|
||||
PipelineStatus AudioTimeStretchProcessor::produce_block_while_locked(AudioBlock& into) const
|
||||
{
|
||||
if (m_input == nullptr || !m_sample_specification.is_valid())
|
||||
return PipelineStatus::Pending;
|
||||
|
||||
VERIFY(m_playback_rate != 0.0f);
|
||||
|
||||
auto pull_input = [&](AudioBlock& input_block) -> PipelineStatus {
|
||||
auto status = m_input->status();
|
||||
while (status == PipelineStatus::MovedPosition) {
|
||||
m_input->pull(input_block);
|
||||
VERIFY(input_block.is_empty());
|
||||
status = m_input->status();
|
||||
}
|
||||
if (status == PipelineStatus::HaveData)
|
||||
m_input->pull(input_block);
|
||||
else
|
||||
input_block.clear();
|
||||
return status;
|
||||
};
|
||||
|
||||
ensure_stretcher_while_locked();
|
||||
maybe_recover_from_stale_upstream_eos_while_locked();
|
||||
|
||||
while (true) {
|
||||
auto result = m_stretcher->retrieve_block();
|
||||
if (!result.is_error()) {
|
||||
into = result.release_value();
|
||||
m_next_output_frame = into.end_frame_index();
|
||||
m_next_emit_media_time = into.media_time_end();
|
||||
return PipelineStatus::HaveData;
|
||||
}
|
||||
if (result.error().category() == DecoderErrorCategory::EndOfStream) {
|
||||
into.clear();
|
||||
m_stretcher_reached_eos = true;
|
||||
return PipelineStatus::EndOfStream;
|
||||
}
|
||||
if (result.error().category() != DecoderErrorCategory::NeedsMoreInput) {
|
||||
into.clear();
|
||||
return PipelineStatus::Error;
|
||||
}
|
||||
|
||||
auto status = pull_input(m_input_block);
|
||||
if (status == PipelineStatus::EndOfStream) {
|
||||
VERIFY(m_input_block.is_empty());
|
||||
m_stretcher->signal_end_of_stream();
|
||||
m_stretcher_reached_eos = false;
|
||||
continue;
|
||||
}
|
||||
if (m_input_block.is_empty()) {
|
||||
into.clear();
|
||||
return status;
|
||||
}
|
||||
VERIFY(status == PipelineStatus::HaveData);
|
||||
VERIFY(m_input_block.sample_specification() == m_sample_specification);
|
||||
m_stretcher->push_block(m_input_block);
|
||||
}
|
||||
}
|
||||
|
||||
PipelineStatus AudioTimeStretchProcessor::status() const
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
auto status = PipelineStatus::HaveData;
|
||||
if (m_pending_block.is_empty())
|
||||
status = produce_block_while_locked(m_pending_block);
|
||||
if (!m_pending_block.is_empty())
|
||||
status = PipelineStatus::HaveData;
|
||||
|
||||
if (m_moved_position_pending)
|
||||
status = PipelineStatus::MovedPosition;
|
||||
m_downstream_needs_wake = is_waiting_for_data(status);
|
||||
return status;
|
||||
}
|
||||
|
||||
void AudioTimeStretchProcessor::pull(AudioBlock& into)
|
||||
{
|
||||
Sync::MutexLocker locker { m_mutex };
|
||||
|
||||
if (m_moved_position_pending) {
|
||||
m_moved_position_pending = false;
|
||||
into.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!m_pending_block.is_empty()) {
|
||||
into.initialize(m_pending_block.sample_specification(), m_pending_block.first_frame_index(), m_pending_block.frame_count());
|
||||
for (size_t channel = 0; channel < into.channel_count(); channel++)
|
||||
AK::TypedTransfer<float>::copy(into.channel_data(channel).data(), m_pending_block.channel_data(channel).data(), into.frame_count());
|
||||
into.set_media_time_start(m_pending_block.media_time_start());
|
||||
into.set_media_time_duration(m_pending_block.media_time_duration());
|
||||
m_pending_block.clear();
|
||||
return;
|
||||
}
|
||||
|
||||
into.clear();
|
||||
}
|
||||
|
||||
}
|
||||
67
Libraries/LibMedia/Processors/AudioTimeStretchProcessor.h
Normal file
67
Libraries/LibMedia/Processors/AudioTimeStretchProcessor.h
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
/*
|
||||
* Copyright (c) 2026-present, the Ladybird developers.
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/NonnullRefPtr.h>
|
||||
#include <AK/OwnPtr.h>
|
||||
#include <AK/RefPtr.h>
|
||||
#include <LibMedia/Audio/SampleSpecification.h>
|
||||
#include <LibMedia/Audio/TimeStretcher.h>
|
||||
#include <LibMedia/AudioBlock.h>
|
||||
#include <LibMedia/Export.h>
|
||||
#include <LibMedia/PipelineStatus.h>
|
||||
#include <LibMedia/Processors/AudioProcessor.h>
|
||||
#include <LibMedia/Producers/AudioProducer.h>
|
||||
#include <LibSync/Mutex.h>
|
||||
|
||||
namespace Media {
|
||||
|
||||
class MEDIA_API AudioTimeStretchProcessor final : public AudioProcessor {
|
||||
public:
|
||||
static ErrorOr<NonnullRefPtr<AudioTimeStretchProcessor>> try_create();
|
||||
AudioTimeStretchProcessor();
|
||||
virtual ~AudioTimeStretchProcessor() override;
|
||||
|
||||
virtual ErrorOr<void> connect_input(NonnullRefPtr<AudioProducer> const&) override;
|
||||
virtual void disconnect_input(NonnullRefPtr<AudioProducer> const&) override;
|
||||
|
||||
virtual void seek(AK::Duration) override;
|
||||
virtual ErrorOr<void> set_output_sample_specification(Audio::SampleSpecification) override;
|
||||
virtual void start() override;
|
||||
|
||||
virtual PipelineStatus status() const override;
|
||||
virtual void pull(AudioBlock&) override;
|
||||
virtual void set_wake_handler(PipelineWakeHandler) override;
|
||||
virtual void set_playback_rate(float) override;
|
||||
|
||||
private:
|
||||
void ensure_stretcher_while_locked() const;
|
||||
void maybe_recover_from_stale_upstream_eos_while_locked() const;
|
||||
PipelineStatus produce_block_while_locked(AudioBlock&) const;
|
||||
void dispatch_wake();
|
||||
|
||||
mutable Sync::Mutex m_mutex;
|
||||
Audio::SampleSpecification m_sample_specification;
|
||||
RefPtr<AudioProducer> m_input;
|
||||
|
||||
float m_playback_rate { 1.0f };
|
||||
mutable OwnPtr<Audio::TimeStretcher> m_stretcher;
|
||||
bool m_started { false };
|
||||
|
||||
mutable i64 m_next_output_frame { 0 };
|
||||
mutable AK::Duration m_next_emit_media_time;
|
||||
mutable bool m_stretcher_reached_eos { false };
|
||||
|
||||
mutable AudioBlock m_input_block;
|
||||
mutable AudioBlock m_pending_block;
|
||||
mutable bool m_downstream_needs_wake { true };
|
||||
bool m_moved_position_pending { false };
|
||||
|
||||
PipelineWakeHandler m_wake_handler;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -25,6 +25,10 @@ LICENSE_HEADER_CHECK_EXCLUDES = {
|
|||
"AK/Checked.h",
|
||||
"AK/Function.h",
|
||||
"Libraries/LibCore/SocketpairWindows.cpp",
|
||||
"Libraries/LibMedia/Audio/WSOLAAlgorithm.cpp",
|
||||
"Libraries/LibMedia/Audio/WSOLAAlgorithm.h",
|
||||
"Libraries/LibMedia/Audio/WSOLAInternals.cpp",
|
||||
"Libraries/LibMedia/Audio/WSOLAInternals.h",
|
||||
}
|
||||
|
||||
# We check that "#pragma once" is present
|
||||
|
|
|
|||
Loading…
Reference in a new issue