diff --git a/Libraries/LibMedia/CMakeLists.txt b/Libraries/LibMedia/CMakeLists.txt index 5c4e56cc7b..a7a4bd5cc1 100644 --- a/Libraries/LibMedia/CMakeLists.txt +++ b/Libraries/LibMedia/CMakeLists.txt @@ -11,6 +11,7 @@ set(SOURCES Containers/Matroska/MatroskaDemuxer.cpp Containers/Matroska/Reader.cpp PlaybackManager.cpp + PlaybackStates/PlaybackStateHandler.cpp Providers/AudioDataProvider.cpp Providers/GenericTimeProvider.cpp Providers/VideoDataProvider.cpp diff --git a/Libraries/LibMedia/PlaybackManager.cpp b/Libraries/LibMedia/PlaybackManager.cpp index 6bb6cb2ddf..fbafe87f9a 100644 --- a/Libraries/LibMedia/PlaybackManager.cpp +++ b/Libraries/LibMedia/PlaybackManager.cpp @@ -173,7 +173,7 @@ NonnullRefPtr PlaybackManager::get_or_create_the_displaying if (track_data.display == nullptr) { track_data.display = MUST(Media::DisplayingVideoSink::try_create(m_time_provider)); track_data.display->set_provider(track, track_data.provider); - track_data.provider->seek(m_time_provider->current_time()); + track_data.provider->seek(m_time_provider->current_time(), SeekMode::Accurate); } VERIFY(track_data.display->provider(track) == track_data.provider); @@ -223,6 +223,12 @@ void PlaybackManager::pause() m_handler->pause(); } +void PlaybackManager::seek(AK::Duration timestamp, SeekMode mode) +{ + m_handler->seek(timestamp, mode); + m_is_in_error_state = false; +} + bool PlaybackManager::is_playing() { return m_handler->is_playing(); diff --git a/Libraries/LibMedia/PlaybackManager.h b/Libraries/LibMedia/PlaybackManager.h index 3f3f504e43..2a730fdaa7 100644 --- a/Libraries/LibMedia/PlaybackManager.h +++ b/Libraries/LibMedia/PlaybackManager.h @@ -70,6 +70,7 @@ public: void play(); void pause(); + void seek(AK::Duration timestamp, SeekMode); bool is_playing(); PlaybackState state(); diff --git a/Libraries/LibMedia/PlaybackStates/Forward.h b/Libraries/LibMedia/PlaybackStates/Forward.h index 7f893c495b..976e768d0d 100644 --- a/Libraries/LibMedia/PlaybackStates/Forward.h +++ b/Libraries/LibMedia/PlaybackStates/Forward.h @@ -11,7 +11,9 @@ #define ENUMERATE_PLAYBACK_STATE_HANDLERS(X) \ X(PlaybackStateHandler) \ X(PlayingStateHandler) \ - X(PausedStateHandler) + X(PausedStateHandler) \ + X(ResumingStateHandler) \ + X(SeekingStateHandler) namespace Media { diff --git a/Libraries/LibMedia/PlaybackStates/PlaybackState.h b/Libraries/LibMedia/PlaybackStates/PlaybackState.h index 636ee511b8..b1d0f3b3fc 100644 --- a/Libraries/LibMedia/PlaybackStates/PlaybackState.h +++ b/Libraries/LibMedia/PlaybackStates/PlaybackState.h @@ -13,6 +13,7 @@ namespace Media { enum class PlaybackState : u8 { Playing, Paused, + Seeking, }; } diff --git a/Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.cpp b/Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.cpp new file mode 100644 index 0000000000..70d421a27a --- /dev/null +++ b/Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.cpp @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2025, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +#include "PlaybackStateHandler.h" + +namespace Media { + +void PlaybackStateHandler::seek(AK::Duration timestamp, SeekMode mode) +{ + manager().replace_state_handler(manager().is_playing(), timestamp, mode); +} + +} diff --git a/Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.h b/Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.h index e7af3f3e4e..1a3d175732 100644 --- a/Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.h +++ b/Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.h @@ -6,8 +6,10 @@ #pragma once +#include #include #include +#include namespace Media { @@ -24,6 +26,7 @@ public: virtual void play() = 0; virtual void pause() = 0; + virtual void seek(AK::Duration timestamp, SeekMode); virtual bool is_playing() = 0; virtual PlaybackState state() = 0; diff --git a/Libraries/LibMedia/PlaybackStates/ResumingStateHandler.h b/Libraries/LibMedia/PlaybackStates/ResumingStateHandler.h new file mode 100644 index 0000000000..2be57d57ad --- /dev/null +++ b/Libraries/LibMedia/PlaybackStates/ResumingStateHandler.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2025, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace Media { + +class ResumingStateHandler : public PlaybackStateHandler { +public: + ResumingStateHandler(PlaybackManager& manager, bool playing) + : PlaybackStateHandler(manager) + , m_playing(playing) + { + } + virtual ~ResumingStateHandler() override = default; + + void resume() + { + if (m_playing) + manager().replace_state_handler(); + else + manager().replace_state_handler(); + } + + virtual void on_enter() override { } + virtual void on_exit() override { } + + virtual void play() override + { + m_playing = true; + } + virtual void pause() override + { + m_playing = false; + } + + virtual bool is_playing() override + { + return m_playing; + } + +private: + bool m_playing { false }; +}; + +} diff --git a/Libraries/LibMedia/PlaybackStates/SeekingStateHandler.h b/Libraries/LibMedia/PlaybackStates/SeekingStateHandler.h new file mode 100644 index 0000000000..a0f92b43ff --- /dev/null +++ b/Libraries/LibMedia/PlaybackStates/SeekingStateHandler.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2025, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Media { + +class SeekingStateHandler final : public ResumingStateHandler { +public: + SeekingStateHandler(PlaybackManager& manager, bool playing, AK::Duration timestamp, SeekMode mode) + : ResumingStateHandler(manager, playing) + , m_target_timestamp(timestamp) + , m_mode(mode) + { + } + virtual ~SeekingStateHandler() override = default; + + virtual void on_enter() override + { + ResumingStateHandler::on_enter(); + begin_seek(); + } + + virtual void seek(AK::Duration timestamp, SeekMode mode) override + { + m_target_timestamp = timestamp; + m_mode = mode; + begin_seek(); + } + + virtual PlaybackState state() override + { + return PlaybackState::Seeking; + } + +private: + struct SeekData : public RefCounted { + SeekData(PlaybackManager& manager) + : manager(manager) + { + } + + NonnullRefPtr manager; + + size_t id { 0 }; + + AK::Duration chosen_timestamp { AK::Duration::zero() }; + + size_t video_seeks_in_flight { 0 }; + size_t video_seeks_completed { 0 }; + + size_t audio_seeks_in_flight { 0 }; + size_t audio_seeks_completed { 0 }; + }; + + static void possibly_complete_seek(SeekData& seek_data) + { + if (seek_data.video_seeks_completed != seek_data.video_seeks_in_flight) + return; + if (seek_data.audio_seeks_completed != seek_data.audio_seeks_in_flight) + return; + + auto& seek_handler = as(*seek_data.manager->m_handler); + + // Providers guarantee that their callbacks don't get called if a new seek is started, but we + // can end up with video seeks in flight while an audio seek is completing. Ensure that the + // old audio seek doesn't cause us to exit the seeking state before the current seek completes. + if (seek_handler.m_current_seek_id != seek_data.id) + return; + + seek_data.manager->m_time_provider->set_time(seek_data.chosen_timestamp); + + for (auto& video_track_data : seek_data.manager->m_video_track_datas) { + if (video_track_data.display == nullptr) + continue; + video_track_data.display->resume_updates(); + } + + seek_handler.resume(); + } + + static size_t count_audio_tracks(PlaybackManager& manager) + { + size_t count = 0; + for (auto const& audio_track_data : manager.m_audio_track_datas) { + if (manager.m_audio_sink->provider(audio_track_data.track) == nullptr) + continue; + count++; + } + return count; + } + + void begin_seek() + { + auto seek_data = make_ref_counted(manager()); + seek_data->id = ++m_current_seek_id; + + for (auto const& video_track_data : manager().m_video_track_datas) { + if (video_track_data.display == nullptr) + continue; + seek_data->video_seeks_in_flight++; + video_track_data.display->pause_updates(); + } + + seek_data->audio_seeks_in_flight = count_audio_tracks(manager()); + + if (seek_data->video_seeks_in_flight == 0) { + seek_data->chosen_timestamp = m_target_timestamp; + for (auto const& audio_track_data : seek_data->manager->m_audio_track_datas) { + if (seek_data->manager->m_audio_sink->provider(audio_track_data.track) == nullptr) + continue; + audio_track_data.provider->seek(seek_data->chosen_timestamp, [seek_data]() { + seek_data->audio_seeks_completed++; + possibly_complete_seek(*seek_data); + }); + } + return; + } + + for (auto const& video_track_data : manager().m_video_track_datas) { + if (video_track_data.display == nullptr) + continue; + video_track_data.provider->seek(m_target_timestamp, m_mode, [seek_data](AK::Duration provider_timestamp) { + seek_data->chosen_timestamp = max(seek_data->chosen_timestamp, provider_timestamp); + seek_data->video_seeks_completed++; + + if (seek_data->video_seeks_completed == seek_data->video_seeks_in_flight) { + // Since we're running this in a callback that can run any time after begin_seek() was called, + // we need to ensure that our audio track count is up-to-date. + seek_data->audio_seeks_in_flight = count_audio_tracks(*seek_data->manager); + + if (seek_data->audio_seeks_in_flight == 0) { + possibly_complete_seek(*seek_data); + return; + } + + for (auto const& audio_track_data : seek_data->manager->m_audio_track_datas) { + if (seek_data->manager->m_audio_sink->provider(audio_track_data.track) == nullptr) + continue; + audio_track_data.provider->seek(seek_data->chosen_timestamp, [seek_data]() { + seek_data->audio_seeks_completed++; + possibly_complete_seek(*seek_data); + }); + } + } + }); + } + } + + AK::Duration m_target_timestamp; + SeekMode m_mode { SeekMode::Accurate }; + size_t m_current_seek_id { 0 }; +}; + +} diff --git a/Libraries/LibMedia/Providers/AudioDataProvider.cpp b/Libraries/LibMedia/Providers/AudioDataProvider.cpp index ce27e4a512..eb0fcc7be8 100644 --- a/Libraries/LibMedia/Providers/AudioDataProvider.cpp +++ b/Libraries/LibMedia/Providers/AudioDataProvider.cpp @@ -26,8 +26,10 @@ DecoderErrorOr> AudioDataProvider::try_create(N auto provider = DECODER_TRY_ALLOC(try_make_ref_counted(thread_data)); auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int { - while (!thread_data->should_thread_exit()) + while (!thread_data->should_thread_exit()) { + thread_data->handle_seek(); thread_data->push_data_and_decode_a_block(); + } return 0; })); thread->start(); @@ -51,9 +53,9 @@ void AudioDataProvider::set_error_handler(ErrorHandler&& handler) m_thread_data->set_error_handler(move(handler)); } -void AudioDataProvider::seek(AK::Duration timestamp) +void AudioDataProvider::seek(AK::Duration timestamp, SeekCompletionHandler&& completion_handler) { - m_thread_data->seek(timestamp); + m_thread_data->seek(timestamp, move(completion_handler)); } AudioDataProvider::ThreadData::ThreadData(NonnullRefPtr const& demuxer, Track const& track, NonnullOwnPtr&& decoder) @@ -89,16 +91,13 @@ void AudioDataProvider::ThreadData::exit() m_wait_condition.broadcast(); } -void AudioDataProvider::ThreadData::seek(AK::Duration timestamp) +void AudioDataProvider::ThreadData::seek(AK::Duration timestamp, SeekCompletionHandler&& completion_handler) { - auto demuxer_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp); - if (demuxer_result.is_error()) { - m_error_handler(demuxer_result.release_error()); - } else { - auto locker = take_lock(); - m_is_in_error_state = false; - m_wait_condition.broadcast(); - } + auto locker = take_lock(); + m_seek_completion_handler = move(completion_handler); + m_seek_id++; + m_seek_timestamp = timestamp; + m_wait_condition.broadcast(); } bool AudioDataProvider::ThreadData::should_thread_exit() const @@ -106,6 +105,121 @@ bool AudioDataProvider::ThreadData::should_thread_exit() const return m_exit; } +template +void AudioDataProvider::ThreadData::process_seek_on_main_thread(u32 seek_id, T&& function) +{ + m_last_processed_seek_id = seek_id; + m_main_thread_event_loop.deferred_invoke([this, seek_id, function] mutable { + if (m_seek_id != seek_id) + return; + function(); + }); +} + +void AudioDataProvider::ThreadData::resolve_seek(u32 seek_id) +{ + process_seek_on_main_thread(seek_id, [this] { + { + auto locker = take_lock(); + m_is_in_error_state = false; + m_wait_condition.broadcast(); + } + auto handler = move(m_seek_completion_handler); + if (handler) + handler(); + }); +} + +bool AudioDataProvider::ThreadData::handle_seek() +{ + auto seek_id = m_seek_id.load(); + if (m_last_processed_seek_id == seek_id) + return false; + + auto handle_error = [&](DecoderError&& error) { + { + auto locker = take_lock(); + m_queue.clear(); + } + + process_seek_on_main_thread(seek_id, + [this, error = move(error)] mutable { + m_error_handler(move(error)); + m_seek_completion_handler = nullptr; + }); + }; + + AK::Duration timestamp; + + while (true) { + { + auto locker = take_lock(); + seek_id = m_seek_id; + timestamp = m_seek_timestamp; + } + + auto demuxer_seek_result_or_error = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp); + if (demuxer_seek_result_or_error.is_error() && demuxer_seek_result_or_error.error().category() != DecoderErrorCategory::EndOfStream) { + handle_error(demuxer_seek_result_or_error.release_error()); + return true; + } + auto demuxer_seek_result = demuxer_seek_result_or_error.value_or(DemuxerSeekResult::MovedPosition); + + if (demuxer_seek_result == DemuxerSeekResult::MovedPosition) + m_decoder->flush(); + + auto new_seek_id = seek_id; + AudioBlock last_block; + + while (new_seek_id == seek_id) { + auto coded_frame_result = m_demuxer->get_next_sample_for_track(m_track); + if (coded_frame_result.is_error()) { + if (coded_frame_result.error().category() == DecoderErrorCategory::EndOfStream) { + resolve_seek(seek_id); + return true; + } + handle_error(coded_frame_result.release_error()); + return true; + } + + auto coded_frame = coded_frame_result.release_value(); + auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data()); + if (decode_result.is_error()) { + handle_error(decode_result.release_error()); + return true; + } + + while (new_seek_id == seek_id) { + AudioBlock current_block; + auto block_result = m_decoder->write_next_block(current_block); + if (block_result.is_error()) { + if (block_result.error().category() == DecoderErrorCategory::NeedsMoreInput) + break; + handle_error(block_result.release_error()); + return true; + } + + if (current_block.start_timestamp() > timestamp) { + auto locker = take_lock(); + m_queue.clear(); + + if (!last_block.is_empty()) + m_queue.enqueue(move(last_block)); + + m_queue.enqueue(move(current_block)); + + resolve_seek(seek_id); + return true; + } + + last_block = move(current_block); + + new_seek_id = m_seek_id; + } + } + } +} + void AudioDataProvider::ThreadData::push_data_and_decode_a_block() { #if PLAYBACK_MANAGER_DEBUG @@ -121,8 +235,11 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block() m_error_handler(move(error)); }); dbgln_if(PLAYBACK_MANAGER_DEBUG, "Audio Data Provider: Encountered an error, waiting for a seek to start decoding again..."); - while (m_is_in_error_state) + while (m_is_in_error_state) { + if (handle_seek()) + break; m_wait_condition.wait(); + } }; auto sample_result = m_demuxer->get_next_sample_for_track(m_track); @@ -146,6 +263,8 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block() auto locker = take_lock(); while (m_queue.size() >= m_queue_max_size) { + if (handle_seek()) + return; m_wait_condition.wait(); if (should_thread_exit()) return; diff --git a/Libraries/LibMedia/Providers/AudioDataProvider.h b/Libraries/LibMedia/Providers/AudioDataProvider.h index 2d504342c9..efcd076c17 100644 --- a/Libraries/LibMedia/Providers/AudioDataProvider.h +++ b/Libraries/LibMedia/Providers/AudioDataProvider.h @@ -32,6 +32,7 @@ public: using AudioQueue = Queue; using ErrorHandler = Function; + using SeekCompletionHandler = Function; static DecoderErrorOr> try_create(NonnullRefPtr const& demuxer, Track const& track); AudioDataProvider(NonnullRefPtr const&); @@ -41,7 +42,7 @@ public: AudioBlock retrieve_block(); - void seek(AK::Duration timestamp); + void seek(AK::Duration timestamp, SeekCompletionHandler&& = nullptr); private: class ThreadData final : public AtomicRefCounted { @@ -52,12 +53,16 @@ private: void set_error_handler(ErrorHandler&&); bool should_thread_exit() const; + bool handle_seek(); + template + void process_seek_on_main_thread(u32 seek_id, T&&); + void resolve_seek(u32 seek_id); void push_data_and_decode_a_block(); void exit(); void set_stopped(bool); bool is_stopped() const; - void seek(AK::Duration timestamp); + void seek(AK::Duration timestamp, SeekCompletionHandler&&); [[nodiscard]] Threading::MutexLocker take_lock() { return Threading::MutexLocker(m_mutex); } void wake() { m_wait_condition.broadcast(); } @@ -80,6 +85,11 @@ private: AudioQueue m_queue; ErrorHandler m_error_handler; bool m_is_in_error_state { false }; + + u32 m_last_processed_seek_id { 0 }; + Atomic m_seek_id { 0 }; + SeekCompletionHandler m_seek_completion_handler; + AK::Duration m_seek_timestamp; }; NonnullRefPtr m_thread_data; diff --git a/Libraries/LibMedia/Providers/VideoDataProvider.cpp b/Libraries/LibMedia/Providers/VideoDataProvider.cpp index 5e005da5c4..a25770f422 100644 --- a/Libraries/LibMedia/Providers/VideoDataProvider.cpp +++ b/Libraries/LibMedia/Providers/VideoDataProvider.cpp @@ -27,8 +27,10 @@ DecoderErrorOr> VideoDataProvider::try_create(N auto provider = DECODER_TRY_ALLOC(try_make_ref_counted(thread_data)); auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int { - while (!thread_data->should_thread_exit()) + while (!thread_data->should_thread_exit()) { + thread_data->handle_seek(); thread_data->push_data_and_decode_some_frames(); + } return 0; })); thread->start(); @@ -68,9 +70,9 @@ TimedImage VideoDataProvider::retrieve_frame() return result; } -void VideoDataProvider::seek(AK::Duration timestamp) +void VideoDataProvider::seek(AK::Duration timestamp, SeekMode seek_mode, SeekCompletionHandler&& completion_handler) { - m_thread_data->seek(timestamp); + m_thread_data->seek(timestamp, seek_mode, move(completion_handler)); } VideoDataProvider::ThreadData::ThreadData(NonnullRefPtr const& demuxer, Track const& track, NonnullOwnPtr&& decoder, RefPtr const& time_provider) @@ -107,16 +109,14 @@ TimedImage VideoDataProvider::ThreadData::take_frame() return m_queue.dequeue(); } -void VideoDataProvider::ThreadData::seek(AK::Duration timestamp) +void VideoDataProvider::ThreadData::seek(AK::Duration timestamp, SeekMode seek_mode, SeekCompletionHandler&& completion_handler) { - auto seek_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp); - if (seek_result.is_error()) { - m_error_handler(seek_result.release_error()); - } else { - auto locker = take_lock(); - m_is_in_error_state = false; - m_wait_condition.broadcast(); - } + auto locker = take_lock(); + m_seek_id++; + m_seek_completion_handler = move(completion_handler); + m_seek_timestamp = timestamp; + m_seek_mode = seek_mode; + m_wait_condition.broadcast(); } bool VideoDataProvider::ThreadData::should_thread_exit() const @@ -155,6 +155,178 @@ void VideoDataProvider::ThreadData::queue_frame(TimedImage&& frame) m_queue.enqueue(move(frame)); } +template +void VideoDataProvider::ThreadData::process_seek_on_main_thread(u32 seek_id, T&& function) +{ + m_last_processed_seek_id = seek_id; + m_main_thread_event_loop.deferred_invoke([this, seek_id, function] mutable { + if (m_seek_id != seek_id) + return; + function(); + }); +} + +void VideoDataProvider::ThreadData::resolve_seek(u32 seek_id, AK::Duration const& timestamp) +{ + m_is_in_error_state = false; + process_seek_on_main_thread(seek_id, [this, timestamp] { + auto handler = move(m_seek_completion_handler); + if (handler) + handler(timestamp); + }); +} + +bool VideoDataProvider::ThreadData::handle_seek() +{ +#define CONVERT_AND_QUEUE_A_FRAME(frame) \ + do { \ + auto __bitmap_result = frame->to_bitmap(); \ + if (__bitmap_result.is_error()) { \ + handle_error(__bitmap_result.release_error()); \ + return true; \ + } \ + queue_frame(TimedImage(frame->timestamp(), __bitmap_result.release_value())); \ + } while (false) + + auto seek_id = m_seek_id.load(); + if (m_last_processed_seek_id == seek_id) + return false; + + auto handle_error = [&](DecoderError&& error) { + m_is_in_error_state = true; + { + auto locker = take_lock(); + m_queue.clear(); + } + process_seek_on_main_thread(seek_id, + [this, error = move(error)] mutable { + m_error_handler(move(error)); + m_seek_completion_handler = nullptr; + }); + }; + + AK::Duration timestamp; + SeekMode mode { SeekMode::Accurate }; + + while (true) { + { + auto locker = take_lock(); + seek_id = m_seek_id; + timestamp = m_seek_timestamp; + mode = m_seek_mode; + } + + auto seek_options = mode == SeekMode::Accurate ? DemuxerSeekOptions::None : DemuxerSeekOptions::Force; + auto demuxer_seek_result_or_error = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp, seek_options); + if (demuxer_seek_result_or_error.is_error() && demuxer_seek_result_or_error.error().category() != DecoderErrorCategory::EndOfStream) { + handle_error(demuxer_seek_result_or_error.release_error()); + return true; + } + auto demuxer_seek_result = demuxer_seek_result_or_error.value_or(DemuxerSeekResult::MovedPosition); + + if (demuxer_seek_result == DemuxerSeekResult::MovedPosition) + m_decoder->flush(); + + auto is_desired_coded_frame = [mode, timestamp](CodedFrame const& frame) { + if (mode == SeekMode::Accurate) + return true; + if (mode == SeekMode::FastBefore) + return frame.is_keyframe(); + if (mode == SeekMode::FastAfter) + return frame.is_keyframe() && frame.timestamp() > timestamp; + VERIFY_NOT_REACHED(); + }; + + auto is_desired_decoded_frame = [mode, timestamp](VideoFrame const& frame) { + if (mode == SeekMode::Accurate) + return frame.timestamp() > timestamp; + return true; + }; + + auto resolved_time = [mode, timestamp](VideoFrame const& frame) { + if (mode == SeekMode::Accurate) + return timestamp; + if (mode == SeekMode::FastBefore) + return min(timestamp, frame.timestamp()); + if (mode == SeekMode::FastAfter) + return max(timestamp, frame.timestamp()); + VERIFY_NOT_REACHED(); + }; + + auto new_seek_id = m_seek_id.load(); + auto found_desired_keyframe = false; + OwnPtr last_frame; + + while (new_seek_id == seek_id) { + auto coded_frame_result = m_demuxer->get_next_sample_for_track(m_track); + if (coded_frame_result.is_error()) { + if (coded_frame_result.error().category() == DecoderErrorCategory::EndOfStream) { + if (mode == SeekMode::FastAfter) { + // If we're fast seeking after the provided timestamp and reach the end of the stream, that means we + // nothing to display. Restart the seek as an accurate seek. + auto locker = take_lock(); + seek_id = ++m_seek_id; + m_seek_mode = SeekMode::Accurate; + continue; + } + + if (last_frame != nullptr) + CONVERT_AND_QUEUE_A_FRAME(last_frame); + + resolve_seek(seek_id, timestamp); + return true; + } + handle_error(coded_frame_result.release_error()); + return true; + } + + auto coded_frame = coded_frame_result.release_value(); + + if (!found_desired_keyframe) + found_desired_keyframe = is_desired_coded_frame(coded_frame); + + if (!found_desired_keyframe) + continue; + + auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data()); + if (decode_result.is_error()) { + handle_error(decode_result.release_error()); + return true; + } + + while (new_seek_id == seek_id) { + auto frame_result = m_decoder->get_decoded_frame(); + if (frame_result.is_error()) { + if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput) + break; + handle_error(frame_result.release_error()); + return true; + } + + auto current_frame = frame_result.release_value(); + set_cicp_values(*current_frame, coded_frame); + if (is_desired_decoded_frame(*current_frame)) { + auto locker = take_lock(); + m_queue.clear(); + + if (last_frame != nullptr) + CONVERT_AND_QUEUE_A_FRAME(last_frame); + + CONVERT_AND_QUEUE_A_FRAME(current_frame); + + VERIFY(!m_queue.is_empty()); + resolve_seek(seek_id, resolved_time(*current_frame)); + return true; + } + + last_frame = move(current_frame); + + new_seek_id = m_seek_id; + } + } + } +} + void VideoDataProvider::ThreadData::push_data_and_decode_some_frames() { // FIXME: Check if the PlaybackManager's current time is ahead of the next keyframe, and seek to it if so. @@ -170,8 +342,11 @@ void VideoDataProvider::ThreadData::push_data_and_decode_some_frames() m_error_handler(move(error)); }); dbgln_if(PLAYBACK_MANAGER_DEBUG, "Video Data Provider: Encountered an error, waiting for a seek to start decoding again..."); - while (m_is_in_error_state) + while (m_is_in_error_state) { + if (handle_seek()) + break; m_wait_condition.wait(); + } }; auto sample_result = m_demuxer->get_next_sample_for_track(m_track); @@ -209,6 +384,8 @@ void VideoDataProvider::ThreadData::push_data_and_decode_some_frames() { auto locker = take_lock(); while (m_queue.size() >= m_queue_max_size) { + if (handle_seek()) + return; m_wait_condition.wait(); if (should_thread_exit()) return; diff --git a/Libraries/LibMedia/Providers/VideoDataProvider.h b/Libraries/LibMedia/Providers/VideoDataProvider.h index 45785dfd1f..824a80887e 100644 --- a/Libraries/LibMedia/Providers/VideoDataProvider.h +++ b/Libraries/LibMedia/Providers/VideoDataProvider.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -33,6 +34,7 @@ public: using ImageQueue = Queue; using ErrorHandler = Function; + using SeekCompletionHandler = Function; static DecoderErrorOr> try_create(NonnullRefPtr const&, Track const&, RefPtr const& = nullptr); static DecoderErrorOr> try_create(NonnullRefPtr const&, Track const&, RefPtr const& = nullptr); @@ -44,7 +46,7 @@ public: TimedImage retrieve_frame(); - void seek(AK::Duration timestamp); + void seek(AK::Duration timestamp, SeekMode, SeekCompletionHandler&& = nullptr); private: class ThreadData final : public AtomicRefCounted { @@ -59,11 +61,15 @@ private: ImageQueue& queue(); TimedImage take_frame(); - void seek(AK::Duration timestamp); + void seek(AK::Duration timestamp, SeekMode, SeekCompletionHandler&&); bool should_thread_exit() const; static void set_cicp_values(VideoFrame&, CodedFrame const&); void queue_frame(TimedImage&&); + bool handle_seek(); + template + void process_seek_on_main_thread(u32 seek_id, T&&); + void resolve_seek(u32 seek_id, AK::Duration const& timestamp); void push_data_and_decode_some_frames(); [[nodiscard]] Threading::MutexLocker take_lock() { return Threading::MutexLocker(m_mutex); } @@ -86,6 +92,12 @@ private: ImageQueue m_queue; ErrorHandler m_error_handler; bool m_is_in_error_state { false }; + + u32 m_last_processed_seek_id { 0 }; + Atomic m_seek_id { 0 }; + SeekCompletionHandler m_seek_completion_handler; + AK::Duration m_seek_timestamp; + SeekMode m_seek_mode { SeekMode::Accurate }; }; NonnullRefPtr m_thread_data; diff --git a/Libraries/LibMedia/SeekMode.h b/Libraries/LibMedia/SeekMode.h new file mode 100644 index 0000000000..a33db335d0 --- /dev/null +++ b/Libraries/LibMedia/SeekMode.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include + +namespace Media { + +enum class SeekMode : u8 { + Accurate, + FastBefore, + FastAfter, +}; + +constexpr StringView seek_mode_to_string(SeekMode seek_mode) +{ + switch (seek_mode) { + case SeekMode::Accurate: + return "Accurate"sv; + case SeekMode::FastBefore: + return "FastBefore"sv; + case SeekMode::FastAfter: + return "FastAfter"sv; + } + VERIFY_NOT_REACHED(); +} + +} + +namespace AK { + +template<> +struct Formatter final : Formatter { + ErrorOr format(FormatBuilder& builder, Media::SeekMode color_primaries) + { + return Formatter::format(builder, Media::seek_mode_to_string(color_primaries)); + } +}; + +} diff --git a/Libraries/LibMedia/Sinks/DisplayingVideoSink.cpp b/Libraries/LibMedia/Sinks/DisplayingVideoSink.cpp index 69aa7fe069..0ddef3984a 100644 --- a/Libraries/LibMedia/Sinks/DisplayingVideoSink.cpp +++ b/Libraries/LibMedia/Sinks/DisplayingVideoSink.cpp @@ -47,8 +47,16 @@ RefPtr DisplayingVideoSink::provider(Track const& track) cons DisplayingVideoSinkUpdateResult DisplayingVideoSink::update() { + if (m_pause_updates) + return DisplayingVideoSinkUpdateResult::NoChange; + auto current_time = m_time_provider->current_time(); auto result = DisplayingVideoSinkUpdateResult::NoChange; + if (m_cleared_current_frame) { + result = DisplayingVideoSinkUpdateResult::NewFrameAvailable; + m_cleared_current_frame = false; + } + while (true) { if (!m_next_frame.is_valid()) { m_next_frame = m_provider->retrieve_frame(); @@ -68,4 +76,17 @@ RefPtr DisplayingVideoSink::current_frame() return m_current_frame; } +void DisplayingVideoSink::pause_updates() +{ + m_pause_updates = true; +} + +void DisplayingVideoSink::resume_updates() +{ + m_next_frame.clear(); + m_current_frame = nullptr; + m_pause_updates = false; + m_cleared_current_frame = true; +} + } diff --git a/Libraries/LibMedia/Sinks/DisplayingVideoSink.h b/Libraries/LibMedia/Sinks/DisplayingVideoSink.h index 6417d2bb8b..1acb034289 100644 --- a/Libraries/LibMedia/Sinks/DisplayingVideoSink.h +++ b/Libraries/LibMedia/Sinks/DisplayingVideoSink.h @@ -38,6 +38,9 @@ public: DisplayingVideoSinkUpdateResult update(); RefPtr current_frame(); + void pause_updates(); + void resume_updates(); + private: static constexpr size_t DEFAULT_QUEUE_SIZE = 8; @@ -49,6 +52,8 @@ private: TimedImage m_next_frame; RefPtr m_current_frame; + bool m_pause_updates { false }; + bool m_cleared_current_frame { false }; }; } diff --git a/Libraries/LibMedia/TimedImage.h b/Libraries/LibMedia/TimedImage.h index 1c85c14a29..eb1f9dbf0a 100644 --- a/Libraries/LibMedia/TimedImage.h +++ b/Libraries/LibMedia/TimedImage.h @@ -42,6 +42,11 @@ public: m_timestamp = AK::Duration::zero(); return m_image.release_nonnull(); } + void clear() + { + m_timestamp = AK::Duration::zero(); + m_image = nullptr; + } private: AK::Duration m_timestamp { AK::Duration::zero() }; diff --git a/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn b/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn index 784a72dd04..005af80592 100644 --- a/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn +++ b/Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn @@ -13,6 +13,7 @@ shared_library("LibMedia") { "Containers/Matroska/MatroskaDemuxer.cpp", "Containers/Matroska/Reader.cpp", "PlaybackManager.cpp", + "PlaybackStates/PlaybackStateHandler.cpp", "Providers/AudioDataProvider.cpp", "Providers/GenericTimeProvider.cpp", "Providers/VideoDataProvider.cpp",