/* * Copyright (c) 2022-2026, Gregory Bertilson * * SPDX-License-Identifier: BSD-2-Clause */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "PlaybackManager.h" namespace Media { DecoderErrorOr> PlaybackManager::create_demuxer_for_stream(NonnullRefPtr const& stream) { if (Matroska::Reader::is_matroska_or_webm(stream->create_cursor())) return Matroska::MatroskaDemuxer::from_stream(stream); return FFmpeg::FFmpegDemuxer::from_stream(stream); } DecoderErrorOr PlaybackManager::prepare_playback_from_demuxer(WeakPlaybackManager const& self, NonnullRefPtr const& demuxer, Core::EventLoop& main_thread_event_loop) { // Create the video tracks and their producers. auto all_video_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Video)); auto supported_video_tracks = VideoTracks(); auto supported_video_track_datas = VideoTrackDatas(); supported_video_tracks.ensure_capacity(all_video_tracks.size()); supported_video_track_datas.ensure_capacity(all_video_tracks.size()); for (auto const& track : all_video_tracks) { auto video_producer_result = DecodedVideoProducer::try_create(main_thread_event_loop, demuxer, track); if (video_producer_result.is_error()) continue; supported_video_tracks.append(track); supported_video_track_datas.empend(VideoTrackData(track, video_producer_result.release_value(), nullptr)); } supported_video_tracks.shrink_to_fit(); supported_video_track_datas.shrink_to_fit(); // Create all the audio tracks, their producers, and the audio output. auto all_audio_tracks = TRY(demuxer->get_tracks_for_type(TrackType::Audio)); auto supported_audio_tracks = AudioTracks(); auto supported_audio_track_datas = AudioTrackDatas(); supported_audio_tracks.ensure_capacity(all_audio_tracks.size()); supported_audio_track_datas.ensure_capacity(all_audio_tracks.size()); for (auto const& track : all_audio_tracks) { auto audio_producer_result = DecodedAudioProducer::try_create(main_thread_event_loop, demuxer, track); if (audio_producer_result.is_error()) continue; auto audio_producer = audio_producer_result.release_value(); supported_audio_tracks.append(track); supported_audio_track_datas.empend(AudioTrackData(track, move(audio_producer))); } supported_audio_tracks.shrink_to_fit(); supported_audio_track_datas.shrink_to_fit(); if (supported_video_tracks.is_empty() && supported_audio_tracks.is_empty()) return DecoderError::with_description(DecoderErrorCategory::NotImplemented, "No supported video or audio tracks found"sv); auto preferred_video_track = demuxer->get_preferred_track_for_type(TrackType::Video).value_or({}); if (preferred_video_track.has_value() && !supported_video_tracks.contains_slow(*preferred_video_track)) preferred_video_track = {}; auto preferred_audio_track = demuxer->get_preferred_track_for_type(TrackType::Audio).value_or({}); if (preferred_audio_track.has_value() && !supported_audio_tracks.contains_slow(*preferred_audio_track)) preferred_audio_track = {}; auto duration = demuxer->total_duration().value_or(AK::Duration::zero()); auto start_time_realtime = demuxer->start_time_realtime(); main_thread_event_loop.deferred_invoke([self, video_tracks = move(supported_video_tracks), video_track_datas = move(supported_video_track_datas), preferred_video_track, audio_tracks = move(supported_audio_tracks), audio_track_datas = move(supported_audio_track_datas), preferred_audio_track, duration, start_time_realtime] mutable { if (!self) return; for (auto const& existing_track : self->m_video_tracks) { if (video_tracks.contains_slow(existing_track)) { self->on_unsupported_format_error(DecoderError::with_description(DecoderErrorCategory::Invalid, "Duplicate video track found"sv)); return; } } for (auto const& existing_track : self->m_audio_tracks) { if (audio_tracks.contains_slow(existing_track)) { self->on_unsupported_format_error(DecoderError::with_description(DecoderErrorCategory::Invalid, "Duplicate audio track found"sv)); return; } } auto first_new_video_index = self->m_video_tracks.size(); auto first_new_audio_index = self->m_audio_tracks.size(); self->m_video_tracks.extend(move(video_tracks)); self->m_video_track_datas.extend(move(video_track_datas)); self->m_audio_tracks.extend(move(audio_tracks)); self->m_audio_track_datas.extend(move(audio_track_datas)); if (!self->m_preferred_video_track.has_value()) self->m_preferred_video_track = preferred_video_track; if (!self->m_preferred_audio_track.has_value()) self->m_preferred_audio_track = preferred_audio_track; self->m_start_time_realtime = start_time_realtime; self->check_for_duration_change(duration); self->set_up_producers(); if (!self->m_audio_output_disabled && !self->m_audio_sink && !self->m_audio_tracks.is_empty()) { self->m_audio_mixer = MUST(AudioMixer::try_create()); self->m_audio_time_stretch_processor = MUST(AudioTimeStretchProcessor::try_create()); self->m_audio_sink = MUST(AudioPlaybackSink::try_create( [self](PipelineStatus status) { if (!self) return; self->on_audio_sink_state_changed(status); })); MUST(self->m_audio_time_stretch_processor->connect_input(*self->m_audio_mixer)); MUST(self->m_audio_sink->connect_input(*self->m_audio_time_stretch_processor)); self->set_time_provider(*self->m_audio_sink); self->m_audio_sink->on_audio_output_error = [self](Error&& error) { if (!self) return; dbgln("Audio output initialization failed with error: {}", error); self->disable_audio(); }; } if (self->on_track_added) { for (size_t i = first_new_audio_index; i < self->m_audio_tracks.size(); i++) self->on_track_added(self->m_audio_tracks[i]); for (size_t i = first_new_video_index; i < self->m_video_tracks.size(); i++) self->on_track_added(self->m_video_tracks[i]); } if (self->on_metadata_parsed) self->on_metadata_parsed(); }); return {}; } NonnullOwnPtr PlaybackManager::create() { auto playback_manager = adopt_own(*new (nothrow) PlaybackManager()); playback_manager->m_handler = make(*playback_manager); playback_manager->m_handler->on_enter(); return playback_manager; } PlaybackManager::PlaybackManager() : m_weak_link(make_ref_counted(*this)) , m_time_provider(make_ref_counted()) { } PlaybackManager::~PlaybackManager() { m_time_provider->pause(); m_weak_link->revoke({}); } static void handle_media_init_error(WeakPlaybackManager self, Core::EventLoop& main_thread_event_loop, DecoderError error) { main_thread_event_loop.deferred_invoke([self = move(self), error = move(error)] mutable { if (!self) return; if (self->on_unsupported_format_error) self->on_unsupported_format_error(move(error)); }); } void PlaybackManager::add_media_source(NonnullRefPtr const& stream) { auto self = weak(); auto& main_thread_event_loop = Core::EventLoop::current(); Threading::ThreadPool::the().submit([self = move(self), stream, &main_thread_event_loop] mutable { auto demuxer_or_error = create_demuxer_for_stream(stream); if (demuxer_or_error.is_error()) { handle_media_init_error(move(self), main_thread_event_loop, demuxer_or_error.release_error()); return; } auto maybe_error = prepare_playback_from_demuxer(self, demuxer_or_error.release_value(), main_thread_event_loop); if (maybe_error.is_error()) handle_media_init_error(move(self), main_thread_event_loop, maybe_error.release_error()); }); } void PlaybackManager::add_media_source(NonnullRefPtr const& demuxer) { auto self = weak(); auto& main_thread_event_loop = Core::EventLoop::current(); Threading::ThreadPool::the().submit([self = move(self), demuxer, &main_thread_event_loop] mutable { auto maybe_error = prepare_playback_from_demuxer(self, demuxer, main_thread_event_loop); if (maybe_error.is_error()) handle_media_init_error(move(self), main_thread_event_loop, maybe_error.release_error()); }); } WeakPlaybackManager PlaybackManager::weak() { return WeakPlaybackManager(m_weak_link); } void PlaybackManager::set_up_producers() { for (auto const& video_track_data : m_video_track_datas) { video_track_data.producer->set_error_handler([self = weak()](DecoderError&& error) { if (!self) return; self->dispatch_error(move(error)); }); video_track_data.producer->set_duration_change_handler([self = weak()](AK::Duration time) { if (!self) return; self->check_for_duration_change(time); }); } for (auto const& audio_track_data : m_audio_track_datas) { audio_track_data.producer->set_error_handler([self = weak()](DecoderError&& error) { if (!self) return; self->dispatch_error(move(error)); }); audio_track_data.producer->set_duration_change_handler([self = weak()](AK::Duration time) { if (!self) return; self->check_for_duration_change(time); }); } } AK::Duration PlaybackManager::current_time() const { auto time = m_handler->current_time(); return min(time, duration()); } void PlaybackManager::on_audio_sink_state_changed(PipelineStatus status) { m_audio_sink_status = status; update_pipeline_state(); } void PlaybackManager::on_video_sink_state_changed(Track const& track, PipelineStatus status) { auto& track_data = get_video_data_for_track(track); track_data.sink_status = status; update_pipeline_state(); } PipelineStatus PlaybackManager::combined_pipeline_status() const { auto status = PipelineStatus::EndOfStream; if (m_audio_sink != nullptr) status = select_combined_pipeline_status(status, m_audio_sink_status); for (auto const& track_data : m_video_track_datas) { if (track_data.display == nullptr) continue; status = select_combined_pipeline_status(status, track_data.sink_status); } return status; } void PlaybackManager::update_pipeline_state() { m_handler->on_pipeline_status_changed(combined_pipeline_status()); } void PlaybackManager::reset_pipeline_state() { for (auto& track_data : m_video_track_datas) { if (track_data.display == nullptr) continue; track_data.sink_status = PipelineStatus::Pending; } m_audio_sink_status = m_audio_sink != nullptr ? PipelineStatus::Pending : PipelineStatus::HaveData; } void PlaybackManager::check_for_duration_change(AK::Duration duration) { if (m_duration >= duration) return; m_duration = duration; if (on_duration_change) on_duration_change(m_duration); } void PlaybackManager::dispatch_error(DecoderError&& error) { VERIFY(error.category() != DecoderErrorCategory::EndOfStream); if (m_is_in_error_state) return; m_is_in_error_state = true; if (on_error) on_error(move(error)); } void PlaybackManager::set_time_provider(NonnullRefPtr const& provider) { auto time = current_time(); provider->seek(time); m_time_provider = provider; for (auto& track_data : m_video_track_datas) { if (!track_data.display) continue; track_data.display->set_time_provider(provider); } provider->set_playback_rate(m_playback_rate); if (is_playing()) provider->resume(); } void PlaybackManager::disable_audio() { m_audio_mixer = nullptr; m_audio_time_stretch_processor = nullptr; m_audio_sink = nullptr; set_time_provider(make_ref_counted()); on_audio_sink_state_changed(PipelineStatus::EndOfStream); } NonnullRefPtr PlaybackManager::get_or_create_the_displaying_video_sink_for_track(Track const& track) { auto& track_data = get_video_data_for_track(track); if (track_data.display == nullptr) { track_data.sink_status = PipelineStatus::HaveData; auto display = MUST(Media::DisplayingVideoSink::try_create(m_time_provider, [self = weak(), track](PipelineStatus status) { if (!self) return; self->on_video_sink_state_changed(track, status); })); MUST(display->connect_input(track_data.producer)); track_data.display = move(display); update_pipeline_state(); } return *track_data.display; } void PlaybackManager::remove_the_displaying_video_sink_for_track(Track const& track) { auto& track_data = get_video_data_for_track(track); VERIFY(track_data.display); track_data.display->disconnect_input(track_data.producer); track_data.display = nullptr; track_data.sink_status = PipelineStatus::HaveData; update_pipeline_state(); } void PlaybackManager::enable_an_audio_track(Track const& track) { auto& track_data = get_audio_data_for_track(track); VERIFY(!track_data.enabled); m_audio_sink_status = PipelineStatus::HaveData; if (m_audio_mixer) { m_audio_mixer->seek(current_time()); MUST(m_audio_mixer->connect_input(track_data.producer)); } track_data.enabled = true; update_pipeline_state(); } void PlaybackManager::disable_an_audio_track(Track const& track) { auto& track_data = get_audio_data_for_track(track); VERIFY(track_data.enabled); m_audio_sink_status = PipelineStatus::HaveData; if (m_audio_mixer) { m_audio_mixer->seek(current_time()); m_audio_mixer->disconnect_input(track_data.producer); } track_data.enabled = false; update_pipeline_state(); } bool PlaybackManager::track_is_enabled(Track const& track) const { if (track.type() == TrackType::Video) { auto const& track_data = get_video_data_for_track(track); return track_data.display != nullptr; } VERIFY(track.type() == TrackType::Audio); auto const& track_data = get_audio_data_for_track(track); return track_data.enabled; } void PlaybackManager::start() { m_handler->start(); } void PlaybackManager::play() { m_handler->play(); } void PlaybackManager::pause() { m_handler->pause(); } void PlaybackManager::seek(AK::Duration timestamp, SeekMode mode) { reset_pipeline_state(); m_handler->seek(timestamp, mode); m_is_in_error_state = false; update_pipeline_state(); } bool PlaybackManager::is_playing() { return m_handler->is_playing(); } PlaybackState PlaybackManager::state() { return m_handler->state(); } AvailableData PlaybackManager::available_data() { return m_handler->available_data(); } TimeRanges PlaybackManager::buffered_time_ranges() const { TimeRanges intersection { { AK::Duration::zero(), m_duration } }; auto intersect_ranges = [&](auto const& track_datas) { for (auto const& track_data : track_datas) { if (!track_is_enabled(track_data.track)) continue; auto range = track_data.producer->buffered_time_ranges(); intersection = intersection.intersection(range); } }; intersect_ranges(m_video_track_datas); intersect_ranges(m_audio_track_datas); return intersection; } void PlaybackManager::set_volume(double volume) { if (m_audio_sink) m_audio_sink->set_volume(volume); } void PlaybackManager::set_playback_rate(float rate) { m_playback_rate = rate; m_time_provider->set_playback_rate(rate); } }