From 353e1c2b4dad88065197a294e32e788925373b6d Mon Sep 17 00:00:00 2001 From: Zaggy1024 Date: Sat, 29 Oct 2022 18:16:14 -0500 Subject: [PATCH] LibVideo: Add PlaybackManager to load and decode videos This file will be the basis for abstracting away the out-of-thread or later out-of-process decoding from applications displaying videos. For now, the demuxer is hardcoded to be MatroskaParser, since that is all we support so far. The demuxer should later be selected based on the file header. The playback and decoding are currently all done on one thread using timers. The design of the code is such that adding threading should be trivial, at least based on an earlier version of the code. For now, though, it's better that this runs in one thread, as the multithreaded approach causes the Video Player to lock up permanently after a few frames are decoded. --- AK/Debug.h.in | 4 + Meta/CMake/all_the_debug_macros.cmake | 1 + Userland/Libraries/LibVideo/CMakeLists.txt | 1 + .../Libraries/LibVideo/PlaybackManager.cpp | 244 ++++++++++++++++++ Userland/Libraries/LibVideo/PlaybackManager.h | 180 +++++++++++++ 5 files changed, 430 insertions(+) create mode 100644 Userland/Libraries/LibVideo/PlaybackManager.cpp create mode 100644 Userland/Libraries/LibVideo/PlaybackManager.h diff --git a/AK/Debug.h.in b/AK/Debug.h.in index c07eba6544..b5f3d95933 100644 --- a/AK/Debug.h.in +++ b/AK/Debug.h.in @@ -342,6 +342,10 @@ #cmakedefine01 PDF_DEBUG #endif +#ifndef PLAYBACK_MANAGER_DEBUG +#cmakedefine01 PLAYBACK_MANAGER_DEBUG +#endif + #ifndef PNG_DEBUG #cmakedefine01 PNG_DEBUG #endif diff --git a/Meta/CMake/all_the_debug_macros.cmake b/Meta/CMake/all_the_debug_macros.cmake index a36d64fcce..577f40ca46 100644 --- a/Meta/CMake/all_the_debug_macros.cmake +++ b/Meta/CMake/all_the_debug_macros.cmake @@ -141,6 +141,7 @@ set(PATA_DEBUG ON) set(PATH_DEBUG ON) set(PCI_DEBUG ON) set(PDF_DEBUG ON) +set(PLAYBACK_MANAGER_DEBUG ON) set(PNG_DEBUG ON) set(POLL_SELECT_DEBUG ON) set(PORTABLE_IMAGE_LOADER_DEBUG ON) diff --git a/Userland/Libraries/LibVideo/CMakeLists.txt b/Userland/Libraries/LibVideo/CMakeLists.txt index 5f4500e0b1..ab1a5192ef 100644 --- a/Userland/Libraries/LibVideo/CMakeLists.txt +++ b/Userland/Libraries/LibVideo/CMakeLists.txt @@ -4,6 +4,7 @@ set(SOURCES Color/TransferCharacteristics.cpp MatroskaDemuxer.cpp MatroskaReader.cpp + PlaybackManager.cpp VideoFrame.cpp VP9/BitStream.cpp VP9/Decoder.cpp diff --git a/Userland/Libraries/LibVideo/PlaybackManager.cpp b/Userland/Libraries/LibVideo/PlaybackManager.cpp new file mode 100644 index 0000000000..2999fe3df5 --- /dev/null +++ b/Userland/Libraries/LibVideo/PlaybackManager.cpp @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2022, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include + +#include "MatroskaDemuxer.h" +#include "PlaybackManager.h" + +namespace Video { + +// We post DecoderErrors to the event queue to be handled, since some will occur off the main thread. +#define TRY_OR_POST_ERROR_AND_RETURN(expression, return_value) \ + ({ \ + auto _temporary_result = ((expression)); \ + if (_temporary_result.is_error()) { \ + dbgln("Playback error encountered: {}", _temporary_result.error().string_literal()); \ + m_main_loop.post_event(*this, make(_temporary_result.release_error())); \ + return return_value; \ + } \ + _temporary_result.release_value(); \ + }) + +#define TRY_OR_POST_ERROR(expression) TRY_OR_POST_ERROR_AND_RETURN(expression, ) + +DecoderErrorOr> PlaybackManager::from_file(Object* event_handler, StringView filename) +{ + NonnullOwnPtr demuxer = TRY(MatroskaDemuxer::from_file(filename)); + auto video_tracks = demuxer->get_tracks_for_type(TrackType::Video); + if (video_tracks.is_empty()) + return DecoderError::with_description(DecoderErrorCategory::Invalid, "No video track is present"sv); + auto track = video_tracks[0]; + + dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier()); + + NonnullOwnPtr decoder = make(); + return PlaybackManager::construct(event_handler, demuxer, track, decoder); +} + +PlaybackManager::PlaybackManager(Object* event_handler, NonnullOwnPtr& demuxer, Track video_track, NonnullOwnPtr& decoder) + : Object(event_handler) + , m_main_loop(Core::EventLoop::current()) + , m_demuxer(move(demuxer)) + , m_selected_video_track(video_track) + , m_decoder(move(decoder)) + , m_frame_queue(make()) + , m_present_timer(Core::Timer::construct()) + , m_decode_timer(Core::Timer::construct()) +{ + m_present_timer->set_single_shot(true); + m_present_timer->set_interval(0); + m_present_timer->on_timeout = [&] { update_presented_frame(); }; + + m_decode_timer->set_single_shot(true); + m_decode_timer->set_interval(0); + m_decode_timer->on_timeout = [&] { on_decode_timer(); }; +} + +void PlaybackManager::set_playback_status(PlaybackStatus status) +{ + if (status != m_status) { + auto old_status = m_status; + m_status = status; + dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set playback status from {} to {}", playback_status_to_string(old_status), playback_status_to_string(m_status)); + + if (status == PlaybackStatus::Playing) { + if (old_status == PlaybackStatus::Stopped) { + restart_playback(); + m_frame_queue->clear(); + m_skipped_frames = 0; + } + m_last_present_in_real_time = Time::now_monotonic(); + m_present_timer->start(); + } else { + m_last_present_in_media_time = current_playback_time(); + m_last_present_in_real_time = Time::zero(); + m_present_timer->stop(); + } + + m_main_loop.post_event(*this, make(status, old_status)); + } +} + +void PlaybackManager::event(Core::Event& event) +{ + if (event.type() == DecoderErrorOccurred) { + auto& error_event = static_cast(event); + VERIFY(error_event.error().category() != DecoderErrorCategory::EndOfStream); + } + + // Allow events to bubble up in all cases. + event.ignore(); +} + +void PlaybackManager::resume_playback() +{ + set_playback_status(PlaybackStatus::Playing); +} + +void PlaybackManager::pause_playback() +{ + set_playback_status(PlaybackStatus::Paused); +} + +bool PlaybackManager::prepare_next_frame() +{ + if (m_next_frame.has_value()) + return true; + if (m_frame_queue->is_empty()) + return false; + auto frame_item = m_frame_queue->dequeue(); + m_next_frame.emplace(frame_item); + m_decode_timer->start(); + return true; +} + +Time PlaybackManager::current_playback_time() +{ + if (is_playing()) + return m_last_present_in_media_time + (Time::now_monotonic() - m_last_present_in_real_time); + return m_last_present_in_media_time; +} + +Time PlaybackManager::duration() +{ + return m_demuxer->duration(); +} + +void PlaybackManager::update_presented_frame() +{ + bool out_of_queued_frames = false; + Optional frame_item_to_display; + + while (true) { + out_of_queued_frames = out_of_queued_frames || !prepare_next_frame(); + if (out_of_queued_frames) + break; + VERIFY(m_next_frame.has_value()); + if (m_next_frame->timestamp > current_playback_time() || m_next_frame->is_eos_marker()) + break; + + if (frame_item_to_display.has_value()) { + dbgln_if(PLAYBACK_MANAGER_DEBUG, "At {}ms: Dropped frame with timestamp {}ms for the next at {}ms", current_playback_time().to_milliseconds(), frame_item_to_display->timestamp.to_milliseconds(), m_next_frame->timestamp.to_milliseconds()); + m_skipped_frames++; + } + frame_item_to_display = m_next_frame.release_value(); + } + + if (!out_of_queued_frames && frame_item_to_display.has_value()) { + m_main_loop.post_event(*this, make(frame_item_to_display->bitmap)); + m_last_present_in_media_time = current_playback_time(); + m_last_present_in_real_time = Time::now_monotonic(); + frame_item_to_display.clear(); + } + + if (frame_item_to_display.has_value()) { + VERIFY(!m_next_frame.has_value()); + m_next_frame = frame_item_to_display; + dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set next frame back to dequeued item at timestamp {}ms", m_next_frame->timestamp.to_milliseconds()); + } + + if (!is_playing()) + return; + + if (!out_of_queued_frames) { + if (m_next_frame->is_eos_marker()) { + set_playback_status(PlaybackStatus::Stopped); + m_next_frame.clear(); + return; + } + + auto frame_time_ms = (m_next_frame.value().timestamp - current_playback_time()).to_milliseconds(); + VERIFY(frame_time_ms <= NumericLimits::max()); + dbgln_if(PLAYBACK_MANAGER_DEBUG, "Time until next frame is {}ms", frame_time_ms); + m_present_timer->start(max(static_cast(frame_time_ms), 0)); + return; + } + + set_playback_status(PlaybackStatus::Buffering); + m_decode_timer->start(); +} + +void PlaybackManager::restart_playback() +{ + m_last_present_in_media_time = Time::zero(); + m_last_present_in_real_time = Time::zero(); + TRY_OR_POST_ERROR(m_demuxer->seek_to_most_recent_keyframe(m_selected_video_track, 0)); +} + +bool PlaybackManager::decode_and_queue_one_sample() +{ + if (m_frame_queue->size() >= FRAME_BUFFER_COUNT) + return false; +#if PLAYBACK_MANAGER_DEBUG + auto start_time = Time::now_monotonic(); +#endif + + auto frame_sample_result = m_demuxer->get_next_video_sample_for_track(m_selected_video_track); + if (frame_sample_result.is_error()) { + if (frame_sample_result.error().category() == DecoderErrorCategory::EndOfStream) { + m_frame_queue->enqueue(FrameQueueItem::eos_marker()); + return false; + } + m_main_loop.post_event(*this, make(frame_sample_result.release_error())); + return false; + } + auto frame_sample = frame_sample_result.release_value(); + + TRY_OR_POST_ERROR_AND_RETURN(m_decoder->receive_sample(frame_sample->data()), false); + auto decoded_frame = TRY_OR_POST_ERROR_AND_RETURN(m_decoder->get_decoded_frame(), false); + + auto& cicp = decoded_frame->cicp(); + cicp.adopt_specified_values(frame_sample->container_cicp()); + cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::ColorRange::Studio }); + + auto bitmap = TRY_OR_POST_ERROR_AND_RETURN(decoded_frame->to_bitmap(), false); + m_frame_queue->enqueue(FrameQueueItem { bitmap, frame_sample->timestamp() }); + +#if PLAYBACK_MANAGER_DEBUG + auto end_time = Time::now_monotonic(); + dbgln("Decoding took {}ms", (end_time - start_time).to_milliseconds()); +#endif + + return true; +} + +void PlaybackManager::on_decode_timer() +{ + if (!decode_and_queue_one_sample() && is_buffering()) { + set_playback_status(PlaybackStatus::Playing); + return; + } + + // Continually decode until buffering is complete + if (is_buffering()) + m_decode_timer->start(); +} + +} diff --git a/Userland/Libraries/LibVideo/PlaybackManager.h b/Userland/Libraries/LibVideo/PlaybackManager.h new file mode 100644 index 0000000000..77a91fc17a --- /dev/null +++ b/Userland/Libraries/LibVideo/PlaybackManager.h @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2022, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Demuxer.h" +#include "MatroskaDocument.h" +#include "VideoDecoder.h" + +namespace Video { + +enum class PlaybackStatus { + Playing, + Paused, + Buffering, + Seeking, + Stopped, +}; + +struct FrameQueueItem { + static FrameQueueItem eos_marker() + { + return { nullptr, Time::max() }; + } + + RefPtr bitmap; + Time timestamp; + + bool is_eos_marker() const { return !bitmap; } +}; + +static constexpr size_t FRAME_BUFFER_COUNT = 4; +using VideoFrameQueue = Queue; + +class PlaybackManager : public Core::Object { + C_OBJECT(PlaybackManager) + +public: + static DecoderErrorOr> from_file(Object* event_handler, StringView file); + static DecoderErrorOr> from_data(Object* event_handler, Span data); + + PlaybackManager(Object* event_handler, NonnullOwnPtr& demuxer, Track video_track, NonnullOwnPtr& decoder); + ~PlaybackManager() override = default; + + void resume_playback(); + void pause_playback(); + void restart_playback(); + bool is_playing() const { return m_status == PlaybackStatus::Playing; } + bool is_buffering() const { return m_status == PlaybackStatus::Buffering; } + bool is_stopped() const { return m_status == PlaybackStatus::Stopped; } + + u64 number_of_skipped_frames() const { return m_skipped_frames; } + + void event(Core::Event& event) override; + + Time current_playback_time(); + Time duration(); + + Function, Time)> on_frame_present; + +private: + void set_playback_status(PlaybackStatus status); + + bool prepare_next_frame(); + void update_presented_frame(); + + // Runs off the main thread + bool decode_and_queue_one_sample(); + void on_decode_timer(); + + Core::EventLoop& m_main_loop; + + PlaybackStatus m_status { PlaybackStatus::Stopped }; + Time m_last_present_in_media_time = Time::zero(); + Time m_last_present_in_real_time = Time::zero(); + + NonnullOwnPtr m_demuxer; + Track m_selected_video_track; + NonnullOwnPtr m_decoder; + + NonnullOwnPtr m_frame_queue; + Optional m_next_frame; + + NonnullRefPtr m_present_timer; + unsigned m_decoding_buffer_time_ms = 16; + + NonnullRefPtr m_decode_timer; + + u64 m_skipped_frames; +}; + +enum EventType : unsigned { + DecoderErrorOccurred = (('v' << 2) | ('i' << 1) | 'd') << 4, + VideoFramePresent, + PlaybackStatusChange, +}; + +class DecoderErrorEvent : public Core::Event { +public: + explicit DecoderErrorEvent(DecoderError error) + : Core::Event(DecoderErrorOccurred) + , m_error(move(error)) + { + } + virtual ~DecoderErrorEvent() = default; + + DecoderError error() { return m_error; } + +private: + DecoderError m_error; +}; + +class VideoFramePresentEvent : public Core::Event { +public: + VideoFramePresentEvent() = default; + explicit VideoFramePresentEvent(RefPtr frame) + : Core::Event(VideoFramePresent) + , m_frame(move(frame)) + { + } + virtual ~VideoFramePresentEvent() = default; + + RefPtr frame() { return m_frame; } + +private: + RefPtr m_frame; +}; + +class PlaybackStatusChangeEvent : public Core::Event { +public: + PlaybackStatusChangeEvent() = default; + explicit PlaybackStatusChangeEvent(PlaybackStatus status, PlaybackStatus previous_status) + : Core::Event(PlaybackStatusChange) + , m_status(status) + , m_previous_status(previous_status) + { + } + virtual ~PlaybackStatusChangeEvent() = default; + + PlaybackStatus status(); + PlaybackStatus previous_status(); + +private: + PlaybackStatus m_status; + PlaybackStatus m_previous_status; +}; + +inline StringView playback_status_to_string(PlaybackStatus status) +{ + switch (status) { + case PlaybackStatus::Playing: + return "Playing"sv; + case PlaybackStatus::Paused: + return "Paused"sv; + case PlaybackStatus::Buffering: + return "Buffering"sv; + case PlaybackStatus::Seeking: + return "Seeking"sv; + case PlaybackStatus::Stopped: + return "Stopped"sv; + } + return "Unknown"sv; +}; + +}