LibVideo: Add PlaybackManager to load and decode videos

This file will be the basis for abstracting away the out-of-thread or
later out-of-process decoding from applications displaying videos. For
now, the demuxer is hardcoded to be MatroskaParser, since that is all
we support so far. The demuxer should later be selected based on the
file header.

The playback and decoding are currently all done on one thread using
timers. The design of the code is such that adding threading should
be trivial, at least based on an earlier version of the code. For now,
though, it's better that this runs in one thread, as the multithreaded
approach causes the Video Player to lock up permanently after a few
frames are decoded.

AK/Debug.h.in

@@ -342,6 +342,10 @@
 #cmakedefine01 PDF_DEBUG
 #endif
 
+#ifndef PLAYBACK_MANAGER_DEBUG
+#cmakedefine01 PLAYBACK_MANAGER_DEBUG
+#endif
+
 #ifndef PNG_DEBUG
 #cmakedefine01 PNG_DEBUG
 #endif

Meta/CMake/all_the_debug_macros.cmake

@@ -141,6 +141,7 @@ set(PATA_DEBUG ON)
 set(PATH_DEBUG ON)
 set(PCI_DEBUG ON)
 set(PDF_DEBUG ON)
+set(PLAYBACK_MANAGER_DEBUG ON)
 set(PNG_DEBUG ON)
 set(POLL_SELECT_DEBUG ON)
 set(PORTABLE_IMAGE_LOADER_DEBUG ON)

Userland/Libraries/LibVideo/CMakeLists.txt

@@ -4,6 +4,7 @@ set(SOURCES
     Color/TransferCharacteristics.cpp
     MatroskaDemuxer.cpp
     MatroskaReader.cpp
+    PlaybackManager.cpp
     VideoFrame.cpp
     VP9/BitStream.cpp
     VP9/Decoder.cpp

Userland/Libraries/LibVideo/PlaybackManager.cpp

@@ -0,0 +1,244 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/Format.h>
+#include <LibCore/Timer.h>
+#include <LibVideo/MatroskaReader.h>
+#include <LibVideo/VP9/Decoder.h>
+
+#include "MatroskaDemuxer.h"
+#include "PlaybackManager.h"
+
+namespace Video {
+
+// We post DecoderErrors to the event queue to be handled, since some will occur off the main thread.
+#define TRY_OR_POST_ERROR_AND_RETURN(expression, return_value)                                         \
+    ({                                                                                                 \
+        auto _temporary_result = ((expression));                                                       \
+        if (_temporary_result.is_error()) {                                                            \
+            dbgln("Playback error encountered: {}", _temporary_result.error().string_literal());       \
+            m_main_loop.post_event(*this, make<DecoderErrorEvent>(_temporary_result.release_error())); \
+            return return_value;                                                                       \
+        }                                                                                              \
+        _temporary_result.release_value();                                                             \
+    })
+
+#define TRY_OR_POST_ERROR(expression) TRY_OR_POST_ERROR_AND_RETURN(expression, )
+
+DecoderErrorOr<NonnullRefPtr<PlaybackManager>> PlaybackManager::from_file(Object* event_handler, StringView filename)
+{
+    NonnullOwnPtr<Demuxer> demuxer = TRY(MatroskaDemuxer::from_file(filename));
+    auto video_tracks = demuxer->get_tracks_for_type(TrackType::Video);
+    if (video_tracks.is_empty())
+        return DecoderError::with_description(DecoderErrorCategory::Invalid, "No video track is present"sv);
+    auto track = video_tracks[0];
+
+    dbgln_if(PLAYBACK_MANAGER_DEBUG, "Selecting video track number {}", track.identifier());
+
+    NonnullOwnPtr<VideoDecoder> decoder = make<VP9::Decoder>();
+    return PlaybackManager::construct(event_handler, demuxer, track, decoder);
+}
+
+PlaybackManager::PlaybackManager(Object* event_handler, NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>& decoder)
+    : Object(event_handler)
+    , m_main_loop(Core::EventLoop::current())
+    , m_demuxer(move(demuxer))
+    , m_selected_video_track(video_track)
+    , m_decoder(move(decoder))
+    , m_frame_queue(make<VideoFrameQueue>())
+    , m_present_timer(Core::Timer::construct())
+    , m_decode_timer(Core::Timer::construct())
+{
+    m_present_timer->set_single_shot(true);
+    m_present_timer->set_interval(0);
+    m_present_timer->on_timeout = [&] { update_presented_frame(); };
+
+    m_decode_timer->set_single_shot(true);
+    m_decode_timer->set_interval(0);
+    m_decode_timer->on_timeout = [&] { on_decode_timer(); };
+}
+
+void PlaybackManager::set_playback_status(PlaybackStatus status)
+{
+    if (status != m_status) {
+        auto old_status = m_status;
+        m_status = status;
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set playback status from {} to {}", playback_status_to_string(old_status), playback_status_to_string(m_status));
+
+        if (status == PlaybackStatus::Playing) {
+            if (old_status == PlaybackStatus::Stopped) {
+                restart_playback();
+                m_frame_queue->clear();
+                m_skipped_frames = 0;
+            }
+            m_last_present_in_real_time = Time::now_monotonic();
+            m_present_timer->start();
+        } else {
+            m_last_present_in_media_time = current_playback_time();
+            m_last_present_in_real_time = Time::zero();
+            m_present_timer->stop();
+        }
+
+        m_main_loop.post_event(*this, make<PlaybackStatusChangeEvent>(status, old_status));
+    }
+}
+
+void PlaybackManager::event(Core::Event& event)
+{
+    if (event.type() == DecoderErrorOccurred) {
+        auto& error_event = static_cast<DecoderErrorEvent&>(event);
+        VERIFY(error_event.error().category() != DecoderErrorCategory::EndOfStream);
+    }
+
+    // Allow events to bubble up in all cases.
+    event.ignore();
+}
+
+void PlaybackManager::resume_playback()
+{
+    set_playback_status(PlaybackStatus::Playing);
+}
+
+void PlaybackManager::pause_playback()
+{
+    set_playback_status(PlaybackStatus::Paused);
+}
+
+bool PlaybackManager::prepare_next_frame()
+{
+    if (m_next_frame.has_value())
+        return true;
+    if (m_frame_queue->is_empty())
+        return false;
+    auto frame_item = m_frame_queue->dequeue();
+    m_next_frame.emplace(frame_item);
+    m_decode_timer->start();
+    return true;
+}
+
+Time PlaybackManager::current_playback_time()
+{
+    if (is_playing())
+        return m_last_present_in_media_time + (Time::now_monotonic() - m_last_present_in_real_time);
+    return m_last_present_in_media_time;
+}
+
+Time PlaybackManager::duration()
+{
+    return m_demuxer->duration();
+}
+
+void PlaybackManager::update_presented_frame()
+{
+    bool out_of_queued_frames = false;
+    Optional<FrameQueueItem> frame_item_to_display;
+
+    while (true) {
+        out_of_queued_frames = out_of_queued_frames || !prepare_next_frame();
+        if (out_of_queued_frames)
+            break;
+        VERIFY(m_next_frame.has_value());
+        if (m_next_frame->timestamp > current_playback_time() || m_next_frame->is_eos_marker())
+            break;
+
+        if (frame_item_to_display.has_value()) {
+            dbgln_if(PLAYBACK_MANAGER_DEBUG, "At {}ms: Dropped frame with timestamp {}ms for the next at {}ms", current_playback_time().to_milliseconds(), frame_item_to_display->timestamp.to_milliseconds(), m_next_frame->timestamp.to_milliseconds());
+            m_skipped_frames++;
+        }
+        frame_item_to_display = m_next_frame.release_value();
+    }
+
+    if (!out_of_queued_frames && frame_item_to_display.has_value()) {
+        m_main_loop.post_event(*this, make<VideoFramePresentEvent>(frame_item_to_display->bitmap));
+        m_last_present_in_media_time = current_playback_time();
+        m_last_present_in_real_time = Time::now_monotonic();
+        frame_item_to_display.clear();
+    }
+
+    if (frame_item_to_display.has_value()) {
+        VERIFY(!m_next_frame.has_value());
+        m_next_frame = frame_item_to_display;
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Set next frame back to dequeued item at timestamp {}ms", m_next_frame->timestamp.to_milliseconds());
+    }
+
+    if (!is_playing())
+        return;
+
+    if (!out_of_queued_frames) {
+        if (m_next_frame->is_eos_marker()) {
+            set_playback_status(PlaybackStatus::Stopped);
+            m_next_frame.clear();
+            return;
+        }
+
+        auto frame_time_ms = (m_next_frame.value().timestamp - current_playback_time()).to_milliseconds();
+        VERIFY(frame_time_ms <= NumericLimits<int>::max());
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Time until next frame is {}ms", frame_time_ms);
+        m_present_timer->start(max(static_cast<int>(frame_time_ms), 0));
+        return;
+    }
+
+    set_playback_status(PlaybackStatus::Buffering);
+    m_decode_timer->start();
+}
+
+void PlaybackManager::restart_playback()
+{
+    m_last_present_in_media_time = Time::zero();
+    m_last_present_in_real_time = Time::zero();
+    TRY_OR_POST_ERROR(m_demuxer->seek_to_most_recent_keyframe(m_selected_video_track, 0));
+}
+
+bool PlaybackManager::decode_and_queue_one_sample()
+{
+    if (m_frame_queue->size() >= FRAME_BUFFER_COUNT)
+        return false;
+#if PLAYBACK_MANAGER_DEBUG
+    auto start_time = Time::now_monotonic();
+#endif
+
+    auto frame_sample_result = m_demuxer->get_next_video_sample_for_track(m_selected_video_track);
+    if (frame_sample_result.is_error()) {
+        if (frame_sample_result.error().category() == DecoderErrorCategory::EndOfStream) {
+            m_frame_queue->enqueue(FrameQueueItem::eos_marker());
+            return false;
+        }
+        m_main_loop.post_event(*this, make<DecoderErrorEvent>(frame_sample_result.release_error()));
+        return false;
+    }
+    auto frame_sample = frame_sample_result.release_value();
+
+    TRY_OR_POST_ERROR_AND_RETURN(m_decoder->receive_sample(frame_sample->data()), false);
+    auto decoded_frame = TRY_OR_POST_ERROR_AND_RETURN(m_decoder->get_decoded_frame(), false);
+
+    auto& cicp = decoded_frame->cicp();
+    cicp.adopt_specified_values(frame_sample->container_cicp());
+    cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::ColorRange::Studio });
+
+    auto bitmap = TRY_OR_POST_ERROR_AND_RETURN(decoded_frame->to_bitmap(), false);
+    m_frame_queue->enqueue(FrameQueueItem { bitmap, frame_sample->timestamp() });
+
+#if PLAYBACK_MANAGER_DEBUG
+    auto end_time = Time::now_monotonic();
+    dbgln("Decoding took {}ms", (end_time - start_time).to_milliseconds());
+#endif
+
+    return true;
+}
+
+void PlaybackManager::on_decode_timer()
+{
+    if (!decode_and_queue_one_sample() && is_buffering()) {
+        set_playback_status(PlaybackStatus::Playing);
+        return;
+    }
+
+    // Continually decode until buffering is complete
+    if (is_buffering())
+        m_decode_timer->start();
+}
+
+}

Userland/Libraries/LibVideo/PlaybackManager.h

@@ -0,0 +1,180 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Atomic.h>
+#include <AK/Function.h>
+#include <AK/NonnullOwnPtr.h>
+#include <AK/Queue.h>
+#include <AK/Time.h>
+#include <LibCore/EventLoop.h>
+#include <LibCore/SharedCircularQueue.h>
+#include <LibGfx/Bitmap.h>
+#include <LibThreading/ConditionVariable.h>
+#include <LibThreading/Mutex.h>
+#include <LibThreading/Thread.h>
+
+#include "Demuxer.h"
+#include "MatroskaDocument.h"
+#include "VideoDecoder.h"
+
+namespace Video {
+
+enum class PlaybackStatus {
+    Playing,
+    Paused,
+    Buffering,
+    Seeking,
+    Stopped,
+};
+
+struct FrameQueueItem {
+    static FrameQueueItem eos_marker()
+    {
+        return { nullptr, Time::max() };
+    }
+
+    RefPtr<Gfx::Bitmap> bitmap;
+    Time timestamp;
+
+    bool is_eos_marker() const { return !bitmap; }
+};
+
+static constexpr size_t FRAME_BUFFER_COUNT = 4;
+using VideoFrameQueue = Queue<FrameQueueItem, FRAME_BUFFER_COUNT>;
+
+class PlaybackManager : public Core::Object {
+    C_OBJECT(PlaybackManager)
+
+public:
+    static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> from_file(Object* event_handler, StringView file);
+    static DecoderErrorOr<NonnullRefPtr<PlaybackManager>> from_data(Object* event_handler, Span<u8> data);
+
+    PlaybackManager(Object* event_handler, NonnullOwnPtr<Demuxer>& demuxer, Track video_track, NonnullOwnPtr<VideoDecoder>& decoder);
+    ~PlaybackManager() override = default;
+
+    void resume_playback();
+    void pause_playback();
+    void restart_playback();
+    bool is_playing() const { return m_status == PlaybackStatus::Playing; }
+    bool is_buffering() const { return m_status == PlaybackStatus::Buffering; }
+    bool is_stopped() const { return m_status == PlaybackStatus::Stopped; }
+
+    u64 number_of_skipped_frames() const { return m_skipped_frames; }
+
+    void event(Core::Event& event) override;
+
+    Time current_playback_time();
+    Time duration();
+
+    Function<void(NonnullRefPtr<Gfx::Bitmap>, Time)> on_frame_present;
+
+private:
+    void set_playback_status(PlaybackStatus status);
+
+    bool prepare_next_frame();
+    void update_presented_frame();
+
+    // Runs off the main thread
+    bool decode_and_queue_one_sample();
+    void on_decode_timer();
+
+    Core::EventLoop& m_main_loop;
+
+    PlaybackStatus m_status { PlaybackStatus::Stopped };
+    Time m_last_present_in_media_time = Time::zero();
+    Time m_last_present_in_real_time = Time::zero();
+
+    NonnullOwnPtr<Demuxer> m_demuxer;
+    Track m_selected_video_track;
+    NonnullOwnPtr<VideoDecoder> m_decoder;
+
+    NonnullOwnPtr<VideoFrameQueue> m_frame_queue;
+    Optional<FrameQueueItem> m_next_frame;
+
+    NonnullRefPtr<Core::Timer> m_present_timer;
+    unsigned m_decoding_buffer_time_ms = 16;
+
+    NonnullRefPtr<Core::Timer> m_decode_timer;
+
+    u64 m_skipped_frames;
+};
+
+enum EventType : unsigned {
+    DecoderErrorOccurred = (('v' << 2) | ('i' << 1) | 'd') << 4,
+    VideoFramePresent,
+    PlaybackStatusChange,
+};
+
+class DecoderErrorEvent : public Core::Event {
+public:
+    explicit DecoderErrorEvent(DecoderError error)
+        : Core::Event(DecoderErrorOccurred)
+        , m_error(move(error))
+    {
+    }
+    virtual ~DecoderErrorEvent() = default;
+
+    DecoderError error() { return m_error; }
+
+private:
+    DecoderError m_error;
+};
+
+class VideoFramePresentEvent : public Core::Event {
+public:
+    VideoFramePresentEvent() = default;
+    explicit VideoFramePresentEvent(RefPtr<Gfx::Bitmap> frame)
+        : Core::Event(VideoFramePresent)
+        , m_frame(move(frame))
+    {
+    }
+    virtual ~VideoFramePresentEvent() = default;
+
+    RefPtr<Gfx::Bitmap> frame() { return m_frame; }
+
+private:
+    RefPtr<Gfx::Bitmap> m_frame;
+};
+
+class PlaybackStatusChangeEvent : public Core::Event {
+public:
+    PlaybackStatusChangeEvent() = default;
+    explicit PlaybackStatusChangeEvent(PlaybackStatus status, PlaybackStatus previous_status)
+        : Core::Event(PlaybackStatusChange)
+        , m_status(status)
+        , m_previous_status(previous_status)
+    {
+    }
+    virtual ~PlaybackStatusChangeEvent() = default;
+
+    PlaybackStatus status();
+    PlaybackStatus previous_status();
+
+private:
+    PlaybackStatus m_status;
+    PlaybackStatus m_previous_status;
+};
+
+inline StringView playback_status_to_string(PlaybackStatus status)
+{
+    switch (status) {
+    case PlaybackStatus::Playing:
+        return "Playing"sv;
+    case PlaybackStatus::Paused:
+        return "Paused"sv;
+    case PlaybackStatus::Buffering:
+        return "Buffering"sv;
+    case PlaybackStatus::Seeking:
+        return "Seeking"sv;
+    case PlaybackStatus::Stopped:
+        return "Stopped"sv;
+    }
+    return "Unknown"sv;
+};
+
+}