LibWeb/Ladybird: Use the abstract audio output in a new audio plugin

The implementation of this plugin is meant to eventually replace all
current audio plugins in Ladybird. The benefits over the current Qt-
based audio playback plugin in Ladybird are:

- Low latency: With direct access to PulseAudio, we can ask for a
specific latency to output to allow minimal delay when pausing or
seeking a stream.
- Accurate timestamps: The Qt audio playback API does not expose audio
time properly. When we have access directly to PulseAudio APIs, we can
enable their timing interpolation to get an accurate monotonically-
increasing timestamp of the playing audio.
- Resiliency: With more control over how the underlying audio API is
called, we have the power to fix most bugs we might encounter. The
PulseAudio wrappers already avoid some bugs that occur with QAudioSink
when running through WSLg.

Ladybird/WebContent/CMakeLists.txt

@@ -29,3 +29,7 @@ target_link_libraries(WebContent PRIVATE Qt::Core Qt::Network Qt::Multimedia Lib
 if (ANDROID)
     link_android_libs(WebContent)
 endif()
+
+if (HAVE_PULSEAUDIO)
+    target_compile_definitions(WebContent PRIVATE HAVE_PULSEAUDIO=1)
+endif()

Ladybird/WebContent/main.cpp

@@ -27,6 +27,7 @@
 #include <LibWeb/Loader/FrameLoader.h>
 #include <LibWeb/Loader/ResourceLoader.h>
 #include <LibWeb/PermissionsPolicy/AutoplayAllowlist.h>
+#include <LibWeb/Platform/AudioCodecPluginAgnostic.h>
 #include <LibWeb/Platform/EventLoopPluginSerenity.h>
 #include <LibWeb/WebSockets/WebSocket.h>
 #include <LibWebView/RequestServerAdapter.h>
@@ -54,7 +55,11 @@ ErrorOr<int> serenity_main(Main::Arguments arguments)
     Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPlugin);
 
     Web::Platform::AudioCodecPlugin::install_creation_hook([](auto loader) {
+#if defined(HAVE_PULSEAUDIO)
+        return Web::Platform::AudioCodecPluginAgnostic::create(move(loader));
+#else
         return Ladybird::AudioCodecPluginQt::create(move(loader));
+#endif
     });
 
     Web::FrameLoader::set_default_favicon_path(DeprecatedString::formatted("{}/res/icons/16x16/app-browser.png", s_serenity_resource_root));

Userland/Libraries/LibWeb/CMakeLists.txt

@@ -489,6 +489,7 @@ set(SOURCES
     PermissionsPolicy/AutoplayAllowlist.cpp
     PixelUnits.cpp
     Platform/AudioCodecPlugin.cpp
+    Platform/AudioCodecPluginAgnostic.cpp
     Platform/EventLoopPlugin.cpp
     Platform/EventLoopPluginSerenity.cpp
     Platform/FontPlugin.cpp

Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.cpp

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/MemoryStream.h>
+#include <AK/WeakPtr.h>
+#include <LibAudio/Loader.h>
+#include <LibCore/EventLoop.h>
+#include <LibCore/ThreadedPromise.h>
+#include <LibCore/Timer.h>
+
+#include "AudioCodecPluginAgnostic.h"
+
+namespace Web::Platform {
+
+constexpr int update_interval = 10;
+
+static Duration timestamp_from_samples(i64 samples, u32 sample_rate)
+{
+    return Duration::from_milliseconds(samples * 1000 / sample_rate);
+}
+
+static Duration get_loader_timestamp(NonnullRefPtr<Audio::Loader> const& loader)
+{
+    return timestamp_from_samples(loader->loaded_samples(), loader->sample_rate());
+}
+
+ErrorOr<NonnullOwnPtr<AudioCodecPluginAgnostic>> AudioCodecPluginAgnostic::create(NonnullRefPtr<Audio::Loader> const& loader)
+{
+    auto duration = timestamp_from_samples(loader->total_samples(), loader->sample_rate());
+
+    auto update_timer = TRY(Core::Timer::try_create());
+    update_timer->set_interval(update_interval);
+
+    auto plugin = TRY(adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginAgnostic(loader, duration, move(update_timer))));
+
+    constexpr u32 latency_ms = 100;
+    RefPtr<Audio::PlaybackStream> output = TRY(Audio::PlaybackStream::create(
+        Audio::OutputState::Suspended, loader->sample_rate(), loader->num_channels(), latency_ms,
+        [&plugin = *plugin, loader](Bytes buffer, Audio::PcmSampleFormat format, size_t sample_count) -> ReadonlyBytes {
+            VERIFY(format == Audio::PcmSampleFormat::Float32);
+            auto samples = loader->get_more_samples(sample_count).release_value_but_fixme_should_propagate_errors();
+            VERIFY(samples.size() <= sample_count);
+            FixedMemoryStream writing_stream { buffer };
+
+            for (auto& sample : samples) {
+                MUST(writing_stream.write_value(sample.left));
+                MUST(writing_stream.write_value(sample.right));
+            }
+
+            // FIXME: Check if we have loaded samples past the current known duration, and if so, update it
+            //        and notify the media element.
+            return buffer.trim(writing_stream.offset());
+        }));
+
+    output->set_underrun_callback([&plugin = *plugin, loader, output]() {
+        auto new_device_time = output->total_time_played().release_value_but_fixme_should_propagate_errors();
+        auto new_media_time = timestamp_from_samples(loader->loaded_samples(), loader->sample_rate());
+        plugin.m_main_thread_event_loop.deferred_invoke([&plugin, new_device_time, new_media_time]() {
+            plugin.m_last_resume_in_device_time = new_device_time;
+            plugin.m_last_resume_in_media_time = new_media_time;
+        });
+    });
+
+    plugin->m_output = move(output);
+
+    return plugin;
+}
+
+AudioCodecPluginAgnostic::AudioCodecPluginAgnostic(NonnullRefPtr<Audio::Loader> loader, Duration duration, NonnullRefPtr<Core::Timer> update_timer)
+    : m_loader(move(loader))
+    , m_duration(duration)
+    , m_main_thread_event_loop(Core::EventLoop::current())
+    , m_update_timer(move(update_timer))
+{
+    m_update_timer->on_timeout = [this]() {
+        update_timestamp();
+    };
+    m_update_timer->start();
+}
+
+void AudioCodecPluginAgnostic::resume_playback()
+{
+    m_paused = false;
+    m_output->resume()
+        ->when_resolved([this](Duration new_device_time) {
+            m_main_thread_event_loop.deferred_invoke([this, new_device_time]() {
+                m_last_resume_in_device_time = new_device_time;
+                m_update_timer->start();
+            });
+        })
+        .when_rejected([](Error&&) {
+            // FIXME: Propagate errors.
+        });
+}
+
+void AudioCodecPluginAgnostic::pause_playback()
+{
+    m_paused = true;
+    m_output->drain_buffer_and_suspend()
+        ->when_resolved([this]() -> ErrorOr<void> {
+            auto new_media_time = timestamp_from_samples(m_loader->loaded_samples(), m_loader->sample_rate());
+            auto new_device_time = TRY(m_output->total_time_played());
+            m_main_thread_event_loop.deferred_invoke([this, new_media_time, new_device_time]() {
+                m_last_resume_in_media_time = new_media_time;
+                m_last_resume_in_device_time = new_device_time;
+                m_update_timer->stop();
+                update_timestamp();
+            });
+            return {};
+        })
+        .when_rejected([](Error&&) {
+            // FIXME: Propagate errors.
+        });
+}
+
+void AudioCodecPluginAgnostic::set_volume(double volume)
+{
+    m_output->set_volume(volume)->when_rejected([](Error&&) {
+        // FIXME: Propagate errors.
+    });
+}
+
+void AudioCodecPluginAgnostic::seek(double position)
+{
+    m_output->discard_buffer_and_suspend()
+        ->when_resolved([this, position, was_paused = m_paused]() -> ErrorOr<void> {
+            auto sample_position = static_cast<i32>(position * m_loader->sample_rate());
+            auto seek_result = m_loader->seek(sample_position);
+            if (seek_result.is_error())
+                return Error::from_string_literal("Seeking in audio loader failed");
+
+            auto new_media_time = get_loader_timestamp(m_loader);
+            auto new_device_time = m_output->total_time_played().release_value_but_fixme_should_propagate_errors();
+
+            m_main_thread_event_loop.deferred_invoke([this, was_paused, new_device_time, new_media_time]() {
+                m_last_resume_in_device_time = new_device_time;
+                m_last_resume_in_media_time = new_media_time;
+                if (was_paused) {
+                    update_timestamp();
+                } else {
+                    m_output->resume()->when_rejected([](Error&&) {
+                        // FIXME: Propagate errors.
+                    });
+                }
+            });
+
+            return {};
+        })
+        .when_rejected([](Error&&) {
+            // FIXME: Propagate errors.
+        });
+}
+
+Duration AudioCodecPluginAgnostic::duration()
+{
+    return m_duration;
+}
+
+void AudioCodecPluginAgnostic::update_timestamp()
+{
+    auto current_device_time_result = m_output->total_time_played();
+    if (!current_device_time_result.is_error())
+        m_last_good_device_time = current_device_time_result.release_value();
+    auto current_device_time_delta = m_last_good_device_time - m_last_resume_in_device_time;
+
+    auto current_media_time = m_last_resume_in_media_time + current_device_time_delta;
+    current_media_time = min(current_media_time, m_duration);
+    on_playback_position_updated(current_media_time);
+}
+
+}

Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2023, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibAudio/PlaybackStream.h>
+#include <LibWeb/Platform/AudioCodecPlugin.h>
+
+namespace Web::Platform {
+
+class AudioCodecPluginAgnostic final : public AudioCodecPlugin {
+public:
+    static ErrorOr<NonnullOwnPtr<AudioCodecPluginAgnostic>> create(NonnullRefPtr<Audio::Loader> const&);
+
+    virtual void resume_playback() override;
+    virtual void pause_playback() override;
+    virtual void set_volume(double) override;
+    virtual void seek(double) override;
+
+    virtual Duration duration() override;
+
+private:
+    explicit AudioCodecPluginAgnostic(NonnullRefPtr<Audio::Loader> loader, Duration, NonnullRefPtr<Core::Timer> update_timer);
+
+    void update_timestamp();
+
+    NonnullRefPtr<Audio::Loader> m_loader;
+    RefPtr<Audio::PlaybackStream> m_output { nullptr };
+    Duration m_duration { Duration::zero() };
+    Duration m_last_resume_in_media_time { Duration::zero() };
+    Duration m_last_resume_in_device_time { Duration::zero() };
+    Duration m_last_good_device_time { Duration::zero() };
+    Core::EventLoop& m_main_thread_event_loop;
+    NonnullRefPtr<Core::Timer> m_update_timer;
+    bool m_paused { true };
+};
+
+}