From 2a4f2110b9d46ca55337cd1745c1382a1e00d149 Mon Sep 17 00:00:00 2001 From: Andrew Kaster Date: Mon, 26 Jun 2023 19:14:40 -0600 Subject: [PATCH] Ladybird: Move the AudioThread class into its own file This implementation detail of audio support in ladybird is a QObject that needs moc'd by the moc tools. Putting it in its own file follows the pattern we have for all the other QObjects in Ladybird. --- Ladybird/AudioCodecPluginLadybird.cpp | 278 +------------------------- Ladybird/AudioThread.cpp | 212 ++++++++++++++++++++ Ladybird/AudioThread.h | 103 ++++++++++ Ladybird/WebContent/CMakeLists.txt | 1 + 4 files changed, 317 insertions(+), 277 deletions(-) create mode 100644 Ladybird/AudioThread.cpp create mode 100644 Ladybird/AudioThread.h diff --git a/Ladybird/AudioCodecPluginLadybird.cpp b/Ladybird/AudioCodecPluginLadybird.cpp index cba16fdadc..e10a7ea418 100644 --- a/Ladybird/AudioCodecPluginLadybird.cpp +++ b/Ladybird/AudioCodecPluginLadybird.cpp @@ -5,285 +5,11 @@ */ #include "AudioCodecPluginLadybird.h" -#include -#include +#include "AudioThread.h" #include -#include -#include -#include -#include -#include -#include -#include namespace Ladybird { -static constexpr u32 UPDATE_RATE_MS = 10; - -struct AudioTask { - enum class Type { - Stop, - Play, - Pause, - Seek, - Volume, - RecreateAudioDevice, - }; - - Type type; - Optional data {}; -}; - -using AudioTaskQueue = Core::SharedSingleProducerCircularQueue; - -class AudioThread final : public QThread { // We have to use QThread, otherwise internal Qt media QTimer objects do not work. - Q_OBJECT - -public: - static ErrorOr> create(NonnullRefPtr loader) - { - auto task_queue = TRY(AudioTaskQueue::create()); - return adopt_nonnull_own_or_enomem(new (nothrow) AudioThread(move(loader), move(task_queue))); - } - - ErrorOr stop() - { - TRY(queue_task({ AudioTask::Type::Stop })); - wait(); - - return {}; - } - - Duration duration() const - { - return m_duration; - } - - ErrorOr queue_task(AudioTask task) - { - return m_task_queue.blocking_enqueue(move(task), []() { - usleep(UPDATE_RATE_MS * 1000); - }); - } - -Q_SIGNALS: - void playback_position_updated(Duration); - -private: - AudioThread(NonnullRefPtr loader, AudioTaskQueue task_queue) - : m_loader(move(loader)) - , m_task_queue(move(task_queue)) - { - auto duration = static_cast(m_loader->total_samples()) / static_cast(m_loader->sample_rate()); - m_duration = Duration::from_milliseconds(static_cast(duration * 1000.0)); - } - - enum class Paused { - Yes, - No, - }; - - struct AudioDevice { - static AudioDevice create(Audio::Loader const& loader) - { - auto const& device_info = QMediaDevices::defaultAudioOutput(); - - auto format = device_info.preferredFormat(); - format.setSampleRate(static_cast(loader.sample_rate())); - format.setChannelCount(2); - - auto audio_output = make(device_info, format); - return AudioDevice { move(audio_output) }; - } - - AudioDevice(AudioDevice&&) = default; - - AudioDevice& operator=(AudioDevice&& device) - { - if (audio_output) { - audio_output->stop(); - io_device = nullptr; - } - - swap(audio_output, device.audio_output); - swap(io_device, device.io_device); - return *this; - } - - ~AudioDevice() - { - if (audio_output) - audio_output->stop(); - } - - OwnPtr audio_output; - QIODevice* io_device { nullptr }; - - private: - explicit AudioDevice(NonnullOwnPtr output) - : audio_output(move(output)) - { - io_device = audio_output->start(); - } - }; - - void run() override - { - auto devices = make(); - auto audio_device = AudioDevice::create(m_loader); - - connect(devices, &QMediaDevices::audioOutputsChanged, this, [this]() { - queue_task({ AudioTask::Type::RecreateAudioDevice }).release_value_but_fixme_should_propagate_errors(); - }); - - auto paused = Paused::Yes; - - while (true) { - auto& audio_output = audio_device.audio_output; - auto* io_device = audio_device.io_device; - - if (auto result = m_task_queue.dequeue(); result.is_error()) { - VERIFY(result.error() == AudioTaskQueue::QueueStatus::Empty); - } else { - auto task = result.release_value(); - - switch (task.type) { - case AudioTask::Type::Stop: - return; - - case AudioTask::Type::Play: - audio_output->resume(); - paused = Paused::No; - break; - - case AudioTask::Type::Pause: - audio_output->suspend(); - paused = Paused::Yes; - break; - - case AudioTask::Type::Seek: - VERIFY(task.data.has_value()); - m_position = Web::Platform::AudioCodecPlugin::set_loader_position(m_loader, *task.data, m_duration); - - if (paused == Paused::Yes) - Q_EMIT playback_position_updated(m_position); - - break; - - case AudioTask::Type::Volume: - VERIFY(task.data.has_value()); - audio_output->setVolume(*task.data); - break; - - case AudioTask::Type::RecreateAudioDevice: - audio_device = AudioDevice::create(m_loader); - continue; - } - } - - if (paused == Paused::No) { - if (auto result = play_next_samples(*audio_output, *io_device); result.is_error()) { - // FIXME: Propagate the error to the HTMLMediaElement. - } else { - Q_EMIT playback_position_updated(m_position); - paused = result.value(); - } - } - - usleep(UPDATE_RATE_MS * 1000); - } - } - - ErrorOr play_next_samples(QAudioSink& audio_output, QIODevice& io_device) - { - bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples(); - - if (all_samples_loaded) { - audio_output.suspend(); - (void)m_loader->reset(); - - m_position = m_duration; - return Paused::Yes; - } - - auto bytes_available = audio_output.bytesFree(); - auto bytes_per_sample = audio_output.format().bytesPerSample(); - auto channel_count = audio_output.format().channelCount(); - auto samples_to_load = bytes_available / bytes_per_sample / channel_count; - - auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, samples_to_load)); - enqueue_samples(audio_output, io_device, move(samples)); - - m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader); - return Paused::No; - } - - void enqueue_samples(QAudioSink const& audio_output, QIODevice& io_device, FixedArray samples) - { - auto buffer_size = samples.size() * audio_output.format().bytesPerSample() * audio_output.format().channelCount(); - - if (buffer_size > static_cast(m_sample_buffer.size())) - m_sample_buffer.resize(buffer_size); - - FixedMemoryStream stream { Bytes { m_sample_buffer.data(), buffer_size } }; - - for (auto const& sample : samples) { - switch (audio_output.format().sampleFormat()) { - case QAudioFormat::UInt8: - write_sample(stream, sample.left); - write_sample(stream, sample.right); - break; - case QAudioFormat::Int16: - write_sample(stream, sample.left); - write_sample(stream, sample.right); - break; - case QAudioFormat::Int32: - write_sample(stream, sample.left); - write_sample(stream, sample.right); - break; - case QAudioFormat::Float: - write_sample(stream, sample.left); - write_sample(stream, sample.right); - break; - default: - VERIFY_NOT_REACHED(); - } - } - - io_device.write(m_sample_buffer.data(), buffer_size); - } - - template - void write_sample(FixedMemoryStream& stream, float sample) - { - // The values that need to be written to the stream vary depending on the output channel format, and isn't - // particularly well documented. The value derivations performed below were adapted from a Qt example: - // https://code.qt.io/cgit/qt/qtmultimedia.git/tree/examples/multimedia/audiooutput/audiooutput.cpp?h=6.4.2#n46 - LittleEndian pcm; - - if constexpr (IsSame) - pcm = static_cast((sample + 1.0f) / 2 * NumericLimits::max()); - else if constexpr (IsSame) - pcm = static_cast(sample * NumericLimits::max()); - else if constexpr (IsSame) - pcm = static_cast(sample * NumericLimits::max()); - else if constexpr (IsSame) - pcm = sample; - else - static_assert(DependentFalse); - - MUST(stream.write_value(pcm)); - } - - NonnullRefPtr m_loader; - AudioTaskQueue m_task_queue; - - QByteArray m_sample_buffer; - - Duration m_duration; - Duration m_position; -}; - ErrorOr> AudioCodecPluginLadybird::create(NonnullRefPtr loader) { auto audio_thread = TRY(AudioThread::create(move(loader))); @@ -339,5 +65,3 @@ Duration AudioCodecPluginLadybird::duration() } } - -#include "AudioCodecPluginLadybird.moc" diff --git a/Ladybird/AudioThread.cpp b/Ladybird/AudioThread.cpp new file mode 100644 index 0000000000..5fb321d9ad --- /dev/null +++ b/Ladybird/AudioThread.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2023, Tim Flynn + * Copyright (c) 2023, Andrew Kaster + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "AudioThread.h" +#include + +namespace Ladybird { + +struct AudioDevice { + static AudioDevice create(Audio::Loader const& loader) + { + auto const& device_info = QMediaDevices::defaultAudioOutput(); + + auto format = device_info.preferredFormat(); + format.setSampleRate(static_cast(loader.sample_rate())); + format.setChannelCount(2); + + auto audio_output = make(device_info, format); + return AudioDevice { move(audio_output) }; + } + + AudioDevice(AudioDevice&&) = default; + + AudioDevice& operator=(AudioDevice&& device) + { + if (audio_output) { + audio_output->stop(); + io_device = nullptr; + } + + swap(audio_output, device.audio_output); + swap(io_device, device.io_device); + return *this; + } + + ~AudioDevice() + { + if (audio_output) + audio_output->stop(); + } + + OwnPtr audio_output; + QIODevice* io_device { nullptr }; + +private: + explicit AudioDevice(NonnullOwnPtr output) + : audio_output(move(output)) + { + io_device = audio_output->start(); + } +}; + +ErrorOr> AudioThread::create(NonnullRefPtr loader) +{ + auto task_queue = TRY(AudioTaskQueue::create()); + return adopt_nonnull_own_or_enomem(new (nothrow) AudioThread(move(loader), move(task_queue))); +} + +ErrorOr AudioThread::stop() +{ + TRY(queue_task({ AudioTask::Type::Stop })); + wait(); + + return {}; +} + +ErrorOr AudioThread::queue_task(AudioTask task) +{ + return m_task_queue.blocking_enqueue(move(task), []() { + usleep(UPDATE_RATE_MS * 1000); + }); +} + +AudioThread::AudioThread(NonnullRefPtr loader, AudioTaskQueue task_queue) + : m_loader(move(loader)) + , m_task_queue(move(task_queue)) +{ + auto duration = static_cast(m_loader->total_samples()) / static_cast(m_loader->sample_rate()); + m_duration = Duration::from_milliseconds(static_cast(duration * 1000.0)); +} + +void AudioThread::run() +{ + auto devices = make(); + auto audio_device = AudioDevice::create(m_loader); + + connect(devices, &QMediaDevices::audioOutputsChanged, this, [this]() { + queue_task({ AudioTask::Type::RecreateAudioDevice }).release_value_but_fixme_should_propagate_errors(); + }); + + auto paused = Paused::Yes; + + while (true) { + auto& audio_output = audio_device.audio_output; + auto* io_device = audio_device.io_device; + + if (auto result = m_task_queue.dequeue(); result.is_error()) { + VERIFY(result.error() == AudioTaskQueue::QueueStatus::Empty); + } else { + auto task = result.release_value(); + + switch (task.type) { + case AudioTask::Type::Stop: + return; + + case AudioTask::Type::Play: + audio_output->resume(); + paused = Paused::No; + break; + + case AudioTask::Type::Pause: + audio_output->suspend(); + paused = Paused::Yes; + break; + + case AudioTask::Type::Seek: + VERIFY(task.data.has_value()); + m_position = Web::Platform::AudioCodecPlugin::set_loader_position(m_loader, *task.data, m_duration); + + if (paused == Paused::Yes) + Q_EMIT playback_position_updated(m_position); + + break; + + case AudioTask::Type::Volume: + VERIFY(task.data.has_value()); + audio_output->setVolume(*task.data); + break; + + case AudioTask::Type::RecreateAudioDevice: + audio_device = AudioDevice::create(m_loader); + continue; + } + } + + if (paused == Paused::No) { + if (auto result = play_next_samples(*audio_output, *io_device); result.is_error()) { + // FIXME: Propagate the error to the HTMLMediaElement. + } else { + Q_EMIT playback_position_updated(m_position); + paused = result.value(); + } + } + + usleep(UPDATE_RATE_MS * 1000); + } +} + +ErrorOr AudioThread::play_next_samples(QAudioSink& audio_output, QIODevice& io_device) +{ + bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples(); + + if (all_samples_loaded) { + audio_output.suspend(); + (void)m_loader->reset(); + + m_position = m_duration; + return Paused::Yes; + } + + auto bytes_available = audio_output.bytesFree(); + auto bytes_per_sample = audio_output.format().bytesPerSample(); + auto channel_count = audio_output.format().channelCount(); + auto samples_to_load = bytes_available / bytes_per_sample / channel_count; + + auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, samples_to_load)); + enqueue_samples(audio_output, io_device, move(samples)); + + m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader); + return Paused::No; +} + +void AudioThread::enqueue_samples(QAudioSink const& audio_output, QIODevice& io_device, FixedArray samples) +{ + auto buffer_size = samples.size() * audio_output.format().bytesPerSample() * audio_output.format().channelCount(); + + if (buffer_size > static_cast(m_sample_buffer.size())) + m_sample_buffer.resize(buffer_size); + + FixedMemoryStream stream { Bytes { m_sample_buffer.data(), buffer_size } }; + + for (auto const& sample : samples) { + switch (audio_output.format().sampleFormat()) { + case QAudioFormat::UInt8: + write_sample(stream, sample.left); + write_sample(stream, sample.right); + break; + case QAudioFormat::Int16: + write_sample(stream, sample.left); + write_sample(stream, sample.right); + break; + case QAudioFormat::Int32: + write_sample(stream, sample.left); + write_sample(stream, sample.right); + break; + case QAudioFormat::Float: + write_sample(stream, sample.left); + write_sample(stream, sample.right); + break; + default: + VERIFY_NOT_REACHED(); + } + } + + io_device.write(m_sample_buffer.data(), buffer_size); +} + +} diff --git a/Ladybird/AudioThread.h b/Ladybird/AudioThread.h new file mode 100644 index 0000000000..a6da5057c8 --- /dev/null +++ b/Ladybird/AudioThread.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023, Tim Flynn + * Copyright (c) 2023, Andrew Kaster + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +static constexpr u32 UPDATE_RATE_MS = 10; + +struct AudioTask { + enum class Type { + Stop, + Play, + Pause, + Seek, + Volume, + RecreateAudioDevice, + }; + + Type type; + Optional data {}; +}; + +using AudioTaskQueue = Core::SharedSingleProducerCircularQueue; + +class AudioThread final : public QThread { // We have to use QThread, otherwise internal Qt media QTimer objects do not work. + Q_OBJECT + +public: + static ErrorOr> create(NonnullRefPtr loader); + + ErrorOr stop(); + + Duration duration() const { return m_duration; } + + ErrorOr queue_task(AudioTask task); + +Q_SIGNALS: + void playback_position_updated(Duration); + +private: + AudioThread(NonnullRefPtr loader, AudioTaskQueue task_queue); + + enum class Paused { + Yes, + No, + }; + + void run() override; + + ErrorOr play_next_samples(QAudioSink& audio_output, QIODevice& io_device); + + void enqueue_samples(QAudioSink const& audio_output, QIODevice& io_device, FixedArray samples); + + template + void write_sample(FixedMemoryStream& stream, float sample) + { + // The values that need to be written to the stream vary depending on the output channel format, and isn't + // particularly well documented. The value derivations performed below were adapted from a Qt example: + // https://code.qt.io/cgit/qt/qtmultimedia.git/tree/examples/multimedia/audiooutput/audiooutput.cpp?h=6.4.2#n46 + LittleEndian pcm; + + if constexpr (IsSame) + pcm = static_cast((sample + 1.0f) / 2 * NumericLimits::max()); + else if constexpr (IsSame) + pcm = static_cast(sample * NumericLimits::max()); + else if constexpr (IsSame) + pcm = static_cast(sample * NumericLimits::max()); + else if constexpr (IsSame) + pcm = sample; + else + static_assert(DependentFalse); + + MUST(stream.write_value(pcm)); + } + + NonnullRefPtr m_loader; + AudioTaskQueue m_task_queue; + + QByteArray m_sample_buffer; + + Duration m_duration; + Duration m_position; +}; + +} diff --git a/Ladybird/WebContent/CMakeLists.txt b/Ladybird/WebContent/CMakeLists.txt index 88e5315f43..93204007dc 100644 --- a/Ladybird/WebContent/CMakeLists.txt +++ b/Ladybird/WebContent/CMakeLists.txt @@ -7,6 +7,7 @@ set(WEBCONTENT_SOURCES ${WEBCONTENT_SOURCE_DIR}/WebContentConsoleClient.cpp ${WEBCONTENT_SOURCE_DIR}/WebDriverConnection.cpp ../AudioCodecPluginLadybird.cpp + ../AudioThread.cpp ../EventLoopImplementationQt.cpp ../FontPluginQt.cpp ../ImageCodecPluginLadybird.cpp