From 1c4dd0caad3fbbd19eea21f877e7bf7086f98698 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Tue, 20 Jun 2023 12:43:04 -0400 Subject: [PATCH] Ladybird+LibWeb+WebConent: Drive audio in Ladybird off the main thread The main thread in the WebContent process is often busy with layout and running JavaScript. This can cause audio to sound jittery and crack. To avoid this behavior, we now drive audio on a secondary thread. Note: Browser on Serenity uses AudioServer, the connection for which is already handled on a secondary thread within LibAudio. So this only applies to Lagom. Rather than using LibThreading, our hands are tied to QThread for now. Internally, the Qt media objects use a QTimer, which is forbidden from running on a thread that is not a QThread (the debug console is spammed with messages pointing this out). Ideally, in the future AudioServer will be able to run for non-Serenity platforms, and most of this can be aligned with the Serenity implementation. --- Ladybird/AudioCodecPluginLadybird.cpp | 276 ++++++++++++++---- Ladybird/AudioCodecPluginLadybird.h | 32 +- Ladybird/WebContent/main.cpp | 5 +- Userland/Libraries/LibWeb/HTML/AudioTrack.cpp | 89 +----- Userland/Libraries/LibWeb/HTML/AudioTrack.h | 8 +- .../LibWeb/Platform/AudioCodecPlugin.cpp | 34 ++- .../LibWeb/Platform/AudioCodecPlugin.h | 21 +- .../WebContent/AudioCodecPluginSerenity.cpp | 88 ++++-- .../WebContent/AudioCodecPluginSerenity.h | 30 +- Userland/Services/WebContent/main.cpp | 5 +- 10 files changed, 383 insertions(+), 205 deletions(-) diff --git a/Ladybird/AudioCodecPluginLadybird.cpp b/Ladybird/AudioCodecPluginLadybird.cpp index 894ccd13f0..80b157e723 100644 --- a/Ladybird/AudioCodecPluginLadybird.cpp +++ b/Ladybird/AudioCodecPluginLadybird.cpp @@ -7,88 +7,258 @@ #include "AudioCodecPluginLadybird.h" #include #include +#include #include +#include #include #include -#include +#include #include +#include namespace Ladybird { -ErrorOr> AudioCodecPluginLadybird::create() -{ - auto devices = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QMediaDevices())); - auto const& device_info = devices->defaultAudioOutput(); +static constexpr u32 UPDATE_RATE_MS = 50; - auto format = device_info.preferredFormat(); - format.setSampleFormat(QAudioFormat::Int16); - format.setChannelCount(2); +struct AudioTask { + enum class Type { + Stop, + Play, + Pause, + Seek, + Volume, + }; - if (!device_info.isFormatSupported(format)) - return Error::from_string_literal("Audio device format not supported"); + Type type; + Optional data {}; +}; - auto audio_output = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QAudioSink(device_info, format))); +using AudioTaskQueue = Core::SharedSingleProducerCircularQueue; - return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(devices), move(audio_output))); -} +class AudioThread final : public QThread { // We have to use QThread, otherwise internal Qt media QTimer objects do not work. + Q_OBJECT -AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr devices, NonnullOwnPtr audio_output) - : m_devices(move(devices)) - , m_audio_output(move(audio_output)) - , m_io_device(m_audio_output->start()) -{ -} - -AudioCodecPluginLadybird::~AudioCodecPluginLadybird() = default; - -size_t AudioCodecPluginLadybird::device_sample_rate() -{ - return m_audio_output->format().sampleRate(); -} - -void AudioCodecPluginLadybird::enqueue_samples(FixedArray samples) -{ - QByteArray buffer; - buffer.resize(samples.size() * 2 * sizeof(u16)); - - FixedMemoryStream stream { Bytes { buffer.data(), static_cast(buffer.size()) } }; - - for (auto& sample : samples) { - LittleEndian pcm; - - pcm = static_cast(sample.left * NumericLimits::max()); - MUST(stream.write_value(pcm)); - - pcm = static_cast(sample.right * NumericLimits::max()); - MUST(stream.write_value(pcm)); +public: + static ErrorOr> create(NonnullRefPtr loader) + { + auto task_queue = TRY(AudioTaskQueue::create()); + return adopt_nonnull_own_or_enomem(new (nothrow) AudioThread(move(loader), move(task_queue))); } - m_io_device->write(buffer.data(), buffer.size()); + ErrorOr stop() + { + TRY(queue_task({ AudioTask::Type::Stop })); + wait(); + + return {}; + } + + Duration duration() const + { + return m_duration; + } + + ErrorOr queue_task(AudioTask task) + { + return m_task_queue.blocking_enqueue(move(task), []() { + usleep(UPDATE_RATE_MS * 1000); + }); + } + +Q_SIGNALS: + void playback_position_updated(Duration); + +private: + AudioThread(NonnullRefPtr loader, AudioTaskQueue task_queue) + : m_loader(move(loader)) + , m_task_queue(move(task_queue)) + { + auto duration = static_cast(m_loader->total_samples()) / static_cast(m_loader->sample_rate()); + m_duration = Duration::from_milliseconds(static_cast(duration * 1000.0)); + + m_samples_to_load_per_buffer = static_cast(UPDATE_RATE_MS / 1000.0 * static_cast(m_loader->sample_rate())); + } + + enum class Paused { + Yes, + No, + }; + + void run() override + { + auto devices = make(); + auto const& device_info = devices->defaultAudioOutput(); + + auto format = device_info.preferredFormat(); + format.setSampleFormat(QAudioFormat::Int16); + format.setChannelCount(2); + + auto audio_output = make(device_info, format); + auto* io_device = audio_output->start(); + + auto paused = Paused::Yes; + + while (true) { + if (auto result = m_task_queue.dequeue(); result.is_error()) { + VERIFY(result.error() == AudioTaskQueue::QueueStatus::Empty); + } else { + auto task = result.release_value(); + + switch (task.type) { + case AudioTask::Type::Stop: + return; + + case AudioTask::Type::Play: + audio_output->resume(); + paused = Paused::No; + break; + + case AudioTask::Type::Pause: + audio_output->suspend(); + paused = Paused::Yes; + break; + + case AudioTask::Type::Seek: { + VERIFY(task.data.has_value()); + auto position = *task.data; + + auto duration = static_cast(this->duration().to_milliseconds()) / 1000.0; + position = position / duration * static_cast(m_loader->total_samples()); + + m_loader->seek(static_cast(position)).release_value_but_fixme_should_propagate_errors(); + + if (paused == Paused::Yes) { + m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader, audio_output->format().sampleRate()); + Q_EMIT playback_position_updated(m_position); + } + + break; + } + + case AudioTask::Type::Volume: + VERIFY(task.data.has_value()); + audio_output->setVolume(*task.data); + break; + } + } + + if (paused == Paused::No) { + if (auto result = play_next_samples(*audio_output, *io_device); result.is_error()) { + // FIXME: Propagate the error to the HTMLMediaElement. + } else { + Q_EMIT playback_position_updated(m_position); + paused = result.value(); + } + } + + usleep(UPDATE_RATE_MS * 1000); + } + } + + ErrorOr play_next_samples(QAudioSink& audio_output, QIODevice& io_device) + { + bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples(); + + if (all_samples_loaded) { + audio_output.suspend(); + (void)m_loader->reset(); + + m_position = m_duration; + return Paused::Yes; + } + + auto samples = TRY(Web::Platform::AudioCodecPlugin::read_samples_from_loader(*m_loader, m_samples_to_load_per_buffer, audio_output.format().sampleRate())); + enqueue_samples(io_device, move(samples)); + + m_position = Web::Platform::AudioCodecPlugin::current_loader_position(m_loader, audio_output.format().sampleRate()); + return Paused::No; + } + + void enqueue_samples(QIODevice& io_device, FixedArray samples) + { + auto buffer_size = samples.size() * 2 * sizeof(u16); + if (buffer_size > static_cast(m_sample_buffer.size())) + m_sample_buffer.resize(buffer_size); + + FixedMemoryStream stream { Bytes { m_sample_buffer.data(), buffer_size } }; + + for (auto& sample : samples) { + LittleEndian pcm; + + pcm = static_cast(sample.left * NumericLimits::max()); + MUST(stream.write_value(pcm)); + + pcm = static_cast(sample.right * NumericLimits::max()); + MUST(stream.write_value(pcm)); + } + + io_device.write(m_sample_buffer.data(), buffer_size); + } + + NonnullRefPtr m_loader; + AudioTaskQueue m_task_queue; + + size_t m_samples_to_load_per_buffer { 0 }; + QByteArray m_sample_buffer; + + Duration m_duration; + Duration m_position; +}; + +ErrorOr> AudioCodecPluginLadybird::create(NonnullRefPtr loader) +{ + auto audio_thread = TRY(AudioThread::create(move(loader))); + audio_thread->start(); + + return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(audio_thread))); } -size_t AudioCodecPluginLadybird::remaining_samples() const +AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr audio_thread) + : m_audio_thread(move(audio_thread)) { - return 0; + connect(m_audio_thread, &AudioThread::playback_position_updated, this, [this](auto position) { + if (on_playback_position_updated) + on_playback_position_updated(position); + }); +} + +AudioCodecPluginLadybird::~AudioCodecPluginLadybird() +{ + m_audio_thread->stop().release_value_but_fixme_should_propagate_errors(); } void AudioCodecPluginLadybird::resume_playback() { - m_audio_output->resume(); + m_audio_thread->queue_task({ AudioTask::Type::Play }).release_value_but_fixme_should_propagate_errors(); } void AudioCodecPluginLadybird::pause_playback() { - m_audio_output->suspend(); -} - -void AudioCodecPluginLadybird::playback_ended() -{ - m_audio_output->suspend(); + m_audio_thread->queue_task({ AudioTask::Type::Pause }).release_value_but_fixme_should_propagate_errors(); } void AudioCodecPluginLadybird::set_volume(double volume) { - m_audio_output->setVolume(volume); + + AudioTask task { AudioTask::Type::Volume }; + task.data = volume; + + m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors(); +} + +void AudioCodecPluginLadybird::seek(double position) +{ + AudioTask task { AudioTask::Type::Seek }; + task.data = position; + + m_audio_thread->queue_task(move(task)).release_value_but_fixme_should_propagate_errors(); +} + +Duration AudioCodecPluginLadybird::duration() +{ + return m_audio_thread->duration(); } } + +#include "AudioCodecPluginLadybird.moc" diff --git a/Ladybird/AudioCodecPluginLadybird.h b/Ladybird/AudioCodecPluginLadybird.h index 5519e3fc1b..207f466be7 100644 --- a/Ladybird/AudioCodecPluginLadybird.h +++ b/Ladybird/AudioCodecPluginLadybird.h @@ -8,37 +8,35 @@ #include #include +#include #include #include - -class QAudioSink; -class QIODevice; -class QMediaDevices; +#include namespace Ladybird { -class AudioCodecPluginLadybird final : public Web::Platform::AudioCodecPlugin { +class AudioThread; + +class AudioCodecPluginLadybird final + : public QObject + , public Web::Platform::AudioCodecPlugin { + Q_OBJECT + public: - static ErrorOr> create(); + static ErrorOr> create(NonnullRefPtr); virtual ~AudioCodecPluginLadybird() override; - virtual size_t device_sample_rate() override; - - virtual void enqueue_samples(FixedArray) override; - virtual size_t remaining_samples() const override; - virtual void resume_playback() override; virtual void pause_playback() override; - virtual void playback_ended() override; - virtual void set_volume(double) override; + virtual void seek(double) override; + + virtual Duration duration() override; private: - AudioCodecPluginLadybird(NonnullOwnPtr, NonnullOwnPtr); + explicit AudioCodecPluginLadybird(NonnullOwnPtr); - NonnullOwnPtr m_devices; - NonnullOwnPtr m_audio_output; - QIODevice* m_io_device { nullptr }; + NonnullOwnPtr m_audio_thread; }; } diff --git a/Ladybird/WebContent/main.cpp b/Ladybird/WebContent/main.cpp index 33430b415f..cd0e088f70 100644 --- a/Ladybird/WebContent/main.cpp +++ b/Ladybird/WebContent/main.cpp @@ -13,6 +13,7 @@ #include "../WebSocketClientManagerLadybird.h" #include #include +#include #include #include #include @@ -59,8 +60,8 @@ ErrorOr serenity_main(Main::Arguments arguments) Web::Platform::EventLoopPlugin::install(*new Web::Platform::EventLoopPluginSerenity); Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPluginLadybird); - Web::Platform::AudioCodecPlugin::install_creation_hook([] { - return Ladybird::AudioCodecPluginLadybird::create(); + Web::Platform::AudioCodecPlugin::install_creation_hook([](auto loader) { + return Ladybird::AudioCodecPluginLadybird::create(move(loader)); }); Web::ResourceLoader::initialize(RequestManagerQt::create()); diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrack.cpp b/Userland/Libraries/LibWeb/HTML/AudioTrack.cpp index 75a0360941..4c48a96e9e 100644 --- a/Userland/Libraries/LibWeb/HTML/AudioTrack.cpp +++ b/Userland/Libraries/LibWeb/HTML/AudioTrack.cpp @@ -5,10 +5,7 @@ */ #include -#include #include -#include -#include #include #include #include @@ -20,25 +17,23 @@ #include #include #include -#include namespace Web::HTML { static IDAllocator s_audio_track_id_allocator; -// Number of milliseconds of audio data contained in each audio buffer -static constexpr u32 BUFFER_SIZE_MS = 50; - AudioTrack::AudioTrack(JS::Realm& realm, JS::NonnullGCPtr media_element, NonnullRefPtr loader) : PlatformObject(realm) , m_media_element(media_element) - , m_audio_plugin(Platform::AudioCodecPlugin::create().release_value_but_fixme_should_propagate_errors()) - , m_loader(move(loader)) - , m_sample_timer(Platform::Timer::create_repeating(BUFFER_SIZE_MS, [this]() { - play_next_samples(); - })) + , m_audio_plugin(Platform::AudioCodecPlugin::create(move(loader)).release_value_but_fixme_should_propagate_errors()) { - m_audio_plugin->device_sample_rate(); + m_audio_plugin->on_playback_position_updated = [this](auto position) { + if (auto* layout_node = m_media_element->layout_node()) + layout_node->set_needs_display(); + + auto playback_position = static_cast(position.to_milliseconds()) / 1000.0; + m_media_element->set_current_playback_position(playback_position); + }; } AudioTrack::~AudioTrack() @@ -63,30 +58,16 @@ JS::ThrowCompletionOr AudioTrack::initialize(JS::Realm& realm) void AudioTrack::play(Badge) { m_audio_plugin->resume_playback(); - m_sample_timer->start(); } void AudioTrack::pause(Badge) { m_audio_plugin->pause_playback(); - m_sample_timer->stop(); } -Duration AudioTrack::position() const +Duration AudioTrack::duration() { - auto samples_played = static_cast(m_loader->loaded_samples()); - auto sample_rate = static_cast(m_loader->sample_rate()); - - auto source_to_device_ratio = sample_rate / static_cast(m_audio_plugin->device_sample_rate()); - samples_played *= source_to_device_ratio; - - return Duration::from_milliseconds(static_cast(samples_played / sample_rate * 1000.0)); -} - -Duration AudioTrack::duration() const -{ - auto duration = static_cast(m_loader->total_samples()) / static_cast(m_loader->sample_rate()); - return Duration::from_milliseconds(static_cast(duration * 1000.0)); + return m_audio_plugin->duration(); } void AudioTrack::seek(double position, MediaSeekMode seek_mode) @@ -94,11 +75,7 @@ void AudioTrack::seek(double position, MediaSeekMode seek_mode) // FIXME: Implement seeking mode. (void)seek_mode; - auto duration = static_cast(this->duration().to_milliseconds()) / 1000.0; - position = position / duration * static_cast(m_loader->total_samples()); - - m_loader->seek(position).release_value_but_fixme_should_propagate_errors(); - m_media_element->set_current_playback_position(this->position().to_milliseconds() / 1000.0); + m_audio_plugin->seek(position); } void AudioTrack::update_volume() @@ -134,48 +111,4 @@ void AudioTrack::set_enabled(bool enabled) m_enabled = enabled; } -Optional> AudioTrack::get_next_samples() -{ - bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples(); - bool audio_server_done = m_audio_plugin->remaining_samples() == 0; - - if (all_samples_loaded && audio_server_done) - return {}; - - auto samples_to_load_per_buffer = static_cast(BUFFER_SIZE_MS / 1000.0f * static_cast(m_loader->sample_rate())); - - auto buffer_or_error = m_loader->get_more_samples(samples_to_load_per_buffer); - if (buffer_or_error.is_error()) { - dbgln("Error while loading samples: {}", buffer_or_error.error().description); - return {}; - } - - return buffer_or_error.release_value(); -} - -void AudioTrack::play_next_samples() -{ - if (auto* layout_node = m_media_element->layout_node()) - layout_node->set_needs_display(); - - auto samples = get_next_samples(); - if (!samples.has_value()) { - m_audio_plugin->playback_ended(); - (void)m_loader->reset(); - - auto playback_position = static_cast(duration().to_milliseconds()) / 1000.0; - m_media_element->set_current_playback_position(playback_position); - - return; - } - - Audio::ResampleHelper resampler(m_loader->sample_rate(), m_audio_plugin->device_sample_rate()); - auto resampled = FixedArray::create(resampler.resample(samples.release_value()).span()).release_value_but_fixme_should_propagate_errors(); - - m_audio_plugin->enqueue_samples(move(resampled)); - - auto playback_position = static_cast(position().to_milliseconds()) / 1000.0; - m_media_element->set_current_playback_position(playback_position); -} - } diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrack.h b/Userland/Libraries/LibWeb/HTML/AudioTrack.h index 424e52cbe9..3de202f28f 100644 --- a/Userland/Libraries/LibWeb/HTML/AudioTrack.h +++ b/Userland/Libraries/LibWeb/HTML/AudioTrack.h @@ -24,8 +24,7 @@ public: void play(Badge); void pause(Badge); - Duration position() const; - Duration duration() const; + Duration duration(); void seek(double, MediaSeekMode); void update_volume(); @@ -44,9 +43,6 @@ private: virtual JS::ThrowCompletionOr initialize(JS::Realm&) override; virtual void visit_edges(Cell::Visitor&) override; - Optional> get_next_samples(); - void play_next_samples(); - // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-id String m_id; @@ -66,8 +62,6 @@ private: JS::GCPtr m_audio_track_list; NonnullOwnPtr m_audio_plugin; - NonnullRefPtr m_loader; - NonnullRefPtr m_sample_timer; }; } diff --git a/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp index bd5e7d6d56..8b8e19a681 100644 --- a/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp +++ b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp @@ -4,25 +4,51 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include +#include +#include #include namespace Web::Platform { -static Function>()> s_creation_hook; +static AudioCodecPlugin::AudioCodecPluginCreator s_creation_hook; AudioCodecPlugin::AudioCodecPlugin() = default; AudioCodecPlugin::~AudioCodecPlugin() = default; -void AudioCodecPlugin::install_creation_hook(Function>()> creation_hook) +void AudioCodecPlugin::install_creation_hook(AudioCodecPluginCreator creation_hook) { VERIFY(!s_creation_hook); s_creation_hook = move(creation_hook); } -ErrorOr> AudioCodecPlugin::create() +ErrorOr> AudioCodecPlugin::create(NonnullRefPtr loader) { VERIFY(s_creation_hook); - return s_creation_hook(); + return s_creation_hook(move(loader)); +} + +ErrorOr> AudioCodecPlugin::read_samples_from_loader(Audio::Loader& loader, size_t samples_to_load, size_t device_sample_rate) +{ + auto buffer_or_error = loader.get_more_samples(samples_to_load); + if (buffer_or_error.is_error()) { + dbgln("Error while loading samples: {}", buffer_or_error.error().description); + return Error::from_string_literal("Error while loading samples"); + } + + Audio::ResampleHelper resampler(loader.sample_rate(), device_sample_rate); + return FixedArray::create(resampler.resample(buffer_or_error.release_value()).span()); +} + +Duration AudioCodecPlugin::current_loader_position(Audio::Loader const& loader, size_t device_sample_rate) +{ + auto samples_played = static_cast(loader.loaded_samples()); + auto sample_rate = static_cast(loader.sample_rate()); + + auto source_to_device_ratio = sample_rate / static_cast(device_sample_rate); + samples_played *= source_to_device_ratio; + + return Duration::from_milliseconds(static_cast(samples_played / sample_rate * 1000.0)); } } diff --git a/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h index 8626eee61c..1b0efb1fd3 100644 --- a/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h +++ b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h @@ -9,28 +9,31 @@ #include #include #include -#include +#include #include namespace Web::Platform { class AudioCodecPlugin { public: - static void install_creation_hook(Function>()>); - static ErrorOr> create(); + using AudioCodecPluginCreator = Function>(NonnullRefPtr)>; + + static void install_creation_hook(AudioCodecPluginCreator); + static ErrorOr> create(NonnullRefPtr); virtual ~AudioCodecPlugin(); - virtual size_t device_sample_rate() = 0; - - virtual void enqueue_samples(FixedArray) = 0; - virtual size_t remaining_samples() const = 0; + static ErrorOr> read_samples_from_loader(Audio::Loader&, size_t samples_to_load, size_t device_sample_rate); + static Duration current_loader_position(Audio::Loader const&, size_t device_sample_rate); virtual void resume_playback() = 0; virtual void pause_playback() = 0; - virtual void playback_ended() = 0; - virtual void set_volume(double) = 0; + virtual void seek(double) = 0; + + virtual Duration duration() = 0; + + Function on_playback_position_updated; protected: AudioCodecPlugin(); diff --git a/Userland/Services/WebContent/AudioCodecPluginSerenity.cpp b/Userland/Services/WebContent/AudioCodecPluginSerenity.cpp index 5c6d31b951..9934a355de 100644 --- a/Userland/Services/WebContent/AudioCodecPluginSerenity.cpp +++ b/Userland/Services/WebContent/AudioCodecPluginSerenity.cpp @@ -5,55 +5,82 @@ */ #include +#include +#include +#include #include namespace WebContent { -ErrorOr> AudioCodecPluginSerenity::create() +// These constants and this implementation is based heavily on SoundPlayer::PlaybackManager. +static constexpr u32 UPDATE_RATE_MS = 50; +static constexpr u32 BUFFER_SIZE_MS = 100; +static constexpr size_t ALWAYS_ENQUEUED_BUFFER_COUNT = 5; + +ErrorOr> AudioCodecPluginSerenity::create(NonnullRefPtr loader) { auto connection = TRY(Audio::ConnectionToServer::try_create()); - return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection))); + return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection), move(loader))); } -AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr connection) +AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr connection, NonnullRefPtr loader) : m_connection(move(connection)) + , m_loader(move(loader)) + , m_sample_timer(Web::Platform::Timer::create_repeating(UPDATE_RATE_MS, [this]() { + if (play_next_samples().is_error()) { + // FIXME: Propagate the error to the HTMLMediaElement. + } else { + if (on_playback_position_updated) + on_playback_position_updated(m_position); + } + })) { + auto duration = static_cast(m_loader->total_samples()) / static_cast(m_loader->sample_rate()); + m_duration = Duration::from_milliseconds(static_cast(duration * 1000.0)); + + m_device_sample_rate = m_connection->get_sample_rate(); + m_device_samples_per_buffer = static_cast(BUFFER_SIZE_MS / 1000.0 * static_cast(m_device_sample_rate)); + m_samples_to_load_per_buffer = static_cast(BUFFER_SIZE_MS / 1000.0 * static_cast(m_loader->sample_rate())); } AudioCodecPluginSerenity::~AudioCodecPluginSerenity() = default; -size_t AudioCodecPluginSerenity::device_sample_rate() +ErrorOr AudioCodecPluginSerenity::play_next_samples() { - if (!m_device_sample_rate.has_value()) - m_device_sample_rate = m_connection->get_sample_rate(); - return *m_device_sample_rate; -} + while (m_connection->remaining_samples() < m_device_samples_per_buffer * ALWAYS_ENQUEUED_BUFFER_COUNT) { + bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples(); + bool audio_server_done = m_connection->remaining_samples() == 0; -void AudioCodecPluginSerenity::enqueue_samples(FixedArray samples) -{ - m_connection->async_enqueue(move(samples)).release_value_but_fixme_should_propagate_errors(); -} + if (all_samples_loaded && audio_server_done) { + pause_playback(); -size_t AudioCodecPluginSerenity::remaining_samples() const -{ - return m_connection->remaining_samples(); + m_connection->clear_client_buffer(); + m_connection->async_clear_buffer(); + (void)m_loader->reset(); + + m_position = m_duration; + break; + } + + auto samples = TRY(read_samples_from_loader(m_loader, m_samples_to_load_per_buffer, m_device_sample_rate)); + TRY(m_connection->async_enqueue(move(samples))); + + m_position = current_loader_position(m_loader, m_device_sample_rate); + } + + return {}; } void AudioCodecPluginSerenity::resume_playback() { m_connection->async_start_playback(); + m_sample_timer->start(); } void AudioCodecPluginSerenity::pause_playback() -{ - m_connection->async_start_playback(); -} - -void AudioCodecPluginSerenity::playback_ended() { m_connection->async_pause_playback(); - m_connection->clear_client_buffer(); - m_connection->async_clear_buffer(); + m_sample_timer->stop(); } void AudioCodecPluginSerenity::set_volume(double volume) @@ -61,4 +88,21 @@ void AudioCodecPluginSerenity::set_volume(double volume) m_connection->async_set_self_volume(volume); } +void AudioCodecPluginSerenity::seek(double position) +{ + auto duration = static_cast(this->duration().to_milliseconds()) / 1000.0; + position = position / duration * static_cast(m_loader->total_samples()); + + m_loader->seek(static_cast(position)).release_value_but_fixme_should_propagate_errors(); + m_position = current_loader_position(m_loader, m_device_sample_rate); + + if (on_playback_position_updated) + on_playback_position_updated(m_position); +} + +Duration AudioCodecPluginSerenity::duration() +{ + return m_duration; +} + } diff --git a/Userland/Services/WebContent/AudioCodecPluginSerenity.h b/Userland/Services/WebContent/AudioCodecPluginSerenity.h index b188d6a613..5e6fb1aafb 100644 --- a/Userland/Services/WebContent/AudioCodecPluginSerenity.h +++ b/Userland/Services/WebContent/AudioCodecPluginSerenity.h @@ -7,34 +7,42 @@ #pragma once #include +#include #include -#include +#include #include +#include #include namespace WebContent { class AudioCodecPluginSerenity final : public Web::Platform::AudioCodecPlugin { public: - static ErrorOr> create(); + static ErrorOr> create(NonnullRefPtr); virtual ~AudioCodecPluginSerenity() override; - virtual size_t device_sample_rate() override; - - virtual void enqueue_samples(FixedArray) override; - virtual size_t remaining_samples() const override; - virtual void resume_playback() override; virtual void pause_playback() override; - virtual void playback_ended() override; - virtual void set_volume(double) override; + virtual void seek(double) override; + + virtual Duration duration() override; private: - explicit AudioCodecPluginSerenity(NonnullRefPtr); + AudioCodecPluginSerenity(NonnullRefPtr, NonnullRefPtr); + + ErrorOr play_next_samples(); NonnullRefPtr m_connection; - Optional m_device_sample_rate; + NonnullRefPtr m_loader; + NonnullRefPtr m_sample_timer; + + Duration m_duration; + Duration m_position; + + size_t m_device_sample_rate { 0 }; + size_t m_device_samples_per_buffer { 0 }; + size_t m_samples_to_load_per_buffer { 0 }; }; } diff --git a/Userland/Services/WebContent/main.cpp b/Userland/Services/WebContent/main.cpp index 93a9f09558..91a7b1d5c2 100644 --- a/Userland/Services/WebContent/main.cpp +++ b/Userland/Services/WebContent/main.cpp @@ -6,6 +6,7 @@ #include "AudioCodecPluginSerenity.h" #include "ImageCodecPluginSerenity.h" +#include #include #include #include @@ -46,8 +47,8 @@ ErrorOr serenity_main(Main::Arguments) Web::Platform::ImageCodecPlugin::install(*new WebContent::ImageCodecPluginSerenity); Web::Platform::FontPlugin::install(*new Web::Platform::FontPluginSerenity); - Web::Platform::AudioCodecPlugin::install_creation_hook([] { - return WebContent::AudioCodecPluginSerenity::create(); + Web::Platform::AudioCodecPlugin::install_creation_hook([](auto loader) { + return WebContent::AudioCodecPluginSerenity::create(move(loader)); }); Web::WebSockets::WebSocketClientManager::initialize(TRY(WebView::WebSocketClientManagerAdapter::try_create()));