Ladybird+LibWeb+WebConent: Drive audio in Ladybird off the main thread

The main thread in the WebContent process is often busy with layout and
running JavaScript. This can cause audio to sound jittery and crack. To
avoid this behavior, we now drive audio on a secondary thread.

Note: Browser on Serenity uses AudioServer, the connection for which is
already handled on a secondary thread within LibAudio. So this only
applies to Lagom.

Rather than using LibThreading, our hands are tied to QThread for now.
Internally, the Qt media objects use a QTimer, which is forbidden from
running on a thread that is not a QThread (the debug console is spammed
with messages pointing this out). Ideally, in the future AudioServer
will be able to run for non-Serenity platforms, and most of this can be
aligned with the Serenity implementation.

Userland/Services/WebContent/AudioCodecPluginSerenity.cpp

@@ -5,55 +5,82 @@
  */
 
 #include <LibAudio/ConnectionToServer.h>
+#include <LibAudio/Loader.h>
+#include <LibAudio/Sample.h>
+#include <LibWeb/Platform/Timer.h>
 #include <WebContent/AudioCodecPluginSerenity.h>
 
 namespace WebContent {
 
-ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> AudioCodecPluginSerenity::create()
+// These constants and this implementation is based heavily on SoundPlayer::PlaybackManager.
+static constexpr u32 UPDATE_RATE_MS = 50;
+static constexpr u32 BUFFER_SIZE_MS = 100;
+static constexpr size_t ALWAYS_ENQUEUED_BUFFER_COUNT = 5;
+
+ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> AudioCodecPluginSerenity::create(NonnullRefPtr<Audio::Loader> loader)
 {
     auto connection = TRY(Audio::ConnectionToServer::try_create());
-    return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection)));
+    return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection), move(loader)));
 }
 
-AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer> connection)
+AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer> connection, NonnullRefPtr<Audio::Loader> loader)
     : m_connection(move(connection))
+    , m_loader(move(loader))
+    , m_sample_timer(Web::Platform::Timer::create_repeating(UPDATE_RATE_MS, [this]() {
+        if (play_next_samples().is_error()) {
+            // FIXME: Propagate the error to the HTMLMediaElement.
+        } else {
+            if (on_playback_position_updated)
+                on_playback_position_updated(m_position);
+        }
+    }))
 {
+    auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
+    m_duration = Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
+
+    m_device_sample_rate = m_connection->get_sample_rate();
+    m_device_samples_per_buffer = static_cast<size_t>(BUFFER_SIZE_MS / 1000.0 * static_cast<double>(m_device_sample_rate));
+    m_samples_to_load_per_buffer = static_cast<size_t>(BUFFER_SIZE_MS / 1000.0 * static_cast<double>(m_loader->sample_rate()));
 }
 
 AudioCodecPluginSerenity::~AudioCodecPluginSerenity() = default;
 
-size_t AudioCodecPluginSerenity::device_sample_rate()
+ErrorOr<void> AudioCodecPluginSerenity::play_next_samples()
 {
-    if (!m_device_sample_rate.has_value())
-        m_device_sample_rate = m_connection->get_sample_rate();
-    return *m_device_sample_rate;
-}
+    while (m_connection->remaining_samples() < m_device_samples_per_buffer * ALWAYS_ENQUEUED_BUFFER_COUNT) {
+        bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
+        bool audio_server_done = m_connection->remaining_samples() == 0;
 
-void AudioCodecPluginSerenity::enqueue_samples(FixedArray<Audio::Sample> samples)
-{
-    m_connection->async_enqueue(move(samples)).release_value_but_fixme_should_propagate_errors();
-}
+        if (all_samples_loaded && audio_server_done) {
+            pause_playback();
 
-size_t AudioCodecPluginSerenity::remaining_samples() const
-{
-    return m_connection->remaining_samples();
+            m_connection->clear_client_buffer();
+            m_connection->async_clear_buffer();
+            (void)m_loader->reset();
+
+            m_position = m_duration;
+            break;
+        }
+
+        auto samples = TRY(read_samples_from_loader(m_loader, m_samples_to_load_per_buffer, m_device_sample_rate));
+        TRY(m_connection->async_enqueue(move(samples)));
+
+        m_position = current_loader_position(m_loader, m_device_sample_rate);
+    }
+
+    return {};
 }
 
 void AudioCodecPluginSerenity::resume_playback()
 {
     m_connection->async_start_playback();
+    m_sample_timer->start();
 }
 
 void AudioCodecPluginSerenity::pause_playback()
-{
-    m_connection->async_start_playback();
-}
-
-void AudioCodecPluginSerenity::playback_ended()
 {
     m_connection->async_pause_playback();
-    m_connection->clear_client_buffer();
-    m_connection->async_clear_buffer();
+    m_sample_timer->stop();
 }
 
 void AudioCodecPluginSerenity::set_volume(double volume)
@@ -61,4 +88,21 @@ void AudioCodecPluginSerenity::set_volume(double volume)
     m_connection->async_set_self_volume(volume);
 }
 
+void AudioCodecPluginSerenity::seek(double position)
+{
+    auto duration = static_cast<double>(this->duration().to_milliseconds()) / 1000.0;
+    position = position / duration * static_cast<double>(m_loader->total_samples());
+
+    m_loader->seek(static_cast<int>(position)).release_value_but_fixme_should_propagate_errors();
+    m_position = current_loader_position(m_loader, m_device_sample_rate);
+
+    if (on_playback_position_updated)
+        on_playback_position_updated(m_position);
+}
+
+Duration AudioCodecPluginSerenity::duration()
+{
+    return m_duration;
+}
+
 }

Userland/Services/WebContent/AudioCodecPluginSerenity.h

@@ -7,34 +7,42 @@
 #pragma once
 
 #include <AK/Error.h>
+#include <AK/NonnullOwnPtr.h>
 #include <AK/NonnullRefPtr.h>
-#include <AK/Optional.h>
+#include <AK/Time.h>
 #include <LibAudio/Forward.h>
+#include <LibWeb/Forward.h>
 #include <LibWeb/Platform/AudioCodecPlugin.h>
 
 namespace WebContent {
 
 class AudioCodecPluginSerenity final : public Web::Platform::AudioCodecPlugin {
 public:
-    static ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> create();
+    static ErrorOr<NonnullOwnPtr<AudioCodecPluginSerenity>> create(NonnullRefPtr<Audio::Loader>);
     virtual ~AudioCodecPluginSerenity() override;
 
-    virtual size_t device_sample_rate() override;
-
-    virtual void enqueue_samples(FixedArray<Audio::Sample>) override;
-    virtual size_t remaining_samples() const override;
-
     virtual void resume_playback() override;
     virtual void pause_playback() override;
-    virtual void playback_ended() override;
-
     virtual void set_volume(double) override;
+    virtual void seek(double) override;
+
+    virtual Duration duration() override;
 
 private:
-    explicit AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer>);
+    AudioCodecPluginSerenity(NonnullRefPtr<Audio::ConnectionToServer>, NonnullRefPtr<Audio::Loader>);
+
+    ErrorOr<void> play_next_samples();
 
     NonnullRefPtr<Audio::ConnectionToServer> m_connection;
-    Optional<size_t> m_device_sample_rate;
+    NonnullRefPtr<Audio::Loader> m_loader;
+    NonnullRefPtr<Web::Platform::Timer> m_sample_timer;
+
+    Duration m_duration;
+    Duration m_position;
+
+    size_t m_device_sample_rate { 0 };
+    size_t m_device_samples_per_buffer { 0 };
+    size_t m_samples_to_load_per_buffer { 0 };
 };
 
 }

Userland/Services/WebContent/main.cpp

@@ -6,6 +6,7 @@
 
 #include "AudioCodecPluginSerenity.h"
 #include "ImageCodecPluginSerenity.h"
+#include <LibAudio/Loader.h>
 #include <LibCore/EventLoop.h>
 #include <LibCore/LocalServer.h>
 #include <LibCore/StandardPaths.h>
@@ -46,8 +47,8 @@ ErrorOr<int> serenity_main(Main::Arguments)
     Web::Platform::ImageCodecPlugin::install(*new WebContent::ImageCodecPluginSerenity);
     Web::Platform::FontPlugin::install(*new Web::Platform::FontPluginSerenity);
 
-    Web::Platform::AudioCodecPlugin::install_creation_hook([] {
-        return WebContent::AudioCodecPluginSerenity::create();
+    Web::Platform::AudioCodecPlugin::install_creation_hook([](auto loader) {
+        return WebContent::AudioCodecPluginSerenity::create(move(loader));
     });
 
     Web::WebSockets::WebSocketClientManager::initialize(TRY(WebView::WebSocketClientManagerAdapter::try_create()));