From ad440f9e9ac6c9692cd7f61e06d2dab2a5d0c80c Mon Sep 17 00:00:00 2001 From: Zaggy1024 Date: Tue, 4 Jul 2023 05:02:19 -0500 Subject: [PATCH] LibWeb/Ladybird: Use the abstract audio output in a new audio plugin The implementation of this plugin is meant to eventually replace all current audio plugins in Ladybird. The benefits over the current Qt- based audio playback plugin in Ladybird are: - Low latency: With direct access to PulseAudio, we can ask for a specific latency to output to allow minimal delay when pausing or seeking a stream. - Accurate timestamps: The Qt audio playback API does not expose audio time properly. When we have access directly to PulseAudio APIs, we can enable their timing interpolation to get an accurate monotonically- increasing timestamp of the playing audio. - Resiliency: With more control over how the underlying audio API is called, we have the power to fix most bugs we might encounter. The PulseAudio wrappers already avoid some bugs that occur with QAudioSink when running through WSLg. --- Ladybird/WebContent/CMakeLists.txt | 4 + Ladybird/WebContent/main.cpp | 5 + Userland/Libraries/LibWeb/CMakeLists.txt | 1 + .../Platform/AudioCodecPluginAgnostic.cpp | 174 ++++++++++++++++++ .../Platform/AudioCodecPluginAgnostic.h | 41 +++++ 5 files changed, 225 insertions(+) create mode 100644 Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.cpp create mode 100644 Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.h diff --git a/Ladybird/WebContent/CMakeLists.txt b/Ladybird/WebContent/CMakeLists.txt index f7ce3dd4e6..79cb33dc01 100644 --- a/Ladybird/WebContent/CMakeLists.txt +++ b/Ladybird/WebContent/CMakeLists.txt @@ -29,3 +29,7 @@ target_link_libraries(WebContent PRIVATE Qt::Core Qt::Network Qt::Multimedia Lib if (ANDROID) link_android_libs(WebContent) endif() + +if (HAVE_PULSEAUDIO) + target_compile_definitions(WebContent PRIVATE HAVE_PULSEAUDIO=1) +endif() diff --git a/Ladybird/WebContent/main.cpp b/Ladybird/WebContent/main.cpp index 233e25adf3..6968890c3b 100644 --- a/Ladybird/WebContent/main.cpp +++ b/Ladybird/WebContent/main.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,11 @@ ErrorOr serenity_main(Main::Arguments arguments) Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPlugin); Web::Platform::AudioCodecPlugin::install_creation_hook([](auto loader) { +#if defined(HAVE_PULSEAUDIO) + return Web::Platform::AudioCodecPluginAgnostic::create(move(loader)); +#else return Ladybird::AudioCodecPluginQt::create(move(loader)); +#endif }); Web::FrameLoader::set_default_favicon_path(DeprecatedString::formatted("{}/res/icons/16x16/app-browser.png", s_serenity_resource_root)); diff --git a/Userland/Libraries/LibWeb/CMakeLists.txt b/Userland/Libraries/LibWeb/CMakeLists.txt index a2385507b1..b8fe79275e 100644 --- a/Userland/Libraries/LibWeb/CMakeLists.txt +++ b/Userland/Libraries/LibWeb/CMakeLists.txt @@ -489,6 +489,7 @@ set(SOURCES PermissionsPolicy/AutoplayAllowlist.cpp PixelUnits.cpp Platform/AudioCodecPlugin.cpp + Platform/AudioCodecPluginAgnostic.cpp Platform/EventLoopPlugin.cpp Platform/EventLoopPluginSerenity.cpp Platform/FontPlugin.cpp diff --git a/Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.cpp b/Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.cpp new file mode 100644 index 0000000000..9acd9cb40d --- /dev/null +++ b/Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.cpp @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2023, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include + +#include "AudioCodecPluginAgnostic.h" + +namespace Web::Platform { + +constexpr int update_interval = 10; + +static Duration timestamp_from_samples(i64 samples, u32 sample_rate) +{ + return Duration::from_milliseconds(samples * 1000 / sample_rate); +} + +static Duration get_loader_timestamp(NonnullRefPtr const& loader) +{ + return timestamp_from_samples(loader->loaded_samples(), loader->sample_rate()); +} + +ErrorOr> AudioCodecPluginAgnostic::create(NonnullRefPtr const& loader) +{ + auto duration = timestamp_from_samples(loader->total_samples(), loader->sample_rate()); + + auto update_timer = TRY(Core::Timer::try_create()); + update_timer->set_interval(update_interval); + + auto plugin = TRY(adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginAgnostic(loader, duration, move(update_timer)))); + + constexpr u32 latency_ms = 100; + RefPtr output = TRY(Audio::PlaybackStream::create( + Audio::OutputState::Suspended, loader->sample_rate(), loader->num_channels(), latency_ms, + [&plugin = *plugin, loader](Bytes buffer, Audio::PcmSampleFormat format, size_t sample_count) -> ReadonlyBytes { + VERIFY(format == Audio::PcmSampleFormat::Float32); + auto samples = loader->get_more_samples(sample_count).release_value_but_fixme_should_propagate_errors(); + VERIFY(samples.size() <= sample_count); + FixedMemoryStream writing_stream { buffer }; + + for (auto& sample : samples) { + MUST(writing_stream.write_value(sample.left)); + MUST(writing_stream.write_value(sample.right)); + } + + // FIXME: Check if we have loaded samples past the current known duration, and if so, update it + // and notify the media element. + return buffer.trim(writing_stream.offset()); + })); + + output->set_underrun_callback([&plugin = *plugin, loader, output]() { + auto new_device_time = output->total_time_played().release_value_but_fixme_should_propagate_errors(); + auto new_media_time = timestamp_from_samples(loader->loaded_samples(), loader->sample_rate()); + plugin.m_main_thread_event_loop.deferred_invoke([&plugin, new_device_time, new_media_time]() { + plugin.m_last_resume_in_device_time = new_device_time; + plugin.m_last_resume_in_media_time = new_media_time; + }); + }); + + plugin->m_output = move(output); + + return plugin; +} + +AudioCodecPluginAgnostic::AudioCodecPluginAgnostic(NonnullRefPtr loader, Duration duration, NonnullRefPtr update_timer) + : m_loader(move(loader)) + , m_duration(duration) + , m_main_thread_event_loop(Core::EventLoop::current()) + , m_update_timer(move(update_timer)) +{ + m_update_timer->on_timeout = [this]() { + update_timestamp(); + }; + m_update_timer->start(); +} + +void AudioCodecPluginAgnostic::resume_playback() +{ + m_paused = false; + m_output->resume() + ->when_resolved([this](Duration new_device_time) { + m_main_thread_event_loop.deferred_invoke([this, new_device_time]() { + m_last_resume_in_device_time = new_device_time; + m_update_timer->start(); + }); + }) + .when_rejected([](Error&&) { + // FIXME: Propagate errors. + }); +} + +void AudioCodecPluginAgnostic::pause_playback() +{ + m_paused = true; + m_output->drain_buffer_and_suspend() + ->when_resolved([this]() -> ErrorOr { + auto new_media_time = timestamp_from_samples(m_loader->loaded_samples(), m_loader->sample_rate()); + auto new_device_time = TRY(m_output->total_time_played()); + m_main_thread_event_loop.deferred_invoke([this, new_media_time, new_device_time]() { + m_last_resume_in_media_time = new_media_time; + m_last_resume_in_device_time = new_device_time; + m_update_timer->stop(); + update_timestamp(); + }); + return {}; + }) + .when_rejected([](Error&&) { + // FIXME: Propagate errors. + }); +} + +void AudioCodecPluginAgnostic::set_volume(double volume) +{ + m_output->set_volume(volume)->when_rejected([](Error&&) { + // FIXME: Propagate errors. + }); +} + +void AudioCodecPluginAgnostic::seek(double position) +{ + m_output->discard_buffer_and_suspend() + ->when_resolved([this, position, was_paused = m_paused]() -> ErrorOr { + auto sample_position = static_cast(position * m_loader->sample_rate()); + auto seek_result = m_loader->seek(sample_position); + if (seek_result.is_error()) + return Error::from_string_literal("Seeking in audio loader failed"); + + auto new_media_time = get_loader_timestamp(m_loader); + auto new_device_time = m_output->total_time_played().release_value_but_fixme_should_propagate_errors(); + + m_main_thread_event_loop.deferred_invoke([this, was_paused, new_device_time, new_media_time]() { + m_last_resume_in_device_time = new_device_time; + m_last_resume_in_media_time = new_media_time; + if (was_paused) { + update_timestamp(); + } else { + m_output->resume()->when_rejected([](Error&&) { + // FIXME: Propagate errors. + }); + } + }); + + return {}; + }) + .when_rejected([](Error&&) { + // FIXME: Propagate errors. + }); +} + +Duration AudioCodecPluginAgnostic::duration() +{ + return m_duration; +} + +void AudioCodecPluginAgnostic::update_timestamp() +{ + auto current_device_time_result = m_output->total_time_played(); + if (!current_device_time_result.is_error()) + m_last_good_device_time = current_device_time_result.release_value(); + auto current_device_time_delta = m_last_good_device_time - m_last_resume_in_device_time; + + auto current_media_time = m_last_resume_in_media_time + current_device_time_delta; + current_media_time = min(current_media_time, m_duration); + on_playback_position_updated(current_media_time); +} + +} diff --git a/Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.h b/Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.h new file mode 100644 index 0000000000..12b5103720 --- /dev/null +++ b/Userland/Libraries/LibWeb/Platform/AudioCodecPluginAgnostic.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023, Gregory Bertilson + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include + +namespace Web::Platform { + +class AudioCodecPluginAgnostic final : public AudioCodecPlugin { +public: + static ErrorOr> create(NonnullRefPtr const&); + + virtual void resume_playback() override; + virtual void pause_playback() override; + virtual void set_volume(double) override; + virtual void seek(double) override; + + virtual Duration duration() override; + +private: + explicit AudioCodecPluginAgnostic(NonnullRefPtr loader, Duration, NonnullRefPtr update_timer); + + void update_timestamp(); + + NonnullRefPtr m_loader; + RefPtr m_output { nullptr }; + Duration m_duration { Duration::zero() }; + Duration m_last_resume_in_media_time { Duration::zero() }; + Duration m_last_resume_in_device_time { Duration::zero() }; + Duration m_last_good_device_time { Duration::zero() }; + Core::EventLoop& m_main_thread_event_loop; + NonnullRefPtr m_update_timer; + bool m_paused { true }; +}; + +}