From a34e3692523aa7d2cc3f94d8dc7345dfedaad03d Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Mon, 12 Jun 2023 13:44:10 -0400 Subject: [PATCH] Ladybird+LibWeb+WebContent: Create a platform plugin for playing audio This creates (and installs upon WebContent startup) a platform plugin to play audio data. On Serenity, we use AudioServer to play audio over IPC. Unfortunately, AudioServer is currently coupled with Serenity's audio devices, and thus cannot be used in Ladybird on Lagom. Instead, we use a Qt audio device to play the audio, which requires the Qt multimedia package. While we use Qt to play the audio, note that we can still use LibAudio to decode the audio data and retrieve samples - we simply send Qt the raw PCM signals. --- Documentation/BuildInstructionsLadybird.md | 6 +- Ladybird/AudioCodecPluginLadybird.cpp | 89 +++++++++++++++++++ Ladybird/AudioCodecPluginLadybird.h | 42 +++++++++ Ladybird/CMakeLists.txt | 2 +- Ladybird/WebContent/CMakeLists.txt | 3 +- Ladybird/WebContent/main.cpp | 5 ++ Meta/Azure/Setup.yml | 2 +- Userland/Libraries/LibWeb/CMakeLists.txt | 3 +- Userland/Libraries/LibWeb/Forward.h | 1 + .../LibWeb/Platform/AudioCodecPlugin.cpp | 28 ++++++ .../LibWeb/Platform/AudioCodecPlugin.h | 37 ++++++++ .../WebContent/AudioCodecPluginSerenity.cpp | 59 ++++++++++++ .../WebContent/AudioCodecPluginSerenity.h | 38 ++++++++ Userland/Services/WebContent/CMakeLists.txt | 3 +- Userland/Services/WebContent/main.cpp | 8 +- 15 files changed, 317 insertions(+), 9 deletions(-) create mode 100644 Ladybird/AudioCodecPluginLadybird.cpp create mode 100644 Ladybird/AudioCodecPluginLadybird.h create mode 100644 Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp create mode 100644 Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h create mode 100644 Userland/Services/WebContent/AudioCodecPluginSerenity.cpp create mode 100644 Userland/Services/WebContent/AudioCodecPluginSerenity.h diff --git a/Documentation/BuildInstructionsLadybird.md b/Documentation/BuildInstructionsLadybird.md index f8ffe0c564..9a479bdb9c 100644 --- a/Documentation/BuildInstructionsLadybird.md +++ b/Documentation/BuildInstructionsLadybird.md @@ -7,7 +7,7 @@ Qt6 development packages and a C++20 capable compiler are required. gcc-12 or cl On Debian/Ubuntu required packages include, but are not limited to: ``` -sudo apt install build-essential cmake libgl1-mesa-dev ninja-build qt6-base-dev libqt6svg6-dev qt6-tools-dev-tools +sudo apt install build-essential cmake libgl1-mesa-dev ninja-build qt6-base-dev libqt6svg6-dev qt6-tools-dev-tools qt6-multimedia-dev ``` For Ubuntu 20.04 and above, ensure that the Qt6 Wayland packages are available: @@ -19,12 +19,12 @@ sudo apt install qt6-wayland On Arch Linux/Manjaro: ``` -sudo pacman -S --needed base-devel cmake libgl ninja qt6-base qt6-svg qt6-tools qt6-wayland +sudo pacman -S --needed base-devel cmake libgl ninja qt6-base qt6-svg qt6-tools qt6-wayland qt6-multimedia ``` On Fedora or derivatives: ``` -sudo dnf install cmake libglvnd-devel ninja-build qt6-qtbase-devel qt6-qtsvg-devel qt6-qttools-devel qt6-qtwayland-devel +sudo dnf install cmake libglvnd-devel ninja-build qt6-qtbase-devel qt6-qtsvg-devel qt6-qttools-devel qt6-qtwayland-devel qt6-qtmultimedia-devel ``` On openSUSE: diff --git a/Ladybird/AudioCodecPluginLadybird.cpp b/Ladybird/AudioCodecPluginLadybird.cpp new file mode 100644 index 0000000000..8baf41a9c2 --- /dev/null +++ b/Ladybird/AudioCodecPluginLadybird.cpp @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "AudioCodecPluginLadybird.h" +#include +#include +#include +#include +#include +#include +#include + +namespace Ladybird { + +ErrorOr> AudioCodecPluginLadybird::create() +{ + auto devices = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QMediaDevices())); + auto const& device_info = devices->defaultAudioOutput(); + + auto format = device_info.preferredFormat(); + format.setSampleFormat(QAudioFormat::Int16); + format.setChannelCount(2); + + if (!device_info.isFormatSupported(format)) + return Error::from_string_literal("Audio device format not supported"); + + auto audio_output = TRY(adopt_nonnull_own_or_enomem(new (nothrow) QAudioSink(device_info, format))); + + return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginLadybird(move(devices), move(audio_output))); +} + +AudioCodecPluginLadybird::AudioCodecPluginLadybird(NonnullOwnPtr devices, NonnullOwnPtr audio_output) + : m_devices(move(devices)) + , m_audio_output(move(audio_output)) + , m_io_device(m_audio_output->start()) +{ +} + +AudioCodecPluginLadybird::~AudioCodecPluginLadybird() = default; + +size_t AudioCodecPluginLadybird::device_sample_rate() +{ + return m_audio_output->format().sampleRate(); +} + +void AudioCodecPluginLadybird::enqueue_samples(FixedArray samples) +{ + QByteArray buffer; + buffer.resize(samples.size() * 2 * sizeof(u16)); + + FixedMemoryStream stream { Bytes { buffer.data(), static_cast(buffer.size()) } }; + + for (auto& sample : samples) { + LittleEndian pcm; + + pcm = static_cast(sample.left * NumericLimits::max()); + MUST(stream.write_value(pcm)); + + pcm = static_cast(sample.right * NumericLimits::max()); + MUST(stream.write_value(pcm)); + } + + m_io_device->write(buffer.data(), buffer.size()); +} + +size_t AudioCodecPluginLadybird::remaining_samples() const +{ + return 0; +} + +void AudioCodecPluginLadybird::resume_playback() +{ + m_audio_output->resume(); +} + +void AudioCodecPluginLadybird::pause_playback() +{ + m_audio_output->suspend(); +} + +void AudioCodecPluginLadybird::playback_ended() +{ + m_audio_output->suspend(); +} + +} diff --git a/Ladybird/AudioCodecPluginLadybird.h b/Ladybird/AudioCodecPluginLadybird.h new file mode 100644 index 0000000000..bd17f1d778 --- /dev/null +++ b/Ladybird/AudioCodecPluginLadybird.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +class QAudioSink; +class QIODevice; +class QMediaDevices; + +namespace Ladybird { + +class AudioCodecPluginLadybird final : public Web::Platform::AudioCodecPlugin { +public: + static ErrorOr> create(); + virtual ~AudioCodecPluginLadybird() override; + + virtual size_t device_sample_rate() override; + + virtual void enqueue_samples(FixedArray) override; + virtual size_t remaining_samples() const override; + + virtual void resume_playback() override; + virtual void pause_playback() override; + virtual void playback_ended() override; + +private: + AudioCodecPluginLadybird(NonnullOwnPtr, NonnullOwnPtr); + + NonnullOwnPtr m_devices; + NonnullOwnPtr m_audio_output; + QIODevice* m_io_device { nullptr }; +}; + +} diff --git a/Ladybird/CMakeLists.txt b/Ladybird/CMakeLists.txt index e1e5bb6274..98ad917395 100644 --- a/Ladybird/CMakeLists.txt +++ b/Ladybird/CMakeLists.txt @@ -73,7 +73,7 @@ add_compile_options(-Wno-user-defined-literals) set(CMAKE_AUTOMOC ON) set(CMAKE_AUTORCC ON) set(CMAKE_AUTOUIC ON) -find_package(Qt6 REQUIRED COMPONENTS Core Widgets Network Svg) +find_package(Qt6 REQUIRED COMPONENTS Core Widgets Network Svg Multimedia) set(BROWSER_SOURCE_DIR ${SERENITY_SOURCE_DIR}/Userland/Applications/Browser/) diff --git a/Ladybird/WebContent/CMakeLists.txt b/Ladybird/WebContent/CMakeLists.txt index 71c28a143e..88e5315f43 100644 --- a/Ladybird/WebContent/CMakeLists.txt +++ b/Ladybird/WebContent/CMakeLists.txt @@ -6,6 +6,7 @@ set(WEBCONTENT_SOURCES ${WEBCONTENT_SOURCE_DIR}/PageHost.cpp ${WEBCONTENT_SOURCE_DIR}/WebContentConsoleClient.cpp ${WEBCONTENT_SOURCE_DIR}/WebDriverConnection.cpp + ../AudioCodecPluginLadybird.cpp ../EventLoopImplementationQt.cpp ../FontPluginQt.cpp ../ImageCodecPluginLadybird.cpp @@ -25,4 +26,4 @@ qt_add_executable(WebContent ${WEBCONTENT_SOURCES}) target_include_directories(WebContent PRIVATE ${SERENITY_SOURCE_DIR}/Userland/Services/) target_include_directories(WebContent PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/..) -target_link_libraries(WebContent PRIVATE Qt::Core Qt::Gui Qt::Network LibCore LibFileSystem LibGfx LibIPC LibJS LibMain LibWeb LibWebSocket) +target_link_libraries(WebContent PRIVATE Qt::Core Qt::Gui Qt::Network Qt::Multimedia LibAudio LibCore LibFileSystem LibGfx LibIPC LibJS LibMain LibWeb LibWebSocket) diff --git a/Ladybird/WebContent/main.cpp b/Ladybird/WebContent/main.cpp index ee35a2b258..7413485324 100644 --- a/Ladybird/WebContent/main.cpp +++ b/Ladybird/WebContent/main.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include "../AudioCodecPluginLadybird.h" #include "../EventLoopImplementationQt.h" #include "../FontPluginQt.h" #include "../ImageCodecPluginLadybird.h" @@ -57,6 +58,10 @@ ErrorOr serenity_main(Main::Arguments arguments) Web::Platform::EventLoopPlugin::install(*new Web::Platform::EventLoopPluginSerenity); Web::Platform::ImageCodecPlugin::install(*new Ladybird::ImageCodecPluginLadybird); + Web::Platform::AudioCodecPlugin::install_creation_hook([] { + return Ladybird::AudioCodecPluginLadybird::create(); + }); + Web::ResourceLoader::initialize(RequestManagerQt::create()); Web::WebSockets::WebSocketClientManager::initialize(Ladybird::WebSocketClientManagerLadybird::create()); diff --git a/Meta/Azure/Setup.yml b/Meta/Azure/Setup.yml index 6422a99f76..975b961116 100644 --- a/Meta/Azure/Setup.yml +++ b/Meta/Azure/Setup.yml @@ -21,7 +21,7 @@ steps: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - sudo add-apt-repository 'deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-15 main' sudo apt-get update - sudo apt-get install ccache gcc-12 g++-12 clang-15 libstdc++-12-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev libgl1-mesa-dev + sudo apt-get install ccache gcc-12 g++-12 clang-15 libstdc++-12-dev ninja-build unzip qt6-base-dev qt6-tools-dev-tools libqt6svg6-dev qt6-multimedia-dev libgl1-mesa-dev sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100 sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-15 100 diff --git a/Userland/Libraries/LibWeb/CMakeLists.txt b/Userland/Libraries/LibWeb/CMakeLists.txt index 08266fcc02..321682d2b1 100644 --- a/Userland/Libraries/LibWeb/CMakeLists.txt +++ b/Userland/Libraries/LibWeb/CMakeLists.txt @@ -473,6 +473,7 @@ set(SOURCES PerformanceTimeline/EntryTypes.cpp PerformanceTimeline/PerformanceEntry.cpp PermissionsPolicy/AutoplayAllowlist.cpp + Platform/AudioCodecPlugin.cpp Platform/EventLoopPlugin.cpp Platform/EventLoopPluginSerenity.cpp Platform/FontPlugin.cpp @@ -595,7 +596,7 @@ set(GENERATED_SOURCES serenity_lib(LibWeb web) # NOTE: We link with LibSoftGPU here instead of lazy loading it via dlopen() so that we do not have to unveil the library and pledge prot_exec. -target_link_libraries(LibWeb PRIVATE LibCore LibCrypto LibJS LibMarkdown LibHTTP LibGemini LibGL LibGUI LibGfx LibIPC LibLocale LibRegex LibSoftGPU LibSyntax LibTextCodec LibUnicode LibVideo LibWasm LibXML LibIDL) +target_link_libraries(LibWeb PRIVATE LibCore LibCrypto LibJS LibMarkdown LibHTTP LibGemini LibGL LibGUI LibGfx LibIPC LibLocale LibRegex LibSoftGPU LibSyntax LibTextCodec LibUnicode LibAudio LibVideo LibWasm LibXML LibIDL) link_with_locale_data(LibWeb) generate_js_bindings(LibWeb) diff --git a/Userland/Libraries/LibWeb/Forward.h b/Userland/Libraries/LibWeb/Forward.h index 7421de5325..b529234407 100644 --- a/Userland/Libraries/LibWeb/Forward.h +++ b/Userland/Libraries/LibWeb/Forward.h @@ -509,6 +509,7 @@ class AutoplayAllowlist; } namespace Web::Platform { +class AudioCodecPlugin; class Timer; } diff --git a/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp new file mode 100644 index 0000000000..bd5e7d6d56 --- /dev/null +++ b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.cpp @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +namespace Web::Platform { + +static Function>()> s_creation_hook; + +AudioCodecPlugin::AudioCodecPlugin() = default; +AudioCodecPlugin::~AudioCodecPlugin() = default; + +void AudioCodecPlugin::install_creation_hook(Function>()> creation_hook) +{ + VERIFY(!s_creation_hook); + s_creation_hook = move(creation_hook); +} + +ErrorOr> AudioCodecPlugin::create() +{ + VERIFY(s_creation_hook); + return s_creation_hook(); +} + +} diff --git a/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h new file mode 100644 index 0000000000..0f759706a0 --- /dev/null +++ b/Userland/Libraries/LibWeb/Platform/AudioCodecPlugin.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Web::Platform { + +class AudioCodecPlugin { +public: + static void install_creation_hook(Function>()>); + static ErrorOr> create(); + + virtual ~AudioCodecPlugin(); + + virtual size_t device_sample_rate() = 0; + + virtual void enqueue_samples(FixedArray) = 0; + virtual size_t remaining_samples() const = 0; + + virtual void resume_playback() = 0; + virtual void pause_playback() = 0; + virtual void playback_ended() = 0; + +protected: + AudioCodecPlugin(); +}; + +} diff --git a/Userland/Services/WebContent/AudioCodecPluginSerenity.cpp b/Userland/Services/WebContent/AudioCodecPluginSerenity.cpp new file mode 100644 index 0000000000..0632504a87 --- /dev/null +++ b/Userland/Services/WebContent/AudioCodecPluginSerenity.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +namespace WebContent { + +ErrorOr> AudioCodecPluginSerenity::create() +{ + auto connection = TRY(Audio::ConnectionToServer::try_create()); + return adopt_nonnull_own_or_enomem(new (nothrow) AudioCodecPluginSerenity(move(connection))); +} + +AudioCodecPluginSerenity::AudioCodecPluginSerenity(NonnullRefPtr connection) + : m_connection(move(connection)) +{ +} + +AudioCodecPluginSerenity::~AudioCodecPluginSerenity() = default; + +size_t AudioCodecPluginSerenity::device_sample_rate() +{ + if (!m_device_sample_rate.has_value()) + m_device_sample_rate = m_connection->get_sample_rate(); + return *m_device_sample_rate; +} + +void AudioCodecPluginSerenity::enqueue_samples(FixedArray samples) +{ + m_connection->async_enqueue(move(samples)).release_value_but_fixme_should_propagate_errors(); +} + +size_t AudioCodecPluginSerenity::remaining_samples() const +{ + return m_connection->remaining_samples(); +} + +void AudioCodecPluginSerenity::resume_playback() +{ + m_connection->async_start_playback(); +} + +void AudioCodecPluginSerenity::pause_playback() +{ + m_connection->async_start_playback(); +} + +void AudioCodecPluginSerenity::playback_ended() +{ + m_connection->async_pause_playback(); + m_connection->clear_client_buffer(); + m_connection->async_clear_buffer(); +} + +} diff --git a/Userland/Services/WebContent/AudioCodecPluginSerenity.h b/Userland/Services/WebContent/AudioCodecPluginSerenity.h new file mode 100644 index 0000000000..39bf6a31cb --- /dev/null +++ b/Userland/Services/WebContent/AudioCodecPluginSerenity.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace WebContent { + +class AudioCodecPluginSerenity final : public Web::Platform::AudioCodecPlugin { +public: + static ErrorOr> create(); + virtual ~AudioCodecPluginSerenity() override; + + virtual size_t device_sample_rate() override; + + virtual void enqueue_samples(FixedArray) override; + virtual size_t remaining_samples() const override; + + virtual void resume_playback() override; + virtual void pause_playback() override; + virtual void playback_ended() override; + +private: + explicit AudioCodecPluginSerenity(NonnullRefPtr); + + NonnullRefPtr m_connection; + Optional m_device_sample_rate; +}; + +} diff --git a/Userland/Services/WebContent/CMakeLists.txt b/Userland/Services/WebContent/CMakeLists.txt index ef0cf40da7..6980a21fc8 100644 --- a/Userland/Services/WebContent/CMakeLists.txt +++ b/Userland/Services/WebContent/CMakeLists.txt @@ -11,6 +11,7 @@ compile_ipc(WebDriverClient.ipc WebDriverClientEndpoint.h) compile_ipc(WebDriverServer.ipc WebDriverServerEndpoint.h) set(SOURCES + AudioCodecPluginSerenity.cpp ConnectionFromClient.cpp ConsoleGlobalEnvironmentExtensions.cpp ImageCodecPluginSerenity.cpp @@ -28,5 +29,5 @@ set(GENERATED_SOURCES ) serenity_bin(WebContent) -target_link_libraries(WebContent PRIVATE LibCore LibFileSystem LibIPC LibGfx LibImageDecoderClient LibJS LibWebView LibWeb LibLocale LibMain) +target_link_libraries(WebContent PRIVATE LibCore LibFileSystem LibIPC LibGfx LibAudio LibImageDecoderClient LibJS LibWebView LibWeb LibLocale LibMain) link_with_locale_data(WebContent) diff --git a/Userland/Services/WebContent/main.cpp b/Userland/Services/WebContent/main.cpp index d6c5b77ffe..93a9f09558 100644 --- a/Userland/Services/WebContent/main.cpp +++ b/Userland/Services/WebContent/main.cpp @@ -4,6 +4,7 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include "AudioCodecPluginSerenity.h" #include "ImageCodecPluginSerenity.h" #include #include @@ -25,7 +26,7 @@ ErrorOr serenity_main(Main::Arguments) { Core::EventLoop event_loop; - TRY(Core::System::pledge("stdio recvfd sendfd accept unix rpath thread")); + TRY(Core::System::pledge("stdio recvfd sendfd accept unix rpath thread proc")); // This must be first; we can't check if /tmp/webdriver exists once we've unveiled other paths. auto webdriver_socket_path = DeprecatedString::formatted("{}/webdriver", TRY(Core::StandardPaths::runtime_directory())); @@ -35,6 +36,7 @@ ErrorOr serenity_main(Main::Arguments) TRY(Core::System::unveil("/res", "r")); TRY(Core::System::unveil("/etc/timezone", "r")); TRY(Core::System::unveil("/usr/lib", "r")); + TRY(Core::System::unveil("/tmp/session/%sid/portal/audio", "rw")); TRY(Core::System::unveil("/tmp/session/%sid/portal/request", "rw")); TRY(Core::System::unveil("/tmp/session/%sid/portal/image", "rw")); TRY(Core::System::unveil("/tmp/session/%sid/portal/websocket", "rw")); @@ -44,6 +46,10 @@ ErrorOr serenity_main(Main::Arguments) Web::Platform::ImageCodecPlugin::install(*new WebContent::ImageCodecPluginSerenity); Web::Platform::FontPlugin::install(*new Web::Platform::FontPluginSerenity); + Web::Platform::AudioCodecPlugin::install_creation_hook([] { + return WebContent::AudioCodecPluginSerenity::create(); + }); + Web::WebSockets::WebSocketClientManager::initialize(TRY(WebView::WebSocketClientManagerAdapter::try_create())); Web::ResourceLoader::initialize(TRY(WebView::RequestServerAdapter::try_create())); TRY(Web::Bindings::initialize_main_thread_vm());