mirror of
https://github.com/RGBCube/serenity
synced 2025-07-27 18:07:34 +00:00
Ladybird+LibWeb+WebConent: Drive audio in Ladybird off the main thread
The main thread in the WebContent process is often busy with layout and running JavaScript. This can cause audio to sound jittery and crack. To avoid this behavior, we now drive audio on a secondary thread. Note: Browser on Serenity uses AudioServer, the connection for which is already handled on a secondary thread within LibAudio. So this only applies to Lagom. Rather than using LibThreading, our hands are tied to QThread for now. Internally, the Qt media objects use a QTimer, which is forbidden from running on a thread that is not a QThread (the debug console is spammed with messages pointing this out). Ideally, in the future AudioServer will be able to run for non-Serenity platforms, and most of this can be aligned with the Serenity implementation.
This commit is contained in:
parent
0fd35b4dd8
commit
1c4dd0caad
10 changed files with 383 additions and 205 deletions
|
@ -5,10 +5,7 @@
|
|||
*/
|
||||
|
||||
#include <AK/IDAllocator.h>
|
||||
#include <AK/Time.h>
|
||||
#include <LibAudio/Loader.h>
|
||||
#include <LibAudio/Resampler.h>
|
||||
#include <LibAudio/Sample.h>
|
||||
#include <LibJS/Runtime/Realm.h>
|
||||
#include <LibJS/Runtime/VM.h>
|
||||
#include <LibWeb/Bindings/AudioTrackPrototype.h>
|
||||
|
@ -20,25 +17,23 @@
|
|||
#include <LibWeb/HTML/HTMLMediaElement.h>
|
||||
#include <LibWeb/Layout/Node.h>
|
||||
#include <LibWeb/Platform/AudioCodecPlugin.h>
|
||||
#include <LibWeb/Platform/Timer.h>
|
||||
|
||||
namespace Web::HTML {
|
||||
|
||||
static IDAllocator s_audio_track_id_allocator;
|
||||
|
||||
// Number of milliseconds of audio data contained in each audio buffer
|
||||
static constexpr u32 BUFFER_SIZE_MS = 50;
|
||||
|
||||
AudioTrack::AudioTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> media_element, NonnullRefPtr<Audio::Loader> loader)
|
||||
: PlatformObject(realm)
|
||||
, m_media_element(media_element)
|
||||
, m_audio_plugin(Platform::AudioCodecPlugin::create().release_value_but_fixme_should_propagate_errors())
|
||||
, m_loader(move(loader))
|
||||
, m_sample_timer(Platform::Timer::create_repeating(BUFFER_SIZE_MS, [this]() {
|
||||
play_next_samples();
|
||||
}))
|
||||
, m_audio_plugin(Platform::AudioCodecPlugin::create(move(loader)).release_value_but_fixme_should_propagate_errors())
|
||||
{
|
||||
m_audio_plugin->device_sample_rate();
|
||||
m_audio_plugin->on_playback_position_updated = [this](auto position) {
|
||||
if (auto* layout_node = m_media_element->layout_node())
|
||||
layout_node->set_needs_display();
|
||||
|
||||
auto playback_position = static_cast<double>(position.to_milliseconds()) / 1000.0;
|
||||
m_media_element->set_current_playback_position(playback_position);
|
||||
};
|
||||
}
|
||||
|
||||
AudioTrack::~AudioTrack()
|
||||
|
@ -63,30 +58,16 @@ JS::ThrowCompletionOr<void> AudioTrack::initialize(JS::Realm& realm)
|
|||
void AudioTrack::play(Badge<HTMLAudioElement>)
|
||||
{
|
||||
m_audio_plugin->resume_playback();
|
||||
m_sample_timer->start();
|
||||
}
|
||||
|
||||
void AudioTrack::pause(Badge<HTMLAudioElement>)
|
||||
{
|
||||
m_audio_plugin->pause_playback();
|
||||
m_sample_timer->stop();
|
||||
}
|
||||
|
||||
Duration AudioTrack::position() const
|
||||
Duration AudioTrack::duration()
|
||||
{
|
||||
auto samples_played = static_cast<double>(m_loader->loaded_samples());
|
||||
auto sample_rate = static_cast<double>(m_loader->sample_rate());
|
||||
|
||||
auto source_to_device_ratio = sample_rate / static_cast<double>(m_audio_plugin->device_sample_rate());
|
||||
samples_played *= source_to_device_ratio;
|
||||
|
||||
return Duration::from_milliseconds(static_cast<i64>(samples_played / sample_rate * 1000.0));
|
||||
}
|
||||
|
||||
Duration AudioTrack::duration() const
|
||||
{
|
||||
auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
|
||||
return Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
|
||||
return m_audio_plugin->duration();
|
||||
}
|
||||
|
||||
void AudioTrack::seek(double position, MediaSeekMode seek_mode)
|
||||
|
@ -94,11 +75,7 @@ void AudioTrack::seek(double position, MediaSeekMode seek_mode)
|
|||
// FIXME: Implement seeking mode.
|
||||
(void)seek_mode;
|
||||
|
||||
auto duration = static_cast<double>(this->duration().to_milliseconds()) / 1000.0;
|
||||
position = position / duration * static_cast<double>(m_loader->total_samples());
|
||||
|
||||
m_loader->seek(position).release_value_but_fixme_should_propagate_errors();
|
||||
m_media_element->set_current_playback_position(this->position().to_milliseconds() / 1000.0);
|
||||
m_audio_plugin->seek(position);
|
||||
}
|
||||
|
||||
void AudioTrack::update_volume()
|
||||
|
@ -134,48 +111,4 @@ void AudioTrack::set_enabled(bool enabled)
|
|||
m_enabled = enabled;
|
||||
}
|
||||
|
||||
Optional<FixedArray<Audio::Sample>> AudioTrack::get_next_samples()
|
||||
{
|
||||
bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
|
||||
bool audio_server_done = m_audio_plugin->remaining_samples() == 0;
|
||||
|
||||
if (all_samples_loaded && audio_server_done)
|
||||
return {};
|
||||
|
||||
auto samples_to_load_per_buffer = static_cast<size_t>(BUFFER_SIZE_MS / 1000.0f * static_cast<float>(m_loader->sample_rate()));
|
||||
|
||||
auto buffer_or_error = m_loader->get_more_samples(samples_to_load_per_buffer);
|
||||
if (buffer_or_error.is_error()) {
|
||||
dbgln("Error while loading samples: {}", buffer_or_error.error().description);
|
||||
return {};
|
||||
}
|
||||
|
||||
return buffer_or_error.release_value();
|
||||
}
|
||||
|
||||
void AudioTrack::play_next_samples()
|
||||
{
|
||||
if (auto* layout_node = m_media_element->layout_node())
|
||||
layout_node->set_needs_display();
|
||||
|
||||
auto samples = get_next_samples();
|
||||
if (!samples.has_value()) {
|
||||
m_audio_plugin->playback_ended();
|
||||
(void)m_loader->reset();
|
||||
|
||||
auto playback_position = static_cast<double>(duration().to_milliseconds()) / 1000.0;
|
||||
m_media_element->set_current_playback_position(playback_position);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
Audio::ResampleHelper<Audio::Sample> resampler(m_loader->sample_rate(), m_audio_plugin->device_sample_rate());
|
||||
auto resampled = FixedArray<Audio::Sample>::create(resampler.resample(samples.release_value()).span()).release_value_but_fixme_should_propagate_errors();
|
||||
|
||||
m_audio_plugin->enqueue_samples(move(resampled));
|
||||
|
||||
auto playback_position = static_cast<double>(position().to_milliseconds()) / 1000.0;
|
||||
m_media_element->set_current_playback_position(playback_position);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -24,8 +24,7 @@ public:
|
|||
void play(Badge<HTMLAudioElement>);
|
||||
void pause(Badge<HTMLAudioElement>);
|
||||
|
||||
Duration position() const;
|
||||
Duration duration() const;
|
||||
Duration duration();
|
||||
void seek(double, MediaSeekMode);
|
||||
|
||||
void update_volume();
|
||||
|
@ -44,9 +43,6 @@ private:
|
|||
virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override;
|
||||
virtual void visit_edges(Cell::Visitor&) override;
|
||||
|
||||
Optional<FixedArray<Audio::Sample>> get_next_samples();
|
||||
void play_next_samples();
|
||||
|
||||
// https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-id
|
||||
String m_id;
|
||||
|
||||
|
@ -66,8 +62,6 @@ private:
|
|||
JS::GCPtr<AudioTrackList> m_audio_track_list;
|
||||
|
||||
NonnullOwnPtr<Platform::AudioCodecPlugin> m_audio_plugin;
|
||||
NonnullRefPtr<Audio::Loader> m_loader;
|
||||
NonnullRefPtr<Platform::Timer> m_sample_timer;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
@ -4,25 +4,51 @@
|
|||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <LibAudio/Loader.h>
|
||||
#include <LibAudio/Resampler.h>
|
||||
#include <LibAudio/Sample.h>
|
||||
#include <LibWeb/Platform/AudioCodecPlugin.h>
|
||||
|
||||
namespace Web::Platform {
|
||||
|
||||
static Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()> s_creation_hook;
|
||||
static AudioCodecPlugin::AudioCodecPluginCreator s_creation_hook;
|
||||
|
||||
AudioCodecPlugin::AudioCodecPlugin() = default;
|
||||
AudioCodecPlugin::~AudioCodecPlugin() = default;
|
||||
|
||||
void AudioCodecPlugin::install_creation_hook(Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()> creation_hook)
|
||||
void AudioCodecPlugin::install_creation_hook(AudioCodecPluginCreator creation_hook)
|
||||
{
|
||||
VERIFY(!s_creation_hook);
|
||||
s_creation_hook = move(creation_hook);
|
||||
}
|
||||
|
||||
ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> AudioCodecPlugin::create()
|
||||
ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> AudioCodecPlugin::create(NonnullRefPtr<Audio::Loader> loader)
|
||||
{
|
||||
VERIFY(s_creation_hook);
|
||||
return s_creation_hook();
|
||||
return s_creation_hook(move(loader));
|
||||
}
|
||||
|
||||
ErrorOr<FixedArray<Audio::Sample>> AudioCodecPlugin::read_samples_from_loader(Audio::Loader& loader, size_t samples_to_load, size_t device_sample_rate)
|
||||
{
|
||||
auto buffer_or_error = loader.get_more_samples(samples_to_load);
|
||||
if (buffer_or_error.is_error()) {
|
||||
dbgln("Error while loading samples: {}", buffer_or_error.error().description);
|
||||
return Error::from_string_literal("Error while loading samples");
|
||||
}
|
||||
|
||||
Audio::ResampleHelper<Audio::Sample> resampler(loader.sample_rate(), device_sample_rate);
|
||||
return FixedArray<Audio::Sample>::create(resampler.resample(buffer_or_error.release_value()).span());
|
||||
}
|
||||
|
||||
Duration AudioCodecPlugin::current_loader_position(Audio::Loader const& loader, size_t device_sample_rate)
|
||||
{
|
||||
auto samples_played = static_cast<double>(loader.loaded_samples());
|
||||
auto sample_rate = static_cast<double>(loader.sample_rate());
|
||||
|
||||
auto source_to_device_ratio = sample_rate / static_cast<double>(device_sample_rate);
|
||||
samples_played *= source_to_device_ratio;
|
||||
|
||||
return Duration::from_milliseconds(static_cast<i64>(samples_played / sample_rate * 1000.0));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -9,28 +9,31 @@
|
|||
#include <AK/FixedArray.h>
|
||||
#include <AK/Function.h>
|
||||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <AK/NonnullRefPtr.h>
|
||||
#include <LibAudio/Forward.h>
|
||||
|
||||
namespace Web::Platform {
|
||||
|
||||
class AudioCodecPlugin {
|
||||
public:
|
||||
static void install_creation_hook(Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>()>);
|
||||
static ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> create();
|
||||
using AudioCodecPluginCreator = Function<ErrorOr<NonnullOwnPtr<AudioCodecPlugin>>(NonnullRefPtr<Audio::Loader>)>;
|
||||
|
||||
static void install_creation_hook(AudioCodecPluginCreator);
|
||||
static ErrorOr<NonnullOwnPtr<AudioCodecPlugin>> create(NonnullRefPtr<Audio::Loader>);
|
||||
|
||||
virtual ~AudioCodecPlugin();
|
||||
|
||||
virtual size_t device_sample_rate() = 0;
|
||||
|
||||
virtual void enqueue_samples(FixedArray<Audio::Sample>) = 0;
|
||||
virtual size_t remaining_samples() const = 0;
|
||||
static ErrorOr<FixedArray<Audio::Sample>> read_samples_from_loader(Audio::Loader&, size_t samples_to_load, size_t device_sample_rate);
|
||||
static Duration current_loader_position(Audio::Loader const&, size_t device_sample_rate);
|
||||
|
||||
virtual void resume_playback() = 0;
|
||||
virtual void pause_playback() = 0;
|
||||
virtual void playback_ended() = 0;
|
||||
|
||||
virtual void set_volume(double) = 0;
|
||||
virtual void seek(double) = 0;
|
||||
|
||||
virtual Duration duration() = 0;
|
||||
|
||||
Function<void(Duration)> on_playback_position_updated;
|
||||
|
||||
protected:
|
||||
AudioCodecPlugin();
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue