From c89fd6dff07ae45390fbd64a86b393ab08274e53 Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Mon, 12 Jun 2023 13:52:30 -0400 Subject: [PATCH] LibWeb: Implement the AudioTrack and AudioTrackList interfaces These are used to own and manage the playing of audio data. --- Userland/Libraries/LibWeb/CMakeLists.txt | 2 + Userland/Libraries/LibWeb/Forward.h | 2 + Userland/Libraries/LibWeb/HTML/AudioTrack.cpp | 164 ++++++++++++++++++ Userland/Libraries/LibWeb/HTML/AudioTrack.h | 70 ++++++++ Userland/Libraries/LibWeb/HTML/AudioTrack.idl | 9 + .../Libraries/LibWeb/HTML/AudioTrackList.cpp | 123 +++++++++++++ .../Libraries/LibWeb/HTML/AudioTrackList.h | 57 ++++++ .../Libraries/LibWeb/HTML/AudioTrackList.idl | 15 ++ .../LibWeb/HTML/HTMLMediaElement.cpp | 5 + .../Libraries/LibWeb/HTML/HTMLMediaElement.h | 4 + .../LibWeb/HTML/HTMLMediaElement.idl | 2 + Userland/Libraries/LibWeb/idl_files.cmake | 2 + 12 files changed, 455 insertions(+) create mode 100644 Userland/Libraries/LibWeb/HTML/AudioTrack.cpp create mode 100644 Userland/Libraries/LibWeb/HTML/AudioTrack.h create mode 100644 Userland/Libraries/LibWeb/HTML/AudioTrack.idl create mode 100644 Userland/Libraries/LibWeb/HTML/AudioTrackList.cpp create mode 100644 Userland/Libraries/LibWeb/HTML/AudioTrackList.h create mode 100644 Userland/Libraries/LibWeb/HTML/AudioTrackList.idl diff --git a/Userland/Libraries/LibWeb/CMakeLists.txt b/Userland/Libraries/LibWeb/CMakeLists.txt index 321682d2b1..61105a4bd6 100644 --- a/Userland/Libraries/LibWeb/CMakeLists.txt +++ b/Userland/Libraries/LibWeb/CMakeLists.txt @@ -214,6 +214,8 @@ set(SOURCES Geometry/DOMRectReadOnly.cpp HTML/AnimatedBitmapDecodedImageData.cpp HTML/AttributeNames.cpp + HTML/AudioTrack.cpp + HTML/AudioTrackList.cpp HTML/BrowsingContext.cpp HTML/BrowsingContextGroup.cpp HTML/Canvas/CanvasDrawImage.cpp diff --git a/Userland/Libraries/LibWeb/Forward.h b/Userland/Libraries/LibWeb/Forward.h index b529234407..5a0458fd1b 100644 --- a/Userland/Libraries/LibWeb/Forward.h +++ b/Userland/Libraries/LibWeb/Forward.h @@ -290,6 +290,8 @@ class DOMRectReadOnly; } namespace Web::HTML { +class AudioTrack; +class AudioTrackList; class BrowsingContext; class BrowsingContextGroup; class CanvasRenderingContext2D; diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrack.cpp b/Userland/Libraries/LibWeb/HTML/AudioTrack.cpp new file mode 100644 index 0000000000..c816be4490 --- /dev/null +++ b/Userland/Libraries/LibWeb/HTML/AudioTrack.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Web::HTML { + +static IDAllocator s_audio_track_id_allocator; + +// Number of milliseconds of audio data contained in each audio buffer +static constexpr u32 BUFFER_SIZE_MS = 50; + +AudioTrack::AudioTrack(JS::Realm& realm, JS::NonnullGCPtr media_element, NonnullRefPtr loader) + : PlatformObject(realm) + , m_media_element(media_element) + , m_audio_plugin(Platform::AudioCodecPlugin::create().release_value_but_fixme_should_propagate_errors()) + , m_loader(move(loader)) + , m_sample_timer(Platform::Timer::create_repeating(BUFFER_SIZE_MS, [this]() { + play_next_samples(); + })) +{ + m_audio_plugin->device_sample_rate(); +} + +AudioTrack::~AudioTrack() +{ + auto id = m_id.to_number(); + VERIFY(id.has_value()); + + s_audio_track_id_allocator.deallocate(id.value()); +} + +JS::ThrowCompletionOr AudioTrack::initialize(JS::Realm& realm) +{ + MUST_OR_THROW_OOM(Base::initialize(realm)); + set_prototype(&Bindings::ensure_web_prototype(realm, "AudioTrack")); + + auto id = s_audio_track_id_allocator.allocate(); + m_id = TRY_OR_THROW_OOM(realm.vm(), String::number(id)); + + return {}; +} + +void AudioTrack::play(Badge) +{ + m_audio_plugin->resume_playback(); + m_sample_timer->start(); +} + +void AudioTrack::pause(Badge) +{ + m_audio_plugin->pause_playback(); + m_sample_timer->stop(); +} + +Duration AudioTrack::position() const +{ + auto samples_played = static_cast(m_loader->loaded_samples()); + auto sample_rate = static_cast(m_loader->sample_rate()); + + auto source_to_device_ratio = sample_rate / static_cast(m_audio_plugin->device_sample_rate()); + samples_played *= source_to_device_ratio; + + return Duration::from_milliseconds(static_cast(samples_played / sample_rate * 1000.0)); +} + +Duration AudioTrack::duration() const +{ + auto duration = static_cast(m_loader->total_samples()) / static_cast(m_loader->sample_rate()); + return Duration::from_milliseconds(static_cast(duration * 1000.0)); +} + +void AudioTrack::visit_edges(Cell::Visitor& visitor) +{ + Base::visit_edges(visitor); + visitor.visit(m_media_element); + visitor.visit(m_audio_track_list); +} + +// https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-enabled +void AudioTrack::set_enabled(bool enabled) +{ + // On setting, it must enable the track if the new value is true, and disable it otherwise. (If the track is no + // longer in an AudioTrackList object, then the track being enabled or disabled has no effect beyond changing the + // value of the attribute on the AudioTrack object.) + if (m_enabled == enabled) + return; + + if (m_audio_track_list) { + // Whenever an audio track in an AudioTrackList that was disabled is enabled, and whenever one that was enabled + // is disabled, the user agent must queue a media element task given the media element to fire an event named + // change at the AudioTrackList object. + m_media_element->queue_a_media_element_task([this]() { + m_audio_track_list->dispatch_event(DOM::Event::create(realm(), HTML::EventNames::change).release_value_but_fixme_should_propagate_errors()); + }); + } + + m_enabled = enabled; +} + +Optional> AudioTrack::get_next_samples() +{ + bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples(); + bool audio_server_done = m_audio_plugin->remaining_samples() == 0; + + if (all_samples_loaded && audio_server_done) + return {}; + + auto samples_to_load_per_buffer = static_cast(BUFFER_SIZE_MS / 1000.0f * static_cast(m_loader->sample_rate())); + + auto buffer_or_error = m_loader->get_more_samples(samples_to_load_per_buffer); + if (buffer_or_error.is_error()) { + dbgln("Error while loading samples: {}", buffer_or_error.error().description); + return {}; + } + + return buffer_or_error.release_value(); +} + +void AudioTrack::play_next_samples() +{ + if (auto* layout_node = m_media_element->layout_node()) + layout_node->set_needs_display(); + + auto samples = get_next_samples(); + if (!samples.has_value()) { + m_audio_plugin->playback_ended(); + (void)m_loader->reset(); + + auto playback_position = static_cast(duration().to_milliseconds()) / 1000.0; + m_media_element->set_current_playback_position(playback_position); + + return; + } + + Audio::ResampleHelper resampler(m_loader->sample_rate(), m_audio_plugin->device_sample_rate()); + auto resampled = FixedArray::create(resampler.resample(samples.release_value()).span()).release_value_but_fixme_should_propagate_errors(); + + m_audio_plugin->enqueue_samples(move(resampled)); + + auto playback_position = static_cast(position().to_milliseconds()) / 1000.0; + m_media_element->set_current_playback_position(playback_position); +} + +} diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrack.h b/Userland/Libraries/LibWeb/HTML/AudioTrack.h new file mode 100644 index 0000000000..82efba881a --- /dev/null +++ b/Userland/Libraries/LibWeb/HTML/AudioTrack.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include + +namespace Web::HTML { + +class AudioTrack final : public Bindings::PlatformObject { + WEB_PLATFORM_OBJECT(AudioTrack, Bindings::PlatformObject); + +public: + virtual ~AudioTrack() override; + + void set_audio_track_list(Badge, JS::GCPtr audio_track_list) { m_audio_track_list = audio_track_list; } + + void play(Badge); + void pause(Badge); + + Duration position() const; + Duration duration() const; + + String const& id() const { return m_id; } + String const& kind() const { return m_kind; } + String const& label() const { return m_label; } + String const& language() const { return m_language; } + + bool enabled() const { return m_enabled; } + void set_enabled(bool enabled); + +private: + AudioTrack(JS::Realm&, JS::NonnullGCPtr, NonnullRefPtr); + + virtual JS::ThrowCompletionOr initialize(JS::Realm&) override; + virtual void visit_edges(Cell::Visitor&) override; + + Optional> get_next_samples(); + void play_next_samples(); + + // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-id + String m_id; + + // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-kind + String m_kind; + + // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-label + String m_label; + + // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-language + String m_language; + + // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-enabled + bool m_enabled { false }; + + JS::NonnullGCPtr m_media_element; + JS::GCPtr m_audio_track_list; + + NonnullOwnPtr m_audio_plugin; + NonnullRefPtr m_loader; + NonnullRefPtr m_sample_timer; +}; + +} diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrack.idl b/Userland/Libraries/LibWeb/HTML/AudioTrack.idl new file mode 100644 index 0000000000..88364e6c80 --- /dev/null +++ b/Userland/Libraries/LibWeb/HTML/AudioTrack.idl @@ -0,0 +1,9 @@ +// https://html.spec.whatwg.org/multipage/media.html#audiotrack +[Exposed=Window] +interface AudioTrack { + readonly attribute DOMString id; + readonly attribute DOMString kind; + readonly attribute DOMString label; + readonly attribute DOMString language; + attribute boolean enabled; +}; diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrackList.cpp b/Userland/Libraries/LibWeb/HTML/AudioTrackList.cpp new file mode 100644 index 0000000000..65cb89a998 --- /dev/null +++ b/Userland/Libraries/LibWeb/HTML/AudioTrackList.cpp @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include + +namespace Web::HTML { + +AudioTrackList::AudioTrackList(JS::Realm& realm) + : DOM::EventTarget(realm) + , m_audio_tracks(realm.heap()) +{ +} + +JS::ThrowCompletionOr AudioTrackList::initialize(JS::Realm& realm) +{ + MUST_OR_THROW_OOM(Base::initialize(realm)); + set_prototype(&Bindings::ensure_web_prototype(realm, "AudioTrackList")); + + return {}; +} + +// https://html.spec.whatwg.org/multipage/media.html#dom-tracklist-item +JS::ThrowCompletionOr> AudioTrackList::internal_get_own_property(JS::PropertyKey const& property_name) const +{ + // To determine the value of an indexed property for a given index index in an AudioTrackList or VideoTrackList + // object list, the user agent must return the AudioTrack or VideoTrack object that represents the indexth track + // in list. + if (property_name.is_number()) { + if (auto index = property_name.as_number(); index < m_audio_tracks.size()) { + JS::PropertyDescriptor descriptor; + descriptor.value = m_audio_tracks.at(index); + + return descriptor; + } + } + + return Base::internal_get_own_property(property_name); +} + +ErrorOr AudioTrackList::add_track(Badge, JS::NonnullGCPtr audio_track) +{ + TRY(m_audio_tracks.try_append(audio_track)); + audio_track->set_audio_track_list({}, this); + + return {}; +} + +void AudioTrackList::remove_all_tracks(Badge) +{ + m_audio_tracks.clear(); +} + +// https://html.spec.whatwg.org/multipage/media.html#dom-audiotracklist-gettrackbyid +JS::GCPtr AudioTrackList::get_track_by_id(StringView id) const +{ + // The AudioTrackList getTrackById(id) and VideoTrackList getTrackById(id) methods must return the first AudioTrack + // or VideoTrack object (respectively) in the AudioTrackList or VideoTrackList object (respectively) whose identifier + // is equal to the value of the id argument (in the natural order of the list, as defined above). + auto it = m_audio_tracks.find_if([&](auto const& audio_track) { + return audio_track->id() == id; + }); + + // When no tracks match the given argument, the methods must return null. + if (it == m_audio_tracks.end()) + return nullptr; + + return *it; +} + +bool AudioTrackList::has_enabled_track() const +{ + auto it = m_audio_tracks.find_if([&](auto const& audio_track) { + return audio_track->enabled(); + }); + + return it != m_audio_tracks.end(); +} + +// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onchange +void AudioTrackList::set_onchange(WebIDL::CallbackType* event_handler) +{ + set_event_handler_attribute(HTML::EventNames::change, event_handler); +} + +// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onchange +WebIDL::CallbackType* AudioTrackList::onchange() +{ + return event_handler_attribute(HTML::EventNames::change); +} + +// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onaddtrack +void AudioTrackList::set_onaddtrack(WebIDL::CallbackType* event_handler) +{ + set_event_handler_attribute(HTML::EventNames::addtrack, event_handler); +} + +// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onaddtrack +WebIDL::CallbackType* AudioTrackList::onaddtrack() +{ + return event_handler_attribute(HTML::EventNames::addtrack); +} + +// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onremovetrack +void AudioTrackList::set_onremovetrack(WebIDL::CallbackType* event_handler) +{ + set_event_handler_attribute(HTML::EventNames::removetrack, event_handler); +} + +// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onremovetrack +WebIDL::CallbackType* AudioTrackList::onremovetrack() +{ + return event_handler_attribute(HTML::EventNames::removetrack); +} + +} diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrackList.h b/Userland/Libraries/LibWeb/HTML/AudioTrackList.h new file mode 100644 index 0000000000..9898fb6592 --- /dev/null +++ b/Userland/Libraries/LibWeb/HTML/AudioTrackList.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023, Tim Flynn + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Web::HTML { + +class AudioTrackList final : public DOM::EventTarget { + WEB_PLATFORM_OBJECT(AudioTrackList, DOM::EventTarget); + +public: + ErrorOr add_track(Badge, JS::NonnullGCPtr); + void remove_all_tracks(Badge); + + // https://html.spec.whatwg.org/multipage/media.html#dom-audiotracklist-length + size_t length() const { return m_audio_tracks.size(); } + + JS::GCPtr get_track_by_id(StringView id) const; + bool has_enabled_track() const; + + template + void for_each_enabled_track(Callback&& callback) + { + for (auto& audio_track : m_audio_tracks) { + if (audio_track->enabled()) + callback(*audio_track); + } + } + + void set_onchange(WebIDL::CallbackType*); + WebIDL::CallbackType* onchange(); + + void set_onaddtrack(WebIDL::CallbackType*); + WebIDL::CallbackType* onaddtrack(); + + void set_onremovetrack(WebIDL::CallbackType*); + WebIDL::CallbackType* onremovetrack(); + +private: + explicit AudioTrackList(JS::Realm&); + + virtual JS::ThrowCompletionOr initialize(JS::Realm&) override; + virtual JS::ThrowCompletionOr> internal_get_own_property(JS::PropertyKey const& property_name) const override; + + JS::MarkedVector> m_audio_tracks; +}; + +} diff --git a/Userland/Libraries/LibWeb/HTML/AudioTrackList.idl b/Userland/Libraries/LibWeb/HTML/AudioTrackList.idl new file mode 100644 index 0000000000..633f1a2586 --- /dev/null +++ b/Userland/Libraries/LibWeb/HTML/AudioTrackList.idl @@ -0,0 +1,15 @@ +#import +#import +#import + +// https://html.spec.whatwg.org/multipage/media.html#audiotracklist +[Exposed=Window] +interface AudioTrackList : EventTarget { + readonly attribute unsigned long length; + getter AudioTrack (unsigned long index); + AudioTrack? getTrackById(DOMString id); + + attribute EventHandler onchange; + attribute EventHandler onaddtrack; + attribute EventHandler onremovetrack; +}; diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp index 0a9c93eb92..3b251232ae 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -48,6 +50,7 @@ JS::ThrowCompletionOr HTMLMediaElement::initialize(JS::Realm& realm) MUST_OR_THROW_OOM(Base::initialize(realm)); set_prototype(&Bindings::ensure_web_prototype(realm, "HTMLMediaElement")); + m_audio_tracks = TRY(realm.heap().allocate(realm, realm)); m_video_tracks = TRY(realm.heap().allocate(realm, realm)); m_document_observer = TRY(realm.heap().allocate(realm, realm, document())); @@ -73,6 +76,7 @@ void HTMLMediaElement::visit_edges(Cell::Visitor& visitor) { Base::visit_edges(visitor); visitor.visit(m_error); + visitor.visit(m_audio_tracks); visitor.visit(m_video_tracks); visitor.visit(m_document_observer); visitor.visit(m_source_element_selector); @@ -1122,6 +1126,7 @@ void HTMLMediaElement::forget_media_resource_specific_tracks() // of text tracks all the media-resource-specific text tracks, then empty the media element's audioTracks attribute's AudioTrackList object, then // empty the media element's videoTracks attribute's VideoTrackList object. No events (in particular, no removetrack events) are fired as part of // this; the error and emptied events, fired by the algorithms that invoke this one, can be used instead. + m_audio_tracks->remove_all_tracks({}); m_video_tracks->remove_all_tracks({}); } diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h index 75af877a06..6ecf072bd6 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h @@ -85,6 +85,7 @@ public: WebIDL::ExceptionOr> play(); WebIDL::ExceptionOr pause(); + JS::NonnullGCPtr audio_tracks() const { return *m_audio_tracks; } JS::NonnullGCPtr video_tracks() const { return *m_video_tracks; } void set_layout_mouse_position(Badge, Optional mouse_position) { m_mouse_position = move(mouse_position); } @@ -211,6 +212,9 @@ private: // https://html.spec.whatwg.org/multipage/media.html#dom-media-paused bool m_paused { true }; + // https://html.spec.whatwg.org/multipage/media.html#dom-media-audiotracks + JS::GCPtr m_audio_tracks; + // https://html.spec.whatwg.org/multipage/media.html#dom-media-videotracks JS::GCPtr m_video_tracks; diff --git a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl index 9c65ac5a7e..6f6806ab67 100644 --- a/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl +++ b/Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl @@ -1,3 +1,4 @@ +#import #import #import #import @@ -53,5 +54,6 @@ interface HTMLMediaElement : HTMLElement { [Reflect, CEReactions] attribute boolean controls; // tracks + [SameObject] readonly attribute AudioTrackList audioTracks; [SameObject] readonly attribute VideoTrackList videoTracks; }; diff --git a/Userland/Libraries/LibWeb/idl_files.cmake b/Userland/Libraries/LibWeb/idl_files.cmake index 01e56a75be..75a88e8b81 100644 --- a/Userland/Libraries/LibWeb/idl_files.cmake +++ b/Userland/Libraries/LibWeb/idl_files.cmake @@ -68,6 +68,8 @@ libweb_js_bindings(Geometry/DOMPointReadOnly) libweb_js_bindings(Geometry/DOMRect) libweb_js_bindings(Geometry/DOMRectList) libweb_js_bindings(Geometry/DOMRectReadOnly) +libweb_js_bindings(HTML/AudioTrack) +libweb_js_bindings(HTML/AudioTrackList) libweb_js_bindings(HTML/CanvasGradient) libweb_js_bindings(HTML/CanvasPattern) libweb_js_bindings(HTML/CanvasRenderingContext2D)