LibWeb: Implement the AudioTrack and AudioTrackList interfaces

These are used to own and manage the playing of audio data.

Userland/Libraries/LibWeb/CMakeLists.txt

@@ -214,6 +214,8 @@ set(SOURCES
     Geometry/DOMRectReadOnly.cpp
     HTML/AnimatedBitmapDecodedImageData.cpp
     HTML/AttributeNames.cpp
+    HTML/AudioTrack.cpp
+    HTML/AudioTrackList.cpp
     HTML/BrowsingContext.cpp
     HTML/BrowsingContextGroup.cpp
     HTML/Canvas/CanvasDrawImage.cpp

Userland/Libraries/LibWeb/Forward.h

@@ -290,6 +290,8 @@ class DOMRectReadOnly;
 }
 
 namespace Web::HTML {
+class AudioTrack;
+class AudioTrackList;
 class BrowsingContext;
 class BrowsingContextGroup;
 class CanvasRenderingContext2D;

Userland/Libraries/LibWeb/HTML/AudioTrack.cpp

@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <AK/IDAllocator.h>
+#include <AK/Time.h>
+#include <LibAudio/Loader.h>
+#include <LibAudio/Resampler.h>
+#include <LibAudio/Sample.h>
+#include <LibJS/Runtime/Realm.h>
+#include <LibJS/Runtime/VM.h>
+#include <LibWeb/Bindings/AudioTrackPrototype.h>
+#include <LibWeb/Bindings/Intrinsics.h>
+#include <LibWeb/DOM/Event.h>
+#include <LibWeb/HTML/AudioTrack.h>
+#include <LibWeb/HTML/AudioTrackList.h>
+#include <LibWeb/HTML/EventNames.h>
+#include <LibWeb/HTML/HTMLMediaElement.h>
+#include <LibWeb/Layout/Node.h>
+#include <LibWeb/Platform/AudioCodecPlugin.h>
+#include <LibWeb/Platform/Timer.h>
+
+namespace Web::HTML {
+
+static IDAllocator s_audio_track_id_allocator;
+
+// Number of milliseconds of audio data contained in each audio buffer
+static constexpr u32 BUFFER_SIZE_MS = 50;
+
+AudioTrack::AudioTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> media_element, NonnullRefPtr<Audio::Loader> loader)
+    : PlatformObject(realm)
+    , m_media_element(media_element)
+    , m_audio_plugin(Platform::AudioCodecPlugin::create().release_value_but_fixme_should_propagate_errors())
+    , m_loader(move(loader))
+    , m_sample_timer(Platform::Timer::create_repeating(BUFFER_SIZE_MS, [this]() {
+        play_next_samples();
+    }))
+{
+    m_audio_plugin->device_sample_rate();
+}
+
+AudioTrack::~AudioTrack()
+{
+    auto id = m_id.to_number<int>();
+    VERIFY(id.has_value());
+
+    s_audio_track_id_allocator.deallocate(id.value());
+}
+
+JS::ThrowCompletionOr<void> AudioTrack::initialize(JS::Realm& realm)
+{
+    MUST_OR_THROW_OOM(Base::initialize(realm));
+    set_prototype(&Bindings::ensure_web_prototype<Bindings::AudioTrackPrototype>(realm, "AudioTrack"));
+
+    auto id = s_audio_track_id_allocator.allocate();
+    m_id = TRY_OR_THROW_OOM(realm.vm(), String::number(id));
+
+    return {};
+}
+
+void AudioTrack::play(Badge<HTMLAudioElement>)
+{
+    m_audio_plugin->resume_playback();
+    m_sample_timer->start();
+}
+
+void AudioTrack::pause(Badge<HTMLAudioElement>)
+{
+    m_audio_plugin->pause_playback();
+    m_sample_timer->stop();
+}
+
+Duration AudioTrack::position() const
+{
+    auto samples_played = static_cast<double>(m_loader->loaded_samples());
+    auto sample_rate = static_cast<double>(m_loader->sample_rate());
+
+    auto source_to_device_ratio = sample_rate / static_cast<double>(m_audio_plugin->device_sample_rate());
+    samples_played *= source_to_device_ratio;
+
+    return Duration::from_milliseconds(static_cast<i64>(samples_played / sample_rate * 1000.0));
+}
+
+Duration AudioTrack::duration() const
+{
+    auto duration = static_cast<double>(m_loader->total_samples()) / static_cast<double>(m_loader->sample_rate());
+    return Duration::from_milliseconds(static_cast<i64>(duration * 1000.0));
+}
+
+void AudioTrack::visit_edges(Cell::Visitor& visitor)
+{
+    Base::visit_edges(visitor);
+    visitor.visit(m_media_element);
+    visitor.visit(m_audio_track_list);
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-enabled
+void AudioTrack::set_enabled(bool enabled)
+{
+    // On setting, it must enable the track if the new value is true, and disable it otherwise. (If the track is no
+    // longer in an AudioTrackList object, then the track being enabled or disabled has no effect beyond changing the
+    // value of the attribute on the AudioTrack object.)
+    if (m_enabled == enabled)
+        return;
+
+    if (m_audio_track_list) {
+        // Whenever an audio track in an AudioTrackList that was disabled is enabled, and whenever one that was enabled
+        // is disabled, the user agent must queue a media element task given the media element to fire an event named
+        // change at the AudioTrackList object.
+        m_media_element->queue_a_media_element_task([this]() {
+            m_audio_track_list->dispatch_event(DOM::Event::create(realm(), HTML::EventNames::change).release_value_but_fixme_should_propagate_errors());
+        });
+    }
+
+    m_enabled = enabled;
+}
+
+Optional<FixedArray<Audio::Sample>> AudioTrack::get_next_samples()
+{
+    bool all_samples_loaded = m_loader->loaded_samples() >= m_loader->total_samples();
+    bool audio_server_done = m_audio_plugin->remaining_samples() == 0;
+
+    if (all_samples_loaded && audio_server_done)
+        return {};
+
+    auto samples_to_load_per_buffer = static_cast<size_t>(BUFFER_SIZE_MS / 1000.0f * static_cast<float>(m_loader->sample_rate()));
+
+    auto buffer_or_error = m_loader->get_more_samples(samples_to_load_per_buffer);
+    if (buffer_or_error.is_error()) {
+        dbgln("Error while loading samples: {}", buffer_or_error.error().description);
+        return {};
+    }
+
+    return buffer_or_error.release_value();
+}
+
+void AudioTrack::play_next_samples()
+{
+    if (auto* layout_node = m_media_element->layout_node())
+        layout_node->set_needs_display();
+
+    auto samples = get_next_samples();
+    if (!samples.has_value()) {
+        m_audio_plugin->playback_ended();
+        (void)m_loader->reset();
+
+        auto playback_position = static_cast<double>(duration().to_milliseconds()) / 1000.0;
+        m_media_element->set_current_playback_position(playback_position);
+
+        return;
+    }
+
+    Audio::ResampleHelper<Audio::Sample> resampler(m_loader->sample_rate(), m_audio_plugin->device_sample_rate());
+    auto resampled = FixedArray<Audio::Sample>::create(resampler.resample(samples.release_value()).span()).release_value_but_fixme_should_propagate_errors();
+
+    m_audio_plugin->enqueue_samples(move(resampled));
+
+    auto playback_position = static_cast<double>(position().to_milliseconds()) / 1000.0;
+    m_media_element->set_current_playback_position(playback_position);
+}
+
+}

Userland/Libraries/LibWeb/HTML/AudioTrack.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/String.h>
+#include <AK/Time.h>
+#include <LibAudio/Forward.h>
+#include <LibWeb/Bindings/PlatformObject.h>
+
+namespace Web::HTML {
+
+class AudioTrack final : public Bindings::PlatformObject {
+    WEB_PLATFORM_OBJECT(AudioTrack, Bindings::PlatformObject);
+
+public:
+    virtual ~AudioTrack() override;
+
+    void set_audio_track_list(Badge<AudioTrackList>, JS::GCPtr<AudioTrackList> audio_track_list) { m_audio_track_list = audio_track_list; }
+
+    void play(Badge<HTMLAudioElement>);
+    void pause(Badge<HTMLAudioElement>);
+
+    Duration position() const;
+    Duration duration() const;
+
+    String const& id() const { return m_id; }
+    String const& kind() const { return m_kind; }
+    String const& label() const { return m_label; }
+    String const& language() const { return m_language; }
+
+    bool enabled() const { return m_enabled; }
+    void set_enabled(bool enabled);
+
+private:
+    AudioTrack(JS::Realm&, JS::NonnullGCPtr<HTMLMediaElement>, NonnullRefPtr<Audio::Loader>);
+
+    virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override;
+    virtual void visit_edges(Cell::Visitor&) override;
+
+    Optional<FixedArray<Audio::Sample>> get_next_samples();
+    void play_next_samples();
+
+    // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-id
+    String m_id;
+
+    // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-kind
+    String m_kind;
+
+    // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-label
+    String m_label;
+
+    // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-language
+    String m_language;
+
+    // https://html.spec.whatwg.org/multipage/media.html#dom-audiotrack-enabled
+    bool m_enabled { false };
+
+    JS::NonnullGCPtr<HTMLMediaElement> m_media_element;
+    JS::GCPtr<AudioTrackList> m_audio_track_list;
+
+    NonnullOwnPtr<Platform::AudioCodecPlugin> m_audio_plugin;
+    NonnullRefPtr<Audio::Loader> m_loader;
+    NonnullRefPtr<Platform::Timer> m_sample_timer;
+};
+
+}

Userland/Libraries/LibWeb/HTML/AudioTrack.idl

@@ -0,0 +1,9 @@
+// https://html.spec.whatwg.org/multipage/media.html#audiotrack
+[Exposed=Window]
+interface AudioTrack {
+    readonly attribute DOMString id;
+    readonly attribute DOMString kind;
+    readonly attribute DOMString label;
+    readonly attribute DOMString language;
+    attribute boolean enabled;
+};

Userland/Libraries/LibWeb/HTML/AudioTrackList.cpp

@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibJS/Runtime/Realm.h>
+#include <LibJS/Runtime/VM.h>
+#include <LibWeb/Bindings/AudioTrackListPrototype.h>
+#include <LibWeb/Bindings/Intrinsics.h>
+#include <LibWeb/HTML/AudioTrackList.h>
+#include <LibWeb/HTML/EventNames.h>
+
+namespace Web::HTML {
+
+AudioTrackList::AudioTrackList(JS::Realm& realm)
+    : DOM::EventTarget(realm)
+    , m_audio_tracks(realm.heap())
+{
+}
+
+JS::ThrowCompletionOr<void> AudioTrackList::initialize(JS::Realm& realm)
+{
+    MUST_OR_THROW_OOM(Base::initialize(realm));
+    set_prototype(&Bindings::ensure_web_prototype<Bindings::AudioTrackListPrototype>(realm, "AudioTrackList"));
+
+    return {};
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#dom-tracklist-item
+JS::ThrowCompletionOr<Optional<JS::PropertyDescriptor>> AudioTrackList::internal_get_own_property(JS::PropertyKey const& property_name) const
+{
+    // To determine the value of an indexed property for a given index index in an AudioTrackList or VideoTrackList
+    // object list, the user agent must return the AudioTrack or VideoTrack object that represents the indexth track
+    // in list.
+    if (property_name.is_number()) {
+        if (auto index = property_name.as_number(); index < m_audio_tracks.size()) {
+            JS::PropertyDescriptor descriptor;
+            descriptor.value = m_audio_tracks.at(index);
+
+            return descriptor;
+        }
+    }
+
+    return Base::internal_get_own_property(property_name);
+}
+
+ErrorOr<void> AudioTrackList::add_track(Badge<HTMLMediaElement>, JS::NonnullGCPtr<AudioTrack> audio_track)
+{
+    TRY(m_audio_tracks.try_append(audio_track));
+    audio_track->set_audio_track_list({}, this);
+
+    return {};
+}
+
+void AudioTrackList::remove_all_tracks(Badge<HTMLMediaElement>)
+{
+    m_audio_tracks.clear();
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#dom-audiotracklist-gettrackbyid
+JS::GCPtr<AudioTrack> AudioTrackList::get_track_by_id(StringView id) const
+{
+    // The AudioTrackList getTrackById(id) and VideoTrackList getTrackById(id) methods must return the first AudioTrack
+    // or VideoTrack object (respectively) in the AudioTrackList or VideoTrackList object (respectively) whose identifier
+    // is equal to the value of the id argument (in the natural order of the list, as defined above).
+    auto it = m_audio_tracks.find_if([&](auto const& audio_track) {
+        return audio_track->id() == id;
+    });
+
+    // When no tracks match the given argument, the methods must return null.
+    if (it == m_audio_tracks.end())
+        return nullptr;
+
+    return *it;
+}
+
+bool AudioTrackList::has_enabled_track() const
+{
+    auto it = m_audio_tracks.find_if([&](auto const& audio_track) {
+        return audio_track->enabled();
+    });
+
+    return it != m_audio_tracks.end();
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onchange
+void AudioTrackList::set_onchange(WebIDL::CallbackType* event_handler)
+{
+    set_event_handler_attribute(HTML::EventNames::change, event_handler);
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onchange
+WebIDL::CallbackType* AudioTrackList::onchange()
+{
+    return event_handler_attribute(HTML::EventNames::change);
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onaddtrack
+void AudioTrackList::set_onaddtrack(WebIDL::CallbackType* event_handler)
+{
+    set_event_handler_attribute(HTML::EventNames::addtrack, event_handler);
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onaddtrack
+WebIDL::CallbackType* AudioTrackList::onaddtrack()
+{
+    return event_handler_attribute(HTML::EventNames::addtrack);
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onremovetrack
+void AudioTrackList::set_onremovetrack(WebIDL::CallbackType* event_handler)
+{
+    set_event_handler_attribute(HTML::EventNames::removetrack, event_handler);
+}
+
+// https://html.spec.whatwg.org/multipage/media.html#handler-tracklist-onremovetrack
+WebIDL::CallbackType* AudioTrackList::onremovetrack()
+{
+    return event_handler_attribute(HTML::EventNames::removetrack);
+}
+
+}

Userland/Libraries/LibWeb/HTML/AudioTrackList.h

@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Badge.h>
+#include <AK/String.h>
+#include <LibJS/Heap/MarkedVector.h>
+#include <LibWeb/DOM/EventTarget.h>
+#include <LibWeb/HTML/AudioTrack.h>
+
+namespace Web::HTML {
+
+class AudioTrackList final : public DOM::EventTarget {
+    WEB_PLATFORM_OBJECT(AudioTrackList, DOM::EventTarget);
+
+public:
+    ErrorOr<void> add_track(Badge<HTMLMediaElement>, JS::NonnullGCPtr<AudioTrack>);
+    void remove_all_tracks(Badge<HTMLMediaElement>);
+
+    // https://html.spec.whatwg.org/multipage/media.html#dom-audiotracklist-length
+    size_t length() const { return m_audio_tracks.size(); }
+
+    JS::GCPtr<AudioTrack> get_track_by_id(StringView id) const;
+    bool has_enabled_track() const;
+
+    template<typename Callback>
+    void for_each_enabled_track(Callback&& callback)
+    {
+        for (auto& audio_track : m_audio_tracks) {
+            if (audio_track->enabled())
+                callback(*audio_track);
+        }
+    }
+
+    void set_onchange(WebIDL::CallbackType*);
+    WebIDL::CallbackType* onchange();
+
+    void set_onaddtrack(WebIDL::CallbackType*);
+    WebIDL::CallbackType* onaddtrack();
+
+    void set_onremovetrack(WebIDL::CallbackType*);
+    WebIDL::CallbackType* onremovetrack();
+
+private:
+    explicit AudioTrackList(JS::Realm&);
+
+    virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override;
+    virtual JS::ThrowCompletionOr<Optional<JS::PropertyDescriptor>> internal_get_own_property(JS::PropertyKey const& property_name) const override;
+
+    JS::MarkedVector<JS::NonnullGCPtr<AudioTrack>> m_audio_tracks;
+};
+
+}

Userland/Libraries/LibWeb/HTML/AudioTrackList.idl

@@ -0,0 +1,15 @@
+#import <DOM/EventHandler.idl>
+#import <DOM/EventTarget.idl>
+#import <HTML/AudioTrack.idl>
+
+// https://html.spec.whatwg.org/multipage/media.html#audiotracklist
+[Exposed=Window]
+interface AudioTrackList : EventTarget {
+    readonly attribute unsigned long length;
+    getter AudioTrack (unsigned long index);
+    AudioTrack? getTrackById(DOMString id);
+
+    attribute EventHandler onchange;
+    attribute EventHandler onaddtrack;
+    attribute EventHandler onremovetrack;
+};

Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp

@@ -17,6 +17,8 @@
 #include <LibWeb/Fetch/Infrastructure/FetchController.h>
 #include <LibWeb/Fetch/Infrastructure/HTTP/Requests.h>
 #include <LibWeb/Fetch/Infrastructure/HTTP/Responses.h>
+#include <LibWeb/HTML/AudioTrack.h>
+#include <LibWeb/HTML/AudioTrackList.h>
 #include <LibWeb/HTML/CORSSettingAttribute.h>
 #include <LibWeb/HTML/HTMLAudioElement.h>
 #include <LibWeb/HTML/HTMLMediaElement.h>
@@ -48,6 +50,7 @@ JS::ThrowCompletionOr<void> HTMLMediaElement::initialize(JS::Realm& realm)
     MUST_OR_THROW_OOM(Base::initialize(realm));
     set_prototype(&Bindings::ensure_web_prototype<Bindings::HTMLMediaElementPrototype>(realm, "HTMLMediaElement"));
 
+    m_audio_tracks = TRY(realm.heap().allocate<AudioTrackList>(realm, realm));
     m_video_tracks = TRY(realm.heap().allocate<VideoTrackList>(realm, realm));
     m_document_observer = TRY(realm.heap().allocate<DOM::DocumentObserver>(realm, realm, document()));
 
@@ -73,6 +76,7 @@ void HTMLMediaElement::visit_edges(Cell::Visitor& visitor)
 {
     Base::visit_edges(visitor);
     visitor.visit(m_error);
+    visitor.visit(m_audio_tracks);
     visitor.visit(m_video_tracks);
     visitor.visit(m_document_observer);
     visitor.visit(m_source_element_selector);
@@ -1122,6 +1126,7 @@ void HTMLMediaElement::forget_media_resource_specific_tracks()
     // of text tracks all the media-resource-specific text tracks, then empty the media element's audioTracks attribute's AudioTrackList object, then
     // empty the media element's videoTracks attribute's VideoTrackList object. No events (in particular, no removetrack events) are fired as part of
     // this; the error and emptied events, fired by the algorithms that invoke this one, can be used instead.
+    m_audio_tracks->remove_all_tracks({});
     m_video_tracks->remove_all_tracks({});
 }
 

Userland/Libraries/LibWeb/HTML/HTMLMediaElement.h

@@ -85,6 +85,7 @@ public:
     WebIDL::ExceptionOr<JS::NonnullGCPtr<JS::Promise>> play();
     WebIDL::ExceptionOr<void> pause();
 
+    JS::NonnullGCPtr<AudioTrackList> audio_tracks() const { return *m_audio_tracks; }
     JS::NonnullGCPtr<VideoTrackList> video_tracks() const { return *m_video_tracks; }
 
     void set_layout_mouse_position(Badge<Painting::MediaPaintable>, Optional<CSSPixelPoint> mouse_position) { m_mouse_position = move(mouse_position); }
@@ -211,6 +212,9 @@ private:
     // https://html.spec.whatwg.org/multipage/media.html#dom-media-paused
     bool m_paused { true };
 
+    // https://html.spec.whatwg.org/multipage/media.html#dom-media-audiotracks
+    JS::GCPtr<AudioTrackList> m_audio_tracks;
+
     // https://html.spec.whatwg.org/multipage/media.html#dom-media-videotracks
     JS::GCPtr<VideoTrackList> m_video_tracks;
 

Userland/Libraries/LibWeb/HTML/HTMLMediaElement.idl

@@ -1,3 +1,4 @@
+#import <HTML/AudioTrackList.idl>
 #import <HTML/HTMLElement.idl>
 #import <HTML/MediaError.idl>
 #import <HTML/TimeRanges.idl>
@@ -53,5 +54,6 @@ interface HTMLMediaElement : HTMLElement {
     [Reflect, CEReactions] attribute boolean controls;
 
     // tracks
+    [SameObject] readonly attribute AudioTrackList audioTracks;
     [SameObject] readonly attribute VideoTrackList videoTracks;
 };

Userland/Libraries/LibWeb/idl_files.cmake

@@ -68,6 +68,8 @@ libweb_js_bindings(Geometry/DOMPointReadOnly)
 libweb_js_bindings(Geometry/DOMRect)
 libweb_js_bindings(Geometry/DOMRectList)
 libweb_js_bindings(Geometry/DOMRectReadOnly)
+libweb_js_bindings(HTML/AudioTrack)
+libweb_js_bindings(HTML/AudioTrackList)
 libweb_js_bindings(HTML/CanvasGradient)
 libweb_js_bindings(HTML/CanvasPattern)
 libweb_js_bindings(HTML/CanvasRenderingContext2D)