LibWeb: Port HTMLVideoElement to play videos with Video::PlaybackManager

This has several advantages over the current manual demuxing currently
being performed. PlaybackManager hides the specific demuxer being used,
which will allow more codecs to be added transparently to LibWeb. It
also provides buffering and controls playback rate for us.

Further, it will allow us to much more easily implement the "media
timeline" to render a timestamp and implement seeking.

Userland/Libraries/LibWeb/HTML/HTMLMediaElement.cpp

@@ -6,7 +6,7 @@
  */
 
 #include <LibJS/Runtime/Promise.h>
-#include <LibVideo/Containers/Matroska/MatroskaDemuxer.h>
+#include <LibVideo/PlaybackManager.h>
 #include <LibWeb/Bindings/HTMLMediaElementPrototype.h>
 #include <LibWeb/Bindings/Intrinsics.h>
 #include <LibWeb/DOM/Document.h>
@@ -26,10 +26,31 @@
 #include <LibWeb/HTML/VideoTrack.h>
 #include <LibWeb/HTML/VideoTrackList.h>
 #include <LibWeb/MimeSniff/MimeType.h>
+#include <LibWeb/Platform/Timer.h>
 #include <LibWeb/WebIDL/Promise.h>
 
 namespace Web::HTML {
 
+class MediaElementPlaybackTimer final : public Video::PlaybackTimer {
+public:
+    static ErrorOr<NonnullOwnPtr<MediaElementPlaybackTimer>> create(int interval_ms, Function<void()> timeout_handler)
+    {
+        auto timer = Platform::Timer::create_single_shot(interval_ms, move(timeout_handler));
+        return adopt_nonnull_own_or_enomem(new (nothrow) MediaElementPlaybackTimer(move(timer)));
+    }
+
+    virtual void start() override { m_timer->start(); }
+    virtual void start(int interval_ms) override { m_timer->start(interval_ms); }
+
+private:
+    explicit MediaElementPlaybackTimer(NonnullRefPtr<Platform::Timer> timer)
+        : m_timer(move(timer))
+    {
+    }
+
+    NonnullRefPtr<Platform::Timer> m_timer;
+};
+
 HTMLMediaElement::HTMLMediaElement(DOM::Document& document, DOM::QualifiedName qualified_name)
     : HTMLElement(document, move(qualified_name))
     , m_pending_play_promises(heap())
@@ -579,10 +600,13 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void()>
     auto& realm = this->realm();
     auto& vm = realm.vm();
 
+    auto playback_manager = Video::PlaybackManager::from_data(m_media_data, [](auto interval_ms, auto timeout_handler) {
+        return MediaElementPlaybackTimer::create(interval_ms, move(timeout_handler));
+    });
+
     // -> If the media data cannot be fetched at all, due to network errors, causing the user agent to give up trying to fetch the resource
     // -> If the media data can be fetched but is found by inspection to be in an unsupported format, or can otherwise not be rendered at all
-    auto demuxer = Video::Matroska::MatroskaDemuxer::from_data(m_media_data);
+    if (playback_manager.is_error()) {
-    if (demuxer.is_error()) {
         // 1. The user agent should cancel the fetching process.
         m_fetch_controller->terminate();
 
@@ -595,7 +619,7 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void()>
     JS::GCPtr<VideoTrack> video_track;
 
     // -> If the media resource is found to have an audio track
-    if (auto audio_tracks = demuxer.value()->get_tracks_for_type(Video::TrackType::Audio); !audio_tracks.is_error() && !audio_tracks.value().is_empty()) {
+    {
         // FIXME: 1. Create an AudioTrack object to represent the audio track.
         // FIXME: 2. Update the media element's audioTracks attribute's AudioTrackList object with the new AudioTrack object.
         // FIXME: 3. Let enable be unknown.
@@ -609,9 +633,10 @@ WebIDL::ExceptionOr<void> HTMLMediaElement::process_media_data(Function<void()>
     }
 
     // -> If the media resource is found to have a video track
-    if (auto video_tracks = demuxer.value()->get_tracks_for_type(Video::TrackType::Video); !video_tracks.is_error() && !video_tracks.value().is_empty()) {
+    // NOTE: Creating a Video::PlaybackManager above will have failed if there was not a video track.
+    {
         // 1. Create a VideoTrack object to represent the video track.
-        video_track = TRY(vm.heap().allocate<VideoTrack>(realm, realm, *this, demuxer.release_value(), video_tracks.value()[0]));
+        video_track = TRY(vm.heap().allocate<VideoTrack>(realm, realm, *this, playback_manager.release_value()));
 
         // 2. Update the media element's videoTracks attribute's VideoTrackList object with the new VideoTrack object.
         TRY_OR_THROW_OOM(vm, m_video_tracks->add_track({}, *video_track));

Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp

@@ -11,13 +11,9 @@
 #include <LibWeb/HTML/HTMLVideoElement.h>
 #include <LibWeb/HTML/VideoTrack.h>
 #include <LibWeb/Layout/VideoBox.h>
-#include <LibWeb/Platform/Timer.h>
 
 namespace Web::HTML {
 
-// FIXME: Determine a reasonable framerate somehow. For now, this is roughly 24fps.
-static constexpr int s_frame_delay_ms = 42;
-
 HTMLVideoElement::HTMLVideoElement(DOM::Document& document, DOM::QualifiedName qualified_name)
     : HTMLMediaElement(document, move(qualified_name))
 {
@@ -81,32 +77,28 @@ void HTMLVideoElement::set_video_track(JS::GCPtr<HTML::VideoTrack> video_track)
     set_needs_style_update(true);
     document().set_needs_layout();
 
-    if (m_video_timer)
+    if (m_video_track)
-        m_video_timer->stop();
+        m_video_track->pause_video({});
 
     m_video_track = video_track;
 }
 
+void HTMLVideoElement::set_current_frame(Badge<VideoTrack>, RefPtr<Gfx::Bitmap> frame)
+{
+    m_current_frame = move(frame);
+    layout_node()->set_needs_display();
+}
+
 void HTMLVideoElement::on_playing()
 {
-    if (!m_video_timer) {
+    if (m_video_track)
-        m_video_timer = Platform::Timer::create_repeating(s_frame_delay_ms, [this]() {
+        m_video_track->play_video({});
-            if (auto frame = m_video_track->next_frame())
-                m_current_frame = move(frame);
-            else
-                m_video_timer->stop();
-
-            layout_node()->set_needs_display();
-        });
-    }
-
-    m_video_timer->start();
 }
 
 void HTMLVideoElement::on_paused()
 {
-    if (m_video_timer)
+    if (m_video_track)
-        m_video_timer->stop();
+        m_video_track->pause_video({});
 }
 
 }

Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h

@@ -28,6 +28,8 @@ public:
     u32 video_height() const;
 
     void set_video_track(JS::GCPtr<VideoTrack>);
+
+    void set_current_frame(Badge<VideoTrack>, RefPtr<Gfx::Bitmap> frame);
     RefPtr<Gfx::Bitmap> const& current_frame() const { return m_current_frame; }
 
 private:
@@ -42,7 +44,6 @@ private:
     virtual void on_paused() override;
 
     JS::GCPtr<HTML::VideoTrack> m_video_track;
-    RefPtr<Platform::Timer> m_video_timer;
     RefPtr<Gfx::Bitmap> m_current_frame;
 
     u32 m_video_width { 0 };

Userland/Libraries/LibWeb/HTML/VideoTrack.cpp

@@ -8,6 +8,8 @@
 #include <LibGfx/Bitmap.h>
 #include <LibJS/Runtime/Realm.h>
 #include <LibJS/Runtime/VM.h>
+#include <LibVideo/PlaybackManager.h>
+#include <LibVideo/Track.h>
 #include <LibWeb/Bindings/Intrinsics.h>
 #include <LibWeb/Bindings/VideoTrackPrototype.h>
 #include <LibWeb/DOM/Event.h>
@@ -21,12 +23,23 @@ namespace Web::HTML {
 
 static IDAllocator s_video_track_id_allocator;
 
-VideoTrack::VideoTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> media_element, NonnullOwnPtr<Video::Matroska::MatroskaDemuxer> demuxer, Video::Track track)
+VideoTrack::VideoTrack(JS::Realm& realm, JS::NonnullGCPtr<HTMLMediaElement> media_element, NonnullOwnPtr<Video::PlaybackManager> playback_manager)
     : PlatformObject(realm)
     , m_media_element(media_element)
-    , m_demuxer(move(demuxer))
+    , m_playback_manager(move(playback_manager))
-    , m_track(track)
 {
+    m_playback_manager->on_video_frame = [this](auto frame) {
+        if (is<HTMLVideoElement>(*m_media_element))
+            verify_cast<HTMLVideoElement>(*m_media_element).set_current_frame({}, move(frame));
+    };
+
+    m_playback_manager->on_decoder_error = [](auto) {
+        // FIXME: Propagate this error to HTMLMediaElement's error attribute.
+    };
+
+    m_playback_manager->on_fatal_playback_error = [](auto) {
+        // FIXME: Propagate this error to HTMLMediaElement's error attribute.
+    };
 }
 
 VideoTrack::~VideoTrack()
@@ -54,63 +67,29 @@ void VideoTrack::visit_edges(Cell::Visitor& visitor)
     visitor.visit(m_video_track_list);
 }
 
-RefPtr<Gfx::Bitmap> VideoTrack::next_frame()
+void VideoTrack::play_video(Badge<HTMLVideoElement>)
 {
-    auto frame_sample = m_demuxer->get_next_video_sample_for_track(m_track);
+    m_playback_manager->resume_playback();
-    if (frame_sample.is_error()) {
+}
-        if (frame_sample.error().category() != Video::DecoderErrorCategory::EndOfStream)
-            dbgln("VideoTrack: Error getting next video sample: {}", frame_sample.error().description());
-        return {};
-    }
 
-    OwnPtr<Video::VideoFrame> decoded_frame;
+void VideoTrack::pause_video(Badge<HTMLVideoElement>)
+{
+    m_playback_manager->pause_playback();
+}
 
-    while (!decoded_frame) {
+Time VideoTrack::duration() const
-        auto result = m_decoder.receive_sample(frame_sample.value()->data());
+{
-        if (result.is_error()) {
+    return m_playback_manager->selected_video_track().video_data().duration;
-            dbgln("VideoTrack: Error receiving video sample data: {}", result.error().description());
+}
-            return {};
-        }
 
-        while (true) {
+u64 VideoTrack::pixel_width() const
-            auto frame_result = m_decoder.get_decoded_frame();
+{
-            if (frame_result.is_error()) {
+    return m_playback_manager->selected_video_track().video_data().pixel_width;
-                if (frame_result.error().category() == Video::DecoderErrorCategory::NeedsMoreInput)
+}
-                    break;
 
-                dbgln("VideoTrack: Error decoding video frame: {}", frame_result.error().description());
+u64 VideoTrack::pixel_height() const
-                return {};
+{
-            }
+    return m_playback_manager->selected_video_track().video_data().pixel_height;
-
-            decoded_frame = frame_result.release_value();
-            VERIFY(decoded_frame);
-        }
-    }
-
-    auto& cicp = decoded_frame->cicp();
-    cicp.adopt_specified_values(frame_sample.value()->container_cicp());
-    cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::VideoFullRangeFlag::Studio });
-
-    // BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
-    // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
-    // handling those as sRGB instead, which causes no transfer function change in the output,
-    // unless display color management is later implemented.
-    switch (cicp.transfer_characteristics()) {
-    case Video::TransferCharacteristics::BT601:
-    case Video::TransferCharacteristics::BT709:
-    case Video::TransferCharacteristics::BT2020BitDepth10:
-    case Video::TransferCharacteristics::BT2020BitDepth12:
-        cicp.set_transfer_characteristics(Video::TransferCharacteristics::SRGB);
-        break;
-    default:
-        break;
-    }
-
-    auto bitmap = decoded_frame->to_bitmap();
-    if (bitmap.is_error())
-        return {};
-
-    return bitmap.release_value();
 }
 
 // https://html.spec.whatwg.org/multipage/media.html#dom-videotrack-selected

Userland/Libraries/LibWeb/HTML/VideoTrack.h

@@ -9,9 +9,7 @@
 #include <AK/String.h>
 #include <AK/Time.h>
 #include <LibGfx/Forward.h>
-#include <LibVideo/Containers/Matroska/MatroskaDemuxer.h>
+#include <LibVideo/Forward.h>
-#include <LibVideo/Track.h>
-#include <LibVideo/VP9/Decoder.h>
 #include <LibWeb/Bindings/PlatformObject.h>
 
 namespace Web::HTML {
@@ -24,11 +22,12 @@ public:
 
     void set_video_track_list(Badge<VideoTrackList>, JS::GCPtr<VideoTrackList> video_track_list) { m_video_track_list = video_track_list; }
 
-    RefPtr<Gfx::Bitmap> next_frame();
+    void play_video(Badge<HTMLVideoElement>);
+    void pause_video(Badge<HTMLVideoElement>);
 
-    Time duration() const { return m_track.video_data().duration; }
+    Time duration() const;
-    u64 pixel_width() const { return m_track.video_data().pixel_width; }
+    u64 pixel_width() const;
-    u64 pixel_height() const { return m_track.video_data().pixel_height; }
+    u64 pixel_height() const;
 
     String const& id() const { return m_id; }
     String const& kind() const { return m_kind; }
@@ -39,7 +38,7 @@ public:
     void set_selected(bool selected);
 
 private:
-    explicit VideoTrack(JS::Realm&, JS::NonnullGCPtr<HTMLMediaElement>, NonnullOwnPtr<Video::Matroska::MatroskaDemuxer>, Video::Track);
+    VideoTrack(JS::Realm&, JS::NonnullGCPtr<HTMLMediaElement>, NonnullOwnPtr<Video::PlaybackManager>);
 
     virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override;
     virtual void visit_edges(Cell::Visitor&) override;
@@ -62,9 +61,7 @@ private:
     JS::NonnullGCPtr<HTMLMediaElement> m_media_element;
     JS::GCPtr<VideoTrackList> m_video_track_list;
 
-    NonnullOwnPtr<Video::Matroska::MatroskaDemuxer> m_demuxer;
+    NonnullOwnPtr<Video::PlaybackManager> m_playback_manager;
-    Video::VP9::Decoder m_decoder;
-    Video::Track m_track;
 };
 
 }