LibWeb: Create a basic layout node for HTMLVideoElement

Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp

@@ -4,11 +4,19 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <LibGfx/Bitmap.h>
 #include <LibWeb/Bindings/Intrinsics.h>
+#include <LibWeb/DOM/Document.h>
 #include <LibWeb/HTML/HTMLVideoElement.h>
+#include <LibWeb/HTML/VideoTrack.h>
+#include <LibWeb/Layout/VideoBox.h>
+#include <LibWeb/Platform/Timer.h>
 
 namespace Web::HTML {
 
+// FIXME: Determine a reasonable framerate somehow. For now, this is roughly 24fps.
+static constexpr int s_frame_delay_ms = 42;
+
 HTMLVideoElement::HTMLVideoElement(DOM::Document& document, DOM::QualifiedName qualified_name)
     : HTMLMediaElement(document, move(qualified_name))
 {
@@ -24,6 +32,27 @@ JS::ThrowCompletionOr<void> HTMLVideoElement::initialize(JS::Realm& realm)
     return {};
 }
 
+void HTMLVideoElement::visit_edges(Cell::Visitor& visitor)
+{
+    Base::visit_edges(visitor);
+    visitor.visit(m_video_track);
+}
+
+JS::GCPtr<Layout::Node> HTMLVideoElement::create_layout_node(NonnullRefPtr<CSS::StyleProperties> style)
+{
+    return heap().allocate_without_realm<Layout::VideoBox>(document(), *this, move(style));
+}
+
+Layout::VideoBox* HTMLVideoElement::layout_node()
+{
+    return static_cast<Layout::VideoBox*>(Node::layout_node());
+}
+
+Layout::VideoBox const* HTMLVideoElement::layout_node() const
+{
+    return static_cast<Layout::VideoBox const*>(Node::layout_node());
+}
+
 // https://html.spec.whatwg.org/multipage/media.html#dom-video-videowidth
 u32 HTMLVideoElement::video_width() const
 {
@@ -46,4 +75,30 @@ u32 HTMLVideoElement::video_height() const
     return m_video_height;
 }
 
+void HTMLVideoElement::set_video_track(JS::GCPtr<HTML::VideoTrack> video_track)
+{
+    set_needs_style_update(true);
+    document().set_needs_layout();
+
+    if (m_video_timer)
+        m_video_timer->stop();
+
+    m_video_track = video_track;
+    if (!m_video_track)
+        return;
+
+    if (!m_video_timer) {
+        m_video_timer = Platform::Timer::create_repeating(s_frame_delay_ms, [this]() {
+            if (auto frame = m_video_track->next_frame())
+                m_current_frame = move(frame);
+            else
+                m_video_timer->stop();
+
+            layout_node()->set_needs_display();
+        });
+    }
+
+    m_video_timer->start();
+}
+
 }

Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h

@@ -6,6 +6,8 @@
 
 #pragma once
 
+#include <LibGfx/Forward.h>
+#include <LibWeb/Forward.h>
 #include <LibWeb/HTML/HTMLMediaElement.h>
 
 namespace Web::HTML {
@@ -16,16 +18,29 @@ class HTMLVideoElement final : public HTMLMediaElement {
 public:
     virtual ~HTMLVideoElement() override;
 
+    Layout::VideoBox* layout_node();
+    Layout::VideoBox const* layout_node() const;
+
     void set_video_width(u32 video_width) { m_video_width = video_width; }
     u32 video_width() const;
 
     void set_video_height(u32 video_height) { m_video_height = video_height; }
     u32 video_height() const;
 
+    void set_video_track(JS::GCPtr<VideoTrack>);
+    RefPtr<Gfx::Bitmap> const& current_frame() const { return m_current_frame; }
+
 private:
     HTMLVideoElement(DOM::Document&, DOM::QualifiedName);
 
     virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override;
+    virtual void visit_edges(Cell::Visitor&) override;
+
+    virtual JS::GCPtr<Layout::Node> create_layout_node(NonnullRefPtr<CSS::StyleProperties>) override;
+
+    JS::GCPtr<HTML::VideoTrack> m_video_track;
+    RefPtr<Platform::Timer> m_video_timer;
+    RefPtr<Gfx::Bitmap> m_current_frame;
 
     u32 m_video_width { 0 };
     u32 m_video_height { 0 };

Userland/Libraries/LibWeb/HTML/VideoTrack.cpp

@@ -5,6 +5,7 @@
  */
 
 #include <AK/IDAllocator.h>
+#include <LibGfx/Bitmap.h>
 #include <LibJS/Runtime/Realm.h>
 #include <LibJS/Runtime/VM.h>
 #include <LibWeb/Bindings/Intrinsics.h>
@@ -12,6 +13,7 @@
 #include <LibWeb/DOM/Event.h>
 #include <LibWeb/HTML/EventNames.h>
 #include <LibWeb/HTML/HTMLMediaElement.h>
+#include <LibWeb/HTML/HTMLVideoElement.h>
 #include <LibWeb/HTML/VideoTrack.h>
 #include <LibWeb/HTML/VideoTrackList.h>
 
@@ -52,6 +54,65 @@ void VideoTrack::visit_edges(Cell::Visitor& visitor)
     visitor.visit(m_video_track_list);
 }
 
+RefPtr<Gfx::Bitmap> VideoTrack::next_frame()
+{
+    auto frame_sample = m_demuxer->get_next_video_sample_for_track(m_track);
+    if (frame_sample.is_error()) {
+        if (frame_sample.error().category() != Video::DecoderErrorCategory::EndOfStream)
+            dbgln("VideoTrack: Error getting next video sample: {}", frame_sample.error().description());
+        return {};
+    }
+
+    OwnPtr<Video::VideoFrame> decoded_frame;
+
+    while (!decoded_frame) {
+        auto result = m_decoder.receive_sample(frame_sample.value()->data());
+        if (result.is_error()) {
+            dbgln("VideoTrack: Error receiving video sample data: {}", frame_sample.error().description());
+            return {};
+        }
+
+        while (true) {
+            auto frame_result = m_decoder.get_decoded_frame();
+            if (frame_result.is_error()) {
+                if (frame_result.error().category() == Video::DecoderErrorCategory::NeedsMoreInput)
+                    break;
+
+                dbgln("VideoTrack: Error decoding video frame: {}", frame_result.error().description());
+                return {};
+            }
+
+            decoded_frame = frame_result.release_value();
+            VERIFY(decoded_frame);
+        }
+    }
+
+    auto& cicp = decoded_frame->cicp();
+    cicp.adopt_specified_values(frame_sample.value()->container_cicp());
+    cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::VideoFullRangeFlag::Studio });
+
+    // BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
+    // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
+    // handling those as sRGB instead, which causes no transfer function change in the output,
+    // unless display color management is later implemented.
+    switch (cicp.transfer_characteristics()) {
+    case Video::TransferCharacteristics::BT601:
+    case Video::TransferCharacteristics::BT709:
+    case Video::TransferCharacteristics::BT2020BitDepth10:
+    case Video::TransferCharacteristics::BT2020BitDepth12:
+        cicp.set_transfer_characteristics(Video::TransferCharacteristics::SRGB);
+        break;
+    default:
+        break;
+    }
+
+    auto bitmap = decoded_frame->to_bitmap();
+    if (bitmap.is_error())
+        return {};
+
+    return bitmap.release_value();
+}
+
 // https://html.spec.whatwg.org/multipage/media.html#dom-videotrack-selected
 void VideoTrack::set_selected(bool selected)
 {
@@ -83,6 +144,12 @@ void VideoTrack::set_selected(bool selected)
     }
 
     m_selected = selected;
+
+    // AD-HOC: Inform the video element node that we have (un)selected a video track for layout.
+    if (is<HTMLVideoElement>(*m_media_element)) {
+        auto& video_element = verify_cast<HTMLVideoElement>(*m_media_element);
+        video_element.set_video_track(m_selected ? this : nullptr);
+    }
 }
 
 }

Userland/Libraries/LibWeb/HTML/VideoTrack.h

@@ -8,8 +8,10 @@
 
 #include <AK/String.h>
 #include <AK/Time.h>
+#include <LibGfx/Forward.h>
 #include <LibVideo/Containers/Matroska/MatroskaDemuxer.h>
 #include <LibVideo/Track.h>
+#include <LibVideo/VP9/Decoder.h>
 #include <LibWeb/Bindings/PlatformObject.h>
 
 namespace Web::HTML {
@@ -22,6 +24,8 @@ public:
 
     void set_video_track_list(Badge<VideoTrackList>, JS::GCPtr<VideoTrackList> video_track_list) { m_video_track_list = video_track_list; }
 
+    RefPtr<Gfx::Bitmap> next_frame();
+
     Time duration() const { return m_track.video_data().duration; }
     u64 pixel_width() const { return m_track.video_data().pixel_width; }
     u64 pixel_height() const { return m_track.video_data().pixel_height; }
@@ -59,6 +63,7 @@ private:
     JS::GCPtr<VideoTrackList> m_video_track_list;
 
     NonnullOwnPtr<Video::Matroska::MatroskaDemuxer> m_demuxer;
+    Video::VP9::Decoder m_decoder;
     Video::Track m_track;
 };