LibWeb: Create a basic layout node for HTMLVideoElement

Userland/Libraries/LibWeb/CMakeLists.txt

@@ -405,6 +405,7 @@ set(SOURCES
     Layout/TableWrapper.cpp
     Layout/TextNode.cpp
     Layout/TreeBuilder.cpp
+    Layout/VideoBox.cpp
     Loader/ContentFilter.cpp
     Loader/FileRequest.cpp
     Loader/FrameLoader.cpp
@@ -446,6 +447,7 @@ set(SOURCES
     Painting/ShadowPainting.cpp
     Painting/StackingContext.cpp
     Painting/TextPaintable.cpp
+    Painting/VideoPaintable.cpp
     PerformanceTimeline/EntryTypes.cpp
     PerformanceTimeline/PerformanceEntry.cpp
     Platform/EventLoopPlugin.cpp

Userland/Libraries/LibWeb/Forward.h

@@ -394,6 +394,7 @@ class PaintableBox;
 class PaintableWithLines;
 class StackingContext;
 class TextPaintable;
+class VideoPaintable;
 struct BorderRadiusData;
 struct BorderRadiiData;
 struct LinearGradientData;
@@ -491,6 +492,7 @@ class RadioButton;
 class ReplacedBox;
 class TableWrapper;
 class TextNode;
+class VideoBox;
 }
 
 namespace Web {

Userland/Libraries/LibWeb/HTML/HTMLVideoElement.cpp

@@ -4,11 +4,19 @@
  * SPDX-License-Identifier: BSD-2-Clause
  */
 
+#include <LibGfx/Bitmap.h>
 #include <LibWeb/Bindings/Intrinsics.h>
+#include <LibWeb/DOM/Document.h>
 #include <LibWeb/HTML/HTMLVideoElement.h>
+#include <LibWeb/HTML/VideoTrack.h>
+#include <LibWeb/Layout/VideoBox.h>
+#include <LibWeb/Platform/Timer.h>
 
 namespace Web::HTML {
 
+// FIXME: Determine a reasonable framerate somehow. For now, this is roughly 24fps.
+static constexpr int s_frame_delay_ms = 42;
+
 HTMLVideoElement::HTMLVideoElement(DOM::Document& document, DOM::QualifiedName qualified_name)
     : HTMLMediaElement(document, move(qualified_name))
 {
@@ -24,6 +32,27 @@ JS::ThrowCompletionOr<void> HTMLVideoElement::initialize(JS::Realm& realm)
     return {};
 }
 
+void HTMLVideoElement::visit_edges(Cell::Visitor& visitor)
+{
+    Base::visit_edges(visitor);
+    visitor.visit(m_video_track);
+}
+
+JS::GCPtr<Layout::Node> HTMLVideoElement::create_layout_node(NonnullRefPtr<CSS::StyleProperties> style)
+{
+    return heap().allocate_without_realm<Layout::VideoBox>(document(), *this, move(style));
+}
+
+Layout::VideoBox* HTMLVideoElement::layout_node()
+{
+    return static_cast<Layout::VideoBox*>(Node::layout_node());
+}
+
+Layout::VideoBox const* HTMLVideoElement::layout_node() const
+{
+    return static_cast<Layout::VideoBox const*>(Node::layout_node());
+}
+
 // https://html.spec.whatwg.org/multipage/media.html#dom-video-videowidth
 u32 HTMLVideoElement::video_width() const
 {
@@ -46,4 +75,30 @@ u32 HTMLVideoElement::video_height() const
     return m_video_height;
 }
 
+void HTMLVideoElement::set_video_track(JS::GCPtr<HTML::VideoTrack> video_track)
+{
+    set_needs_style_update(true);
+    document().set_needs_layout();
+
+    if (m_video_timer)
+        m_video_timer->stop();
+
+    m_video_track = video_track;
+    if (!m_video_track)
+        return;
+
+    if (!m_video_timer) {
+        m_video_timer = Platform::Timer::create_repeating(s_frame_delay_ms, [this]() {
+            if (auto frame = m_video_track->next_frame())
+                m_current_frame = move(frame);
+            else
+                m_video_timer->stop();
+
+            layout_node()->set_needs_display();
+        });
+    }
+
+    m_video_timer->start();
+}
+
 }

Userland/Libraries/LibWeb/HTML/HTMLVideoElement.h

@@ -6,6 +6,8 @@
 
 #pragma once
 
+#include <LibGfx/Forward.h>
+#include <LibWeb/Forward.h>
 #include <LibWeb/HTML/HTMLMediaElement.h>
 
 namespace Web::HTML {
@@ -16,16 +18,29 @@ class HTMLVideoElement final : public HTMLMediaElement {
 public:
     virtual ~HTMLVideoElement() override;
 
+    Layout::VideoBox* layout_node();
+    Layout::VideoBox const* layout_node() const;
+
     void set_video_width(u32 video_width) { m_video_width = video_width; }
     u32 video_width() const;
 
     void set_video_height(u32 video_height) { m_video_height = video_height; }
     u32 video_height() const;
 
+    void set_video_track(JS::GCPtr<VideoTrack>);
+    RefPtr<Gfx::Bitmap> const& current_frame() const { return m_current_frame; }
+
 private:
     HTMLVideoElement(DOM::Document&, DOM::QualifiedName);
 
     virtual JS::ThrowCompletionOr<void> initialize(JS::Realm&) override;
+    virtual void visit_edges(Cell::Visitor&) override;
+
+    virtual JS::GCPtr<Layout::Node> create_layout_node(NonnullRefPtr<CSS::StyleProperties>) override;
+
+    JS::GCPtr<HTML::VideoTrack> m_video_track;
+    RefPtr<Platform::Timer> m_video_timer;
+    RefPtr<Gfx::Bitmap> m_current_frame;
 
     u32 m_video_width { 0 };
     u32 m_video_height { 0 };

Userland/Libraries/LibWeb/HTML/VideoTrack.cpp

@@ -5,6 +5,7 @@
  */
 
 #include <AK/IDAllocator.h>
+#include <LibGfx/Bitmap.h>
 #include <LibJS/Runtime/Realm.h>
 #include <LibJS/Runtime/VM.h>
 #include <LibWeb/Bindings/Intrinsics.h>
@@ -12,6 +13,7 @@
 #include <LibWeb/DOM/Event.h>
 #include <LibWeb/HTML/EventNames.h>
 #include <LibWeb/HTML/HTMLMediaElement.h>
+#include <LibWeb/HTML/HTMLVideoElement.h>
 #include <LibWeb/HTML/VideoTrack.h>
 #include <LibWeb/HTML/VideoTrackList.h>
 
@@ -52,6 +54,65 @@ void VideoTrack::visit_edges(Cell::Visitor& visitor)
     visitor.visit(m_video_track_list);
 }
 
+RefPtr<Gfx::Bitmap> VideoTrack::next_frame()
+{
+    auto frame_sample = m_demuxer->get_next_video_sample_for_track(m_track);
+    if (frame_sample.is_error()) {
+        if (frame_sample.error().category() != Video::DecoderErrorCategory::EndOfStream)
+            dbgln("VideoTrack: Error getting next video sample: {}", frame_sample.error().description());
+        return {};
+    }
+
+    OwnPtr<Video::VideoFrame> decoded_frame;
+
+    while (!decoded_frame) {
+        auto result = m_decoder.receive_sample(frame_sample.value()->data());
+        if (result.is_error()) {
+            dbgln("VideoTrack: Error receiving video sample data: {}", frame_sample.error().description());
+            return {};
+        }
+
+        while (true) {
+            auto frame_result = m_decoder.get_decoded_frame();
+            if (frame_result.is_error()) {
+                if (frame_result.error().category() == Video::DecoderErrorCategory::NeedsMoreInput)
+                    break;
+
+                dbgln("VideoTrack: Error decoding video frame: {}", frame_result.error().description());
+                return {};
+            }
+
+            decoded_frame = frame_result.release_value();
+            VERIFY(decoded_frame);
+        }
+    }
+
+    auto& cicp = decoded_frame->cicp();
+    cicp.adopt_specified_values(frame_sample.value()->container_cicp());
+    cicp.default_code_points_if_unspecified({ Video::ColorPrimaries::BT709, Video::TransferCharacteristics::BT709, Video::MatrixCoefficients::BT709, Video::VideoFullRangeFlag::Studio });
+
+    // BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
+    // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
+    // handling those as sRGB instead, which causes no transfer function change in the output,
+    // unless display color management is later implemented.
+    switch (cicp.transfer_characteristics()) {
+    case Video::TransferCharacteristics::BT601:
+    case Video::TransferCharacteristics::BT709:
+    case Video::TransferCharacteristics::BT2020BitDepth10:
+    case Video::TransferCharacteristics::BT2020BitDepth12:
+        cicp.set_transfer_characteristics(Video::TransferCharacteristics::SRGB);
+        break;
+    default:
+        break;
+    }
+
+    auto bitmap = decoded_frame->to_bitmap();
+    if (bitmap.is_error())
+        return {};
+
+    return bitmap.release_value();
+}
+
 // https://html.spec.whatwg.org/multipage/media.html#dom-videotrack-selected
 void VideoTrack::set_selected(bool selected)
 {
@@ -83,6 +144,12 @@ void VideoTrack::set_selected(bool selected)
     }
 
     m_selected = selected;
+
+    // AD-HOC: Inform the video element node that we have (un)selected a video track for layout.
+    if (is<HTMLVideoElement>(*m_media_element)) {
+        auto& video_element = verify_cast<HTMLVideoElement>(*m_media_element);
+        video_element.set_video_track(m_selected ? this : nullptr);
+    }
 }
 
 }

Userland/Libraries/LibWeb/HTML/VideoTrack.h

@@ -8,8 +8,10 @@
 
 #include <AK/String.h>
 #include <AK/Time.h>
+#include <LibGfx/Forward.h>
 #include <LibVideo/Containers/Matroska/MatroskaDemuxer.h>
 #include <LibVideo/Track.h>
+#include <LibVideo/VP9/Decoder.h>
 #include <LibWeb/Bindings/PlatformObject.h>
 
 namespace Web::HTML {
@@ -22,6 +24,8 @@ public:
 
     void set_video_track_list(Badge<VideoTrackList>, JS::GCPtr<VideoTrackList> video_track_list) { m_video_track_list = video_track_list; }
 
+    RefPtr<Gfx::Bitmap> next_frame();
+
     Time duration() const { return m_track.video_data().duration; }
     u64 pixel_width() const { return m_track.video_data().pixel_width; }
     u64 pixel_height() const { return m_track.video_data().pixel_height; }
@@ -59,6 +63,7 @@ private:
     JS::GCPtr<VideoTrackList> m_video_track_list;
 
     NonnullOwnPtr<Video::Matroska::MatroskaDemuxer> m_demuxer;
+    Video::VP9::Decoder m_decoder;
     Video::Track m_track;
 };
 

Userland/Libraries/LibWeb/Layout/VideoBox.cpp

@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibWeb/HTML/HTMLVideoElement.h>
+#include <LibWeb/Layout/VideoBox.h>
+#include <LibWeb/Painting/VideoPaintable.h>
+
+namespace Web::Layout {
+
+VideoBox::VideoBox(DOM::Document& document, DOM::Element& element, NonnullRefPtr<CSS::StyleProperties> style)
+    : ReplacedBox(document, element, move(style))
+{
+    browsing_context().register_viewport_client(*this);
+}
+
+void VideoBox::finalize()
+{
+    Base::finalize();
+
+    // NOTE: We unregister from the browsing context in finalize() to avoid trouble
+    //       in the scenario where our BrowsingContext has already been swept by GC.
+    browsing_context().unregister_viewport_client(*this);
+}
+
+HTML::HTMLVideoElement& VideoBox::dom_node()
+{
+    return static_cast<HTML::HTMLVideoElement&>(ReplacedBox::dom_node());
+}
+
+HTML::HTMLVideoElement const& VideoBox::dom_node() const
+{
+    return static_cast<HTML::HTMLVideoElement const&>(ReplacedBox::dom_node());
+}
+
+int VideoBox::preferred_width() const
+{
+    return dom_node().attribute(HTML::AttributeNames::width).to_int().value_or(dom_node().video_width());
+}
+
+int VideoBox::preferred_height() const
+{
+    return dom_node().attribute(HTML::AttributeNames::height).to_int().value_or(dom_node().video_height());
+}
+
+void VideoBox::prepare_for_replaced_layout()
+{
+    auto width = static_cast<float>(dom_node().video_width());
+    set_intrinsic_width(width);
+
+    auto height = static_cast<float>(dom_node().video_height());
+    set_intrinsic_height(height);
+
+    if (width != 0 && height != 0)
+        set_intrinsic_aspect_ratio(width / height);
+    else
+        set_intrinsic_aspect_ratio({});
+}
+
+void VideoBox::browsing_context_did_set_viewport_rect(CSSPixelRect const&)
+{
+    // FIXME: Several steps in HTMLMediaElement indicate we may optionally handle whether the media object
+    //        is in view. Implement those steps.
+}
+
+JS::GCPtr<Painting::Paintable> VideoBox::create_paintable() const
+{
+    return Painting::VideoPaintable::create(*this);
+}
+
+}

Userland/Libraries/LibWeb/Layout/VideoBox.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibWeb/Forward.h>
+#include <LibWeb/HTML/BrowsingContext.h>
+#include <LibWeb/Layout/ReplacedBox.h>
+
+namespace Web::Layout {
+
+class VideoBox final
+    : public ReplacedBox
+    , public HTML::BrowsingContext::ViewportClient {
+    JS_CELL(VideoBox, ReplacedBox);
+
+public:
+    virtual void prepare_for_replaced_layout() override;
+
+    HTML::HTMLVideoElement& dom_node();
+    HTML::HTMLVideoElement const& dom_node() const;
+
+    virtual JS::GCPtr<Painting::Paintable> create_paintable() const override;
+
+private:
+    VideoBox(DOM::Document&, DOM::Element&, NonnullRefPtr<CSS::StyleProperties>);
+
+    // ^BrowsingContext::ViewportClient
+    virtual void browsing_context_did_set_viewport_rect(CSSPixelRect const&) final;
+
+    // ^JS::Cell
+    virtual void finalize() override;
+
+    int preferred_width() const;
+    int preferred_height() const;
+};
+
+}

Userland/Libraries/LibWeb/Painting/VideoPaintable.cpp

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibWeb/HTML/HTMLVideoElement.h>
+#include <LibWeb/Layout/VideoBox.h>
+#include <LibWeb/Painting/BorderRadiusCornerClipper.h>
+#include <LibWeb/Painting/VideoPaintable.h>
+
+namespace Web::Painting {
+
+JS::NonnullGCPtr<VideoPaintable> VideoPaintable::create(Layout::VideoBox const& layout_box)
+{
+    return layout_box.heap().allocate_without_realm<VideoPaintable>(layout_box);
+}
+
+VideoPaintable::VideoPaintable(Layout::VideoBox const& layout_box)
+    : PaintableBox(layout_box)
+{
+}
+
+Layout::VideoBox const& VideoPaintable::layout_box() const
+{
+    return static_cast<Layout::VideoBox const&>(layout_node());
+}
+
+void VideoPaintable::paint(PaintContext& context, PaintPhase phase) const
+{
+    if (!is_visible())
+        return;
+
+    // FIXME: This should be done at a different level.
+    if (is_out_of_view(context))
+        return;
+
+    PaintableBox::paint(context, phase);
+
+    if (phase != PaintPhase::Foreground)
+        return;
+
+    if (auto const& bitmap = layout_box().dom_node().current_frame()) {
+        auto image_rect = context.rounded_device_rect(absolute_rect());
+        ScopedCornerRadiusClip corner_clip { context, context.painter(), image_rect, normalized_border_radii_data(ShrinkRadiiForBorders::Yes) };
+        context.painter().draw_scaled_bitmap(image_rect.to_type<int>(), *bitmap, bitmap->rect(), 1.0f, to_gfx_scaling_mode(computed_values().image_rendering()));
+    }
+}
+
+}

Userland/Libraries/LibWeb/Painting/VideoPaintable.h

@@ -0,0 +1,28 @@
+/*
+ * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <LibWeb/Forward.h>
+#include <LibWeb/Painting/PaintableBox.h>
+
+namespace Web::Painting {
+
+class VideoPaintable final : public PaintableBox {
+    JS_CELL(VideoPaintable, PaintableBox);
+
+public:
+    static JS::NonnullGCPtr<VideoPaintable> create(Layout::VideoBox const&);
+
+    virtual void paint(PaintContext&, PaintPhase) const override;
+
+    Layout::VideoBox const& layout_box() const;
+
+private:
+    VideoPaintable(Layout::VideoBox const&);
+};
+
+}