LibWeb: Share decoded images at the Resource level :^)

This patch adds ImageResource as a subclass of Resource. This new class also keeps a Gfx::ImageDecoder so that we can share decoded bitmaps between all clients of an image resource inside LibWeb. With this, we now share both encoded and decoded data for images. :^) I had to change how the purgeable-volatile flag is updated to keep the volatile-images-outside-the-visible-viewport optimization working. HTMLImageElement now inherits from ImageResourceClient (a subclass of ResourceClient with additional image-specific stuff) and informs its ImageResource about whether it's inside the viewport or outside. This is pretty awesome! :^)
2025-10-31 18:22:45 +00:00 · 2020-06-02 20:27:26 +02:00 · 2020-06-02 20:27:26 +02:00 · d4ddb0013c
commit d4ddb0013c
parent 1c6e4e04a8
14 changed files with 200 additions and 30 deletions
--- a/Base/home/anon/www/many-buggies.html
+++ b/Base/home/anon/www/many-buggies.html
@ -0,0 +1,9 @@
+<html>
+    <body>
+<img src=http://serenityos.org/buggie.png>
+<img src=http://serenityos.org/buggie.png>
+<img src=http://serenityos.org/buggie.png>
+<img src=http://serenityos.org/buggie.png>
+<img src=http://serenityos.org/buggie.png>
+    </body>
+</html>
--- a/Base/home/anon/www/welcome.html
+++ b/Base/home/anon/www/welcome.html
@ -28,6 +28,7 @@ span#ua {
    <p>Your user agent is: <b><span id="ua"></span></b></p>
    <p>Some small test pages:</p>
    <ul>
+        <li><a href="many.html">many buggies</a></li>
        <li><a href="palette.html">system palette color css extension</a></li>
        <li><a href="inline-block-link.html">link inside display: inline-block</a></li>
        <li><a href="set-interval.html">setInterval() test</a></li>
--- a/Libraries/LibWeb/CMakeLists.txt
+++ b/Libraries/LibWeb/CMakeLists.txt
@ -83,6 +83,7 @@ set(SOURCES
    Layout/LayoutWidget.cpp
    Layout/LineBox.cpp
    Layout/LineBoxFragment.cpp
+    Loader/ImageResource.cpp
    Loader/Resource.cpp
    Loader/ResourceLoader.cpp
    PageView.cpp
--- a/Libraries/LibWeb/CSS/StyleValue.cpp
+++ b/Libraries/LibWeb/CSS/StyleValue.cpp
@ -293,7 +293,7 @@ ImageStyleValue::ImageStyleValue(const URL& url, Document& document)
 {
    LoadRequest request;
    request.set_url(url);
-    set_resource(ResourceLoader::the().load_resource(request));
+    set_resource(ResourceLoader::the().load_resource(Resource::Type::Generic, request));
 }

 void ImageStyleValue::resource_did_load()
--- a/Libraries/LibWeb/DOM/HTMLImageElement.cpp
+++ b/Libraries/LibWeb/DOM/HTMLImageElement.cpp
@ -56,7 +56,7 @@ void HTMLImageElement::load_image(const String& src)
 {
    LoadRequest request;
    request.set_url(document().complete_url(src));
-    set_resource(ResourceLoader::the().load_resource(request));
+    set_resource(ResourceLoader::the().load_resource(Resource::Type::Image, request));
 }

 void HTMLImageElement::resource_did_load()
@ -70,7 +70,8 @@ void HTMLImageElement::resource_did_load()

    dbg() << "HTMLImageElement: Resource did load, encoded data looks tasty: " << this->src();

-    m_image_decoder = Gfx::ImageDecoder::create(resource()->encoded_data());
+    ASSERT(!m_image_decoder);
+    m_image_decoder = resource()->ensure_decoder();

    if (m_image_decoder->is_animated() && m_image_decoder->frame_count() > 1) {
        const auto& first_frame = m_image_decoder->frame(0);
@ -91,6 +92,11 @@ void HTMLImageElement::resource_did_fail()
    dispatch_event(Event::create("error"));
 }

+void HTMLImageElement::resource_did_replace_decoder()
+{
+    m_image_decoder = resource()->ensure_decoder();
+}
+
 void HTMLImageElement::animate()
 {
    if (!layout_node()) {
@ -161,19 +167,17 @@ const Gfx::Bitmap* HTMLImageElement::bitmap() const
    return m_image_decoder->bitmap();
 }

-void HTMLImageElement::set_volatile(Badge<LayoutDocument>, bool v)
+void HTMLImageElement::set_visible_in_viewport(Badge<LayoutDocument>, bool visible_in_viewport)
 {
-    if (!m_image_decoder)
+    if (m_visible_in_viewport == visible_in_viewport)
        return;
-    if (v) {
-        m_image_decoder->set_volatile();
-        return;
-    }
-    bool has_image = m_image_decoder->set_nonvolatile();
-    if (has_image)
-        return;
-    ASSERT(resource());
-    m_image_decoder = Gfx::ImageDecoder::create(resource()->encoded_data());
+    m_visible_in_viewport = visible_in_viewport;
+
+    // FIXME: Don't update volatility every time. If we're here, we're probably scanning through
+    //        the whole document, updating "is visible in viewport" flags, and this could lead
+    //        to the same bitmap being marked volatile back and forth unnecessarily.
+    if (resource())
+        resource()->update_volatility();
 }

 }
--- a/Libraries/LibWeb/DOM/HTMLImageElement.h
+++ b/Libraries/LibWeb/DOM/HTMLImageElement.h
@ -30,7 +30,7 @@
 #include <LibCore/Forward.h>
 #include <LibGfx/Forward.h>
 #include <LibWeb/DOM/HTMLElement.h>
-#include <LibWeb/Loader/Resource.h>
+#include <LibWeb/Loader/ImageResource.h>

 namespace Web {

@ -38,7 +38,7 @@ class LayoutDocument;

 class HTMLImageElement final
    : public HTMLElement
-    , public ResourceClient {
+    , public ImageResourceClient {
 public:
    using WrapperType = Bindings::HTMLImageElementWrapper;

@ -55,11 +55,14 @@ public:
    const Gfx::Bitmap* bitmap() const;
    const Gfx::ImageDecoder* image_decoder() const { return m_image_decoder; }

-    void set_volatile(Badge<LayoutDocument>, bool);
+    void set_visible_in_viewport(Badge<LayoutDocument>, bool);

 private:
+    // ^ImageResource
    virtual void resource_did_load() override;
    virtual void resource_did_fail() override;
+    virtual void resource_did_replace_decoder() override;
+    virtual bool is_visible_in_viewport() const override { return m_visible_in_viewport; }

    void load_image(const String& src);

@ -72,6 +75,8 @@ private:
    size_t m_current_frame_index { 0 };
    size_t m_loops_completed { 0 };
    NonnullRefPtr<Core::Timer> m_timer;
+
+    bool m_visible_in_viewport { false };
 };

 template<>
--- a/Libraries/LibWeb/DOM/HTMLLinkElement.cpp
+++ b/Libraries/LibWeb/DOM/HTMLLinkElement.cpp
@ -76,7 +76,7 @@ void HTMLLinkElement::load_stylesheet(const URL& url)
 {
    LoadRequest request;
    request.set_url(url);
-    set_resource(ResourceLoader::the().load_resource(request));
+    set_resource(ResourceLoader::the().load_resource(Resource::Type::Generic, request));
 }

 }
--- a/Libraries/LibWeb/Layout/LayoutDocument.cpp
+++ b/Libraries/LibWeb/Layout/LayoutDocument.cpp
@ -63,7 +63,7 @@ void LayoutDocument::did_set_viewport_rect(Badge<Frame>, const Gfx::Rect& a_view
 {
    Gfx::FloatRect viewport_rect(a_viewport_rect.x(), a_viewport_rect.y(), a_viewport_rect.width(), a_viewport_rect.height());
    for_each_in_subtree_of_type<LayoutImage>([&](auto& layout_image) {
-        const_cast<HTMLImageElement&>(layout_image.node()).set_volatile({}, !viewport_rect.intersects(layout_image.rect()));
+        const_cast<HTMLImageElement&>(layout_image.node()).set_visible_in_viewport({}, viewport_rect.intersects(layout_image.rect()));
        return IterationDecision::Continue;
    });
 }
--- a/Libraries/LibWeb/Loader/ImageResource.cpp
+++ b/Libraries/LibWeb/Loader/ImageResource.cpp
@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <AK/Function.h>
+#include <LibGfx/Bitmap.h>
+#include <LibGfx/ImageDecoder.h>
+#include <LibWeb/Loader/ImageResource.h>
+
+namespace Web {
+
+ImageResource::ImageResource(const LoadRequest& request)
+    : Resource(Type::Image, request)
+{
+}
+
+ImageResource::~ImageResource()
+{
+}
+
+Gfx::ImageDecoder& ImageResource::ensure_decoder()
+{
+    if (!m_decoder)
+        m_decoder = Gfx::ImageDecoder::create(encoded_data());
+    return *m_decoder;
+}
+
+void ImageResource::update_volatility()
+{
+    if (!m_decoder)
+        return;
+
+    bool visible_in_viewport = false;
+    for_each_client([&](auto& client) {
+        if (static_cast<const ImageResourceClient&>(client).is_visible_in_viewport())
+            visible_in_viewport = true;
+    });
+
+    if (!visible_in_viewport) {
+        m_decoder->set_volatile();
+        return;
+    }
+
+    bool still_has_decoded_image = m_decoder->set_nonvolatile();
+    if (still_has_decoded_image)
+        return;
+
+    m_decoder = nullptr;
+    for_each_client([&](auto& client) {
+        static_cast<ImageResourceClient&>(client).resource_did_replace_decoder();
+    });
+}
+
+ImageResourceClient::~ImageResourceClient()
+{
+}
+
+}
--- a/Libraries/LibWeb/Loader/ImageResource.h
+++ b/Libraries/LibWeb/Loader/ImageResource.h
@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2020, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <LibWeb/Loader/Resource.h>
+
+namespace Web {
+
+class ImageResource final : public Resource {
+    friend class Resource;
+
+public:
+    virtual ~ImageResource() override;
+    Gfx::ImageDecoder& ensure_decoder();
+
+    void update_volatility();
+
+private:
+    explicit ImageResource(const LoadRequest&);
+    RefPtr<Gfx::ImageDecoder> m_decoder;
+};
+
+class ImageResourceClient : public ResourceClient {
+public:
+    virtual ~ImageResourceClient();
+
+    virtual bool is_visible_in_viewport() const { return false; }
+    virtual void resource_did_replace_decoder() {}
+
+protected:
+    ImageResource* resource() { return static_cast<ImageResource*>(ResourceClient::resource()); }
+    const ImageResource* resource() const { return static_cast<const ImageResource*>(ResourceClient::resource()); }
+};
+
+}
--- a/Libraries/LibWeb/Loader/Resource.cpp
+++ b/Libraries/LibWeb/Loader/Resource.cpp
@ -30,13 +30,16 @@

 namespace Web {

-NonnullRefPtr<Resource> Resource::create(Badge<ResourceLoader>, const LoadRequest& request)
+NonnullRefPtr<Resource> Resource::create(Badge<ResourceLoader>, Type type, const LoadRequest& request)
 {
-    return adopt(*new Resource(request));
+    if (type == Type::Image)
+        return adopt(*new ImageResource(request));
+    return adopt(*new Resource(type, request));
 }

-Resource::Resource(const LoadRequest& request)
+Resource::Resource(Type type, const LoadRequest& request)
    : m_request(request)
+    , m_type(type)
 {
 }

--- a/Libraries/LibWeb/Loader/Resource.h
+++ b/Libraries/LibWeb/Loader/Resource.h
@ -34,6 +34,7 @@
 #include <AK/URL.h>
 #include <AK/WeakPtr.h>
 #include <AK/Weakable.h>
+#include <LibGfx/Forward.h>
 #include <LibWeb/Forward.h>
 #include <LibWeb/Loader/LoadRequest.h>

@ -46,8 +47,13 @@ class Resource : public RefCounted<Resource> {
    AK_MAKE_NONMOVABLE(Resource);

 public:
-    static NonnullRefPtr<Resource> create(Badge<ResourceLoader>, const LoadRequest&);
-    ~Resource();
+    enum class Type {
+        Generic,
+        Image,
+    };
+
+    static NonnullRefPtr<Resource> create(Badge<ResourceLoader>, Type, const LoadRequest&);
+    virtual ~Resource();

    bool is_loaded() const { return m_loaded; }

@ -67,11 +73,13 @@ public:
    void did_load(Badge<ResourceLoader>, const ByteBuffer& data, const HashMap<String, String, CaseInsensitiveStringTraits>& headers);
    void did_fail(Badge<ResourceLoader>, const String& error);

-private:
-    explicit Resource(const LoadRequest&);
+protected:
+    explicit Resource(Type, const LoadRequest&);

+private:
    LoadRequest m_request;
    ByteBuffer m_encoded_data;
+    Type m_type { Type::Generic };
    bool m_loaded { false };
    bool m_failed { false };
    String m_error;
--- a/Libraries/LibWeb/Loader/ResourceLoader.cpp
+++ b/Libraries/LibWeb/Loader/ResourceLoader.cpp
@ -72,7 +72,7 @@ void ResourceLoader::load_sync(const URL& url, Function<void(const ByteBuffer&,

 static HashMap<LoadRequest, NonnullRefPtr<Resource>> s_resource_cache;

-RefPtr<Resource> ResourceLoader::load_resource(const LoadRequest& request)
+RefPtr<Resource> ResourceLoader::load_resource(Resource::Type type, const LoadRequest& request)
 {
    if (!request.is_valid())
        return nullptr;
@ -83,7 +83,7 @@ RefPtr<Resource> ResourceLoader::load_resource(const LoadRequest& request)
        return it->value;
    }

-    auto resource = Resource::create({}, request);
+    auto resource = Resource::create({}, type, request);

    s_resource_cache.set(request, resource);

--- a/Libraries/LibWeb/Loader/ResourceLoader.h
+++ b/Libraries/LibWeb/Loader/ResourceLoader.h
@ -29,7 +29,7 @@
 #include <AK/Function.h>
 #include <AK/URL.h>
 #include <LibCore/Object.h>
-#include <LibWeb/Forward.h>
+#include <LibWeb/Loader/Resource.h>

 namespace Protocol {
 class Client;
@ -42,7 +42,7 @@ class ResourceLoader : public Core::Object {
 public:
    static ResourceLoader& the();

-    RefPtr<Resource> load_resource(const LoadRequest&);
+    RefPtr<Resource> load_resource(Resource::Type, const LoadRequest&);

    void load(const URL&, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);
    void load_sync(const URL&, Function<void(const ByteBuffer&, const HashMap<String, String, CaseInsensitiveStringTraits>& response_headers)> success_callback, Function<void(const String&)> error_callback = nullptr);