diff --git a/Meta/gn/secondary/Userland/Libraries/LibWeb/BUILD.gn b/Meta/gn/secondary/Userland/Libraries/LibWeb/BUILD.gn
index 20ffc5352a..3d06e9babe 100644
--- a/Meta/gn/secondary/Userland/Libraries/LibWeb/BUILD.gn
+++ b/Meta/gn/secondary/Userland/Libraries/LibWeb/BUILD.gn
@@ -306,7 +306,6 @@ shared_library("LibWeb") {
   public_configs = [ ":configs" ]
   sources = [
     "Dump.cpp",
-    "FontCache.cpp",
     "Namespace.cpp",
     "PixelUnits.cpp",
   ]
diff --git a/Userland/Libraries/LibGfx/CMakeLists.txt b/Userland/Libraries/LibGfx/CMakeLists.txt
index c4f2e8a20a..7348ba6d7c 100644
--- a/Userland/Libraries/LibGfx/CMakeLists.txt
+++ b/Userland/Libraries/LibGfx/CMakeLists.txt
@@ -25,6 +25,7 @@ set(SOURCES
     Font/OpenType/Tables.cpp
     Font/ScaledFont.cpp
     Font/Typeface.cpp
+    Font/VectorFont.cpp
     Font/WOFF/Font.cpp
     Font/WOFF2/Font.cpp
     GradientPainting.cpp
diff --git a/Userland/Libraries/LibGfx/Font/ScaledFont.cpp b/Userland/Libraries/LibGfx/Font/ScaledFont.cpp
index 15524c0275..84f3bfc5c5 100644
--- a/Userland/Libraries/LibGfx/Font/ScaledFont.cpp
+++ b/Userland/Libraries/LibGfx/Font/ScaledFont.cpp
@@ -155,7 +155,9 @@ u8 ScaledFont::glyph_fixed_width() const
 
 RefPtr<Font> ScaledFont::with_size(float point_size) const
 {
-    return adopt_ref(*new Gfx::ScaledFont(*m_font, point_size, point_size));
+    if (point_size == m_point_height && point_size == m_point_width)
+        return const_cast<ScaledFont*>(this);
+    return m_font->scaled_font(point_size);
 }
 
 Gfx::FontPixelMetrics ScaledFont::pixel_metrics() const
diff --git a/Userland/Libraries/LibGfx/Font/Typeface.cpp b/Userland/Libraries/LibGfx/Font/Typeface.cpp
index 56a1a47762..87e338e17a 100644
--- a/Userland/Libraries/LibGfx/Font/Typeface.cpp
+++ b/Userland/Libraries/LibGfx/Font/Typeface.cpp
@@ -64,7 +64,7 @@ RefPtr<Font> Typeface::get_font(float point_size, Font::AllowInexactSizeMatch al
     VERIFY(point_size >= 0);
 
     if (m_vector_font)
-        return adopt_ref(*new Gfx::ScaledFont(*m_vector_font, point_size, point_size));
+        return m_vector_font->scaled_font(point_size);
 
     RefPtr<BitmapFont> best_match;
     int size = roundf(point_size);
diff --git a/Userland/Libraries/LibGfx/Font/VectorFont.cpp b/Userland/Libraries/LibGfx/Font/VectorFont.cpp
new file mode 100644
index 0000000000..f58e01b0fe
--- /dev/null
+++ b/Userland/Libraries/LibGfx/Font/VectorFont.cpp
@@ -0,0 +1,31 @@
+/*
+ * Copyright (c) 2023, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibGfx/Font/ScaledFont.h>
+#include <LibGfx/Font/VectorFont.h>
+
+namespace Gfx {
+
+VectorFont::VectorFont() = default;
+VectorFont::~VectorFont() = default;
+
+NonnullRefPtr<ScaledFont> VectorFont::scaled_font(float point_size) const
+{
+    auto it = m_scaled_fonts.find(point_size);
+    if (it != m_scaled_fonts.end())
+        return *it->value;
+
+    // FIXME: It might be nice to have a global cap on the number of fonts we cache
+    //        instead of doing it at the per-VectorFont level like this.
+    constexpr size_t max_cached_font_size_count = 128;
+    if (m_scaled_fonts.size() > max_cached_font_size_count)
+        m_scaled_fonts.remove(m_scaled_fonts.begin());
+
+    auto scaled_font = adopt_ref(*new ScaledFont(*this, point_size, point_size));
+    m_scaled_fonts.set(point_size, scaled_font);
+    return scaled_font;
+}
+}
diff --git a/Userland/Libraries/LibGfx/Font/VectorFont.h b/Userland/Libraries/LibGfx/Font/VectorFont.h
index 07de33c0d9..14b3d337ab 100644
--- a/Userland/Libraries/LibGfx/Font/VectorFont.h
+++ b/Userland/Libraries/LibGfx/Font/VectorFont.h
@@ -14,6 +14,8 @@
 
 namespace Gfx {
 
+class ScaledFont;
+
 struct ScaledFontMetrics {
     float ascender { 0 };
     float descender { 0 };
@@ -35,7 +37,7 @@ struct ScaledGlyphMetrics {
 
 class VectorFont : public RefCounted<VectorFont> {
 public:
-    virtual ~VectorFont() { }
+    virtual ~VectorFont();
     virtual ScaledFontMetrics metrics(float x_scale, float y_scale) const = 0;
     virtual ScaledGlyphMetrics glyph_metrics(u32 glyph_id, float x_scale, float y_scale, float point_width, float point_height) const = 0;
     virtual float glyphs_horizontal_kerning(u32 left_glyph_id, u32 right_glyph_id, float x_scale) const = 0;
@@ -52,6 +54,14 @@ public:
     virtual u8 slope() const = 0;
     virtual bool is_fixed_width() const = 0;
     virtual bool has_color_bitmaps() const = 0;
+
+    [[nodiscard]] NonnullRefPtr<ScaledFont> scaled_font(float point_size) const;
+
+protected:
+    VectorFont();
+
+private:
+    mutable HashMap<float, NonnullRefPtr<ScaledFont>> m_scaled_fonts;
 };
 
 }
diff --git a/Userland/Libraries/LibGfx/Forward.h b/Userland/Libraries/LibGfx/Forward.h
index 43c6b9c28d..6e25167856 100644
--- a/Userland/Libraries/LibGfx/Forward.h
+++ b/Userland/Libraries/LibGfx/Forward.h
@@ -21,6 +21,7 @@ class Font;
 class GlyphBitmap;
 class ImageDecoder;
 struct FontPixelMetrics;
+class ScaledFont;
 
 template<typename T>
 class Line;
diff --git a/Userland/Libraries/LibWeb/CMakeLists.txt b/Userland/Libraries/LibWeb/CMakeLists.txt
index c091c60e37..32fd5a534c 100644
--- a/Userland/Libraries/LibWeb/CMakeLists.txt
+++ b/Userland/Libraries/LibWeb/CMakeLists.txt
@@ -217,7 +217,6 @@ set(SOURCES
     FileAPI/File.cpp
     FileAPI/FileList.cpp
     FileAPI/FileReader.cpp
-    FontCache.cpp
     Geometry/DOMMatrix.cpp
     Geometry/DOMMatrixReadOnly.cpp
     Geometry/DOMPoint.cpp
diff --git a/Userland/Libraries/LibWeb/CSS/StyleComputer.cpp b/Userland/Libraries/LibWeb/CSS/StyleComputer.cpp
index 80868ef3bf..0c0e2b67cc 100644
--- a/Userland/Libraries/LibWeb/CSS/StyleComputer.cpp
+++ b/Userland/Libraries/LibWeb/CSS/StyleComputer.cpp
@@ -55,7 +55,6 @@
 #include <LibWeb/CSS/StyleValues/UnsetStyleValue.h>
 #include <LibWeb/DOM/Document.h>
 #include <LibWeb/DOM/Element.h>
-#include <LibWeb/FontCache.h>
 #include <LibWeb/HTML/HTMLBRElement.h>
 #include <LibWeb/HTML/HTMLHtmlElement.h>
 #include <LibWeb/Layout/Node.h>
@@ -122,19 +121,7 @@ public:
             start_loading_next_url();
             return nullptr;
         }
-
-        if (auto it = m_cached_fonts.find(point_size); it != m_cached_fonts.end())
-            return it->value;
-
-        // FIXME: It might be nicer to have a global cap on the number of fonts we cache
-        //        instead of doing it at the per-font level like this.
-        constexpr size_t max_cached_font_size_count = 64;
-        if (m_cached_fonts.size() > max_cached_font_size_count)
-            m_cached_fonts.remove(m_cached_fonts.begin());
-
-        auto font = adopt_ref(*new Gfx::ScaledFont(*m_vector_font, point_size, point_size));
-        m_cached_fonts.set(point_size, font);
-        return font;
+        return m_vector_font->scaled_font(point_size);
     }
 
 private:
@@ -189,8 +176,6 @@ private:
     Vector<Gfx::UnicodeRange> m_unicode_ranges;
     RefPtr<Gfx::VectorFont> m_vector_font;
     Vector<AK::URL> m_urls;
-
-    HashMap<float, NonnullRefPtr<Gfx::ScaledFont>> mutable m_cached_fonts;
 };
 
 struct StyleComputer::MatchingFontCandidate {
@@ -1796,14 +1781,11 @@ RefPtr<Gfx::FontCascadeList const> StyleComputer::compute_font_for_style_values(
     // FIXME: Implement the full font-matching algorithm: https://www.w3.org/TR/css-fonts-4/#font-matching-algorithm
 
     // Note: This is modified by the find_font() lambda
-    FontSelector font_selector;
     bool monospace = false;
 
     float const font_size_in_pt = font_size_in_px * 0.75f;
 
     auto find_font = [&](FlyString const& family) -> RefPtr<Gfx::FontCascadeList const> {
-        font_selector = { family, font_size_in_pt, weight, width, slope };
-
         FontFaceKey key {
             .family_name = family,
             .weight = weight,
@@ -2381,9 +2363,8 @@ CSSPixelRect StyleComputer::viewport_rect() const
     return {};
 }
 
-void StyleComputer::did_load_font(FlyString const& family_name)
+void StyleComputer::did_load_font(FlyString const&)
 {
-    m_font_cache.did_load_font({}, family_name);
     document().invalidate_style();
 }
 
diff --git a/Userland/Libraries/LibWeb/CSS/StyleComputer.h b/Userland/Libraries/LibWeb/CSS/StyleComputer.h
index be878be6df..e33a6a7e3c 100644
--- a/Userland/Libraries/LibWeb/CSS/StyleComputer.h
+++ b/Userland/Libraries/LibWeb/CSS/StyleComputer.h
@@ -16,7 +16,6 @@
 #include <LibWeb/CSS/CSSStyleDeclaration.h>
 #include <LibWeb/CSS/Selector.h>
 #include <LibWeb/CSS/StyleProperties.h>
-#include <LibWeb/FontCache.h>
 #include <LibWeb/Forward.h>
 
 namespace Web::CSS {
@@ -48,8 +47,6 @@ public:
     DOM::Document& document() { return m_document; }
     DOM::Document const& document() const { return m_document; }
 
-    FontCache& font_cache() const { return m_font_cache; }
-
     NonnullRefPtr<StyleProperties> create_document_style() const;
 
     ErrorOr<NonnullRefPtr<StyleProperties>> compute_style(DOM::Element&, Optional<CSS::Selector::PseudoElement::Type> = {}) const;
@@ -184,8 +181,6 @@ private:
     OwnPtr<RuleCache> m_user_agent_rule_cache;
     JS::Handle<CSSStyleSheet> m_user_style_sheet;
 
-    mutable FontCache m_font_cache;
-
     using FontLoaderList = Vector<NonnullOwnPtr<FontLoader>>;
     HashMap<FontFaceKey, FontLoaderList> m_loaded_fonts;
 
diff --git a/Userland/Libraries/LibWeb/CSS/StyleProperties.cpp b/Userland/Libraries/LibWeb/CSS/StyleProperties.cpp
index ad767d6551..8ce1989e5c 100644
--- a/Userland/Libraries/LibWeb/CSS/StyleProperties.cpp
+++ b/Userland/Libraries/LibWeb/CSS/StyleProperties.cpp
@@ -28,7 +28,6 @@
 #include <LibWeb/CSS/StyleValues/StringStyleValue.h>
 #include <LibWeb/CSS/StyleValues/StyleValueList.h>
 #include <LibWeb/CSS/StyleValues/TransformationStyleValue.h>
-#include <LibWeb/FontCache.h>
 #include <LibWeb/Layout/BlockContainer.h>
 #include <LibWeb/Layout/Node.h>
 #include <LibWeb/Platform/FontPlugin.h>
diff --git a/Userland/Libraries/LibWeb/FontCache.cpp b/Userland/Libraries/LibWeb/FontCache.cpp
deleted file mode 100644
index d5d593bfb0..0000000000
--- a/Userland/Libraries/LibWeb/FontCache.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright (c) 2018-2023, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#include <LibGfx/Font/Font.h>
-#include <LibWeb/FontCache.h>
-
-namespace Web {
-
-RefPtr<Gfx::Font const> FontCache::get(FontSelector const& font_selector) const
-{
-    auto cached_font = m_fonts.get(font_selector);
-    if (cached_font.has_value())
-        return cached_font.value();
-    return nullptr;
-}
-
-NonnullRefPtr<Gfx::Font const> FontCache::scaled_font(Gfx::Font const& font, float scale_factor)
-{
-    auto device_font_pt_size = font.point_size() * scale_factor;
-    FontSelector font_selector = { font.family(), device_font_pt_size, font.weight(), font.width(), font.slope() };
-    if (auto cached_font = get(font_selector)) {
-        return *cached_font;
-    }
-
-    if (auto font_with_device_pt_size = font.with_size(device_font_pt_size)) {
-        set(font_selector, *font_with_device_pt_size);
-        return font_with_device_pt_size.release_nonnull();
-    }
-
-    return font;
-}
-
-void FontCache::set(FontSelector const& font_selector, NonnullRefPtr<Gfx::Font const> font)
-{
-    m_fonts.set(font_selector, move(font));
-}
-
-void FontCache::did_load_font(Badge<CSS::StyleComputer>, FlyString const& family_name)
-{
-    m_fonts.remove_all_matching([&family_name](auto& key, auto&) -> bool {
-        return key.family == family_name;
-    });
-}
-
-}
diff --git a/Userland/Libraries/LibWeb/FontCache.h b/Userland/Libraries/LibWeb/FontCache.h
deleted file mode 100644
index 536436c9c8..0000000000
--- a/Userland/Libraries/LibWeb/FontCache.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright (c) 2018-2023, Andreas Kling <kling@serenityos.org>
- *
- * SPDX-License-Identifier: BSD-2-Clause
- */
-
-#pragma once
-
-#include <AK/FlyString.h>
-#include <AK/HashMap.h>
-#include <LibGfx/Font/Font.h>
-#include <LibGfx/Forward.h>
-#include <LibWeb/Forward.h>
-
-namespace Web {
-
-struct FontSelector {
-    FlyString family;
-    float point_size { 0 };
-    int weight { 0 };
-    int width { 0 };
-    int slope { 0 };
-
-    bool operator==(FontSelector const& other) const
-    {
-        return family == other.family && point_size == other.point_size && weight == other.weight && width == other.width && slope == other.slope;
-    }
-};
-
-class FontCache {
-public:
-    FontCache() = default;
-    RefPtr<Gfx::Font const> get(FontSelector const&) const;
-    void set(FontSelector const&, NonnullRefPtr<Gfx::Font const>);
-
-    NonnullRefPtr<Gfx::Font const> scaled_font(Gfx::Font const&, float scale_factor);
-
-    void did_load_font(Badge<CSS::StyleComputer>, FlyString const& family_name);
-
-private:
-    mutable HashMap<FontSelector, NonnullRefPtr<Gfx::Font const>> m_fonts;
-};
-
-}
-
-namespace AK {
-template<>
-struct Traits<Web::FontSelector> : public DefaultTraits<Web::FontSelector> {
-    static unsigned hash(Web::FontSelector const& key) { return pair_int_hash(pair_int_hash(key.family.hash(), key.weight), key.point_size); }
-};
-}
diff --git a/Userland/Libraries/LibWeb/Layout/Node.h b/Userland/Libraries/LibWeb/Layout/Node.h
index e123355ea4..944ee5ca99 100644
--- a/Userland/Libraries/LibWeb/Layout/Node.h
+++ b/Userland/Libraries/LibWeb/Layout/Node.h
@@ -17,7 +17,6 @@
 #include <LibWeb/CSS/StyleProperties.h>
 #include <LibWeb/CSS/StyleValues/ImageStyleValue.h>
 #include <LibWeb/DOM/Document.h>
-#include <LibWeb/FontCache.h>
 #include <LibWeb/Forward.h>
 #include <LibWeb/Layout/BoxModelMetrics.h>
 #include <LibWeb/Painting/PaintContext.h>
@@ -300,7 +299,8 @@ inline Gfx::Font const& Node::scaled_font(PaintContext& context) const
 
 inline Gfx::Font const& Node::scaled_font(float scale_factor) const
 {
-    return document().style_computer().font_cache().scaled_font(first_available_font(), scale_factor);
+    auto const& font = first_available_font();
+    return *font.with_size(font.point_size() * scale_factor);
 }
 
 inline const CSS::ImmutableComputedValues& Node::computed_values() const
diff --git a/Userland/Libraries/LibWeb/Painting/ButtonPaintable.cpp b/Userland/Libraries/LibWeb/Painting/ButtonPaintable.cpp
index 53503fe5a8..0d37b19f37 100644
--- a/Userland/Libraries/LibWeb/Painting/ButtonPaintable.cpp
+++ b/Userland/Libraries/LibWeb/Painting/ButtonPaintable.cpp
@@ -5,7 +5,6 @@
  */
 
 #include <LibGUI/Event.h>
-#include <LibWeb/FontCache.h>
 #include <LibWeb/HTML/BrowsingContext.h>
 #include <LibWeb/HTML/HTMLImageElement.h>
 #include <LibWeb/Layout/ButtonBox.h>
@@ -65,7 +64,7 @@ void ButtonPaintable::paint(PaintContext& context, PaintPhase phase) const
         painter.draw_text(
             text_rect.to_type<int>(),
             static_cast<HTML::HTMLInputElement const&>(dom_node).value(),
-            document().style_computer().font_cache().scaled_font(layout_box().first_available_font(), context.device_pixels_per_css_pixel()),
+            layout_box().scaled_font(context.device_pixels_per_css_pixel()),
             Gfx::TextAlignment::Center,
             computed_values().color());
         painter.restore();
diff --git a/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp b/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp
index 35273d00c3..51922db177 100644
--- a/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp
+++ b/Userland/Libraries/LibWeb/Painting/PaintableBox.cpp
@@ -618,10 +618,9 @@ static void paint_text_fragment(PaintContext& context, Layout::TextNode const& t
         DevicePixelPoint baseline_start { fragment_absolute_device_rect.x(), fragment_absolute_device_rect.y() + context.rounded_device_pixels(fragment.baseline()) };
         Vector<Gfx::DrawGlyphOrEmoji> scaled_glyph_run;
         scaled_glyph_run.ensure_capacity(fragment.glyph_run().size());
-        auto& font_cache = text_node.document().style_computer().font_cache();
         for (auto glyph : fragment.glyph_run()) {
             glyph.visit([&](auto& glyph) {
-                glyph.font = font_cache.scaled_font(*glyph.font, context.device_pixels_per_css_pixel());
+                glyph.font = *glyph.font->with_size(glyph.font->point_size() * static_cast<float>(context.device_pixels_per_css_pixel()));
                 glyph.position = glyph.position.scaled(context.device_pixels_per_css_pixel());
             });
             scaled_glyph_run.append(move(glyph));