diff --git a/Userland/Applications/CharacterMap/CharacterMapWidget.cpp b/Userland/Applications/CharacterMap/CharacterMapWidget.cpp
index 9c23b14cab..de5afd1f18 100644
--- a/Userland/Applications/CharacterMap/CharacterMapWidget.cpp
+++ b/Userland/Applications/CharacterMap/CharacterMapWidget.cpp
@@ -166,7 +166,8 @@ void CharacterMapWidget::initialize_menubar(GUI::Window& window)
 
 void CharacterMapWidget::did_change_font()
 {
-    m_glyph_map->set_font(font());
+    // No need to track glyph modifications by cloning
+    m_glyph_map->GUI::AbstractScrollableWidget::set_font(font());
     m_font_name_label->set_text(font().human_readable_name());
     m_output_box->set_font(font());
 }
diff --git a/Userland/Applications/FontEditor/MainWidget.cpp b/Userland/Applications/FontEditor/MainWidget.cpp
index 98367ad18e..45d6e1b477 100644
--- a/Userland/Applications/FontEditor/MainWidget.cpp
+++ b/Userland/Applications/FontEditor/MainWidget.cpp
@@ -583,6 +583,8 @@ ErrorOr<void> MainWidget::initialize(String const& path, RefPtr<Gfx::BitmapFont>
     if (m_edited_font == edited_font)
         return {};
 
+    TRY(m_glyph_map_widget->set_font(*edited_font));
+
     auto selection = m_glyph_map_widget->selection().normalized();
     m_undo_selection = TRY(try_make_ref_counted<UndoSelection>(selection.start(), selection.size(), m_glyph_map_widget->active_glyph(), *edited_font, *m_glyph_map_widget));
     m_undo_stack->clear();
@@ -593,7 +595,6 @@ ErrorOr<void> MainWidget::initialize(String const& path, RefPtr<Gfx::BitmapFont>
     if (m_preview_label)
         m_preview_label->set_font(*m_edited_font);
 
-    m_glyph_map_widget->set_font(*m_edited_font);
     m_glyph_editor_widget->set_font(*m_edited_font);
     m_glyph_editor_widget->set_fixed_size(m_glyph_editor_widget->preferred_width(), m_glyph_editor_widget->preferred_height());
 
diff --git a/Userland/Libraries/LibGUI/GlyphMapWidget.cpp b/Userland/Libraries/LibGUI/GlyphMapWidget.cpp
index 8691d02499..84a947fe1e 100644
--- a/Userland/Libraries/LibGUI/GlyphMapWidget.cpp
+++ b/Userland/Libraries/LibGUI/GlyphMapWidget.cpp
@@ -456,11 +456,12 @@ bool GlyphMapWidget::glyph_is_modified(u32 glyph)
     return m_modified_glyphs.contains(glyph);
 }
 
-void GlyphMapWidget::set_font(Gfx::Font const& font)
+ErrorOr<void> GlyphMapWidget::set_font(Gfx::Font const& font)
 {
-    AbstractScrollableWidget::set_font(font);
-    m_original_font = font.clone();
+    m_original_font = TRY(font.try_clone());
     m_modified_glyphs.clear();
+    AbstractScrollableWidget::set_font(font);
+    return {};
 }
 
 }
diff --git a/Userland/Libraries/LibGUI/GlyphMapWidget.h b/Userland/Libraries/LibGUI/GlyphMapWidget.h
index b751e3a61f..c184e76fd2 100644
--- a/Userland/Libraries/LibGUI/GlyphMapWidget.h
+++ b/Userland/Libraries/LibGUI/GlyphMapWidget.h
@@ -21,7 +21,7 @@ class GlyphMapWidget final : public AbstractScrollableWidget {
 public:
     virtual ~GlyphMapWidget() override = default;
 
-    void set_font(Gfx::Font const&);
+    ErrorOr<void> set_font(Gfx::Font const&);
 
     class Selection {
     public: