diff --git a/Userland/Libraries/LibGUI/GlyphMapWidget.cpp b/Userland/Libraries/LibGUI/GlyphMapWidget.cpp
index 010913ee81..408e587f22 100644
--- a/Userland/Libraries/LibGUI/GlyphMapWidget.cpp
+++ b/Userland/Libraries/LibGUI/GlyphMapWidget.cpp
@@ -167,12 +167,8 @@ void GlyphMapWidget::mousedown_event(MouseEvent& event)
         auto glyph = maybe_glyph.value();
         if (event.shift())
             m_selection.extend_to(glyph);
-        else {
-            m_selection.set_size(1);
-            m_selection.set_start(glyph);
-        }
         m_in_drag_select = true;
-        set_active_glyph(glyph, ShouldResetSelection::No);
+        set_active_glyph(glyph, event.shift() ? ShouldResetSelection::No : ShouldResetSelection::Yes);
     }
 }