diff --git a/Libraries/LibGUI/AbstractView.cpp b/Libraries/LibGUI/AbstractView.cpp
index e2ed2c9698..fe5a8bf355 100644
--- a/Libraries/LibGUI/AbstractView.cpp
+++ b/Libraries/LibGUI/AbstractView.cpp
@@ -364,9 +364,12 @@ void AbstractView::doubleclick_event(MouseEvent& event)
 
     auto index = index_at_event_position(event.position());
 
-    if (!index.is_valid())
+    if (!index.is_valid()) {
         clear_selection();
-    else if (!m_selection.contains(index))
+        return;
+    }
+
+    if (!m_selection.contains(index))
         set_selection(index);
 
     if (is_editable() && edit_triggers() & EditTrigger::DoubleClicked)