diff --git a/Servers/WindowServer/WSButton.cpp b/Servers/WindowServer/WSButton.cpp
index ed1324a468..24e639164f 100644
--- a/Servers/WindowServer/WSButton.cpp
+++ b/Servers/WindowServer/WSButton.cpp
@@ -40,7 +40,9 @@ void WSButton::on_mouse_event(const WSMouseEvent& event)
     }
 
     if (event.type() == WSEvent::MouseUp && event.button() == MouseButton::Left) {
-        WSWindowManager::the().set_cursor_tracking_button(nullptr);
+        if (wm.cursor_tracking_button() != this)
+            return;
+        wm.set_cursor_tracking_button(nullptr);
         bool old_pressed = m_pressed;
         m_pressed = false;
         if (rect().contains(event.position())) {
@@ -61,6 +63,8 @@ void WSButton::on_mouse_event(const WSMouseEvent& event)
     }
 
     if (event.type() == WSEvent::MouseMove && event.buttons() & (unsigned)MouseButton::Left) {
+        if (wm.cursor_tracking_button() != this)
+            return;
         bool old_pressed = m_pressed;
         m_pressed = m_hovered;
         if (old_pressed != m_pressed)
diff --git a/Servers/WindowServer/WSWindowManager.h b/Servers/WindowServer/WSWindowManager.h
index a2d2bdb664..45d4d53717 100644
--- a/Servers/WindowServer/WSWindowManager.h
+++ b/Servers/WindowServer/WSWindowManager.h
@@ -101,9 +101,11 @@ public:
     const WSCursor& move_cursor() const { return *m_move_cursor; }
 
     void set_active_window(WSWindow*);
-    void set_cursor_tracking_button(WSButton*);
     void set_hovered_button(WSButton*);
 
+    WSButton* cursor_tracking_button() { return m_cursor_tracking_button.ptr(); }
+    void set_cursor_tracking_button(WSButton*);
+
     void set_resize_candidate(WSWindow&, ResizeDirection);
     void clear_resize_candidate();