diff --git a/Kernel/Syscalls/chdir.cpp b/Kernel/Syscalls/chdir.cpp
index 001aa661c2..852392358b 100644
--- a/Kernel/Syscalls/chdir.cpp
+++ b/Kernel/Syscalls/chdir.cpp
@@ -15,10 +15,15 @@ ErrorOr<FlatPtr> Process::sys$chdir(Userspace<char const*> user_path, size_t pat
     VERIFY_NO_PROCESS_BIG_LOCK(this);
     TRY(require_promise(Pledge::rpath));
     auto path = TRY(get_syscall_path_argument(user_path, path_length));
-    return m_current_directory.with([&](auto& current_directory) -> ErrorOr<FlatPtr> {
-        current_directory = TRY(VirtualFileSystem::the().open_directory(path->view(), *current_directory));
-        return 0;
+    auto current_directory = m_current_directory.with([](auto& current_directory) -> NonnullRefPtr<Custody> {
+        return *current_directory;
     });
+    RefPtr<Custody> new_directory = TRY(VirtualFileSystem::the().open_directory(path->view(), *current_directory));
+    m_current_directory.with([&](auto& current_directory) {
+        // NOTE: We use swap() here to avoid manipulating the ref counts while holding the lock.
+        swap(current_directory, new_directory);
+    });
+    return 0;
 }
 
 ErrorOr<FlatPtr> Process::sys$fchdir(int fd)