diff --git a/Kernel/VM/MemoryManager.cpp b/Kernel/VM/MemoryManager.cpp
index 55c53e01fc..3856a72b1d 100644
--- a/Kernel/VM/MemoryManager.cpp
+++ b/Kernel/VM/MemoryManager.cpp
@@ -227,7 +227,8 @@ PageTableEntry& MemoryManager::ensure_pte(PageDirectory& page_directory, Virtual
         pde.set_present(true);
         pde.set_writable(true);
         pde.set_global(&page_directory == m_kernel_page_directory.ptr());
-        page_directory.m_physical_pages.set(page_directory_index, move(page_table));
+        auto result = page_directory.m_physical_pages.set(move(page_table));
+        ASSERT(result == AK::HashSetResult::InsertedNewEntry);
     }
 
     return quickmap_pt(PhysicalAddress((FlatPtr)pde.page_table_base()))[page_table_index];
diff --git a/Kernel/VM/PageDirectory.h b/Kernel/VM/PageDirectory.h
index 7602a8ccd7..d6de42b570 100644
--- a/Kernel/VM/PageDirectory.h
+++ b/Kernel/VM/PageDirectory.h
@@ -66,7 +66,7 @@ private:
     RangeAllocator m_identity_range_allocator;
     RefPtr<PhysicalPage> m_directory_table;
     RefPtr<PhysicalPage> m_directory_pages[4];
-    HashMap<unsigned, RefPtr<PhysicalPage>> m_physical_pages;
+    HashTable<RefPtr<PhysicalPage>> m_physical_pages;
 };
 
 }