Kernel: Factor address space management out of the Process class

This patch adds Space, a class representing a process's address space.

- Each Process has a Space.
- The Space owns the PageDirectory and all Regions in the Process.

This allows us to reorganize sys$execve() so that it constructs and
populates a new Space fully before committing to it.

Previously, we would construct the new address space while still
running in the old one, and encountering an error meant we had to do
tedious and error-prone rollback.

Those problems are now gone, replaced by what's hopefully a set of much
smaller problems and missing cleanups. :^)

Kernel/VM/MemoryManager.cpp

@@ -401,29 +401,29 @@ Region* MemoryManager::kernel_region_from_vaddr(VirtualAddress vaddr)
     return nullptr;
 }
 
-Region* MemoryManager::user_region_from_vaddr(Process& process, VirtualAddress vaddr)
+Region* MemoryManager::user_region_from_vaddr(Space& space, VirtualAddress vaddr)
 {
-    ScopedSpinLock lock(s_mm_lock);
     // FIXME: Use a binary search tree (maybe red/black?) or some other more appropriate data structure!
-    for (auto& region : process.m_regions) {
+    ScopedSpinLock lock(space.get_lock());
+    for (auto& region : space.regions()) {
         if (region.contains(vaddr))
             return &region;
     }
     return nullptr;
 }
 
-Region* MemoryManager::find_region_from_vaddr(Process& process, VirtualAddress vaddr)
+Region* MemoryManager::find_region_from_vaddr(Space& space, VirtualAddress vaddr)
 {
     ScopedSpinLock lock(s_mm_lock);
-    if (auto* region = user_region_from_vaddr(process, vaddr))
+    if (auto* region = user_region_from_vaddr(space, vaddr))
         return region;
     return kernel_region_from_vaddr(vaddr);
 }
 
-const Region* MemoryManager::find_region_from_vaddr(const Process& process, VirtualAddress vaddr)
+const Region* MemoryManager::find_region_from_vaddr(const Space& space, VirtualAddress vaddr)
 {
     ScopedSpinLock lock(s_mm_lock);
-    if (auto* region = user_region_from_vaddr(const_cast<Process&>(process), vaddr))
+    if (auto* region = user_region_from_vaddr(const_cast<Space&>(space), vaddr))
         return region;
     return kernel_region_from_vaddr(vaddr);
 }
@@ -436,8 +436,8 @@ Region* MemoryManager::find_region_from_vaddr(VirtualAddress vaddr)
     auto page_directory = PageDirectory::find_by_cr3(read_cr3());
     if (!page_directory)
         return nullptr;
-    ASSERT(page_directory->process());
-    return user_region_from_vaddr(*page_directory->process(), vaddr);
+    ASSERT(page_directory->space());
+    return user_region_from_vaddr(*page_directory->space(), vaddr);
 }
 
 PageFaultResponse MemoryManager::handle_page_fault(const PageFault& fault)
@@ -734,13 +734,18 @@ RefPtr<PhysicalPage> MemoryManager::allocate_supervisor_physical_page()
 }
 
 void MemoryManager::enter_process_paging_scope(Process& process)
+{
+    enter_space(process.space());
+}
+
+void MemoryManager::enter_space(Space& space)
 {
     auto current_thread = Thread::current();
     ASSERT(current_thread != nullptr);
     ScopedSpinLock lock(s_mm_lock);
 
-    current_thread->tss().cr3 = process.page_directory().cr3();
-    write_cr3(process.page_directory().cr3());
+    current_thread->tss().cr3 = space.page_directory().cr3();
+    write_cr3(space.page_directory().cr3());
 }
 
 void MemoryManager::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
@@ -846,7 +851,7 @@ bool MemoryManager::validate_user_stack(const Process& process, VirtualAddress v
     if (!is_user_address(vaddr))
         return false;
     ScopedSpinLock lock(s_mm_lock);
-    auto* region = user_region_from_vaddr(const_cast<Process&>(process), vaddr);
+    auto* region = user_region_from_vaddr(const_cast<Process&>(process).space(), vaddr);
     return region && region->is_user_accessible() && region->is_stack();
 }
 

Kernel/VM/MemoryManager.h

@@ -143,7 +143,8 @@ public:
 
     PageFaultResponse handle_page_fault(const PageFault&);
 
-    void enter_process_paging_scope(Process&);
+    static void enter_process_paging_scope(Process&);
+    static void enter_space(Space&);
 
     bool validate_user_stack(const Process&, VirtualAddress) const;
 
@@ -196,8 +197,8 @@ public:
         }
     }
 
-    static Region* find_region_from_vaddr(Process&, VirtualAddress);
-    static const Region* find_region_from_vaddr(const Process&, VirtualAddress);
+    static Region* find_region_from_vaddr(Space&, VirtualAddress);
+    static const Region* find_region_from_vaddr(const Space&, VirtualAddress);
 
     void dump_kernel_regions();
 
@@ -225,7 +226,7 @@ private:
     static void flush_tlb_local(VirtualAddress, size_t page_count = 1);
     static void flush_tlb(const PageDirectory*, VirtualAddress, size_t page_count = 1);
 
-    static Region* user_region_from_vaddr(Process&, VirtualAddress);
+    static Region* user_region_from_vaddr(Space&, VirtualAddress);
     static Region* kernel_region_from_vaddr(VirtualAddress);
 
     static Region* find_region_from_vaddr(VirtualAddress);

Kernel/VM/PageDirectory.cpp

@@ -73,7 +73,7 @@ PageDirectory::PageDirectory()
     m_directory_pages[3] = PhysicalPage::create(boot_pd3_paddr, true, false);
 }
 
-PageDirectory::PageDirectory(Process& process, const RangeAllocator* parent_range_allocator)
+PageDirectory::PageDirectory(const RangeAllocator* parent_range_allocator)
 {
     ScopedSpinLock lock(s_mm_lock);
     if (parent_range_allocator) {
@@ -142,8 +142,8 @@ PageDirectory::PageDirectory(Process& process, const RangeAllocator* parent_rang
     auto* new_pd = MM.quickmap_pd(*this, 0);
     memcpy(new_pd, &buffer, sizeof(PageDirectoryEntry));
 
-    // If we got here, we successfully created it. Set m_process now
-    m_process = &process;
+    // If we got here, we successfully created it. Set m_space now
+    m_valid = true;
 
     cr3_map().set(cr3(), this);
 }
@@ -151,7 +151,7 @@ PageDirectory::PageDirectory(Process& process, const RangeAllocator* parent_rang
 PageDirectory::~PageDirectory()
 {
     ScopedSpinLock lock(s_mm_lock);
-    if (m_process)
+    if (m_space)
         cr3_map().remove(cr3());
 }
 

Kernel/VM/PageDirectory.h

@@ -40,10 +40,10 @@ class PageDirectory : public RefCounted<PageDirectory> {
     friend class MemoryManager;
 
 public:
-    static RefPtr<PageDirectory> create_for_userspace(Process& process, const RangeAllocator* parent_range_allocator = nullptr)
+    static RefPtr<PageDirectory> create_for_userspace(const RangeAllocator* parent_range_allocator = nullptr)
     {
-        auto page_directory = adopt(*new PageDirectory(process, parent_range_allocator));
-        if (!page_directory->process())
+        auto page_directory = adopt(*new PageDirectory(parent_range_allocator));
+        if (!page_directory->is_valid())
             return {};
         return page_directory;
     }
@@ -55,24 +55,31 @@ public:
     u32 cr3() const { return m_directory_table->paddr().get(); }
 
     RangeAllocator& range_allocator() { return m_range_allocator; }
+    const RangeAllocator& range_allocator() const { return m_range_allocator; }
+
     RangeAllocator& identity_range_allocator() { return m_identity_range_allocator; }
 
-    Process* process() { return m_process; }
-    const Process* process() const { return m_process; }
+    bool is_valid() const { return m_valid; }
+
+    Space* space() { return m_space; }
+    const Space* space() const { return m_space; }
+
+    void set_space(Badge<Space>, Space& space) { m_space = &space; }
 
     RecursiveSpinLock& get_lock() { return m_lock; }
 
 private:
-    PageDirectory(Process&, const RangeAllocator* parent_range_allocator);
+    explicit PageDirectory(const RangeAllocator* parent_range_allocator);
     PageDirectory();
 
-    Process* m_process { nullptr };
+    Space* m_space { nullptr };
     RangeAllocator m_range_allocator;
     RangeAllocator m_identity_range_allocator;
     RefPtr<PhysicalPage> m_directory_table;
     RefPtr<PhysicalPage> m_directory_pages[4];
     HashMap<u32, RefPtr<PhysicalPage>> m_page_tables;
     RecursiveSpinLock m_lock;
+    bool m_valid { false };
 };
 
 }

Kernel/VM/Space.cpp

@@ -0,0 +1,222 @@
+/*
+ * Copyright (c) 2021, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <AK/QuickSort.h>
+#include <Kernel/Process.h>
+#include <Kernel/SpinLock.h>
+#include <Kernel/VM/AnonymousVMObject.h>
+#include <Kernel/VM/MemoryManager.h>
+#include <Kernel/VM/Space.h>
+
+namespace Kernel {
+
+OwnPtr<Space> Space::create(Process& process, const Space* parent)
+{
+    auto page_directory = PageDirectory::create_for_userspace(parent ? &parent->page_directory().range_allocator() : nullptr);
+    if (!page_directory)
+        return {};
+    auto space = adopt_own(*new Space(process, page_directory.release_nonnull()));
+    space->page_directory().set_space({}, *space);
+    return space;
+}
+
+Space::Space(Process& process, NonnullRefPtr<PageDirectory> page_directory)
+    : m_process(&process)
+    , m_page_directory(move(page_directory))
+{
+}
+
+Space::~Space()
+{
+}
+
+Optional<Range> Space::allocate_range(VirtualAddress vaddr, size_t size, size_t alignment)
+{
+    vaddr.mask(PAGE_MASK);
+    size = PAGE_ROUND_UP(size);
+    if (vaddr.is_null())
+        return page_directory().range_allocator().allocate_anywhere(size, alignment);
+    return page_directory().range_allocator().allocate_specific(vaddr, size);
+}
+
+Region& Space::allocate_split_region(const Region& source_region, const Range& range, size_t offset_in_vmobject)
+{
+    auto& region = add_region(Region::create_user_accessible(
+        m_process, range, source_region.vmobject(), offset_in_vmobject, source_region.name(), source_region.access(), source_region.is_cacheable(), source_region.is_shared()));
+    region.set_syscall_region(source_region.is_syscall_region());
+    region.set_mmap(source_region.is_mmap());
+    region.set_stack(source_region.is_stack());
+    size_t page_offset_in_source_region = (offset_in_vmobject - source_region.offset_in_vmobject()) / PAGE_SIZE;
+    for (size_t i = 0; i < region.page_count(); ++i) {
+        if (source_region.should_cow(page_offset_in_source_region + i))
+            region.set_should_cow(i, true);
+    }
+    return region;
+}
+
+KResultOr<Region*> Space::allocate_region(const Range& range, const String& name, int prot, AllocationStrategy strategy)
+{
+    ASSERT(range.is_valid());
+    auto vmobject = AnonymousVMObject::create_with_size(range.size(), strategy);
+    if (!vmobject)
+        return ENOMEM;
+    auto region = Region::create_user_accessible(m_process, range, vmobject.release_nonnull(), 0, name, prot_to_region_access_flags(prot), true, false);
+    if (!region->map(page_directory()))
+        return ENOMEM;
+    return &add_region(move(region));
+}
+
+KResultOr<Region*> Space::allocate_region_with_vmobject(const Range& range, NonnullRefPtr<VMObject> vmobject, size_t offset_in_vmobject, const String& name, int prot, bool shared)
+{
+    ASSERT(range.is_valid());
+    size_t end_in_vmobject = offset_in_vmobject + range.size();
+    if (end_in_vmobject <= offset_in_vmobject) {
+        dbgln("allocate_region_with_vmobject: Overflow (offset + size)");
+        return EINVAL;
+    }
+    if (offset_in_vmobject >= vmobject->size()) {
+        dbgln("allocate_region_with_vmobject: Attempt to allocate a region with an offset past the end of its VMObject.");
+        return EINVAL;
+    }
+    if (end_in_vmobject > vmobject->size()) {
+        dbgln("allocate_region_with_vmobject: Attempt to allocate a region with an end past the end of its VMObject.");
+        return EINVAL;
+    }
+    offset_in_vmobject &= PAGE_MASK;
+    auto& region = add_region(Region::create_user_accessible(m_process, range, move(vmobject), offset_in_vmobject, name, prot_to_region_access_flags(prot), true, shared));
+    if (!region.map(page_directory())) {
+        // FIXME: What is an appropriate error code here, really?
+        return ENOMEM;
+    }
+    return &region;
+}
+
+bool Space::deallocate_region(Region& region)
+{
+    OwnPtr<Region> region_protector;
+    ScopedSpinLock lock(m_lock);
+
+    if (m_region_lookup_cache.region.unsafe_ptr() == &region)
+        m_region_lookup_cache.region = nullptr;
+    for (size_t i = 0; i < m_regions.size(); ++i) {
+        if (&m_regions[i] == &region) {
+            region_protector = m_regions.unstable_take(i);
+            return true;
+        }
+    }
+    return false;
+}
+
+Region* Space::find_region_from_range(const Range& range)
+{
+    ScopedSpinLock lock(m_lock);
+    if (m_region_lookup_cache.range.has_value() && m_region_lookup_cache.range.value() == range && m_region_lookup_cache.region)
+        return m_region_lookup_cache.region.unsafe_ptr();
+
+    size_t size = PAGE_ROUND_UP(range.size());
+    for (auto& region : m_regions) {
+        if (region.vaddr() == range.base() && region.size() == size) {
+            m_region_lookup_cache.range = range;
+            m_region_lookup_cache.region = region;
+            return &region;
+        }
+    }
+    return nullptr;
+}
+
+Region* Space::find_region_containing(const Range& range)
+{
+    ScopedSpinLock lock(m_lock);
+    for (auto& region : m_regions) {
+        if (region.contains(range))
+            return &region;
+    }
+    return nullptr;
+}
+
+Region& Space::add_region(NonnullOwnPtr<Region> region)
+{
+    auto* ptr = region.ptr();
+    ScopedSpinLock lock(m_lock);
+    m_regions.append(move(region));
+    return *ptr;
+}
+
+// Carve out a virtual address range from a region and return the two regions on either side
+Vector<Region*, 2> Space::split_region_around_range(const Region& source_region, const Range& desired_range)
+{
+    Range old_region_range = source_region.range();
+    auto remaining_ranges_after_unmap = old_region_range.carve(desired_range);
+
+    ASSERT(!remaining_ranges_after_unmap.is_empty());
+    auto make_replacement_region = [&](const Range& new_range) -> Region& {
+        ASSERT(old_region_range.contains(new_range));
+        size_t new_range_offset_in_vmobject = source_region.offset_in_vmobject() + (new_range.base().get() - old_region_range.base().get());
+        return allocate_split_region(source_region, new_range, new_range_offset_in_vmobject);
+    };
+    Vector<Region*, 2> new_regions;
+    for (auto& new_range : remaining_ranges_after_unmap) {
+        new_regions.unchecked_append(&make_replacement_region(new_range));
+    }
+    return new_regions;
+}
+
+void Space::dump_regions()
+{
+    klog() << "Process regions:";
+    klog() << "BEGIN       END         SIZE        ACCESS  NAME";
+
+    ScopedSpinLock lock(m_lock);
+
+    Vector<Region*> sorted_regions;
+    sorted_regions.ensure_capacity(m_regions.size());
+    for (auto& region : m_regions)
+        sorted_regions.append(&region);
+    quick_sort(sorted_regions, [](auto& a, auto& b) {
+        return a->vaddr() < b->vaddr();
+    });
+
+    for (auto& sorted_region : sorted_regions) {
+        auto& region = *sorted_region;
+        dmesgln("{:08x} -- {:08x} {:08x} {:c}{:c}{:c}{:c}{:c} {}", region.vaddr().get(), region.vaddr().offset(region.size() - 1).get(), region.size(),
+            region.is_readable() ? 'R' : ' ',
+            region.is_writable() ? 'W' : ' ',
+            region.is_executable() ? 'X' : ' ',
+            region.is_shared() ? 'S' : ' ',
+            region.is_stack() ? 'T' : ' ',
+            region.is_syscall_region() ? 'C' : ' ',
+            region.name());
+    }
+    MM.dump_kernel_regions();
+}
+
+void Space::remove_all_regions(Badge<Process>)
+{
+    ScopedSpinLock lock(m_lock);
+    m_regions.clear();
+}
+
+}

Kernel/VM/Space.h

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#pragma once
+
+#include <AK/NonnullOwnPtrVector.h>
+#include <AK/WeakPtr.h>
+#include <Kernel/UnixTypes.h>
+#include <Kernel/VM/AllocationStrategy.h>
+#include <Kernel/VM/PageDirectory.h>
+
+namespace Kernel {
+
+class Space {
+public:
+    static OwnPtr<Space> create(Process&, const Space* parent);
+    ~Space();
+
+    PageDirectory& page_directory() { return *m_page_directory; }
+    const PageDirectory& page_directory() const { return *m_page_directory; }
+
+    Region& add_region(NonnullOwnPtr<Region>);
+
+    size_t region_count() const { return m_regions.size(); }
+
+    NonnullOwnPtrVector<Region>& regions() { return m_regions; }
+    const NonnullOwnPtrVector<Region>& regions() const { return m_regions; }
+
+    void dump_regions();
+
+    Optional<Range> allocate_range(VirtualAddress, size_t, size_t alignment = PAGE_SIZE);
+
+    KResultOr<Region*> allocate_region_with_vmobject(const Range&, NonnullRefPtr<VMObject>, size_t offset_in_vmobject, const String& name, int prot, bool shared);
+    KResultOr<Region*> allocate_region(const Range&, const String& name, int prot = PROT_READ | PROT_WRITE, AllocationStrategy strategy = AllocationStrategy::Reserve);
+    bool deallocate_region(Region& region);
+
+    Region& allocate_split_region(const Region& source_region, const Range&, size_t offset_in_vmobject);
+    Vector<Region*, 2> split_region_around_range(const Region& source_region, const Range&);
+
+    Region* find_region_from_range(const Range&);
+    Region* find_region_containing(const Range&);
+
+    bool enforces_syscall_regions() const { return m_enforces_syscall_regions; }
+    void set_enforces_syscall_regions(bool b) { m_enforces_syscall_regions = b; }
+
+    void remove_all_regions(Badge<Process>);
+
+    SpinLock<u32>& get_lock() const { return m_lock; }
+
+private:
+    Space(Process&, NonnullRefPtr<PageDirectory>);
+
+    Process* m_process { nullptr };
+    mutable SpinLock<u32> m_lock;
+
+    RefPtr<PageDirectory> m_page_directory;
+
+    NonnullOwnPtrVector<Region> m_regions;
+
+    struct RegionLookupCache {
+        Optional<Range> range;
+        WeakPtr<Region> region;
+    };
+    RegionLookupCache m_region_lookup_cache;
+
+    bool m_enforces_syscall_regions { false };
+};
+
+}