Kernel: Implement lazy committed page allocation

By designating a committed page pool we can guarantee to have physical
pages available for lazy allocation in mappings. However, when forking
we will overcommit. The assumption is that worst-case it's better for
the fork to die due to insufficient physical memory on COW access than
the parent that created the region. If a fork wants to ensure that all
memory is available (trigger a commit) then it can use madvise.

This also means that fork now can gracefully fail if we don't have
enough physical pages available.

Kernel/VM/PurgeableVMObject.cpp

@@ -44,10 +44,10 @@ inline LogStream& operator<<(const LogStream& stream, const VolatilePageRange& r
 
 static void dump_volatile_page_ranges(const Vector<VolatilePageRange>& ranges)
 {
-   for (size_t i = 0; i < ranges.size(); i++) {
-       const auto& range = ranges[i];
-       klog() << "  [" << i << "] " << range;
-   }
+    for (size_t i = 0; i < ranges.size(); i++) {
+        const auto& range = ranges[i];
+        klog() << "  [" << i << "] " << range;
+    }
 }
 #endif
 
@@ -185,7 +185,7 @@ bool VolatilePageRanges::intersects(const VolatilePageRange& range) const
 }
 
 PurgeablePageRanges::PurgeablePageRanges(const VMObject& vmobject)
-    : m_volatile_ranges({0, vmobject.is_purgeable() ? static_cast<const PurgeableVMObject&>(vmobject).page_count() : 0})
+    : m_volatile_ranges({ 0, vmobject.is_purgeable() ? static_cast<const PurgeableVMObject&>(vmobject).page_count() : 0 })
 {
 }
 
@@ -193,8 +193,23 @@ bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range)
 {
     if (range.is_empty())
         return false;
+
+    // Since we may need to call into PurgeableVMObject we need to acquire
+    // its lock as well, and acquire it first. This is important so that
+    // we don't deadlock when a page fault (e.g. on another processor)
+    // happens that is meant to lazy-allocate a committed page. It would
+    // call into PurgeableVMObject::range_made_volatile, which then would
+    // also call into this object and need to acquire m_lock. By acquiring
+    // the vmobject lock first in both cases, we avoid deadlocking.
+    // We can access m_vmobject without any locks for that purpose because
+    // add_volatile_range and remove_volatile_range can only be called
+    // by same object that calls set_vmobject.
+    ScopedSpinLock vmobject_lock(m_vmobject->m_lock);
     ScopedSpinLock lock(m_volatile_ranges_lock);
-    return m_volatile_ranges.add(range);
+    bool added = m_volatile_ranges.add(range);
+    if (added)
+        m_vmobject->range_made_volatile(range);
+    return added;
 }
 
 bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged)
@@ -202,6 +217,7 @@ bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range,
     if (range.is_empty())
         return false;
     ScopedSpinLock lock(m_volatile_ranges_lock);
+    ASSERT(m_vmobject);
     return m_volatile_ranges.remove(range, was_purged);
 }
 
@@ -213,35 +229,73 @@ bool PurgeablePageRanges::is_volatile_range(const VolatilePageRange& range) cons
     return m_volatile_ranges.intersects(range);
 }
 
+bool PurgeablePageRanges::is_volatile(size_t index) const
+{
+    ScopedSpinLock lock(m_volatile_ranges_lock);
+    return m_volatile_ranges.contains(index);
+}
+
 void PurgeablePageRanges::set_was_purged(const VolatilePageRange& range)
 {
     ScopedSpinLock lock(m_volatile_ranges_lock);
-    m_volatile_ranges.add({range.base, range.count, true});
+    m_volatile_ranges.add({ range.base, range.count, true });
 }
 
-NonnullRefPtr<PurgeableVMObject> PurgeableVMObject::create_with_size(size_t size)
+void PurgeablePageRanges::set_vmobject(PurgeableVMObject* vmobject)
 {
+    // No lock needed here
+    if (vmobject) {
+        ASSERT(!m_vmobject);
+        m_vmobject = vmobject;
+    } else {
+        ASSERT(m_vmobject);
+        m_vmobject = nullptr;
+    }
+}
+
+RefPtr<PurgeableVMObject> PurgeableVMObject::create_with_size(size_t size)
+{
+    // We need to attempt to commit before actually creating the object
+    if (!MM.commit_user_physical_pages(ceil_div(size, PAGE_SIZE)))
+        return {};
     return adopt(*new PurgeableVMObject(size));
 }
 
 PurgeableVMObject::PurgeableVMObject(size_t size)
-    : AnonymousVMObject(size)
+    : AnonymousVMObject(size, false)
+    , m_unused_committed_pages(page_count())
 {
+    for (size_t i = 0; i < page_count(); ++i)
+        physical_pages()[i] = MM.lazy_committed_page();
 }
 
 PurgeableVMObject::PurgeableVMObject(const PurgeableVMObject& other)
     : AnonymousVMObject(other)
     , m_purgeable_ranges() // do *not* clone this
+    , m_unused_committed_pages(other.m_unused_committed_pages)
 {
-    // TODO: what about m_lock?
+    // We can't really "copy" a spinlock. But we're holding it. Clear in the clone
+    ASSERT(other.m_lock.is_locked());
+    m_lock.initialize();
 }
 
 PurgeableVMObject::~PurgeableVMObject()
 {
+    if (m_unused_committed_pages > 0)
+        MM.uncommit_user_physical_pages(m_unused_committed_pages);
 }
 
-NonnullRefPtr<VMObject> PurgeableVMObject::clone()
+RefPtr<VMObject> PurgeableVMObject::clone()
 {
+    // We need to acquire our lock so we copy a sane state
+    ScopedSpinLock lock(m_lock);
+    if (m_unused_committed_pages > 0) {
+        // We haven't used up all committed pages. In order to be able
+        // to clone ourselves, we need to be able to commit the same number
+        // of pages first
+        if (!MM.commit_user_physical_pages(m_unused_committed_pages))
+            return {};
+    }
     return adopt(*new PurgeableVMObject(*this));
 }
 
@@ -275,8 +329,10 @@ int PurgeableVMObject::purge_impl()
         auto range_end = range.base + range.count;
         for (size_t i = range.base; i < range_end; i++) {
             auto& phys_page = m_physical_pages[i];
-            if (phys_page && !phys_page->is_shared_zero_page())
+            if (phys_page && !phys_page->is_shared_zero_page()) {
+                ASSERT(!phys_page->is_lazy_committed_page());
                 ++purged_in_range;
+            }
             phys_page = MM.shared_zero_page();
         }
 
@@ -291,7 +347,7 @@ int PurgeableVMObject::purge_impl()
                     } else {
                         klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " (no ownership) at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
                     }
-                    region.remap_page_range(range.base, range.count, false);
+                    region.remap_page_range(range.base, range.count);
                 }
             });
         }
@@ -303,6 +359,7 @@ int PurgeableVMObject::purge_impl()
 void PurgeableVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
 {
     ScopedSpinLock lock(m_lock);
+    purgeable_page_ranges.set_vmobject(this);
     ASSERT(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges));
     m_purgeable_ranges.append(&purgeable_page_ranges);
 }
@@ -313,6 +370,7 @@ void PurgeableVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& pu
     for (size_t i = 0; i < m_purgeable_ranges.size(); i++) {
         if (m_purgeable_ranges[i] != &purgeable_page_ranges)
             continue;
+        purgeable_page_ranges.set_vmobject(nullptr);
         m_purgeable_ranges.remove(i);
         return;
     }
@@ -330,4 +388,72 @@ bool PurgeableVMObject::is_any_volatile() const
     return false;
 }
 
+size_t PurgeableVMObject::remove_lazy_commit_pages(const VolatilePageRange& range)
+{
+    ASSERT(m_lock.is_locked());
+
+    size_t removed_count = 0;
+    auto range_end = range.base + range.count;
+    for (size_t i = range.base; i < range_end; i++) {
+        auto& phys_page = m_physical_pages[i];
+        if (phys_page && phys_page->is_lazy_committed_page()) {
+            phys_page = MM.shared_zero_page();
+            removed_count++;
+            ASSERT(m_unused_committed_pages > 0);
+            m_unused_committed_pages--;
+            //            if (--m_unused_committed_pages == 0)
+            //                break;
+        }
+    }
+    return removed_count;
+}
+
+void PurgeableVMObject::range_made_volatile(const VolatilePageRange& range)
+{
+    ASSERT(m_lock.is_locked());
+
+    if (m_unused_committed_pages == 0)
+        return;
+
+    // We need to check this range for any pages that are marked for
+    // lazy committed allocation and turn them into shared zero pages
+    // and also adjust the m_unused_committed_pages for each such page.
+    // Take into account all the other views as well.
+    size_t uncommit_page_count = 0;
+    for_each_volatile_range([&](const auto& r) {
+        auto intersected = range.intersected(r);
+        if (!intersected.is_empty()) {
+            uncommit_page_count += remove_lazy_commit_pages(intersected);
+            //            if (m_unused_committed_pages == 0)
+            //                return IterationDecision::Break;
+        }
+        return IterationDecision::Continue;
+    });
+
+    // Return those committed pages back to the system
+    if (uncommit_page_count > 0)
+        MM.uncommit_user_physical_pages(uncommit_page_count);
+}
+
+RefPtr<PhysicalPage> PurgeableVMObject::allocate_committed_page(size_t page_index)
+{
+    {
+        ScopedSpinLock lock(m_lock);
+
+        ASSERT(m_unused_committed_pages > 0);
+
+        // We should't have any committed page tags in volatile regions
+        ASSERT([&]() {
+            for (auto* purgeable_ranges : m_purgeable_ranges) {
+                if (purgeable_ranges->is_volatile(page_index))
+                    return false;
+            }
+            return true;
+        }());
+
+        m_unused_committed_pages--;
+    }
+    return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
+}
+
 }