From 843d0d0d15dbcf3dd9591e235f0be1871d528e47 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Thu, 5 Aug 2021 15:33:26 +0200 Subject: [PATCH] Kernel: Detach AnonymousVMObject from shared COW pages set once emptied We currently overcommit for COW when forking a process and cloning its memory regions. Both the parent and child process share a set of. committed COW pages. If there's COW sharing across more than two processeses within a lineage (e.g parent, child & grandchild), it's possible to exhaust these pages. When the shared set is emptied, the next COW fault in each process must detach from the shared set and fall back to on demand allocation. This patch makes sure that we detach from the shared set once we discover it to be empty (during COW fault handling). This fixes an issue where we'd try to allocate from an exhausted shared set while building GNU binutils inside SerenityOS. --- Kernel/VM/AnonymousVMObject.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Kernel/VM/AnonymousVMObject.cpp b/Kernel/VM/AnonymousVMObject.cpp index ad9ab7e92c..6807c44da0 100644 --- a/Kernel/VM/AnonymousVMObject.cpp +++ b/Kernel/VM/AnonymousVMObject.cpp @@ -307,15 +307,14 @@ PageFaultResponse AnonymousVMObject::handle_cow_fault(size_t page_index, Virtual auto& page_slot = physical_pages()[page_index]; + // If we were sharing committed COW pages with another process, and the other process + // has exhausted the supply, we can stop counting the shared pages. + if (m_shared_committed_cow_pages && m_shared_committed_cow_pages->is_empty()) + m_shared_committed_cow_pages = nullptr; + if (page_slot->ref_count() == 1) { dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page but nobody is sharing it anymore. Remap r/w"); set_should_cow(page_index, false); - - // If we were sharing committed COW pages with another process, and the other process - // has exhausted the supply, we can stop counting the shared pages. - if (m_shared_committed_cow_pages && m_shared_committed_cow_pages->is_empty()) - m_shared_committed_cow_pages = nullptr; - return PageFaultResponse::Continue; }