From bc5d6992a44be838c225bfd88be0aa4f978ddd66 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 2 Sep 2020 22:57:09 -0600 Subject: [PATCH] Kernel: Memory purging improvements This adds the ability for a Region to define volatile/nonvolatile areas within mapped memory using madvise(). This also means that memory purging takes into account all views of the PurgeableVMObject and only purges memory that is not needed by all of them. When calling madvise() to change an area to nonvolatile memory, return whether memory from that area was purged. At that time also try to remap all memory that is requested to be nonvolatile, and if insufficient pages are available notify the caller of that fact. --- Kernel/FileSystem/ProcFS.cpp | 2 +- Kernel/Heap/SlabAllocator.cpp | 2 +- Kernel/Process.cpp | 10 +- Kernel/SharedBuffer.cpp | 25 +++ Kernel/SharedBuffer.h | 7 + Kernel/Syscalls/mmap.cpp | 33 ++-- Kernel/Syscalls/shbuf.cpp | 24 ++- Kernel/VM/MemoryManager.cpp | 2 +- Kernel/VM/MemoryManager.h | 3 +- Kernel/VM/PurgeableVMObject.cpp | 272 ++++++++++++++++++++++++-- Kernel/VM/PurgeableVMObject.h | 197 ++++++++++++++++++- Kernel/VM/Region.cpp | 114 ++++++++++- Kernel/VM/Region.h | 25 ++- Meta/CMake/all_the_debug_macros.cmake | 1 + 14 files changed, 655 insertions(+), 62 deletions(-) diff --git a/Kernel/FileSystem/ProcFS.cpp b/Kernel/FileSystem/ProcFS.cpp index 5cd22d6de8..05b87abe03 100644 --- a/Kernel/FileSystem/ProcFS.cpp +++ b/Kernel/FileSystem/ProcFS.cpp @@ -331,7 +331,7 @@ static OwnPtr procfs$pid_vm(InodeIdentifier identifier) region_object.add("user_accessible", region.is_user_accessible()); region_object.add("purgeable", region.vmobject().is_purgeable()); if (region.vmobject().is_purgeable()) { - region_object.add("volatile", static_cast(region.vmobject()).is_volatile()); + region_object.add("volatile", static_cast(region.vmobject()).is_any_volatile()); } region_object.add("cacheable", region.is_cacheable()); region_object.add("kernel", region.is_kernel()); diff --git a/Kernel/Heap/SlabAllocator.cpp b/Kernel/Heap/SlabAllocator.cpp index d8f8210752..a306537ca9 100644 --- a/Kernel/Heap/SlabAllocator.cpp +++ b/Kernel/Heap/SlabAllocator.cpp @@ -130,7 +130,7 @@ static SlabAllocator<32> s_slab_allocator_32; static SlabAllocator<64> s_slab_allocator_64; static SlabAllocator<128> s_slab_allocator_128; -static_assert(sizeof(Region) <= s_slab_allocator_64.slab_size()); +static_assert(sizeof(Region) <= s_slab_allocator_128.slab_size()); template void for_each_allocator(Callback callback) diff --git a/Kernel/Process.cpp b/Kernel/Process.cpp index d58a2e0a71..dd750f8889 100644 --- a/Kernel/Process.cpp +++ b/Kernel/Process.cpp @@ -130,7 +130,7 @@ Range Process::allocate_range(VirtualAddress vaddr, size_t size, size_t alignmen Region& Process::allocate_split_region(const Region& source_region, const Range& range, size_t offset_in_vmobject) { - auto& region = add_region(Region::create_user_accessible(range, source_region.vmobject(), offset_in_vmobject, source_region.name(), source_region.access())); + auto& region = add_region(Region::create_user_accessible(this, range, source_region.vmobject(), offset_in_vmobject, source_region.name(), source_region.access())); region.set_mmap(source_region.is_mmap()); region.set_stack(source_region.is_stack()); size_t page_offset_in_source_region = (offset_in_vmobject - source_region.offset_in_vmobject()) / PAGE_SIZE; @@ -145,7 +145,7 @@ Region* Process::allocate_region(const Range& range, const String& name, int pro { ASSERT(range.is_valid()); auto vmobject = AnonymousVMObject::create_with_size(range.size()); - auto region = Region::create_user_accessible(range, vmobject, 0, name, prot_to_region_access_flags(prot)); + auto region = Region::create_user_accessible(this, range, vmobject, 0, name, prot_to_region_access_flags(prot)); region->map(page_directory()); if (should_commit && !region->commit()) return nullptr; @@ -177,7 +177,7 @@ Region* Process::allocate_region_with_vmobject(const Range& range, NonnullRefPtr return nullptr; } offset_in_vmobject &= PAGE_MASK; - auto& region = add_region(Region::create_user_accessible(range, move(vmobject), offset_in_vmobject, name, prot_to_region_access_flags(prot))); + auto& region = add_region(Region::create_user_accessible(this, range, move(vmobject), offset_in_vmobject, name, prot_to_region_access_flags(prot))); region.map(page_directory()); return ®ion; } @@ -762,7 +762,7 @@ size_t Process::amount_purgeable_volatile() const size_t amount = 0; ScopedSpinLock lock(m_lock); for (auto& region : m_regions) { - if (region.vmobject().is_purgeable() && static_cast(region.vmobject()).is_volatile()) + if (region.vmobject().is_purgeable() && static_cast(region.vmobject()).is_any_volatile()) amount += region.amount_resident(); } return amount; @@ -773,7 +773,7 @@ size_t Process::amount_purgeable_nonvolatile() const size_t amount = 0; ScopedSpinLock lock(m_lock); for (auto& region : m_regions) { - if (region.vmobject().is_purgeable() && !static_cast(region.vmobject()).is_volatile()) + if (region.vmobject().is_purgeable() && !static_cast(region.vmobject()).is_any_volatile()) amount += region.amount_resident(); } return amount; diff --git a/Kernel/SharedBuffer.cpp b/Kernel/SharedBuffer.cpp index 0e0682a590..5d62e3e4a5 100644 --- a/Kernel/SharedBuffer.cpp +++ b/Kernel/SharedBuffer.cpp @@ -219,4 +219,29 @@ void SharedBuffer::seal() } } +auto SharedBuffer::set_volatile_all(bool is_volatile, bool& was_purged) -> SetVolatileError +{ + was_purged = false; + auto pid = Process::current()->pid(); + LOCKER(shared_buffers().lock()); + for (size_t i = 0; i < m_refs.size(); ++i) { + auto& ref = m_refs[i]; + if (ref.pid == pid) { + if (Region* region = ref.region.unsafe_ptr()) { + switch (region->set_volatile(region->vaddr(), region->size(), is_volatile, was_purged)) { + case Region::SetVolatileError::Success: + if (!was_purged && was_purged) + klog() << "Region @ " << region->vaddr() << " - " << region->vaddr().offset(region->size()) << " was purged!"; + return SetVolatileError::Success; + case Region::SetVolatileError::NotPurgeable: + return SetVolatileError::NotPurgeable; + case Region::SetVolatileError::OutOfMemory: + return SetVolatileError::OutOfMemory; + } + } + } + } + return SetVolatileError::NotMapped; +} + } diff --git a/Kernel/SharedBuffer.h b/Kernel/SharedBuffer.h index acdcfe34d6..02704b3d42 100644 --- a/Kernel/SharedBuffer.h +++ b/Kernel/SharedBuffer.h @@ -75,6 +75,13 @@ public: size_t size() const { return m_vmobject->size(); } void destroy_if_unused(); void seal(); + enum class SetVolatileError { + Success = 0, + NotPurgeable, + OutOfMemory, + NotMapped + }; + SetVolatileError set_volatile_all(bool is_volatile, bool& was_purged); PurgeableVMObject& vmobject() { return m_vmobject; } const PurgeableVMObject& vmobject() const { return m_vmobject; } int id() const { return m_shbuf_id; } diff --git a/Kernel/Syscalls/mmap.cpp b/Kernel/Syscalls/mmap.cpp index a5209cb476..20e9f379ae 100644 --- a/Kernel/Syscalls/mmap.cpp +++ b/Kernel/Syscalls/mmap.cpp @@ -282,31 +282,30 @@ int Process::sys$madvise(void* address, size_t size, int advice) return -EINVAL; if (!region->is_mmap()) return -EPERM; - if ((advice & MADV_SET_VOLATILE) && (advice & MADV_SET_NONVOLATILE)) + bool set_volatile = advice & MADV_SET_VOLATILE; + bool set_nonvolatile = advice & MADV_SET_NONVOLATILE; + if (set_volatile && set_nonvolatile) return -EINVAL; - if (advice & MADV_SET_VOLATILE) { + if (set_volatile || set_nonvolatile) { if (!region->vmobject().is_purgeable()) return -EPERM; - auto& vmobject = static_cast(region->vmobject()); - vmobject.set_volatile(true); + bool was_purged = false; + switch (region->set_volatile(VirtualAddress(address), size, set_volatile, was_purged)) { + case Region::SetVolatileError::Success: + break; + case Region::SetVolatileError::NotPurgeable: + return -EPERM; + case Region::SetVolatileError::OutOfMemory: + return -ENOMEM; + } + if (set_nonvolatile) + return was_purged ? 1 : 0; return 0; } - if (advice & MADV_SET_NONVOLATILE) { - if (!region->vmobject().is_purgeable()) - return -EPERM; - auto& vmobject = static_cast(region->vmobject()); - if (!vmobject.is_volatile()) - return 0; - vmobject.set_volatile(false); - bool was_purged = vmobject.was_purged(); - vmobject.set_was_purged(false); - return was_purged ? 1 : 0; - } if (advice & MADV_GET_VOLATILE) { if (!region->vmobject().is_purgeable()) return -EPERM; - auto& vmobject = static_cast(region->vmobject()); - return vmobject.is_volatile() ? 0 : 1; + return region->is_volatile(VirtualAddress(address), size) ? 0 : 1; } return -EINVAL; } diff --git a/Kernel/Syscalls/shbuf.cpp b/Kernel/Syscalls/shbuf.cpp index 03ff104c62..fc2abfa805 100644 --- a/Kernel/Syscalls/shbuf.cpp +++ b/Kernel/Syscalls/shbuf.cpp @@ -174,14 +174,28 @@ int Process::sys$shbuf_set_volatile(int shbuf_id, bool state) #ifdef SHARED_BUFFER_DEBUG klog() << "Set shared buffer " << shbuf_id << " volatile: " << state; #endif + + bool was_purged = false; + auto set_volatile = [&]() -> int { + switch (shared_buffer.set_volatile_all(state, was_purged)) { + case SharedBuffer::SetVolatileError::Success: + break; + case SharedBuffer::SetVolatileError::NotPurgeable: + return -EPERM; + case SharedBuffer::SetVolatileError::OutOfMemory: + return -ENOMEM; + case SharedBuffer::SetVolatileError::NotMapped: + return -EINVAL; + } + return 0; + }; + if (!state) { - bool was_purged = shared_buffer.vmobject().was_purged(); - shared_buffer.vmobject().set_volatile(state); - shared_buffer.vmobject().set_was_purged(false); + if (int err = set_volatile()) + return err; return was_purged ? 1 : 0; } - shared_buffer.vmobject().set_volatile(true); - return 0; + return set_volatile(); } } diff --git a/Kernel/VM/MemoryManager.cpp b/Kernel/VM/MemoryManager.cpp index 622b0ba4ca..d5905401a3 100644 --- a/Kernel/VM/MemoryManager.cpp +++ b/Kernel/VM/MemoryManager.cpp @@ -451,7 +451,7 @@ OwnPtr MemoryManager::allocate_kernel_region_with_vmobject(const Range& ScopedSpinLock lock(s_mm_lock); OwnPtr region; if (user_accessible) - region = Region::create_user_accessible(range, vmobject, 0, name, access, cacheable); + region = Region::create_user_accessible(nullptr, range, vmobject, 0, name, access, cacheable); else region = Region::create_kernel_only(range, vmobject, 0, name, access, cacheable); if (region) diff --git a/Kernel/VM/MemoryManager.h b/Kernel/VM/MemoryManager.h index 980b18b1bc..8b9eeb46ff 100644 --- a/Kernel/VM/MemoryManager.h +++ b/Kernel/VM/MemoryManager.h @@ -38,7 +38,8 @@ namespace Kernel { -#define PAGE_ROUND_UP(x) ((((u32)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1))) +#define PAGE_ROUND_UP(x) ((((FlatPtr)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1))) +#define PAGE_ROUND_DOWN(x) (((FlatPtr)(x)) & ~(PAGE_SIZE - 1)) template inline T* low_physical_to_virtual(T* physical) diff --git a/Kernel/VM/PurgeableVMObject.cpp b/Kernel/VM/PurgeableVMObject.cpp index 8f1af0e1cf..c944b3882d 100644 --- a/Kernel/VM/PurgeableVMObject.cpp +++ b/Kernel/VM/PurgeableVMObject.cpp @@ -24,12 +24,201 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include +#include +#include #include #include #include +//#define VOLATILE_PAGE_RANGES_DEBUG + namespace Kernel { +#ifdef VOLATILE_PAGE_RANGES_DEBUG +inline LogStream& operator<<(const LogStream& stream, const VolatilePageRange& range) +{ + stream << "{" << range.base << " (" << range.count << ") purged: " << range.was_purged << "}"; + return const_cast(stream); +} + +static void dump_volatile_page_ranges(const Vector& ranges) +{ + for (size_t i = 0; i < ranges.size(); i++) { + const auto& range = ranges[i]; + klog() << " [" << i << "] " << range; + } +} +#endif + +bool VolatilePageRanges::add(const VolatilePageRange& range) +{ + auto add_range = m_total_range.intersected(range); + if (add_range.is_empty()) + return false; + add_range.was_purged = range.was_purged; + +#ifdef VOLATILE_PAGE_RANGES_DEBUG + klog() << "ADD " << range << " (total range: " << m_total_range << ") -->"; + dump_volatile_page_ranges(m_ranges); + ScopeGuard debug_guard([&]() { + klog() << "After adding " << range << " (total range: " << m_total_range << ")"; + dump_volatile_page_ranges(m_ranges); + klog() << "<-- ADD " << range << " (total range: " << m_total_range << ")"; + }); +#endif + + size_t nearby_index = 0; + auto* existing_range = binary_search( + m_ranges.span(), add_range, &nearby_index, [](auto& a, auto& b) { + if (a.intersects_or_adjacent(b)) + return 0; + return (signed)(a.base - (b.base + b.count - 1)); + }); + + size_t inserted_index = 0; + if (existing_range) { + if (*existing_range == add_range) + return false; + + if (existing_range->was_purged != add_range.was_purged) { + // Found an intersecting or adjacent range, but the purge flag + // doesn't match. Subtract what we're adding from it, and + existing_range->subtract_intersecting(add_range); + if (existing_range->is_empty()) { + *existing_range = add_range; + } else { + m_ranges.insert(++nearby_index, add_range); + existing_range = &m_ranges[nearby_index]; + } + } else { + // Found an intersecting or adjacent range that can be merged + existing_range->combine_intersecting_or_adjacent(add_range); + } + inserted_index = nearby_index; + } else { + // Insert into the sorted list + m_ranges.insert_before_matching( + VolatilePageRange(add_range), [&](auto& entry) { + return entry.base >= add_range.base + add_range.count; + }, + nearby_index, &inserted_index); + existing_range = &m_ranges[inserted_index]; + } + + // See if we can merge any of the following ranges + inserted_index++; + while (inserted_index < m_ranges.size()) { + auto& next_range = m_ranges[inserted_index]; + if (!next_range.intersects_or_adjacent(*existing_range)) + break; + if (next_range.was_purged != existing_range->was_purged) { + // The purged flag of following range is not the same. + // Subtract the added/combined range from it + next_range.subtract_intersecting(*existing_range); + if (next_range.is_empty()) + m_ranges.remove(inserted_index); + } else { + existing_range->combine_intersecting_or_adjacent(next_range); + m_ranges.remove(inserted_index); + } + } + return true; +} + +bool VolatilePageRanges::remove(const VolatilePageRange& range, bool& was_purged) +{ + auto remove_range = m_total_range.intersected(range); + if (remove_range.is_empty()) + return false; + +#ifdef VOLATILE_PAGE_RANGES_DEBUG + klog() << "REMOVE " << range << " (total range: " << m_total_range << ") -->"; + dump_volatile_page_ranges(m_ranges); + ScopeGuard debug_guard([&]() { + klog() << "After removing " << range << " (total range: " << m_total_range << ")"; + dump_volatile_page_ranges(m_ranges); + klog() << "<-- REMOVE " << range << " (total range: " << m_total_range << ") was_purged: " << was_purged; + }); +#endif + + size_t nearby_index = 0; + auto* existing_range = binary_search( + m_ranges.span(), remove_range, &nearby_index, [](auto& a, auto& b) { + if (a.intersects(b)) + return 0; + return (signed)(a.base - (b.base + b.count - 1)); + }); + if (!existing_range) + return false; + + was_purged = existing_range->was_purged; + if (existing_range->range_equals(remove_range)) { + m_ranges.remove(nearby_index); + } else { + // See if we need to remove any of the following ranges + ASSERT(existing_range == &m_ranges[nearby_index]); // sanity check + while (nearby_index < m_ranges.size()) { + existing_range = &m_ranges[nearby_index]; + if (!existing_range->intersects(range)) + break; + was_purged |= existing_range->was_purged; + existing_range->subtract_intersecting(remove_range); + if (existing_range->is_empty()) { + m_ranges.remove(nearby_index); + break; + } + } + } + return true; +} + +bool VolatilePageRanges::intersects(const VolatilePageRange& range) const +{ + auto* existing_range = binary_search( + m_ranges.span(), range, nullptr, [](auto& a, auto& b) { + if (a.intersects(b)) + return 0; + return (signed)(a.base - (b.base + b.count - 1)); + }); + return existing_range != nullptr; +} + +PurgeablePageRanges::PurgeablePageRanges(const VMObject& vmobject) + : m_volatile_ranges({0, vmobject.is_purgeable() ? static_cast(vmobject).page_count() : 0}) +{ +} + +bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range) +{ + if (range.is_empty()) + return false; + ScopedSpinLock lock(m_volatile_ranges_lock); + return m_volatile_ranges.add(range); +} + +bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged) +{ + if (range.is_empty()) + return false; + ScopedSpinLock lock(m_volatile_ranges_lock); + return m_volatile_ranges.remove(range, was_purged); +} + +bool PurgeablePageRanges::is_volatile_range(const VolatilePageRange& range) const +{ + if (range.is_empty()) + return false; + ScopedSpinLock lock(m_volatile_ranges_lock); + return m_volatile_ranges.intersects(range); +} + +void PurgeablePageRanges::set_was_purged(const VolatilePageRange& range) +{ + ScopedSpinLock lock(m_volatile_ranges_lock); + m_volatile_ranges.add({range.base, range.count, true}); +} + NonnullRefPtr PurgeableVMObject::create_with_size(size_t size) { return adopt(*new PurgeableVMObject(size)); @@ -42,9 +231,9 @@ PurgeableVMObject::PurgeableVMObject(size_t size) PurgeableVMObject::PurgeableVMObject(const PurgeableVMObject& other) : AnonymousVMObject(other) - , m_was_purged(other.m_was_purged) - , m_volatile(other.m_volatile) + , m_purgeable_ranges() // do *not* clone this { + // TODO: what about m_lock? } PurgeableVMObject::~PurgeableVMObject() @@ -70,26 +259,75 @@ int PurgeableVMObject::purge_with_interrupts_disabled(Badge) return purge_impl(); } +void PurgeableVMObject::set_was_purged(const VolatilePageRange& range) +{ + ASSERT(m_lock.is_locked()); + for (auto* purgeable_ranges : m_purgeable_ranges) + purgeable_ranges->set_was_purged(range); +} + int PurgeableVMObject::purge_impl() { - if (!m_volatile) - return 0; int purged_page_count = 0; - for (size_t i = 0; i < m_physical_pages.size(); ++i) { - if (m_physical_pages[i] && !m_physical_pages[i]->is_shared_zero_page()) - ++purged_page_count; - m_physical_pages[i] = MM.shared_zero_page(); - } - m_was_purged = true; - - if (purged_page_count > 0) { - for_each_region([&](auto& region) { - if (®ion.vmobject() == this) - region.remap(); - }); - } + ScopedSpinLock lock(m_lock); + for_each_volatile_range([&](const auto& range) { + int purged_in_range = 0; + auto range_end = range.base + range.count; + for (size_t i = range.base; i < range_end; i++) { + auto& phys_page = m_physical_pages[i]; + if (phys_page && !phys_page->is_shared_zero_page()) + ++purged_in_range; + phys_page = MM.shared_zero_page(); + } + if (purged_in_range > 0) { + purged_page_count += purged_in_range; + set_was_purged(range); + for_each_region([&](auto& region) { + if (®ion.vmobject() == this) { + if (auto owner = region.get_owner()) { + // we need to hold a reference the process here (if there is one) as we may not own this region + klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " owned by " << *owner << " at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count); + } else { + klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " (no ownership) at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count); + } + region.remap_page_range(range.base, range.count); + } + }); + } + return IterationDecision::Continue; + }); return purged_page_count; } +void PurgeableVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges) +{ + ScopedSpinLock lock(m_lock); + ASSERT(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges)); + m_purgeable_ranges.append(&purgeable_page_ranges); +} + +void PurgeableVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges) +{ + ScopedSpinLock lock(m_lock); + for (size_t i = 0; i < m_purgeable_ranges.size(); i++) { + if (m_purgeable_ranges[i] != &purgeable_page_ranges) + continue; + m_purgeable_ranges.remove(i); + return; + } + ASSERT_NOT_REACHED(); +} + +bool PurgeableVMObject::is_any_volatile() const +{ + ScopedSpinLock lock(m_lock); + for (auto& volatile_ranges : m_purgeable_ranges) { + ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock); + if (!volatile_ranges->is_empty()) + return true; + } + return false; +} + } diff --git a/Kernel/VM/PurgeableVMObject.h b/Kernel/VM/PurgeableVMObject.h index 5e100aa5f0..c3fd4d2791 100644 --- a/Kernel/VM/PurgeableVMObject.h +++ b/Kernel/VM/PurgeableVMObject.h @@ -26,10 +26,159 @@ #pragma once +#include #include namespace Kernel { +struct VolatilePageRange { + size_t base { 0 }; + size_t count { 0 }; + bool was_purged { false }; + + bool is_empty() const { return count == 0; } + + bool intersects(const VolatilePageRange& other) const + { + return other.base < base + count || other.base + other.count > base; + } + + bool intersects_or_adjacent(const VolatilePageRange& other) const + { + return other.base <= base + count || other.base + other.count >= base; + } + + bool contains(const VolatilePageRange& other) const + { + return base <= other.base && base + count >= other.base + other.count; + } + + VolatilePageRange intersected(const VolatilePageRange& other) const + { + auto b = max(base, other.base); + auto e = min(base + count, other.base + other.count); + if (b >= e) + return {}; + return { b, e - b, was_purged }; + } + + void combine_intersecting_or_adjacent(const VolatilePageRange& other) + { + ASSERT(intersects_or_adjacent(other)); + if (base <= other.base) { + count = (other.base - base) + other.count; + } else { + count = (base - other.base) + count; + base = other.base; + } + was_purged |= other.was_purged; + } + + void subtract_intersecting(const VolatilePageRange& other) + { + if (!intersects(other)) + return; + if (other.contains(*this)) { + count = 0; + return; + } + if (base <= other.base) { + count = (other.base - base); + } else { + auto new_base = other.base + other.count; + count = (base + count) - new_base; + base = new_base; + } + } + + bool range_equals(const VolatilePageRange& other) const + { + return base == other.base && count == other.count; + } + bool operator==(const VolatilePageRange& other) const + { + return base == other.base && count == other.count && was_purged == other.was_purged; + } + bool operator!=(const VolatilePageRange& other) const + { + return base != other.base || count != other.count || was_purged != other.was_purged; + } +}; + +class VolatilePageRanges { +public: + VolatilePageRanges(const VolatilePageRange& total_range) + : m_total_range(total_range) + { + } + VolatilePageRanges(const VolatilePageRanges& other) + : m_ranges(other.m_ranges) + , m_total_range(other.m_total_range) + { + } + + bool is_empty() const { return m_ranges.is_empty(); } + void clear() { m_ranges.clear(); } + + bool is_all() const + { + if (m_ranges.size() != 1) + return false; + return m_ranges[0] == m_total_range; + } + + void set_all() + { + if (m_ranges.size() != 1) + m_ranges = { m_total_range }; + else + m_ranges[0] = m_total_range; + } + + bool intersects(const VolatilePageRange&) const; + + bool add(const VolatilePageRange&); + bool remove(const VolatilePageRange&, bool&); + + Vector& ranges() { return m_ranges; } + const Vector& ranges() const { return m_ranges; } + +private: + Vector m_ranges; + VolatilePageRange m_total_range; +}; + +class PurgeableVMObject; + +class PurgeablePageRanges { + friend class PurgeableVMObject; +public: + PurgeablePageRanges(const VMObject&); + + void set_purgeable_page_ranges(const PurgeablePageRanges& other) + { + if (this == &other) + return; + ScopedSpinLock lock(m_volatile_ranges_lock); + ScopedSpinLock other_lock(other.m_volatile_ranges_lock); + m_volatile_ranges = other.m_volatile_ranges; + return; + } + + bool add_volatile_range(const VolatilePageRange& range); + bool remove_volatile_range(const VolatilePageRange& range, bool& was_purged); + bool is_volatile_range(const VolatilePageRange& range) const; + + bool is_empty() const { return m_volatile_ranges.is_empty(); } + + void set_was_purged(const VolatilePageRange&); + + const VolatilePageRanges& volatile_ranges() const { return m_volatile_ranges; } +protected: + VolatilePageRanges m_volatile_ranges; + mutable SpinLock m_volatile_ranges_lock; +}; + class PurgeableVMObject final : public AnonymousVMObject { public: virtual ~PurgeableVMObject() override; @@ -37,14 +186,49 @@ public: static NonnullRefPtr create_with_size(size_t); virtual NonnullRefPtr clone() override; + void register_purgeable_page_ranges(PurgeablePageRanges&); + void unregister_purgeable_page_ranges(PurgeablePageRanges&); + int purge(); int purge_with_interrupts_disabled(Badge); - bool was_purged() const { return m_was_purged; } - void set_was_purged(bool b) { m_was_purged = b; } + bool is_any_volatile() const; - bool is_volatile() const { return m_volatile; } - void set_volatile(bool b) { m_volatile = b; } + template + IterationDecision for_each_volatile_range(F f) + { + ASSERT(m_lock.is_locked()); + // This is a little ugly. Basically, we're trying to find the + // volatile ranges that all share, because those are the only + // pages we can actually purge + for (auto* purgeable_range : m_purgeable_ranges) { + for (auto& r1 : purgeable_range->volatile_ranges().ranges()) { + VolatilePageRange range(r1); + for (auto* purgeable_range2 : m_purgeable_ranges) { + if (purgeable_range2 == purgeable_range) + continue; + if (purgeable_range2->is_empty()) { + // If just one doesn't allow any purging, we can + // immediately bail + return IterationDecision::Continue; + } + for (const auto& r2 : purgeable_range2->volatile_ranges().ranges()) { + range = range.intersected(r2); + if (range.is_empty()) + break; + } + if (range.is_empty()) + break; + } + if (range.is_empty()) + continue; + IterationDecision decision = f(range); + if (decision != IterationDecision::Continue) + return decision; + } + } + return IterationDecision::Continue; + } private: explicit PurgeableVMObject(size_t); @@ -53,6 +237,7 @@ private: virtual const char* class_name() const override { return "PurgeableVMObject"; } int purge_impl(); + void set_was_purged(const VolatilePageRange&); PurgeableVMObject& operator=(const PurgeableVMObject&) = delete; PurgeableVMObject& operator=(PurgeableVMObject&&) = delete; @@ -60,8 +245,8 @@ private: virtual bool is_purgeable() const override { return true; } - bool m_was_purged { false }; - bool m_volatile { false }; + Vector m_purgeable_ranges; + mutable SpinLock m_lock; }; } diff --git a/Kernel/VM/Region.cpp b/Kernel/VM/Region.cpp index ac4899203f..22d49c60a1 100644 --- a/Kernel/VM/Region.cpp +++ b/Kernel/VM/Region.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -41,7 +42,8 @@ namespace Kernel { Region::Region(const Range& range, NonnullRefPtr vmobject, size_t offset_in_vmobject, const String& name, u8 access, bool cacheable, bool kernel) - : m_range(range) + : PurgeablePageRanges(vmobject) + , m_range(range) , m_offset_in_vmobject(offset_in_vmobject) , m_vmobject(move(vmobject)) , m_name(name) @@ -49,11 +51,14 @@ Region::Region(const Range& range, NonnullRefPtr vmobject, size_t offs , m_cacheable(cacheable) , m_kernel(kernel) { + register_purgeable_page_ranges(); MM.register_region(*this); } Region::~Region() { + unregister_purgeable_page_ranges(); + // Make sure we disable interrupts so we don't get interrupted between unmapping and unregistering. // Unmapping the region will give the VM back to the RangeAllocator, so an interrupt handler would // find the address<->region mappings in an invalid state there. @@ -62,9 +67,26 @@ Region::~Region() unmap(ShouldDeallocateVirtualMemoryRange::Yes); ASSERT(!m_page_directory); } + MM.unregister_region(*this); } +void Region::register_purgeable_page_ranges() +{ + if (m_vmobject->is_purgeable()) { + auto& vmobject = static_cast(*m_vmobject); + vmobject.register_purgeable_page_ranges(*this); + } +} + +void Region::unregister_purgeable_page_ranges() +{ + if (m_vmobject->is_purgeable()) { + auto& vmobject = static_cast(*m_vmobject); + vmobject.unregister_purgeable_page_ranges(*this); + } +} + NonnullOwnPtr Region::clone() { ASSERT(Process::current()); @@ -74,7 +96,8 @@ NonnullOwnPtr Region::clone() ASSERT(m_mmap); ASSERT(!m_shared); ASSERT(vmobject().is_anonymous()); - auto zeroed_region = Region::create_user_accessible(m_range, AnonymousVMObject::create_with_size(size()), 0, m_name, m_access); + auto zeroed_region = Region::create_user_accessible(get_owner().ptr(), m_range, AnonymousVMObject::create_with_size(size()), 0, m_name, m_access); + zeroed_region->set_purgeable_page_ranges(*this); zeroed_region->set_mmap(m_mmap); zeroed_region->set_inherit_mode(m_inherit_mode); return zeroed_region; @@ -89,7 +112,8 @@ NonnullOwnPtr Region::clone() ASSERT(vmobject().is_shared_inode()); // Create a new region backed by the same VMObject. - auto region = Region::create_user_accessible(m_range, m_vmobject, m_offset_in_vmobject, m_name, m_access); + auto region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject, m_offset_in_vmobject, m_name, m_access); + region->set_purgeable_page_ranges(*this); region->set_mmap(m_mmap); region->set_shared(m_shared); return region; @@ -104,7 +128,8 @@ NonnullOwnPtr Region::clone() // Set up a COW region. The parent (this) region becomes COW as well! ensure_cow_map().fill(true); remap(); - auto clone_region = Region::create_user_accessible(m_range, m_vmobject->clone(), m_offset_in_vmobject, m_name, m_access); + auto clone_region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject->clone(), m_offset_in_vmobject, m_name, m_access); + clone_region->set_purgeable_page_ranges(*this); clone_region->ensure_cow_map(); if (m_stack) { ASSERT(is_readable()); @@ -116,6 +141,59 @@ NonnullOwnPtr Region::clone() return clone_region; } +void Region::set_vmobject(NonnullRefPtr&& obj) +{ + if (m_vmobject.ptr() == obj.ptr()) + return; + unregister_purgeable_page_ranges(); + m_vmobject = move(obj); + register_purgeable_page_ranges(); +} + +bool Region::is_volatile(VirtualAddress vaddr, size_t size) const +{ + if (!m_vmobject->is_purgeable()) + return false; + + auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject); + size_t first_page_index = PAGE_ROUND_DOWN(offset_in_vmobject) / PAGE_SIZE; + size_t last_page_index = PAGE_ROUND_UP(offset_in_vmobject + size) / PAGE_SIZE; + return is_volatile_range({ first_page_index, last_page_index - first_page_index }); +} + +auto Region::set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged) -> SetVolatileError +{ + was_purged = false; + if (!m_vmobject->is_purgeable()) + return SetVolatileError::NotPurgeable; + + auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject); + if (is_volatile) { + // If marking pages as volatile, be prudent by not marking + // partial pages volatile to prevent potentially non-volatile + // data to be discarded. So rund up the first page and round + // down the last page. + size_t first_page_index = PAGE_ROUND_UP(offset_in_vmobject) / PAGE_SIZE; + size_t last_page_index = PAGE_ROUND_DOWN(offset_in_vmobject + size) / PAGE_SIZE; + if (first_page_index != last_page_index) + add_volatile_range({ first_page_index, last_page_index - first_page_index }); + } else { + // If marking pages as non-volatile, round down the first page + // and round up the last page to make sure the beginning and + // end of the range doesn't inadvertedly get discarded. + size_t first_page_index = PAGE_ROUND_DOWN(offset_in_vmobject) / PAGE_SIZE; + size_t last_page_index = PAGE_ROUND_UP(offset_in_vmobject + size) / PAGE_SIZE; + if (remove_volatile_range({ first_page_index, last_page_index - first_page_index }, was_purged)) { + // Attempt to remap the page range. We want to make sure we have + // enough memory, if not we need to inform the caller of that + // fact + if (!remap_page_range(first_page_index, last_page_index - first_page_index)) + return SetVolatileError::OutOfMemory; + } + } + return SetVolatileError::Success; +} + bool Region::commit() { ScopedSpinLock lock(s_mm_lock); @@ -190,9 +268,11 @@ size_t Region::amount_shared() const return bytes; } -NonnullOwnPtr Region::create_user_accessible(const Range& range, NonnullRefPtr vmobject, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable) +NonnullOwnPtr Region::create_user_accessible(Process* owner, const Range& range, NonnullRefPtr vmobject, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable) { auto region = make(range, move(vmobject), offset_in_vmobject, name, access, cacheable, false); + if (owner) + region->m_owner = owner->make_weak_ptr(); region->m_user_accessible = true; return region; } @@ -259,6 +339,25 @@ bool Region::map_individual_page_impl(size_t page_index) return true; } +bool Region::remap_page_range(size_t page_index, size_t page_count) +{ + bool success = true; + ScopedSpinLock lock(s_mm_lock); + ASSERT(m_page_directory); + ScopedSpinLock page_lock(m_page_directory->get_lock()); + size_t index = page_index; + while (index < page_index + page_count) { + if (!map_individual_page_impl(index)) { + success = false; + break; + } + index++; + } + if (index > page_index) + MM.flush_tlb(vaddr_from_page_index(page_index), index - page_index); + return success; +} + bool Region::remap_page(size_t page_index, bool with_flush) { ScopedSpinLock lock(s_mm_lock); @@ -534,4 +633,9 @@ PageFaultResponse Region::handle_inode_fault(size_t page_index_in_region) return PageFaultResponse::Continue; } +RefPtr Region::get_owner() +{ + return m_owner.strong_ref(); +} + } diff --git a/Kernel/VM/Region.h b/Kernel/VM/Region.h index 575b595186..46159d8c8c 100644 --- a/Kernel/VM/Region.h +++ b/Kernel/VM/Region.h @@ -28,9 +28,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -47,7 +49,8 @@ enum class PageFaultResponse { class Region final : public InlineLinkedListNode - , public Weakable { + , public Weakable + , public PurgeablePageRanges { friend class MemoryManager; MAKE_SLAB_ALLOCATED(Region) @@ -63,7 +66,7 @@ public: ZeroedOnFork, }; - static NonnullOwnPtr create_user_accessible(const Range&, NonnullRefPtr, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true); + static NonnullOwnPtr create_user_accessible(Process*, const Range&, NonnullRefPtr, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true); static NonnullOwnPtr create_kernel_only(const Range&, NonnullRefPtr, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true); ~Region(); @@ -83,7 +86,7 @@ public: const VMObject& vmobject() const { return *m_vmobject; } VMObject& vmobject() { return *m_vmobject; } - void set_vmobject(NonnullRefPtr&& obj) { m_vmobject = obj; } + void set_vmobject(NonnullRefPtr&&); bool is_shared() const { return m_shared; } void set_shared(bool shared) { m_shared = shared; } @@ -190,6 +193,18 @@ public: void set_inherit_mode(InheritMode inherit_mode) { m_inherit_mode = inherit_mode; } + bool remap_page_range(size_t page_index, size_t page_count); + + bool is_volatile(VirtualAddress vaddr, size_t size) const; + enum class SetVolatileError { + Success = 0, + NotPurgeable, + OutOfMemory + }; + SetVolatileError set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged); + + RefPtr get_owner(); + private: Bitmap& ensure_cow_map() const; @@ -210,6 +225,9 @@ private: bool map_individual_page_impl(size_t page_index); + void register_purgeable_page_ranges(); + void unregister_purgeable_page_ranges(); + RefPtr m_page_directory; Range m_range; size_t m_offset_in_vmobject { 0 }; @@ -224,6 +242,7 @@ private: bool m_mmap : 1 { false }; bool m_kernel : 1 { false }; mutable OwnPtr m_cow_map; + WeakPtr m_owner; }; inline unsigned prot_to_region_access_flags(int prot) diff --git a/Meta/CMake/all_the_debug_macros.cmake b/Meta/CMake/all_the_debug_macros.cmake index e0e637f07a..f687d1804e 100644 --- a/Meta/CMake/all_the_debug_macros.cmake +++ b/Meta/CMake/all_the_debug_macros.cmake @@ -156,6 +156,7 @@ add_compile_definitions("UPDATE_COALESCING_DEBUG") add_compile_definitions("VERY_DEBUG") add_compile_definitions("VFS_DEBUG") add_compile_definitions("VMWAREBACKDOOR_DEBUG") +add_compile_definitions("VOLATILE_PAGE_RANGES_DEBUG") add_compile_definitions("VRA_DEBUG") add_compile_definitions("WAITBLOCK_DEBUG") add_compile_definitions("WAITQUEUE_DEBUG")