diff --git a/Kernel/FileSystem/ProcFS.cpp b/Kernel/FileSystem/ProcFS.cpp index 5cd22d6de8..05b87abe03 100644 --- a/Kernel/FileSystem/ProcFS.cpp +++ b/Kernel/FileSystem/ProcFS.cpp @@ -331,7 +331,7 @@ static OwnPtr procfs$pid_vm(InodeIdentifier identifier) region_object.add("user_accessible", region.is_user_accessible()); region_object.add("purgeable", region.vmobject().is_purgeable()); if (region.vmobject().is_purgeable()) { - region_object.add("volatile", static_cast(region.vmobject()).is_volatile()); + region_object.add("volatile", static_cast(region.vmobject()).is_any_volatile()); } region_object.add("cacheable", region.is_cacheable()); region_object.add("kernel", region.is_kernel()); diff --git a/Kernel/Heap/SlabAllocator.cpp b/Kernel/Heap/SlabAllocator.cpp index d8f8210752..a306537ca9 100644 --- a/Kernel/Heap/SlabAllocator.cpp +++ b/Kernel/Heap/SlabAllocator.cpp @@ -130,7 +130,7 @@ static SlabAllocator<32> s_slab_allocator_32; static SlabAllocator<64> s_slab_allocator_64; static SlabAllocator<128> s_slab_allocator_128; -static_assert(sizeof(Region) <= s_slab_allocator_64.slab_size()); +static_assert(sizeof(Region) <= s_slab_allocator_128.slab_size()); template void for_each_allocator(Callback callback) diff --git a/Kernel/Process.cpp b/Kernel/Process.cpp index d58a2e0a71..dd750f8889 100644 --- a/Kernel/Process.cpp +++ b/Kernel/Process.cpp @@ -130,7 +130,7 @@ Range Process::allocate_range(VirtualAddress vaddr, size_t size, size_t alignmen Region& Process::allocate_split_region(const Region& source_region, const Range& range, size_t offset_in_vmobject) { - auto& region = add_region(Region::create_user_accessible(range, source_region.vmobject(), offset_in_vmobject, source_region.name(), source_region.access())); + auto& region = add_region(Region::create_user_accessible(this, range, source_region.vmobject(), offset_in_vmobject, source_region.name(), source_region.access())); region.set_mmap(source_region.is_mmap()); region.set_stack(source_region.is_stack()); size_t page_offset_in_source_region = (offset_in_vmobject - source_region.offset_in_vmobject()) / PAGE_SIZE; @@ -145,7 +145,7 @@ Region* Process::allocate_region(const Range& range, const String& name, int pro { ASSERT(range.is_valid()); auto vmobject = AnonymousVMObject::create_with_size(range.size()); - auto region = Region::create_user_accessible(range, vmobject, 0, name, prot_to_region_access_flags(prot)); + auto region = Region::create_user_accessible(this, range, vmobject, 0, name, prot_to_region_access_flags(prot)); region->map(page_directory()); if (should_commit && !region->commit()) return nullptr; @@ -177,7 +177,7 @@ Region* Process::allocate_region_with_vmobject(const Range& range, NonnullRefPtr return nullptr; } offset_in_vmobject &= PAGE_MASK; - auto& region = add_region(Region::create_user_accessible(range, move(vmobject), offset_in_vmobject, name, prot_to_region_access_flags(prot))); + auto& region = add_region(Region::create_user_accessible(this, range, move(vmobject), offset_in_vmobject, name, prot_to_region_access_flags(prot))); region.map(page_directory()); return ®ion; } @@ -762,7 +762,7 @@ size_t Process::amount_purgeable_volatile() const size_t amount = 0; ScopedSpinLock lock(m_lock); for (auto& region : m_regions) { - if (region.vmobject().is_purgeable() && static_cast(region.vmobject()).is_volatile()) + if (region.vmobject().is_purgeable() && static_cast(region.vmobject()).is_any_volatile()) amount += region.amount_resident(); } return amount; @@ -773,7 +773,7 @@ size_t Process::amount_purgeable_nonvolatile() const size_t amount = 0; ScopedSpinLock lock(m_lock); for (auto& region : m_regions) { - if (region.vmobject().is_purgeable() && !static_cast(region.vmobject()).is_volatile()) + if (region.vmobject().is_purgeable() && !static_cast(region.vmobject()).is_any_volatile()) amount += region.amount_resident(); } return amount; diff --git a/Kernel/SharedBuffer.cpp b/Kernel/SharedBuffer.cpp index 0e0682a590..5d62e3e4a5 100644 --- a/Kernel/SharedBuffer.cpp +++ b/Kernel/SharedBuffer.cpp @@ -219,4 +219,29 @@ void SharedBuffer::seal() } } +auto SharedBuffer::set_volatile_all(bool is_volatile, bool& was_purged) -> SetVolatileError +{ + was_purged = false; + auto pid = Process::current()->pid(); + LOCKER(shared_buffers().lock()); + for (size_t i = 0; i < m_refs.size(); ++i) { + auto& ref = m_refs[i]; + if (ref.pid == pid) { + if (Region* region = ref.region.unsafe_ptr()) { + switch (region->set_volatile(region->vaddr(), region->size(), is_volatile, was_purged)) { + case Region::SetVolatileError::Success: + if (!was_purged && was_purged) + klog() << "Region @ " << region->vaddr() << " - " << region->vaddr().offset(region->size()) << " was purged!"; + return SetVolatileError::Success; + case Region::SetVolatileError::NotPurgeable: + return SetVolatileError::NotPurgeable; + case Region::SetVolatileError::OutOfMemory: + return SetVolatileError::OutOfMemory; + } + } + } + } + return SetVolatileError::NotMapped; +} + } diff --git a/Kernel/SharedBuffer.h b/Kernel/SharedBuffer.h index acdcfe34d6..02704b3d42 100644 --- a/Kernel/SharedBuffer.h +++ b/Kernel/SharedBuffer.h @@ -75,6 +75,13 @@ public: size_t size() const { return m_vmobject->size(); } void destroy_if_unused(); void seal(); + enum class SetVolatileError { + Success = 0, + NotPurgeable, + OutOfMemory, + NotMapped + }; + SetVolatileError set_volatile_all(bool is_volatile, bool& was_purged); PurgeableVMObject& vmobject() { return m_vmobject; } const PurgeableVMObject& vmobject() const { return m_vmobject; } int id() const { return m_shbuf_id; } diff --git a/Kernel/Syscalls/mmap.cpp b/Kernel/Syscalls/mmap.cpp index a5209cb476..20e9f379ae 100644 --- a/Kernel/Syscalls/mmap.cpp +++ b/Kernel/Syscalls/mmap.cpp @@ -282,31 +282,30 @@ int Process::sys$madvise(void* address, size_t size, int advice) return -EINVAL; if (!region->is_mmap()) return -EPERM; - if ((advice & MADV_SET_VOLATILE) && (advice & MADV_SET_NONVOLATILE)) + bool set_volatile = advice & MADV_SET_VOLATILE; + bool set_nonvolatile = advice & MADV_SET_NONVOLATILE; + if (set_volatile && set_nonvolatile) return -EINVAL; - if (advice & MADV_SET_VOLATILE) { + if (set_volatile || set_nonvolatile) { if (!region->vmobject().is_purgeable()) return -EPERM; - auto& vmobject = static_cast(region->vmobject()); - vmobject.set_volatile(true); + bool was_purged = false; + switch (region->set_volatile(VirtualAddress(address), size, set_volatile, was_purged)) { + case Region::SetVolatileError::Success: + break; + case Region::SetVolatileError::NotPurgeable: + return -EPERM; + case Region::SetVolatileError::OutOfMemory: + return -ENOMEM; + } + if (set_nonvolatile) + return was_purged ? 1 : 0; return 0; } - if (advice & MADV_SET_NONVOLATILE) { - if (!region->vmobject().is_purgeable()) - return -EPERM; - auto& vmobject = static_cast(region->vmobject()); - if (!vmobject.is_volatile()) - return 0; - vmobject.set_volatile(false); - bool was_purged = vmobject.was_purged(); - vmobject.set_was_purged(false); - return was_purged ? 1 : 0; - } if (advice & MADV_GET_VOLATILE) { if (!region->vmobject().is_purgeable()) return -EPERM; - auto& vmobject = static_cast(region->vmobject()); - return vmobject.is_volatile() ? 0 : 1; + return region->is_volatile(VirtualAddress(address), size) ? 0 : 1; } return -EINVAL; } diff --git a/Kernel/Syscalls/shbuf.cpp b/Kernel/Syscalls/shbuf.cpp index 03ff104c62..fc2abfa805 100644 --- a/Kernel/Syscalls/shbuf.cpp +++ b/Kernel/Syscalls/shbuf.cpp @@ -174,14 +174,28 @@ int Process::sys$shbuf_set_volatile(int shbuf_id, bool state) #ifdef SHARED_BUFFER_DEBUG klog() << "Set shared buffer " << shbuf_id << " volatile: " << state; #endif + + bool was_purged = false; + auto set_volatile = [&]() -> int { + switch (shared_buffer.set_volatile_all(state, was_purged)) { + case SharedBuffer::SetVolatileError::Success: + break; + case SharedBuffer::SetVolatileError::NotPurgeable: + return -EPERM; + case SharedBuffer::SetVolatileError::OutOfMemory: + return -ENOMEM; + case SharedBuffer::SetVolatileError::NotMapped: + return -EINVAL; + } + return 0; + }; + if (!state) { - bool was_purged = shared_buffer.vmobject().was_purged(); - shared_buffer.vmobject().set_volatile(state); - shared_buffer.vmobject().set_was_purged(false); + if (int err = set_volatile()) + return err; return was_purged ? 1 : 0; } - shared_buffer.vmobject().set_volatile(true); - return 0; + return set_volatile(); } } diff --git a/Kernel/VM/MemoryManager.cpp b/Kernel/VM/MemoryManager.cpp index 622b0ba4ca..d5905401a3 100644 --- a/Kernel/VM/MemoryManager.cpp +++ b/Kernel/VM/MemoryManager.cpp @@ -451,7 +451,7 @@ OwnPtr MemoryManager::allocate_kernel_region_with_vmobject(const Range& ScopedSpinLock lock(s_mm_lock); OwnPtr region; if (user_accessible) - region = Region::create_user_accessible(range, vmobject, 0, name, access, cacheable); + region = Region::create_user_accessible(nullptr, range, vmobject, 0, name, access, cacheable); else region = Region::create_kernel_only(range, vmobject, 0, name, access, cacheable); if (region) diff --git a/Kernel/VM/MemoryManager.h b/Kernel/VM/MemoryManager.h index 980b18b1bc..8b9eeb46ff 100644 --- a/Kernel/VM/MemoryManager.h +++ b/Kernel/VM/MemoryManager.h @@ -38,7 +38,8 @@ namespace Kernel { -#define PAGE_ROUND_UP(x) ((((u32)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1))) +#define PAGE_ROUND_UP(x) ((((FlatPtr)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1))) +#define PAGE_ROUND_DOWN(x) (((FlatPtr)(x)) & ~(PAGE_SIZE - 1)) template inline T* low_physical_to_virtual(T* physical) diff --git a/Kernel/VM/PurgeableVMObject.cpp b/Kernel/VM/PurgeableVMObject.cpp index 8f1af0e1cf..c944b3882d 100644 --- a/Kernel/VM/PurgeableVMObject.cpp +++ b/Kernel/VM/PurgeableVMObject.cpp @@ -24,12 +24,201 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include +#include +#include #include #include #include +//#define VOLATILE_PAGE_RANGES_DEBUG + namespace Kernel { +#ifdef VOLATILE_PAGE_RANGES_DEBUG +inline LogStream& operator<<(const LogStream& stream, const VolatilePageRange& range) +{ + stream << "{" << range.base << " (" << range.count << ") purged: " << range.was_purged << "}"; + return const_cast(stream); +} + +static void dump_volatile_page_ranges(const Vector& ranges) +{ + for (size_t i = 0; i < ranges.size(); i++) { + const auto& range = ranges[i]; + klog() << " [" << i << "] " << range; + } +} +#endif + +bool VolatilePageRanges::add(const VolatilePageRange& range) +{ + auto add_range = m_total_range.intersected(range); + if (add_range.is_empty()) + return false; + add_range.was_purged = range.was_purged; + +#ifdef VOLATILE_PAGE_RANGES_DEBUG + klog() << "ADD " << range << " (total range: " << m_total_range << ") -->"; + dump_volatile_page_ranges(m_ranges); + ScopeGuard debug_guard([&]() { + klog() << "After adding " << range << " (total range: " << m_total_range << ")"; + dump_volatile_page_ranges(m_ranges); + klog() << "<-- ADD " << range << " (total range: " << m_total_range << ")"; + }); +#endif + + size_t nearby_index = 0; + auto* existing_range = binary_search( + m_ranges.span(), add_range, &nearby_index, [](auto& a, auto& b) { + if (a.intersects_or_adjacent(b)) + return 0; + return (signed)(a.base - (b.base + b.count - 1)); + }); + + size_t inserted_index = 0; + if (existing_range) { + if (*existing_range == add_range) + return false; + + if (existing_range->was_purged != add_range.was_purged) { + // Found an intersecting or adjacent range, but the purge flag + // doesn't match. Subtract what we're adding from it, and + existing_range->subtract_intersecting(add_range); + if (existing_range->is_empty()) { + *existing_range = add_range; + } else { + m_ranges.insert(++nearby_index, add_range); + existing_range = &m_ranges[nearby_index]; + } + } else { + // Found an intersecting or adjacent range that can be merged + existing_range->combine_intersecting_or_adjacent(add_range); + } + inserted_index = nearby_index; + } else { + // Insert into the sorted list + m_ranges.insert_before_matching( + VolatilePageRange(add_range), [&](auto& entry) { + return entry.base >= add_range.base + add_range.count; + }, + nearby_index, &inserted_index); + existing_range = &m_ranges[inserted_index]; + } + + // See if we can merge any of the following ranges + inserted_index++; + while (inserted_index < m_ranges.size()) { + auto& next_range = m_ranges[inserted_index]; + if (!next_range.intersects_or_adjacent(*existing_range)) + break; + if (next_range.was_purged != existing_range->was_purged) { + // The purged flag of following range is not the same. + // Subtract the added/combined range from it + next_range.subtract_intersecting(*existing_range); + if (next_range.is_empty()) + m_ranges.remove(inserted_index); + } else { + existing_range->combine_intersecting_or_adjacent(next_range); + m_ranges.remove(inserted_index); + } + } + return true; +} + +bool VolatilePageRanges::remove(const VolatilePageRange& range, bool& was_purged) +{ + auto remove_range = m_total_range.intersected(range); + if (remove_range.is_empty()) + return false; + +#ifdef VOLATILE_PAGE_RANGES_DEBUG + klog() << "REMOVE " << range << " (total range: " << m_total_range << ") -->"; + dump_volatile_page_ranges(m_ranges); + ScopeGuard debug_guard([&]() { + klog() << "After removing " << range << " (total range: " << m_total_range << ")"; + dump_volatile_page_ranges(m_ranges); + klog() << "<-- REMOVE " << range << " (total range: " << m_total_range << ") was_purged: " << was_purged; + }); +#endif + + size_t nearby_index = 0; + auto* existing_range = binary_search( + m_ranges.span(), remove_range, &nearby_index, [](auto& a, auto& b) { + if (a.intersects(b)) + return 0; + return (signed)(a.base - (b.base + b.count - 1)); + }); + if (!existing_range) + return false; + + was_purged = existing_range->was_purged; + if (existing_range->range_equals(remove_range)) { + m_ranges.remove(nearby_index); + } else { + // See if we need to remove any of the following ranges + ASSERT(existing_range == &m_ranges[nearby_index]); // sanity check + while (nearby_index < m_ranges.size()) { + existing_range = &m_ranges[nearby_index]; + if (!existing_range->intersects(range)) + break; + was_purged |= existing_range->was_purged; + existing_range->subtract_intersecting(remove_range); + if (existing_range->is_empty()) { + m_ranges.remove(nearby_index); + break; + } + } + } + return true; +} + +bool VolatilePageRanges::intersects(const VolatilePageRange& range) const +{ + auto* existing_range = binary_search( + m_ranges.span(), range, nullptr, [](auto& a, auto& b) { + if (a.intersects(b)) + return 0; + return (signed)(a.base - (b.base + b.count - 1)); + }); + return existing_range != nullptr; +} + +PurgeablePageRanges::PurgeablePageRanges(const VMObject& vmobject) + : m_volatile_ranges({0, vmobject.is_purgeable() ? static_cast(vmobject).page_count() : 0}) +{ +} + +bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range) +{ + if (range.is_empty()) + return false; + ScopedSpinLock lock(m_volatile_ranges_lock); + return m_volatile_ranges.add(range); +} + +bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged) +{ + if (range.is_empty()) + return false; + ScopedSpinLock lock(m_volatile_ranges_lock); + return m_volatile_ranges.remove(range, was_purged); +} + +bool PurgeablePageRanges::is_volatile_range(const VolatilePageRange& range) const +{ + if (range.is_empty()) + return false; + ScopedSpinLock lock(m_volatile_ranges_lock); + return m_volatile_ranges.intersects(range); +} + +void PurgeablePageRanges::set_was_purged(const VolatilePageRange& range) +{ + ScopedSpinLock lock(m_volatile_ranges_lock); + m_volatile_ranges.add({range.base, range.count, true}); +} + NonnullRefPtr PurgeableVMObject::create_with_size(size_t size) { return adopt(*new PurgeableVMObject(size)); @@ -42,9 +231,9 @@ PurgeableVMObject::PurgeableVMObject(size_t size) PurgeableVMObject::PurgeableVMObject(const PurgeableVMObject& other) : AnonymousVMObject(other) - , m_was_purged(other.m_was_purged) - , m_volatile(other.m_volatile) + , m_purgeable_ranges() // do *not* clone this { + // TODO: what about m_lock? } PurgeableVMObject::~PurgeableVMObject() @@ -70,26 +259,75 @@ int PurgeableVMObject::purge_with_interrupts_disabled(Badge) return purge_impl(); } +void PurgeableVMObject::set_was_purged(const VolatilePageRange& range) +{ + ASSERT(m_lock.is_locked()); + for (auto* purgeable_ranges : m_purgeable_ranges) + purgeable_ranges->set_was_purged(range); +} + int PurgeableVMObject::purge_impl() { - if (!m_volatile) - return 0; int purged_page_count = 0; - for (size_t i = 0; i < m_physical_pages.size(); ++i) { - if (m_physical_pages[i] && !m_physical_pages[i]->is_shared_zero_page()) - ++purged_page_count; - m_physical_pages[i] = MM.shared_zero_page(); - } - m_was_purged = true; - - if (purged_page_count > 0) { - for_each_region([&](auto& region) { - if (®ion.vmobject() == this) - region.remap(); - }); - } + ScopedSpinLock lock(m_lock); + for_each_volatile_range([&](const auto& range) { + int purged_in_range = 0; + auto range_end = range.base + range.count; + for (size_t i = range.base; i < range_end; i++) { + auto& phys_page = m_physical_pages[i]; + if (phys_page && !phys_page->is_shared_zero_page()) + ++purged_in_range; + phys_page = MM.shared_zero_page(); + } + if (purged_in_range > 0) { + purged_page_count += purged_in_range; + set_was_purged(range); + for_each_region([&](auto& region) { + if (®ion.vmobject() == this) { + if (auto owner = region.get_owner()) { + // we need to hold a reference the process here (if there is one) as we may not own this region + klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " owned by " << *owner << " at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count); + } else { + klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " (no ownership) at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count); + } + region.remap_page_range(range.base, range.count); + } + }); + } + return IterationDecision::Continue; + }); return purged_page_count; } +void PurgeableVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges) +{ + ScopedSpinLock lock(m_lock); + ASSERT(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges)); + m_purgeable_ranges.append(&purgeable_page_ranges); +} + +void PurgeableVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges) +{ + ScopedSpinLock lock(m_lock); + for (size_t i = 0; i < m_purgeable_ranges.size(); i++) { + if (m_purgeable_ranges[i] != &purgeable_page_ranges) + continue; + m_purgeable_ranges.remove(i); + return; + } + ASSERT_NOT_REACHED(); +} + +bool PurgeableVMObject::is_any_volatile() const +{ + ScopedSpinLock lock(m_lock); + for (auto& volatile_ranges : m_purgeable_ranges) { + ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock); + if (!volatile_ranges->is_empty()) + return true; + } + return false; +} + } diff --git a/Kernel/VM/PurgeableVMObject.h b/Kernel/VM/PurgeableVMObject.h index 5e100aa5f0..c3fd4d2791 100644 --- a/Kernel/VM/PurgeableVMObject.h +++ b/Kernel/VM/PurgeableVMObject.h @@ -26,10 +26,159 @@ #pragma once +#include #include namespace Kernel { +struct VolatilePageRange { + size_t base { 0 }; + size_t count { 0 }; + bool was_purged { false }; + + bool is_empty() const { return count == 0; } + + bool intersects(const VolatilePageRange& other) const + { + return other.base < base + count || other.base + other.count > base; + } + + bool intersects_or_adjacent(const VolatilePageRange& other) const + { + return other.base <= base + count || other.base + other.count >= base; + } + + bool contains(const VolatilePageRange& other) const + { + return base <= other.base && base + count >= other.base + other.count; + } + + VolatilePageRange intersected(const VolatilePageRange& other) const + { + auto b = max(base, other.base); + auto e = min(base + count, other.base + other.count); + if (b >= e) + return {}; + return { b, e - b, was_purged }; + } + + void combine_intersecting_or_adjacent(const VolatilePageRange& other) + { + ASSERT(intersects_or_adjacent(other)); + if (base <= other.base) { + count = (other.base - base) + other.count; + } else { + count = (base - other.base) + count; + base = other.base; + } + was_purged |= other.was_purged; + } + + void subtract_intersecting(const VolatilePageRange& other) + { + if (!intersects(other)) + return; + if (other.contains(*this)) { + count = 0; + return; + } + if (base <= other.base) { + count = (other.base - base); + } else { + auto new_base = other.base + other.count; + count = (base + count) - new_base; + base = new_base; + } + } + + bool range_equals(const VolatilePageRange& other) const + { + return base == other.base && count == other.count; + } + bool operator==(const VolatilePageRange& other) const + { + return base == other.base && count == other.count && was_purged == other.was_purged; + } + bool operator!=(const VolatilePageRange& other) const + { + return base != other.base || count != other.count || was_purged != other.was_purged; + } +}; + +class VolatilePageRanges { +public: + VolatilePageRanges(const VolatilePageRange& total_range) + : m_total_range(total_range) + { + } + VolatilePageRanges(const VolatilePageRanges& other) + : m_ranges(other.m_ranges) + , m_total_range(other.m_total_range) + { + } + + bool is_empty() const { return m_ranges.is_empty(); } + void clear() { m_ranges.clear(); } + + bool is_all() const + { + if (m_ranges.size() != 1) + return false; + return m_ranges[0] == m_total_range; + } + + void set_all() + { + if (m_ranges.size() != 1) + m_ranges = { m_total_range }; + else + m_ranges[0] = m_total_range; + } + + bool intersects(const VolatilePageRange&) const; + + bool add(const VolatilePageRange&); + bool remove(const VolatilePageRange&, bool&); + + Vector& ranges() { return m_ranges; } + const Vector& ranges() const { return m_ranges; } + +private: + Vector m_ranges; + VolatilePageRange m_total_range; +}; + +class PurgeableVMObject; + +class PurgeablePageRanges { + friend class PurgeableVMObject; +public: + PurgeablePageRanges(const VMObject&); + + void set_purgeable_page_ranges(const PurgeablePageRanges& other) + { + if (this == &other) + return; + ScopedSpinLock lock(m_volatile_ranges_lock); + ScopedSpinLock other_lock(other.m_volatile_ranges_lock); + m_volatile_ranges = other.m_volatile_ranges; + return; + } + + bool add_volatile_range(const VolatilePageRange& range); + bool remove_volatile_range(const VolatilePageRange& range, bool& was_purged); + bool is_volatile_range(const VolatilePageRange& range) const; + + bool is_empty() const { return m_volatile_ranges.is_empty(); } + + void set_was_purged(const VolatilePageRange&); + + const VolatilePageRanges& volatile_ranges() const { return m_volatile_ranges; } +protected: + VolatilePageRanges m_volatile_ranges; + mutable SpinLock m_volatile_ranges_lock; +}; + class PurgeableVMObject final : public AnonymousVMObject { public: virtual ~PurgeableVMObject() override; @@ -37,14 +186,49 @@ public: static NonnullRefPtr create_with_size(size_t); virtual NonnullRefPtr clone() override; + void register_purgeable_page_ranges(PurgeablePageRanges&); + void unregister_purgeable_page_ranges(PurgeablePageRanges&); + int purge(); int purge_with_interrupts_disabled(Badge); - bool was_purged() const { return m_was_purged; } - void set_was_purged(bool b) { m_was_purged = b; } + bool is_any_volatile() const; - bool is_volatile() const { return m_volatile; } - void set_volatile(bool b) { m_volatile = b; } + template + IterationDecision for_each_volatile_range(F f) + { + ASSERT(m_lock.is_locked()); + // This is a little ugly. Basically, we're trying to find the + // volatile ranges that all share, because those are the only + // pages we can actually purge + for (auto* purgeable_range : m_purgeable_ranges) { + for (auto& r1 : purgeable_range->volatile_ranges().ranges()) { + VolatilePageRange range(r1); + for (auto* purgeable_range2 : m_purgeable_ranges) { + if (purgeable_range2 == purgeable_range) + continue; + if (purgeable_range2->is_empty()) { + // If just one doesn't allow any purging, we can + // immediately bail + return IterationDecision::Continue; + } + for (const auto& r2 : purgeable_range2->volatile_ranges().ranges()) { + range = range.intersected(r2); + if (range.is_empty()) + break; + } + if (range.is_empty()) + break; + } + if (range.is_empty()) + continue; + IterationDecision decision = f(range); + if (decision != IterationDecision::Continue) + return decision; + } + } + return IterationDecision::Continue; + } private: explicit PurgeableVMObject(size_t); @@ -53,6 +237,7 @@ private: virtual const char* class_name() const override { return "PurgeableVMObject"; } int purge_impl(); + void set_was_purged(const VolatilePageRange&); PurgeableVMObject& operator=(const PurgeableVMObject&) = delete; PurgeableVMObject& operator=(PurgeableVMObject&&) = delete; @@ -60,8 +245,8 @@ private: virtual bool is_purgeable() const override { return true; } - bool m_was_purged { false }; - bool m_volatile { false }; + Vector m_purgeable_ranges; + mutable SpinLock m_lock; }; } diff --git a/Kernel/VM/Region.cpp b/Kernel/VM/Region.cpp index ac4899203f..22d49c60a1 100644 --- a/Kernel/VM/Region.cpp +++ b/Kernel/VM/Region.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include @@ -41,7 +42,8 @@ namespace Kernel { Region::Region(const Range& range, NonnullRefPtr vmobject, size_t offset_in_vmobject, const String& name, u8 access, bool cacheable, bool kernel) - : m_range(range) + : PurgeablePageRanges(vmobject) + , m_range(range) , m_offset_in_vmobject(offset_in_vmobject) , m_vmobject(move(vmobject)) , m_name(name) @@ -49,11 +51,14 @@ Region::Region(const Range& range, NonnullRefPtr vmobject, size_t offs , m_cacheable(cacheable) , m_kernel(kernel) { + register_purgeable_page_ranges(); MM.register_region(*this); } Region::~Region() { + unregister_purgeable_page_ranges(); + // Make sure we disable interrupts so we don't get interrupted between unmapping and unregistering. // Unmapping the region will give the VM back to the RangeAllocator, so an interrupt handler would // find the address<->region mappings in an invalid state there. @@ -62,9 +67,26 @@ Region::~Region() unmap(ShouldDeallocateVirtualMemoryRange::Yes); ASSERT(!m_page_directory); } + MM.unregister_region(*this); } +void Region::register_purgeable_page_ranges() +{ + if (m_vmobject->is_purgeable()) { + auto& vmobject = static_cast(*m_vmobject); + vmobject.register_purgeable_page_ranges(*this); + } +} + +void Region::unregister_purgeable_page_ranges() +{ + if (m_vmobject->is_purgeable()) { + auto& vmobject = static_cast(*m_vmobject); + vmobject.unregister_purgeable_page_ranges(*this); + } +} + NonnullOwnPtr Region::clone() { ASSERT(Process::current()); @@ -74,7 +96,8 @@ NonnullOwnPtr Region::clone() ASSERT(m_mmap); ASSERT(!m_shared); ASSERT(vmobject().is_anonymous()); - auto zeroed_region = Region::create_user_accessible(m_range, AnonymousVMObject::create_with_size(size()), 0, m_name, m_access); + auto zeroed_region = Region::create_user_accessible(get_owner().ptr(), m_range, AnonymousVMObject::create_with_size(size()), 0, m_name, m_access); + zeroed_region->set_purgeable_page_ranges(*this); zeroed_region->set_mmap(m_mmap); zeroed_region->set_inherit_mode(m_inherit_mode); return zeroed_region; @@ -89,7 +112,8 @@ NonnullOwnPtr Region::clone() ASSERT(vmobject().is_shared_inode()); // Create a new region backed by the same VMObject. - auto region = Region::create_user_accessible(m_range, m_vmobject, m_offset_in_vmobject, m_name, m_access); + auto region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject, m_offset_in_vmobject, m_name, m_access); + region->set_purgeable_page_ranges(*this); region->set_mmap(m_mmap); region->set_shared(m_shared); return region; @@ -104,7 +128,8 @@ NonnullOwnPtr Region::clone() // Set up a COW region. The parent (this) region becomes COW as well! ensure_cow_map().fill(true); remap(); - auto clone_region = Region::create_user_accessible(m_range, m_vmobject->clone(), m_offset_in_vmobject, m_name, m_access); + auto clone_region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject->clone(), m_offset_in_vmobject, m_name, m_access); + clone_region->set_purgeable_page_ranges(*this); clone_region->ensure_cow_map(); if (m_stack) { ASSERT(is_readable()); @@ -116,6 +141,59 @@ NonnullOwnPtr Region::clone() return clone_region; } +void Region::set_vmobject(NonnullRefPtr&& obj) +{ + if (m_vmobject.ptr() == obj.ptr()) + return; + unregister_purgeable_page_ranges(); + m_vmobject = move(obj); + register_purgeable_page_ranges(); +} + +bool Region::is_volatile(VirtualAddress vaddr, size_t size) const +{ + if (!m_vmobject->is_purgeable()) + return false; + + auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject); + size_t first_page_index = PAGE_ROUND_DOWN(offset_in_vmobject) / PAGE_SIZE; + size_t last_page_index = PAGE_ROUND_UP(offset_in_vmobject + size) / PAGE_SIZE; + return is_volatile_range({ first_page_index, last_page_index - first_page_index }); +} + +auto Region::set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged) -> SetVolatileError +{ + was_purged = false; + if (!m_vmobject->is_purgeable()) + return SetVolatileError::NotPurgeable; + + auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject); + if (is_volatile) { + // If marking pages as volatile, be prudent by not marking + // partial pages volatile to prevent potentially non-volatile + // data to be discarded. So rund up the first page and round + // down the last page. + size_t first_page_index = PAGE_ROUND_UP(offset_in_vmobject) / PAGE_SIZE; + size_t last_page_index = PAGE_ROUND_DOWN(offset_in_vmobject + size) / PAGE_SIZE; + if (first_page_index != last_page_index) + add_volatile_range({ first_page_index, last_page_index - first_page_index }); + } else { + // If marking pages as non-volatile, round down the first page + // and round up the last page to make sure the beginning and + // end of the range doesn't inadvertedly get discarded. + size_t first_page_index = PAGE_ROUND_DOWN(offset_in_vmobject) / PAGE_SIZE; + size_t last_page_index = PAGE_ROUND_UP(offset_in_vmobject + size) / PAGE_SIZE; + if (remove_volatile_range({ first_page_index, last_page_index - first_page_index }, was_purged)) { + // Attempt to remap the page range. We want to make sure we have + // enough memory, if not we need to inform the caller of that + // fact + if (!remap_page_range(first_page_index, last_page_index - first_page_index)) + return SetVolatileError::OutOfMemory; + } + } + return SetVolatileError::Success; +} + bool Region::commit() { ScopedSpinLock lock(s_mm_lock); @@ -190,9 +268,11 @@ size_t Region::amount_shared() const return bytes; } -NonnullOwnPtr Region::create_user_accessible(const Range& range, NonnullRefPtr vmobject, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable) +NonnullOwnPtr Region::create_user_accessible(Process* owner, const Range& range, NonnullRefPtr vmobject, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable) { auto region = make(range, move(vmobject), offset_in_vmobject, name, access, cacheable, false); + if (owner) + region->m_owner = owner->make_weak_ptr(); region->m_user_accessible = true; return region; } @@ -259,6 +339,25 @@ bool Region::map_individual_page_impl(size_t page_index) return true; } +bool Region::remap_page_range(size_t page_index, size_t page_count) +{ + bool success = true; + ScopedSpinLock lock(s_mm_lock); + ASSERT(m_page_directory); + ScopedSpinLock page_lock(m_page_directory->get_lock()); + size_t index = page_index; + while (index < page_index + page_count) { + if (!map_individual_page_impl(index)) { + success = false; + break; + } + index++; + } + if (index > page_index) + MM.flush_tlb(vaddr_from_page_index(page_index), index - page_index); + return success; +} + bool Region::remap_page(size_t page_index, bool with_flush) { ScopedSpinLock lock(s_mm_lock); @@ -534,4 +633,9 @@ PageFaultResponse Region::handle_inode_fault(size_t page_index_in_region) return PageFaultResponse::Continue; } +RefPtr Region::get_owner() +{ + return m_owner.strong_ref(); +} + } diff --git a/Kernel/VM/Region.h b/Kernel/VM/Region.h index 575b595186..46159d8c8c 100644 --- a/Kernel/VM/Region.h +++ b/Kernel/VM/Region.h @@ -28,9 +28,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -47,7 +49,8 @@ enum class PageFaultResponse { class Region final : public InlineLinkedListNode - , public Weakable { + , public Weakable + , public PurgeablePageRanges { friend class MemoryManager; MAKE_SLAB_ALLOCATED(Region) @@ -63,7 +66,7 @@ public: ZeroedOnFork, }; - static NonnullOwnPtr create_user_accessible(const Range&, NonnullRefPtr, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true); + static NonnullOwnPtr create_user_accessible(Process*, const Range&, NonnullRefPtr, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true); static NonnullOwnPtr create_kernel_only(const Range&, NonnullRefPtr, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true); ~Region(); @@ -83,7 +86,7 @@ public: const VMObject& vmobject() const { return *m_vmobject; } VMObject& vmobject() { return *m_vmobject; } - void set_vmobject(NonnullRefPtr&& obj) { m_vmobject = obj; } + void set_vmobject(NonnullRefPtr&&); bool is_shared() const { return m_shared; } void set_shared(bool shared) { m_shared = shared; } @@ -190,6 +193,18 @@ public: void set_inherit_mode(InheritMode inherit_mode) { m_inherit_mode = inherit_mode; } + bool remap_page_range(size_t page_index, size_t page_count); + + bool is_volatile(VirtualAddress vaddr, size_t size) const; + enum class SetVolatileError { + Success = 0, + NotPurgeable, + OutOfMemory + }; + SetVolatileError set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged); + + RefPtr get_owner(); + private: Bitmap& ensure_cow_map() const; @@ -210,6 +225,9 @@ private: bool map_individual_page_impl(size_t page_index); + void register_purgeable_page_ranges(); + void unregister_purgeable_page_ranges(); + RefPtr m_page_directory; Range m_range; size_t m_offset_in_vmobject { 0 }; @@ -224,6 +242,7 @@ private: bool m_mmap : 1 { false }; bool m_kernel : 1 { false }; mutable OwnPtr m_cow_map; + WeakPtr m_owner; }; inline unsigned prot_to_region_access_flags(int prot) diff --git a/Meta/CMake/all_the_debug_macros.cmake b/Meta/CMake/all_the_debug_macros.cmake index e0e637f07a..f687d1804e 100644 --- a/Meta/CMake/all_the_debug_macros.cmake +++ b/Meta/CMake/all_the_debug_macros.cmake @@ -156,6 +156,7 @@ add_compile_definitions("UPDATE_COALESCING_DEBUG") add_compile_definitions("VERY_DEBUG") add_compile_definitions("VFS_DEBUG") add_compile_definitions("VMWAREBACKDOOR_DEBUG") +add_compile_definitions("VOLATILE_PAGE_RANGES_DEBUG") add_compile_definitions("VRA_DEBUG") add_compile_definitions("WAITBLOCK_DEBUG") add_compile_definitions("WAITQUEUE_DEBUG")