mirror of
https://github.com/RGBCube/serenity
synced 2025-07-27 00:27:45 +00:00
Kernel: Merge PurgeableVMObject into AnonymousVMObject
This implements memory commitments and lazy-allocation of committed memory.
This commit is contained in:
Tom
Andreas Kling
parent
b2a52f6208
commit
476f17b3f1
35 changed files with 937 additions and 564 deletions
37
Kernel/VM/AllocationStrategy.h
Normal file
37
Kernel/VM/AllocationStrategy.h
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Copyright (c) 2020, The SerenityOS developers.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace Kernel {
|
||||
|
||||
enum class AllocationStrategy {
|
||||
Reserve = 0,
|
||||
AllocateNow,
|
||||
None
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -24,15 +24,67 @@
|
|||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <Kernel/Process.h>
|
||||
#include <Kernel/VM/AnonymousVMObject.h>
|
||||
#include <Kernel/VM/MemoryManager.h>
|
||||
#include <Kernel/VM/PhysicalPage.h>
|
||||
|
||||
//#define COMMIT_DEBUG
|
||||
//#define PAGE_FAULT_DEBUG
|
||||
|
||||
namespace Kernel {
|
||||
|
||||
NonnullRefPtr<AnonymousVMObject> AnonymousVMObject::create_with_size(size_t size)
|
||||
RefPtr<VMObject> AnonymousVMObject::clone()
|
||||
{
|
||||
return adopt(*new AnonymousVMObject(size));
|
||||
// We need to acquire our lock so we copy a sane state
|
||||
ScopedSpinLock lock(m_lock);
|
||||
|
||||
// We're the parent. Since we're about to become COW we need to
|
||||
// commit the number of pages that we need to potentially allocate
|
||||
// so that the parent is still guaranteed to be able to have all
|
||||
// non-volatile memory available.
|
||||
size_t need_cow_pages = 0;
|
||||
{
|
||||
// We definitely need to commit non-volatile areas
|
||||
for_each_nonvolatile_range([&](const VolatilePageRange& nonvolatile_range) {
|
||||
need_cow_pages += nonvolatile_range.count;
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
}
|
||||
|
||||
#ifdef COMMIT_DEBUG
|
||||
klog() << "Cloning " << this << ", need " << need_cow_pages << " committed cow pages";
|
||||
#endif
|
||||
if (!MM.commit_user_physical_pages(need_cow_pages))
|
||||
return {};
|
||||
// Create or replace the committed cow pages. When cloning a previously
|
||||
// cloned vmobject, we want to essentially "fork", leaving us and the
|
||||
// new clone with one set of shared committed cow pages, and the original
|
||||
// one would keep the one it still has. This ensures that the original
|
||||
// one and this one, as well as the clone have sufficient resources
|
||||
// to cow all pages as needed
|
||||
m_shared_committed_cow_pages = adopt(*new CommittedCowPages(need_cow_pages));
|
||||
|
||||
// Both original and clone become COW. So create a COW map for ourselves
|
||||
// or reset all pages to be copied again if we were previously cloned
|
||||
ensure_or_reset_cow_map();
|
||||
|
||||
return adopt(*new AnonymousVMObject(*this));
|
||||
}
|
||||
|
||||
RefPtr<AnonymousVMObject> AnonymousVMObject::create_with_size(size_t size, AllocationStrategy commit)
|
||||
{
|
||||
if (commit == AllocationStrategy::Reserve || commit == AllocationStrategy::AllocateNow) {
|
||||
// We need to attempt to commit before actually creating the object
|
||||
if (!MM.commit_user_physical_pages(ceil_div(size, PAGE_SIZE)))
|
||||
return {};
|
||||
}
|
||||
return adopt(*new AnonymousVMObject(size, commit));
|
||||
}
|
||||
|
||||
NonnullRefPtr<AnonymousVMObject> AnonymousVMObject::create_with_physical_page(PhysicalPage& page)
|
||||
{
|
||||
return adopt(*new AnonymousVMObject(page));
|
||||
}
|
||||
|
||||
RefPtr<AnonymousVMObject> AnonymousVMObject::create_for_physical_range(PhysicalAddress paddr, size_t size)
|
||||
|
|
@ -44,49 +96,403 @@ RefPtr<AnonymousVMObject> AnonymousVMObject::create_for_physical_range(PhysicalA
|
|||
return adopt(*new AnonymousVMObject(paddr, size));
|
||||
}
|
||||
|
||||
NonnullRefPtr<AnonymousVMObject> AnonymousVMObject::create_with_physical_page(PhysicalPage& page)
|
||||
{
|
||||
auto vmobject = create_with_size(PAGE_SIZE);
|
||||
vmobject->m_physical_pages[0] = page;
|
||||
return vmobject;
|
||||
}
|
||||
|
||||
AnonymousVMObject::AnonymousVMObject(size_t size, bool initialize_pages)
|
||||
AnonymousVMObject::AnonymousVMObject(size_t size, AllocationStrategy strategy)
|
||||
: VMObject(size)
|
||||
, m_volatile_ranges_cache({ 0, page_count() })
|
||||
, m_unused_committed_pages(strategy == AllocationStrategy::Reserve ? page_count() : 0)
|
||||
{
|
||||
if (initialize_pages) {
|
||||
#ifndef MAP_SHARED_ZERO_PAGE_LAZILY
|
||||
if (strategy == AllocationStrategy::AllocateNow) {
|
||||
// Allocate all pages right now. We know we can get all because we committed the amount needed
|
||||
for (size_t i = 0; i < page_count(); ++i)
|
||||
physical_pages()[i] = MM.shared_zero_page();
|
||||
#endif
|
||||
physical_pages()[i] = MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
} else {
|
||||
auto& initial_page = (strategy == AllocationStrategy::Reserve) ? MM.lazy_committed_page() : MM.shared_zero_page();
|
||||
for (size_t i = 0; i < page_count(); ++i)
|
||||
physical_pages()[i] = initial_page;
|
||||
}
|
||||
}
|
||||
|
||||
AnonymousVMObject::AnonymousVMObject(PhysicalAddress paddr, size_t size)
|
||||
: VMObject(size)
|
||||
, m_volatile_ranges_cache({ 0, page_count() })
|
||||
{
|
||||
ASSERT(paddr.page_base() == paddr);
|
||||
for (size_t i = 0; i < page_count(); ++i)
|
||||
physical_pages()[i] = PhysicalPage::create(paddr.offset(i * PAGE_SIZE), false, false);
|
||||
}
|
||||
|
||||
AnonymousVMObject::AnonymousVMObject(PhysicalPage& page)
|
||||
: VMObject(PAGE_SIZE)
|
||||
, m_volatile_ranges_cache({ 0, page_count() })
|
||||
{
|
||||
physical_pages()[0] = page;
|
||||
}
|
||||
|
||||
AnonymousVMObject::AnonymousVMObject(const AnonymousVMObject& other)
|
||||
: VMObject(other)
|
||||
, m_volatile_ranges_cache({ 0, page_count() }) // do *not* clone this
|
||||
, m_volatile_ranges_cache_dirty(true) // do *not* clone this
|
||||
, m_purgeable_ranges() // do *not* clone this
|
||||
, m_unused_committed_pages(other.m_unused_committed_pages)
|
||||
, m_cow_map() // do *not* clone this
|
||||
, m_shared_committed_cow_pages(other.m_shared_committed_cow_pages) // share the pool
|
||||
{
|
||||
// We can't really "copy" a spinlock. But we're holding it. Clear in the clone
|
||||
ASSERT(other.m_lock.is_locked());
|
||||
m_lock.initialize();
|
||||
|
||||
// The clone also becomes COW
|
||||
ensure_or_reset_cow_map();
|
||||
|
||||
if (m_unused_committed_pages > 0) {
|
||||
// The original vmobject didn't use up all commited pages. When
|
||||
// cloning (fork) we will overcommit. For this purpose we drop all
|
||||
// lazy-commit references and replace them with shared zero pages.
|
||||
for (size_t i = 0; i < page_count(); i++) {
|
||||
auto& phys_page = m_physical_pages[i];
|
||||
if (phys_page && phys_page->is_lazy_committed_page()) {
|
||||
phys_page = MM.shared_zero_page();
|
||||
if (--m_unused_committed_pages == 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
ASSERT(m_unused_committed_pages == 0);
|
||||
}
|
||||
}
|
||||
|
||||
AnonymousVMObject::~AnonymousVMObject()
|
||||
{
|
||||
// Return any unused committed pages
|
||||
if (m_unused_committed_pages > 0)
|
||||
MM.uncommit_user_physical_pages(m_unused_committed_pages);
|
||||
}
|
||||
|
||||
RefPtr<VMObject> AnonymousVMObject::clone()
|
||||
int AnonymousVMObject::purge()
|
||||
{
|
||||
return adopt(*new AnonymousVMObject(*this));
|
||||
LOCKER(m_paging_lock);
|
||||
return purge_impl();
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(size_t)
|
||||
int AnonymousVMObject::purge_with_interrupts_disabled(Badge<MemoryManager>)
|
||||
{
|
||||
return {};
|
||||
ASSERT_INTERRUPTS_DISABLED();
|
||||
if (m_paging_lock.is_locked())
|
||||
return 0;
|
||||
return purge_impl();
|
||||
}
|
||||
|
||||
void AnonymousVMObject::set_was_purged(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
for (auto* purgeable_ranges : m_purgeable_ranges)
|
||||
purgeable_ranges->set_was_purged(range);
|
||||
}
|
||||
|
||||
int AnonymousVMObject::purge_impl()
|
||||
{
|
||||
int purged_page_count = 0;
|
||||
ScopedSpinLock lock(m_lock);
|
||||
for_each_volatile_range([&](const auto& range) {
|
||||
int purged_in_range = 0;
|
||||
auto range_end = range.base + range.count;
|
||||
for (size_t i = range.base; i < range_end; i++) {
|
||||
auto& phys_page = m_physical_pages[i];
|
||||
if (phys_page && !phys_page->is_shared_zero_page()) {
|
||||
ASSERT(!phys_page->is_lazy_committed_page());
|
||||
++purged_in_range;
|
||||
}
|
||||
phys_page = MM.shared_zero_page();
|
||||
}
|
||||
|
||||
if (purged_in_range > 0) {
|
||||
purged_page_count += purged_in_range;
|
||||
set_was_purged(range);
|
||||
for_each_region([&](auto& region) {
|
||||
if (®ion.vmobject() == this) {
|
||||
if (auto owner = region.get_owner()) {
|
||||
// we need to hold a reference the process here (if there is one) as we may not own this region
|
||||
klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " owned by " << *owner << " at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
|
||||
} else {
|
||||
klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " (no ownership) at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
|
||||
}
|
||||
region.remap_page_range(range.base, range.count);
|
||||
}
|
||||
});
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
return purged_page_count;
|
||||
}
|
||||
|
||||
void AnonymousVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
purgeable_page_ranges.set_vmobject(this);
|
||||
ASSERT(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges));
|
||||
m_purgeable_ranges.append(&purgeable_page_ranges);
|
||||
}
|
||||
|
||||
void AnonymousVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
for (size_t i = 0; i < m_purgeable_ranges.size(); i++) {
|
||||
if (m_purgeable_ranges[i] != &purgeable_page_ranges)
|
||||
continue;
|
||||
purgeable_page_ranges.set_vmobject(nullptr);
|
||||
m_purgeable_ranges.remove(i);
|
||||
return;
|
||||
}
|
||||
ASSERT_NOT_REACHED();
|
||||
}
|
||||
|
||||
bool AnonymousVMObject::is_any_volatile() const
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
for (auto& volatile_ranges : m_purgeable_ranges) {
|
||||
ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock);
|
||||
if (!volatile_ranges->is_empty())
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t AnonymousVMObject::remove_lazy_commit_pages(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
|
||||
size_t removed_count = 0;
|
||||
auto range_end = range.base + range.count;
|
||||
for (size_t i = range.base; i < range_end; i++) {
|
||||
auto& phys_page = m_physical_pages[i];
|
||||
if (phys_page && phys_page->is_lazy_committed_page()) {
|
||||
phys_page = MM.shared_zero_page();
|
||||
removed_count++;
|
||||
ASSERT(m_unused_committed_pages > 0);
|
||||
if (--m_unused_committed_pages == 0)
|
||||
break;
|
||||
}
|
||||
}
|
||||
return removed_count;
|
||||
}
|
||||
|
||||
void AnonymousVMObject::update_volatile_cache()
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
ASSERT(m_volatile_ranges_cache_dirty);
|
||||
|
||||
m_volatile_ranges_cache.clear();
|
||||
for_each_nonvolatile_range([&](const VolatilePageRange& range) {
|
||||
m_volatile_ranges_cache.add_unchecked(range);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
m_volatile_ranges_cache_dirty = false;
|
||||
}
|
||||
|
||||
void AnonymousVMObject::range_made_volatile(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
|
||||
if (m_unused_committed_pages == 0)
|
||||
return;
|
||||
|
||||
// We need to check this range for any pages that are marked for
|
||||
// lazy committed allocation and turn them into shared zero pages
|
||||
// and also adjust the m_unused_committed_pages for each such page.
|
||||
// Take into account all the other views as well.
|
||||
size_t uncommit_page_count = 0;
|
||||
for_each_volatile_range([&](const auto& r) {
|
||||
auto intersected = range.intersected(r);
|
||||
if (!intersected.is_empty()) {
|
||||
uncommit_page_count += remove_lazy_commit_pages(intersected);
|
||||
if (m_unused_committed_pages == 0)
|
||||
return IterationDecision::Break;
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
// Return those committed pages back to the system
|
||||
if (uncommit_page_count > 0) {
|
||||
#ifdef COMMIT_DEBUG
|
||||
klog() << "Uncommit " << uncommit_page_count << " lazy-commit pages from " << this;
|
||||
#endif
|
||||
MM.uncommit_user_physical_pages(uncommit_page_count);
|
||||
}
|
||||
|
||||
m_volatile_ranges_cache_dirty = true;
|
||||
}
|
||||
|
||||
void AnonymousVMObject::range_made_nonvolatile(const VolatilePageRange&)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
m_volatile_ranges_cache_dirty = true;
|
||||
}
|
||||
|
||||
size_t AnonymousVMObject::count_needed_commit_pages_for_nonvolatile_range(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
ASSERT(!range.is_empty());
|
||||
|
||||
size_t need_commit_pages = 0;
|
||||
auto range_end = range.base + range.count;
|
||||
for (size_t page_index = range.base; page_index < range_end; page_index++) {
|
||||
// COW pages are accounted for in m_shared_committed_cow_pages
|
||||
if (m_cow_map && m_cow_map->get(page_index))
|
||||
continue;
|
||||
auto& phys_page = m_physical_pages[page_index];
|
||||
if (phys_page && phys_page->is_shared_zero_page())
|
||||
need_commit_pages++;
|
||||
}
|
||||
return need_commit_pages;
|
||||
}
|
||||
|
||||
size_t AnonymousVMObject::mark_committed_pages_for_nonvolatile_range(const VolatilePageRange& range, size_t mark_total)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
ASSERT(!range.is_empty());
|
||||
ASSERT(mark_total > 0);
|
||||
|
||||
size_t pages_updated = 0;
|
||||
auto range_end = range.base + range.count;
|
||||
for (size_t page_index = range.base; page_index < range_end; page_index++) {
|
||||
// COW pages are accounted for in m_shared_committed_cow_pages
|
||||
if (m_cow_map && m_cow_map->get(page_index))
|
||||
continue;
|
||||
auto& phys_page = m_physical_pages[page_index];
|
||||
if (phys_page && phys_page->is_shared_zero_page()) {
|
||||
phys_page = MM.lazy_committed_page();
|
||||
if (++pages_updated == mark_total)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef COMMIT_DEBUG
|
||||
klog() << "Added " << pages_updated << " lazy-commit pages to " << this;
|
||||
#endif
|
||||
m_unused_committed_pages += pages_updated;
|
||||
return pages_updated;
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(size_t page_index)
|
||||
{
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
|
||||
ASSERT(m_unused_committed_pages > 0);
|
||||
|
||||
// We should't have any committed page tags in volatile regions
|
||||
ASSERT([&]() {
|
||||
for (auto* purgeable_ranges : m_purgeable_ranges) {
|
||||
if (purgeable_ranges->is_volatile(page_index))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}());
|
||||
|
||||
m_unused_committed_pages--;
|
||||
}
|
||||
return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
}
|
||||
|
||||
Bitmap& AnonymousVMObject::ensure_cow_map()
|
||||
{
|
||||
if (!m_cow_map)
|
||||
m_cow_map = make<Bitmap>(page_count(), true);
|
||||
return *m_cow_map;
|
||||
}
|
||||
|
||||
void AnonymousVMObject::ensure_or_reset_cow_map()
|
||||
{
|
||||
if (!m_cow_map)
|
||||
m_cow_map = make<Bitmap>(page_count(), true);
|
||||
else
|
||||
m_cow_map->fill(true);
|
||||
}
|
||||
|
||||
bool AnonymousVMObject::should_cow(size_t page_index, bool is_shared) const
|
||||
{
|
||||
auto& page = physical_pages()[page_index];
|
||||
if (page && (page->is_shared_zero_page() || page->is_lazy_committed_page()))
|
||||
return true;
|
||||
if (is_shared)
|
||||
return false;
|
||||
return m_cow_map && m_cow_map->get(page_index);
|
||||
}
|
||||
|
||||
void AnonymousVMObject::set_should_cow(size_t page_index, bool cow)
|
||||
{
|
||||
ensure_cow_map().set(page_index, cow);
|
||||
}
|
||||
|
||||
size_t AnonymousVMObject::cow_pages() const
|
||||
{
|
||||
if (!m_cow_map)
|
||||
return 0;
|
||||
return m_cow_map->count_slow(true);
|
||||
}
|
||||
|
||||
bool AnonymousVMObject::is_nonvolatile(size_t page_index)
|
||||
{
|
||||
if (m_volatile_ranges_cache_dirty)
|
||||
update_volatile_cache();
|
||||
return !m_volatile_ranges_cache.contains(page_index);
|
||||
}
|
||||
|
||||
PageFaultResponse AnonymousVMObject::handle_cow_fault(size_t page_index, VirtualAddress vaddr)
|
||||
{
|
||||
ASSERT_INTERRUPTS_DISABLED();
|
||||
ScopedSpinLock lock(m_lock);
|
||||
auto& page_slot = physical_pages()[page_index];
|
||||
bool have_committed = m_shared_committed_cow_pages && is_nonvolatile(page_index);
|
||||
if (page_slot->ref_count() == 1) {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> It's a COW page but nobody is sharing it anymore. Remap r/w";
|
||||
#endif
|
||||
set_should_cow(page_index, false);
|
||||
if (have_committed) {
|
||||
if (m_shared_committed_cow_pages->return_one())
|
||||
m_shared_committed_cow_pages = nullptr;
|
||||
}
|
||||
return PageFaultResponse::Continue;
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> page;
|
||||
if (have_committed) {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> It's a committed COW page and it's time to COW!";
|
||||
#endif
|
||||
page = m_shared_committed_cow_pages->allocate_one();
|
||||
} else {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> It's a COW page and it's time to COW!";
|
||||
#endif
|
||||
page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::No);
|
||||
if (page.is_null()) {
|
||||
klog() << "MM: handle_cow_fault was unable to allocate a physical page";
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
}
|
||||
}
|
||||
|
||||
u8* dest_ptr = MM.quickmap_page(*page);
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> COW " << page->paddr() << " <- " << page_slot->paddr();
|
||||
#endif
|
||||
{
|
||||
SmapDisabler disabler;
|
||||
void* fault_at;
|
||||
if (!safe_memcpy(dest_ptr, vaddr.as_ptr(), PAGE_SIZE, fault_at)) {
|
||||
if ((u8*)fault_at >= dest_ptr && (u8*)fault_at <= dest_ptr + PAGE_SIZE)
|
||||
dbg() << " >> COW: error copying page " << page_slot->paddr() << "/" << vaddr << " to " << page->paddr() << "/" << VirtualAddress(dest_ptr) << ": failed to write to page at " << VirtualAddress(fault_at);
|
||||
else if ((u8*)fault_at >= vaddr.as_ptr() && (u8*)fault_at <= vaddr.as_ptr() + PAGE_SIZE)
|
||||
dbg() << " >> COW: error copying page " << page_slot->paddr() << "/" << vaddr << " to " << page->paddr() << "/" << VirtualAddress(dest_ptr) << ": failed to read from page at " << VirtualAddress(fault_at);
|
||||
else
|
||||
ASSERT_NOT_REACHED();
|
||||
}
|
||||
}
|
||||
page_slot = move(page);
|
||||
MM.unquickmap_page();
|
||||
set_should_cow(page_index, false);
|
||||
return PageFaultResponse::Continue;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,35 +27,132 @@
|
|||
#pragma once
|
||||
|
||||
#include <Kernel/PhysicalAddress.h>
|
||||
#include <Kernel/VM/AllocationStrategy.h>
|
||||
#include <Kernel/VM/PageFaultResponse.h>
|
||||
#include <Kernel/VM/PurgeablePageRanges.h>
|
||||
#include <Kernel/VM/VMObject.h>
|
||||
|
||||
namespace Kernel {
|
||||
|
||||
class AnonymousVMObject : public VMObject {
|
||||
friend class PurgeablePageRanges;
|
||||
|
||||
public:
|
||||
virtual ~AnonymousVMObject() override;
|
||||
|
||||
static NonnullRefPtr<AnonymousVMObject> create_with_size(size_t);
|
||||
static RefPtr<AnonymousVMObject> create_for_physical_range(PhysicalAddress, size_t);
|
||||
static NonnullRefPtr<AnonymousVMObject> create_with_physical_page(PhysicalPage&);
|
||||
static RefPtr<AnonymousVMObject> create_with_size(size_t, AllocationStrategy);
|
||||
static RefPtr<AnonymousVMObject> create_for_physical_range(PhysicalAddress paddr, size_t size);
|
||||
static NonnullRefPtr<AnonymousVMObject> create_with_physical_page(PhysicalPage& page);
|
||||
virtual RefPtr<VMObject> clone() override;
|
||||
|
||||
virtual RefPtr<PhysicalPage> allocate_committed_page(size_t);
|
||||
RefPtr<PhysicalPage> allocate_committed_page(size_t);
|
||||
PageFaultResponse handle_cow_fault(size_t, VirtualAddress);
|
||||
size_t cow_pages() const;
|
||||
bool should_cow(size_t page_index, bool) const;
|
||||
void set_should_cow(size_t page_index, bool);
|
||||
|
||||
protected:
|
||||
explicit AnonymousVMObject(size_t, bool initialize_pages = true);
|
||||
void register_purgeable_page_ranges(PurgeablePageRanges&);
|
||||
void unregister_purgeable_page_ranges(PurgeablePageRanges&);
|
||||
|
||||
int purge();
|
||||
int purge_with_interrupts_disabled(Badge<MemoryManager>);
|
||||
|
||||
bool is_any_volatile() const;
|
||||
|
||||
template<typename F>
|
||||
IterationDecision for_each_volatile_range(F f) const
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
// This is a little ugly. Basically, we're trying to find the
|
||||
// volatile ranges that all share, because those are the only
|
||||
// pages we can actually purge
|
||||
for (auto* purgeable_range : m_purgeable_ranges) {
|
||||
ScopedSpinLock purgeable_lock(purgeable_range->m_volatile_ranges_lock);
|
||||
for (auto& r1 : purgeable_range->volatile_ranges().ranges()) {
|
||||
VolatilePageRange range(r1);
|
||||
for (auto* purgeable_range2 : m_purgeable_ranges) {
|
||||
if (purgeable_range2 == purgeable_range)
|
||||
continue;
|
||||
ScopedSpinLock purgeable2_lock(purgeable_range2->m_volatile_ranges_lock);
|
||||
if (purgeable_range2->is_empty()) {
|
||||
// If just one doesn't allow any purging, we can
|
||||
// immediately bail
|
||||
return IterationDecision::Continue;
|
||||
}
|
||||
for (const auto& r2 : purgeable_range2->volatile_ranges().ranges()) {
|
||||
range = range.intersected(r2);
|
||||
if (range.is_empty())
|
||||
break;
|
||||
}
|
||||
if (range.is_empty())
|
||||
break;
|
||||
}
|
||||
if (range.is_empty())
|
||||
continue;
|
||||
IterationDecision decision = f(range);
|
||||
if (decision != IterationDecision::Continue)
|
||||
return decision;
|
||||
}
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
}
|
||||
|
||||
template<typename F>
|
||||
IterationDecision for_each_nonvolatile_range(F f) const
|
||||
{
|
||||
size_t base = 0;
|
||||
for_each_volatile_range([&](const VolatilePageRange& volatile_range) {
|
||||
if (volatile_range.base == base)
|
||||
return IterationDecision::Continue;
|
||||
IterationDecision decision = f({ base, volatile_range.base - base });
|
||||
if (decision != IterationDecision::Continue)
|
||||
return decision;
|
||||
base = volatile_range.base + volatile_range.count;
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
if (base < page_count())
|
||||
return f({ base, page_count() - base });
|
||||
return IterationDecision::Continue;
|
||||
}
|
||||
|
||||
size_t get_lazy_committed_page_count() const;
|
||||
|
||||
private:
|
||||
explicit AnonymousVMObject(size_t, AllocationStrategy);
|
||||
explicit AnonymousVMObject(PhysicalAddress, size_t);
|
||||
explicit AnonymousVMObject(PhysicalPage&);
|
||||
explicit AnonymousVMObject(const AnonymousVMObject&);
|
||||
|
||||
virtual const char* class_name() const override { return "AnonymousVMObject"; }
|
||||
|
||||
private:
|
||||
AnonymousVMObject(PhysicalAddress, size_t);
|
||||
int purge_impl();
|
||||
void update_volatile_cache();
|
||||
void set_was_purged(const VolatilePageRange&);
|
||||
size_t remove_lazy_commit_pages(const VolatilePageRange&);
|
||||
void range_made_volatile(const VolatilePageRange&);
|
||||
void range_made_nonvolatile(const VolatilePageRange&);
|
||||
size_t count_needed_commit_pages_for_nonvolatile_range(const VolatilePageRange&);
|
||||
size_t mark_committed_pages_for_nonvolatile_range(const VolatilePageRange&, size_t);
|
||||
bool is_nonvolatile(size_t page_index);
|
||||
|
||||
AnonymousVMObject& operator=(const AnonymousVMObject&) = delete;
|
||||
AnonymousVMObject& operator=(AnonymousVMObject&&) = delete;
|
||||
AnonymousVMObject(AnonymousVMObject&&) = delete;
|
||||
|
||||
virtual bool is_anonymous() const override { return true; }
|
||||
|
||||
Bitmap& ensure_cow_map();
|
||||
void ensure_or_reset_cow_map();
|
||||
|
||||
VolatilePageRanges m_volatile_ranges_cache;
|
||||
bool m_volatile_ranges_cache_dirty { true };
|
||||
Vector<PurgeablePageRanges*> m_purgeable_ranges;
|
||||
size_t m_unused_committed_pages { 0 };
|
||||
|
||||
mutable OwnPtr<Bitmap> m_cow_map;
|
||||
|
||||
// We share a pool of committed cow-pages with clones
|
||||
RefPtr<CommittedCowPages> m_shared_committed_cow_pages;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,7 +39,6 @@
|
|||
#include <Kernel/VM/MemoryManager.h>
|
||||
#include <Kernel/VM/PageDirectory.h>
|
||||
#include <Kernel/VM/PhysicalRegion.h>
|
||||
#include <Kernel/VM/PurgeableVMObject.h>
|
||||
#include <Kernel/VM/SharedInodeVMObject.h>
|
||||
|
||||
//#define MM_DEBUG
|
||||
|
|
@ -381,7 +380,6 @@ Region* MemoryManager::find_region_from_vaddr(VirtualAddress vaddr)
|
|||
PageFaultResponse MemoryManager::handle_page_fault(const PageFault& fault)
|
||||
{
|
||||
ASSERT_INTERRUPTS_DISABLED();
|
||||
ASSERT(Thread::current() != nullptr);
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
if (Processor::current().in_irq()) {
|
||||
dbg() << "CPU[" << Processor::current().id() << "] BUG! Page fault while handling IRQ! code=" << fault.code() << ", vaddr=" << fault.vaddr() << ", irq level: " << Processor::current().in_irq();
|
||||
|
|
@ -408,26 +406,20 @@ OwnPtr<Region> MemoryManager::allocate_contiguous_kernel_region(size_t size, con
|
|||
if (!range.is_valid())
|
||||
return nullptr;
|
||||
auto vmobject = ContiguousVMObject::create_with_size(size);
|
||||
auto region = allocate_kernel_region_with_vmobject(range, vmobject, name, access, user_accessible, cacheable);
|
||||
if (!region)
|
||||
return nullptr;
|
||||
return region;
|
||||
return allocate_kernel_region_with_vmobject(range, vmobject, name, access, user_accessible, cacheable);
|
||||
}
|
||||
|
||||
OwnPtr<Region> MemoryManager::allocate_kernel_region(size_t size, const StringView& name, u8 access, bool user_accessible, bool should_commit, bool cacheable)
|
||||
OwnPtr<Region> MemoryManager::allocate_kernel_region(size_t size, const StringView& name, u8 access, bool user_accessible, AllocationStrategy strategy, bool cacheable)
|
||||
{
|
||||
ASSERT(!(size % PAGE_SIZE));
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
auto range = kernel_page_directory().range_allocator().allocate_anywhere(size);
|
||||
if (!range.is_valid())
|
||||
return nullptr;
|
||||
auto vmobject = AnonymousVMObject::create_with_size(size);
|
||||
auto region = allocate_kernel_region_with_vmobject(range, vmobject, name, access, user_accessible, cacheable);
|
||||
if (!region)
|
||||
auto vmobject = AnonymousVMObject::create_with_size(size, strategy);
|
||||
if (!vmobject)
|
||||
return nullptr;
|
||||
if (should_commit && !region->commit())
|
||||
return nullptr;
|
||||
return region;
|
||||
return allocate_kernel_region_with_vmobject(range, vmobject.release_nonnull(), name, access, user_accessible, cacheable);
|
||||
}
|
||||
|
||||
OwnPtr<Region> MemoryManager::allocate_kernel_region(PhysicalAddress paddr, size_t size, const StringView& name, u8 access, bool user_accessible, bool cacheable)
|
||||
|
|
@ -458,7 +450,7 @@ OwnPtr<Region> MemoryManager::allocate_kernel_region_identity(PhysicalAddress pa
|
|||
|
||||
OwnPtr<Region> MemoryManager::allocate_user_accessible_kernel_region(size_t size, const StringView& name, u8 access, bool cacheable)
|
||||
{
|
||||
return allocate_kernel_region(size, name, access, true, true, cacheable);
|
||||
return allocate_kernel_region(size, name, access, true, AllocationStrategy::Reserve, cacheable);
|
||||
}
|
||||
|
||||
OwnPtr<Region> MemoryManager::allocate_kernel_region_with_vmobject(const Range& range, VMObject& vmobject, const StringView& name, u8 access, bool user_accessible, bool cacheable)
|
||||
|
|
@ -576,11 +568,11 @@ RefPtr<PhysicalPage> MemoryManager::allocate_user_physical_page(ShouldZeroFill s
|
|||
// We didn't have a single free physical page. Let's try to free something up!
|
||||
// First, we look for a purgeable VMObject in the volatile state.
|
||||
for_each_vmobject([&](auto& vmobject) {
|
||||
if (!vmobject.is_purgeable())
|
||||
if (!vmobject.is_anonymous())
|
||||
return IterationDecision::Continue;
|
||||
int purged_page_count = static_cast<PurgeableVMObject&>(vmobject).purge_with_interrupts_disabled({});
|
||||
int purged_page_count = static_cast<AnonymousVMObject&>(vmobject).purge_with_interrupts_disabled({});
|
||||
if (purged_page_count) {
|
||||
klog() << "MM: Purge saved the day! Purged " << purged_page_count << " pages from PurgeableVMObject{" << &vmobject << "}";
|
||||
klog() << "MM: Purge saved the day! Purged " << purged_page_count << " pages from AnonymousVMObject{" << &vmobject << "}";
|
||||
page = find_free_user_physical_page(false);
|
||||
purged_pages = true;
|
||||
ASSERT(page);
|
||||
|
|
@ -890,7 +882,7 @@ void MemoryManager::dump_kernel_regions()
|
|||
klog() << "BEGIN END SIZE ACCESS NAME";
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
for (auto& region : MM.m_kernel_regions) {
|
||||
klog() << String::format("%08x", region.vaddr().get()) << " -- " << String::format("%08x", region.vaddr().offset(region.size() - 1).get()) << " " << String::format("%08x", region.size()) << " " << (region.is_readable() ? 'R' : ' ') << (region.is_writable() ? 'W' : ' ') << (region.is_executable() ? 'X' : ' ') << (region.is_shared() ? 'S' : ' ') << (region.is_stack() ? 'T' : ' ') << (region.vmobject().is_purgeable() ? 'P' : ' ') << " " << region.name().characters();
|
||||
klog() << String::format("%08x", region.vaddr().get()) << " -- " << String::format("%08x", region.vaddr().offset(region.size() - 1).get()) << " " << String::format("%08x", region.size()) << " " << (region.is_readable() ? 'R' : ' ') << (region.is_writable() ? 'W' : ' ') << (region.is_executable() ? 'X' : ' ') << (region.is_shared() ? 'S' : ' ') << (region.is_stack() ? 'T' : ' ') << (region.vmobject().is_anonymous() ? 'A' : ' ') << " " << region.name().characters();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
#include <Kernel/Arch/i386/CPU.h>
|
||||
#include <Kernel/Forward.h>
|
||||
#include <Kernel/SpinLock.h>
|
||||
#include <Kernel/VM/AllocationStrategy.h>
|
||||
#include <Kernel/VM/PhysicalPage.h>
|
||||
#include <Kernel/VM/Region.h>
|
||||
#include <Kernel/VM/VMObject.h>
|
||||
|
|
@ -83,6 +84,7 @@ class MemoryManager {
|
|||
friend class PageDirectory;
|
||||
friend class PhysicalPage;
|
||||
friend class PhysicalRegion;
|
||||
friend class AnonymousVMObject;
|
||||
friend class Region;
|
||||
friend class VMObject;
|
||||
friend OwnPtr<KBuffer> procfs$mm(InodeIdentifier);
|
||||
|
|
@ -120,7 +122,7 @@ public:
|
|||
void deallocate_supervisor_physical_page(const PhysicalPage&);
|
||||
|
||||
OwnPtr<Region> allocate_contiguous_kernel_region(size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
||||
OwnPtr<Region> allocate_kernel_region(size_t, const StringView& name, u8 access, bool user_accessible = false, bool should_commit = true, bool cacheable = true);
|
||||
OwnPtr<Region> allocate_kernel_region(size_t, const StringView& name, u8 access, bool user_accessible = false, AllocationStrategy strategy = AllocationStrategy::Reserve, bool cacheable = true);
|
||||
OwnPtr<Region> allocate_kernel_region(PhysicalAddress, size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
||||
OwnPtr<Region> allocate_kernel_region_identity(PhysicalAddress, size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
||||
OwnPtr<Region> allocate_kernel_region_with_vmobject(VMObject&, size_t, const StringView& name, u8 access, bool user_accessible = false, bool cacheable = true);
|
||||
|
|
|
|||
37
Kernel/VM/PageFaultResponse.h
Normal file
37
Kernel/VM/PageFaultResponse.h
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
/*
|
||||
* Copyright (c) 2020, The SerenityOS developers.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace Kernel {
|
||||
|
||||
enum class PageFaultResponse {
|
||||
ShouldCrash,
|
||||
OutOfMemory,
|
||||
Continue,
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -27,10 +27,12 @@
|
|||
#include <AK/BinarySearch.h>
|
||||
#include <AK/ScopeGuard.h>
|
||||
#include <Kernel/Process.h>
|
||||
#include <Kernel/VM/AnonymousVMObject.h>
|
||||
#include <Kernel/VM/MemoryManager.h>
|
||||
#include <Kernel/VM/PhysicalPage.h>
|
||||
#include <Kernel/VM/PurgeableVMObject.h>
|
||||
#include <Kernel/VM/PurgeablePageRanges.h>
|
||||
|
||||
//#define PAGE_FAULT_DEBUG
|
||||
//#define VOLATILE_PAGE_RANGES_DEBUG
|
||||
|
||||
namespace Kernel {
|
||||
|
|
@ -51,6 +53,14 @@ static void dump_volatile_page_ranges(const Vector<VolatilePageRange>& ranges)
|
|||
}
|
||||
#endif
|
||||
|
||||
void VolatilePageRanges::add_unchecked(const VolatilePageRange& range)
|
||||
{
|
||||
auto add_range = m_total_range.intersected(range);
|
||||
if (add_range.is_empty())
|
||||
return;
|
||||
m_ranges.append(range);
|
||||
}
|
||||
|
||||
bool VolatilePageRanges::add(const VolatilePageRange& range)
|
||||
{
|
||||
auto add_range = m_total_range.intersected(range);
|
||||
|
|
@ -185,7 +195,7 @@ bool VolatilePageRanges::intersects(const VolatilePageRange& range) const
|
|||
}
|
||||
|
||||
PurgeablePageRanges::PurgeablePageRanges(const VMObject& vmobject)
|
||||
: m_volatile_ranges({ 0, vmobject.is_purgeable() ? static_cast<const PurgeableVMObject&>(vmobject).page_count() : 0 })
|
||||
: m_volatile_ranges({ 0, vmobject.is_anonymous() ? vmobject.page_count() : 0 })
|
||||
{
|
||||
}
|
||||
|
||||
|
|
@ -194,11 +204,11 @@ bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range)
|
|||
if (range.is_empty())
|
||||
return false;
|
||||
|
||||
// Since we may need to call into PurgeableVMObject we need to acquire
|
||||
// Since we may need to call into AnonymousVMObject we need to acquire
|
||||
// its lock as well, and acquire it first. This is important so that
|
||||
// we don't deadlock when a page fault (e.g. on another processor)
|
||||
// happens that is meant to lazy-allocate a committed page. It would
|
||||
// call into PurgeableVMObject::range_made_volatile, which then would
|
||||
// call into AnonymousVMObject::range_made_volatile, which then would
|
||||
// also call into this object and need to acquire m_lock. By acquiring
|
||||
// the vmobject lock first in both cases, we avoid deadlocking.
|
||||
// We can access m_vmobject without any locks for that purpose because
|
||||
|
|
@ -212,13 +222,47 @@ bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range)
|
|||
return added;
|
||||
}
|
||||
|
||||
bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged)
|
||||
auto PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged) -> RemoveVolatileError
|
||||
{
|
||||
if (range.is_empty())
|
||||
return false;
|
||||
if (range.is_empty()) {
|
||||
was_purged = false;
|
||||
return RemoveVolatileError::Success;
|
||||
}
|
||||
ScopedSpinLock vmobject_lock(m_vmobject->m_lock); // see comment in add_volatile_range
|
||||
ScopedSpinLock lock(m_volatile_ranges_lock);
|
||||
ASSERT(m_vmobject);
|
||||
return m_volatile_ranges.remove(range, was_purged);
|
||||
|
||||
// Before we actually remove this range, we need to check if we need
|
||||
// to commit any pages, which may fail. If it fails, we don't actually
|
||||
// want to make any modifications. COW pages are already accounted for
|
||||
// in m_shared_committed_cow_pages
|
||||
size_t need_commit_pages = 0;
|
||||
m_volatile_ranges.for_each_intersecting_range(range, [&](const VolatilePageRange& intersected_range) {
|
||||
need_commit_pages += m_vmobject->count_needed_commit_pages_for_nonvolatile_range(intersected_range);
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
if (need_commit_pages > 0) {
|
||||
// See if we can grab enough pages for what we're marking non-volatile
|
||||
if (!MM.commit_user_physical_pages(need_commit_pages))
|
||||
return RemoveVolatileError::OutOfMemory;
|
||||
|
||||
// Now that we are committed to these pages, mark them for lazy-commit allocation
|
||||
auto pages_to_mark = need_commit_pages;
|
||||
m_volatile_ranges.for_each_intersecting_range(range, [&](const VolatilePageRange& intersected_range) {
|
||||
auto pages_marked = m_vmobject->mark_committed_pages_for_nonvolatile_range(intersected_range, pages_to_mark);
|
||||
pages_to_mark -= pages_marked;
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
}
|
||||
|
||||
// Now actually remove the range
|
||||
if (m_volatile_ranges.remove(range, was_purged)) {
|
||||
m_vmobject->range_made_nonvolatile(range);
|
||||
return RemoveVolatileError::Success;
|
||||
}
|
||||
|
||||
ASSERT(need_commit_pages == 0); // We should have not touched anything
|
||||
return RemoveVolatileError::SuccessNoChange;
|
||||
}
|
||||
|
||||
bool PurgeablePageRanges::is_volatile_range(const VolatilePageRange& range) const
|
||||
|
|
@ -241,7 +285,7 @@ void PurgeablePageRanges::set_was_purged(const VolatilePageRange& range)
|
|||
m_volatile_ranges.add({ range.base, range.count, true });
|
||||
}
|
||||
|
||||
void PurgeablePageRanges::set_vmobject(PurgeableVMObject* vmobject)
|
||||
void PurgeablePageRanges::set_vmobject(AnonymousVMObject* vmobject)
|
||||
{
|
||||
// No lock needed here
|
||||
if (vmobject) {
|
||||
|
|
@ -253,207 +297,33 @@ void PurgeablePageRanges::set_vmobject(PurgeableVMObject* vmobject)
|
|||
}
|
||||
}
|
||||
|
||||
RefPtr<PurgeableVMObject> PurgeableVMObject::create_with_size(size_t size)
|
||||
CommittedCowPages::CommittedCowPages(size_t committed_pages)
|
||||
: m_committed_pages(committed_pages)
|
||||
{
|
||||
// We need to attempt to commit before actually creating the object
|
||||
if (!MM.commit_user_physical_pages(ceil_div(size, PAGE_SIZE)))
|
||||
return {};
|
||||
return adopt(*new PurgeableVMObject(size));
|
||||
}
|
||||
|
||||
PurgeableVMObject::PurgeableVMObject(size_t size)
|
||||
: AnonymousVMObject(size, false)
|
||||
, m_unused_committed_pages(page_count())
|
||||
CommittedCowPages::~CommittedCowPages()
|
||||
{
|
||||
for (size_t i = 0; i < page_count(); ++i)
|
||||
physical_pages()[i] = MM.lazy_committed_page();
|
||||
// Return unused committed pages
|
||||
if (m_committed_pages > 0)
|
||||
MM.uncommit_user_physical_pages(m_committed_pages);
|
||||
}
|
||||
|
||||
PurgeableVMObject::PurgeableVMObject(const PurgeableVMObject& other)
|
||||
: AnonymousVMObject(other)
|
||||
, m_purgeable_ranges() // do *not* clone this
|
||||
, m_unused_committed_pages(other.m_unused_committed_pages)
|
||||
NonnullRefPtr<PhysicalPage> CommittedCowPages::allocate_one()
|
||||
{
|
||||
// We can't really "copy" a spinlock. But we're holding it. Clear in the clone
|
||||
ASSERT(other.m_lock.is_locked());
|
||||
m_lock.initialize();
|
||||
}
|
||||
ASSERT(m_committed_pages > 0);
|
||||
m_committed_pages--;
|
||||
|
||||
PurgeableVMObject::~PurgeableVMObject()
|
||||
{
|
||||
if (m_unused_committed_pages > 0)
|
||||
MM.uncommit_user_physical_pages(m_unused_committed_pages);
|
||||
}
|
||||
|
||||
RefPtr<VMObject> PurgeableVMObject::clone()
|
||||
{
|
||||
// We need to acquire our lock so we copy a sane state
|
||||
ScopedSpinLock lock(m_lock);
|
||||
if (m_unused_committed_pages > 0) {
|
||||
// We haven't used up all committed pages. In order to be able
|
||||
// to clone ourselves, we need to be able to commit the same number
|
||||
// of pages first
|
||||
if (!MM.commit_user_physical_pages(m_unused_committed_pages))
|
||||
return {};
|
||||
}
|
||||
return adopt(*new PurgeableVMObject(*this));
|
||||
}
|
||||
|
||||
int PurgeableVMObject::purge()
|
||||
{
|
||||
LOCKER(m_paging_lock);
|
||||
return purge_impl();
|
||||
}
|
||||
|
||||
int PurgeableVMObject::purge_with_interrupts_disabled(Badge<MemoryManager>)
|
||||
{
|
||||
ASSERT_INTERRUPTS_DISABLED();
|
||||
if (m_paging_lock.is_locked())
|
||||
return 0;
|
||||
return purge_impl();
|
||||
}
|
||||
|
||||
void PurgeableVMObject::set_was_purged(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
for (auto* purgeable_ranges : m_purgeable_ranges)
|
||||
purgeable_ranges->set_was_purged(range);
|
||||
}
|
||||
|
||||
int PurgeableVMObject::purge_impl()
|
||||
{
|
||||
int purged_page_count = 0;
|
||||
ScopedSpinLock lock(m_lock);
|
||||
for_each_volatile_range([&](const auto& range) {
|
||||
int purged_in_range = 0;
|
||||
auto range_end = range.base + range.count;
|
||||
for (size_t i = range.base; i < range_end; i++) {
|
||||
auto& phys_page = m_physical_pages[i];
|
||||
if (phys_page && !phys_page->is_shared_zero_page()) {
|
||||
ASSERT(!phys_page->is_lazy_committed_page());
|
||||
++purged_in_range;
|
||||
}
|
||||
phys_page = MM.shared_zero_page();
|
||||
}
|
||||
|
||||
if (purged_in_range > 0) {
|
||||
purged_page_count += purged_in_range;
|
||||
set_was_purged(range);
|
||||
for_each_region([&](auto& region) {
|
||||
if (®ion.vmobject() == this) {
|
||||
if (auto owner = region.get_owner()) {
|
||||
// we need to hold a reference the process here (if there is one) as we may not own this region
|
||||
klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " owned by " << *owner << " at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
|
||||
} else {
|
||||
klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " (no ownership) at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
|
||||
}
|
||||
region.remap_page_range(range.base, range.count);
|
||||
}
|
||||
});
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
return purged_page_count;
|
||||
}
|
||||
|
||||
void PurgeableVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
purgeable_page_ranges.set_vmobject(this);
|
||||
ASSERT(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges));
|
||||
m_purgeable_ranges.append(&purgeable_page_ranges);
|
||||
}
|
||||
|
||||
void PurgeableVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
for (size_t i = 0; i < m_purgeable_ranges.size(); i++) {
|
||||
if (m_purgeable_ranges[i] != &purgeable_page_ranges)
|
||||
continue;
|
||||
purgeable_page_ranges.set_vmobject(nullptr);
|
||||
m_purgeable_ranges.remove(i);
|
||||
return;
|
||||
}
|
||||
ASSERT_NOT_REACHED();
|
||||
}
|
||||
|
||||
bool PurgeableVMObject::is_any_volatile() const
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
for (auto& volatile_ranges : m_purgeable_ranges) {
|
||||
ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock);
|
||||
if (!volatile_ranges->is_empty())
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
size_t PurgeableVMObject::remove_lazy_commit_pages(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
|
||||
size_t removed_count = 0;
|
||||
auto range_end = range.base + range.count;
|
||||
for (size_t i = range.base; i < range_end; i++) {
|
||||
auto& phys_page = m_physical_pages[i];
|
||||
if (phys_page && phys_page->is_lazy_committed_page()) {
|
||||
phys_page = MM.shared_zero_page();
|
||||
removed_count++;
|
||||
ASSERT(m_unused_committed_pages > 0);
|
||||
m_unused_committed_pages--;
|
||||
// if (--m_unused_committed_pages == 0)
|
||||
// break;
|
||||
}
|
||||
}
|
||||
return removed_count;
|
||||
}
|
||||
|
||||
void PurgeableVMObject::range_made_volatile(const VolatilePageRange& range)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
|
||||
if (m_unused_committed_pages == 0)
|
||||
return;
|
||||
|
||||
// We need to check this range for any pages that are marked for
|
||||
// lazy committed allocation and turn them into shared zero pages
|
||||
// and also adjust the m_unused_committed_pages for each such page.
|
||||
// Take into account all the other views as well.
|
||||
size_t uncommit_page_count = 0;
|
||||
for_each_volatile_range([&](const auto& r) {
|
||||
auto intersected = range.intersected(r);
|
||||
if (!intersected.is_empty()) {
|
||||
uncommit_page_count += remove_lazy_commit_pages(intersected);
|
||||
// if (m_unused_committed_pages == 0)
|
||||
// return IterationDecision::Break;
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
// Return those committed pages back to the system
|
||||
if (uncommit_page_count > 0)
|
||||
MM.uncommit_user_physical_pages(uncommit_page_count);
|
||||
}
|
||||
|
||||
RefPtr<PhysicalPage> PurgeableVMObject::allocate_committed_page(size_t page_index)
|
||||
{
|
||||
{
|
||||
ScopedSpinLock lock(m_lock);
|
||||
|
||||
ASSERT(m_unused_committed_pages > 0);
|
||||
|
||||
// We should't have any committed page tags in volatile regions
|
||||
ASSERT([&]() {
|
||||
for (auto* purgeable_ranges : m_purgeable_ranges) {
|
||||
if (purgeable_ranges->is_volatile(page_index))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}());
|
||||
|
||||
m_unused_committed_pages--;
|
||||
}
|
||||
return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
}
|
||||
|
||||
bool CommittedCowPages::return_one()
|
||||
{
|
||||
ASSERT(m_committed_pages > 0);
|
||||
m_committed_pages--;
|
||||
|
||||
MM.uncommit_user_physical_pages(1);
|
||||
return m_committed_pages == 0;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -26,8 +26,9 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Bitmap.h>
|
||||
#include <AK/RefCounted.h>
|
||||
#include <Kernel/SpinLock.h>
|
||||
#include <Kernel/VM/AnonymousVMObject.h>
|
||||
|
||||
namespace Kernel {
|
||||
|
||||
|
|
@ -118,7 +119,7 @@ public:
|
|||
}
|
||||
|
||||
bool is_empty() const { return m_ranges.is_empty(); }
|
||||
void clear() { m_ranges.clear(); }
|
||||
void clear() { m_ranges.clear_with_capacity(); }
|
||||
|
||||
bool is_all() const
|
||||
{
|
||||
|
|
@ -142,8 +143,60 @@ public:
|
|||
}
|
||||
|
||||
bool add(const VolatilePageRange&);
|
||||
void add_unchecked(const VolatilePageRange&);
|
||||
bool remove(const VolatilePageRange&, bool&);
|
||||
|
||||
template<typename F>
|
||||
IterationDecision for_each_intersecting_range(const VolatilePageRange& range, F f)
|
||||
{
|
||||
auto r = m_total_range.intersected(range);
|
||||
if (r.is_empty())
|
||||
return IterationDecision::Continue;
|
||||
|
||||
size_t nearby_index = 0;
|
||||
auto* existing_range = binary_search(
|
||||
m_ranges.span(), r, &nearby_index, [](auto& a, auto& b) {
|
||||
if (a.intersects(b))
|
||||
return 0;
|
||||
return (signed)(a.base - (b.base + b.count - 1));
|
||||
});
|
||||
if (!existing_range)
|
||||
return IterationDecision::Continue;
|
||||
|
||||
if (existing_range->range_equals(r))
|
||||
return f(r);
|
||||
ASSERT(existing_range == &m_ranges[nearby_index]); // sanity check
|
||||
while (nearby_index < m_ranges.size()) {
|
||||
existing_range = &m_ranges[nearby_index];
|
||||
if (!existing_range->intersects(range))
|
||||
break;
|
||||
|
||||
IterationDecision decision = f(existing_range->intersected(r));
|
||||
if (decision != IterationDecision::Continue)
|
||||
return decision;
|
||||
|
||||
nearby_index++;
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
}
|
||||
|
||||
template<typename F>
|
||||
IterationDecision for_each_nonvolatile_range(F f) const
|
||||
{
|
||||
size_t base = m_total_range.base;
|
||||
for (const auto& volatile_range : m_ranges) {
|
||||
if (volatile_range.base == base)
|
||||
continue;
|
||||
IterationDecision decision = f({ base, volatile_range.base - base });
|
||||
if (decision != IterationDecision::Continue)
|
||||
return decision;
|
||||
base = volatile_range.base + volatile_range.count;
|
||||
}
|
||||
if (base < m_total_range.base + m_total_range.count)
|
||||
return f({ base, (m_total_range.base + m_total_range.count) - base });
|
||||
return IterationDecision::Continue;
|
||||
}
|
||||
|
||||
Vector<VolatilePageRange>& ranges() { return m_ranges; }
|
||||
const Vector<VolatilePageRange>& ranges() const { return m_ranges; }
|
||||
|
||||
|
|
@ -152,15 +205,15 @@ private:
|
|||
VolatilePageRange m_total_range;
|
||||
};
|
||||
|
||||
class PurgeableVMObject;
|
||||
class AnonymousVMObject;
|
||||
|
||||
class PurgeablePageRanges {
|
||||
friend class PurgeableVMObject;
|
||||
friend class AnonymousVMObject;
|
||||
|
||||
public:
|
||||
PurgeablePageRanges(const VMObject&);
|
||||
|
||||
void set_purgeable_page_ranges(const PurgeablePageRanges& other)
|
||||
void copy_purgeable_page_ranges(const PurgeablePageRanges& other)
|
||||
{
|
||||
if (this == &other)
|
||||
return;
|
||||
|
|
@ -171,7 +224,12 @@ public:
|
|||
}
|
||||
|
||||
bool add_volatile_range(const VolatilePageRange& range);
|
||||
bool remove_volatile_range(const VolatilePageRange& range, bool& was_purged);
|
||||
enum class RemoveVolatileError {
|
||||
Success = 0,
|
||||
SuccessNoChange,
|
||||
OutOfMemory
|
||||
};
|
||||
RemoveVolatileError remove_volatile_range(const VolatilePageRange& range, bool& was_purged);
|
||||
bool is_volatile_range(const VolatilePageRange& range) const;
|
||||
bool is_volatile(size_t) const;
|
||||
|
||||
|
|
@ -182,92 +240,27 @@ public:
|
|||
const VolatilePageRanges& volatile_ranges() const { return m_volatile_ranges; }
|
||||
|
||||
protected:
|
||||
void set_vmobject(PurgeableVMObject*);
|
||||
void set_vmobject(AnonymousVMObject*);
|
||||
|
||||
VolatilePageRanges m_volatile_ranges;
|
||||
mutable RecursiveSpinLock m_volatile_ranges_lock;
|
||||
PurgeableVMObject* m_vmobject { nullptr };
|
||||
AnonymousVMObject* m_vmobject { nullptr };
|
||||
};
|
||||
|
||||
class PurgeableVMObject final : public AnonymousVMObject {
|
||||
friend class PurgeablePageRanges;
|
||||
class CommittedCowPages : public RefCounted<CommittedCowPages> {
|
||||
AK_MAKE_NONCOPYABLE(CommittedCowPages);
|
||||
|
||||
public:
|
||||
virtual ~PurgeableVMObject() override;
|
||||
CommittedCowPages() = delete;
|
||||
|
||||
static RefPtr<PurgeableVMObject> create_with_size(size_t);
|
||||
virtual RefPtr<VMObject> clone() override;
|
||||
CommittedCowPages(size_t);
|
||||
~CommittedCowPages();
|
||||
|
||||
virtual RefPtr<PhysicalPage> allocate_committed_page(size_t) override;
|
||||
|
||||
void register_purgeable_page_ranges(PurgeablePageRanges&);
|
||||
void unregister_purgeable_page_ranges(PurgeablePageRanges&);
|
||||
|
||||
int purge();
|
||||
int purge_with_interrupts_disabled(Badge<MemoryManager>);
|
||||
|
||||
bool is_any_volatile() const;
|
||||
|
||||
template<typename F>
|
||||
IterationDecision for_each_volatile_range(F f)
|
||||
{
|
||||
ASSERT(m_lock.is_locked());
|
||||
// This is a little ugly. Basically, we're trying to find the
|
||||
// volatile ranges that all share, because those are the only
|
||||
// pages we can actually purge
|
||||
for (auto* purgeable_range : m_purgeable_ranges) {
|
||||
ScopedSpinLock purgeable_lock(purgeable_range->m_volatile_ranges_lock);
|
||||
for (auto& r1 : purgeable_range->volatile_ranges().ranges()) {
|
||||
VolatilePageRange range(r1);
|
||||
for (auto* purgeable_range2 : m_purgeable_ranges) {
|
||||
if (purgeable_range2 == purgeable_range)
|
||||
continue;
|
||||
ScopedSpinLock purgeable2_lock(purgeable_range2->m_volatile_ranges_lock);
|
||||
if (purgeable_range2->is_empty()) {
|
||||
// If just one doesn't allow any purging, we can
|
||||
// immediately bail
|
||||
return IterationDecision::Continue;
|
||||
}
|
||||
for (const auto& r2 : purgeable_range2->volatile_ranges().ranges()) {
|
||||
range = range.intersected(r2);
|
||||
if (range.is_empty())
|
||||
break;
|
||||
}
|
||||
if (range.is_empty())
|
||||
break;
|
||||
}
|
||||
if (range.is_empty())
|
||||
continue;
|
||||
IterationDecision decision = f(range);
|
||||
if (decision != IterationDecision::Continue)
|
||||
return decision;
|
||||
}
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
}
|
||||
|
||||
size_t get_lazy_committed_page_count() const;
|
||||
NonnullRefPtr<PhysicalPage> allocate_one();
|
||||
bool return_one();
|
||||
|
||||
private:
|
||||
explicit PurgeableVMObject(size_t);
|
||||
explicit PurgeableVMObject(const PurgeableVMObject&);
|
||||
|
||||
virtual const char* class_name() const override { return "PurgeableVMObject"; }
|
||||
|
||||
int purge_impl();
|
||||
void set_was_purged(const VolatilePageRange&);
|
||||
size_t remove_lazy_commit_pages(const VolatilePageRange&);
|
||||
void range_made_volatile(const VolatilePageRange&);
|
||||
|
||||
PurgeableVMObject& operator=(const PurgeableVMObject&) = delete;
|
||||
PurgeableVMObject& operator=(PurgeableVMObject&&) = delete;
|
||||
PurgeableVMObject(PurgeableVMObject&&) = delete;
|
||||
|
||||
virtual bool is_purgeable() const override { return true; }
|
||||
|
||||
Vector<PurgeablePageRanges*> m_purgeable_ranges;
|
||||
mutable SpinLock<u8> m_lock;
|
||||
size_t m_unused_committed_pages { 0 };
|
||||
size_t m_committed_pages;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -32,7 +32,6 @@
|
|||
#include <Kernel/VM/AnonymousVMObject.h>
|
||||
#include <Kernel/VM/MemoryManager.h>
|
||||
#include <Kernel/VM/PageDirectory.h>
|
||||
#include <Kernel/VM/PurgeableVMObject.h>
|
||||
#include <Kernel/VM/Region.h>
|
||||
#include <Kernel/VM/SharedInodeVMObject.h>
|
||||
|
||||
|
|
@ -73,16 +72,16 @@ Region::~Region()
|
|||
|
||||
void Region::register_purgeable_page_ranges()
|
||||
{
|
||||
if (m_vmobject->is_purgeable()) {
|
||||
auto& vmobject = static_cast<PurgeableVMObject&>(*m_vmobject);
|
||||
if (m_vmobject->is_anonymous()) {
|
||||
auto& vmobject = static_cast<AnonymousVMObject&>(*m_vmobject);
|
||||
vmobject.register_purgeable_page_ranges(*this);
|
||||
}
|
||||
}
|
||||
|
||||
void Region::unregister_purgeable_page_ranges()
|
||||
{
|
||||
if (m_vmobject->is_purgeable()) {
|
||||
auto& vmobject = static_cast<PurgeableVMObject&>(*m_vmobject);
|
||||
if (m_vmobject->is_anonymous()) {
|
||||
auto& vmobject = static_cast<AnonymousVMObject&>(*m_vmobject);
|
||||
vmobject.unregister_purgeable_page_ranges(*this);
|
||||
}
|
||||
}
|
||||
|
|
@ -96,8 +95,11 @@ OwnPtr<Region> Region::clone()
|
|||
ASSERT(m_mmap);
|
||||
ASSERT(!m_shared);
|
||||
ASSERT(vmobject().is_anonymous());
|
||||
auto zeroed_region = Region::create_user_accessible(get_owner().ptr(), m_range, AnonymousVMObject::create_with_size(size()), 0, m_name, m_access);
|
||||
zeroed_region->set_purgeable_page_ranges(*this);
|
||||
auto new_vmobject = AnonymousVMObject::create_with_size(size(), AllocationStrategy::Reserve); // TODO: inherit committed non-volatile areas?
|
||||
if (!new_vmobject)
|
||||
return {};
|
||||
auto zeroed_region = Region::create_user_accessible(get_owner().ptr(), m_range, new_vmobject.release_nonnull(), 0, m_name, m_access);
|
||||
zeroed_region->copy_purgeable_page_ranges(*this);
|
||||
zeroed_region->set_mmap(m_mmap);
|
||||
zeroed_region->set_inherit_mode(m_inherit_mode);
|
||||
return zeroed_region;
|
||||
|
|
@ -113,7 +115,8 @@ OwnPtr<Region> Region::clone()
|
|||
|
||||
// Create a new region backed by the same VMObject.
|
||||
auto region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject, m_offset_in_vmobject, m_name, m_access);
|
||||
region->set_purgeable_page_ranges(*this);
|
||||
if (m_vmobject->is_anonymous())
|
||||
region->copy_purgeable_page_ranges(*this);
|
||||
region->set_mmap(m_mmap);
|
||||
region->set_shared(m_shared);
|
||||
return region;
|
||||
|
|
@ -122,7 +125,7 @@ OwnPtr<Region> Region::clone()
|
|||
if (vmobject().is_inode())
|
||||
ASSERT(vmobject().is_private_inode());
|
||||
|
||||
auto vmobject_clone = m_vmobject->clone();
|
||||
auto vmobject_clone = vmobject().clone();
|
||||
if (!vmobject_clone)
|
||||
return {};
|
||||
|
||||
|
|
@ -130,11 +133,10 @@ OwnPtr<Region> Region::clone()
|
|||
dbg() << "Region::clone(): CoWing " << name() << " (" << vaddr() << ")";
|
||||
#endif
|
||||
// Set up a COW region. The parent (this) region becomes COW as well!
|
||||
ensure_cow_map().fill(true);
|
||||
remap();
|
||||
auto clone_region = Region::create_user_accessible(get_owner().ptr(), m_range, vmobject_clone.release_nonnull(), m_offset_in_vmobject, m_name, m_access);
|
||||
clone_region->set_purgeable_page_ranges(*this);
|
||||
clone_region->ensure_cow_map();
|
||||
if (m_vmobject->is_anonymous())
|
||||
clone_region->copy_purgeable_page_ranges(*this);
|
||||
if (m_stack) {
|
||||
ASSERT(is_readable());
|
||||
ASSERT(is_writable());
|
||||
|
|
@ -156,7 +158,7 @@ void Region::set_vmobject(NonnullRefPtr<VMObject>&& obj)
|
|||
|
||||
bool Region::is_volatile(VirtualAddress vaddr, size_t size) const
|
||||
{
|
||||
if (!m_vmobject->is_purgeable())
|
||||
if (!m_vmobject->is_anonymous())
|
||||
return false;
|
||||
|
||||
auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject);
|
||||
|
|
@ -168,7 +170,7 @@ bool Region::is_volatile(VirtualAddress vaddr, size_t size) const
|
|||
auto Region::set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged) -> SetVolatileError
|
||||
{
|
||||
was_purged = false;
|
||||
if (!m_vmobject->is_purgeable())
|
||||
if (!m_vmobject->is_anonymous())
|
||||
return SetVolatileError::NotPurgeable;
|
||||
|
||||
auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject);
|
||||
|
|
@ -187,70 +189,22 @@ auto Region::set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, b
|
|||
// end of the range doesn't inadvertedly get discarded.
|
||||
size_t first_page_index = PAGE_ROUND_DOWN(offset_in_vmobject) / PAGE_SIZE;
|
||||
size_t last_page_index = PAGE_ROUND_UP(offset_in_vmobject + size) / PAGE_SIZE;
|
||||
if (remove_volatile_range({ first_page_index, last_page_index - first_page_index }, was_purged)) {
|
||||
// Attempt to remap the page range. We want to make sure we have
|
||||
// enough memory, if not we need to inform the caller of that
|
||||
// fact
|
||||
if (!remap_page_range(first_page_index, last_page_index - first_page_index))
|
||||
return SetVolatileError::OutOfMemory;
|
||||
switch (remove_volatile_range({ first_page_index, last_page_index - first_page_index }, was_purged)) {
|
||||
case PurgeablePageRanges::RemoveVolatileError::Success:
|
||||
case PurgeablePageRanges::RemoveVolatileError::SuccessNoChange:
|
||||
break;
|
||||
case PurgeablePageRanges::RemoveVolatileError::OutOfMemory:
|
||||
return SetVolatileError::OutOfMemory;
|
||||
}
|
||||
}
|
||||
return SetVolatileError::Success;
|
||||
}
|
||||
|
||||
bool Region::can_commit() const
|
||||
size_t Region::cow_pages() const
|
||||
{
|
||||
return vmobject().is_anonymous() || vmobject().is_purgeable();
|
||||
}
|
||||
|
||||
bool Region::commit()
|
||||
{
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
#ifdef MM_DEBUG
|
||||
dbg() << "MM: Commit " << page_count() << " pages in Region " << this << " (VMO=" << &vmobject() << ") at " << vaddr();
|
||||
#endif
|
||||
for (size_t i = 0; i < page_count(); ++i) {
|
||||
if (!commit(i)) {
|
||||
// Flush what we did commit
|
||||
if (i > 0)
|
||||
MM.flush_tlb(vaddr(), i + 1);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
MM.flush_tlb(vaddr(), page_count());
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Region::commit(size_t page_index)
|
||||
{
|
||||
ASSERT(vmobject().is_anonymous() || vmobject().is_purgeable());
|
||||
ASSERT(s_mm_lock.own_lock());
|
||||
auto& vmobject_physical_page_entry = physical_page_slot(page_index);
|
||||
if (!vmobject_physical_page_entry.is_null() && !vmobject_physical_page_entry->is_shared_zero_page())
|
||||
return true;
|
||||
RefPtr<PhysicalPage> physical_page;
|
||||
if (vmobject_physical_page_entry->is_lazy_committed_page()) {
|
||||
physical_page = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page(page_index);
|
||||
} else {
|
||||
physical_page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
if (!physical_page) {
|
||||
klog() << "MM: commit was unable to allocate a physical page";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
vmobject_physical_page_entry = move(physical_page);
|
||||
remap_page(page_index, false); // caller is in charge of flushing tlb
|
||||
return true;
|
||||
}
|
||||
|
||||
u32 Region::cow_pages() const
|
||||
{
|
||||
if (!m_cow_map)
|
||||
if (!vmobject().is_anonymous())
|
||||
return 0;
|
||||
u32 count = 0;
|
||||
for (size_t i = 0; i < m_cow_map->size(); ++i)
|
||||
count += m_cow_map->get(i);
|
||||
return count;
|
||||
return static_cast<const AnonymousVMObject&>(vmobject()).cow_pages();
|
||||
}
|
||||
|
||||
size_t Region::amount_dirty() const
|
||||
|
|
@ -300,25 +254,16 @@ NonnullOwnPtr<Region> Region::create_kernel_only(const Range& range, NonnullRefP
|
|||
|
||||
bool Region::should_cow(size_t page_index) const
|
||||
{
|
||||
auto* page = physical_page(page_index);
|
||||
if (page && (page->is_shared_zero_page() || page->is_lazy_committed_page()))
|
||||
return true;
|
||||
if (m_shared)
|
||||
if (!vmobject().is_anonymous())
|
||||
return false;
|
||||
return m_cow_map && m_cow_map->get(page_index);
|
||||
return static_cast<const AnonymousVMObject&>(vmobject()).should_cow(first_page_index() + page_index, m_shared);
|
||||
}
|
||||
|
||||
void Region::set_should_cow(size_t page_index, bool cow)
|
||||
{
|
||||
ASSERT(!m_shared);
|
||||
ensure_cow_map().set(page_index, cow);
|
||||
}
|
||||
|
||||
Bitmap& Region::ensure_cow_map() const
|
||||
{
|
||||
if (!m_cow_map)
|
||||
m_cow_map = make<Bitmap>(page_count(), true);
|
||||
return *m_cow_map;
|
||||
if (vmobject().is_anonymous())
|
||||
static_cast<AnonymousVMObject&>(vmobject()).set_should_cow(first_page_index() + page_index, cow);
|
||||
}
|
||||
|
||||
bool Region::map_individual_page_impl(size_t page_index)
|
||||
|
|
@ -339,7 +284,7 @@ bool Region::map_individual_page_impl(size_t page_index)
|
|||
pte->set_cache_disabled(!m_cacheable);
|
||||
pte->set_physical_page_base(page->paddr().get());
|
||||
pte->set_present(true);
|
||||
if (should_cow(page_index))
|
||||
if (page->is_shared_zero_page() || page->is_lazy_committed_page() || should_cow(page_index))
|
||||
pte->set_writable(false);
|
||||
else
|
||||
pte->set_writable(is_writable());
|
||||
|
|
@ -387,7 +332,8 @@ bool Region::remap_page(size_t page_index, bool with_flush)
|
|||
void Region::unmap(ShouldDeallocateVirtualMemoryRange deallocate_range)
|
||||
{
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
ASSERT(m_page_directory);
|
||||
if (!m_page_directory)
|
||||
return;
|
||||
ScopedSpinLock page_lock(m_page_directory->get_lock());
|
||||
size_t count = page_count();
|
||||
for (size_t i = 0; i < count; ++i) {
|
||||
|
|
@ -444,6 +390,7 @@ void Region::remap()
|
|||
|
||||
PageFaultResponse Region::handle_fault(const PageFault& fault)
|
||||
{
|
||||
ScopedSpinLock lock(s_mm_lock);
|
||||
auto page_index_in_region = page_index_from_address(fault.vaddr());
|
||||
if (fault.type() == PageFault::Type::PageNotPresent) {
|
||||
if (fault.is_read() && !is_readable()) {
|
||||
|
|
@ -482,12 +429,12 @@ PageFaultResponse Region::handle_fault(const PageFault& fault)
|
|||
ASSERT(fault.type() == PageFault::Type::ProtectionViolation);
|
||||
if (fault.access() == PageFault::Access::Write && is_writable() && should_cow(page_index_in_region)) {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << "PV(cow) fault in Region{" << this << "}[" << page_index_in_region << "]";
|
||||
dbg() << "PV(cow) fault in Region{" << this << "}[" << page_index_in_region << "] at " << fault.vaddr();
|
||||
#endif
|
||||
auto* phys_page = physical_page(page_index_in_region);
|
||||
if (phys_page->is_shared_zero_page() || phys_page->is_lazy_committed_page()) {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << "NP(zero) fault in Region{" << this << "}[" << page_index_in_region << "]";
|
||||
dbg() << "NP(zero) fault in Region{" << this << "}[" << page_index_in_region << "] at " << fault.vaddr();
|
||||
#endif
|
||||
return handle_zero_fault(page_index_in_region);
|
||||
}
|
||||
|
|
@ -521,17 +468,20 @@ PageFaultResponse Region::handle_zero_fault(size_t page_index_in_region)
|
|||
|
||||
if (page_slot->is_lazy_committed_page()) {
|
||||
page_slot = static_cast<AnonymousVMObject&>(*m_vmobject).allocate_committed_page(page_index_in_region);
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> ALLOCATED COMMITTED " << page_slot->paddr();
|
||||
#endif
|
||||
} else {
|
||||
page_slot = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
||||
if (page_slot.is_null()) {
|
||||
klog() << "MM: handle_zero_fault was unable to allocate a physical page";
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
}
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> ALLOCATED " << page_slot->paddr();
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> ZERO " << page_slot->paddr();
|
||||
#endif
|
||||
if (!remap_page(page_index_in_region)) {
|
||||
klog() << "MM: handle_zero_fault was unable to allocate a page table to map " << page_slot;
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
|
|
@ -542,53 +492,17 @@ PageFaultResponse Region::handle_zero_fault(size_t page_index_in_region)
|
|||
PageFaultResponse Region::handle_cow_fault(size_t page_index_in_region)
|
||||
{
|
||||
ASSERT_INTERRUPTS_DISABLED();
|
||||
auto& page_slot = physical_page_slot(page_index_in_region);
|
||||
if (page_slot->ref_count() == 1) {
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> It's a COW page but nobody is sharing it anymore. Remap r/w";
|
||||
#endif
|
||||
set_should_cow(page_index_in_region, false);
|
||||
if (!remap_page(page_index_in_region))
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
return PageFaultResponse::Continue;
|
||||
}
|
||||
|
||||
auto current_thread = Thread::current();
|
||||
if (current_thread)
|
||||
current_thread->did_cow_fault();
|
||||
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> It's a COW page and it's time to COW!";
|
||||
#endif
|
||||
auto page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::No);
|
||||
if (page.is_null()) {
|
||||
klog() << "MM: handle_cow_fault was unable to allocate a physical page";
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
}
|
||||
if (!vmobject().is_anonymous())
|
||||
return PageFaultResponse::ShouldCrash;
|
||||
|
||||
u8* dest_ptr = MM.quickmap_page(*page);
|
||||
const u8* src_ptr = vaddr().offset(page_index_in_region * PAGE_SIZE).as_ptr();
|
||||
#ifdef PAGE_FAULT_DEBUG
|
||||
dbg() << " >> COW " << page->paddr() << " <- " << page_slot->paddr();
|
||||
#endif
|
||||
{
|
||||
SmapDisabler disabler;
|
||||
void* fault_at;
|
||||
if (!safe_memcpy(dest_ptr, src_ptr, PAGE_SIZE, fault_at)) {
|
||||
if ((u8*)fault_at >= dest_ptr && (u8*)fault_at <= dest_ptr + PAGE_SIZE)
|
||||
dbg() << " >> COW: error copying page " << page_slot->paddr() << "/" << VirtualAddress(src_ptr) << " to " << page->paddr() << "/" << VirtualAddress(dest_ptr) << ": failed to write to page at " << VirtualAddress(fault_at);
|
||||
else if ((u8*)fault_at >= src_ptr && (u8*)fault_at <= src_ptr + PAGE_SIZE)
|
||||
dbg() << " >> COW: error copying page " << page_slot->paddr() << "/" << VirtualAddress(src_ptr) << " to " << page->paddr() << "/" << VirtualAddress(dest_ptr) << ": failed to read from page at " << VirtualAddress(fault_at);
|
||||
else
|
||||
ASSERT_NOT_REACHED();
|
||||
}
|
||||
}
|
||||
page_slot = move(page);
|
||||
MM.unquickmap_page();
|
||||
set_should_cow(page_index_in_region, false);
|
||||
auto response = reinterpret_cast<AnonymousVMObject&>(vmobject()).handle_cow_fault(first_page_index() + page_index_in_region, vaddr().offset(page_index_in_region * PAGE_SIZE));
|
||||
if (!remap_page(page_index_in_region))
|
||||
return PageFaultResponse::OutOfMemory;
|
||||
return PageFaultResponse::Continue;
|
||||
return response;
|
||||
}
|
||||
|
||||
PageFaultResponse Region::handle_inode_fault(size_t page_index_in_region)
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@
|
|||
#include <AK/Weakable.h>
|
||||
#include <Kernel/Arch/i386/CPU.h>
|
||||
#include <Kernel/Heap/SlabAllocator.h>
|
||||
#include <Kernel/VM/PurgeableVMObject.h>
|
||||
#include <Kernel/VM/PageFaultResponse.h>
|
||||
#include <Kernel/VM/PurgeablePageRanges.h>
|
||||
#include <Kernel/VM/RangeAllocator.h>
|
||||
#include <Kernel/VM/VMObject.h>
|
||||
|
||||
|
|
@ -41,12 +42,6 @@ namespace Kernel {
|
|||
class Inode;
|
||||
class VMObject;
|
||||
|
||||
enum class PageFaultResponse {
|
||||
ShouldCrash,
|
||||
OutOfMemory,
|
||||
Continue,
|
||||
};
|
||||
|
||||
class Region final
|
||||
: public InlineLinkedListNode<Region>
|
||||
, public Weakable<Region>
|
||||
|
|
@ -159,9 +154,6 @@ public:
|
|||
return m_offset_in_vmobject;
|
||||
}
|
||||
|
||||
bool can_commit() const;
|
||||
bool commit();
|
||||
|
||||
size_t amount_resident() const;
|
||||
size_t amount_shared() const;
|
||||
size_t amount_dirty() const;
|
||||
|
|
@ -169,7 +161,7 @@ public:
|
|||
bool should_cow(size_t page_index) const;
|
||||
void set_should_cow(size_t page_index, bool);
|
||||
|
||||
u32 cow_pages() const;
|
||||
size_t cow_pages() const;
|
||||
|
||||
void set_readable(bool b) { set_access_bit(Access::Read, b); }
|
||||
void set_writable(bool b) { set_access_bit(Access::Write, b); }
|
||||
|
|
@ -207,8 +199,6 @@ public:
|
|||
RefPtr<Process> get_owner();
|
||||
|
||||
private:
|
||||
Bitmap& ensure_cow_map() const;
|
||||
|
||||
void set_access_bit(Access access, bool b)
|
||||
{
|
||||
if (b)
|
||||
|
|
@ -217,7 +207,6 @@ private:
|
|||
m_access &= ~access;
|
||||
}
|
||||
|
||||
bool commit(size_t page_index);
|
||||
bool remap_page(size_t index, bool with_flush = true);
|
||||
|
||||
PageFaultResponse handle_cow_fault(size_t page_index);
|
||||
|
|
@ -242,7 +231,6 @@ private:
|
|||
bool m_stack : 1 { false };
|
||||
bool m_mmap : 1 { false };
|
||||
bool m_kernel : 1 { false };
|
||||
mutable OwnPtr<Bitmap> m_cow_map;
|
||||
WeakPtr<Process> m_owner;
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -50,7 +50,6 @@ public:
|
|||
virtual RefPtr<VMObject> clone() = 0;
|
||||
|
||||
virtual bool is_anonymous() const { return false; }
|
||||
virtual bool is_purgeable() const { return false; }
|
||||
virtual bool is_inode() const { return false; }
|
||||
virtual bool is_shared_inode() const { return false; }
|
||||
virtual bool is_private_inode() const { return false; }
|
||||
|
|
@ -78,6 +77,8 @@ protected:
|
|||
Vector<RefPtr<PhysicalPage>> m_physical_pages;
|
||||
Lock m_paging_lock { "VMObject" };
|
||||
|
||||
mutable SpinLock<u8> m_lock;
|
||||
|
||||
private:
|
||||
VMObject& operator=(const VMObject&) = delete;
|
||||
VMObject& operator=(VMObject&&) = delete;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue