mirror of
https://github.com/RGBCube/serenity
synced 2025-05-16 18:35:07 +00:00
082ed6f417
This patch greatly simplifies VMObject locking by doing two things: 1. Giving VMObject an IntrusiveList of all its mapping Region objects. 2. Removing VMObject::m_paging_lock in favor of VMObject::m_lock Before (1), VMObject::for_each_region() was forced to acquire the global MM lock (since it worked by walking MemoryManager's list of all regions and checking for regions that pointed to itself.) With each VMObject having its own list of Regions, VMObject's own m_lock is all we need. Before (2), page fault handlers used a separate mutex for preventing overlapping work. This design required multiple temporary unlocks and was generally extremely hard to reason about. Instead, page fault handlers now use VMObject's own m_lock as well.
465 lines
17 KiB
C++
465 lines
17 KiB
C++
/*
|
|
* Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#include <Kernel/Arch/x86/SmapDisabler.h>
|
|
#include <Kernel/Debug.h>
|
|
#include <Kernel/Process.h>
|
|
#include <Kernel/VM/AnonymousVMObject.h>
|
|
#include <Kernel/VM/MemoryManager.h>
|
|
#include <Kernel/VM/PhysicalPage.h>
|
|
|
|
namespace Kernel {
|
|
|
|
RefPtr<VMObject> AnonymousVMObject::try_clone()
|
|
{
|
|
// We need to acquire our lock so we copy a sane state
|
|
ScopedSpinLock lock(m_lock);
|
|
|
|
// We're the parent. Since we're about to become COW we need to
|
|
// commit the number of pages that we need to potentially allocate
|
|
// so that the parent is still guaranteed to be able to have all
|
|
// non-volatile memory available.
|
|
size_t need_cow_pages = 0;
|
|
|
|
// We definitely need to commit non-volatile areas
|
|
for_each_nonvolatile_range([&](VolatilePageRange const& nonvolatile_range) {
|
|
need_cow_pages += nonvolatile_range.count;
|
|
});
|
|
|
|
dbgln_if(COMMIT_DEBUG, "Cloning {:p}, need {} committed cow pages", this, need_cow_pages);
|
|
|
|
if (!MM.commit_user_physical_pages(need_cow_pages))
|
|
return {};
|
|
|
|
// Create or replace the committed cow pages. When cloning a previously
|
|
// cloned vmobject, we want to essentially "fork", leaving us and the
|
|
// new clone with one set of shared committed cow pages, and the original
|
|
// one would keep the one it still has. This ensures that the original
|
|
// one and this one, as well as the clone have sufficient resources
|
|
// to cow all pages as needed
|
|
m_shared_committed_cow_pages = try_create<CommittedCowPages>(need_cow_pages);
|
|
|
|
if (!m_shared_committed_cow_pages) {
|
|
MM.uncommit_user_physical_pages(need_cow_pages);
|
|
return {};
|
|
}
|
|
|
|
// Both original and clone become COW. So create a COW map for ourselves
|
|
// or reset all pages to be copied again if we were previously cloned
|
|
ensure_or_reset_cow_map();
|
|
|
|
// FIXME: If this allocation fails, we need to rollback all changes.
|
|
return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(*this));
|
|
}
|
|
|
|
RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_size(size_t size, AllocationStrategy commit)
|
|
{
|
|
if (commit == AllocationStrategy::Reserve || commit == AllocationStrategy::AllocateNow) {
|
|
// We need to attempt to commit before actually creating the object
|
|
if (!MM.commit_user_physical_pages(ceil_div(size, static_cast<size_t>(PAGE_SIZE))))
|
|
return {};
|
|
}
|
|
return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(size, commit));
|
|
}
|
|
|
|
RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_with_physical_pages(Span<NonnullRefPtr<PhysicalPage>> physical_pages)
|
|
{
|
|
return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(physical_pages));
|
|
}
|
|
|
|
RefPtr<AnonymousVMObject> AnonymousVMObject::try_create_for_physical_range(PhysicalAddress paddr, size_t size)
|
|
{
|
|
if (paddr.offset(size) < paddr) {
|
|
dbgln("Shenanigans! try_create_for_physical_range({}, {}) would wrap around", paddr, size);
|
|
return nullptr;
|
|
}
|
|
return adopt_ref_if_nonnull(new (nothrow) AnonymousVMObject(paddr, size));
|
|
}
|
|
|
|
AnonymousVMObject::AnonymousVMObject(size_t size, AllocationStrategy strategy)
|
|
: VMObject(size)
|
|
, m_volatile_ranges_cache({ 0, page_count() })
|
|
, m_unused_committed_pages(strategy == AllocationStrategy::Reserve ? page_count() : 0)
|
|
{
|
|
if (strategy == AllocationStrategy::AllocateNow) {
|
|
// Allocate all pages right now. We know we can get all because we committed the amount needed
|
|
for (size_t i = 0; i < page_count(); ++i)
|
|
physical_pages()[i] = MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
|
} else {
|
|
auto& initial_page = (strategy == AllocationStrategy::Reserve) ? MM.lazy_committed_page() : MM.shared_zero_page();
|
|
for (size_t i = 0; i < page_count(); ++i)
|
|
physical_pages()[i] = initial_page;
|
|
}
|
|
}
|
|
|
|
AnonymousVMObject::AnonymousVMObject(PhysicalAddress paddr, size_t size)
|
|
: VMObject(size)
|
|
, m_volatile_ranges_cache({ 0, page_count() })
|
|
{
|
|
VERIFY(paddr.page_base() == paddr);
|
|
for (size_t i = 0; i < page_count(); ++i)
|
|
physical_pages()[i] = PhysicalPage::create(paddr.offset(i * PAGE_SIZE), MayReturnToFreeList::No);
|
|
}
|
|
|
|
AnonymousVMObject::AnonymousVMObject(Span<NonnullRefPtr<PhysicalPage>> physical_pages)
|
|
: VMObject(physical_pages.size() * PAGE_SIZE)
|
|
, m_volatile_ranges_cache({ 0, page_count() })
|
|
{
|
|
for (size_t i = 0; i < physical_pages.size(); ++i) {
|
|
m_physical_pages[i] = physical_pages[i];
|
|
}
|
|
}
|
|
|
|
AnonymousVMObject::AnonymousVMObject(AnonymousVMObject const& other)
|
|
: VMObject(other)
|
|
, m_volatile_ranges_cache({ 0, page_count() }) // do *not* clone this
|
|
, m_volatile_ranges_cache_dirty(true) // do *not* clone this
|
|
, m_purgeable_ranges() // do *not* clone this
|
|
, m_unused_committed_pages(other.m_unused_committed_pages)
|
|
, m_cow_map() // do *not* clone this
|
|
, m_shared_committed_cow_pages(other.m_shared_committed_cow_pages) // share the pool
|
|
{
|
|
// We can't really "copy" a spinlock. But we're holding it. Clear in the clone
|
|
VERIFY(other.m_lock.is_locked());
|
|
m_lock.initialize();
|
|
|
|
// The clone also becomes COW
|
|
ensure_or_reset_cow_map();
|
|
|
|
if (m_unused_committed_pages > 0) {
|
|
// The original vmobject didn't use up all committed pages. When
|
|
// cloning (fork) we will overcommit. For this purpose we drop all
|
|
// lazy-commit references and replace them with shared zero pages.
|
|
for (size_t i = 0; i < page_count(); i++) {
|
|
auto& phys_page = m_physical_pages[i];
|
|
if (phys_page && phys_page->is_lazy_committed_page()) {
|
|
phys_page = MM.shared_zero_page();
|
|
if (--m_unused_committed_pages == 0)
|
|
break;
|
|
}
|
|
}
|
|
VERIFY(m_unused_committed_pages == 0);
|
|
}
|
|
}
|
|
|
|
AnonymousVMObject::~AnonymousVMObject()
|
|
{
|
|
// Return any unused committed pages
|
|
if (m_unused_committed_pages > 0)
|
|
MM.uncommit_user_physical_pages(m_unused_committed_pages);
|
|
}
|
|
|
|
int AnonymousVMObject::purge()
|
|
{
|
|
int purged_page_count = 0;
|
|
ScopedSpinLock lock(m_lock);
|
|
for_each_volatile_range([&](auto const& range) {
|
|
int purged_in_range = 0;
|
|
auto range_end = range.base + range.count;
|
|
for (size_t i = range.base; i < range_end; i++) {
|
|
auto& phys_page = m_physical_pages[i];
|
|
if (phys_page && !phys_page->is_shared_zero_page()) {
|
|
VERIFY(!phys_page->is_lazy_committed_page());
|
|
++purged_in_range;
|
|
}
|
|
phys_page = MM.shared_zero_page();
|
|
}
|
|
|
|
if (purged_in_range > 0) {
|
|
purged_page_count += purged_in_range;
|
|
set_was_purged(range);
|
|
for_each_region([&](auto& region) {
|
|
if (auto owner = region.get_owner()) {
|
|
// we need to hold a reference the process here (if there is one) as we may not own this region
|
|
dmesgln("Purged {} pages from region {} owned by {} at {} - {}",
|
|
purged_in_range,
|
|
region.name(),
|
|
*owner,
|
|
region.vaddr_from_page_index(range.base),
|
|
region.vaddr_from_page_index(range.base + range.count));
|
|
} else {
|
|
dmesgln("Purged {} pages from region {} (no ownership) at {} - {}",
|
|
purged_in_range,
|
|
region.name(),
|
|
region.vaddr_from_page_index(range.base),
|
|
region.vaddr_from_page_index(range.base + range.count));
|
|
}
|
|
region.remap_vmobject_page_range(range.base, range.count);
|
|
});
|
|
}
|
|
});
|
|
return purged_page_count;
|
|
}
|
|
|
|
void AnonymousVMObject::set_was_purged(VolatilePageRange const& range)
|
|
{
|
|
VERIFY(m_lock.is_locked());
|
|
for (auto* purgeable_ranges : m_purgeable_ranges)
|
|
purgeable_ranges->set_was_purged(range);
|
|
}
|
|
|
|
void AnonymousVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
|
|
{
|
|
ScopedSpinLock lock(m_lock);
|
|
purgeable_page_ranges.set_vmobject(this);
|
|
VERIFY(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges));
|
|
m_purgeable_ranges.append(&purgeable_page_ranges);
|
|
}
|
|
|
|
void AnonymousVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
|
|
{
|
|
ScopedSpinLock lock(m_lock);
|
|
for (size_t i = 0; i < m_purgeable_ranges.size(); i++) {
|
|
if (m_purgeable_ranges[i] != &purgeable_page_ranges)
|
|
continue;
|
|
purgeable_page_ranges.set_vmobject(nullptr);
|
|
m_purgeable_ranges.remove(i);
|
|
return;
|
|
}
|
|
VERIFY_NOT_REACHED();
|
|
}
|
|
|
|
bool AnonymousVMObject::is_any_volatile() const
|
|
{
|
|
ScopedSpinLock lock(m_lock);
|
|
for (auto& volatile_ranges : m_purgeable_ranges) {
|
|
ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock);
|
|
if (!volatile_ranges->is_empty())
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
size_t AnonymousVMObject::remove_lazy_commit_pages(VolatilePageRange const& range)
|
|
{
|
|
VERIFY(m_lock.is_locked());
|
|
|
|
size_t removed_count = 0;
|
|
auto range_end = range.base + range.count;
|
|
for (size_t i = range.base; i < range_end; i++) {
|
|
auto& phys_page = m_physical_pages[i];
|
|
if (phys_page && phys_page->is_lazy_committed_page()) {
|
|
phys_page = MM.shared_zero_page();
|
|
removed_count++;
|
|
VERIFY(m_unused_committed_pages > 0);
|
|
if (--m_unused_committed_pages == 0)
|
|
break;
|
|
}
|
|
}
|
|
return removed_count;
|
|
}
|
|
|
|
void AnonymousVMObject::update_volatile_cache()
|
|
{
|
|
VERIFY(m_lock.is_locked());
|
|
VERIFY(m_volatile_ranges_cache_dirty);
|
|
|
|
m_volatile_ranges_cache.clear();
|
|
for_each_nonvolatile_range([&](VolatilePageRange const& range) {
|
|
m_volatile_ranges_cache.add_unchecked(range);
|
|
});
|
|
|
|
m_volatile_ranges_cache_dirty = false;
|
|
}
|
|
|
|
void AnonymousVMObject::range_made_volatile(VolatilePageRange const& range)
|
|
{
|
|
VERIFY(m_lock.is_locked());
|
|
|
|
if (m_unused_committed_pages == 0)
|
|
return;
|
|
|
|
// We need to check this range for any pages that are marked for
|
|
// lazy committed allocation and turn them into shared zero pages
|
|
// and also adjust the m_unused_committed_pages for each such page.
|
|
// Take into account all the other views as well.
|
|
size_t uncommit_page_count = 0;
|
|
for_each_volatile_range([&](auto const& r) {
|
|
auto intersected = range.intersected(r);
|
|
if (!intersected.is_empty()) {
|
|
uncommit_page_count += remove_lazy_commit_pages(intersected);
|
|
if (m_unused_committed_pages == 0)
|
|
return IterationDecision::Break;
|
|
}
|
|
return IterationDecision::Continue;
|
|
});
|
|
|
|
// Return those committed pages back to the system
|
|
if (uncommit_page_count > 0) {
|
|
dbgln_if(COMMIT_DEBUG, "Uncommit {} lazy-commit pages from {:p}", uncommit_page_count, this);
|
|
MM.uncommit_user_physical_pages(uncommit_page_count);
|
|
}
|
|
|
|
m_volatile_ranges_cache_dirty = true;
|
|
}
|
|
|
|
void AnonymousVMObject::range_made_nonvolatile(VolatilePageRange const&)
|
|
{
|
|
VERIFY(m_lock.is_locked());
|
|
m_volatile_ranges_cache_dirty = true;
|
|
}
|
|
|
|
size_t AnonymousVMObject::count_needed_commit_pages_for_nonvolatile_range(VolatilePageRange const& range)
|
|
{
|
|
VERIFY(m_lock.is_locked());
|
|
VERIFY(!range.is_empty());
|
|
|
|
size_t need_commit_pages = 0;
|
|
auto range_end = range.base + range.count;
|
|
for (size_t page_index = range.base; page_index < range_end; page_index++) {
|
|
// COW pages are accounted for in m_shared_committed_cow_pages
|
|
if (!m_cow_map.is_null() && m_cow_map.get(page_index))
|
|
continue;
|
|
auto& phys_page = m_physical_pages[page_index];
|
|
if (phys_page && phys_page->is_shared_zero_page())
|
|
need_commit_pages++;
|
|
}
|
|
return need_commit_pages;
|
|
}
|
|
|
|
size_t AnonymousVMObject::mark_committed_pages_for_nonvolatile_range(VolatilePageRange const& range, size_t mark_total)
|
|
{
|
|
VERIFY(m_lock.is_locked());
|
|
VERIFY(!range.is_empty());
|
|
VERIFY(mark_total > 0);
|
|
|
|
size_t pages_updated = 0;
|
|
auto range_end = range.base + range.count;
|
|
for (size_t page_index = range.base; page_index < range_end; page_index++) {
|
|
// COW pages are accounted for in m_shared_committed_cow_pages
|
|
if (!m_cow_map.is_null() && m_cow_map.get(page_index))
|
|
continue;
|
|
auto& phys_page = m_physical_pages[page_index];
|
|
if (phys_page && phys_page->is_shared_zero_page()) {
|
|
phys_page = MM.lazy_committed_page();
|
|
if (++pages_updated == mark_total)
|
|
break;
|
|
}
|
|
}
|
|
|
|
dbgln_if(COMMIT_DEBUG, "Added {} lazy-commit pages to {:p}", pages_updated, this);
|
|
|
|
m_unused_committed_pages += pages_updated;
|
|
return pages_updated;
|
|
}
|
|
|
|
NonnullRefPtr<PhysicalPage> AnonymousVMObject::allocate_committed_page(Badge<Region>, size_t page_index)
|
|
{
|
|
{
|
|
ScopedSpinLock lock(m_lock);
|
|
|
|
VERIFY(m_unused_committed_pages > 0);
|
|
|
|
// We shouldn't have any committed page tags in volatile regions
|
|
VERIFY([&]() {
|
|
for (auto* purgeable_ranges : m_purgeable_ranges) {
|
|
if (purgeable_ranges->is_volatile(page_index))
|
|
return false;
|
|
}
|
|
return true;
|
|
}());
|
|
|
|
m_unused_committed_pages--;
|
|
}
|
|
return MM.allocate_committed_user_physical_page(MemoryManager::ShouldZeroFill::Yes);
|
|
}
|
|
|
|
Bitmap& AnonymousVMObject::ensure_cow_map()
|
|
{
|
|
if (m_cow_map.is_null())
|
|
m_cow_map = Bitmap { page_count(), true };
|
|
return m_cow_map;
|
|
}
|
|
|
|
void AnonymousVMObject::ensure_or_reset_cow_map()
|
|
{
|
|
if (m_cow_map.is_null())
|
|
ensure_cow_map();
|
|
else
|
|
m_cow_map.fill(true);
|
|
}
|
|
|
|
bool AnonymousVMObject::should_cow(size_t page_index, bool is_shared) const
|
|
{
|
|
auto& page = physical_pages()[page_index];
|
|
if (page && (page->is_shared_zero_page() || page->is_lazy_committed_page()))
|
|
return true;
|
|
if (is_shared)
|
|
return false;
|
|
return !m_cow_map.is_null() && m_cow_map.get(page_index);
|
|
}
|
|
|
|
void AnonymousVMObject::set_should_cow(size_t page_index, bool cow)
|
|
{
|
|
ensure_cow_map().set(page_index, cow);
|
|
}
|
|
|
|
size_t AnonymousVMObject::cow_pages() const
|
|
{
|
|
if (m_cow_map.is_null())
|
|
return 0;
|
|
return m_cow_map.count_slow(true);
|
|
}
|
|
|
|
bool AnonymousVMObject::is_nonvolatile(size_t page_index)
|
|
{
|
|
if (m_volatile_ranges_cache_dirty)
|
|
update_volatile_cache();
|
|
return !m_volatile_ranges_cache.contains(page_index);
|
|
}
|
|
|
|
PageFaultResponse AnonymousVMObject::handle_cow_fault(size_t page_index, VirtualAddress vaddr)
|
|
{
|
|
VERIFY_INTERRUPTS_DISABLED();
|
|
ScopedSpinLock lock(m_lock);
|
|
auto& page_slot = physical_pages()[page_index];
|
|
bool have_committed = m_shared_committed_cow_pages && is_nonvolatile(page_index);
|
|
if (page_slot->ref_count() == 1) {
|
|
dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page but nobody is sharing it anymore. Remap r/w");
|
|
set_should_cow(page_index, false);
|
|
if (have_committed) {
|
|
if (m_shared_committed_cow_pages->return_one())
|
|
m_shared_committed_cow_pages = nullptr;
|
|
}
|
|
return PageFaultResponse::Continue;
|
|
}
|
|
|
|
RefPtr<PhysicalPage> page;
|
|
if (have_committed) {
|
|
dbgln_if(PAGE_FAULT_DEBUG, " >> It's a committed COW page and it's time to COW!");
|
|
page = m_shared_committed_cow_pages->allocate_one();
|
|
} else {
|
|
dbgln_if(PAGE_FAULT_DEBUG, " >> It's a COW page and it's time to COW!");
|
|
page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::No);
|
|
if (page.is_null()) {
|
|
dmesgln("MM: handle_cow_fault was unable to allocate a physical page");
|
|
return PageFaultResponse::OutOfMemory;
|
|
}
|
|
}
|
|
|
|
u8* dest_ptr = MM.quickmap_page(*page);
|
|
dbgln_if(PAGE_FAULT_DEBUG, " >> COW {} <- {}", page->paddr(), page_slot->paddr());
|
|
{
|
|
SmapDisabler disabler;
|
|
void* fault_at;
|
|
if (!safe_memcpy(dest_ptr, vaddr.as_ptr(), PAGE_SIZE, fault_at)) {
|
|
if ((u8*)fault_at >= dest_ptr && (u8*)fault_at <= dest_ptr + PAGE_SIZE)
|
|
dbgln(" >> COW: error copying page {}/{} to {}/{}: failed to write to page at {}",
|
|
page_slot->paddr(), vaddr, page->paddr(), VirtualAddress(dest_ptr), VirtualAddress(fault_at));
|
|
else if ((u8*)fault_at >= vaddr.as_ptr() && (u8*)fault_at <= vaddr.as_ptr() + PAGE_SIZE)
|
|
dbgln(" >> COW: error copying page {}/{} to {}/{}: failed to read from page at {}",
|
|
page_slot->paddr(), vaddr, page->paddr(), VirtualAddress(dest_ptr), VirtualAddress(fault_at));
|
|
else
|
|
VERIFY_NOT_REACHED();
|
|
}
|
|
}
|
|
page_slot = move(page);
|
|
MM.unquickmap_page();
|
|
set_should_cow(page_index, false);
|
|
return PageFaultResponse::Continue;
|
|
}
|
|
|
|
}
|