From b9738fa8ac001945d440d2c078d8a3ec2b2a893b Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Wed, 3 Apr 2019 15:13:07 +0200 Subject: [PATCH] Kernel: Move VM-related files into Kernel/VM/. Also break MemoryManager.{cpp,h} into one file per class. --- Kernel/Devices/BXVGADevice.cpp | 2 +- Kernel/FileDescriptor.cpp | 2 +- Kernel/FileSystem/FileSystem.cpp | 2 +- Kernel/FileSystem/ProcFS.cpp | 2 +- Kernel/Makefile | 6 +- Kernel/Net/E1000NetworkAdapter.h | 2 +- Kernel/Process.cpp | 2 +- Kernel/Thread.cpp | 2 +- Kernel/VM/.gitignore | 2 + Kernel/{ => VM}/MemoryManager.cpp | 370 +----------------------------- Kernel/{ => VM}/MemoryManager.h | 200 +--------------- Kernel/VM/PageDirectory.cpp | 32 +++ Kernel/VM/PageDirectory.h | 26 +++ Kernel/VM/PhysicalPage.cpp | 42 ++++ Kernel/VM/PhysicalPage.h | 46 ++++ Kernel/VM/Region.cpp | 142 ++++++++++++ Kernel/VM/Region.h | 97 ++++++++ Kernel/VM/VMObject.cpp | 167 ++++++++++++++ Kernel/VM/VMObject.h | 57 +++++ Kernel/i386.cpp | 2 +- Kernel/init.cpp | 2 +- 21 files changed, 630 insertions(+), 575 deletions(-) create mode 100644 Kernel/VM/.gitignore rename Kernel/{ => VM}/MemoryManager.cpp (68%) rename Kernel/{ => VM}/MemoryManager.h (53%) create mode 100644 Kernel/VM/PageDirectory.cpp create mode 100644 Kernel/VM/PageDirectory.h create mode 100644 Kernel/VM/PhysicalPage.cpp create mode 100644 Kernel/VM/PhysicalPage.h create mode 100644 Kernel/VM/Region.cpp create mode 100644 Kernel/VM/Region.h create mode 100644 Kernel/VM/VMObject.cpp create mode 100644 Kernel/VM/VMObject.h diff --git a/Kernel/Devices/BXVGADevice.cpp b/Kernel/Devices/BXVGADevice.cpp index 8ff72b7fee..b014a22304 100644 --- a/Kernel/Devices/BXVGADevice.cpp +++ b/Kernel/Devices/BXVGADevice.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include diff --git a/Kernel/FileDescriptor.cpp b/Kernel/FileDescriptor.cpp index 78e23f2602..86c60d447d 100644 --- a/Kernel/FileDescriptor.cpp +++ b/Kernel/FileDescriptor.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include Retained FileDescriptor::create(RetainPtr&& inode) { diff --git a/Kernel/FileSystem/FileSystem.cpp b/Kernel/FileSystem/FileSystem.cpp index 0b237b3129..da82a87234 100644 --- a/Kernel/FileSystem/FileSystem.cpp +++ b/Kernel/FileSystem/FileSystem.cpp @@ -3,7 +3,7 @@ #include #include #include "FileSystem.h" -#include "MemoryManager.h" +#include #include static dword s_lastFileSystemID; diff --git a/Kernel/FileSystem/ProcFS.cpp b/Kernel/FileSystem/ProcFS.cpp index d08e896f6d..8bedb9f40f 100644 --- a/Kernel/FileSystem/ProcFS.cpp +++ b/Kernel/FileSystem/ProcFS.cpp @@ -2,7 +2,7 @@ #include "Process.h" #include #include "system.h" -#include "MemoryManager.h" +#include #include "StdLib.h" #include "i386.h" #include "KSyms.h" diff --git a/Kernel/Makefile b/Kernel/Makefile index d13b8ad291..aaab46fa2e 100644 --- a/Kernel/Makefile +++ b/Kernel/Makefile @@ -11,7 +11,11 @@ KERNEL_OBJS = \ PIC.o \ Syscall.o \ Devices/IDEDiskDevice.o \ - MemoryManager.o \ + VM/MemoryManager.o \ + VM/Region.o \ + VM/VMObject.o \ + VM/PageDirectory.o \ + VM/PhysicalPage.o \ Console.o \ IRQHandler.o \ kprintf.o \ diff --git a/Kernel/Net/E1000NetworkAdapter.h b/Kernel/Net/E1000NetworkAdapter.h index c0afa4cc26..cf9bdb3ba5 100644 --- a/Kernel/Net/E1000NetworkAdapter.h +++ b/Kernel/Net/E1000NetworkAdapter.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include #include diff --git a/Kernel/Process.cpp b/Kernel/Process.cpp index 4c097db3d4..4a00c7cf4d 100644 --- a/Kernel/Process.cpp +++ b/Kernel/Process.cpp @@ -8,7 +8,7 @@ #include #include #include -#include "MemoryManager.h" +#include #include "i8253.h" #include "RTC.h" #include diff --git a/Kernel/Thread.cpp b/Kernel/Thread.cpp index 3703725fc1..48823f94cb 100644 --- a/Kernel/Thread.cpp +++ b/Kernel/Thread.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include InlineLinkedList* g_threads; diff --git a/Kernel/VM/.gitignore b/Kernel/VM/.gitignore new file mode 100644 index 0000000000..6142305dc1 --- /dev/null +++ b/Kernel/VM/.gitignore @@ -0,0 +1,2 @@ +*.o +*.d diff --git a/Kernel/MemoryManager.cpp b/Kernel/VM/MemoryManager.cpp similarity index 68% rename from Kernel/MemoryManager.cpp rename to Kernel/VM/MemoryManager.cpp index 03b95f07c3..06e9ef65ef 100644 --- a/Kernel/MemoryManager.cpp +++ b/Kernel/VM/MemoryManager.cpp @@ -1,10 +1,9 @@ -#include "MemoryManager.h" +#include #include #include #include "i386.h" #include "StdLib.h" #include "Process.h" -#include #include "CMOS.h" //#define MM_DEBUG @@ -43,16 +42,6 @@ MemoryManager::~MemoryManager() { } -PageDirectory::PageDirectory(PhysicalAddress paddr) -{ - m_directory_page = PhysicalPage::create_eternal(paddr, true); -} - -PageDirectory::PageDirectory() -{ - MM.populate_page_directory(*this); -} - void MemoryManager::populate_page_directory(PageDirectory& page_directory) { page_directory.m_directory_page = allocate_supervisor_physical_page(); @@ -296,25 +285,6 @@ bool MemoryManager::copy_on_write(Region& region, unsigned page_index_in_region) return true; } -bool Region::page_in() -{ - ASSERT(m_page_directory); - ASSERT(!vmo().is_anonymous()); - ASSERT(vmo().inode()); -#ifdef MM_DEBUG - dbgprintf("MM: page_in %u pages\n", page_count()); -#endif - for (size_t i = 0; i < page_count(); ++i) { - auto& vmo_page = vmo().physical_pages()[first_page_index() + i]; - if (vmo_page.is_null()) { - bool success = MM.page_in_from_inode(*this, i); - if (!success) - return false; - } - MM.remap_region_page(*this, i, true); - } - return true; -} bool MemoryManager::page_in_from_inode(Region& region, unsigned page_index_in_region) { @@ -625,305 +595,6 @@ bool MemoryManager::validate_user_write(const Process& process, LinearAddress la return region && region->is_writable(); } -Retained Region::clone() -{ - ASSERT(current); - if (m_shared || (m_readable && !m_writable)) { -#ifdef MM_DEBUG - dbgprintf("%s<%u> Region::clone(): sharing %s (L%x)\n", - current->process().name().characters(), - current->pid(), - m_name.characters(), - laddr().get()); -#endif - // Create a new region backed by the same VMObject. - return adopt(*new Region(laddr(), size(), m_vmo.copy_ref(), m_offset_in_vmo, String(m_name), m_readable, m_writable)); - } - -#ifdef MM_DEBUG - dbgprintf("%s<%u> Region::clone(): cowing %s (L%x)\n", - current->process().name().characters(), - current->pid(), - m_name.characters(), - laddr().get()); -#endif - // Set up a COW region. The parent (this) region becomes COW as well! - for (size_t i = 0; i < page_count(); ++i) - m_cow_map.set(i, true); - MM.remap_region(current->process().page_directory(), *this); - return adopt(*new Region(laddr(), size(), m_vmo->clone(), m_offset_in_vmo, String(m_name), m_readable, m_writable, true)); -} - -Region::Region(LinearAddress a, size_t s, String&& n, bool r, bool w, bool cow) - : m_laddr(a) - , m_size(s) - , m_vmo(VMObject::create_anonymous(s)) - , m_name(move(n)) - , m_readable(r) - , m_writable(w) - , m_cow_map(Bitmap::create(m_vmo->page_count(), cow)) -{ - m_vmo->set_name(m_name); - MM.register_region(*this); -} - -Region::Region(LinearAddress a, size_t s, RetainPtr&& inode, String&& n, bool r, bool w) - : m_laddr(a) - , m_size(s) - , m_vmo(VMObject::create_file_backed(move(inode))) - , m_name(move(n)) - , m_readable(r) - , m_writable(w) - , m_cow_map(Bitmap::create(m_vmo->page_count())) -{ - MM.register_region(*this); -} - -Region::Region(LinearAddress a, size_t s, Retained&& vmo, size_t offset_in_vmo, String&& n, bool r, bool w, bool cow) - : m_laddr(a) - , m_size(s) - , m_offset_in_vmo(offset_in_vmo) - , m_vmo(move(vmo)) - , m_name(move(n)) - , m_readable(r) - , m_writable(w) - , m_cow_map(Bitmap::create(m_vmo->page_count(), cow)) -{ - MM.register_region(*this); -} - -Region::~Region() -{ - if (m_page_directory) { - MM.unmap_region(*this); - ASSERT(!m_page_directory); - } - MM.unregister_region(*this); -} - -Retained PhysicalPage::create_eternal(PhysicalAddress paddr, bool supervisor) -{ - void* slot = kmalloc_eternal(sizeof(PhysicalPage)); - new (slot) PhysicalPage(paddr, supervisor); - return adopt(*(PhysicalPage*)slot); -} - -Retained PhysicalPage::create(PhysicalAddress paddr, bool supervisor) -{ - void* slot = kmalloc(sizeof(PhysicalPage)); - new (slot) PhysicalPage(paddr, supervisor, false); - return adopt(*(PhysicalPage*)slot); -} - -PhysicalPage::PhysicalPage(PhysicalAddress paddr, bool supervisor, bool may_return_to_freelist) - : m_may_return_to_freelist(may_return_to_freelist) - , m_supervisor(supervisor) - , m_paddr(paddr) -{ - if (supervisor) - ++MemoryManager::s_super_physical_pages_in_existence; - else - ++MemoryManager::s_user_physical_pages_in_existence; -} - -void PhysicalPage::return_to_freelist() -{ - ASSERT((paddr().get() & ~PAGE_MASK) == 0); - InterruptDisabler disabler; - m_retain_count = 1; - if (m_supervisor) - MM.m_free_supervisor_physical_pages.append(adopt(*this)); - else - MM.m_free_physical_pages.append(adopt(*this)); -#ifdef MM_DEBUG - dbgprintf("MM: P%x released to freelist\n", m_paddr.get()); -#endif -} - -Retained VMObject::create_file_backed(RetainPtr&& inode) -{ - InterruptDisabler disabler; - if (inode->vmo()) - return *inode->vmo(); - auto vmo = adopt(*new VMObject(move(inode))); - vmo->inode()->set_vmo(*vmo); - return vmo; -} - -Retained VMObject::create_anonymous(size_t size) -{ - size = ceil_div(size, PAGE_SIZE) * PAGE_SIZE; - return adopt(*new VMObject(size)); -} - -Retained VMObject::create_for_physical_range(PhysicalAddress paddr, size_t size) -{ - size = ceil_div(size, PAGE_SIZE) * PAGE_SIZE; - auto vmo = adopt(*new VMObject(paddr, size)); - vmo->m_allow_cpu_caching = false; - return vmo; -} - -Retained VMObject::clone() -{ - return adopt(*new VMObject(*this)); -} - -VMObject::VMObject(VMObject& other) - : m_name(other.m_name) - , m_anonymous(other.m_anonymous) - , m_inode_offset(other.m_inode_offset) - , m_size(other.m_size) - , m_inode(other.m_inode) - , m_physical_pages(other.m_physical_pages) -{ - MM.register_vmo(*this); -} - -VMObject::VMObject(size_t size) - : m_anonymous(true) - , m_size(size) -{ - MM.register_vmo(*this); - m_physical_pages.resize(page_count()); -} - -VMObject::VMObject(PhysicalAddress paddr, size_t size) - : m_anonymous(true) - , m_size(size) -{ - MM.register_vmo(*this); - for (size_t i = 0; i < size; i += PAGE_SIZE) { - m_physical_pages.append(PhysicalPage::create(paddr.offset(i), false)); - } - ASSERT(m_physical_pages.size() == page_count()); -} - - -VMObject::VMObject(RetainPtr&& inode) - : m_inode(move(inode)) -{ - ASSERT(m_inode); - m_size = ceil_div(m_inode->size(), PAGE_SIZE) * PAGE_SIZE; - m_physical_pages.resize(page_count()); - MM.register_vmo(*this); -} - -VMObject::~VMObject() -{ - if (m_inode) - ASSERT(m_inode->vmo() == this); - MM.unregister_vmo(*this); -} - -template -void VMObject::for_each_region(Callback callback) -{ - // FIXME: Figure out a better data structure so we don't have to walk every single region every time an inode changes. - // Perhaps VMObject could have a Vector with all of his mappers? - for (auto* region : MM.m_regions) { - if (®ion->vmo() == this) - callback(*region); - } -} - -void VMObject::inode_size_changed(Badge, size_t old_size, size_t new_size) -{ - (void)old_size; - InterruptDisabler disabler; - - size_t old_page_count = page_count(); - m_size = new_size; - - if (page_count() > old_page_count) { - // Add null pages and let the fault handler page these in when that day comes. - for (size_t i = old_page_count; i < page_count(); ++i) - m_physical_pages.append(nullptr); - } else { - // Prune the no-longer valid pages. I'm not sure this is actually correct behavior. - for (size_t i = page_count(); i < old_page_count; ++i) - m_physical_pages.take_last(); - } - - // FIXME: Consolidate with inode_contents_changed() so we only do a single walk. - for_each_region([] (Region& region) { - ASSERT(region.page_directory()); - MM.remap_region(*region.page_directory(), region); - }); -} - -void VMObject::inode_contents_changed(Badge, off_t offset, ssize_t size, const byte* data) -{ - (void)size; - (void)data; - InterruptDisabler disabler; - ASSERT(offset >= 0); - - // FIXME: Only invalidate the parts that actually changed. - for (auto& physical_page : m_physical_pages) - physical_page = nullptr; - -#if 0 - size_t current_offset = offset; - size_t remaining_bytes = size; - const byte* data_ptr = data; - - auto to_page_index = [] (size_t offset) -> size_t { - return offset / PAGE_SIZE; - }; - - if (current_offset & PAGE_MASK) { - size_t page_index = to_page_index(current_offset); - size_t bytes_to_copy = min(size, PAGE_SIZE - (current_offset & PAGE_MASK)); - if (m_physical_pages[page_index]) { - auto* ptr = MM.quickmap_page(*m_physical_pages[page_index]); - memcpy(ptr, data_ptr, bytes_to_copy); - MM.unquickmap_page(); - } - current_offset += bytes_to_copy; - data += bytes_to_copy; - remaining_bytes -= bytes_to_copy; - } - - for (size_t page_index = to_page_index(current_offset); page_index < m_physical_pages.size(); ++page_index) { - size_t bytes_to_copy = PAGE_SIZE - (current_offset & PAGE_MASK); - if (m_physical_pages[page_index]) { - auto* ptr = MM.quickmap_page(*m_physical_pages[page_index]); - memcpy(ptr, data_ptr, bytes_to_copy); - MM.unquickmap_page(); - } - current_offset += bytes_to_copy; - data += bytes_to_copy; - } -#endif - - // FIXME: Consolidate with inode_size_changed() so we only do a single walk. - for_each_region([] (Region& region) { - ASSERT(region.page_directory()); - MM.remap_region(*region.page_directory(), region); - }); -} - -int Region::commit() -{ - InterruptDisabler disabler; -#ifdef MM_DEBUG - dbgprintf("MM: commit %u pages in Region %p (VMO=%p) at L%x\n", vmo().page_count(), this, &vmo(), laddr().get()); -#endif - for (size_t i = first_page_index(); i <= last_page_index(); ++i) { - if (!vmo().physical_pages()[i].is_null()) - continue; - auto physical_page = MM.allocate_physical_page(MemoryManager::ShouldZeroFill::Yes); - if (!physical_page) { - kprintf("MM: commit was unable to allocate a physical page\n"); - return -ENOMEM; - } - vmo().physical_pages()[i] = move(physical_page); - MM.remap_region_page(*this, i, true); - } - return 0; -} - void MemoryManager::register_vmo(VMObject& vmo) { InterruptDisabler disabler; @@ -948,45 +619,6 @@ void MemoryManager::unregister_region(Region& region) m_regions.remove(®ion); } -size_t Region::amount_resident() const -{ - size_t bytes = 0; - for (size_t i = 0; i < page_count(); ++i) { - if (m_vmo->physical_pages()[first_page_index() + i]) - bytes += PAGE_SIZE; - } - return bytes; -} - -size_t Region::amount_shared() const -{ - size_t bytes = 0; - for (size_t i = 0; i < page_count(); ++i) { - auto& physical_page = m_vmo->physical_pages()[first_page_index() + i]; - if (physical_page && physical_page->retain_count() > 1) - bytes += PAGE_SIZE; - } - return bytes; -} - -PageDirectory::~PageDirectory() -{ -#ifdef MM_DEBUG - dbgprintf("MM: ~PageDirectory K%x\n", this); -#endif -} - -void PageDirectory::flush(LinearAddress laddr) -{ -#ifdef MM_DEBUG - dbgprintf("MM: Flush page L%x\n", laddr.get()); -#endif - if (!current) - return; - if (¤t->process().page_directory() == this) - MM.flush_tlb(laddr); -} - ProcessPagingScope::ProcessPagingScope(Process& process) { ASSERT(current); diff --git a/Kernel/MemoryManager.h b/Kernel/VM/MemoryManager.h similarity index 53% rename from Kernel/MemoryManager.h rename to Kernel/VM/MemoryManager.h index bc1340695b..b33ae2605f 100644 --- a/Kernel/MemoryManager.h +++ b/Kernel/VM/MemoryManager.h @@ -11,7 +11,10 @@ #include #include #include -#include +#include +#include +#include +#include #define PAGE_ROUND_UP(x) ((((dword)(x)) + PAGE_SIZE-1) & (~(PAGE_SIZE-1))) @@ -22,201 +25,6 @@ enum class PageFaultResponse { Continue, }; -class PhysicalPage { - friend class MemoryManager; - friend class PageDirectory; - friend class VMObject; -public: - PhysicalAddress paddr() const { return m_paddr; } - - void retain() - { - ASSERT(m_retain_count); - ++m_retain_count; - } - - void release() - { - ASSERT(m_retain_count); - if (!--m_retain_count) { - if (m_may_return_to_freelist) - return_to_freelist(); - else - delete this; - } - } - - static Retained create_eternal(PhysicalAddress, bool supervisor); - static Retained create(PhysicalAddress, bool supervisor); - - unsigned short retain_count() const { return m_retain_count; } - -private: - PhysicalPage(PhysicalAddress paddr, bool supervisor, bool may_return_to_freelist = true); - ~PhysicalPage() { } - - void return_to_freelist(); - - unsigned short m_retain_count { 1 }; - bool m_may_return_to_freelist { true }; - bool m_supervisor { false }; - PhysicalAddress m_paddr; -}; - -class PageDirectory : public Retainable { - friend class MemoryManager; -public: - static Retained create() { return adopt(*new PageDirectory); } - static Retained create_at_fixed_address(PhysicalAddress paddr) { return adopt(*new PageDirectory(paddr)); } - ~PageDirectory(); - - dword cr3() const { return m_directory_page->paddr().get(); } - dword* entries() { return reinterpret_cast(cr3()); } - - void flush(LinearAddress); - -private: - PageDirectory(); - explicit PageDirectory(PhysicalAddress); - - RetainPtr m_directory_page; - HashMap> m_physical_pages; -}; - -class VMObject : public Retainable, public Weakable { - friend class MemoryManager; -public: - static Retained create_file_backed(RetainPtr&&); - static Retained create_anonymous(size_t); - static Retained create_for_physical_range(PhysicalAddress, size_t); - Retained clone(); - - ~VMObject(); - bool is_anonymous() const { return m_anonymous; } - - Inode* inode() { return m_inode.ptr(); } - const Inode* inode() const { return m_inode.ptr(); } - size_t inode_offset() const { return m_inode_offset; } - - String name() const { return m_name; } - void set_name(const String& name) { m_name = name; } - - size_t page_count() const { return m_size / PAGE_SIZE; } - const Vector>& physical_pages() const { return m_physical_pages; } - Vector>& physical_pages() { return m_physical_pages; } - - void inode_contents_changed(Badge, off_t, ssize_t, const byte*); - void inode_size_changed(Badge, size_t old_size, size_t new_size); - - size_t size() const { return m_size; } - -private: - VMObject(RetainPtr&&); - explicit VMObject(VMObject&); - explicit VMObject(size_t); - VMObject(PhysicalAddress, size_t); - - template void for_each_region(Callback); - - String m_name; - bool m_anonymous { false }; - off_t m_inode_offset { 0 }; - size_t m_size { 0 }; - bool m_allow_cpu_caching { true }; - RetainPtr m_inode; - Vector> m_physical_pages; - Lock m_paging_lock; -}; - -class Region : public Retainable { - friend class MemoryManager; -public: - Region(LinearAddress, size_t, String&&, bool r, bool w, bool cow = false); - Region(LinearAddress, size_t, Retained&&, size_t offset_in_vmo, String&&, bool r, bool w, bool cow = false); - Region(LinearAddress, size_t, RetainPtr&&, String&&, bool r, bool w); - ~Region(); - - LinearAddress laddr() const { return m_laddr; } - size_t size() const { return m_size; } - bool is_readable() const { return m_readable; } - bool is_writable() const { return m_writable; } - String name() const { return m_name; } - - void set_name(String&& name) { m_name = move(name); } - - const VMObject& vmo() const { return *m_vmo; } - VMObject& vmo() { return *m_vmo; } - - bool is_shared() const { return m_shared; } - void set_shared(bool shared) { m_shared = shared; } - - bool is_bitmap() const { return m_is_bitmap; } - void set_is_bitmap(bool b) { m_is_bitmap = b; } - - Retained clone(); - bool contains(LinearAddress laddr) const - { - return laddr >= m_laddr && laddr < m_laddr.offset(size()); - } - - unsigned page_index_from_address(LinearAddress laddr) const - { - return (laddr - m_laddr).get() / PAGE_SIZE; - } - - size_t first_page_index() const - { - return m_offset_in_vmo / PAGE_SIZE; - } - - size_t last_page_index() const - { - return (first_page_index() + page_count()) - 1; - } - - size_t page_count() const - { - return m_size / PAGE_SIZE; - } - - bool page_in(); - int commit(); - - size_t amount_resident() const; - size_t amount_shared() const; - - PageDirectory* page_directory() { return m_page_directory.ptr(); } - - void set_page_directory(PageDirectory& page_directory) - { - ASSERT(!m_page_directory || m_page_directory.ptr() == &page_directory); - m_page_directory = page_directory; - } - - void release_page_directory() - { - ASSERT(m_page_directory); - m_page_directory.clear(); - } - - const Bitmap& cow_map() const { return m_cow_map; } - - void set_writable(bool b) { m_writable = b; } - -private: - RetainPtr m_page_directory; - LinearAddress m_laddr; - size_t m_size { 0 }; - size_t m_offset_in_vmo { 0 }; - Retained m_vmo; - String m_name; - bool m_readable { true }; - bool m_writable { true }; - bool m_shared { false }; - bool m_is_bitmap { false }; - Bitmap m_cow_map; -}; - #define MM MemoryManager::the() class MemoryManager { diff --git a/Kernel/VM/PageDirectory.cpp b/Kernel/VM/PageDirectory.cpp new file mode 100644 index 0000000000..d5c10c189a --- /dev/null +++ b/Kernel/VM/PageDirectory.cpp @@ -0,0 +1,32 @@ +#include +#include +#include +#include + +PageDirectory::PageDirectory(PhysicalAddress paddr) +{ + m_directory_page = PhysicalPage::create_eternal(paddr, true); +} + +PageDirectory::PageDirectory() +{ + MM.populate_page_directory(*this); +} + +PageDirectory::~PageDirectory() +{ +#ifdef MM_DEBUG + dbgprintf("MM: ~PageDirectory K%x\n", this); +#endif +} + +void PageDirectory::flush(LinearAddress laddr) +{ +#ifdef MM_DEBUG + dbgprintf("MM: Flush page L%x\n", laddr.get()); +#endif + if (!current) + return; + if (¤t->process().page_directory() == this) + MM.flush_tlb(laddr); +} diff --git a/Kernel/VM/PageDirectory.h b/Kernel/VM/PageDirectory.h new file mode 100644 index 0000000000..ba655178b2 --- /dev/null +++ b/Kernel/VM/PageDirectory.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include +#include + +class PageDirectory : public Retainable { + friend class MemoryManager; +public: + static Retained create() { return adopt(*new PageDirectory); } + static Retained create_at_fixed_address(PhysicalAddress paddr) { return adopt(*new PageDirectory(paddr)); } + ~PageDirectory(); + + dword cr3() const { return m_directory_page->paddr().get(); } + dword* entries() { return reinterpret_cast(cr3()); } + + void flush(LinearAddress); + +private: + PageDirectory(); + explicit PageDirectory(PhysicalAddress); + + RetainPtr m_directory_page; + HashMap> m_physical_pages; +}; diff --git a/Kernel/VM/PhysicalPage.cpp b/Kernel/VM/PhysicalPage.cpp new file mode 100644 index 0000000000..459fde5c60 --- /dev/null +++ b/Kernel/VM/PhysicalPage.cpp @@ -0,0 +1,42 @@ +#include +#include +#include + +Retained PhysicalPage::create_eternal(PhysicalAddress paddr, bool supervisor) +{ + void* slot = kmalloc_eternal(sizeof(PhysicalPage)); + new (slot) PhysicalPage(paddr, supervisor); + return adopt(*(PhysicalPage*)slot); +} + +Retained PhysicalPage::create(PhysicalAddress paddr, bool supervisor) +{ + void* slot = kmalloc(sizeof(PhysicalPage)); + new (slot) PhysicalPage(paddr, supervisor, false); + return adopt(*(PhysicalPage*)slot); +} + +PhysicalPage::PhysicalPage(PhysicalAddress paddr, bool supervisor, bool may_return_to_freelist) + : m_may_return_to_freelist(may_return_to_freelist) + , m_supervisor(supervisor) + , m_paddr(paddr) +{ + if (supervisor) + ++MemoryManager::s_super_physical_pages_in_existence; + else + ++MemoryManager::s_user_physical_pages_in_existence; +} + +void PhysicalPage::return_to_freelist() +{ + ASSERT((paddr().get() & ~PAGE_MASK) == 0); + InterruptDisabler disabler; + m_retain_count = 1; + if (m_supervisor) + MM.m_free_supervisor_physical_pages.append(adopt(*this)); + else + MM.m_free_physical_pages.append(adopt(*this)); +#ifdef MM_DEBUG + dbgprintf("MM: P%x released to freelist\n", m_paddr.get()); +#endif +} diff --git a/Kernel/VM/PhysicalPage.h b/Kernel/VM/PhysicalPage.h new file mode 100644 index 0000000000..f3b15378dc --- /dev/null +++ b/Kernel/VM/PhysicalPage.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include + +class PhysicalPage { + friend class MemoryManager; + friend class PageDirectory; + friend class VMObject; +public: + PhysicalAddress paddr() const { return m_paddr; } + + void retain() + { + ASSERT(m_retain_count); + ++m_retain_count; + } + + void release() + { + ASSERT(m_retain_count); + if (!--m_retain_count) { + if (m_may_return_to_freelist) + return_to_freelist(); + else + delete this; + } + } + + static Retained create_eternal(PhysicalAddress, bool supervisor); + static Retained create(PhysicalAddress, bool supervisor); + + word retain_count() const { return m_retain_count; } + +private: + PhysicalPage(PhysicalAddress paddr, bool supervisor, bool may_return_to_freelist = true); + ~PhysicalPage() { } + + void return_to_freelist(); + + word m_retain_count { 1 }; + bool m_may_return_to_freelist { true }; + bool m_supervisor { false }; + PhysicalAddress m_paddr; +}; diff --git a/Kernel/VM/Region.cpp b/Kernel/VM/Region.cpp new file mode 100644 index 0000000000..1f01b0b33d --- /dev/null +++ b/Kernel/VM/Region.cpp @@ -0,0 +1,142 @@ +#include +#include +#include +#include +#include + +Region::Region(LinearAddress a, size_t s, String&& n, bool r, bool w, bool cow) + : m_laddr(a) + , m_size(s) + , m_vmo(VMObject::create_anonymous(s)) + , m_name(move(n)) + , m_readable(r) + , m_writable(w) + , m_cow_map(Bitmap::create(m_vmo->page_count(), cow)) +{ + m_vmo->set_name(m_name); + MM.register_region(*this); +} + +Region::Region(LinearAddress a, size_t s, RetainPtr&& inode, String&& n, bool r, bool w) + : m_laddr(a) + , m_size(s) + , m_vmo(VMObject::create_file_backed(move(inode))) + , m_name(move(n)) + , m_readable(r) + , m_writable(w) + , m_cow_map(Bitmap::create(m_vmo->page_count())) +{ + MM.register_region(*this); +} + +Region::Region(LinearAddress a, size_t s, Retained&& vmo, size_t offset_in_vmo, String&& n, bool r, bool w, bool cow) + : m_laddr(a) + , m_size(s) + , m_offset_in_vmo(offset_in_vmo) + , m_vmo(move(vmo)) + , m_name(move(n)) + , m_readable(r) + , m_writable(w) + , m_cow_map(Bitmap::create(m_vmo->page_count(), cow)) +{ + MM.register_region(*this); +} + +Region::~Region() +{ + if (m_page_directory) { + MM.unmap_region(*this); + ASSERT(!m_page_directory); + } + MM.unregister_region(*this); +} + +bool Region::page_in() +{ + ASSERT(m_page_directory); + ASSERT(!vmo().is_anonymous()); + ASSERT(vmo().inode()); +#ifdef MM_DEBUG + dbgprintf("MM: page_in %u pages\n", page_count()); +#endif + for (size_t i = 0; i < page_count(); ++i) { + auto& vmo_page = vmo().physical_pages()[first_page_index() + i]; + if (vmo_page.is_null()) { + bool success = MM.page_in_from_inode(*this, i); + if (!success) + return false; + } + MM.remap_region_page(*this, i, true); + } + return true; +} + +Retained Region::clone() +{ + ASSERT(current); + if (m_shared || (m_readable && !m_writable)) { +#ifdef MM_DEBUG + dbgprintf("%s<%u> Region::clone(): sharing %s (L%x)\n", + current->process().name().characters(), + current->pid(), + m_name.characters(), + laddr().get()); +#endif + // Create a new region backed by the same VMObject. + return adopt(*new Region(laddr(), size(), m_vmo.copy_ref(), m_offset_in_vmo, String(m_name), m_readable, m_writable)); + } + +#ifdef MM_DEBUG + dbgprintf("%s<%u> Region::clone(): cowing %s (L%x)\n", + current->process().name().characters(), + current->pid(), + m_name.characters(), + laddr().get()); +#endif + // Set up a COW region. The parent (this) region becomes COW as well! + for (size_t i = 0; i < page_count(); ++i) + m_cow_map.set(i, true); + MM.remap_region(current->process().page_directory(), *this); + return adopt(*new Region(laddr(), size(), m_vmo->clone(), m_offset_in_vmo, String(m_name), m_readable, m_writable, true)); +} + +int Region::commit() +{ + InterruptDisabler disabler; +#ifdef MM_DEBUG + dbgprintf("MM: commit %u pages in Region %p (VMO=%p) at L%x\n", vmo().page_count(), this, &vmo(), laddr().get()); +#endif + for (size_t i = first_page_index(); i <= last_page_index(); ++i) { + if (!vmo().physical_pages()[i].is_null()) + continue; + auto physical_page = MM.allocate_physical_page(MemoryManager::ShouldZeroFill::Yes); + if (!physical_page) { + kprintf("MM: commit was unable to allocate a physical page\n"); + return -ENOMEM; + } + vmo().physical_pages()[i] = move(physical_page); + MM.remap_region_page(*this, i, true); + } + return 0; +} + +size_t Region::amount_resident() const +{ + size_t bytes = 0; + for (size_t i = 0; i < page_count(); ++i) { + if (m_vmo->physical_pages()[first_page_index() + i]) + bytes += PAGE_SIZE; + } + return bytes; +} + +size_t Region::amount_shared() const +{ + size_t bytes = 0; + for (size_t i = 0; i < page_count(); ++i) { + auto& physical_page = m_vmo->physical_pages()[first_page_index() + i]; + if (physical_page && physical_page->retain_count() > 1) + bytes += PAGE_SIZE; + } + return bytes; +} diff --git a/Kernel/VM/Region.h b/Kernel/VM/Region.h new file mode 100644 index 0000000000..1ddc9b50ef --- /dev/null +++ b/Kernel/VM/Region.h @@ -0,0 +1,97 @@ +#pragma once + +#include +#include +#include + +class Inode; +class VMObject; + +class Region : public Retainable { + friend class MemoryManager; +public: + Region(LinearAddress, size_t, String&&, bool r, bool w, bool cow = false); + Region(LinearAddress, size_t, Retained&&, size_t offset_in_vmo, String&&, bool r, bool w, bool cow = false); + Region(LinearAddress, size_t, RetainPtr&&, String&&, bool r, bool w); + ~Region(); + + LinearAddress laddr() const { return m_laddr; } + size_t size() const { return m_size; } + bool is_readable() const { return m_readable; } + bool is_writable() const { return m_writable; } + String name() const { return m_name; } + + void set_name(String&& name) { m_name = move(name); } + + const VMObject& vmo() const { return *m_vmo; } + VMObject& vmo() { return *m_vmo; } + + bool is_shared() const { return m_shared; } + void set_shared(bool shared) { m_shared = shared; } + + bool is_bitmap() const { return m_is_bitmap; } + void set_is_bitmap(bool b) { m_is_bitmap = b; } + + Retained clone(); + bool contains(LinearAddress laddr) const + { + return laddr >= m_laddr && laddr < m_laddr.offset(size()); + } + + unsigned page_index_from_address(LinearAddress laddr) const + { + return (laddr - m_laddr).get() / PAGE_SIZE; + } + + size_t first_page_index() const + { + return m_offset_in_vmo / PAGE_SIZE; + } + + size_t last_page_index() const + { + return (first_page_index() + page_count()) - 1; + } + + size_t page_count() const + { + return m_size / PAGE_SIZE; + } + + bool page_in(); + int commit(); + + size_t amount_resident() const; + size_t amount_shared() const; + + PageDirectory* page_directory() { return m_page_directory.ptr(); } + + void set_page_directory(PageDirectory& page_directory) + { + ASSERT(!m_page_directory || m_page_directory.ptr() == &page_directory); + m_page_directory = page_directory; + } + + void release_page_directory() + { + ASSERT(m_page_directory); + m_page_directory.clear(); + } + + const Bitmap& cow_map() const { return m_cow_map; } + + void set_writable(bool b) { m_writable = b; } + +private: + RetainPtr m_page_directory; + LinearAddress m_laddr; + size_t m_size { 0 }; + size_t m_offset_in_vmo { 0 }; + Retained m_vmo; + String m_name; + bool m_readable { true }; + bool m_writable { true }; + bool m_shared { false }; + bool m_is_bitmap { false }; + Bitmap m_cow_map; +}; diff --git a/Kernel/VM/VMObject.cpp b/Kernel/VM/VMObject.cpp new file mode 100644 index 0000000000..a31edaac7d --- /dev/null +++ b/Kernel/VM/VMObject.cpp @@ -0,0 +1,167 @@ +#include +#include +#include + +Retained VMObject::create_file_backed(RetainPtr&& inode) +{ + InterruptDisabler disabler; + if (inode->vmo()) + return *inode->vmo(); + auto vmo = adopt(*new VMObject(move(inode))); + vmo->inode()->set_vmo(*vmo); + return vmo; +} + +Retained VMObject::create_anonymous(size_t size) +{ + size = ceil_div(size, PAGE_SIZE) * PAGE_SIZE; + return adopt(*new VMObject(size)); +} + +Retained VMObject::create_for_physical_range(PhysicalAddress paddr, size_t size) +{ + size = ceil_div(size, PAGE_SIZE) * PAGE_SIZE; + auto vmo = adopt(*new VMObject(paddr, size)); + vmo->m_allow_cpu_caching = false; + return vmo; +} + +Retained VMObject::clone() +{ + return adopt(*new VMObject(*this)); +} + +VMObject::VMObject(VMObject& other) + : m_name(other.m_name) + , m_anonymous(other.m_anonymous) + , m_inode_offset(other.m_inode_offset) + , m_size(other.m_size) + , m_inode(other.m_inode) + , m_physical_pages(other.m_physical_pages) +{ + MM.register_vmo(*this); +} + +VMObject::VMObject(size_t size) + : m_anonymous(true) + , m_size(size) +{ + MM.register_vmo(*this); + m_physical_pages.resize(page_count()); +} + +VMObject::VMObject(PhysicalAddress paddr, size_t size) + : m_anonymous(true) + , m_size(size) +{ + MM.register_vmo(*this); + for (size_t i = 0; i < size; i += PAGE_SIZE) { + m_physical_pages.append(PhysicalPage::create(paddr.offset(i), false)); + } + ASSERT(m_physical_pages.size() == page_count()); +} + + +VMObject::VMObject(RetainPtr&& inode) + : m_inode(move(inode)) +{ + ASSERT(m_inode); + m_size = ceil_div(m_inode->size(), PAGE_SIZE) * PAGE_SIZE; + m_physical_pages.resize(page_count()); + MM.register_vmo(*this); +} + +VMObject::~VMObject() +{ + if (m_inode) + ASSERT(m_inode->vmo() == this); + MM.unregister_vmo(*this); +} + +template +void VMObject::for_each_region(Callback callback) +{ + // FIXME: Figure out a better data structure so we don't have to walk every single region every time an inode changes. + // Perhaps VMObject could have a Vector with all of his mappers? + for (auto* region : MM.m_regions) { + if (®ion->vmo() == this) + callback(*region); + } +} + +void VMObject::inode_size_changed(Badge, size_t old_size, size_t new_size) +{ + (void)old_size; + InterruptDisabler disabler; + + size_t old_page_count = page_count(); + m_size = new_size; + + if (page_count() > old_page_count) { + // Add null pages and let the fault handler page these in when that day comes. + for (size_t i = old_page_count; i < page_count(); ++i) + m_physical_pages.append(nullptr); + } else { + // Prune the no-longer valid pages. I'm not sure this is actually correct behavior. + for (size_t i = page_count(); i < old_page_count; ++i) + m_physical_pages.take_last(); + } + + // FIXME: Consolidate with inode_contents_changed() so we only do a single walk. + for_each_region([] (Region& region) { + ASSERT(region.page_directory()); + MM.remap_region(*region.page_directory(), region); + }); +} + +void VMObject::inode_contents_changed(Badge, off_t offset, ssize_t size, const byte* data) +{ + (void)size; + (void)data; + InterruptDisabler disabler; + ASSERT(offset >= 0); + + // FIXME: Only invalidate the parts that actually changed. + for (auto& physical_page : m_physical_pages) + physical_page = nullptr; + +#if 0 + size_t current_offset = offset; + size_t remaining_bytes = size; + const byte* data_ptr = data; + + auto to_page_index = [] (size_t offset) -> size_t { + return offset / PAGE_SIZE; + }; + + if (current_offset & PAGE_MASK) { + size_t page_index = to_page_index(current_offset); + size_t bytes_to_copy = min(size, PAGE_SIZE - (current_offset & PAGE_MASK)); + if (m_physical_pages[page_index]) { + auto* ptr = MM.quickmap_page(*m_physical_pages[page_index]); + memcpy(ptr, data_ptr, bytes_to_copy); + MM.unquickmap_page(); + } + current_offset += bytes_to_copy; + data += bytes_to_copy; + remaining_bytes -= bytes_to_copy; + } + + for (size_t page_index = to_page_index(current_offset); page_index < m_physical_pages.size(); ++page_index) { + size_t bytes_to_copy = PAGE_SIZE - (current_offset & PAGE_MASK); + if (m_physical_pages[page_index]) { + auto* ptr = MM.quickmap_page(*m_physical_pages[page_index]); + memcpy(ptr, data_ptr, bytes_to_copy); + MM.unquickmap_page(); + } + current_offset += bytes_to_copy; + data += bytes_to_copy; + } +#endif + + // FIXME: Consolidate with inode_size_changed() so we only do a single walk. + for_each_region([] (Region& region) { + ASSERT(region.page_directory()); + MM.remap_region(*region.page_directory(), region); + }); +} diff --git a/Kernel/VM/VMObject.h b/Kernel/VM/VMObject.h new file mode 100644 index 0000000000..f376fa25a4 --- /dev/null +++ b/Kernel/VM/VMObject.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +class Inode; +class PhysicalPage; + +class VMObject : public Retainable, public Weakable { + friend class MemoryManager; +public: + static Retained create_file_backed(RetainPtr&&); + static Retained create_anonymous(size_t); + static Retained create_for_physical_range(PhysicalAddress, size_t); + Retained clone(); + + ~VMObject(); + bool is_anonymous() const { return m_anonymous; } + + Inode* inode() { return m_inode.ptr(); } + const Inode* inode() const { return m_inode.ptr(); } + size_t inode_offset() const { return m_inode_offset; } + + String name() const { return m_name; } + void set_name(const String& name) { m_name = name; } + + size_t page_count() const { return m_size / PAGE_SIZE; } + const Vector>& physical_pages() const { return m_physical_pages; } + Vector>& physical_pages() { return m_physical_pages; } + + void inode_contents_changed(Badge, off_t, ssize_t, const byte*); + void inode_size_changed(Badge, size_t old_size, size_t new_size); + + size_t size() const { return m_size; } + +private: + VMObject(RetainPtr&&); + explicit VMObject(VMObject&); + explicit VMObject(size_t); + VMObject(PhysicalAddress, size_t); + + template void for_each_region(Callback); + + String m_name; + bool m_anonymous { false }; + off_t m_inode_offset { 0 }; + size_t m_size { 0 }; + bool m_allow_cpu_caching { true }; + RetainPtr m_inode; + Vector> m_physical_pages; + Lock m_paging_lock; +}; diff --git a/Kernel/i386.cpp b/Kernel/i386.cpp index efdc3850d9..717b717bd4 100644 --- a/Kernel/i386.cpp +++ b/Kernel/i386.cpp @@ -3,7 +3,7 @@ #include "i386.h" #include "Assertions.h" #include "Process.h" -#include "MemoryManager.h" +#include #include "IRQHandler.h" #include "PIC.h" #include "Scheduler.h" diff --git a/Kernel/init.cpp b/Kernel/init.cpp index 209ea71552..4cd85e66db 100644 --- a/Kernel/init.cpp +++ b/Kernel/init.cpp @@ -14,7 +14,7 @@ #include #include #include -#include "MemoryManager.h" +#include #include #include "RTC.h" #include