diff --git a/Kernel/FileSystem/ProcFS.cpp b/Kernel/FileSystem/ProcFS.cpp index 0874fe6704..aef44fcbe5 100644 --- a/Kernel/FileSystem/ProcFS.cpp +++ b/Kernel/FileSystem/ProcFS.cpp @@ -387,8 +387,8 @@ ByteBuffer procfs$mm(InodeIdentifier) vmo->name().characters()); } builder.appendf("VMO count: %u\n", MM.m_vmos.size()); - builder.appendf("Free physical pages: %u\n", MM.m_free_physical_pages.size()); - builder.appendf("Free supervisor physical pages: %u\n", MM.m_free_supervisor_physical_pages.size()); + builder.appendf("Free physical pages: %u\n", MM.user_physical_pages() - MM.user_physical_pages_used()); + builder.appendf("Free supervisor physical pages: %u\n", MM.super_physical_pages() - MM.super_physical_pages_used()); return builder.to_byte_buffer(); } @@ -544,10 +544,10 @@ ByteBuffer procfs$memstat(InodeIdentifier) kmalloc_sum_eternal, sum_alloc, sum_free, - MM.user_physical_pages_in_existence() - MM.m_free_physical_pages.size(), - MM.m_free_physical_pages.size(), - MM.super_physical_pages_in_existence() - MM.m_free_supervisor_physical_pages.size(), - MM.m_free_supervisor_physical_pages.size(), + MM.user_physical_pages_used(), + MM.user_physical_pages() - MM.user_physical_pages_used(), + MM.super_physical_pages_used(), + MM.super_physical_pages() - MM.super_physical_pages_used(), g_kmalloc_call_count, g_kfree_call_count); return builder.to_byte_buffer(); diff --git a/Kernel/Makefile b/Kernel/Makefile index cd5d3a5b75..8579d3bf38 100644 --- a/Kernel/Makefile +++ b/Kernel/Makefile @@ -18,6 +18,7 @@ KERNEL_OBJS = \ VM/VMObject.o \ VM/PageDirectory.o \ VM/PhysicalPage.o \ + VM/PhysicalRegion.o \ VM/RangeAllocator.o \ Console.o \ IRQHandler.o \ diff --git a/Kernel/VM/MemoryManager.cpp b/Kernel/VM/MemoryManager.cpp index 00537c5fa5..aa3292fdb1 100644 --- a/Kernel/VM/MemoryManager.cpp +++ b/Kernel/VM/MemoryManager.cpp @@ -12,8 +12,6 @@ //#define PAGE_FAULT_DEBUG static MemoryManager* s_the; -unsigned MemoryManager::s_user_physical_pages_in_existence; -unsigned MemoryManager::s_super_physical_pages_in_existence; MemoryManager& MM { @@ -78,6 +76,14 @@ void MemoryManager::initialize_paging() // 5 MB -> 0xc0000000 Userspace physical pages (available for allocation!) // 0xc0000000-0xffffffff Kernel-only linear address space +#ifdef MM_DEBUG + dbgprintf("MM: Quickmap will use %p\n", m_quickmap_addr.get()); +#endif + m_quickmap_addr = VirtualAddress((1 * MB) - PAGE_SIZE); + + RetainPtr region = nullptr; + bool region_is_super = false; + for (auto* mmap = (multiboot_memory_map_t*)multiboot_info_ptr->mmap_addr; (unsigned long)mmap < multiboot_info_ptr->mmap_addr + multiboot_info_ptr->mmap_length; mmap = (multiboot_memory_map_t*)((unsigned long)mmap + mmap->size + sizeof(mmap->size))) { kprintf("MM: Multiboot mmap: base_addr = 0x%x%08x, length = 0x%x%08x, type = 0x%x\n", (dword)(mmap->addr >> 32), @@ -88,26 +94,48 @@ void MemoryManager::initialize_paging() if (mmap->type != MULTIBOOT_MEMORY_AVAILABLE) continue; + // FIXME: Maybe make use of stuff below the 1MB mark? if (mmap->addr < (1 * MB)) continue; - for (size_t page_base = mmap->addr; page_base < (mmap->addr + mmap->len); page_base += PAGE_SIZE) { - if (page_base < (4 * MB)) { - // Skip over pages managed by kmalloc. - continue; - } +#ifdef MM_DEBUG + kprintf("MM: considering memory at %p - %p\n", + (dword)mmap->addr, (dword)(mmap->addr + mmap->len)); +#endif - if (page_base < (5 * MB)) - m_free_supervisor_physical_pages.append(PhysicalPage::create_eternal(PhysicalAddress(page_base), true)); - else - m_free_physical_pages.append(PhysicalPage::create_eternal(PhysicalAddress(page_base), false)); + for (size_t page_base = mmap->addr; page_base < (mmap->addr + mmap->len); page_base += PAGE_SIZE) { + auto addr = PhysicalAddress(page_base); + + if (page_base < 4 * MB) { + // nothing + } else if (page_base >= 4 * MB && page_base < 5 * MB) { + if (region.is_null() || !region_is_super || region->upper().offset(PAGE_SIZE) != addr) { + m_super_physical_regions.append(PhysicalRegion::create(addr, addr)); + region = m_super_physical_regions.last(); + region_is_super = true; + } else { + region->expand(region->lower(), addr); + } + } else { + if (region.is_null() || region_is_super || region->upper().offset(PAGE_SIZE) != addr) { + m_user_physical_regions.append(PhysicalRegion::create(addr, addr)); + region = m_user_physical_regions.last(); + region_is_super = false; + } else { + region->expand(region->lower(), addr); + } + } } } - m_quickmap_addr = VirtualAddress((1 * MB) - PAGE_SIZE); + for (auto& region : m_super_physical_regions) + m_super_physical_pages += region->finalize_capacity(); + + for (auto& region : m_user_physical_regions) + m_user_physical_pages += region->finalize_capacity(); + #ifdef MM_DEBUG - dbgprintf("MM: Quickmap will use P%x\n", m_quickmap_addr.get()); dbgprintf("MM: Installing page directory\n"); #endif @@ -282,7 +310,7 @@ bool MemoryManager::zero_page(Region& region, unsigned page_index_in_region) remap_region_page(region, page_index_in_region, true); return true; } - auto physical_page = allocate_physical_page(ShouldZeroFill::Yes); + auto physical_page = allocate_user_physical_page(ShouldZeroFill::Yes); #ifdef PAGE_FAULT_DEBUG dbgprintf(" >> ZERO P%x\n", physical_page->paddr().get()); #endif @@ -309,7 +337,7 @@ bool MemoryManager::copy_on_write(Region& region, unsigned page_index_in_region) dbgprintf(" >> It's a COW page and it's time to COW!\n"); #endif auto physical_page_to_copy = move(vmo.physical_pages()[page_index_in_region]); - auto physical_page = allocate_physical_page(ShouldZeroFill::No); + auto physical_page = allocate_user_physical_page(ShouldZeroFill::No); byte* dest_ptr = quickmap_page(*physical_page); const byte* src_ptr = region.vaddr().offset(page_index_in_region * PAGE_SIZE).as_ptr(); #ifdef PAGE_FAULT_DEBUG @@ -360,7 +388,7 @@ bool MemoryManager::page_in_from_inode(Region& region, unsigned page_index_in_re memset(page_buffer + nread, 0, PAGE_SIZE - nread); } cli(); - vmo_page = allocate_physical_page(ShouldZeroFill::No); + vmo_page = allocate_user_physical_page(ShouldZeroFill::No); if (vmo_page.is_null()) { kprintf("MM: page_in_from_inode was unable to allocate a physical page\n"); return false; @@ -430,40 +458,114 @@ RetainPtr MemoryManager::allocate_kernel_region(size_t size, String&& na return region; } -RetainPtr MemoryManager::allocate_physical_page(ShouldZeroFill should_zero_fill) +void MemoryManager::deallocate_user_physical_page(PhysicalPage& page) +{ + for (auto& region : m_user_physical_regions) { + if (!region->contains(page)) { + kprintf( + "MM: deallocate_user_physical_page: %p not in %p -> %p\n", + page.paddr(), region->lower().get(), region->upper().get()); + continue; + } + + region->return_page(page); + m_user_physical_pages_used--; + + return; + } + + kprintf("MM: deallocate_user_physical_page couldn't figure out region for user page @ %p\n", page.paddr()); + ASSERT_NOT_REACHED(); +} + +RetainPtr MemoryManager::allocate_user_physical_page(ShouldZeroFill should_zero_fill) { InterruptDisabler disabler; - if (1 > m_free_physical_pages.size()) { - kprintf("FUCK! No physical pages available.\n"); + + RetainPtr page = nullptr; + + for (auto& region : m_user_physical_regions) { + page = region->take_free_page(false); + if (page.is_null()) + continue; + } + + if (!page) { + if (m_user_physical_regions.is_empty()) { + kprintf("MM: no user physical regions available (?)\n"); + } + + kprintf("MM: no user physical pages available\n"); ASSERT_NOT_REACHED(); return {}; } + #ifdef MM_DEBUG - dbgprintf("MM: allocate_physical_page vending P%x (%u remaining)\n", m_free_physical_pages.last()->paddr().get(), m_free_physical_pages.size()); + dbgprintf("MM: allocate_user_physical_page vending P%p\n", page->paddr().get()); #endif - auto physical_page = m_free_physical_pages.take_last(); + if (should_zero_fill == ShouldZeroFill::Yes) { - auto* ptr = (dword*)quickmap_page(*physical_page); + auto* ptr = (dword*)quickmap_page(*page); fast_dword_fill(ptr, 0, PAGE_SIZE / sizeof(dword)); unquickmap_page(); } - return physical_page; + + m_user_physical_pages_used++; + + return page; +} + +void MemoryManager::deallocate_supervisor_physical_page(PhysicalPage& page) +{ + for (auto& region : m_super_physical_regions) { + if (!region->contains(page)) { + kprintf( + "MM: deallocate_supervisor_physical_page: %p not in %p -> %p\n", + page.paddr(), region->lower().get(), region->upper().get()); + continue; + } + + region->return_page(page); + m_super_physical_pages_used--; + + return; + } + + kprintf("MM: deallocate_supervisor_physical_page couldn't figure out region for super page @ %p\n", page.paddr()); + ASSERT_NOT_REACHED(); } RetainPtr MemoryManager::allocate_supervisor_physical_page() { InterruptDisabler disabler; - if (1 > m_free_supervisor_physical_pages.size()) { - kprintf("FUCK! No physical pages available.\n"); + + RetainPtr page = nullptr; + + for (auto& region : m_super_physical_regions) { + page = region->take_free_page(true); + if (page.is_null()) + continue; + } + + if (!page) { + if (m_super_physical_regions.is_empty()) { + kprintf("MM: no super physical regions available (?)\n"); + } + + kprintf("MM: no super physical pages available\n"); ASSERT_NOT_REACHED(); return {}; } + #ifdef MM_DEBUG - dbgprintf("MM: allocate_supervisor_physical_page vending P%x (%u remaining)\n", m_free_supervisor_physical_pages.last()->paddr().get(), m_free_supervisor_physical_pages.size()); + dbgprintf("MM: allocate_supervisor_physical_page vending P%p\n", page->paddr().get()); #endif - auto physical_page = m_free_supervisor_physical_pages.take_last(); - fast_dword_fill((dword*)physical_page->paddr().as_ptr(), 0, PAGE_SIZE / sizeof(dword)); - return physical_page; + + fast_dword_fill((dword*)page->paddr().as_ptr(), 0, PAGE_SIZE / sizeof(dword)); + + m_super_physical_pages_used++; + + return page; } void MemoryManager::enter_process_paging_scope(Process& process) diff --git a/Kernel/VM/MemoryManager.h b/Kernel/VM/MemoryManager.h index ef2e274faa..29a985adc1 100644 --- a/Kernel/VM/MemoryManager.h +++ b/Kernel/VM/MemoryManager.h @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ class MemoryManager { AK_MAKE_ETERNAL friend class PageDirectory; friend class PhysicalPage; + friend class PhysicalRegion; friend class Region; friend class VMObject; friend ByteBuffer procfs$mm(InodeIdentifier); @@ -59,19 +61,23 @@ public: Yes }; - RetainPtr allocate_physical_page(ShouldZeroFill); + RetainPtr allocate_user_physical_page(ShouldZeroFill); RetainPtr allocate_supervisor_physical_page(); + void deallocate_user_physical_page(PhysicalPage&); + void deallocate_supervisor_physical_page(PhysicalPage&); void remap_region(PageDirectory&, Region&); - int user_physical_pages_in_existence() const { return s_user_physical_pages_in_existence; } - int super_physical_pages_in_existence() const { return s_super_physical_pages_in_existence; } - void map_for_kernel(VirtualAddress, PhysicalAddress); RetainPtr allocate_kernel_region(size_t, String&& name); void map_region_at_address(PageDirectory&, Region&, VirtualAddress, bool user_accessible); + unsigned user_physical_pages() const { return m_user_physical_pages; } + unsigned user_physical_pages_used() const { return m_user_physical_pages_used; } + unsigned super_physical_pages() const { return m_super_physical_pages; } + unsigned super_physical_pages_used() const { return m_super_physical_pages_used; } + private: MemoryManager(); ~MemoryManager(); @@ -206,9 +212,6 @@ private: dword* m_pte; }; - static unsigned s_user_physical_pages_in_existence; - static unsigned s_super_physical_pages_in_existence; - PageTableEntry ensure_pte(PageDirectory&, VirtualAddress); RetainPtr m_kernel_page_directory; @@ -217,8 +220,13 @@ private: VirtualAddress m_quickmap_addr; - Vector> m_free_physical_pages; - Vector> m_free_supervisor_physical_pages; + unsigned m_user_physical_pages { 0 }; + unsigned m_user_physical_pages_used { 0 }; + unsigned m_super_physical_pages { 0 }; + unsigned m_super_physical_pages_used { 0 }; + + Vector> m_user_physical_regions {}; + Vector> m_super_physical_regions {}; HashTable m_vmos; HashTable m_user_regions; diff --git a/Kernel/VM/PhysicalPage.cpp b/Kernel/VM/PhysicalPage.cpp index 0ffa91bafd..f6027589a7 100644 --- a/Kernel/VM/PhysicalPage.cpp +++ b/Kernel/VM/PhysicalPage.cpp @@ -21,21 +21,21 @@ PhysicalPage::PhysicalPage(PhysicalAddress paddr, bool supervisor, bool may_retu , m_supervisor(supervisor) , m_paddr(paddr) { - if (supervisor) - ++MemoryManager::s_super_physical_pages_in_existence; - else - ++MemoryManager::s_user_physical_pages_in_existence; } void PhysicalPage::return_to_freelist() { ASSERT((paddr().get() & ~PAGE_MASK) == 0); + InterruptDisabler disabler; + m_retain_count = 1; + if (m_supervisor) - MM.m_free_supervisor_physical_pages.append(adopt(*this)); + MM.deallocate_supervisor_physical_page(*this); else - MM.m_free_physical_pages.append(adopt(*this)); + MM.deallocate_user_physical_page(*this); + #ifdef MM_DEBUG dbgprintf("MM: P%x released to freelist\n", m_paddr.get()); #endif diff --git a/Kernel/VM/PhysicalRegion.cpp b/Kernel/VM/PhysicalRegion.cpp new file mode 100644 index 0000000000..4743d1b59d --- /dev/null +++ b/Kernel/VM/PhysicalRegion.cpp @@ -0,0 +1,77 @@ +#include +#include +#include +#include +#include +#include +#include + +Retained PhysicalRegion::create(PhysicalAddress lower, PhysicalAddress upper) +{ + return adopt(*new PhysicalRegion(lower, upper)); +} + +PhysicalRegion::PhysicalRegion(PhysicalAddress lower, PhysicalAddress upper) + : m_lower(lower) + , m_upper(upper) + , m_bitmap(Bitmap::create()) +{ +} + +void PhysicalRegion::expand(PhysicalAddress lower, PhysicalAddress upper) +{ + ASSERT(!m_pages); + + m_lower = lower; + m_upper = upper; +} + +unsigned PhysicalRegion::finalize_capacity() +{ + ASSERT(!m_pages); + + m_pages = (m_upper.get() - m_lower.get()) / PAGE_SIZE; + m_bitmap.grow(m_pages, false); + + return size(); +} + +RetainPtr PhysicalRegion::take_free_page(bool supervisor) +{ + ASSERT(m_pages); + + if (m_used == m_pages) + return nullptr; + + for (unsigned page = m_last; page < m_pages; page++) { + if (!m_bitmap.get(page)) { + m_bitmap.set(page, true); + m_used++; + m_last = page + 1; + return PhysicalPage::create(m_lower.offset(page * PAGE_SIZE), supervisor); + } + } + + ASSERT_NOT_REACHED(); + + return nullptr; +} + +void PhysicalRegion::return_page_at(PhysicalAddress addr) +{ + ASSERT(m_pages); + + if (m_used == 0) { + ASSERT_NOT_REACHED(); + } + + int local_offset = addr.get() - m_lower.get(); + ASSERT(local_offset >= 0); + ASSERT(local_offset < m_pages * PAGE_SIZE); + + auto page = local_offset / PAGE_SIZE; + if (page < m_last) m_last = page; + + m_bitmap.set(page, false); + m_used--; +} diff --git a/Kernel/VM/PhysicalRegion.h b/Kernel/VM/PhysicalRegion.h new file mode 100644 index 0000000000..d4f4fb1934 --- /dev/null +++ b/Kernel/VM/PhysicalRegion.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include +#include +#include +#include + +class PhysicalRegion : public Retainable { + AK_MAKE_ETERNAL + +public: + static Retained create(PhysicalAddress lower, PhysicalAddress upper); + ~PhysicalRegion() {} + + void expand(PhysicalAddress lower, PhysicalAddress upper); + unsigned finalize_capacity(); + + PhysicalAddress lower() const { return m_lower; } + PhysicalAddress upper() const { return m_upper; } + unsigned size() const { return m_pages; } + unsigned used() const { return m_used; } + unsigned free() const { return m_pages - m_used; } + bool contains(PhysicalPage& page) const { return page.paddr() >= m_lower && page.paddr() <= m_upper; } + + RetainPtr take_free_page(bool supervisor); + void return_page_at(PhysicalAddress addr); + void return_page(PhysicalPage& page) { return_page_at(page.paddr()); } + +private: + PhysicalRegion(PhysicalAddress lower, PhysicalAddress upper); + + PhysicalAddress m_lower; + PhysicalAddress m_upper; + unsigned m_pages { 0 }; + unsigned m_used { 0 }; + unsigned m_last { 0 }; + Bitmap m_bitmap; +}; diff --git a/Kernel/VM/Region.cpp b/Kernel/VM/Region.cpp index 87acd375d5..d3cfd0c655 100644 --- a/Kernel/VM/Region.cpp +++ b/Kernel/VM/Region.cpp @@ -103,7 +103,7 @@ int Region::commit() for (size_t i = first_page_index(); i <= last_page_index(); ++i) { if (!vmo().physical_pages()[i].is_null()) continue; - auto physical_page = MM.allocate_physical_page(MemoryManager::ShouldZeroFill::Yes); + auto physical_page = MM.allocate_user_physical_page(MemoryManager::ShouldZeroFill::Yes); if (!physical_page) { kprintf("MM: commit was unable to allocate a physical page\n"); return -ENOMEM;