From da33e2a564faa5f9829f312c9f8664e9e021f1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Holz?= Date: Wed, 3 Jan 2024 17:21:45 +0100 Subject: [PATCH] Kernel/riscv64: Add MMU initialization code We initialize the MMU by first setting up the page tables for the kernel image and the initial kernel stack. Then we jump to a identity mapped page which makes the newly created kernel root page table active by setting `satp` and then jumps to `init`. --- Kernel/Arch/riscv64/MMU.cpp | 256 +++++++++++++++++++++++++++++++ Kernel/Arch/riscv64/MMU.h | 18 +++ Kernel/Arch/riscv64/pre_init.cpp | 4 +- Kernel/CMakeLists.txt | 1 + 4 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 Kernel/Arch/riscv64/MMU.cpp create mode 100644 Kernel/Arch/riscv64/MMU.h diff --git a/Kernel/Arch/riscv64/MMU.cpp b/Kernel/Arch/riscv64/MMU.cpp new file mode 100644 index 0000000000..ab522a08aa --- /dev/null +++ b/Kernel/Arch/riscv64/MMU.cpp @@ -0,0 +1,256 @@ +/* + * Copyright (c) 2023, Sönke Holz + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include + +#include +#include +#include +#include +#include +#include + +// These come from the linker script +extern u8 page_tables_phys_start[]; +extern u8 page_tables_phys_end[]; +extern u8 start_of_kernel_image[]; +extern u8 end_of_kernel_image[]; + +namespace Kernel::Memory { + +class PageBumpAllocator { +public: + PageBumpAllocator(u64* start, u64 const* end) + : m_start(start) + , m_end(end) + , m_current(start) + { + if (m_start >= m_end) { + panic_without_mmu("Invalid memory range passed to PageBumpAllocator"sv); + } + if (bit_cast(m_start) % PAGE_TABLE_SIZE != 0 || bit_cast(m_end) % PAGE_TABLE_SIZE != 0) { + panic_without_mmu("Memory range passed into PageBumpAllocator not aligned to PAGE_TABLE_SIZE"sv); + } + } + + u64* take_page() + { + if (m_current >= m_end) { + panic_without_mmu("pre_init page table memory exhausted"sv); + } + + u64* page = m_current; + m_current += (PAGE_TABLE_SIZE / sizeof(FlatPtr)); + + __builtin_memset(page, 0, PAGE_TABLE_SIZE); + return page; + } + +private: + u64 const* m_start; + u64 const* m_end; + u64* m_current; +}; + +static UNMAP_AFTER_INIT FlatPtr calculate_physical_to_link_time_address_offset() +{ + FlatPtr physical_address; + FlatPtr link_time_address; + + asm volatile( + " lla %[physical_address], 1f\n" + "1: la %[link_time_address], 1b\n" + : [physical_address] "=r"(physical_address), + [link_time_address] "=r"(link_time_address)); + + return link_time_address - physical_address; +} + +template +inline T* adjust_by_mapping_base(T* ptr) +{ + return bit_cast(bit_cast(ptr) - calculate_physical_to_link_time_address_offset()); +} + +static UNMAP_AFTER_INIT bool page_table_entry_valid(u64 entry) +{ + return (entry & to_underlying(PageTableEntryBits::Valid)) != 0; +} + +static UNMAP_AFTER_INIT u64* insert_page_table(PageBumpAllocator& allocator, u64* root_table, VirtualAddress virtual_addr) +{ + size_t vpn_1 = (virtual_addr.get() >> VPN_1_OFFSET) & PAGE_TABLE_INDEX_MASK; + size_t vpn_2 = (virtual_addr.get() >> VPN_2_OFFSET) & PAGE_TABLE_INDEX_MASK; + + u64* level2_table = root_table; + + if (!page_table_entry_valid(level2_table[vpn_2])) { + level2_table[vpn_2] = (bit_cast(allocator.take_page()) >> PADDR_PPN_OFFSET) << PTE_PPN_OFFSET; + level2_table[vpn_2] |= to_underlying(PageTableEntryBits::Valid); + } + + u64* level1_table = bit_cast((level2_table[vpn_2] >> PTE_PPN_OFFSET) << PADDR_PPN_OFFSET); + + if (!page_table_entry_valid(level1_table[vpn_1])) { + level1_table[vpn_1] = (bit_cast(allocator.take_page()) >> PADDR_PPN_OFFSET) << PTE_PPN_OFFSET; + level1_table[vpn_1] |= to_underlying(PageTableEntryBits::Valid); + } + + return bit_cast((level1_table[vpn_1] >> PTE_PPN_OFFSET) << PADDR_PPN_OFFSET); +} + +static UNMAP_AFTER_INIT u64* get_page_directory(u64* root_table, VirtualAddress virtual_addr) +{ + size_t vpn_2 = (virtual_addr.get() >> VPN_2_OFFSET) & PAGE_TABLE_INDEX_MASK; + + u64* level2_table = root_table; + + if (!page_table_entry_valid(level2_table[vpn_2])) + return nullptr; + + return bit_cast((level2_table[vpn_2] >> PTE_PPN_OFFSET) << PADDR_PPN_OFFSET); +} + +static UNMAP_AFTER_INIT void insert_entry(PageBumpAllocator& allocator, u64* root_table, VirtualAddress vaddr, PhysicalAddress paddr, PageTableEntryBits flags) +{ + u64* level0_table = insert_page_table(allocator, root_table, vaddr); + + size_t vpn_0 = (vaddr.get() >> VPN_0_OFFSET) & PAGE_TABLE_INDEX_MASK; + level0_table[vpn_0] = (paddr.get() >> PADDR_PPN_OFFSET) << PTE_PPN_OFFSET; + level0_table[vpn_0] |= to_underlying(PageTableEntryBits::Valid | PageTableEntryBits::Accessed | PageTableEntryBits::Dirty | flags); +} + +static UNMAP_AFTER_INIT void insert_entries_for_memory_range(PageBumpAllocator& allocator, u64* root_table, VirtualAddress start, VirtualAddress end, PhysicalAddress paddr, PageTableEntryBits flags) +{ + // Not very efficient, but simple and it works. + for (VirtualAddress vaddr = start; vaddr < end;) { + insert_entry(allocator, root_table, vaddr, paddr, flags); + vaddr = vaddr.offset(PAGE_SIZE); + paddr = paddr.offset(PAGE_SIZE); + } +} + +static UNMAP_AFTER_INIT void setup_quickmap_page_table(PageBumpAllocator& allocator, u64* root_table) +{ + auto kernel_pt1024_base = VirtualAddress { *adjust_by_mapping_base(&kernel_mapping_base) + KERNEL_PT1024_OFFSET }; + + auto quickmap_page_table = PhysicalAddress { bit_cast(insert_page_table(allocator, root_table, kernel_pt1024_base)) }; + *adjust_by_mapping_base(&boot_pd_kernel_pt1023) = bit_cast(quickmap_page_table.offset(calculate_physical_to_link_time_address_offset()).get()); +} + +static UNMAP_AFTER_INIT void build_mappings(PageBumpAllocator& allocator, u64* root_table) +{ + auto start_of_kernel_range = VirtualAddress { bit_cast(+start_of_kernel_image) }; + auto end_of_kernel_range = VirtualAddress { bit_cast(+end_of_kernel_image) }; + + // FIXME: don't use the memory before `start` as the stack + auto start_of_stack_range = VirtualAddress { bit_cast(+start_of_kernel_image) }.offset(-64 * KiB); + auto end_of_stack_range = VirtualAddress { bit_cast(+start_of_kernel_image) }; + + auto start_of_physical_kernel_range = PhysicalAddress { start_of_kernel_range.get() }.offset(-calculate_physical_to_link_time_address_offset()); + auto start_of_physical_stack_range = PhysicalAddress { start_of_stack_range.get() }.offset(-calculate_physical_to_link_time_address_offset()); + + // FIXME: dont map everything RWX + + // Map kernel into high virtual memory + insert_entries_for_memory_range(allocator, root_table, start_of_kernel_range, end_of_kernel_range, start_of_physical_kernel_range, PageTableEntryBits::Readable | PageTableEntryBits::Writeable | PageTableEntryBits::Executable); + + // Map the stack + insert_entries_for_memory_range(allocator, root_table, start_of_stack_range, end_of_stack_range, start_of_physical_stack_range, PageTableEntryBits::Readable | PageTableEntryBits::Writeable); +} + +static UNMAP_AFTER_INIT void setup_kernel_page_directory(u64* root_table) +{ + auto kernel_page_directory = bit_cast(get_page_directory(root_table, VirtualAddress { *adjust_by_mapping_base(&kernel_mapping_base) })); + if (kernel_page_directory == 0) + panic_without_mmu("Could not find kernel page directory!"sv); + + *adjust_by_mapping_base(&boot_pd_kernel) = PhysicalAddress { kernel_page_directory }; + + // There is no level 4 table in Sv39 + *adjust_by_mapping_base(&boot_pml4t) = PhysicalAddress { 0 }; + + *adjust_by_mapping_base(&boot_pdpt) = PhysicalAddress { bit_cast(root_table) }; +} + +// This function has to fit into one page as it will be identity mapped. +[[gnu::aligned(PAGE_SIZE)]] [[noreturn]] UNMAP_AFTER_INIT static void enable_paging(FlatPtr satp, u64* enable_paging_pte) +{ + // Switch current root page table to argument 0. This will immediately take effect, but we won't not crash as this function is identity mapped. + // Also, set up a temporary trap handler to catch any traps while switching page tables. + asm volatile( + " lla t0, 1f \n" + " csrw stvec, t0 \n" + + " csrw satp, %[satp] \n" + " sfence.vma \n" + + // Continue execution at high virtual address, by using an absolute jump. + " ld t0, 3f \n" + " jr t0 \n" + "3: .dword 4f \n" + "4: \n" + + // Add kernel_mapping_base to the stack pointer, such that it is also using the mapping in high virtual memory. + " add sp, sp, %[offset] \n" + + // Zero the PTE which identity maps this function + " add t0, %[offset], %[enable_paging_pte] \n" + " sd zero, (t0) \n" + " sfence.vma \n" + + // TODO: Set `stvec` to a trap handling function + + " li ra, 0 \n" + " li fp, 0 \n" + " tail init \n" + + ".p2align 2 \n" + "1: csrw sie, zero \n" + " wfi \n" + " j 1b \n" + : + : [satp] "r"(satp), [offset] "r"(calculate_physical_to_link_time_address_offset()), [enable_paging_pte] "r"(enable_paging_pte) + : "t0"); + + VERIFY_NOT_REACHED(); +} + +[[noreturn]] UNMAP_AFTER_INIT void init_page_tables_and_jump_to_init() +{ + if (RISCV64::CSR::SATP::read().MODE != RISCV64::CSR::SATP::Mode::Bare) + panic_without_mmu("Kernel booted with MMU enabled"sv); + + *adjust_by_mapping_base(&physical_to_virtual_offset) = calculate_physical_to_link_time_address_offset(); + *adjust_by_mapping_base(&kernel_mapping_base) = KERNEL_MAPPING_BASE; + *adjust_by_mapping_base(&kernel_load_base) = KERNEL_MAPPING_BASE; + + PageBumpAllocator allocator(adjust_by_mapping_base(reinterpret_cast(page_tables_phys_start)), adjust_by_mapping_base(reinterpret_cast(page_tables_phys_end))); + auto* root_table = allocator.take_page(); + build_mappings(allocator, root_table); + setup_quickmap_page_table(allocator, root_table); + setup_kernel_page_directory(root_table); + + // Identity map the `enable_paging` function and save the level 0 table address in order to remove the identity mapping in `enable_paging` again + auto const enable_paging_vaddr = VirtualAddress { bit_cast(&enable_paging) }; + auto const enable_paging_paddr = PhysicalAddress { bit_cast(&enable_paging) }; + + u64* enable_paging_level0_table = insert_page_table(allocator, root_table, enable_paging_vaddr); + + size_t enable_paging_vpn_0 = (enable_paging_vaddr.get() >> VPN_0_OFFSET) & PAGE_TABLE_INDEX_MASK; + enable_paging_level0_table[enable_paging_vpn_0] = (enable_paging_paddr.get() >> PADDR_PPN_OFFSET) << PTE_PPN_OFFSET; + enable_paging_level0_table[enable_paging_vpn_0] |= to_underlying(PageTableEntryBits::Valid | PageTableEntryBits::Accessed | PageTableEntryBits::Dirty | PageTableEntryBits::Readable | PageTableEntryBits::Executable); + + RISCV64::CSR::SATP satp = { + .PPN = bit_cast(root_table) >> PADDR_PPN_OFFSET, + .ASID = 0, + .MODE = RISCV64::CSR::SATP::Mode::Sv39, + }; + + enable_paging(bit_cast(satp), &enable_paging_level0_table[enable_paging_vpn_0]); +} + +} diff --git a/Kernel/Arch/riscv64/MMU.h b/Kernel/Arch/riscv64/MMU.h new file mode 100644 index 0000000000..02ffe967fd --- /dev/null +++ b/Kernel/Arch/riscv64/MMU.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2023, Sönke Holz + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include + +#include +VALIDATE_IS_RISCV64() + +namespace Kernel::Memory { + +[[noreturn]] void init_page_tables_and_jump_to_init(); + +} diff --git a/Kernel/Arch/riscv64/pre_init.cpp b/Kernel/Arch/riscv64/pre_init.cpp index 1f277ad051..16507d5f21 100644 --- a/Kernel/Arch/riscv64/pre_init.cpp +++ b/Kernel/Arch/riscv64/pre_init.cpp @@ -51,9 +51,7 @@ extern "C" [[noreturn]] UNMAP_AFTER_INIT void pre_init(FlatPtr mhartid, Physical // Catch traps in pre_init RISCV64::CSR::write(RISCV64::CSR::Address::STVEC, bit_cast(&early_trap_handler)); - // FIXME: Implement this - - Processor::halt(); + Memory::init_page_tables_and_jump_to_init(); } } diff --git a/Kernel/CMakeLists.txt b/Kernel/CMakeLists.txt index af068e4c3b..773582fae0 100644 --- a/Kernel/CMakeLists.txt +++ b/Kernel/CMakeLists.txt @@ -523,6 +523,7 @@ elseif("${SERENITY_ARCH}" STREQUAL "riscv64") Arch/riscv64/Delay.cpp Arch/riscv64/InterruptManagement.cpp Arch/riscv64/Interrupts.cpp + Arch/riscv64/MMU.cpp Arch/riscv64/PageDirectory.cpp Arch/riscv64/Panic.cpp Arch/riscv64/PCI/Initializer.cpp