From 55d756a81383a3c8cd0b938bef5c13f647e4f6df Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Sun, 8 Jan 2023 17:42:07 +0100 Subject: [PATCH] Kernel/aarch64: Implement initial page fault handling The shared code is moved to a common PageFault.cpp file. --- Kernel/Arch/PageFault.cpp | 150 +++++++++++++++++++++++++++++ Kernel/Arch/PageFault.h | 3 + Kernel/Arch/aarch64/Interrupts.cpp | 43 ++++++++- Kernel/Arch/x86_64/Interrupts.cpp | 131 +------------------------ Kernel/CMakeLists.txt | 1 + 5 files changed, 195 insertions(+), 133 deletions(-) create mode 100644 Kernel/Arch/PageFault.cpp diff --git a/Kernel/Arch/PageFault.cpp b/Kernel/Arch/PageFault.cpp new file mode 100644 index 0000000000..beaa039b2a --- /dev/null +++ b/Kernel/Arch/PageFault.cpp @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2018-2021, Andreas Kling + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Kernel { + +void PageFault::handle(RegisterState& regs) +{ + auto fault_address = m_vaddr.get(); + bool faulted_in_kernel = regs.previous_mode() == ExecutionMode::Kernel; + + if (faulted_in_kernel && Processor::current_in_irq()) { + // If we're faulting in an IRQ handler, first check if we failed + // due to safe_memcpy, safe_strnlen, or safe_memset. If we did, + // gracefully continue immediately. Because we're in an IRQ handler + // we can't really try to resolve the page fault in a meaningful + // way, so we need to do this before calling into + // MemoryManager::handle_page_fault, which would just bail and + // request a crash + if (handle_safe_access_fault(regs, fault_address)) + return; + } + + auto current_thread = Thread::current(); + + if (current_thread) { + current_thread->set_handling_page_fault(true); + PerformanceManager::add_page_fault_event(*current_thread, regs); + } + + ScopeGuard guard = [current_thread] { + if (current_thread) + current_thread->set_handling_page_fault(false); + }; + + if (!faulted_in_kernel) { + VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() }; + bool has_valid_stack_pointer = current_thread->process().address_space().with([&](auto& space) { + return MM.validate_user_stack(*space, userspace_sp); + }); + if (!has_valid_stack_pointer) { + dbgln("Invalid stack pointer: {}", userspace_sp); + return handle_crash(regs, "Bad stack on page fault", SIGSEGV); + } + } + + auto response = MM.handle_page_fault(*this); + + if (response == PageFaultResponse::ShouldCrash || response == PageFaultResponse::OutOfMemory || response == PageFaultResponse::BusError) { + if (faulted_in_kernel && handle_safe_access_fault(regs, fault_address)) { + // If this would be a ring0 (kernel) fault and the fault was triggered by + // safe_memcpy, safe_strnlen, or safe_memset then we resume execution at + // the appropriate _fault label rather than crashing + return; + } + + if (response == PageFaultResponse::BusError && current_thread->has_signal_handler(SIGBUS)) { + current_thread->send_urgent_signal_to_self(SIGBUS); + return; + } + + if (response != PageFaultResponse::OutOfMemory && current_thread) { + if (current_thread->has_signal_handler(SIGSEGV)) { + current_thread->send_urgent_signal_to_self(SIGSEGV); + return; + } + } + + dbgln("Unrecoverable page fault, {}{}{} address {}", + is_reserved_bit_violation() ? "reserved bit violation / " : "", + is_instruction_fetch() ? "instruction fetch / " : "", + is_write() ? "write to" : "read from", + VirtualAddress(fault_address)); + constexpr FlatPtr malloc_scrub_pattern = explode_byte(MALLOC_SCRUB_BYTE); + constexpr FlatPtr free_scrub_pattern = explode_byte(FREE_SCRUB_BYTE); + constexpr FlatPtr kmalloc_scrub_pattern = explode_byte(KMALLOC_SCRUB_BYTE); + constexpr FlatPtr kfree_scrub_pattern = explode_byte(KFREE_SCRUB_BYTE); + if (response == PageFaultResponse::BusError) { + dbgln("Note: Address {} is an access to an undefined memory range of an Inode-backed VMObject", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (malloc_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be uninitialized malloc() memory", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (free_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be recently free()'d memory", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (kmalloc_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be uninitialized kmalloc() memory", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (kfree_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be recently kfree()'d memory", VirtualAddress(fault_address)); + } else if (fault_address < 4096) { + dbgln("Note: Address {} looks like a possible nullptr dereference", VirtualAddress(fault_address)); + } else if constexpr (SANITIZE_PTRS) { + constexpr FlatPtr refptr_scrub_pattern = explode_byte(REFPTR_SCRUB_BYTE); + constexpr FlatPtr nonnullrefptr_scrub_pattern = explode_byte(NONNULLREFPTR_SCRUB_BYTE); + constexpr FlatPtr ownptr_scrub_pattern = explode_byte(OWNPTR_SCRUB_BYTE); + constexpr FlatPtr nonnullownptr_scrub_pattern = explode_byte(NONNULLOWNPTR_SCRUB_BYTE); + constexpr FlatPtr lockrefptr_scrub_pattern = explode_byte(LOCKREFPTR_SCRUB_BYTE); + constexpr FlatPtr nonnulllockrefptr_scrub_pattern = explode_byte(NONNULLLOCKREFPTR_SCRUB_BYTE); + + if ((fault_address & 0xffff0000) == (refptr_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (nonnullrefptr_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (ownptr_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be a recently destroyed OwnPtr", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (nonnullownptr_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be a recently destroyed NonnullOwnPtr", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (lockrefptr_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address)); + } else if ((fault_address & 0xffff0000) == (nonnulllockrefptr_scrub_pattern & 0xffff0000)) { + dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address)); + } + } + + if (current_thread) { + auto& current_process = current_thread->process(); + if (current_process.is_user_process()) { + auto fault_address_string = KString::formatted("{:p}", fault_address); + auto fault_address_view = fault_address_string.is_error() ? ""sv : fault_address_string.value()->view(); + (void)current_process.try_set_coredump_property("fault_address"sv, fault_address_view); + (void)current_process.try_set_coredump_property("fault_type"sv, type() == PageFault::Type::PageNotPresent ? "NotPresent"sv : "ProtectionViolation"sv); + StringView fault_access; + if (is_instruction_fetch()) + fault_access = "Execute"sv; + else + fault_access = access() == PageFault::Access::Read ? "Read"sv : "Write"sv; + (void)current_process.try_set_coredump_property("fault_access"sv, fault_access); + } + } + + if (response == PageFaultResponse::BusError) + return handle_crash(regs, "Page Fault (Bus Error)", SIGBUS, false); + return handle_crash(regs, "Page Fault", SIGSEGV, response == PageFaultResponse::OutOfMemory); + } else if (response == PageFaultResponse::Continue) { + dbgln_if(PAGE_FAULT_DEBUG, "Continuing after resolved page fault"); + } else { + VERIFY_NOT_REACHED(); + } +} + +} diff --git a/Kernel/Arch/PageFault.h b/Kernel/Arch/PageFault.h index faa6fc4e1e..27ad0eb49d 100644 --- a/Kernel/Arch/PageFault.h +++ b/Kernel/Arch/PageFault.h @@ -8,6 +8,7 @@ #include #include +#include #include #include @@ -44,6 +45,8 @@ public: { } + void handle(RegisterState& regs); + enum class Type { PageNotPresent = PageFaultFlags::NotPresent, ProtectionViolation = PageFaultFlags::ProtectionViolation, diff --git a/Kernel/Arch/aarch64/Interrupts.cpp b/Kernel/Arch/aarch64/Interrupts.cpp index c23943e34f..d5560b307f 100644 --- a/Kernel/Arch/aarch64/Interrupts.cpp +++ b/Kernel/Arch/aarch64/Interrupts.cpp @@ -5,12 +5,14 @@ */ #include +#include #include #include #include #include #include #include +#include #include namespace Kernel { @@ -53,10 +55,35 @@ static void dump_exception_syndrome_register(Aarch64::ESR_EL1 const& esr_el1) dbgln("Data Fault Status Code: {}", Aarch64::data_fault_status_code_to_string(esr_el1.ISS)); } -extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame); -extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame) +static PageFault page_fault_from_exception_syndrome_register(VirtualAddress fault_address, Aarch64::ESR_EL1 esr_el1) { + PageFault fault { fault_address }; + + u8 data_fault_status_code = esr_el1.ISS & 0x3f; + if (data_fault_status_code >= 0b001100 && data_fault_status_code <= 0b001111) { + fault.set_type(PageFault::Type::ProtectionViolation); + } else if (data_fault_status_code >= 0b000100 && data_fault_status_code <= 0b000111) { + fault.set_type(PageFault::Type::PageNotPresent); + } else { + PANIC("Unknown DFSC: {}", data_fault_status_code); + } + + fault.set_access((esr_el1.ISS & (1 << 6)) == (1 << 6) ? PageFault::Access::Write : PageFault::Access::Read); + + // FIXME: Set correct mode + fault.set_mode(ExecutionMode::Kernel); + + return fault; +} + +extern "C" void exception_common(Kernel::TrapFrame* trap_frame); +extern "C" void exception_common(Kernel::TrapFrame* trap_frame) +{ + Processor::current().enter_trap(*trap_frame, false); + auto esr_el1 = Kernel::Aarch64::ESR_EL1::read(); + auto fault_address = Aarch64::FAR_EL1::read().virtual_address; + Processor::enable_interrupts(); constexpr bool print_state = true; if constexpr (print_state) { @@ -65,7 +92,17 @@ extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame) dump_exception_syndrome_register(esr_el1); } - Kernel::Processor::halt(); + if (Aarch64::exception_class_is_data_abort(esr_el1.EC)) { + auto page_fault = page_fault_from_exception_syndrome_register(VirtualAddress(fault_address), esr_el1); + page_fault.handle(*trap_frame->regs); + } else { + dump_registers(*trap_frame->regs); + dump_exception_syndrome_register(esr_el1); + PANIC("Unexpected exception!"); + } + + Processor::disable_interrupts(); + Processor::current().exit_trap(*trap_frame); } static Array s_interrupt_handlers; diff --git a/Kernel/Arch/x86_64/Interrupts.cpp b/Kernel/Arch/x86_64/Interrupts.cpp index c9f98f3614..eedb7097c6 100644 --- a/Kernel/Arch/x86_64/Interrupts.cpp +++ b/Kernel/Arch/x86_64/Interrupts.cpp @@ -23,8 +23,6 @@ #include #include -#include - #include #include #include @@ -201,135 +199,8 @@ void page_fault_handler(TrapFrame* trap) dump_registers(regs); } - bool faulted_in_kernel = !(regs.cs & 3); - - if (faulted_in_kernel && Processor::current_in_irq()) { - // If we're faulting in an IRQ handler, first check if we failed - // due to safe_memcpy, safe_strnlen, or safe_memset. If we did, - // gracefully continue immediately. Because we're in an IRQ handler - // we can't really try to resolve the page fault in a meaningful - // way, so we need to do this before calling into - // MemoryManager::handle_page_fault, which would just bail and - // request a crash - if (handle_safe_access_fault(regs, fault_address)) - return; - } - - auto current_thread = Thread::current(); - - if (current_thread) { - current_thread->set_handling_page_fault(true); - PerformanceManager::add_page_fault_event(*current_thread, regs); - } - - ScopeGuard guard = [current_thread] { - if (current_thread) - current_thread->set_handling_page_fault(false); - }; - - VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() }; - - if (!faulted_in_kernel) { - bool has_valid_stack_pointer = current_thread->process().address_space().with([&](auto& space) { - return MM.validate_user_stack(*space, userspace_sp); - }); - if (!has_valid_stack_pointer) { - dbgln("Invalid stack pointer: {}", userspace_sp); - return handle_crash(regs, "Bad stack on page fault", SIGSEGV); - } - } - PageFault fault { regs.exception_code, VirtualAddress { fault_address } }; - auto response = MM.handle_page_fault(fault); - - if (response == PageFaultResponse::ShouldCrash || response == PageFaultResponse::OutOfMemory || response == PageFaultResponse::BusError) { - if (faulted_in_kernel && handle_safe_access_fault(regs, fault_address)) { - // If this would be a ring0 (kernel) fault and the fault was triggered by - // safe_memcpy, safe_strnlen, or safe_memset then we resume execution at - // the appropriate _fault label rather than crashing - return; - } - - if (response == PageFaultResponse::BusError && current_thread->has_signal_handler(SIGBUS)) { - current_thread->send_urgent_signal_to_self(SIGBUS); - return; - } - - if (response != PageFaultResponse::OutOfMemory && current_thread) { - if (current_thread->has_signal_handler(SIGSEGV)) { - current_thread->send_urgent_signal_to_self(SIGSEGV); - return; - } - } - - dbgln("Unrecoverable page fault, {}{}{} address {}", - fault.is_reserved_bit_violation() ? "reserved bit violation / " : "", - fault.is_instruction_fetch() ? "instruction fetch / " : "", - fault.is_write() ? "write to" : "read from", - VirtualAddress(fault_address)); - constexpr FlatPtr malloc_scrub_pattern = explode_byte(MALLOC_SCRUB_BYTE); - constexpr FlatPtr free_scrub_pattern = explode_byte(FREE_SCRUB_BYTE); - constexpr FlatPtr kmalloc_scrub_pattern = explode_byte(KMALLOC_SCRUB_BYTE); - constexpr FlatPtr kfree_scrub_pattern = explode_byte(KFREE_SCRUB_BYTE); - if (response == PageFaultResponse::BusError) { - dbgln("Note: Address {} is an access to an undefined memory range of an Inode-backed VMObject", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (malloc_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be uninitialized malloc() memory", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (free_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be recently free()'d memory", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (kmalloc_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be uninitialized kmalloc() memory", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (kfree_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be recently kfree()'d memory", VirtualAddress(fault_address)); - } else if (fault_address < 4096) { - dbgln("Note: Address {} looks like a possible nullptr dereference", VirtualAddress(fault_address)); - } else if constexpr (SANITIZE_PTRS) { - constexpr FlatPtr refptr_scrub_pattern = explode_byte(REFPTR_SCRUB_BYTE); - constexpr FlatPtr nonnullrefptr_scrub_pattern = explode_byte(NONNULLREFPTR_SCRUB_BYTE); - constexpr FlatPtr ownptr_scrub_pattern = explode_byte(OWNPTR_SCRUB_BYTE); - constexpr FlatPtr nonnullownptr_scrub_pattern = explode_byte(NONNULLOWNPTR_SCRUB_BYTE); - constexpr FlatPtr lockrefptr_scrub_pattern = explode_byte(LOCKREFPTR_SCRUB_BYTE); - constexpr FlatPtr nonnulllockrefptr_scrub_pattern = explode_byte(NONNULLLOCKREFPTR_SCRUB_BYTE); - - if ((fault_address & 0xffff0000) == (refptr_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (nonnullrefptr_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (ownptr_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be a recently destroyed OwnPtr", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (nonnullownptr_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be a recently destroyed NonnullOwnPtr", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (lockrefptr_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address)); - } else if ((fault_address & 0xffff0000) == (nonnulllockrefptr_scrub_pattern & 0xffff0000)) { - dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address)); - } - } - - if (current_thread) { - auto& current_process = current_thread->process(); - if (current_process.is_user_process()) { - auto fault_address_string = KString::formatted("{:p}", fault_address); - auto fault_address_view = fault_address_string.is_error() ? ""sv : fault_address_string.value()->view(); - (void)current_process.try_set_coredump_property("fault_address"sv, fault_address_view); - (void)current_process.try_set_coredump_property("fault_type"sv, fault.type() == PageFault::Type::PageNotPresent ? "NotPresent"sv : "ProtectionViolation"sv); - StringView fault_access; - if (fault.is_instruction_fetch()) - fault_access = "Execute"sv; - else - fault_access = fault.access() == PageFault::Access::Read ? "Read"sv : "Write"sv; - (void)current_process.try_set_coredump_property("fault_access"sv, fault_access); - } - } - - if (response == PageFaultResponse::BusError) - return handle_crash(regs, "Page Fault (Bus Error)", SIGBUS, false); - return handle_crash(regs, "Page Fault", SIGSEGV, response == PageFaultResponse::OutOfMemory); - } else if (response == PageFaultResponse::Continue) { - dbgln_if(PAGE_FAULT_DEBUG, "Continuing after resolved page fault"); - } else { - VERIFY_NOT_REACHED(); - } + fault.handle(regs); } EH_ENTRY_NO_CODE(1, debug); diff --git a/Kernel/CMakeLists.txt b/Kernel/CMakeLists.txt index 5db487d2c5..0da5fcea36 100644 --- a/Kernel/CMakeLists.txt +++ b/Kernel/CMakeLists.txt @@ -17,6 +17,7 @@ set(KERNEL_HEAP_SOURCES set(KERNEL_SOURCES AddressSanitizer.cpp + Arch/PageFault.cpp Bus/PCI/Controller/HostController.cpp Bus/PCI/Controller/MemoryBackedHostBridge.cpp Bus/PCI/Controller/VolumeManagementDevice.cpp