Kernel/aarch64: Implement initial page fault handling

The shared code is moved to a common PageFault.cpp file.

Kernel/Arch/PageFault.cpp

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <Kernel/Arch/CPU.h>
+#include <Kernel/Arch/PageFault.h>
+#include <Kernel/Arch/Processor.h>
+#include <Kernel/Arch/RegisterState.h>
+#include <Kernel/Arch/SafeMem.h>
+#include <Kernel/PerformanceManager.h>
+#include <Kernel/Thread.h>
+#include <LibC/mallocdefs.h>
+
+namespace Kernel {
+
+void PageFault::handle(RegisterState& regs)
+{
+    auto fault_address = m_vaddr.get();
+    bool faulted_in_kernel = regs.previous_mode() == ExecutionMode::Kernel;
+
+    if (faulted_in_kernel && Processor::current_in_irq()) {
+        // If we're faulting in an IRQ handler, first check if we failed
+        // due to safe_memcpy, safe_strnlen, or safe_memset. If we did,
+        // gracefully continue immediately. Because we're in an IRQ handler
+        // we can't really try to resolve the page fault in a meaningful
+        // way, so we need to do this before calling into
+        // MemoryManager::handle_page_fault, which would just bail and
+        // request a crash
+        if (handle_safe_access_fault(regs, fault_address))
+            return;
+    }
+
+    auto current_thread = Thread::current();
+
+    if (current_thread) {
+        current_thread->set_handling_page_fault(true);
+        PerformanceManager::add_page_fault_event(*current_thread, regs);
+    }
+
+    ScopeGuard guard = [current_thread] {
+        if (current_thread)
+            current_thread->set_handling_page_fault(false);
+    };
+
+    if (!faulted_in_kernel) {
+        VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
+        bool has_valid_stack_pointer = current_thread->process().address_space().with([&](auto& space) {
+            return MM.validate_user_stack(*space, userspace_sp);
+        });
+        if (!has_valid_stack_pointer) {
+            dbgln("Invalid stack pointer: {}", userspace_sp);
+            return handle_crash(regs, "Bad stack on page fault", SIGSEGV);
+        }
+    }
+
+    auto response = MM.handle_page_fault(*this);
+
+    if (response == PageFaultResponse::ShouldCrash || response == PageFaultResponse::OutOfMemory || response == PageFaultResponse::BusError) {
+        if (faulted_in_kernel && handle_safe_access_fault(regs, fault_address)) {
+            // If this would be a ring0 (kernel) fault and the fault was triggered by
+            // safe_memcpy, safe_strnlen, or safe_memset then we resume execution at
+            // the appropriate _fault label rather than crashing
+            return;
+        }
+
+        if (response == PageFaultResponse::BusError && current_thread->has_signal_handler(SIGBUS)) {
+            current_thread->send_urgent_signal_to_self(SIGBUS);
+            return;
+        }
+
+        if (response != PageFaultResponse::OutOfMemory && current_thread) {
+            if (current_thread->has_signal_handler(SIGSEGV)) {
+                current_thread->send_urgent_signal_to_self(SIGSEGV);
+                return;
+            }
+        }
+
+        dbgln("Unrecoverable page fault, {}{}{} address {}",
+            is_reserved_bit_violation() ? "reserved bit violation / " : "",
+            is_instruction_fetch() ? "instruction fetch / " : "",
+            is_write() ? "write to" : "read from",
+            VirtualAddress(fault_address));
+        constexpr FlatPtr malloc_scrub_pattern = explode_byte(MALLOC_SCRUB_BYTE);
+        constexpr FlatPtr free_scrub_pattern = explode_byte(FREE_SCRUB_BYTE);
+        constexpr FlatPtr kmalloc_scrub_pattern = explode_byte(KMALLOC_SCRUB_BYTE);
+        constexpr FlatPtr kfree_scrub_pattern = explode_byte(KFREE_SCRUB_BYTE);
+        if (response == PageFaultResponse::BusError) {
+            dbgln("Note: Address {} is an access to an undefined memory range of an Inode-backed VMObject", VirtualAddress(fault_address));
+        } else if ((fault_address & 0xffff0000) == (malloc_scrub_pattern & 0xffff0000)) {
+            dbgln("Note: Address {} looks like it may be uninitialized malloc() memory", VirtualAddress(fault_address));
+        } else if ((fault_address & 0xffff0000) == (free_scrub_pattern & 0xffff0000)) {
+            dbgln("Note: Address {} looks like it may be recently free()'d memory", VirtualAddress(fault_address));
+        } else if ((fault_address & 0xffff0000) == (kmalloc_scrub_pattern & 0xffff0000)) {
+            dbgln("Note: Address {} looks like it may be uninitialized kmalloc() memory", VirtualAddress(fault_address));
+        } else if ((fault_address & 0xffff0000) == (kfree_scrub_pattern & 0xffff0000)) {
+            dbgln("Note: Address {} looks like it may be recently kfree()'d memory", VirtualAddress(fault_address));
+        } else if (fault_address < 4096) {
+            dbgln("Note: Address {} looks like a possible nullptr dereference", VirtualAddress(fault_address));
+        } else if constexpr (SANITIZE_PTRS) {
+            constexpr FlatPtr refptr_scrub_pattern = explode_byte(REFPTR_SCRUB_BYTE);
+            constexpr FlatPtr nonnullrefptr_scrub_pattern = explode_byte(NONNULLREFPTR_SCRUB_BYTE);
+            constexpr FlatPtr ownptr_scrub_pattern = explode_byte(OWNPTR_SCRUB_BYTE);
+            constexpr FlatPtr nonnullownptr_scrub_pattern = explode_byte(NONNULLOWNPTR_SCRUB_BYTE);
+            constexpr FlatPtr lockrefptr_scrub_pattern = explode_byte(LOCKREFPTR_SCRUB_BYTE);
+            constexpr FlatPtr nonnulllockrefptr_scrub_pattern = explode_byte(NONNULLLOCKREFPTR_SCRUB_BYTE);
+
+            if ((fault_address & 0xffff0000) == (refptr_scrub_pattern & 0xffff0000)) {
+                dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
+            } else if ((fault_address & 0xffff0000) == (nonnullrefptr_scrub_pattern & 0xffff0000)) {
+                dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
+            } else if ((fault_address & 0xffff0000) == (ownptr_scrub_pattern & 0xffff0000)) {
+                dbgln("Note: Address {} looks like it may be a recently destroyed OwnPtr", VirtualAddress(fault_address));
+            } else if ((fault_address & 0xffff0000) == (nonnullownptr_scrub_pattern & 0xffff0000)) {
+                dbgln("Note: Address {} looks like it may be a recently destroyed NonnullOwnPtr", VirtualAddress(fault_address));
+            } else if ((fault_address & 0xffff0000) == (lockrefptr_scrub_pattern & 0xffff0000)) {
+                dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
+            } else if ((fault_address & 0xffff0000) == (nonnulllockrefptr_scrub_pattern & 0xffff0000)) {
+                dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
+            }
+        }
+
+        if (current_thread) {
+            auto& current_process = current_thread->process();
+            if (current_process.is_user_process()) {
+                auto fault_address_string = KString::formatted("{:p}", fault_address);
+                auto fault_address_view = fault_address_string.is_error() ? ""sv : fault_address_string.value()->view();
+                (void)current_process.try_set_coredump_property("fault_address"sv, fault_address_view);
+                (void)current_process.try_set_coredump_property("fault_type"sv, type() == PageFault::Type::PageNotPresent ? "NotPresent"sv : "ProtectionViolation"sv);
+                StringView fault_access;
+                if (is_instruction_fetch())
+                    fault_access = "Execute"sv;
+                else
+                    fault_access = access() == PageFault::Access::Read ? "Read"sv : "Write"sv;
+                (void)current_process.try_set_coredump_property("fault_access"sv, fault_access);
+            }
+        }
+
+        if (response == PageFaultResponse::BusError)
+            return handle_crash(regs, "Page Fault (Bus Error)", SIGBUS, false);
+        return handle_crash(regs, "Page Fault", SIGSEGV, response == PageFaultResponse::OutOfMemory);
+    } else if (response == PageFaultResponse::Continue) {
+        dbgln_if(PAGE_FAULT_DEBUG, "Continuing after resolved page fault");
+    } else {
+        VERIFY_NOT_REACHED();
+    }
+}
+
+}

Kernel/Arch/PageFault.h

@@ -8,6 +8,7 @@
 
 #include <AK/Platform.h>
 #include <AK/Types.h>
+#include <Kernel/Arch/RegisterState.h>
 #include <Kernel/ExecutionMode.h>
 #include <Kernel/VirtualAddress.h>
 
@@ -44,6 +45,8 @@ public:
     {
     }
 
+    void handle(RegisterState& regs);
+
     enum class Type {
         PageNotPresent = PageFaultFlags::NotPresent,
         ProtectionViolation = PageFaultFlags::ProtectionViolation,

Kernel/Arch/aarch64/Interrupts.cpp

@@ -5,12 +5,14 @@
  */
 
 #include <Kernel/Arch/Interrupts.h>
+#include <Kernel/Arch/PageFault.h>
 #include <Kernel/Arch/TrapFrame.h>
 #include <Kernel/Arch/aarch64/InterruptManagement.h>
 #include <Kernel/Interrupts/GenericInterruptHandler.h>
 #include <Kernel/Interrupts/SharedIRQHandler.h>
 #include <Kernel/Interrupts/UnhandledInterruptHandler.h>
 #include <Kernel/KSyms.h>
+#include <Kernel/Panic.h>
 #include <Kernel/StdLib.h>
 
 namespace Kernel {
@@ -53,10 +55,35 @@ static void dump_exception_syndrome_register(Aarch64::ESR_EL1 const& esr_el1)
         dbgln("Data Fault Status Code: {}", Aarch64::data_fault_status_code_to_string(esr_el1.ISS));
 }
 
-extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame);
+static PageFault page_fault_from_exception_syndrome_register(VirtualAddress fault_address, Aarch64::ESR_EL1 esr_el1)
-extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame)
 {
+    PageFault fault { fault_address };
+
+    u8 data_fault_status_code = esr_el1.ISS & 0x3f;
+    if (data_fault_status_code >= 0b001100 && data_fault_status_code <= 0b001111) {
+        fault.set_type(PageFault::Type::ProtectionViolation);
+    } else if (data_fault_status_code >= 0b000100 && data_fault_status_code <= 0b000111) {
+        fault.set_type(PageFault::Type::PageNotPresent);
+    } else {
+        PANIC("Unknown DFSC: {}", data_fault_status_code);
+    }
+
+    fault.set_access((esr_el1.ISS & (1 << 6)) == (1 << 6) ? PageFault::Access::Write : PageFault::Access::Read);
+
+    // FIXME: Set correct mode
+    fault.set_mode(ExecutionMode::Kernel);
+
+    return fault;
+}
+
+extern "C" void exception_common(Kernel::TrapFrame* trap_frame);
+extern "C" void exception_common(Kernel::TrapFrame* trap_frame)
+{
+    Processor::current().enter_trap(*trap_frame, false);
+
     auto esr_el1 = Kernel::Aarch64::ESR_EL1::read();
+    auto fault_address = Aarch64::FAR_EL1::read().virtual_address;
+    Processor::enable_interrupts();
 
     constexpr bool print_state = true;
     if constexpr (print_state) {
@@ -65,7 +92,17 @@ extern "C" void exception_common(Kernel::TrapFrame const* const trap_frame)
         dump_exception_syndrome_register(esr_el1);
     }
 
-    Kernel::Processor::halt();
+    if (Aarch64::exception_class_is_data_abort(esr_el1.EC)) {
+        auto page_fault = page_fault_from_exception_syndrome_register(VirtualAddress(fault_address), esr_el1);
+        page_fault.handle(*trap_frame->regs);
+    } else {
+        dump_registers(*trap_frame->regs);
+        dump_exception_syndrome_register(esr_el1);
+        PANIC("Unexpected exception!");
+    }
+
+    Processor::disable_interrupts();
+    Processor::current().exit_trap(*trap_frame);
 }
 
 static Array<GenericInterruptHandler*, 64> s_interrupt_handlers;

Kernel/Arch/x86_64/Interrupts.cpp

@@ -23,8 +23,6 @@
 #include <Kernel/Thread.h>
 #include <Kernel/ThreadTracer.h>
 
-#include <LibC/mallocdefs.h>
-
 #include <Kernel/Arch/CPU.h>
 #include <Kernel/Arch/PageFault.h>
 #include <Kernel/Arch/Processor.h>
@@ -201,135 +199,8 @@ void page_fault_handler(TrapFrame* trap)
         dump_registers(regs);
     }
 
-    bool faulted_in_kernel = !(regs.cs & 3);
-
-    if (faulted_in_kernel && Processor::current_in_irq()) {
-        // If we're faulting in an IRQ handler, first check if we failed
-        // due to safe_memcpy, safe_strnlen, or safe_memset. If we did,
-        // gracefully continue immediately. Because we're in an IRQ handler
-        // we can't really try to resolve the page fault in a meaningful
-        // way, so we need to do this before calling into
-        // MemoryManager::handle_page_fault, which would just bail and
-        // request a crash
-        if (handle_safe_access_fault(regs, fault_address))
-            return;
-    }
-
-    auto current_thread = Thread::current();
-
-    if (current_thread) {
-        current_thread->set_handling_page_fault(true);
-        PerformanceManager::add_page_fault_event(*current_thread, regs);
-    }
-
-    ScopeGuard guard = [current_thread] {
-        if (current_thread)
-            current_thread->set_handling_page_fault(false);
-    };
-
-    VirtualAddress userspace_sp = VirtualAddress { regs.userspace_sp() };
-
-    if (!faulted_in_kernel) {
-        bool has_valid_stack_pointer = current_thread->process().address_space().with([&](auto& space) {
-            return MM.validate_user_stack(*space, userspace_sp);
-        });
-        if (!has_valid_stack_pointer) {
-            dbgln("Invalid stack pointer: {}", userspace_sp);
-            return handle_crash(regs, "Bad stack on page fault", SIGSEGV);
-        }
-    }
-
     PageFault fault { regs.exception_code, VirtualAddress { fault_address } };
-    auto response = MM.handle_page_fault(fault);
+    fault.handle(regs);
-
-    if (response == PageFaultResponse::ShouldCrash || response == PageFaultResponse::OutOfMemory || response == PageFaultResponse::BusError) {
-        if (faulted_in_kernel && handle_safe_access_fault(regs, fault_address)) {
-            // If this would be a ring0 (kernel) fault and the fault was triggered by
-            // safe_memcpy, safe_strnlen, or safe_memset then we resume execution at
-            // the appropriate _fault label rather than crashing
-            return;
-        }
-
-        if (response == PageFaultResponse::BusError && current_thread->has_signal_handler(SIGBUS)) {
-            current_thread->send_urgent_signal_to_self(SIGBUS);
-            return;
-        }
-
-        if (response != PageFaultResponse::OutOfMemory && current_thread) {
-            if (current_thread->has_signal_handler(SIGSEGV)) {
-                current_thread->send_urgent_signal_to_self(SIGSEGV);
-                return;
-            }
-        }
-
-        dbgln("Unrecoverable page fault, {}{}{} address {}",
-            fault.is_reserved_bit_violation() ? "reserved bit violation / " : "",
-            fault.is_instruction_fetch() ? "instruction fetch / " : "",
-            fault.is_write() ? "write to" : "read from",
-            VirtualAddress(fault_address));
-        constexpr FlatPtr malloc_scrub_pattern = explode_byte(MALLOC_SCRUB_BYTE);
-        constexpr FlatPtr free_scrub_pattern = explode_byte(FREE_SCRUB_BYTE);
-        constexpr FlatPtr kmalloc_scrub_pattern = explode_byte(KMALLOC_SCRUB_BYTE);
-        constexpr FlatPtr kfree_scrub_pattern = explode_byte(KFREE_SCRUB_BYTE);
-        if (response == PageFaultResponse::BusError) {
-            dbgln("Note: Address {} is an access to an undefined memory range of an Inode-backed VMObject", VirtualAddress(fault_address));
-        } else if ((fault_address & 0xffff0000) == (malloc_scrub_pattern & 0xffff0000)) {
-            dbgln("Note: Address {} looks like it may be uninitialized malloc() memory", VirtualAddress(fault_address));
-        } else if ((fault_address & 0xffff0000) == (free_scrub_pattern & 0xffff0000)) {
-            dbgln("Note: Address {} looks like it may be recently free()'d memory", VirtualAddress(fault_address));
-        } else if ((fault_address & 0xffff0000) == (kmalloc_scrub_pattern & 0xffff0000)) {
-            dbgln("Note: Address {} looks like it may be uninitialized kmalloc() memory", VirtualAddress(fault_address));
-        } else if ((fault_address & 0xffff0000) == (kfree_scrub_pattern & 0xffff0000)) {
-            dbgln("Note: Address {} looks like it may be recently kfree()'d memory", VirtualAddress(fault_address));
-        } else if (fault_address < 4096) {
-            dbgln("Note: Address {} looks like a possible nullptr dereference", VirtualAddress(fault_address));
-        } else if constexpr (SANITIZE_PTRS) {
-            constexpr FlatPtr refptr_scrub_pattern = explode_byte(REFPTR_SCRUB_BYTE);
-            constexpr FlatPtr nonnullrefptr_scrub_pattern = explode_byte(NONNULLREFPTR_SCRUB_BYTE);
-            constexpr FlatPtr ownptr_scrub_pattern = explode_byte(OWNPTR_SCRUB_BYTE);
-            constexpr FlatPtr nonnullownptr_scrub_pattern = explode_byte(NONNULLOWNPTR_SCRUB_BYTE);
-            constexpr FlatPtr lockrefptr_scrub_pattern = explode_byte(LOCKREFPTR_SCRUB_BYTE);
-            constexpr FlatPtr nonnulllockrefptr_scrub_pattern = explode_byte(NONNULLLOCKREFPTR_SCRUB_BYTE);
-
-            if ((fault_address & 0xffff0000) == (refptr_scrub_pattern & 0xffff0000)) {
-                dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
-            } else if ((fault_address & 0xffff0000) == (nonnullrefptr_scrub_pattern & 0xffff0000)) {
-                dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
-            } else if ((fault_address & 0xffff0000) == (ownptr_scrub_pattern & 0xffff0000)) {
-                dbgln("Note: Address {} looks like it may be a recently destroyed OwnPtr", VirtualAddress(fault_address));
-            } else if ((fault_address & 0xffff0000) == (nonnullownptr_scrub_pattern & 0xffff0000)) {
-                dbgln("Note: Address {} looks like it may be a recently destroyed NonnullOwnPtr", VirtualAddress(fault_address));
-            } else if ((fault_address & 0xffff0000) == (lockrefptr_scrub_pattern & 0xffff0000)) {
-                dbgln("Note: Address {} looks like it may be a recently destroyed LockRefPtr", VirtualAddress(fault_address));
-            } else if ((fault_address & 0xffff0000) == (nonnulllockrefptr_scrub_pattern & 0xffff0000)) {
-                dbgln("Note: Address {} looks like it may be a recently destroyed NonnullLockRefPtr", VirtualAddress(fault_address));
-            }
-        }
-
-        if (current_thread) {
-            auto& current_process = current_thread->process();
-            if (current_process.is_user_process()) {
-                auto fault_address_string = KString::formatted("{:p}", fault_address);
-                auto fault_address_view = fault_address_string.is_error() ? ""sv : fault_address_string.value()->view();
-                (void)current_process.try_set_coredump_property("fault_address"sv, fault_address_view);
-                (void)current_process.try_set_coredump_property("fault_type"sv, fault.type() == PageFault::Type::PageNotPresent ? "NotPresent"sv : "ProtectionViolation"sv);
-                StringView fault_access;
-                if (fault.is_instruction_fetch())
-                    fault_access = "Execute"sv;
-                else
-                    fault_access = fault.access() == PageFault::Access::Read ? "Read"sv : "Write"sv;
-                (void)current_process.try_set_coredump_property("fault_access"sv, fault_access);
-            }
-        }
-
-        if (response == PageFaultResponse::BusError)
-            return handle_crash(regs, "Page Fault (Bus Error)", SIGBUS, false);
-        return handle_crash(regs, "Page Fault", SIGSEGV, response == PageFaultResponse::OutOfMemory);
-    } else if (response == PageFaultResponse::Continue) {
-        dbgln_if(PAGE_FAULT_DEBUG, "Continuing after resolved page fault");
-    } else {
-        VERIFY_NOT_REACHED();
-    }
 }
 
 EH_ENTRY_NO_CODE(1, debug);

Kernel/CMakeLists.txt

@@ -17,6 +17,7 @@ set(KERNEL_HEAP_SOURCES
 
 set(KERNEL_SOURCES
     AddressSanitizer.cpp
+    Arch/PageFault.cpp
     Bus/PCI/Controller/HostController.cpp
     Bus/PCI/Controller/MemoryBackedHostBridge.cpp
     Bus/PCI/Controller/VolumeManagementDevice.cpp