Kernel: Split prekernel exception level code into its own file

Kernel/Prekernel/Arch/aarch64/Aarch64_asm_utils.S

@@ -23,25 +23,20 @@ Lstart:
   bne Lstart
   ret
 
-.global return_from_el2
+.global enter_el2_from_el3
-.type return_from_el2, @function
+.type enter_el2_from_el3, @function
-return_from_el2:
+enter_el2_from_el3:
-  adr x0, jump_to_os_start
+  adr x0, entered_el2
-  msr elr_el2, x0
-  eret
-
-.global return_from_el3
-.type return_from_el3, @function
-return_from_el3:
-  adr x0, jump_to_os_start
   msr elr_el3, x0
   eret
+entered_el2:
+  ret
 
-jump_to_os_start:
+.global enter_el1_from_el2
-  // Let stack start before .text for now.
+.type enter_el1_from_el2, @function
-  // 512 kiB (0x80000) of stack are probably not sufficient, especially once we give the other cores some stack too,
+enter_el1_from_el2:
-  // but for now it's ok.
+  adr x0, entered_el1
-  ldr x14, =start
+  msr elr_el2, x0
-  mov sp, x14
+  eret
-
+entered_el1:
-  b os_start
+  ret

Kernel/Prekernel/Arch/aarch64/Aarch64_asm_utils.h

@@ -6,7 +6,14 @@
 
 #pragma once
 
-extern "C" uint8_t get_current_exception_level();
+enum class ExceptionLevel : u8 {
+    EL0 = 0,
+    EL1 = 1,
+    EL2 = 2,
+    EL3 = 3,
+};
+
+extern "C" ExceptionLevel get_current_exception_level();
 extern "C" void wait_cycles(int n);
 
 // CPU initialization functions

Kernel/Prekernel/Arch/aarch64/Prekernel.h

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2021, James Mintram <me@jamesrm.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+namespace Prekernel {
+
+void drop_to_exception_level_1();
+void init_prekernel_page_tables();
+
+[[noreturn]] void panic(const char* msg);
+
+[[noreturn]] inline void halt()
+{
+    for (;;) {
+        asm volatile("wfi");
+    }
+}
+
+}

Kernel/Prekernel/Arch/aarch64/PrekernelCommon.cpp

@@ -0,0 +1,25 @@
+/*
+ * Copyright (c) 2021, James Mintram <me@jamesrm.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <Kernel/Prekernel/Arch/aarch64/Prekernel.h>
+
+#include <Kernel/Arch/aarch64/Aarch64Asm.h>
+#include <Kernel/Prekernel/Arch/aarch64/UART.h>
+
+namespace Prekernel {
+
+[[noreturn]] void panic(const char* msg)
+{
+    auto& uart = Prekernel::UART::the();
+
+    if (msg) {
+        uart.print_str(msg);
+    }
+
+    Kernel::halt();
+}
+
+}

Kernel/Prekernel/Arch/aarch64/PrekernelExceptions.cpp

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2021, James Mintram <me@jamesrm.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <Kernel/Arch/aarch64/Aarch64Registers.h>
+#include <Kernel/Prekernel/Arch/aarch64/Aarch64_asm_utils.h>
+#include <Kernel/Prekernel/Arch/aarch64/Prekernel.h>
+
+extern "C" void enter_el2_from_el3();
+extern "C" void enter_el1_from_el2();
+
+using namespace Kernel;
+
+namespace Prekernel {
+
+static void drop_to_el2()
+{
+    Aarch64_SCR_EL3 secure_configuration_register_el3 = {};
+
+    secure_configuration_register_el3.ST = 1;  // Don't trap access to Counter-timer Physical Secure registers
+    secure_configuration_register_el3.RW = 1;  // Lower level to use Aarch64
+    secure_configuration_register_el3.NS = 1;  // Non-secure state
+    secure_configuration_register_el3.HCE = 1; // Enable Hypervisor instructions at all levels
+
+    Aarch64_SCR_EL3::write(secure_configuration_register_el3);
+
+    Aarch64_SPSR_EL3 saved_program_status_register_el3 = {};
+
+    // Mask (disable) all interrupts
+    saved_program_status_register_el3.A = 1;
+    saved_program_status_register_el3.I = 1;
+    saved_program_status_register_el3.F = 1;
+    saved_program_status_register_el3.D = 1;
+
+    // Indicate EL1 as exception origin mode (so we go back there)
+    saved_program_status_register_el3.M = Aarch64_SPSR_EL3::Mode::EL2t;
+
+    // Set the register
+    Aarch64_SPSR_EL3::write(saved_program_status_register_el3);
+
+    // This will jump into os_start() below
+    enter_el2_from_el3();
+}
+static void drop_to_el1()
+{
+    Aarch64_HCR_EL2 hypervisor_configuration_register_el2 = {};
+    hypervisor_configuration_register_el2.RW = 1; // EL1 to use 64-bit mode
+    Aarch64_HCR_EL2::write(hypervisor_configuration_register_el2);
+
+    Aarch64_SPSR_EL2 saved_program_status_register_el2 = {};
+
+    // Mask (disable) all interrupts
+    saved_program_status_register_el2.A = 1;
+    saved_program_status_register_el2.I = 1;
+    saved_program_status_register_el2.F = 1;
+
+    // Indicate EL1 as exception origin mode (so we go back there)
+    saved_program_status_register_el2.M = Aarch64_SPSR_EL2::Mode::EL1t;
+
+    Aarch64_SPSR_EL2::write(saved_program_status_register_el2);
+    enter_el1_from_el2();
+}
+
+static void set_up_el1()
+{
+    Aarch64_SCTLR_EL1 system_control_register_el1 = Aarch64_SCTLR_EL1::reset_value();
+
+    system_control_register_el1.UCT = 1;  // Don't trap access to CTR_EL0
+    system_control_register_el1.nTWE = 1; // Don't trap WFE instructions
+    system_control_register_el1.nTWI = 1; // Don't trap WFI instructions
+    system_control_register_el1.DZE = 1;  // Don't trap DC ZVA instructions
+    system_control_register_el1.UMA = 1;  // Don't trap access to DAIF (debugging) flags of EFLAGS register
+    system_control_register_el1.SA0 = 1;  // Enable stack access alignment check for EL0
+    system_control_register_el1.SA = 1;   // Enable stack access alignment check for EL1
+    system_control_register_el1.A = 1;    // Enable memory access alignment check
+
+    Aarch64_SCTLR_EL1::write(system_control_register_el1);
+}
+
+void drop_to_exception_level_1()
+{
+    switch (get_current_exception_level()) {
+    case ExceptionLevel::EL3:
+        drop_to_el2();
+        [[fallthrough]];
+    case ExceptionLevel::EL2:
+        drop_to_el1();
+        [[fallthrough]];
+    case ExceptionLevel::EL1:
+        set_up_el1();
+        break;
+    default: {
+        Prekernel::panic("FATAL: CPU booted in unsupported exception mode!\r\n");
+    }
+    }
+}
+
+}

Kernel/Prekernel/Arch/aarch64/init.cpp

@@ -6,27 +6,21 @@
  */
 
 #include <AK/Types.h>
-#include <Kernel/Arch/aarch64/Aarch64Registers.h>
+
 #include <Kernel/Prekernel/Arch/aarch64/Aarch64_asm_utils.h>
 #include <Kernel/Prekernel/Arch/aarch64/BootPPMParser.h>
 #include <Kernel/Prekernel/Arch/aarch64/Framebuffer.h>
 #include <Kernel/Prekernel/Arch/aarch64/Mailbox.h>
+#include <Kernel/Prekernel/Arch/aarch64/Prekernel.h>
 #include <Kernel/Prekernel/Arch/aarch64/Timer.h>
 #include <Kernel/Prekernel/Arch/aarch64/UART.h>
 #include <Kernel/Prekernel/Arch/aarch64/Utils.h>
 
-extern "C" [[noreturn]] void halt();
-extern "C" [[noreturn]] void init();
-extern "C" [[noreturn]] void os_start();
-
-static void set_up_el1_mode();
-static void set_up_el2_mode();
-static void set_up_el3_mode();
-static void print_current_exception_level(const char* msg);
 static void draw_logo();
 static u32 query_firmware_version();
-[[noreturn]] static void jump_to_os_start_from_el2();
+
-[[noreturn]] static void jump_to_os_start_from_el3();
+extern "C" [[noreturn]] void halt();
+extern "C" [[noreturn]] void init();
 
 extern "C" [[noreturn]] void init()
 {
@@ -41,31 +35,12 @@ extern "C" [[noreturn]] void init()
     uart.print_num(firmware_version);
     uart.print_str("\r\n");
 
-    print_current_exception_level("CPU started in:");
+    uart.print_str("CPU started in: EL");
+    uart.print_num(static_cast<u64>(get_current_exception_level()));
+    uart.print_str("\r\n");
 
-    set_up_el2_mode();
+    uart.print_str("Drop CPU to EL1\r\n");
-    set_up_el1_mode();
+    Prekernel::drop_to_exception_level_1();
-
-    auto current_exception_level = get_current_exception_level();
-    switch (current_exception_level) {
-    case 2:
-        jump_to_os_start_from_el2();
-        break;
-    case 3:
-        set_up_el3_mode();
-        jump_to_os_start_from_el3();
-        break;
-    default:
-        uart.print_str("FATAL: CPU booted in unsupported exception mode!\r\n");
-        halt();
-    }
-}
-
-extern "C" [[noreturn]] void os_start()
-{
-    auto& uart = Prekernel::UART::the();
-
-    print_current_exception_level("CPU switched to:");
 
     auto& framebuffer = Prekernel::Framebuffer::the();
     if (framebuffer.initialized()) {
@@ -107,118 +82,6 @@ void __stack_chk_fail()
     halt();
 }
 
-static void set_up_el1_mode()
-{
-    Kernel::Aarch64_SCTLR_EL1 system_control_register_el1 = {};
-
-    // Those bits are reserved on ARMv8.0
-    system_control_register_el1.LSMAOE = 1;
-    system_control_register_el1.nTLSMD = 1;
-    system_control_register_el1.SPAN = 1;
-    system_control_register_el1.IESB = 1;
-
-    // Don't trap access to CTR_EL0
-    system_control_register_el1.UCT = 1;
-
-    // Don't trap WFE instructions
-    system_control_register_el1.nTWE = 1;
-
-    // Don't trap WFI instructions
-    system_control_register_el1.nTWI = 1;
-
-    // Don't trap DC ZVA instructions
-    system_control_register_el1.DZE = 1;
-
-    // Don't trap access to DAIF (debugging) flags of EFLAGS register
-    system_control_register_el1.UMA = 1;
-
-    // Enable stack access alignment check for EL0
-    system_control_register_el1.SA0 = 1;
-
-    // Enable stack access alignment check for EL1
-    system_control_register_el1.SA = 1;
-
-    // Enable memory access alignment check
-    system_control_register_el1.A = 1;
-
-    Kernel::Aarch64_SCTLR_EL1::write(system_control_register_el1);
-}
-
-static void set_up_el2_mode()
-{
-    Kernel::Aarch64_HCR_EL2 hypervisor_configuration_register_el2 = {};
-
-    // EL1 to use 64-bit mode
-    hypervisor_configuration_register_el2.RW = 1;
-
-    Kernel::Aarch64_HCR_EL2::write(hypervisor_configuration_register_el2);
-}
-
-static void set_up_el3_mode()
-{
-    Kernel::Aarch64_SCR_EL3 secure_configuration_register_el3 = {};
-
-    // Don't trap access to Counter-timer Physical Secure registers
-    secure_configuration_register_el3.ST = 1;
-
-    // Lower level to use Aarch64
-    secure_configuration_register_el3.RW = 1;
-
-    // Enable Hypervisor instructions at all levels
-    secure_configuration_register_el3.HCE = 1;
-
-    Kernel::Aarch64_SCR_EL3::write(secure_configuration_register_el3);
-}
-
-[[noreturn]] static void jump_to_os_start_from_el2()
-{
-    // Processor state to set when returned from this function (in new EL1 world)
-    Kernel::Aarch64_SPSR_EL2 saved_program_status_register_el2 = {};
-
-    // Mask (disable) all interrupts
-    saved_program_status_register_el2.A = 1;
-    saved_program_status_register_el2.I = 1;
-    saved_program_status_register_el2.F = 1;
-
-    // Indicate EL1 as exception origin mode (so we go back there)
-    saved_program_status_register_el2.M = Kernel::Aarch64_SPSR_EL2::Mode::EL1h;
-
-    Kernel::Aarch64_SPSR_EL2::write(saved_program_status_register_el2);
-
-    // This will jump into os_start()
-    return_from_el2();
-}
-
-[[noreturn]] static void jump_to_os_start_from_el3()
-{
-    // Processor state to set when returned from this function (in new EL1 world)
-    Kernel::Aarch64_SPSR_EL3 saved_program_status_register_el3 = {};
-
-    // Mask (disable) all interrupts
-    saved_program_status_register_el3.A = 1;
-    saved_program_status_register_el3.I = 1;
-    saved_program_status_register_el3.F = 1;
-
-    // Indicate EL1 as exception origin mode (so we go back there)
-    saved_program_status_register_el3.M = Kernel::Aarch64_SPSR_EL3::Mode::EL1h;
-
-    Kernel::Aarch64_SPSR_EL3::write(saved_program_status_register_el3);
-
-    // This will jump into os_start() below
-    return_from_el3();
-}
-
-static void print_current_exception_level(const char* msg)
-{
-    auto& uart = Prekernel::UART::the();
-
-    auto exception_level = get_current_exception_level();
-    uart.print_str(msg);
-    uart.print_str(" EL");
-    uart.print_num(exception_level);
-    uart.print_str("\r\n");
-}
-
 class QueryFirmwareVersionMboxMessage : Prekernel::Mailbox::Message {
 public:
     u32 version;

Kernel/Prekernel/CMakeLists.txt

@@ -14,9 +14,15 @@ if ("${SERENITY_ARCH}" STREQUAL "aarch64")
         Arch/aarch64/Timer.cpp
         Arch/aarch64/UART.cpp
         Arch/aarch64/Utils.cpp
-        Arch/aarch64/Aarch64_asm_utils.S
+
-        Arch/aarch64/boot.S
+        # Preload specific
         Arch/aarch64/init.cpp
+        Arch/aarch64/PrekernelExceptions.cpp
+        Arch/aarch64/PrekernelCommon.cpp
+
+        # Assembly
+        Arch/aarch64/boot.S
+        Arch/aarch64/Aarch64_asm_utils.S
     )
 else()
     set(SOURCES