Kernel: Implement booting all CPU cores on x86_64

The AP boot code was partially adapted to build on x86_64 but didn't
properly jump into 64 bit mode. Furthermore, the APIC code was still
using 32 bit pointers.

Fixes #12662

Kernel/Arch/x86/x86_64/Boot/ap_setup.S

@@ -14,9 +14,6 @@ gdt64ptr:
 code64_sel:
 .short 0
 
-.extern init_ap
-.type init_ap, @function
-
 /*
   The apic_ap_start function will be loaded to P0x00008000 where the APIC
   will boot the AP from in real mode. This code also contains space for
@@ -31,6 +28,7 @@ code64_sel:
 */
 .global apic_ap_start
 .type apic_ap_start, @function
+.align 8
 apic_ap_start:
 .code16
     cli
@@ -68,15 +66,11 @@ apic_ap_start32:
     lock; xaddl %eax, (ap_cpu_id - apic_ap_start)(%ebp) /* avoid relocation entries */
     movl %eax, %esi
 
-    /* find our allocated stack based on the generated id */
-    movl (ap_cpu_init_stacks - apic_ap_start)(%ebp, %eax, 4), %esp
-
     /* check if we support NX and enable it if we do */
     movl $0x80000001, %eax
     cpuid
     testl $0x100000, %edx
-    // TODO: Uncomment this
+    je (1f - apic_ap_start + 0x8000)
-    //je (1f - apic_ap_start + 0x8000)
     /* turn on IA32_EFER.NXE */
     movl $0xc0000080, %ecx
     rdmsr
@@ -105,15 +99,25 @@ apic_ap_start32:
     movl %eax, %cr0
 
     /* load the temporary 64-bit gdt from boot that points above 3GB */
-    // FIXME: uncomment this
+    lgdt (ap_cpu_gdt64ptr - apic_ap_start + 0x8000)
-    //mov gdt64ptr, %eax
-    lgdt (%eax)
 
-    /* jump above 3GB into our identity mapped area now */
+    /* Jump into our identity mapped area, stay in low memory for now.
-    // FIXME: this assumes that code64_sel is always 8
+       We need to fully enable 64 bit mode before we can adjust rsp and rip
-    ljmpl $8, $(apic_ap_start64 - apic_ap_start + 0xc0008000)
+       to values higher than 4GB */
+    ljmpl $(ap_cpu_gdt64code - ap_cpu_gdt64), $(apic_ap_start64 - apic_ap_start + 0x8000)
 .code64
 apic_ap_start64:
+    movq (ap_cpu_kernel_map_base - apic_ap_start)(%rbp), %rbp
+    addq $0x8000, %rbp
+
+    /* find our allocated stack based on the generated id */
+    movq (ap_cpu_init_stacks - apic_ap_start)(%rbp, %rsi, 8), %rsp
+
+    /* Now jump from our identity mapped area into high memory */
+    movq $(1f - apic_ap_start), %rax
+    addq %rbp, %rax
+    jmp *%rax
+1:
     mov $0, %ax
     mov %ax, %ss
     mov %ax, %ds
@@ -125,8 +129,6 @@ apic_ap_start64:
     movq %cr3, %rax
     movq %rax, %cr3
 
-    movl $0xc0008000, %ebp
-
     /* now load the final gdt and idt from the identity mapped area */
     movq (ap_cpu_gdtr - apic_ap_start)(%rbp), %rax
     lgdt (%rax)
@@ -139,28 +141,25 @@ apic_ap_start64:
     movq (ap_cpu_init_cr4 - apic_ap_start)(%rbp), %rax
     movq %rax, %cr4
 
-    /* push the Processor pointer this CPU is going to use */
-    movq (ap_cpu_init_processor_info_array - apic_ap_start)(%ebp), %rax
-    leaq kernel_mapping_base(%rip), %r8
-    movq (%r8), %r8
-    addq %r8, %rax
-    movq 0(%rax, %rsi, 4), %rax
-    push %rax
 
-    /* push the cpu id, 0 representing the bsp and call into c++ */
+    /* Save the cpu id into rdi (first argument), 0 representing the bsp */
-    incq %rsi
+    movq %rsi, %rdi
-    push %rsi
+    incq %rdi
 
-    xor %ebp, %ebp
+    /* Save the Processor pointer this CPU is going to use into rsi (second argument) */
+    movq (ap_cpu_init_processor_info_array - apic_ap_start)(%rbp), %rax
+    movq 0(%rax, %rsi, 8), %rsi
+
+    /* Get the entry function */
+    movq (ap_cpu_kernel_entry_function - apic_ap_start)(%rbp), %r10
+
+    movq %rsp, %rbp
     cld
 
     /* We are in identity mapped P0x8000 and the BSP will unload this code
-       once all APs are initialized, so call init_ap but return to our
+       once all APs are initialized, so call the entry function and return to our
-       infinite loop */
+       infinite loop if it ever were to return. */
-    leaq loop(%rip), %rax
+    call *%r10
-    pushq %rax
-    leaq init_ap(%rip), %rax
-    jmp *(%rax)
 
 loop:
     hlt
@@ -186,6 +185,7 @@ ap_cpu_gdt_end:
 ap_cpu_gdtr_initial:
     .2byte ap_cpu_gdt_end - ap_cpu_gdt - 1
     .4byte (ap_cpu_gdt - apic_ap_start) + 0x8000
+.align 8
 .global ap_cpu_gdtr
 ap_cpu_gdtr:
     .8byte 0x0 /* will be set at runtime */
@@ -201,6 +201,28 @@ ap_cpu_init_cr3:
 .global ap_cpu_init_cr4
 ap_cpu_init_cr4:
     .8byte 0x0 /* will be set at runtime */
+.global ap_cpu_gdt64
+ap_cpu_gdt64:
+    .8byte 0x0
+.global ap_cpu_gdt64code
+ap_cpu_gdt64code:
+    .4byte 0xffff
+    .4byte 0xaf9a00
+.global ap_cpu_gdt64data
+ap_cpu_gdt64data:
+    .4byte 0xffff
+    .4byte 0xaf9200
+.global ap_cpu_gdt64ptr
+ap_cpu_gdt64ptr:
+    .2byte ap_cpu_gdt64ptr - ap_cpu_gdt64 - 1
+    .8byte (ap_cpu_gdt64 - apic_ap_start) + 0x8000
+.align 8
+.global ap_cpu_kernel_entry_function
+ap_cpu_kernel_entry_function:
+    .8byte 0x0 /* will be set at runtime */
+.global ap_cpu_kernel_map_base
+ap_cpu_kernel_map_base:
+    .8byte 0x0 /* will be set at runtime */
 .global ap_cpu_init_processor_info_array
 ap_cpu_init_processor_info_array:
     .8byte 0x0 /* will be set at runtime */

Kernel/Interrupts/APIC.cpp

@@ -211,13 +211,19 @@ void APIC::write_icr(const ICRReg& icr)
 
 extern "C" void apic_ap_start(void);
 extern "C" u16 apic_ap_start_size;
-extern "C" u32 ap_cpu_init_stacks;
+extern "C" FlatPtr ap_cpu_init_stacks;
-extern "C" u32 ap_cpu_init_processor_info_array;
+extern "C" FlatPtr ap_cpu_init_processor_info_array;
 extern "C" u32 ap_cpu_init_cr0;
-extern "C" u32 ap_cpu_init_cr3;
+extern "C" FlatPtr ap_cpu_init_cr3;
 extern "C" u32 ap_cpu_init_cr4;
-extern "C" u32 ap_cpu_gdtr;
+extern "C" FlatPtr ap_cpu_gdtr;
-extern "C" u32 ap_cpu_idtr;
+extern "C" FlatPtr ap_cpu_idtr;
+#if ARCH(X86_64)
+extern "C" FlatPtr ap_cpu_kernel_map_base;
+extern "C" FlatPtr ap_cpu_kernel_entry_function;
+#endif
+
+extern "C" [[noreturn]] void init_ap(FlatPtr, Processor*);
 
 void APIC::eoi()
 {
@@ -341,7 +347,8 @@ UNMAP_AFTER_INIT void APIC::setup_ap_boot_environment()
     constexpr u64 apic_startup_region_base = 0x8000;
     VERIFY(apic_startup_region_base + apic_ap_start_size < USER_RANGE_BASE);
     auto apic_startup_region = create_identity_mapped_region(PhysicalAddress(apic_startup_region_base), Memory::page_round_up(apic_ap_start_size + (2 * aps_to_enable * sizeof(u32))).release_value_but_fixme_should_propagate_errors());
-    memcpy(apic_startup_region->vaddr().as_ptr(), reinterpret_cast<const void*>(apic_ap_start), apic_ap_start_size);
+    u8* apic_startup_region_ptr = apic_startup_region->vaddr().as_ptr();
+    memcpy(apic_startup_region_ptr, reinterpret_cast<const void*>(apic_ap_start), apic_ap_start_size);
 
     // Allocate enough stacks for all APs
     m_ap_temporary_boot_stacks.ensure_capacity(aps_to_enable);
@@ -357,7 +364,7 @@ UNMAP_AFTER_INIT void APIC::setup_ap_boot_environment()
     }
 
     // Store pointers to all stacks for the APs to use
-    auto* ap_stack_array = APIC_INIT_VAR_PTR(u32, apic_startup_region->vaddr().as_ptr(), ap_cpu_init_stacks);
+    auto* ap_stack_array = APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_init_stacks);
     VERIFY(aps_to_enable == m_ap_temporary_boot_stacks.size());
     for (size_t i = 0; i < aps_to_enable; i++) {
         ap_stack_array[i] = m_ap_temporary_boot_stacks[i]->vaddr().get() + Thread::default_kernel_stack_size;
@@ -373,20 +380,26 @@ UNMAP_AFTER_INIT void APIC::setup_ap_boot_environment()
         ap_processor_info_array[i] = FlatPtr(m_ap_processor_info[i].ptr());
         dbgln_if(APIC_DEBUG, "APIC: CPU[{}] processor at {}", i + 1, VirtualAddress { ap_processor_info_array[i] });
     }
-    *APIC_INIT_VAR_PTR(u32, apic_startup_region->vaddr().as_ptr(), ap_cpu_init_processor_info_array) = FlatPtr(&ap_processor_info_array[0]);
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_init_processor_info_array) = FlatPtr(&ap_processor_info_array[0]);
 
     // Store the BSP's CR3 value for the APs to use
-    *APIC_INIT_VAR_PTR(u32, apic_startup_region->vaddr().as_ptr(), ap_cpu_init_cr3) = MM.kernel_page_directory().cr3();
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_init_cr3) = MM.kernel_page_directory().cr3();
 
     // Store the BSP's GDT and IDT for the APs to use
     const auto& gdtr = Processor::current().get_gdtr();
-    *APIC_INIT_VAR_PTR(u32, apic_startup_region->vaddr().as_ptr(), ap_cpu_gdtr) = FlatPtr(&gdtr);
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_gdtr) = FlatPtr(&gdtr);
     const auto& idtr = get_idtr();
-    *APIC_INIT_VAR_PTR(u32, apic_startup_region->vaddr().as_ptr(), ap_cpu_idtr) = FlatPtr(&idtr);
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_idtr) = FlatPtr(&idtr);
+
+#if ARCH(X86_64)
+    // TODO: Use these also in i686 builds
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_kernel_map_base) = FlatPtr(kernel_mapping_base);
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_kernel_entry_function) = FlatPtr(&init_ap);
+#endif
 
     // Store the BSP's CR0 and CR4 values for the APs to use
-    *APIC_INIT_VAR_PTR(u32, apic_startup_region->vaddr().as_ptr(), ap_cpu_init_cr0) = read_cr0();
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_init_cr0) = read_cr0();
-    *APIC_INIT_VAR_PTR(u32, apic_startup_region->vaddr().as_ptr(), ap_cpu_init_cr4) = read_cr4();
+    *APIC_INIT_VAR_PTR(FlatPtr, apic_startup_region_ptr, ap_cpu_init_cr4) = read_cr4();
 
     m_ap_boot_environment = move(apic_startup_region);
 }