RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-29 07:47:35 +00:00
Code Issues Activity Actions

Kernel: Pull apart CPU.h

Browse source 
This does not add any functional changes
This commit is contained in:
Hendiadyoin1 2021-06-21 17:34:09 +02:00 • committed by Andreas Kling
parent 37253ebcae
commit 7ca3d413f7
86 changed files with 3866 additions and 3493 deletions
Download patch file Download diff file Expand all files Collapse all files

83
Kernel/Arch/x86/i386/ASM_wrapper.cpp Normal file
View file

@ -0,0 +1,83 @@
/*
* Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Types.h>
#include <Kernel/Arch/x86/ASM_wrapper.h>
#include <Kernel/Arch/x86/CPU.h>
#include <Kernel/Arch/x86/Processor.h>
namespace Kernel {
UNMAP_AFTER_INIT void write_cr0(FlatPtr value)
{
asm volatile("mov %%eax, %%cr0" ::"a"(value));
}
UNMAP_AFTER_INIT void write_cr4(FlatPtr value)
{
asm volatile("mov %%eax, %%cr4" ::"a"(value));
}
FlatPtr read_cr0()
{
FlatPtr cr0;
asm("mov %%cr0, %%eax"
: "=a"(cr0));
return cr0;
}
FlatPtr read_cr2()
{
FlatPtr cr2;
asm("mov %%cr2, %%eax"
: "=a"(cr2));
return cr2;
}
FlatPtr read_cr3()
{
FlatPtr cr3;
asm("mov %%cr3, %%eax"
: "=a"(cr3));
return cr3;
}
void write_cr3(FlatPtr cr3)
{
// NOTE: If you're here from a GPF crash, it's very likely that a PDPT entry is incorrect, not this!
asm volatile("mov %%eax, %%cr3" ::"a"(cr3)
: "memory");
}
FlatPtr read_cr4()
{
FlatPtr cr4;
asm("mov %%cr4, %%eax"
: "=a"(cr4));
return cr4;
}
#define DEFINE_DEBUG_REGISTER(index) \
FlatPtr read_dr##index() \
{ \
FlatPtr value; \
asm("mov %%dr" #index ", %%eax" \
: "=a"(value)); \
return value; \
} \
void write_dr##index(FlatPtr value) \
{ \
asm volatile("mov %%eax, %%dr" #index ::"a"(value)); \
}
DEFINE_DEBUG_REGISTER(0);
DEFINE_DEBUG_REGISTER(1);
DEFINE_DEBUG_REGISTER(2);
DEFINE_DEBUG_REGISTER(3);
DEFINE_DEBUG_REGISTER(6);
DEFINE_DEBUG_REGISTER(7);
}

433
Kernel/Arch/x86/i386/Boot/boot.S Normal file
View file

@ -0,0 +1,433 @@
.set MULTIBOOT_MAGIC, 0x1badb002
.set MULTIBOOT_PAGE_ALIGN, 0x1
.set MULTIBOOT_MEMORY_INFO, 0x2
.set MULTIBOOT_VIDEO_MODE, 0x4
.set multiboot_flags, MULTIBOOT_PAGE_ALIGN | MULTIBOOT_MEMORY_INFO
.set multiboot_checksum, -(MULTIBOOT_MAGIC + multiboot_flags)
.section .multiboot, "a"
.align 4
.long MULTIBOOT_MAGIC
.long multiboot_flags
.long multiboot_checksum
/* for MULTIBOOT_MEMORY_INFO */
.long 0x00000000 /* header_addr */
.long 0x00000000 /* load_addr */
.long 0x00000000 /* load_end_addr */
.long 0x00000000 /* bss_end_addr */
.long 0x00000000 /* entry_addr */
/* for MULTIBOOT_VIDEO_MODE */
.long 0x00000000 /* mode_type */
.long 1280 /* width */
.long 1024 /* height */
.long 32 /* depth */
.section .stack, "aw", @nobits
stack_bottom:
.skip 32768
stack_top:
.global kernel_cmdline
kernel_cmdline:
.skip 4096
.section .page_tables, "aw", @nobits
.align 4096
.global boot_pdpt
boot_pdpt:
.skip 4096
.global boot_pd0
boot_pd0:
.skip 4096
.global boot_pd3
boot_pd3:
.skip 4096
.global boot_pd0_pt0
boot_pd0_pt0:
.skip 4096 * 4
.global boot_pd3_pts
boot_pd3_pts:
.skip 4096 * 16
.global boot_pd3_pt1023
boot_pd3_pt1023:
.skip 4096
.section .boot_text, "ax"
.global start
.type start, @function
.extern init
.type init, @function
.extern multiboot_info_ptr
.type multiboot_info_ptr, @object
/*
construct the following (32-bit PAE) page table layout:
pdpt
0: boot_pd0 (0-1GB)
1: n/a (1-2GB)
2: n/a (2-3GB)
3: boot_pd3 (3-4GB)
boot_pd0 : 512 pde's
0: boot_pd0_pt0 (0-2MB) (id 512 4KB pages)
boot_pd3 : 512 pde's
0: boot_pd3_pts[0] (3072-3074MB) (pseudo 512 4KB pages)
1: boot_pd3_pts[1] (3074-3076MB) (pseudo 512 4KB pages)
2: boot_pd3_pts[2] (3076-3078MB) (pseudo 512 4KB pages)
3: boot_pd3_pts[3] (3078-3080MB) (pseudo 512 4KB pages)
4: boot_pd3_pts[4] (3080-3082MB) (pseudo 512 4KB pages)
5: boot_pd3_pts[5] (3082-3084MB) (pseudo 512 4KB pages)
6: boot_pd3_pts[6] (3084-3086MB) (pseudo 512 4KB pages)
7: boot_pd3_pts[7] (3086-3088MB) (pseudo 512 4KB pages)
8: boot_pd3_pts[8] (3088-3090MB) (pseudo 512 4KB pages)
9: boot_pd3_pts[9] (3090-3076MB) (pseudo 512 4KB pages)
10: boot_pd3_pts[10] (3092-3094MB) (pseudo 512 4KB pages)
11: boot_pd3_pts[11] (3094-3096MB) (pseudo 512 4KB pages)
12: boot_pd3_pts[12] (3096-3098MB) (pseudo 512 4KB pages)
13: boot_pd3_pts[13] (3098-3100MB) (pseudo 512 4KB pages)
14: boot_pd3_pts[14] (3100-3102MB) (pseudo 512 4KB pages)
15: boot_pd3_pts[15] (3102-3104MB) (pseudo 512 4KB pages)
16: boot_pd3_pt1023 (4094-4096MB) (for page table mappings)
the 9 page tables each contain 512 pte's that map individual 4KB pages
*/
start:
cli
cld
/* We don't know where the bootloader might have put the command line.
* It might be at an inconvenient location that we're not about to map,
* so let's just copy it to a convenient location while we have the whole
* memory space identity-mapped anyway. :^)
*/
movl %ebx, %esi
addl $16, %esi
movl (%esi), %esi
movl $1024, %ecx
movl $(kernel_cmdline - 0xc0000000), %edi
rep movsl
/* clear pdpt */
movl $(boot_pdpt - 0xc0000000), %edi
movl $1024, %ecx
xorl %eax, %eax
rep stosl
/* set up pdpt[0] and pdpt[3] */
movl $(boot_pdpt - 0xc0000000), %edi
movl $((boot_pd0 - 0xc0000000) + 1), 0(%edi)
movl $((boot_pd3 - 0xc0000000) + 1), 24(%edi)
/* clear pd0 */
movl $(boot_pd0 - 0xc0000000), %edi
movl $1024, %ecx
xorl %eax, %eax
rep stosl
/* clear pd3 */
movl $(boot_pd3 - 0xc0000000), %edi
movl $1024, %ecx
xorl %eax, %eax
rep stosl
/* clear pd0's pt's */
movl $(boot_pd0_pt0 - 0xc0000000), %edi
movl $(1024 * 4), %ecx
xorl %eax, %eax
rep stosl
/* clear pd3's pt's */
movl $(boot_pd3_pts - 0xc0000000), %edi
movl $(1024 * 17), %ecx
xorl %eax, %eax
rep stosl
/* add boot_pd0_pt0 to boot_pd0 */
movl $(boot_pd0 - 0xc0000000), %edi
movl $(boot_pd0_pt0 - 0xc0000000), %eax
movl %eax, 0(%edi)
/* R/W + Present */
orl $0x3, 0(%edi)
/* add boot_pd3_pts to boot_pd3 */
movl $16, %ecx
movl $(boot_pd3 - 0xc0000000), %edi
movl $(boot_pd3_pts - 0xc0000000), %eax
1:
movl %eax, 0(%edi)
/* R/W + Present */
orl $0x3, 0(%edi)
addl $8, %edi
addl $4096, %eax
loop 1b
/* identity map the 0 to 2MB range */
movl $512, %ecx
movl $(boot_pd0_pt0 - 0xc0000000), %edi
xorl %eax, %eax
1:
movl %eax, 0(%edi)
/* R/W + Present */
orl $0x3, 0(%edi)
addl $8, %edi
addl $4096, %eax
loop 1b
/* pseudo identity map the 3072-3102MB range */
movl $(512 * 16), %ecx
movl $(boot_pd3_pts - 0xc0000000), %edi
xorl %eax, %eax
1:
movl %eax, 0(%edi)
/* R/W + Present */
orl $0x3, 0(%edi)
addl $8, %edi
addl $4096, %eax
loop 1b
/* create an empty page table for the top 2MB at the 4GB mark */
movl $(boot_pd3 - 0xc0000000), %edi
movl $(boot_pd3_pt1023 - 0xc0000000), 4088(%edi)
orl $0x3, 4088(%edi)
movl $0, 4092(%edi)
/* point CR3 to PDPT */
movl $(boot_pdpt - 0xc0000000), %eax
movl %eax, %cr3
/* enable PAE + PSE */
movl %cr4, %eax
orl $0x60, %eax
movl %eax, %cr4
/* enable PG */
movl %cr0, %eax
orl $0x80000000, %eax
movl %eax, %cr0
/* set up stack */
mov $stack_top, %esp
and $-16, %esp
/* jmp to an address above the 3GB mark */
movl $1f,%eax
jmp *%eax
1:
movl %cr3, %eax
movl %eax, %cr3
/* unmap the 0-1MB range, which isn't used after jmp-ing up here */
movl $256, %ecx
movl $(boot_pd0_pt0 - 0xc0000000), %edi
xorl %eax, %eax
1:
movl %eax, 0(%edi)
addl $8, %edi
loop 1b
/* jump into C++ land */
addl $0xc0000000, %ebx
movl %ebx, multiboot_info_ptr
call init
add $4, %esp
cli
loop:
hlt
jmp loop
.extern init_ap
.type init_ap, @function
/*
The apic_ap_start function will be loaded to P0x00008000 where the APIC
will boot the AP from in real mode. This code also contains space for
special variables that *must* remain here. When initializing the APIC,
the code here gets copied to P0x00008000, the variables in here get
populated and then the the boot of the APs will be triggered. Having
the variables here allows us to access them from real mode. Also, the
code here avoids the need for relocation entries.
Basically, the variables between apic_ap_start and end_apic_ap_start
*MUST* remain here and cannot be moved into a .bss or any other location.
*/
.global apic_ap_start
.type apic_ap_start, @function
apic_ap_start:
.code16
cli
jmp $0x800, $(1f - apic_ap_start) /* avoid relocation entries */
1:
mov %cs, %ax
mov %ax, %ds
xor %ax, %ax
mov %ax, %sp
/* load the first temporary gdt */
lgdt (ap_cpu_gdtr_initial - apic_ap_start)
/* enable PM */
movl %cr0, %eax
orl $1, %eax
movl %eax, %cr0
ljmpl $8, $(apic_ap_start32 - apic_ap_start + 0x8000)
apic_ap_start32:
.code32
mov $0x10, %ax
mov %ax, %ss
mov %ax, %ds
mov %ax, %es
mov %ax, %fs
mov %ax, %gs
movl $0x8000, %ebp
/* generate a unique ap cpu id (0 means 1st ap, not bsp!) */
xorl %eax, %eax
incl %eax
lock; xaddl %eax, (ap_cpu_id - apic_ap_start)(%ebp) /* avoid relocation entries */
movl %eax, %esi
/* find our allocated stack based on the generated id */
movl (ap_cpu_init_stacks - apic_ap_start)(%ebp, %eax, 4), %esp
/* check if we support NX and enable it if we do */
movl $0x80000001, %eax
cpuid
testl $0x100000, %edx
je (1f - apic_ap_start + 0x8000)
/* turn on IA32_EFER.NXE */
movl $0xc0000080, %ecx
rdmsr
orl $0x800, %eax
wrmsr
1:
/* load the bsp's cr3 value */
movl (ap_cpu_init_cr3 - apic_ap_start)(%ebp), %eax
movl %eax, %cr3
/* enable PAE + PSE */
movl %cr4, %eax
orl $0x60, %eax
movl %eax, %cr4
/* enable PG */
movl %cr0, %eax
orl $0x80000000, %eax
movl %eax, %cr0
/* load a second temporary gdt that points above 3GB */
lgdt (ap_cpu_gdtr_initial2 - apic_ap_start + 0xc0008000)
/* jump above 3GB into our identity mapped area now */
ljmp $8, $(apic_ap_start32_2 - apic_ap_start + 0xc0008000)
apic_ap_start32_2:
/* flush the TLB */
movl %cr3, %eax
movl %eax, %cr3
movl $0xc0008000, %ebp
/* now load the final gdt and idt from the identity mapped area */
movl (ap_cpu_gdtr - apic_ap_start)(%ebp), %eax
lgdt (%eax)
movl (ap_cpu_idtr - apic_ap_start)(%ebp), %eax
lidt (%eax)
/* set same cr0 and cr4 values as the BSP */
movl (ap_cpu_init_cr0 - apic_ap_start)(%ebp), %eax
movl %eax, %cr0
movl (ap_cpu_init_cr4 - apic_ap_start)(%ebp), %eax
movl %eax, %cr4
/* push the Processor pointer this CPU is going to use */
movl (ap_cpu_init_processor_info_array - apic_ap_start)(%ebp), %eax
addl $0xc0000000, %eax
movl 0(%eax, %esi, 4), %eax
push %eax
/* push the cpu id, 0 representing the bsp and call into c++ */
incl %esi
push %esi
xor %ebp, %ebp
cld
/* We are in identity mapped P0x8000 and the BSP will unload this code
once all APs are initialized, so call init_ap but return to our
infinite loop */
push $loop
ljmp $8, $init_ap
.align 4
.global apic_ap_start_size
apic_ap_start_size:
.2byte end_apic_ap_start - apic_ap_start
.align 4
ap_cpu_id:
.4byte 0x0
ap_cpu_gdt:
/* null */
.8byte 0x0
/* code */
.4byte 0x0000FFFF
.4byte 0x00cf9a00
/* data */
.4byte 0x0000FFFF
.4byte 0x00cf9200
ap_cpu_gdt_end:
ap_cpu_gdtr_initial:
.2byte ap_cpu_gdt_end - ap_cpu_gdt - 1
.4byte (ap_cpu_gdt - apic_ap_start) + 0x8000
ap_cpu_gdtr_initial2:
.2byte ap_cpu_gdt_end - ap_cpu_gdt - 1
.4byte (ap_cpu_gdt - apic_ap_start) + 0xc0008000
.global ap_cpu_gdtr
ap_cpu_gdtr:
.4byte 0x0 /* will be set at runtime */
.global ap_cpu_idtr
ap_cpu_idtr:
.4byte 0x0 /* will be set at runtime */
.global ap_cpu_init_cr0
ap_cpu_init_cr0:
.4byte 0x0 /* will be set at runtime */
.global ap_cpu_init_cr3
ap_cpu_init_cr3:
.4byte 0x0 /* will be set at runtime */
.global ap_cpu_init_cr4
ap_cpu_init_cr4:
.4byte 0x0 /* will be set at runtime */
.global ap_cpu_init_processor_info_array
ap_cpu_init_processor_info_array:
.4byte 0x0 /* will be set at runtime */
.global ap_cpu_init_stacks
ap_cpu_init_stacks:
/* array of allocated stack pointers */
/* NOTE: ap_cpu_init_stacks must be the last variable before
end_apic_ap_start! */
.set end_apic_ap_start, .

132
Kernel/Arch/x86/i386/CPU.cpp Normal file
View file

@ -0,0 +1,132 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Assertions.h>
#include <AK/Types.h>
#include <Kernel/Arch/x86/CPU.h>
#include <Kernel/Arch/x86/Processor.h>
#include <Kernel/Arch/x86/TrapFrame.h>
#include <Kernel/KSyms.h>
#include <Kernel/Process.h>
#include <Kernel/Thread.h>
namespace Kernel {
// The compiler can't see the calls to these functions inside assembly.
// Declare them, to avoid dead code warnings.
extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread) __attribute__((used));
extern "C" void context_first_init(Thread* from_thread, Thread* to_thread, TrapFrame* trap) __attribute__((used));
extern "C" u32 do_init_context(Thread* thread, u32 flags) __attribute__((used));
extern "C" void enter_thread_context(Thread* from_thread, Thread* to_thread)
{
VERIFY(from_thread == to_thread || from_thread->state() != Thread::Running);
VERIFY(to_thread->state() == Thread::Running);
bool has_fxsr = Processor::current().has_feature(CPUFeature::FXSR);
Processor::set_current_thread(*to_thread);
auto& from_tss = from_thread->tss();
auto& to_tss = to_thread->tss();
if (has_fxsr)
asm volatile("fxsave %0"
: "=m"(from_thread->fpu_state()));
else
asm volatile("fnsave %0"
: "=m"(from_thread->fpu_state()));
from_tss.fs = get_fs();
from_tss.gs = get_gs();
set_fs(to_tss.fs);
set_gs(to_tss.gs);
if (from_thread->process().is_traced())
read_debug_registers_into(from_thread->debug_register_state());
if (to_thread->process().is_traced()) {
write_debug_registers_from(to_thread->debug_register_state());
} else {
clear_debug_registers();
}
auto& processor = Processor::current();
auto& tls_descriptor = processor.get_gdt_entry(GDT_SELECTOR_TLS);
tls_descriptor.set_base(to_thread->thread_specific_data());
tls_descriptor.set_limit(to_thread->thread_specific_region_size());
if (from_tss.cr3 != to_tss.cr3)
write_cr3(to_tss.cr3);
to_thread->set_cpu(processor.get_id());
processor.restore_in_critical(to_thread->saved_critical());
if (has_fxsr)
asm volatile("fxrstor %0" ::"m"(to_thread->fpu_state()));
else
asm volatile("frstor %0" ::"m"(to_thread->fpu_state()));
// TODO: ioperm?
}
extern "C" void context_first_init([[maybe_unused]] Thread* from_thread, [[maybe_unused]] Thread* to_thread, [[maybe_unused]] TrapFrame* trap)
{
VERIFY(!are_interrupts_enabled());
VERIFY(is_kernel_mode());
dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {} (context_first_init)", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread);
VERIFY(to_thread == Thread::current());
Scheduler::enter_current(*from_thread, true);
// Since we got here and don't have Scheduler::context_switch in the
// call stack (because this is the first time we switched into this
// context), we need to notify the scheduler so that it can release
// the scheduler lock. We don't want to enable interrupts at this point
// as we're still in the middle of a context switch. Doing so could
// trigger a context switch within a context switch, leading to a crash.
Scheduler::leave_on_first_switch(trap->regs->eflags & ~0x200);
}
extern "C" u32 do_init_context(Thread* thread, u32 flags)
{
VERIFY_INTERRUPTS_DISABLED();
thread->tss().eflags = flags;
return Processor::current().init_context(*thread, true);
}
}
void __assertion_failed(const char* msg, const char* file, unsigned line, const char* func)
{
asm volatile("cli");
critical_dmesgln("ASSERTION FAILED: {}", msg);
critical_dmesgln("{}:{} in {}", file, line, func);
abort();
}
[[noreturn]] void abort()
{
// Switch back to the current process's page tables if there are any.
// Otherwise stack walking will be a disaster.
auto process = Process::current();
if (process)
MM.enter_process_paging_scope(*process);
Kernel::dump_backtrace();
Processor::halt();
abort();
}
[[noreturn]] void _abort()
{
asm volatile("ud2");
__builtin_unreachable();
}

51
Kernel/Arch/x86/i386/InterruptEntry.cpp Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <Kernel/Arch/x86/DescriptorTable.h>
#include <Kernel/Arch/x86/TrapFrame.h>
// clang-format off
asm(
".globl interrupt_common_asm_entry\n"
"interrupt_common_asm_entry: \n"
" pusha\n"
" pushl %ds\n"
" pushl %es\n"
" pushl %fs\n"
" pushl %gs\n"
" pushl %ss\n"
" mov $" __STRINGIFY(GDT_SELECTOR_DATA0) ", %ax\n"
" mov %ax, %ds\n"
" mov %ax, %es\n"
" mov $" __STRINGIFY(GDT_SELECTOR_PROC) ", %ax\n"
" mov %ax, %fs\n"
" pushl %esp \n" // set TrapFrame::regs
" subl $" __STRINGIFY(TRAP_FRAME_SIZE - 4) ", %esp \n"
" movl %esp, %ebx \n" // save pointer to TrapFrame
" pushl %ebx \n"
" cld\n"
" call enter_trap \n"
" movl %ebx, 0(%esp) \n" // push pointer to TrapFrame
" call handle_interrupt\n"
" movl %ebx, 0(%esp) \n" // push pointer to TrapFrame
".globl common_trap_exit \n"
"common_trap_exit: \n"
// another thread may have handled this trap at this point, so don't
// make assumptions about the stack other than there's a TrapFrame
// and a pointer to it.
" call exit_trap \n"
" addl $" __STRINGIFY(TRAP_FRAME_SIZE + 4) ", %esp\n" // pop TrapFrame and pointer to it
".globl interrupt_common_asm_exit \n"
"interrupt_common_asm_exit: \n"
" addl $4, %esp\n" // pop %ss
" popl %gs\n"
" popl %fs\n"
" popl %es\n"
" popl %ds\n"
" popa\n"
" addl $0x4, %esp\n" // skip exception_code, isr_number
" iret\n"
);
// clang-format on

300
Kernel/Arch/x86/i386/Processor.cpp Normal file
View file

@ -0,0 +1,300 @@
/*
* Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/StdLibExtras.h>
#include <Kernel/Arch/x86/CPU.h>
#include <Kernel/Arch/x86/Processor.h>
#include <Kernel/Arch/x86/TrapFrame.h>
#include <Kernel/Panic.h>
#include <Kernel/Process.h>
#include <Kernel/Random.h>
#include <Kernel/Thread.h>
namespace Kernel {
#define ENTER_THREAD_CONTEXT_ARGS_SIZE (2 * 4) // to_thread, from_thread
extern "C" void thread_context_first_enter(void);
extern "C" void do_assume_context(Thread* thread, u32 flags);
extern "C" void exit_kernel_thread(void);
// clang-format off
asm(
// enter_thread_context returns to here first time a thread is executing
".globl thread_context_first_enter \n"
"thread_context_first_enter: \n"
// switch_context will have pushed from_thread and to_thread to our new
// stack prior to thread_context_first_enter() being called, and the
// pointer to TrapFrame was the top of the stack before that
" movl 8(%esp), %ebx \n" // save pointer to TrapFrame
" cld \n"
" call context_first_init \n"
" addl $" __STRINGIFY(ENTER_THREAD_CONTEXT_ARGS_SIZE) ", %esp \n"
" movl %ebx, 0(%esp) \n" // push pointer to TrapFrame
" jmp common_trap_exit \n"
);
// clang-format on
#if ARCH(I386)
// clang-format off
asm(
".global do_assume_context \n"
"do_assume_context: \n"
" movl 4(%esp), %ebx \n"
" movl 8(%esp), %esi \n"
// We're going to call Processor::init_context, so just make sure
// we have enough stack space so we don't stomp over it
" subl $(" __STRINGIFY(4 + REGISTER_STATE_SIZE + TRAP_FRAME_SIZE + 4) "), %esp \n"
" pushl %esi \n"
" pushl %ebx \n"
" cld \n"
" call do_init_context \n"
" addl $8, %esp \n"
" movl %eax, %esp \n" // move stack pointer to what Processor::init_context set up for us
" pushl %ebx \n" // push to_thread
" pushl %ebx \n" // push from_thread
" pushl $thread_context_first_enter \n" // should be same as tss.eip
" jmp enter_thread_context \n"
);
// clang-format on
#endif
String Processor::platform_string() const
{
// FIXME: other platforms
return "i386";
}
u32 Processor::init_context(Thread& thread, bool leave_crit)
{
VERIFY(is_kernel_mode());
VERIFY(g_scheduler_lock.is_locked());
if (leave_crit) {
// Leave the critical section we set up in in Process::exec,
// but because we still have the scheduler lock we should end up with 1
m_in_critical--; // leave it without triggering anything or restoring flags
VERIFY(in_critical() == 1);
}
u32 kernel_stack_top = thread.kernel_stack_top();
// Add a random offset between 0-256 (16-byte aligned)
kernel_stack_top -= round_up_to_power_of_two(get_fast_random<u8>(), 16);
u32 stack_top = kernel_stack_top;
// TODO: handle NT?
VERIFY((cpu_flags() & 0x24000) == 0); // Assume !(NT | VM)
auto& tss = thread.tss();
bool return_to_user = (tss.cs & 3) != 0;
// make room for an interrupt frame
if (!return_to_user) {
// userspace_esp and userspace_ss are not popped off by iret
// unless we're switching back to user mode
stack_top -= sizeof(RegisterState) - 2 * sizeof(u32);
// For kernel threads we'll push the thread function argument
// which should be in tss.esp and exit_kernel_thread as return
// address.
stack_top -= 2 * sizeof(u32);
*reinterpret_cast<u32*>(kernel_stack_top - 2 * sizeof(u32)) = tss.esp;
*reinterpret_cast<u32*>(kernel_stack_top - 3 * sizeof(u32)) = FlatPtr(&exit_kernel_thread);
} else {
stack_top -= sizeof(RegisterState);
}
// we want to end up 16-byte aligned, %esp + 4 should be aligned
stack_top -= sizeof(u32);
*reinterpret_cast<u32*>(kernel_stack_top - sizeof(u32)) = 0;
// set up the stack so that after returning from thread_context_first_enter()
// we will end up either in kernel mode or user mode, depending on how the thread is set up
// However, the first step is to always start in kernel mode with thread_context_first_enter
RegisterState& iretframe = *reinterpret_cast<RegisterState*>(stack_top);
iretframe.ss = tss.ss;
iretframe.gs = tss.gs;
iretframe.fs = tss.fs;
iretframe.es = tss.es;
iretframe.ds = tss.ds;
iretframe.edi = tss.edi;
iretframe.esi = tss.esi;
iretframe.ebp = tss.ebp;
iretframe.esp = 0;
iretframe.ebx = tss.ebx;
iretframe.edx = tss.edx;
iretframe.ecx = tss.ecx;
iretframe.eax = tss.eax;
iretframe.eflags = tss.eflags;
iretframe.eip = tss.eip;
iretframe.cs = tss.cs;
if (return_to_user) {
iretframe.userspace_esp = tss.esp;
iretframe.userspace_ss = tss.ss;
}
// make space for a trap frame
stack_top -= sizeof(TrapFrame);
TrapFrame& trap = *reinterpret_cast<TrapFrame*>(stack_top);
trap.regs = &iretframe;
trap.prev_irq_level = 0;
trap.next_trap = nullptr;
stack_top -= sizeof(u32); // pointer to TrapFrame
*reinterpret_cast<u32*>(stack_top) = stack_top + 4;
if constexpr (CONTEXT_SWITCH_DEBUG) {
if (return_to_user) {
dbgln("init_context {} ({}) set up to execute at eip={}:{}, esp={}, stack_top={}, user_top={}:{}",
thread,
VirtualAddress(&thread),
iretframe.cs, tss.eip,
VirtualAddress(tss.esp),
VirtualAddress(stack_top),
iretframe.userspace_ss,
iretframe.userspace_esp);
} else {
dbgln("init_context {} ({}) set up to execute at eip={}:{}, esp={}, stack_top={}",
thread,
VirtualAddress(&thread),
iretframe.cs, tss.eip,
VirtualAddress(tss.esp),
VirtualAddress(stack_top));
}
}
// make switch_context() always first return to thread_context_first_enter()
// in kernel mode, so set up these values so that we end up popping iretframe
// off the stack right after the context switch completed, at which point
// control is transferred to what iretframe is pointing to.
tss.eip = FlatPtr(&thread_context_first_enter);
tss.esp0 = kernel_stack_top;
tss.esp = stack_top;
tss.cs = GDT_SELECTOR_CODE0;
tss.ds = GDT_SELECTOR_DATA0;
tss.es = GDT_SELECTOR_DATA0;
tss.gs = GDT_SELECTOR_DATA0;
tss.ss = GDT_SELECTOR_DATA0;
tss.fs = GDT_SELECTOR_PROC;
return stack_top;
}
void Processor::switch_context(Thread*& from_thread, Thread*& to_thread)
{
VERIFY(!in_irq());
VERIFY(m_in_critical == 1);
VERIFY(is_kernel_mode());
dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context --> switching out of: {} {}", VirtualAddress(from_thread), *from_thread);
from_thread->save_critical(m_in_critical);
// clang-format off
// Switch to new thread context, passing from_thread and to_thread
// through to the new context using registers edx and eax
asm volatile(
// NOTE: changing how much we push to the stack affects
// SWITCH_CONTEXT_TO_STACK_SIZE and thread_context_first_enter()!
"pushfl \n"
"pushl %%ebx \n"
"pushl %%esi \n"
"pushl %%edi \n"
"pushl %%ebp \n"
"movl %%esp, %[from_esp] \n"
"movl $1f, %[from_eip] \n"
"movl %[to_esp0], %%ebx \n"
"movl %%ebx, %[tss_esp0] \n"
"movl %[to_esp], %%esp \n"
"pushl %[to_thread] \n"
"pushl %[from_thread] \n"
"pushl %[to_eip] \n"
"cld \n"
"jmp enter_thread_context \n"
"1: \n"
"popl %%edx \n"
"popl %%eax \n"
"popl %%ebp \n"
"popl %%edi \n"
"popl %%esi \n"
"popl %%ebx \n"
"popfl \n"
: [from_esp] "=m" (from_thread->tss().esp),
[from_eip] "=m" (from_thread->tss().eip),
[tss_esp0] "=m" (m_tss.esp0),
"=d" (from_thread), // needed so that from_thread retains the correct value
"=a" (to_thread) // needed so that to_thread retains the correct value
: [to_esp] "g" (to_thread->tss().esp),
[to_esp0] "g" (to_thread->tss().esp0),
[to_eip] "c" (to_thread->tss().eip),
[from_thread] "d" (from_thread),
[to_thread] "a" (to_thread)
: "memory"
);
// clang-format on
dbgln_if(CONTEXT_SWITCH_DEBUG, "switch_context <-- from {} {} to {} {}", VirtualAddress(from_thread), *from_thread, VirtualAddress(to_thread), *to_thread);
Processor::current().restore_in_critical(to_thread->saved_critical());
}
void Processor::assume_context(Thread& thread, FlatPtr flags)
{
dbgln_if(CONTEXT_SWITCH_DEBUG, "Assume context for thread {} {}", VirtualAddress(&thread), thread);
VERIFY_INTERRUPTS_DISABLED();
Scheduler::prepare_after_exec();
// in_critical() should be 2 here. The critical section in Process::exec
// and then the scheduler lock
VERIFY(Processor::current().in_critical() == 2);
do_assume_context(&thread, flags);
VERIFY_NOT_REACHED();
}
UNMAP_AFTER_INIT void Processor::initialize_context_switching(Thread& initial_thread)
{
VERIFY(initial_thread.process().is_kernel_process());
auto& tss = initial_thread.tss();
m_tss = tss;
m_tss.esp0 = tss.esp0;
m_tss.ss0 = GDT_SELECTOR_DATA0;
// user mode needs to be able to switch to kernel mode:
m_tss.cs = m_tss.ds = m_tss.es = m_tss.gs = m_tss.ss = GDT_SELECTOR_CODE0 | 3;
m_tss.fs = GDT_SELECTOR_PROC | 3;
m_scheduler_initialized = true;
// clang-format off
asm volatile(
"movl %[new_esp], %%esp \n" // switch to new stack
"pushl %[from_to_thread] \n" // to_thread
"pushl %[from_to_thread] \n" // from_thread
"pushl $" __STRINGIFY(GDT_SELECTOR_CODE0) " \n"
"pushl %[new_eip] \n" // save the entry eip to the stack
"movl %%esp, %%ebx \n"
"addl $20, %%ebx \n" // calculate pointer to TrapFrame
"pushl %%ebx \n"
"cld \n"
"pushl %[cpu] \n" // push argument for init_finished before register is clobbered
"call pre_init_finished \n"
"call init_finished \n"
"addl $4, %%esp \n"
"call post_init_finished \n"
"call enter_trap_no_irq \n"
"addl $4, %%esp \n"
"lret \n"
:: [new_esp] "g" (tss.esp),
[new_eip] "a" (tss.eip),
[from_to_thread] "b" (&initial_thread),
[cpu] "c" (id())
);
// clang-format on
VERIFY_NOT_REACHED();
}
}

78
Kernel/Arch/x86/i386/ProcessorInfo.cpp Normal file
View file

@ -0,0 +1,78 @@
/*
* Copyright (c) 2018-2020, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/StringBuilder.h>
#include <AK/Types.h>
#include <Kernel/Arch/x86/CPUID.h>
#include <Kernel/Arch/x86/Processor.h>
#include <Kernel/Arch/x86/ProcessorInfo.h>
namespace Kernel {
ProcessorInfo::ProcessorInfo(Processor& processor)
: m_processor(processor)
{
u32 max_leaf;
{
CPUID cpuid(0);
StringBuilder builder;
auto emit_u32 = [&](u32 value) {
builder.appendff("{:c}{:c}{:c}{:c}",
value & 0xff,
(value >> 8) & 0xff,
(value >> 16) & 0xff,
(value >> 24) & 0xff);
};
max_leaf = cpuid.eax();
emit_u32(cpuid.ebx());
emit_u32(cpuid.edx());
emit_u32(cpuid.ecx());
m_cpuid = builder.build();
}
{
VERIFY(max_leaf >= 1);
CPUID cpuid(1);
m_stepping = cpuid.eax() & 0xf;
u32 model = (cpuid.eax() >> 4) & 0xf;
u32 family = (cpuid.eax() >> 8) & 0xf;
m_type = (cpuid.eax() >> 12) & 0x3;
u32 extended_model = (cpuid.eax() >> 16) & 0xf;
u32 extended_family = (cpuid.eax() >> 20) & 0xff;
if (family == 15) {
m_display_family = family + extended_family;
m_display_model = model + (extended_model << 4);
} else if (family == 6) {
m_display_family = family;
m_display_model = model + (extended_model << 4);
} else {
m_display_family = family;
m_display_model = model;
}
}
u32 max_extended_leaf = CPUID(0x80000000).eax();
if (max_extended_leaf >= 0x80000004) {
alignas(u32) char buffer[48];
u32* bufptr = reinterpret_cast<u32*>(buffer);
auto copy_brand_string_part_to_buffer = [&](u32 i) {
CPUID cpuid(0x80000002 + i);
*bufptr++ = cpuid.eax();
*bufptr++ = cpuid.ebx();
*bufptr++ = cpuid.ecx();
*bufptr++ = cpuid.edx();
};
copy_brand_string_part_to_buffer(0);
copy_brand_string_part_to_buffer(1);
copy_brand_string_part_to_buffer(2);
m_brandstr = buffer;
}
// Cache the CPU feature string
m_features = m_processor.features_string();
}
}

298
Kernel/Arch/x86/i386/SafeMem.cpp Normal file
View file

@ -0,0 +1,298 @@
/*
* Copyright (c) 2020, the SerenityOS developers.
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <Kernel/Arch/x86/RegisterState.h>
#include <Kernel/Arch/x86/SafeMem.h>
#define CODE_SECTION(section_name) __attribute__((section(section_name)))
extern "C" u8* start_of_safemem_text;
extern "C" u8* end_of_safemem_text;
extern "C" u8* safe_memcpy_ins_1;
extern "C" u8* safe_memcpy_1_faulted;
extern "C" u8* safe_memcpy_ins_2;
extern "C" u8* safe_memcpy_2_faulted;
extern "C" u8* safe_strnlen_ins;
extern "C" u8* safe_strnlen_faulted;
extern "C" u8* safe_memset_ins_1;
extern "C" u8* safe_memset_1_faulted;
extern "C" u8* safe_memset_ins_2;
extern "C" u8* safe_memset_2_faulted;
extern "C" u8* start_of_safemem_atomic_text;
extern "C" u8* end_of_safemem_atomic_text;
extern "C" u8* safe_atomic_fetch_add_relaxed_ins;
extern "C" u8* safe_atomic_fetch_add_relaxed_faulted;
extern "C" u8* safe_atomic_exchange_relaxed_ins;
extern "C" u8* safe_atomic_exchange_relaxed_faulted;
extern "C" u8* safe_atomic_load_relaxed_ins;
extern "C" u8* safe_atomic_load_relaxed_faulted;
extern "C" u8* safe_atomic_store_relaxed_ins;
extern "C" u8* safe_atomic_store_relaxed_faulted;
extern "C" u8* safe_atomic_compare_exchange_relaxed_ins;
extern "C" u8* safe_atomic_compare_exchange_relaxed_faulted;
namespace Kernel {
CODE_SECTION(".text.safemem")
NEVER_INLINE bool safe_memcpy(void* dest_ptr, const void* src_ptr, size_t n, void*& fault_at)
{
fault_at = nullptr;
size_t dest = (size_t)dest_ptr;
size_t src = (size_t)src_ptr;
size_t remainder;
// FIXME: Support starting at an unaligned address.
if (!(dest & 0x3) && !(src & 0x3) && n >= 12) {
size_t size_ts = n / sizeof(size_t);
asm volatile(
"safe_memcpy_ins_1: \n"
"rep movsl \n"
"safe_memcpy_1_faulted: \n" // handle_safe_access_fault() set edx to the fault address!
: "=S"(src),
"=D"(dest),
"=c"(remainder),
[fault_at] "=d"(fault_at)
: "S"(src),
"D"(dest),
"c"(size_ts)
: "memory");
if (remainder != 0)
return false; // fault_at is already set!
n -= size_ts * sizeof(size_t);
if (n == 0) {
fault_at = nullptr;
return true;
}
}
asm volatile(
"safe_memcpy_ins_2: \n"
"rep movsb \n"
"safe_memcpy_2_faulted: \n" // handle_safe_access_fault() set edx to the fault address!
: "=c"(remainder),
[fault_at] "=d"(fault_at)
: "S"(src),
"D"(dest),
"c"(n)
: "memory");
if (remainder != 0)
return false; // fault_at is already set!
fault_at = nullptr;
return true;
}
CODE_SECTION(".text.safemem")
NEVER_INLINE ssize_t safe_strnlen(const char* str, size_t max_n, void*& fault_at)
{
ssize_t count = 0;
fault_at = nullptr;
asm volatile(
"1: \n"
"test %[max_n], %[max_n] \n"
"je 2f \n"
"dec %[max_n] \n"
"safe_strnlen_ins: \n"
"cmpb $0,(%[str], %[count], 1) \n"
"je 2f \n"
"inc %[count] \n"
"jmp 1b \n"
"safe_strnlen_faulted: \n" // handle_safe_access_fault() set edx to the fault address!
"xor %[count_on_error], %[count_on_error] \n"
"dec %[count_on_error] \n" // return -1 on fault
"2:"
: [count_on_error] "=c"(count),
[fault_at] "=d"(fault_at)
: [str] "b"(str),
[count] "c"(count),
[max_n] "d"(max_n));
if (count >= 0)
fault_at = nullptr;
return count;
}
CODE_SECTION(".text.safemem")
NEVER_INLINE bool safe_memset(void* dest_ptr, int c, size_t n, void*& fault_at)
{
fault_at = nullptr;
size_t dest = (size_t)dest_ptr;
size_t remainder;
// FIXME: Support starting at an unaligned address.
if (!(dest & 0x3) && n >= 12) {
size_t size_ts = n / sizeof(size_t);
size_t expanded_c = (u8)c;
expanded_c |= expanded_c << 8;
expanded_c |= expanded_c << 16;
asm volatile(
"safe_memset_ins_1: \n"
"rep stosl \n"
"safe_memset_1_faulted: \n" // handle_safe_access_fault() set edx to the fault address!
: "=D"(dest),
"=c"(remainder),
[fault_at] "=d"(fault_at)
: "D"(dest),
"a"(expanded_c),
"c"(size_ts)
: "memory");
if (remainder != 0)
return false; // fault_at is already set!
n -= size_ts * sizeof(size_t);
if (remainder == 0) {
fault_at = nullptr;
return true;
}
}
asm volatile(
"safe_memset_ins_2: \n"
"rep stosb \n"
"safe_memset_2_faulted: \n" // handle_safe_access_fault() set edx to the fault address!
: "=D"(dest),
"=c"(remainder),
[fault_at] "=d"(fault_at)
: "D"(dest),
"c"(n),
"a"(c)
: "memory");
if (remainder != 0)
return false; // fault_at is already set!
fault_at = nullptr;
return true;
}
CODE_SECTION(".text.safemem.atomic")
NEVER_INLINE Optional<u32> safe_atomic_fetch_add_relaxed(volatile u32* var, u32 val)
{
u32 result;
bool error;
asm volatile(
"xor %[error], %[error] \n"
"safe_atomic_fetch_add_relaxed_ins: \n"
"lock xadd %[result], %[var] \n"
"safe_atomic_fetch_add_relaxed_faulted: \n"
: [error] "=d"(error), [result] "=a"(result), [var] "=m"(*var)
: [val] "a"(val)
: "memory");
if (error)
return {};
return result;
}
CODE_SECTION(".text.safemem.atomic")
NEVER_INLINE Optional<u32> safe_atomic_exchange_relaxed(volatile u32* var, u32 val)
{
u32 result;
bool error;
asm volatile(
"xor %[error], %[error] \n"
"safe_atomic_exchange_relaxed_ins: \n"
"xchg %[val], %[var] \n"
"safe_atomic_exchange_relaxed_faulted: \n"
: [error] "=d"(error), "=a"(result), [var] "=m"(*var)
: [val] "a"(val)
: "memory");
if (error)
return {};
return result;
}
CODE_SECTION(".text.safemem.atomic")
NEVER_INLINE Optional<u32> safe_atomic_load_relaxed(volatile u32* var)
{
u32 result;
bool error;
asm volatile(
"xor %[error], %[error] \n"
"safe_atomic_load_relaxed_ins: \n"
"mov (%[var]), %[result] \n"
"safe_atomic_load_relaxed_faulted: \n"
: [error] "=d"(error), [result] "=c"(result)
: [var] "b"(var)
: "memory");
if (error)
return {};
return result;
}
CODE_SECTION(".text.safemem.atomic")
NEVER_INLINE bool safe_atomic_store_relaxed(volatile u32* var, u32 val)
{
bool error;
asm volatile(
"xor %[error], %[error] \n"
"safe_atomic_store_relaxed_ins: \n"
"xchg %[val], %[var] \n"
"safe_atomic_store_relaxed_faulted: \n"
: [error] "=d"(error), [var] "=m"(*var)
: [val] "r"(val)
: "memory");
return !error;
}
CODE_SECTION(".text.safemem.atomic")
NEVER_INLINE Optional<bool> safe_atomic_compare_exchange_relaxed(volatile u32* var, u32& expected, u32 val)
{
// NOTE: accessing expected is NOT protected as it should always point
// to a valid location in kernel memory!
bool error;
bool did_exchange;
asm volatile(
"xor %[error], %[error] \n"
"safe_atomic_compare_exchange_relaxed_ins: \n"
"lock cmpxchg %[val], %[var] \n"
"safe_atomic_compare_exchange_relaxed_faulted: \n"
: [error] "=d"(error), "=a"(expected), [var] "=m"(*var), "=@ccz"(did_exchange)
: "a"(expected), [val] "b"(val)
: "memory");
if (error)
return {};
return did_exchange;
}
bool handle_safe_access_fault(RegisterState& regs, u32 fault_address)
{
if (regs.eip >= (FlatPtr)&start_of_safemem_text && regs.eip < (FlatPtr)&end_of_safemem_text) {
// If we detect that the fault happened in safe_memcpy() safe_strnlen(),
// or safe_memset() then resume at the appropriate _faulted label
if (regs.eip == (FlatPtr)&safe_memcpy_ins_1)
regs.eip = (FlatPtr)&safe_memcpy_1_faulted;
else if (regs.eip == (FlatPtr)&safe_memcpy_ins_2)
regs.eip = (FlatPtr)&safe_memcpy_2_faulted;
else if (regs.eip == (FlatPtr)&safe_strnlen_ins)
regs.eip = (FlatPtr)&safe_strnlen_faulted;
else if (regs.eip == (FlatPtr)&safe_memset_ins_1)
regs.eip = (FlatPtr)&safe_memset_1_faulted;
else if (regs.eip == (FlatPtr)&safe_memset_ins_2)
regs.eip = (FlatPtr)&safe_memset_2_faulted;
else
return false;
regs.edx = fault_address;
return true;
}
if (regs.eip >= (FlatPtr)&start_of_safemem_atomic_text && regs.eip < (FlatPtr)&end_of_safemem_atomic_text) {
// If we detect that a fault happened in one of the atomic safe_
// functions, resume at the appropriate _faulted label and set
// the edx register to 1 to indicate an error
if (regs.eip == (FlatPtr)&safe_atomic_fetch_add_relaxed_ins)
regs.eip = (FlatPtr)&safe_atomic_fetch_add_relaxed_faulted;
else if (regs.eip == (FlatPtr)&safe_atomic_exchange_relaxed_ins)
regs.eip = (FlatPtr)&safe_atomic_exchange_relaxed_faulted;
else if (regs.eip == (FlatPtr)&safe_atomic_load_relaxed_ins)
regs.eip = (FlatPtr)&safe_atomic_load_relaxed_faulted;
else if (regs.eip == (FlatPtr)&safe_atomic_store_relaxed_ins)
regs.eip = (FlatPtr)&safe_atomic_store_relaxed_faulted;
else if (regs.eip == (FlatPtr)&safe_atomic_compare_exchange_relaxed_ins)
regs.eip = (FlatPtr)&safe_atomic_compare_exchange_relaxed_faulted;
else
return false;
regs.edx = 1;
return true;
}
return false;
}
}