RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-05-15 15:14:58 +00:00
Code Issues Activity Actions
serenity/Kernel/Arch/x86/CPU.h
Nicholas Baron aa4d41fe2c
AK+Kernel+LibELF: Remove the need for IteratorDecision::Continue 
By constraining two implementations, the compiler will select the best
fitting one. All this will require is duplicating the implementation and
simplifying for the `void` case.

This constraining also informs both the caller and compiler by passing
the callback parameter types as part of the constraint
(e.g.: `IterationFunction<int>`).

Some `for_each` functions in LibELF only take functions which return
`void`. This is a minimal correctness check, as it removes one way for a
function to incompletely do something.

There seems to be a possible idiom where inside a lambda, a `return;` is
the same as `continue;` in a for-loop.
2021-05-16 10:36:52 +01:00

1127 lines
29 KiB
C++
Raw Blame History

/*
* Copyright (c) 2018-2021, Andreas Kling <kling@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Atomic.h>
#include <AK/Badge.h>
#include <AK/Concepts.h>
#include <AK/Noncopyable.h>
#include <AK/Vector.h>
#include <Kernel/Arch/x86/DescriptorTable.h>
#include <Kernel/Arch/x86/TSS.h>
#include <Kernel/PhysicalAddress.h>
#include <Kernel/VirtualAddress.h>
#include <LibC/sys/arch/i386/regs.h>
#define READONLY_AFTER_INIT __attribute__((section(".ro_after_init")))
#define UNMAP_AFTER_INIT NEVER_INLINE __attribute__((section(".unmap_after_init")))
#define PAGE_SIZE 4096
#define GENERIC_INTERRUPT_HANDLERS_COUNT (256 - IRQ_VECTOR_BASE)
#define PAGE_MASK ((FlatPtr)0xfffff000u)
namespace Kernel {
class MemoryManager;
class PageDirectory;
class PageTableEntry;
static constexpr u32 safe_eflags_mask = 0xdff;
static constexpr u32 iopl_mask = 3u << 12;
inline u32 get_iopl_from_eflags(u32 eflags)
{
return (eflags & iopl_mask) >> 12;
}
class PageDirectoryEntry {
public:
const PageTableEntry* page_table_base() const { return reinterpret_cast<PageTableEntry*>(m_raw & 0xfffff000u); }
PageTableEntry* page_table_base() { return reinterpret_cast<PageTableEntry*>(m_raw & 0xfffff000u); }
void set_page_table_base(u32 value)
{
m_raw &= 0x8000000000000fffULL;
m_raw |= value & 0xfffff000;
}
bool is_null() const { return m_raw == 0; }
void clear() { m_raw = 0; }
u64 raw() const { return m_raw; }
void copy_from(Badge<PageDirectory>, const PageDirectoryEntry& other) { m_raw = other.m_raw; }
enum Flags {
Present = 1 << 0,
ReadWrite = 1 << 1,
UserSupervisor = 1 << 2,
WriteThrough = 1 << 3,
CacheDisabled = 1 << 4,
Huge = 1 << 7,
Global = 1 << 8,
NoExecute = 0x8000000000000000ULL,
};
bool is_present() const { return raw() & Present; }
void set_present(bool b) { set_bit(Present, b); }
bool is_user_allowed() const { return raw() & UserSupervisor; }
void set_user_allowed(bool b) { set_bit(UserSupervisor, b); }
bool is_huge() const { return raw() & Huge; }
void set_huge(bool b) { set_bit(Huge, b); }
bool is_writable() const { return raw() & ReadWrite; }
void set_writable(bool b) { set_bit(ReadWrite, b); }
bool is_write_through() const { return raw() & WriteThrough; }
void set_write_through(bool b) { set_bit(WriteThrough, b); }
bool is_cache_disabled() const { return raw() & CacheDisabled; }
void set_cache_disabled(bool b) { set_bit(CacheDisabled, b); }
bool is_global() const { return raw() & Global; }
void set_global(bool b) { set_bit(Global, b); }
bool is_execute_disabled() const { return raw() & NoExecute; }
void set_execute_disabled(bool b) { set_bit(NoExecute, b); }
void set_bit(u64 bit, bool value)
{
if (value)
m_raw |= bit;
else
m_raw &= ~bit;
}
private:
u64 m_raw;
};
class PageTableEntry {
public:
void* physical_page_base() { return reinterpret_cast<void*>(m_raw & 0xfffff000u); }
void set_physical_page_base(u32 value)
{
m_raw &= 0x8000000000000fffULL;
m_raw |= value & 0xfffff000;
}
u64 raw() const { return (u32)m_raw; }
enum Flags {
Present = 1 << 0,
ReadWrite = 1 << 1,
UserSupervisor = 1 << 2,
WriteThrough = 1 << 3,
CacheDisabled = 1 << 4,
Global = 1 << 8,
NoExecute = 0x8000000000000000ULL,
};
bool is_present() const { return raw() & Present; }
void set_present(bool b) { set_bit(Present, b); }
bool is_user_allowed() const { return raw() & UserSupervisor; }
void set_user_allowed(bool b) { set_bit(UserSupervisor, b); }
bool is_writable() const { return raw() & ReadWrite; }
void set_writable(bool b) { set_bit(ReadWrite, b); }
bool is_write_through() const { return raw() & WriteThrough; }
void set_write_through(bool b) { set_bit(WriteThrough, b); }
bool is_cache_disabled() const { return raw() & CacheDisabled; }
void set_cache_disabled(bool b) { set_bit(CacheDisabled, b); }
bool is_global() const { return raw() & Global; }
void set_global(bool b) { set_bit(Global, b); }
bool is_execute_disabled() const { return raw() & NoExecute; }
void set_execute_disabled(bool b) { set_bit(NoExecute, b); }
bool is_null() const { return m_raw == 0; }
void clear() { m_raw = 0; }
void set_bit(u64 bit, bool value)
{
if (value)
m_raw |= bit;
else
m_raw &= ~bit;
}
private:
u64 m_raw;
};
static_assert(sizeof(PageDirectoryEntry) == 8);
static_assert(sizeof(PageTableEntry) == 8);
class PageDirectoryPointerTable {
public:
PageDirectoryEntry* directory(size_t index)
{
return (PageDirectoryEntry*)(raw[index] & ~0xfffu);
}
u64 raw[4];
};
class GenericInterruptHandler;
struct RegisterState;
template<typename T>
void read_possibly_unaligned_data(u8* where, T& data)
{
// FIXME: Implement 64 bit unaligned data access
VERIFY(sizeof(T) == 8 || sizeof(T) == 4 || sizeof(T) == 2);
if (((FlatPtr)where % sizeof(T)) == 0) {
data = *(T*)where;
return;
}
if (sizeof(T) == 2) {
data = *(u8*)where | ((u16)(*(where + 1)) << 8);
return;
}
if (sizeof(T) == 4) {
data = *(u8*)where | (((u32) * (where + 1)) << 8) | (((u32) * (where + 2)) << 16) | (((u32) * (u8*)(where + 3)) << 24);
return;
}
VERIFY_NOT_REACHED();
}
template<typename T>
void write_possibly_unaligned_data(u8* where, T data)
{
// FIXME: Implement 64 bit unaligned data access
VERIFY(sizeof(T) == 8 || sizeof(T) == 4 || sizeof(T) == 2);
if (((FlatPtr)where % sizeof(T)) == 0) {
*(T*)where = data;
return;
}
if (sizeof(T) == 2) {
where[0] = (u8)(data & 0xFF);
where[1] = (u8)((data >> 8) & 0xFF);
return;
}
if (sizeof(T) == 4) {
where[0] = (u8)(data & 0xFF);
where[1] = (u8)((data >> 8) & 0xFF);
where[2] = (u8)((data >> 16) & 0xFF);
where[3] = (u8)((data >> 24) & 0xFF);
return;
}
VERIFY_NOT_REACHED();
}
const DescriptorTablePointer& get_gdtr();
const DescriptorTablePointer& get_idtr();
void register_interrupt_handler(u8 number, void (*handler)());
void register_user_callable_interrupt_handler(u8 number, void (*handler)());
GenericInterruptHandler& get_interrupt_handler(u8 interrupt_number);
void register_generic_interrupt_handler(u8 number, GenericInterruptHandler&);
void unregister_generic_interrupt_handler(u8 number, GenericInterruptHandler&);
void flush_idt();
void load_task_register(u16 selector);
[[noreturn]] void handle_crash(RegisterState&, const char* description, int signal, bool out_of_memory = false);
#define LSW(x) ((u32)(x)&0xFFFF)
#define MSW(x) (((u32)(x) >> 16) & 0xFFFF)
#define LSB(x) ((x)&0xFF)
#define MSB(x) (((x) >> 8) & 0xFF)
#define cli() asm volatile("cli" :: \
: "memory")
#define sti() asm volatile("sti" :: \
: "memory")
inline FlatPtr cpu_flags()
{
FlatPtr flags;
asm volatile(
"pushf\n"
"pop %0\n"
: "=rm"(flags)::"memory");
return flags;
}
inline void set_fs(u16 segment)
{
asm volatile(
"mov %%ax, %%fs" ::"a"(segment)
: "memory");
}
inline void set_gs(u16 segment)
{
asm volatile(
"mov %%ax, %%gs" ::"a"(segment)
: "memory");
}
inline u16 get_fs()
{
u16 fs;
asm("mov %%fs, %%eax"
: "=a"(fs));
return fs;
}
inline u16 get_gs()
{
u16 gs;
asm("mov %%gs, %%eax"
: "=a"(gs));
return gs;
}
inline u32 read_fs_u32(u32 offset)
{
u32 val;
asm volatile(
"movl %%fs:%a[off], %k[val]"
: [val] "=r"(val)
: [off] "ir"(offset));
return val;
}
inline FlatPtr read_fs_ptr(u32 offset)
{
return read_fs_u32(offset);
}
inline void write_fs_u32(u32 offset, u32 val)
{
asm volatile(
"movl %k[val], %%fs:%a[off]" ::[off] "ir"(offset), [val] "ir"(val)
: "memory");
}
inline bool are_interrupts_enabled()
{
return cpu_flags() & 0x200;
}
class InterruptDisabler {
public:
InterruptDisabler()
{
m_flags = cpu_flags();
cli();
}
~InterruptDisabler()
{
if (m_flags & 0x200)
sti();
}
private:
u32 m_flags;
};
class NonMaskableInterruptDisabler {
public:
NonMaskableInterruptDisabler();
~NonMaskableInterruptDisabler();
};
/* Map IRQ0-15 @ ISR 0x50-0x5F */
#define IRQ_VECTOR_BASE 0x50
struct PageFaultFlags {
enum Flags {
NotPresent = 0x00,
ProtectionViolation = 0x01,
Read = 0x00,
Write = 0x02,
UserMode = 0x04,
SupervisorMode = 0x00,
ReservedBitViolation = 0x08,
InstructionFetch = 0x10,
};
};
class PageFault {
public:
PageFault(u16 code, VirtualAddress vaddr)
: m_code(code)
, m_vaddr(vaddr)
{
}
enum class Type {
PageNotPresent = PageFaultFlags::NotPresent,
ProtectionViolation = PageFaultFlags::ProtectionViolation,
};
enum class Access {
Read = PageFaultFlags::Read,
Write = PageFaultFlags::Write,
};
VirtualAddress vaddr() const { return m_vaddr; }
u16 code() const { return m_code; }
Type type() const { return (Type)(m_code & 1); }
Access access() const { return (Access)(m_code & 2); }
bool is_not_present() const { return (m_code & 1) == PageFaultFlags::NotPresent; }
bool is_protection_violation() const { return (m_code & 1) == PageFaultFlags::ProtectionViolation; }
bool is_read() const { return (m_code & 2) == PageFaultFlags::Read; }
bool is_write() const { return (m_code & 2) == PageFaultFlags::Write; }
bool is_user() const { return (m_code & 4) == PageFaultFlags::UserMode; }
bool is_supervisor() const { return (m_code & 4) == PageFaultFlags::SupervisorMode; }
bool is_instruction_fetch() const { return (m_code & 16) == PageFaultFlags::InstructionFetch; }
private:
u16 m_code;
VirtualAddress m_vaddr;
};
struct [[gnu::packed]] RegisterState {
FlatPtr ss;
FlatPtr gs;
FlatPtr fs;
FlatPtr es;
FlatPtr ds;
FlatPtr edi;
FlatPtr esi;
FlatPtr ebp;
FlatPtr esp;
FlatPtr ebx;
FlatPtr edx;
FlatPtr ecx;
FlatPtr eax;
u16 exception_code;
u16 isr_number;
#if ARCH(X86_64)
u32 padding;
#endif
FlatPtr eip;
FlatPtr cs;
FlatPtr eflags;
FlatPtr userspace_esp;
FlatPtr userspace_ss;
};
struct [[gnu::packed]] DebugRegisterState {
FlatPtr dr0;
FlatPtr dr1;
FlatPtr dr2;
FlatPtr dr3;
FlatPtr dr6;
FlatPtr dr7;
};
#if ARCH(I386)
# define REGISTER_STATE_SIZE (19 * 4)
#else
# define REGISTER_STATE_SIZE (19 * 8)
#endif
static_assert(REGISTER_STATE_SIZE == sizeof(RegisterState));
void copy_kernel_registers_into_ptrace_registers(PtraceRegisters&, const RegisterState&);
void copy_ptrace_registers_into_kernel_registers(RegisterState&, const PtraceRegisters&);
struct [[gnu::aligned(16)]] FPUState
{
u8 buffer[512];
};
constexpr FlatPtr page_base_of(FlatPtr address)
{
return address & PAGE_MASK;
}
inline FlatPtr page_base_of(const void* address)
{
return page_base_of((FlatPtr)address);
}
constexpr FlatPtr offset_in_page(FlatPtr address)
{
return address & (~PAGE_MASK);
}
inline FlatPtr offset_in_page(const void* address)
{
return offset_in_page((FlatPtr)address);
}
FlatPtr read_cr0();
FlatPtr read_cr2();
FlatPtr read_cr3();
FlatPtr read_cr4();
u64 read_xcr0();
void write_cr0(FlatPtr);
void write_cr3(FlatPtr);
void write_cr4(FlatPtr);
void write_xcr0(u64);
void read_debug_registers_into(DebugRegisterState&);
void write_debug_registers_from(const DebugRegisterState&);
void clear_debug_registers();
FlatPtr read_dr0();
void write_dr0(FlatPtr);
FlatPtr read_dr1();
void write_dr1(FlatPtr);
FlatPtr read_dr2();
void write_dr2(FlatPtr);
FlatPtr read_dr3();
void write_dr3(FlatPtr);
FlatPtr read_dr6();
void write_dr6(FlatPtr);
FlatPtr read_dr7();
void write_dr7(FlatPtr);
static inline bool is_kernel_mode()
{
u16 cs;
asm volatile(
"mov %%cs, %[cs] \n"
: [cs] "=g"(cs));
return (cs & 3) == 0;
}
class CPUID {
public:
explicit CPUID(u32 function) { asm volatile("cpuid"
: "=a"(m_eax), "=b"(m_ebx), "=c"(m_ecx), "=d"(m_edx)
: "a"(function), "c"(0)); }
u32 eax() const { return m_eax; }
u32 ebx() const { return m_ebx; }
u32 ecx() const { return m_ecx; }
u32 edx() const { return m_edx; }
private:
u32 m_eax { 0xffffffff };
u32 m_ebx { 0xffffffff };
u32 m_ecx { 0xffffffff };
u32 m_edx { 0xffffffff };
};
inline void read_tsc(u32& lsw, u32& msw)
{
asm volatile("rdtsc"
: "=d"(msw), "=a"(lsw));
}
inline u64 read_tsc()
{
u32 lsw;
u32 msw;
read_tsc(lsw, msw);
return ((u64)msw << 32) | lsw;
}
// FIXME: This can't hold every CPU feature as-is.
enum class CPUFeature : u32 {
NX = (1 << 0),
PAE = (1 << 1),
PGE = (1 << 2),
RDRAND = (1 << 3),
RDSEED = (1 << 4),
SMAP = (1 << 5),
SMEP = (1 << 6),
SSE = (1 << 7),
TSC = (1 << 8),
RDTSCP = (1 << 9),
CONSTANT_TSC = (1 << 10),
NONSTOP_TSC = (1 << 11),
UMIP = (1 << 12),
SEP = (1 << 13),
SYSCALL = (1 << 14),
MMX = (1 << 15),
SSE2 = (1 << 16),
SSE3 = (1 << 17),
SSSE3 = (1 << 18),
SSE4_1 = (1 << 19),
SSE4_2 = (1 << 20),
XSAVE = (1 << 21),
AVX = (1 << 22),
};
class Thread;
struct TrapFrame;
#define GDT_SELECTOR_CODE0 0x08
#define GDT_SELECTOR_DATA0 0x10
#define GDT_SELECTOR_CODE3 0x18
#define GDT_SELECTOR_DATA3 0x20
#define GDT_SELECTOR_TLS 0x28
#define GDT_SELECTOR_PROC 0x30
#define GDT_SELECTOR_TSS 0x38
// SYSENTER makes certain assumptions on how the GDT is structured:
static_assert(GDT_SELECTOR_CODE0 + 8 == GDT_SELECTOR_DATA0); // SS0 = CS0 + 8
// SYSEXIT makes certain assumptions on how the GDT is structured:
static_assert(GDT_SELECTOR_CODE0 + 16 == GDT_SELECTOR_CODE3); // CS3 = CS0 + 16
static_assert(GDT_SELECTOR_CODE0 + 24 == GDT_SELECTOR_DATA3); // SS3 = CS0 + 32
class ProcessorInfo;
class SchedulerPerProcessorData;
struct MemoryManagerData;
struct ProcessorMessageEntry;
struct ProcessorMessage {
enum Type {
FlushTlb,
Callback,
CallbackWithData
};
Type type;
volatile u32 refs; // atomic
union {
ProcessorMessage* next; // only valid while in the pool
struct {
void (*handler)();
} callback;
struct {
void* data;
void (*handler)(void*);
void (*free)(void*);
} callback_with_data;
struct {
const PageDirectory* page_directory;
u8* ptr;
size_t page_count;
} flush_tlb;
};
volatile bool async;
ProcessorMessageEntry* per_proc_entries;
};
struct ProcessorMessageEntry {
ProcessorMessageEntry* next;
ProcessorMessage* msg;
};
struct DeferredCallEntry {
DeferredCallEntry* next;
union {
struct {
void (*handler)();
} callback;
struct {
void* data;
void (*handler)(void*);
void (*free)(void*);
} callback_with_data;
};
bool have_data;
bool was_allocated;
};
class Processor {
friend class ProcessorInfo;
AK_MAKE_NONCOPYABLE(Processor);
AK_MAKE_NONMOVABLE(Processor);
Processor* m_self;
DescriptorTablePointer m_gdtr;
Descriptor m_gdt[256];
u32 m_gdt_length;
u32 m_cpu;
u32 m_in_irq;
Atomic<u32, AK::MemoryOrder::memory_order_relaxed> m_in_critical;
static Atomic<u32> s_idle_cpu_mask;
TSS m_tss;
static FPUState s_clean_fpu_state;
CPUFeature m_features;
static volatile u32 g_total_processors; // atomic
u8 m_physical_address_bit_width;
ProcessorInfo* m_info;
MemoryManagerData* m_mm_data;
SchedulerPerProcessorData* m_scheduler_data;
Thread* m_current_thread;
Thread* m_idle_thread;
volatile ProcessorMessageEntry* m_message_queue; // atomic, LIFO
bool m_invoke_scheduler_async;
bool m_scheduler_initialized;
Atomic<bool> m_halt_requested;
DeferredCallEntry* m_pending_deferred_calls; // in reverse order
DeferredCallEntry* m_free_deferred_call_pool_entry;
DeferredCallEntry m_deferred_call_pool[5];
void gdt_init();
void write_raw_gdt_entry(u16 selector, u32 low, u32 high);
void write_gdt_entry(u16 selector, Descriptor& descriptor);
static Vector<Processor*>& processors();
static void smp_return_to_pool(ProcessorMessage& msg);
static ProcessorMessage& smp_get_from_pool();
static void smp_cleanup_message(ProcessorMessage& msg);
bool smp_queue_message(ProcessorMessage& msg);
static void smp_unicast_message(u32 cpu, ProcessorMessage& msg, bool async);
static void smp_broadcast_message(ProcessorMessage& msg);
static void smp_broadcast_wait_sync(ProcessorMessage& msg);
static void smp_broadcast_halt();
void deferred_call_pool_init();
void deferred_call_execute_pending();
DeferredCallEntry* deferred_call_get_free();
void deferred_call_return_to_pool(DeferredCallEntry*);
void deferred_call_queue_entry(DeferredCallEntry*);
void cpu_detect();
void cpu_setup();
String features_string() const;
public:
Processor() = default;
void early_initialize(u32 cpu);
void initialize(u32 cpu);
void idle_begin()
{
s_idle_cpu_mask.fetch_or(1u << m_cpu, AK::MemoryOrder::memory_order_relaxed);
}
void idle_end()
{
s_idle_cpu_mask.fetch_and(~(1u << m_cpu), AK::MemoryOrder::memory_order_relaxed);
}
static u32 count()
{
// NOTE: because this value never changes once all APs are booted,
// we don't really need to do an atomic_load() on this variable
return g_total_processors;
}
ALWAYS_INLINE static void wait_check()
{
Processor::current().smp_process_pending_messages();
// TODO: pause
}
[[noreturn]] static void halt();
static void flush_entire_tlb_local()
{
write_cr3(read_cr3());
}
static void flush_tlb_local(VirtualAddress vaddr, size_t page_count);
static void flush_tlb(const PageDirectory*, VirtualAddress, size_t);
Descriptor& get_gdt_entry(u16 selector);
void flush_gdt();
const DescriptorTablePointer& get_gdtr();
static Processor& by_id(u32 cpu);
static size_t processor_count() { return processors().size(); }
template<IteratorFunction<Processor&> Callback>
static inline IterationDecision for_each(Callback callback)
{
auto& procs = processors();
size_t count = procs.size();
for (size_t i = 0; i < count; i++) {
if (callback(*procs[i]) == IterationDecision::Break)
return IterationDecision::Break;
}
return IterationDecision::Continue;
}
template<VoidFunction<Processor&> Callback>
static inline IterationDecision for_each(Callback callback)
{
auto& procs = processors();
size_t count = procs.size();
for (size_t i = 0; i < count; i++)
callback(*procs[i]);
return IterationDecision::Continue;
}
ALWAYS_INLINE u8 physical_address_bit_width() const { return m_physical_address_bit_width; }
ALWAYS_INLINE ProcessorInfo& info() { return *m_info; }
ALWAYS_INLINE static Processor& current()
{
return *(Processor*)read_fs_ptr(__builtin_offsetof(Processor, m_self));
}
ALWAYS_INLINE static bool is_initialized()
{
return get_fs() == GDT_SELECTOR_PROC && read_fs_u32(__builtin_offsetof(Processor, m_self)) != 0;
}
ALWAYS_INLINE void set_scheduler_data(SchedulerPerProcessorData& scheduler_data)
{
m_scheduler_data = &scheduler_data;
}
ALWAYS_INLINE SchedulerPerProcessorData& get_scheduler_data() const
{
return *m_scheduler_data;
}
ALWAYS_INLINE void set_mm_data(MemoryManagerData& mm_data)
{
m_mm_data = &mm_data;
}
ALWAYS_INLINE MemoryManagerData& get_mm_data() const
{
return *m_mm_data;
}
ALWAYS_INLINE void set_idle_thread(Thread& idle_thread)
{
m_idle_thread = &idle_thread;
}
ALWAYS_INLINE static Thread* current_thread()
{
// If we were to use Processor::current here, we'd have to
// disable interrupts to prevent a race where we may get pre-empted
// right after getting the Processor structure and then get moved
// to another processor, which would lead us to get the wrong thread.
// To avoid having to disable interrupts, we can just read the field
// directly in an atomic fashion, similar to Processor::current.
return (Thread*)read_fs_ptr(__builtin_offsetof(Processor, m_current_thread));
}
ALWAYS_INLINE static void set_current_thread(Thread& current_thread)
{
// See comment in Processor::current_thread
write_fs_u32(__builtin_offsetof(Processor, m_current_thread), FlatPtr(&current_thread));
}
ALWAYS_INLINE static Thread* idle_thread()
{
// See comment in Processor::current_thread
return (Thread*)read_fs_u32(__builtin_offsetof(Processor, m_idle_thread));
}
ALWAYS_INLINE u32 get_id() const
{
// NOTE: This variant should only be used when iterating over all
// Processor instances, or when it's guaranteed that the thread
// cannot move to another processor in between calling Processor::current
// and Processor::get_id, or if this fact is not important.
// All other cases should use Processor::id instead!
return m_cpu;
}
ALWAYS_INLINE static u32 id()
{
// See comment in Processor::current_thread
return read_fs_ptr(__builtin_offsetof(Processor, m_cpu));
}
ALWAYS_INLINE static bool is_bootstrap_processor()
{
return Processor::id() == 0;
}
ALWAYS_INLINE u32 raise_irq()
{
return m_in_irq++;
}
ALWAYS_INLINE void restore_irq(u32 prev_irq)
{
VERIFY(prev_irq <= m_in_irq);
if (!prev_irq) {
u32 prev_critical = 0;
if (m_in_critical.compare_exchange_strong(prev_critical, 1)) {
m_in_irq = prev_irq;
deferred_call_execute_pending();
auto prev_raised = m_in_critical.exchange(prev_critical);
VERIFY(prev_raised == prev_critical + 1);
check_invoke_scheduler();
} else if (prev_critical == 0) {
check_invoke_scheduler();
}
} else {
m_in_irq = prev_irq;
}
}
ALWAYS_INLINE u32& in_irq()
{
return m_in_irq;
}
ALWAYS_INLINE void restore_in_critical(u32 critical)
{
m_in_critical = critical;
}
ALWAYS_INLINE void enter_critical(u32& prev_flags)
{
prev_flags = cpu_flags();
cli();
m_in_critical++;
}
ALWAYS_INLINE void leave_critical(u32 prev_flags)
{
cli(); // Need to prevent IRQs from interrupting us here!
VERIFY(m_in_critical > 0);
if (m_in_critical == 1) {
if (!m_in_irq) {
deferred_call_execute_pending();
VERIFY(m_in_critical == 1);
}
m_in_critical--;
if (!m_in_irq)
check_invoke_scheduler();
} else {
m_in_critical--;
}
if (prev_flags & 0x200)
sti();
else
cli();
}
ALWAYS_INLINE u32 clear_critical(u32& prev_flags, bool enable_interrupts)
{
prev_flags = cpu_flags();
u32 prev_crit = m_in_critical.exchange(0, AK::MemoryOrder::memory_order_acquire);
if (!m_in_irq)
check_invoke_scheduler();
if (enable_interrupts)
sti();
return prev_crit;
}
ALWAYS_INLINE void restore_critical(u32 prev_crit, u32 prev_flags)
{
m_in_critical.store(prev_crit, AK::MemoryOrder::memory_order_release);
VERIFY(!prev_crit || !(prev_flags & 0x200));
if (prev_flags & 0x200)
sti();
else
cli();
}
ALWAYS_INLINE u32 in_critical() { return m_in_critical.load(); }
ALWAYS_INLINE const FPUState& clean_fpu_state() const
{
return s_clean_fpu_state;
}
static void smp_enable();
bool smp_process_pending_messages();
template<typename Callback>
static void smp_broadcast(Callback callback, bool async)
{
auto* data = new Callback(move(callback));
smp_broadcast(
[](void* data) {
(*reinterpret_cast<Callback*>(data))();
},
data,
[](void* data) {
delete reinterpret_cast<Callback*>(data);
},
async);
}
static void smp_broadcast(void (*callback)(), bool async);
static void smp_broadcast(void (*callback)(void*), void* data, void (*free_data)(void*), bool async);
template<typename Callback>
static void smp_unicast(u32 cpu, Callback callback, bool async)
{
auto* data = new Callback(move(callback));
smp_unicast(
cpu,
[](void* data) {
(*reinterpret_cast<Callback*>(data))();
},
data,
[](void* data) {
delete reinterpret_cast<Callback*>(data);
},
async);
}
static void smp_unicast(u32 cpu, void (*callback)(), bool async);
static void smp_unicast(u32 cpu, void (*callback)(void*), void* data, void (*free_data)(void*), bool async);
static void smp_broadcast_flush_tlb(const PageDirectory*, VirtualAddress, size_t);
static u32 smp_wake_n_idle_processors(u32 wake_count);
template<typename Callback>
static void deferred_call_queue(Callback callback)
{
auto* data = new Callback(move(callback));
deferred_call_queue(
[](void* data) {
(*reinterpret_cast<Callback*>(data))();
},
data,
[](void* data) {
delete reinterpret_cast<Callback*>(data);
});
}
static void deferred_call_queue(void (*callback)());
static void deferred_call_queue(void (*callback)(void*), void* data, void (*free_data)(void*));
ALWAYS_INLINE bool has_feature(CPUFeature f) const
{
return (static_cast<u32>(m_features) & static_cast<u32>(f)) != 0;
}
void check_invoke_scheduler();
void invoke_scheduler_async() { m_invoke_scheduler_async = true; }
void enter_trap(TrapFrame& trap, bool raise_irq);
void exit_trap(TrapFrame& trap);
[[noreturn]] void initialize_context_switching(Thread& initial_thread);
NEVER_INLINE void switch_context(Thread*& from_thread, Thread*& to_thread);
[[noreturn]] static void assume_context(Thread& thread, FlatPtr flags);
u32 init_context(Thread& thread, bool leave_crit);
static Vector<FlatPtr> capture_stack_trace(Thread& thread, size_t max_frames = 0);
String platform_string() const;
};
class ScopedCritical {
AK_MAKE_NONCOPYABLE(ScopedCritical);
public:
ScopedCritical()
{
enter();
}
~ScopedCritical()
{
if (m_valid)
leave();
}
ScopedCritical(ScopedCritical&& from)
: m_prev_flags(exchange(from.m_prev_flags, 0))
, m_valid(exchange(from.m_valid, false))
{
}
ScopedCritical& operator=(ScopedCritical&& from)
{
if (&from != this) {
m_prev_flags = exchange(from.m_prev_flags, 0);
m_valid = exchange(from.m_valid, false);
}
return *this;
}
void leave()
{
VERIFY(m_valid);
m_valid = false;
Processor::current().leave_critical(m_prev_flags);
}
void enter()
{
VERIFY(!m_valid);
m_valid = true;
Processor::current().enter_critical(m_prev_flags);
}
private:
u32 m_prev_flags { 0 };
bool m_valid { false };
};
struct TrapFrame {
u32 prev_irq_level;
TrapFrame* next_trap;
RegisterState* regs; // must be last
TrapFrame() = delete;
TrapFrame(const TrapFrame&) = delete;
TrapFrame(TrapFrame&&) = delete;
TrapFrame& operator=(const TrapFrame&) = delete;
TrapFrame& operator=(TrapFrame&&) = delete;
};
#if ARCH(I386)
# define TRAP_FRAME_SIZE (3 * 4)
#else
# define TRAP_FRAME_SIZE (3 * 8)
#endif
static_assert(TRAP_FRAME_SIZE == sizeof(TrapFrame));
extern "C" void enter_trap_no_irq(TrapFrame*) __attribute__((used));
extern "C" void enter_trap(TrapFrame*) __attribute__((used));
extern "C" void exit_trap(TrapFrame*) __attribute__((used));
class MSR {
uint32_t m_msr;
public:
static bool have()
{
CPUID id(1);
return (id.edx() & (1 << 5)) != 0;
}
MSR(const MSR&) = delete;
MSR& operator=(const MSR&) = delete;
MSR(uint32_t msr)
: m_msr(msr)
{
}
void get(u32& low, u32& high)
{
asm volatile("rdmsr"
: "=a"(low), "=d"(high)
: "c"(m_msr));
}
void set(u32 low, u32 high)
{
asm volatile("wrmsr" ::"a"(low), "d"(high), "c"(m_msr));
}
};
ALWAYS_INLINE void stac()
{
if (!Processor::current().has_feature(CPUFeature::SMAP))
return;
asm volatile("stac" ::
: "cc");
}
ALWAYS_INLINE void clac()
{
if (!Processor::current().has_feature(CPUFeature::SMAP))
return;
asm volatile("clac" ::
: "cc");
}
}
Reference in a new issue View git blame Copy permalink