1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-25 14:47:44 +00:00

Kernel: Add SMP IPI support

We can now properly initialize all processors without
crashing by sending SMP IPI messages to synchronize memory
between processors.

We now initialize the APs once we have the scheduler running.
This is so that we can process IPI messages from the other
cores.

Also rework interrupt handling a bit so that it's more of a
1:1 mapping. We need to allocate non-sharable interrupts for
IPIs.

This also fixes the occasional hang/crash because all
CPUs now synchronize memory with each other.
This commit is contained in:
Tom 2020-07-06 07:27:22 -06:00 committed by Andreas Kling
parent dec27e5e6f
commit bc107d0b33
27 changed files with 1236 additions and 627 deletions

View file

@ -49,6 +49,7 @@
//#define PAGE_FAULT_DEBUG
//#define CONTEXT_SWITCH_DEBUG
//#define SMP_DEBUG
namespace Kernel {
@ -153,7 +154,7 @@ void handle_crash(RegisterState& regs, const char* description, int signal, bool
auto process = Process::current();
if (!process) {
klog() << description << " with !current";
hang();
Processor::halt();
}
// If a process crashed while inspecting another process,
@ -166,7 +167,7 @@ void handle_crash(RegisterState& regs, const char* description, int signal, bool
if (process->is_ring0()) {
klog() << "Crash in ring 0 :(";
dump_backtrace();
hang();
Processor::halt();
}
cli();
@ -291,7 +292,7 @@ void debug_handler(TrapFrame* trap)
auto current_thread = Thread::current();
if (&current_thread->process() == nullptr || (regs.cs & 3) == 0) {
klog() << "Debug Exception in Ring0";
hang();
Processor::halt();
return;
}
constexpr u8 REASON_SINGLESTEP = 14;
@ -313,7 +314,7 @@ void breakpoint_handler(TrapFrame* trap)
auto current_thread = Thread::current();
if (&current_thread->process() == nullptr || (regs.cs & 3) == 0) {
klog() << "Breakpoint Trap in Ring0";
hang();
Processor::halt();
return;
}
if (current_thread->tracer()) {
@ -336,7 +337,7 @@ void breakpoint_handler(TrapFrame* trap)
asm("movl %%cr4, %%eax" \
: "=a"(cr4)); \
klog() << "CR0=" << String::format("%x", cr0) << " CR2=" << String::format("%x", cr2) << " CR3=" << String::format("%x", cr3) << " CR4=" << String::format("%x", cr4); \
hang(); \
Processor::halt(); \
}
EH(2, "Unknown error")
@ -358,7 +359,7 @@ const DescriptorTablePointer& get_idtr()
static void unimp_trap()
{
klog() << "Unhandled IRQ.";
hang();
Processor::Processor::halt();
}
GenericInterruptHandler& get_interrupt_handler(u8 interrupt_number)
@ -374,6 +375,7 @@ static void revert_to_unused_handler(u8 interrupt_number)
void register_generic_interrupt_handler(u8 interrupt_number, GenericInterruptHandler& handler)
{
ASSERT(interrupt_number < GENERIC_INTERRUPT_HANDLERS_COUNT);
if (s_interrupt_handler[interrupt_number] != nullptr) {
if (s_interrupt_handler[interrupt_number]->type() == HandlerType::UnhandledInterruptHandler) {
s_interrupt_handler[interrupt_number] = &handler;
@ -442,10 +444,7 @@ void flush_idt()
static void idt_init()
{
s_idtr.address = s_idt;
s_idtr.limit = 0x100 * 8 - 1;
for (u8 i = 0xff; i > 0x10; --i)
register_interrupt_handler(i, unimp_trap);
s_idtr.limit = 256 * 8 - 1;
register_interrupt_handler(0x00, divide_error_asm_entry);
register_user_callable_interrupt_handler(0x01, debug_asm_entry);
@ -465,134 +464,185 @@ static void idt_init()
register_interrupt_handler(0x0f, _exception15);
register_interrupt_handler(0x10, _exception16);
register_interrupt_handler(0x50, interrupt_0_asm_entry);
register_interrupt_handler(0x51, interrupt_1_asm_entry);
register_interrupt_handler(0x52, interrupt_2_asm_entry);
register_interrupt_handler(0x53, interrupt_3_asm_entry);
register_interrupt_handler(0x54, interrupt_4_asm_entry);
register_interrupt_handler(0x55, interrupt_5_asm_entry);
register_interrupt_handler(0x56, interrupt_6_asm_entry);
register_interrupt_handler(0x57, interrupt_7_asm_entry);
register_interrupt_handler(0x58, interrupt_8_asm_entry);
register_interrupt_handler(0x59, interrupt_9_asm_entry);
register_interrupt_handler(0x5a, interrupt_10_asm_entry);
register_interrupt_handler(0x5b, interrupt_11_asm_entry);
register_interrupt_handler(0x5c, interrupt_12_asm_entry);
register_interrupt_handler(0x5d, interrupt_13_asm_entry);
register_interrupt_handler(0x5e, interrupt_14_asm_entry);
register_interrupt_handler(0x5f, interrupt_15_asm_entry);
register_interrupt_handler(0x60, interrupt_16_asm_entry);
register_interrupt_handler(0x61, interrupt_17_asm_entry);
register_interrupt_handler(0x62, interrupt_18_asm_entry);
register_interrupt_handler(0x63, interrupt_19_asm_entry);
register_interrupt_handler(0x64, interrupt_20_asm_entry);
register_interrupt_handler(0x65, interrupt_21_asm_entry);
register_interrupt_handler(0x66, interrupt_22_asm_entry);
register_interrupt_handler(0x67, interrupt_23_asm_entry);
register_interrupt_handler(0x68, interrupt_24_asm_entry);
register_interrupt_handler(0x69, interrupt_25_asm_entry);
register_interrupt_handler(0x6a, interrupt_26_asm_entry);
register_interrupt_handler(0x6b, interrupt_27_asm_entry);
register_interrupt_handler(0x6c, interrupt_28_asm_entry);
register_interrupt_handler(0x6d, interrupt_29_asm_entry);
register_interrupt_handler(0x6e, interrupt_30_asm_entry);
register_interrupt_handler(0x6f, interrupt_31_asm_entry);
register_interrupt_handler(0x70, interrupt_32_asm_entry);
register_interrupt_handler(0x71, interrupt_33_asm_entry);
register_interrupt_handler(0x72, interrupt_34_asm_entry);
register_interrupt_handler(0x73, interrupt_35_asm_entry);
register_interrupt_handler(0x74, interrupt_36_asm_entry);
register_interrupt_handler(0x75, interrupt_37_asm_entry);
register_interrupt_handler(0x76, interrupt_38_asm_entry);
register_interrupt_handler(0x77, interrupt_39_asm_entry);
register_interrupt_handler(0x78, interrupt_40_asm_entry);
register_interrupt_handler(0x79, interrupt_41_asm_entry);
register_interrupt_handler(0x7a, interrupt_42_asm_entry);
register_interrupt_handler(0x7b, interrupt_43_asm_entry);
register_interrupt_handler(0x7c, interrupt_44_asm_entry);
register_interrupt_handler(0x7d, interrupt_45_asm_entry);
register_interrupt_handler(0x7e, interrupt_46_asm_entry);
register_interrupt_handler(0x7f, interrupt_47_asm_entry);
register_interrupt_handler(0x80, interrupt_48_asm_entry);
register_interrupt_handler(0x81, interrupt_49_asm_entry);
register_interrupt_handler(0x82, interrupt_50_asm_entry);
register_interrupt_handler(0x83, interrupt_51_asm_entry);
register_interrupt_handler(0x84, interrupt_52_asm_entry);
register_interrupt_handler(0x85, interrupt_53_asm_entry);
register_interrupt_handler(0x86, interrupt_54_asm_entry);
register_interrupt_handler(0x87, interrupt_55_asm_entry);
register_interrupt_handler(0x88, interrupt_56_asm_entry);
register_interrupt_handler(0x89, interrupt_57_asm_entry);
register_interrupt_handler(0x8a, interrupt_58_asm_entry);
register_interrupt_handler(0x8b, interrupt_59_asm_entry);
register_interrupt_handler(0x8c, interrupt_60_asm_entry);
register_interrupt_handler(0x8d, interrupt_61_asm_entry);
register_interrupt_handler(0x8e, interrupt_62_asm_entry);
register_interrupt_handler(0x8f, interrupt_63_asm_entry);
register_interrupt_handler(0x90, interrupt_64_asm_entry);
register_interrupt_handler(0x91, interrupt_65_asm_entry);
register_interrupt_handler(0x92, interrupt_66_asm_entry);
register_interrupt_handler(0x93, interrupt_67_asm_entry);
register_interrupt_handler(0x94, interrupt_68_asm_entry);
register_interrupt_handler(0x95, interrupt_69_asm_entry);
register_interrupt_handler(0x96, interrupt_70_asm_entry);
register_interrupt_handler(0x97, interrupt_71_asm_entry);
register_interrupt_handler(0x98, interrupt_72_asm_entry);
register_interrupt_handler(0x99, interrupt_73_asm_entry);
register_interrupt_handler(0x9a, interrupt_74_asm_entry);
register_interrupt_handler(0x9b, interrupt_75_asm_entry);
register_interrupt_handler(0x9c, interrupt_76_asm_entry);
register_interrupt_handler(0x9d, interrupt_77_asm_entry);
register_interrupt_handler(0x9e, interrupt_78_asm_entry);
register_interrupt_handler(0x9f, interrupt_79_asm_entry);
register_interrupt_handler(0xa0, interrupt_80_asm_entry);
register_interrupt_handler(0xa1, interrupt_81_asm_entry);
register_interrupt_handler(0xa2, interrupt_82_asm_entry);
register_interrupt_handler(0xa3, interrupt_83_asm_entry);
register_interrupt_handler(0xa4, interrupt_84_asm_entry);
register_interrupt_handler(0xa5, interrupt_85_asm_entry);
register_interrupt_handler(0xa6, interrupt_86_asm_entry);
register_interrupt_handler(0xa7, interrupt_87_asm_entry);
register_interrupt_handler(0xa8, interrupt_88_asm_entry);
register_interrupt_handler(0xa9, interrupt_89_asm_entry);
register_interrupt_handler(0xaa, interrupt_90_asm_entry);
register_interrupt_handler(0xab, interrupt_91_asm_entry);
register_interrupt_handler(0xac, interrupt_92_asm_entry);
register_interrupt_handler(0xad, interrupt_93_asm_entry);
register_interrupt_handler(0xae, interrupt_94_asm_entry);
register_interrupt_handler(0xaf, interrupt_95_asm_entry);
register_interrupt_handler(0xb0, interrupt_96_asm_entry);
register_interrupt_handler(0xb1, interrupt_97_asm_entry);
register_interrupt_handler(0xb2, interrupt_98_asm_entry);
register_interrupt_handler(0xb3, interrupt_99_asm_entry);
register_interrupt_handler(0xb4, interrupt_100_asm_entry);
register_interrupt_handler(0xb5, interrupt_101_asm_entry);
register_interrupt_handler(0xb6, interrupt_102_asm_entry);
register_interrupt_handler(0xb7, interrupt_103_asm_entry);
register_interrupt_handler(0xb8, interrupt_104_asm_entry);
register_interrupt_handler(0xb9, interrupt_105_asm_entry);
register_interrupt_handler(0xba, interrupt_106_asm_entry);
register_interrupt_handler(0xbb, interrupt_107_asm_entry);
register_interrupt_handler(0xbc, interrupt_108_asm_entry);
register_interrupt_handler(0xbd, interrupt_109_asm_entry);
register_interrupt_handler(0xbe, interrupt_110_asm_entry);
register_interrupt_handler(0xbf, interrupt_111_asm_entry);
register_interrupt_handler(0xc0, interrupt_112_asm_entry);
register_interrupt_handler(0xc1, interrupt_113_asm_entry);
register_interrupt_handler(0xc2, interrupt_114_asm_entry);
register_interrupt_handler(0xc3, interrupt_115_asm_entry);
register_interrupt_handler(0xc4, interrupt_116_asm_entry);
register_interrupt_handler(0xc5, interrupt_117_asm_entry);
register_interrupt_handler(0xc6, interrupt_118_asm_entry);
register_interrupt_handler(0xc7, interrupt_119_asm_entry);
register_interrupt_handler(0xc8, interrupt_120_asm_entry);
register_interrupt_handler(0xc9, interrupt_121_asm_entry);
register_interrupt_handler(0xca, interrupt_122_asm_entry);
register_interrupt_handler(0xcb, interrupt_123_asm_entry);
register_interrupt_handler(0xcc, interrupt_124_asm_entry);
register_interrupt_handler(0xcd, interrupt_125_asm_entry);
register_interrupt_handler(0xce, interrupt_126_asm_entry);
register_interrupt_handler(0xcf, interrupt_127_asm_entry);
for (u8 i = 0x11; i < 0x50; i++)
register_interrupt_handler(i, unimp_trap);
register_interrupt_handler(0x50, interrupt_80_asm_entry);
register_interrupt_handler(0x51, interrupt_81_asm_entry);
register_interrupt_handler(0x52, interrupt_82_asm_entry);
register_interrupt_handler(0x53, interrupt_83_asm_entry);
register_interrupt_handler(0x54, interrupt_84_asm_entry);
register_interrupt_handler(0x55, interrupt_85_asm_entry);
register_interrupt_handler(0x56, interrupt_86_asm_entry);
register_interrupt_handler(0x57, interrupt_87_asm_entry);
register_interrupt_handler(0x58, interrupt_88_asm_entry);
register_interrupt_handler(0x59, interrupt_89_asm_entry);
register_interrupt_handler(0x5a, interrupt_90_asm_entry);
register_interrupt_handler(0x5b, interrupt_91_asm_entry);
register_interrupt_handler(0x5c, interrupt_92_asm_entry);
register_interrupt_handler(0x5d, interrupt_93_asm_entry);
register_interrupt_handler(0x5e, interrupt_94_asm_entry);
register_interrupt_handler(0x5f, interrupt_95_asm_entry);
register_interrupt_handler(0x60, interrupt_96_asm_entry);
register_interrupt_handler(0x61, interrupt_97_asm_entry);
register_interrupt_handler(0x62, interrupt_98_asm_entry);
register_interrupt_handler(0x63, interrupt_99_asm_entry);
register_interrupt_handler(0x64, interrupt_100_asm_entry);
register_interrupt_handler(0x65, interrupt_101_asm_entry);
register_interrupt_handler(0x66, interrupt_102_asm_entry);
register_interrupt_handler(0x67, interrupt_103_asm_entry);
register_interrupt_handler(0x68, interrupt_104_asm_entry);
register_interrupt_handler(0x69, interrupt_105_asm_entry);
register_interrupt_handler(0x6a, interrupt_106_asm_entry);
register_interrupt_handler(0x6b, interrupt_107_asm_entry);
register_interrupt_handler(0x6c, interrupt_108_asm_entry);
register_interrupt_handler(0x6d, interrupt_109_asm_entry);
register_interrupt_handler(0x6e, interrupt_110_asm_entry);
register_interrupt_handler(0x6f, interrupt_111_asm_entry);
register_interrupt_handler(0x70, interrupt_112_asm_entry);
register_interrupt_handler(0x71, interrupt_113_asm_entry);
register_interrupt_handler(0x72, interrupt_114_asm_entry);
register_interrupt_handler(0x73, interrupt_115_asm_entry);
register_interrupt_handler(0x74, interrupt_116_asm_entry);
register_interrupt_handler(0x75, interrupt_117_asm_entry);
register_interrupt_handler(0x76, interrupt_118_asm_entry);
register_interrupt_handler(0x77, interrupt_119_asm_entry);
register_interrupt_handler(0x78, interrupt_120_asm_entry);
register_interrupt_handler(0x79, interrupt_121_asm_entry);
register_interrupt_handler(0x7a, interrupt_122_asm_entry);
register_interrupt_handler(0x7b, interrupt_123_asm_entry);
register_interrupt_handler(0x7c, interrupt_124_asm_entry);
register_interrupt_handler(0x7d, interrupt_125_asm_entry);
register_interrupt_handler(0x7e, interrupt_126_asm_entry);
register_interrupt_handler(0x7f, interrupt_127_asm_entry);
register_interrupt_handler(0x80, interrupt_128_asm_entry);
register_interrupt_handler(0x81, interrupt_129_asm_entry);
register_interrupt_handler(0x82, interrupt_130_asm_entry);
register_interrupt_handler(0x83, interrupt_131_asm_entry);
register_interrupt_handler(0x84, interrupt_132_asm_entry);
register_interrupt_handler(0x85, interrupt_133_asm_entry);
register_interrupt_handler(0x86, interrupt_134_asm_entry);
register_interrupt_handler(0x87, interrupt_135_asm_entry);
register_interrupt_handler(0x88, interrupt_136_asm_entry);
register_interrupt_handler(0x89, interrupt_137_asm_entry);
register_interrupt_handler(0x8a, interrupt_138_asm_entry);
register_interrupt_handler(0x8b, interrupt_139_asm_entry);
register_interrupt_handler(0x8c, interrupt_140_asm_entry);
register_interrupt_handler(0x8d, interrupt_141_asm_entry);
register_interrupt_handler(0x8e, interrupt_142_asm_entry);
register_interrupt_handler(0x8f, interrupt_143_asm_entry);
register_interrupt_handler(0x90, interrupt_144_asm_entry);
register_interrupt_handler(0x91, interrupt_145_asm_entry);
register_interrupt_handler(0x92, interrupt_146_asm_entry);
register_interrupt_handler(0x93, interrupt_147_asm_entry);
register_interrupt_handler(0x94, interrupt_148_asm_entry);
register_interrupt_handler(0x95, interrupt_149_asm_entry);
register_interrupt_handler(0x96, interrupt_150_asm_entry);
register_interrupt_handler(0x97, interrupt_151_asm_entry);
register_interrupt_handler(0x98, interrupt_152_asm_entry);
register_interrupt_handler(0x99, interrupt_153_asm_entry);
register_interrupt_handler(0x9a, interrupt_154_asm_entry);
register_interrupt_handler(0x9b, interrupt_155_asm_entry);
register_interrupt_handler(0x9c, interrupt_156_asm_entry);
register_interrupt_handler(0x9d, interrupt_157_asm_entry);
register_interrupt_handler(0x9e, interrupt_158_asm_entry);
register_interrupt_handler(0x9f, interrupt_159_asm_entry);
register_interrupt_handler(0xa0, interrupt_160_asm_entry);
register_interrupt_handler(0xa1, interrupt_161_asm_entry);
register_interrupt_handler(0xa2, interrupt_162_asm_entry);
register_interrupt_handler(0xa3, interrupt_163_asm_entry);
register_interrupt_handler(0xa4, interrupt_164_asm_entry);
register_interrupt_handler(0xa5, interrupt_165_asm_entry);
register_interrupt_handler(0xa6, interrupt_166_asm_entry);
register_interrupt_handler(0xa7, interrupt_167_asm_entry);
register_interrupt_handler(0xa8, interrupt_168_asm_entry);
register_interrupt_handler(0xa9, interrupt_169_asm_entry);
register_interrupt_handler(0xaa, interrupt_170_asm_entry);
register_interrupt_handler(0xab, interrupt_171_asm_entry);
register_interrupt_handler(0xac, interrupt_172_asm_entry);
register_interrupt_handler(0xad, interrupt_173_asm_entry);
register_interrupt_handler(0xae, interrupt_174_asm_entry);
register_interrupt_handler(0xaf, interrupt_175_asm_entry);
register_interrupt_handler(0xb0, interrupt_176_asm_entry);
register_interrupt_handler(0xb1, interrupt_177_asm_entry);
register_interrupt_handler(0xb2, interrupt_178_asm_entry);
register_interrupt_handler(0xb3, interrupt_179_asm_entry);
register_interrupt_handler(0xb4, interrupt_180_asm_entry);
register_interrupt_handler(0xb5, interrupt_181_asm_entry);
register_interrupt_handler(0xb6, interrupt_182_asm_entry);
register_interrupt_handler(0xb7, interrupt_183_asm_entry);
register_interrupt_handler(0xb8, interrupt_184_asm_entry);
register_interrupt_handler(0xb9, interrupt_185_asm_entry);
register_interrupt_handler(0xba, interrupt_186_asm_entry);
register_interrupt_handler(0xbb, interrupt_187_asm_entry);
register_interrupt_handler(0xbc, interrupt_188_asm_entry);
register_interrupt_handler(0xbd, interrupt_189_asm_entry);
register_interrupt_handler(0xbe, interrupt_190_asm_entry);
register_interrupt_handler(0xbf, interrupt_191_asm_entry);
register_interrupt_handler(0xc0, interrupt_192_asm_entry);
register_interrupt_handler(0xc1, interrupt_193_asm_entry);
register_interrupt_handler(0xc2, interrupt_194_asm_entry);
register_interrupt_handler(0xc3, interrupt_195_asm_entry);
register_interrupt_handler(0xc4, interrupt_196_asm_entry);
register_interrupt_handler(0xc5, interrupt_197_asm_entry);
register_interrupt_handler(0xc6, interrupt_198_asm_entry);
register_interrupt_handler(0xc7, interrupt_199_asm_entry);
register_interrupt_handler(0xc8, interrupt_200_asm_entry);
register_interrupt_handler(0xc9, interrupt_201_asm_entry);
register_interrupt_handler(0xca, interrupt_202_asm_entry);
register_interrupt_handler(0xcb, interrupt_203_asm_entry);
register_interrupt_handler(0xcc, interrupt_204_asm_entry);
register_interrupt_handler(0xcd, interrupt_205_asm_entry);
register_interrupt_handler(0xce, interrupt_206_asm_entry);
register_interrupt_handler(0xcf, interrupt_207_asm_entry);
register_interrupt_handler(0xd0, interrupt_208_asm_entry);
register_interrupt_handler(0xd1, interrupt_209_asm_entry);
register_interrupt_handler(0xd2, interrupt_210_asm_entry);
register_interrupt_handler(0xd3, interrupt_211_asm_entry);
register_interrupt_handler(0xd4, interrupt_212_asm_entry);
register_interrupt_handler(0xd5, interrupt_213_asm_entry);
register_interrupt_handler(0xd6, interrupt_214_asm_entry);
register_interrupt_handler(0xd7, interrupt_215_asm_entry);
register_interrupt_handler(0xd8, interrupt_216_asm_entry);
register_interrupt_handler(0xd9, interrupt_217_asm_entry);
register_interrupt_handler(0xda, interrupt_218_asm_entry);
register_interrupt_handler(0xdb, interrupt_219_asm_entry);
register_interrupt_handler(0xdc, interrupt_220_asm_entry);
register_interrupt_handler(0xdd, interrupt_221_asm_entry);
register_interrupt_handler(0xde, interrupt_222_asm_entry);
register_interrupt_handler(0xdf, interrupt_223_asm_entry);
register_interrupt_handler(0xe0, interrupt_224_asm_entry);
register_interrupt_handler(0xe1, interrupt_225_asm_entry);
register_interrupt_handler(0xe2, interrupt_226_asm_entry);
register_interrupt_handler(0xe3, interrupt_227_asm_entry);
register_interrupt_handler(0xe4, interrupt_228_asm_entry);
register_interrupt_handler(0xe5, interrupt_229_asm_entry);
register_interrupt_handler(0xe6, interrupt_230_asm_entry);
register_interrupt_handler(0xe7, interrupt_231_asm_entry);
register_interrupt_handler(0xe8, interrupt_232_asm_entry);
register_interrupt_handler(0xe9, interrupt_233_asm_entry);
register_interrupt_handler(0xea, interrupt_234_asm_entry);
register_interrupt_handler(0xeb, interrupt_235_asm_entry);
register_interrupt_handler(0xec, interrupt_236_asm_entry);
register_interrupt_handler(0xed, interrupt_237_asm_entry);
register_interrupt_handler(0xee, interrupt_238_asm_entry);
register_interrupt_handler(0xef, interrupt_239_asm_entry);
register_interrupt_handler(0xf0, interrupt_240_asm_entry);
register_interrupt_handler(0xf1, interrupt_241_asm_entry);
register_interrupt_handler(0xf2, interrupt_242_asm_entry);
register_interrupt_handler(0xf3, interrupt_243_asm_entry);
register_interrupt_handler(0xf4, interrupt_244_asm_entry);
register_interrupt_handler(0xf5, interrupt_245_asm_entry);
register_interrupt_handler(0xf6, interrupt_246_asm_entry);
register_interrupt_handler(0xf7, interrupt_247_asm_entry);
register_interrupt_handler(0xf8, interrupt_248_asm_entry);
register_interrupt_handler(0xf9, interrupt_249_asm_entry);
register_interrupt_handler(0xfa, interrupt_250_asm_entry);
register_interrupt_handler(0xfb, interrupt_251_asm_entry);
register_interrupt_handler(0xfc, interrupt_252_asm_entry);
register_interrupt_handler(0xfd, interrupt_253_asm_entry);
register_interrupt_handler(0xfe, interrupt_254_asm_entry);
register_interrupt_handler(0xff, interrupt_255_asm_entry);
dbg() << "Installing Unhandled Handlers";
@ -688,6 +738,8 @@ FPUState Processor::s_clean_fpu_state;
static Vector<Processor*>* s_processors;
static SpinLock s_processor_lock;
volatile u32 Processor::g_total_processors;
static volatile bool s_smp_enabled;
Vector<Processor*>& Processor::processors()
{
@ -707,6 +759,12 @@ Processor& Processor::by_id(u32 cpu)
return *procs[cpu];
}
[[noreturn]] static inline void halt_this()
{
for (;;) {
asm volatile("cli; hlt");
}
}
void Processor::cpu_detect()
{
@ -868,13 +926,21 @@ void Processor::early_initialize(u32 cpu)
m_invoke_scheduler_async = false;
m_scheduler_initialized = false;
m_message_queue = nullptr;
m_idle_thread = nullptr;
m_current_thread = nullptr;
m_mm_data = nullptr;
m_info = nullptr;
cpu_setup();
m_halt_requested = false;
if (cpu == 0) {
s_smp_enabled = false;
atomic_store(&g_total_processors, 1u, AK::MemoryOrder::memory_order_release);
} else {
atomic_fetch_add(&g_total_processors, 1u, AK::MemoryOrder::memory_order_acq_rel);
}
cpu_setup();
gdt_init();
ASSERT(&current() == this); // sanity check
}
@ -910,9 +976,9 @@ void Processor::initialize(u32 cpu)
if (cpu >= s_processors->size())
s_processors->resize(cpu + 1);
(*s_processors)[cpu] = this;
}
klog() << "CPU[" << cpu << "]: initialized Processor at " << VirtualAddress(FlatPtr(this));
klog() << "CPU[" << cpu << "]: initialized Processor at " << VirtualAddress(FlatPtr(this));
}
}
void Processor::write_raw_gdt_entry(u16 selector, u32 low, u32 high)
@ -979,7 +1045,7 @@ bool Processor::get_context_frame_ptr(Thread& thread, u32& frame_ptr, u32& eip)
// TODO: If this is the case, the thread is currently running
// on another processor. We can't trust the kernel stack as
// it may be changing at any time. We need to probably send
// an ICI to that processor, have it walk the stack and wait
// an IPI to that processor, have it walk the stack and wait
// until it returns the data back to us
dbg() << "CPU[" << proc.id() << "] getting stack for "
<< thread << " on other CPU# " << thread.cpu() << " not yet implemented!";
@ -1288,12 +1354,16 @@ void Processor::initialize_context_switching(Thread& initial_thread)
"addl $20, %%ebx \n" // calculate pointer to TrapFrame
"pushl %%ebx \n"
"cld \n"
"pushl %[cpu] \n"
"call init_finished \n"
"addl $4, %%esp \n"
"call enter_trap_no_irq \n"
"addl $4, %%esp \n"
"lret \n"
:: [new_esp] "g" (tss.esp),
[new_eip] "a" (tss.eip),
[from_to_thread] "b" (&initial_thread)
[from_to_thread] "b" (&initial_thread),
[cpu] "c" (id())
);
ASSERT_NOT_REACHED();
@ -1312,7 +1382,9 @@ void Processor::exit_trap(TrapFrame& trap)
InterruptDisabler disabler;
ASSERT(m_in_irq >= trap.prev_irq_level);
m_in_irq = trap.prev_irq_level;
smp_process_pending_messages();
if (!m_in_irq && !m_in_critical)
check_invoke_scheduler();
}
@ -1327,6 +1399,267 @@ void Processor::check_invoke_scheduler()
}
}
void Processor::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
{
auto ptr = vaddr.as_ptr();
while (page_count > 0) {
asm volatile("invlpg %0"
:
: "m"(*(char*)vaddr.get())
: "memory");
ptr += PAGE_SIZE;
page_count--;
}
}
void Processor::flush_tlb(VirtualAddress vaddr, size_t page_count)
{
flush_tlb_local(vaddr, page_count);
if (s_smp_enabled)
smp_broadcast_flush_tlb(vaddr, page_count);
}
static volatile ProcessorMessage* s_message_pool;
void Processor::smp_return_to_pool(ProcessorMessage& msg)
{
ProcessorMessage* next = nullptr;
do {
msg.next = next;
} while (!atomic_compare_exchange_strong(&s_message_pool, next, &msg, AK::MemoryOrder::memory_order_acq_rel));
}
ProcessorMessage& Processor::smp_get_from_pool()
{
ProcessorMessage* msg;
// The assumption is that messages are never removed from the pool!
for (;;) {
msg = atomic_load(&s_message_pool, AK::MemoryOrder::memory_order_consume);
if (!msg) {
if (!Processor::current().smp_process_pending_messages()) {
// TODO: pause for a bit?
}
continue;
}
// If another processor were to use this message in the meanwhile,
// "msg" is still valid (because it never gets freed). We'd detect
// this because the expected value "msg" and pool would
// no longer match, and the compare_exchange will fail. But accessing
// "msg->next" is always safe here.
if (atomic_compare_exchange_strong(&s_message_pool, msg, msg->next, AK::MemoryOrder::memory_order_acq_rel)) {
// We successfully "popped" this available message
break;
}
}
ASSERT(msg != nullptr);
return *msg;
}
void Processor::smp_enable()
{
size_t msg_pool_size = Processor::count() * 100u;
size_t msg_entries_cnt = Processor::count();
auto msgs = new ProcessorMessage[msg_pool_size];
auto msg_entries = new ProcessorMessageEntry[msg_pool_size * msg_entries_cnt];
size_t msg_entry_i = 0;
for (size_t i = 0; i < msg_pool_size; i++, msg_entry_i += msg_entries_cnt) {
auto& msg = msgs[i];
msg.next = i < msg_pool_size - 1 ? &msgs[i + 1] : nullptr;
msg.per_proc_entries = &msg_entries[msg_entry_i];
for (size_t k = 0; k < msg_entries_cnt; k++)
msg_entries[msg_entry_i + k].msg = &msg;
}
atomic_store(&s_message_pool, &msgs[0], AK::MemoryOrder::memory_order_release);
// Start sending IPI messages
s_smp_enabled = true;
}
void Processor::smp_cleanup_message(ProcessorMessage& msg)
{
switch (msg.type) {
case ProcessorMessage::CallbackWithData:
if (msg.callback_with_data.free)
msg.callback_with_data.free(msg.callback_with_data.data);
break;
default:
break;
}
}
bool Processor::smp_process_pending_messages()
{
bool did_process = false;
u32 prev_flags;
enter_critical(prev_flags);
if (auto pending_msgs = atomic_exchange(&m_message_queue, nullptr, AK::MemoryOrder::memory_order_acq_rel))
{
// We pulled the stack of pending messages in LIFO order, so we need to reverse the list first
auto reverse_list =
[](ProcessorMessageEntry* list) -> ProcessorMessageEntry*
{
ProcessorMessageEntry* rev_list = nullptr;
while (list) {
auto next = list->next;
list->next = rev_list;
rev_list = list;
list = next;
}
return rev_list;
};
pending_msgs = reverse_list(pending_msgs);
// now process in the right order
ProcessorMessageEntry* next_msg;
for (auto cur_msg = pending_msgs; cur_msg; cur_msg = next_msg) {
next_msg = cur_msg->next;
auto msg = cur_msg->msg;
#ifdef SMP_DEBUG
dbg() << "SMP[" << id() << "]: Processing message " << VirtualAddress(msg);
#endif
switch (msg->type) {
case ProcessorMessage::Callback:
msg->callback.handler();
break;
case ProcessorMessage::CallbackWithData:
msg->callback_with_data.handler(msg->callback_with_data.data);
break;
case ProcessorMessage::FlushTlb:
flush_tlb_local(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count);
break;
}
bool is_async = msg->async; // Need to cache this value *before* dropping the ref count!
auto prev_refs = atomic_fetch_sub(&msg->refs, 1u, AK::MemoryOrder::memory_order_acq_rel);
ASSERT(prev_refs != 0);
if (prev_refs == 1) {
// All processors handled this. If this is an async message,
// we need to clean it up and return it to the pool
if (is_async) {
smp_cleanup_message(*msg);
smp_return_to_pool(*msg);
}
}
if (m_halt_requested)
halt_this();
}
did_process = true;
} else if (m_halt_requested) {
halt_this();
}
leave_critical(prev_flags);
return did_process;
}
bool Processor::smp_queue_message(ProcessorMessage& msg)
{
// Note that it's quite possible that the other processor may pop
// the queue at any given time. We rely on the fact that the messages
// are pooled and never get freed!
auto& msg_entry = msg.per_proc_entries[id()];
ASSERT(msg_entry.msg == &msg);
ProcessorMessageEntry* next = nullptr;
do {
msg_entry.next = next;
} while (!atomic_compare_exchange_strong(&m_message_queue, next, &msg_entry, AK::MemoryOrder::memory_order_acq_rel));
return next == nullptr;
}
void Processor::smp_broadcast_message(ProcessorMessage& msg, bool async)
{
auto& cur_proc = Processor::current();
msg.async = async;
#ifdef SMP_DEBUG
dbg() << "SMP[" << cur_proc.id() << "]: Broadcast message " << VirtualAddress(&msg) << " to cpus: " << (count()) << " proc: " << VirtualAddress(&cur_proc);
#endif
atomic_store(&msg.refs, count() - 1, AK::MemoryOrder::memory_order_release);
ASSERT(msg.refs > 0);
for_each(
[&](Processor& proc) -> IterationDecision
{
if (&proc != &cur_proc) {
if (proc.smp_queue_message(msg)) {
// TODO: only send IPI to that CPU if we queued the first
}
}
return IterationDecision::Continue;
});
// Now trigger an IPI on all other APs
APIC::the().broadcast_ipi();
if (!async) {
// If synchronous then we must cleanup and return the message back
// to the pool. Otherwise, the last processor to complete it will return it
while (atomic_load(&msg.refs, AK::MemoryOrder::memory_order_consume) != 0) {
// TODO: pause for a bit?
}
smp_cleanup_message(msg);
smp_return_to_pool(msg);
}
}
void Processor::smp_broadcast(void(*callback)(void*), void* data, void(*free_data)(void*), bool async)
{
auto& msg = smp_get_from_pool();
msg.type = ProcessorMessage::CallbackWithData;
msg.callback_with_data.handler = callback;
msg.callback_with_data.data = data;
msg.callback_with_data.free = free_data;
smp_broadcast_message(msg, async);
}
void Processor::smp_broadcast(void(*callback)(), bool async)
{
auto& msg = smp_get_from_pool();
msg.type = ProcessorMessage::CallbackWithData;
msg.callback.handler = callback;
smp_broadcast_message(msg, async);
}
void Processor::smp_broadcast_flush_tlb(VirtualAddress vaddr, size_t page_count)
{
auto& msg = smp_get_from_pool();
msg.type = ProcessorMessage::FlushTlb;
msg.flush_tlb.ptr = vaddr.as_ptr();
msg.flush_tlb.page_count = page_count;
smp_broadcast_message(msg, false);
}
void Processor::smp_broadcast_halt()
{
// We don't want to use a message, because this could have been triggered
// by being out of memory and we might not be able to get a message
for_each(
[&](Processor& proc) -> IterationDecision
{
proc.m_halt_requested = true;
return IterationDecision::Continue;
});
// Now trigger an IPI on all other APs
APIC::the().broadcast_ipi();
}
void Processor::Processor::halt()
{
if (s_smp_enabled)
smp_broadcast_halt();
halt_this();
}
void Processor::gdt_init()
{
m_gdt_length = 0;