mirror of
https://github.com/RGBCube/serenity
synced 2025-07-26 05:07:35 +00:00
Kernel: Add SMP IPI support
We can now properly initialize all processors without crashing by sending SMP IPI messages to synchronize memory between processors. We now initialize the APs once we have the scheduler running. This is so that we can process IPI messages from the other cores. Also rework interrupt handling a bit so that it's more of a 1:1 mapping. We need to allocate non-sharable interrupts for IPIs. This also fixes the occasional hang/crash because all CPUs now synchronize memory with each other.
This commit is contained in:
Tom
Andreas Kling
parent
dec27e5e6f
commit
bc107d0b33
27 changed files with 1236 additions and 627 deletions
|
|
@ -49,6 +49,7 @@
|
|||
|
||||
//#define PAGE_FAULT_DEBUG
|
||||
//#define CONTEXT_SWITCH_DEBUG
|
||||
//#define SMP_DEBUG
|
||||
|
||||
namespace Kernel {
|
||||
|
||||
|
|
@ -153,7 +154,7 @@ void handle_crash(RegisterState& regs, const char* description, int signal, bool
|
|||
auto process = Process::current();
|
||||
if (!process) {
|
||||
klog() << description << " with !current";
|
||||
hang();
|
||||
Processor::halt();
|
||||
}
|
||||
|
||||
// If a process crashed while inspecting another process,
|
||||
|
|
@ -166,7 +167,7 @@ void handle_crash(RegisterState& regs, const char* description, int signal, bool
|
|||
if (process->is_ring0()) {
|
||||
klog() << "Crash in ring 0 :(";
|
||||
dump_backtrace();
|
||||
hang();
|
||||
Processor::halt();
|
||||
}
|
||||
|
||||
cli();
|
||||
|
|
@ -291,7 +292,7 @@ void debug_handler(TrapFrame* trap)
|
|||
auto current_thread = Thread::current();
|
||||
if (¤t_thread->process() == nullptr || (regs.cs & 3) == 0) {
|
||||
klog() << "Debug Exception in Ring0";
|
||||
hang();
|
||||
Processor::halt();
|
||||
return;
|
||||
}
|
||||
constexpr u8 REASON_SINGLESTEP = 14;
|
||||
|
|
@ -313,7 +314,7 @@ void breakpoint_handler(TrapFrame* trap)
|
|||
auto current_thread = Thread::current();
|
||||
if (¤t_thread->process() == nullptr || (regs.cs & 3) == 0) {
|
||||
klog() << "Breakpoint Trap in Ring0";
|
||||
hang();
|
||||
Processor::halt();
|
||||
return;
|
||||
}
|
||||
if (current_thread->tracer()) {
|
||||
|
|
@ -336,7 +337,7 @@ void breakpoint_handler(TrapFrame* trap)
|
|||
asm("movl %%cr4, %%eax" \
|
||||
: "=a"(cr4)); \
|
||||
klog() << "CR0=" << String::format("%x", cr0) << " CR2=" << String::format("%x", cr2) << " CR3=" << String::format("%x", cr3) << " CR4=" << String::format("%x", cr4); \
|
||||
hang(); \
|
||||
Processor::halt(); \
|
||||
}
|
||||
|
||||
EH(2, "Unknown error")
|
||||
|
|
@ -358,7 +359,7 @@ const DescriptorTablePointer& get_idtr()
|
|||
static void unimp_trap()
|
||||
{
|
||||
klog() << "Unhandled IRQ.";
|
||||
hang();
|
||||
Processor::Processor::halt();
|
||||
}
|
||||
|
||||
GenericInterruptHandler& get_interrupt_handler(u8 interrupt_number)
|
||||
|
|
@ -374,6 +375,7 @@ static void revert_to_unused_handler(u8 interrupt_number)
|
|||
|
||||
void register_generic_interrupt_handler(u8 interrupt_number, GenericInterruptHandler& handler)
|
||||
{
|
||||
ASSERT(interrupt_number < GENERIC_INTERRUPT_HANDLERS_COUNT);
|
||||
if (s_interrupt_handler[interrupt_number] != nullptr) {
|
||||
if (s_interrupt_handler[interrupt_number]->type() == HandlerType::UnhandledInterruptHandler) {
|
||||
s_interrupt_handler[interrupt_number] = &handler;
|
||||
|
|
@ -442,10 +444,7 @@ void flush_idt()
|
|||
static void idt_init()
|
||||
{
|
||||
s_idtr.address = s_idt;
|
||||
s_idtr.limit = 0x100 * 8 - 1;
|
||||
|
||||
for (u8 i = 0xff; i > 0x10; --i)
|
||||
register_interrupt_handler(i, unimp_trap);
|
||||
s_idtr.limit = 256 * 8 - 1;
|
||||
|
||||
register_interrupt_handler(0x00, divide_error_asm_entry);
|
||||
register_user_callable_interrupt_handler(0x01, debug_asm_entry);
|
||||
|
|
@ -465,134 +464,185 @@ static void idt_init()
|
|||
register_interrupt_handler(0x0f, _exception15);
|
||||
register_interrupt_handler(0x10, _exception16);
|
||||
|
||||
register_interrupt_handler(0x50, interrupt_0_asm_entry);
|
||||
register_interrupt_handler(0x51, interrupt_1_asm_entry);
|
||||
register_interrupt_handler(0x52, interrupt_2_asm_entry);
|
||||
register_interrupt_handler(0x53, interrupt_3_asm_entry);
|
||||
register_interrupt_handler(0x54, interrupt_4_asm_entry);
|
||||
register_interrupt_handler(0x55, interrupt_5_asm_entry);
|
||||
register_interrupt_handler(0x56, interrupt_6_asm_entry);
|
||||
register_interrupt_handler(0x57, interrupt_7_asm_entry);
|
||||
register_interrupt_handler(0x58, interrupt_8_asm_entry);
|
||||
register_interrupt_handler(0x59, interrupt_9_asm_entry);
|
||||
register_interrupt_handler(0x5a, interrupt_10_asm_entry);
|
||||
register_interrupt_handler(0x5b, interrupt_11_asm_entry);
|
||||
register_interrupt_handler(0x5c, interrupt_12_asm_entry);
|
||||
register_interrupt_handler(0x5d, interrupt_13_asm_entry);
|
||||
register_interrupt_handler(0x5e, interrupt_14_asm_entry);
|
||||
register_interrupt_handler(0x5f, interrupt_15_asm_entry);
|
||||
register_interrupt_handler(0x60, interrupt_16_asm_entry);
|
||||
register_interrupt_handler(0x61, interrupt_17_asm_entry);
|
||||
register_interrupt_handler(0x62, interrupt_18_asm_entry);
|
||||
register_interrupt_handler(0x63, interrupt_19_asm_entry);
|
||||
register_interrupt_handler(0x64, interrupt_20_asm_entry);
|
||||
register_interrupt_handler(0x65, interrupt_21_asm_entry);
|
||||
register_interrupt_handler(0x66, interrupt_22_asm_entry);
|
||||
register_interrupt_handler(0x67, interrupt_23_asm_entry);
|
||||
register_interrupt_handler(0x68, interrupt_24_asm_entry);
|
||||
register_interrupt_handler(0x69, interrupt_25_asm_entry);
|
||||
register_interrupt_handler(0x6a, interrupt_26_asm_entry);
|
||||
register_interrupt_handler(0x6b, interrupt_27_asm_entry);
|
||||
register_interrupt_handler(0x6c, interrupt_28_asm_entry);
|
||||
register_interrupt_handler(0x6d, interrupt_29_asm_entry);
|
||||
register_interrupt_handler(0x6e, interrupt_30_asm_entry);
|
||||
register_interrupt_handler(0x6f, interrupt_31_asm_entry);
|
||||
register_interrupt_handler(0x70, interrupt_32_asm_entry);
|
||||
register_interrupt_handler(0x71, interrupt_33_asm_entry);
|
||||
register_interrupt_handler(0x72, interrupt_34_asm_entry);
|
||||
register_interrupt_handler(0x73, interrupt_35_asm_entry);
|
||||
register_interrupt_handler(0x74, interrupt_36_asm_entry);
|
||||
register_interrupt_handler(0x75, interrupt_37_asm_entry);
|
||||
register_interrupt_handler(0x76, interrupt_38_asm_entry);
|
||||
register_interrupt_handler(0x77, interrupt_39_asm_entry);
|
||||
register_interrupt_handler(0x78, interrupt_40_asm_entry);
|
||||
register_interrupt_handler(0x79, interrupt_41_asm_entry);
|
||||
register_interrupt_handler(0x7a, interrupt_42_asm_entry);
|
||||
register_interrupt_handler(0x7b, interrupt_43_asm_entry);
|
||||
register_interrupt_handler(0x7c, interrupt_44_asm_entry);
|
||||
register_interrupt_handler(0x7d, interrupt_45_asm_entry);
|
||||
register_interrupt_handler(0x7e, interrupt_46_asm_entry);
|
||||
register_interrupt_handler(0x7f, interrupt_47_asm_entry);
|
||||
register_interrupt_handler(0x80, interrupt_48_asm_entry);
|
||||
register_interrupt_handler(0x81, interrupt_49_asm_entry);
|
||||
register_interrupt_handler(0x82, interrupt_50_asm_entry);
|
||||
register_interrupt_handler(0x83, interrupt_51_asm_entry);
|
||||
register_interrupt_handler(0x84, interrupt_52_asm_entry);
|
||||
register_interrupt_handler(0x85, interrupt_53_asm_entry);
|
||||
register_interrupt_handler(0x86, interrupt_54_asm_entry);
|
||||
register_interrupt_handler(0x87, interrupt_55_asm_entry);
|
||||
register_interrupt_handler(0x88, interrupt_56_asm_entry);
|
||||
register_interrupt_handler(0x89, interrupt_57_asm_entry);
|
||||
register_interrupt_handler(0x8a, interrupt_58_asm_entry);
|
||||
register_interrupt_handler(0x8b, interrupt_59_asm_entry);
|
||||
register_interrupt_handler(0x8c, interrupt_60_asm_entry);
|
||||
register_interrupt_handler(0x8d, interrupt_61_asm_entry);
|
||||
register_interrupt_handler(0x8e, interrupt_62_asm_entry);
|
||||
register_interrupt_handler(0x8f, interrupt_63_asm_entry);
|
||||
register_interrupt_handler(0x90, interrupt_64_asm_entry);
|
||||
register_interrupt_handler(0x91, interrupt_65_asm_entry);
|
||||
register_interrupt_handler(0x92, interrupt_66_asm_entry);
|
||||
register_interrupt_handler(0x93, interrupt_67_asm_entry);
|
||||
register_interrupt_handler(0x94, interrupt_68_asm_entry);
|
||||
register_interrupt_handler(0x95, interrupt_69_asm_entry);
|
||||
register_interrupt_handler(0x96, interrupt_70_asm_entry);
|
||||
register_interrupt_handler(0x97, interrupt_71_asm_entry);
|
||||
register_interrupt_handler(0x98, interrupt_72_asm_entry);
|
||||
register_interrupt_handler(0x99, interrupt_73_asm_entry);
|
||||
register_interrupt_handler(0x9a, interrupt_74_asm_entry);
|
||||
register_interrupt_handler(0x9b, interrupt_75_asm_entry);
|
||||
register_interrupt_handler(0x9c, interrupt_76_asm_entry);
|
||||
register_interrupt_handler(0x9d, interrupt_77_asm_entry);
|
||||
register_interrupt_handler(0x9e, interrupt_78_asm_entry);
|
||||
register_interrupt_handler(0x9f, interrupt_79_asm_entry);
|
||||
register_interrupt_handler(0xa0, interrupt_80_asm_entry);
|
||||
register_interrupt_handler(0xa1, interrupt_81_asm_entry);
|
||||
register_interrupt_handler(0xa2, interrupt_82_asm_entry);
|
||||
register_interrupt_handler(0xa3, interrupt_83_asm_entry);
|
||||
register_interrupt_handler(0xa4, interrupt_84_asm_entry);
|
||||
register_interrupt_handler(0xa5, interrupt_85_asm_entry);
|
||||
register_interrupt_handler(0xa6, interrupt_86_asm_entry);
|
||||
register_interrupt_handler(0xa7, interrupt_87_asm_entry);
|
||||
register_interrupt_handler(0xa8, interrupt_88_asm_entry);
|
||||
register_interrupt_handler(0xa9, interrupt_89_asm_entry);
|
||||
register_interrupt_handler(0xaa, interrupt_90_asm_entry);
|
||||
register_interrupt_handler(0xab, interrupt_91_asm_entry);
|
||||
register_interrupt_handler(0xac, interrupt_92_asm_entry);
|
||||
register_interrupt_handler(0xad, interrupt_93_asm_entry);
|
||||
register_interrupt_handler(0xae, interrupt_94_asm_entry);
|
||||
register_interrupt_handler(0xaf, interrupt_95_asm_entry);
|
||||
register_interrupt_handler(0xb0, interrupt_96_asm_entry);
|
||||
register_interrupt_handler(0xb1, interrupt_97_asm_entry);
|
||||
register_interrupt_handler(0xb2, interrupt_98_asm_entry);
|
||||
register_interrupt_handler(0xb3, interrupt_99_asm_entry);
|
||||
register_interrupt_handler(0xb4, interrupt_100_asm_entry);
|
||||
register_interrupt_handler(0xb5, interrupt_101_asm_entry);
|
||||
register_interrupt_handler(0xb6, interrupt_102_asm_entry);
|
||||
register_interrupt_handler(0xb7, interrupt_103_asm_entry);
|
||||
register_interrupt_handler(0xb8, interrupt_104_asm_entry);
|
||||
register_interrupt_handler(0xb9, interrupt_105_asm_entry);
|
||||
register_interrupt_handler(0xba, interrupt_106_asm_entry);
|
||||
register_interrupt_handler(0xbb, interrupt_107_asm_entry);
|
||||
register_interrupt_handler(0xbc, interrupt_108_asm_entry);
|
||||
register_interrupt_handler(0xbd, interrupt_109_asm_entry);
|
||||
register_interrupt_handler(0xbe, interrupt_110_asm_entry);
|
||||
register_interrupt_handler(0xbf, interrupt_111_asm_entry);
|
||||
register_interrupt_handler(0xc0, interrupt_112_asm_entry);
|
||||
register_interrupt_handler(0xc1, interrupt_113_asm_entry);
|
||||
register_interrupt_handler(0xc2, interrupt_114_asm_entry);
|
||||
register_interrupt_handler(0xc3, interrupt_115_asm_entry);
|
||||
register_interrupt_handler(0xc4, interrupt_116_asm_entry);
|
||||
register_interrupt_handler(0xc5, interrupt_117_asm_entry);
|
||||
register_interrupt_handler(0xc6, interrupt_118_asm_entry);
|
||||
register_interrupt_handler(0xc7, interrupt_119_asm_entry);
|
||||
register_interrupt_handler(0xc8, interrupt_120_asm_entry);
|
||||
register_interrupt_handler(0xc9, interrupt_121_asm_entry);
|
||||
register_interrupt_handler(0xca, interrupt_122_asm_entry);
|
||||
register_interrupt_handler(0xcb, interrupt_123_asm_entry);
|
||||
register_interrupt_handler(0xcc, interrupt_124_asm_entry);
|
||||
register_interrupt_handler(0xcd, interrupt_125_asm_entry);
|
||||
register_interrupt_handler(0xce, interrupt_126_asm_entry);
|
||||
register_interrupt_handler(0xcf, interrupt_127_asm_entry);
|
||||
for (u8 i = 0x11; i < 0x50; i++)
|
||||
register_interrupt_handler(i, unimp_trap);
|
||||
|
||||
register_interrupt_handler(0x50, interrupt_80_asm_entry);
|
||||
register_interrupt_handler(0x51, interrupt_81_asm_entry);
|
||||
register_interrupt_handler(0x52, interrupt_82_asm_entry);
|
||||
register_interrupt_handler(0x53, interrupt_83_asm_entry);
|
||||
register_interrupt_handler(0x54, interrupt_84_asm_entry);
|
||||
register_interrupt_handler(0x55, interrupt_85_asm_entry);
|
||||
register_interrupt_handler(0x56, interrupt_86_asm_entry);
|
||||
register_interrupt_handler(0x57, interrupt_87_asm_entry);
|
||||
register_interrupt_handler(0x58, interrupt_88_asm_entry);
|
||||
register_interrupt_handler(0x59, interrupt_89_asm_entry);
|
||||
register_interrupt_handler(0x5a, interrupt_90_asm_entry);
|
||||
register_interrupt_handler(0x5b, interrupt_91_asm_entry);
|
||||
register_interrupt_handler(0x5c, interrupt_92_asm_entry);
|
||||
register_interrupt_handler(0x5d, interrupt_93_asm_entry);
|
||||
register_interrupt_handler(0x5e, interrupt_94_asm_entry);
|
||||
register_interrupt_handler(0x5f, interrupt_95_asm_entry);
|
||||
register_interrupt_handler(0x60, interrupt_96_asm_entry);
|
||||
register_interrupt_handler(0x61, interrupt_97_asm_entry);
|
||||
register_interrupt_handler(0x62, interrupt_98_asm_entry);
|
||||
register_interrupt_handler(0x63, interrupt_99_asm_entry);
|
||||
register_interrupt_handler(0x64, interrupt_100_asm_entry);
|
||||
register_interrupt_handler(0x65, interrupt_101_asm_entry);
|
||||
register_interrupt_handler(0x66, interrupt_102_asm_entry);
|
||||
register_interrupt_handler(0x67, interrupt_103_asm_entry);
|
||||
register_interrupt_handler(0x68, interrupt_104_asm_entry);
|
||||
register_interrupt_handler(0x69, interrupt_105_asm_entry);
|
||||
register_interrupt_handler(0x6a, interrupt_106_asm_entry);
|
||||
register_interrupt_handler(0x6b, interrupt_107_asm_entry);
|
||||
register_interrupt_handler(0x6c, interrupt_108_asm_entry);
|
||||
register_interrupt_handler(0x6d, interrupt_109_asm_entry);
|
||||
register_interrupt_handler(0x6e, interrupt_110_asm_entry);
|
||||
register_interrupt_handler(0x6f, interrupt_111_asm_entry);
|
||||
register_interrupt_handler(0x70, interrupt_112_asm_entry);
|
||||
register_interrupt_handler(0x71, interrupt_113_asm_entry);
|
||||
register_interrupt_handler(0x72, interrupt_114_asm_entry);
|
||||
register_interrupt_handler(0x73, interrupt_115_asm_entry);
|
||||
register_interrupt_handler(0x74, interrupt_116_asm_entry);
|
||||
register_interrupt_handler(0x75, interrupt_117_asm_entry);
|
||||
register_interrupt_handler(0x76, interrupt_118_asm_entry);
|
||||
register_interrupt_handler(0x77, interrupt_119_asm_entry);
|
||||
register_interrupt_handler(0x78, interrupt_120_asm_entry);
|
||||
register_interrupt_handler(0x79, interrupt_121_asm_entry);
|
||||
register_interrupt_handler(0x7a, interrupt_122_asm_entry);
|
||||
register_interrupt_handler(0x7b, interrupt_123_asm_entry);
|
||||
register_interrupt_handler(0x7c, interrupt_124_asm_entry);
|
||||
register_interrupt_handler(0x7d, interrupt_125_asm_entry);
|
||||
register_interrupt_handler(0x7e, interrupt_126_asm_entry);
|
||||
register_interrupt_handler(0x7f, interrupt_127_asm_entry);
|
||||
register_interrupt_handler(0x80, interrupt_128_asm_entry);
|
||||
register_interrupt_handler(0x81, interrupt_129_asm_entry);
|
||||
register_interrupt_handler(0x82, interrupt_130_asm_entry);
|
||||
register_interrupt_handler(0x83, interrupt_131_asm_entry);
|
||||
register_interrupt_handler(0x84, interrupt_132_asm_entry);
|
||||
register_interrupt_handler(0x85, interrupt_133_asm_entry);
|
||||
register_interrupt_handler(0x86, interrupt_134_asm_entry);
|
||||
register_interrupt_handler(0x87, interrupt_135_asm_entry);
|
||||
register_interrupt_handler(0x88, interrupt_136_asm_entry);
|
||||
register_interrupt_handler(0x89, interrupt_137_asm_entry);
|
||||
register_interrupt_handler(0x8a, interrupt_138_asm_entry);
|
||||
register_interrupt_handler(0x8b, interrupt_139_asm_entry);
|
||||
register_interrupt_handler(0x8c, interrupt_140_asm_entry);
|
||||
register_interrupt_handler(0x8d, interrupt_141_asm_entry);
|
||||
register_interrupt_handler(0x8e, interrupt_142_asm_entry);
|
||||
register_interrupt_handler(0x8f, interrupt_143_asm_entry);
|
||||
register_interrupt_handler(0x90, interrupt_144_asm_entry);
|
||||
register_interrupt_handler(0x91, interrupt_145_asm_entry);
|
||||
register_interrupt_handler(0x92, interrupt_146_asm_entry);
|
||||
register_interrupt_handler(0x93, interrupt_147_asm_entry);
|
||||
register_interrupt_handler(0x94, interrupt_148_asm_entry);
|
||||
register_interrupt_handler(0x95, interrupt_149_asm_entry);
|
||||
register_interrupt_handler(0x96, interrupt_150_asm_entry);
|
||||
register_interrupt_handler(0x97, interrupt_151_asm_entry);
|
||||
register_interrupt_handler(0x98, interrupt_152_asm_entry);
|
||||
register_interrupt_handler(0x99, interrupt_153_asm_entry);
|
||||
register_interrupt_handler(0x9a, interrupt_154_asm_entry);
|
||||
register_interrupt_handler(0x9b, interrupt_155_asm_entry);
|
||||
register_interrupt_handler(0x9c, interrupt_156_asm_entry);
|
||||
register_interrupt_handler(0x9d, interrupt_157_asm_entry);
|
||||
register_interrupt_handler(0x9e, interrupt_158_asm_entry);
|
||||
register_interrupt_handler(0x9f, interrupt_159_asm_entry);
|
||||
register_interrupt_handler(0xa0, interrupt_160_asm_entry);
|
||||
register_interrupt_handler(0xa1, interrupt_161_asm_entry);
|
||||
register_interrupt_handler(0xa2, interrupt_162_asm_entry);
|
||||
register_interrupt_handler(0xa3, interrupt_163_asm_entry);
|
||||
register_interrupt_handler(0xa4, interrupt_164_asm_entry);
|
||||
register_interrupt_handler(0xa5, interrupt_165_asm_entry);
|
||||
register_interrupt_handler(0xa6, interrupt_166_asm_entry);
|
||||
register_interrupt_handler(0xa7, interrupt_167_asm_entry);
|
||||
register_interrupt_handler(0xa8, interrupt_168_asm_entry);
|
||||
register_interrupt_handler(0xa9, interrupt_169_asm_entry);
|
||||
register_interrupt_handler(0xaa, interrupt_170_asm_entry);
|
||||
register_interrupt_handler(0xab, interrupt_171_asm_entry);
|
||||
register_interrupt_handler(0xac, interrupt_172_asm_entry);
|
||||
register_interrupt_handler(0xad, interrupt_173_asm_entry);
|
||||
register_interrupt_handler(0xae, interrupt_174_asm_entry);
|
||||
register_interrupt_handler(0xaf, interrupt_175_asm_entry);
|
||||
register_interrupt_handler(0xb0, interrupt_176_asm_entry);
|
||||
register_interrupt_handler(0xb1, interrupt_177_asm_entry);
|
||||
register_interrupt_handler(0xb2, interrupt_178_asm_entry);
|
||||
register_interrupt_handler(0xb3, interrupt_179_asm_entry);
|
||||
register_interrupt_handler(0xb4, interrupt_180_asm_entry);
|
||||
register_interrupt_handler(0xb5, interrupt_181_asm_entry);
|
||||
register_interrupt_handler(0xb6, interrupt_182_asm_entry);
|
||||
register_interrupt_handler(0xb7, interrupt_183_asm_entry);
|
||||
register_interrupt_handler(0xb8, interrupt_184_asm_entry);
|
||||
register_interrupt_handler(0xb9, interrupt_185_asm_entry);
|
||||
register_interrupt_handler(0xba, interrupt_186_asm_entry);
|
||||
register_interrupt_handler(0xbb, interrupt_187_asm_entry);
|
||||
register_interrupt_handler(0xbc, interrupt_188_asm_entry);
|
||||
register_interrupt_handler(0xbd, interrupt_189_asm_entry);
|
||||
register_interrupt_handler(0xbe, interrupt_190_asm_entry);
|
||||
register_interrupt_handler(0xbf, interrupt_191_asm_entry);
|
||||
register_interrupt_handler(0xc0, interrupt_192_asm_entry);
|
||||
register_interrupt_handler(0xc1, interrupt_193_asm_entry);
|
||||
register_interrupt_handler(0xc2, interrupt_194_asm_entry);
|
||||
register_interrupt_handler(0xc3, interrupt_195_asm_entry);
|
||||
register_interrupt_handler(0xc4, interrupt_196_asm_entry);
|
||||
register_interrupt_handler(0xc5, interrupt_197_asm_entry);
|
||||
register_interrupt_handler(0xc6, interrupt_198_asm_entry);
|
||||
register_interrupt_handler(0xc7, interrupt_199_asm_entry);
|
||||
register_interrupt_handler(0xc8, interrupt_200_asm_entry);
|
||||
register_interrupt_handler(0xc9, interrupt_201_asm_entry);
|
||||
register_interrupt_handler(0xca, interrupt_202_asm_entry);
|
||||
register_interrupt_handler(0xcb, interrupt_203_asm_entry);
|
||||
register_interrupt_handler(0xcc, interrupt_204_asm_entry);
|
||||
register_interrupt_handler(0xcd, interrupt_205_asm_entry);
|
||||
register_interrupt_handler(0xce, interrupt_206_asm_entry);
|
||||
register_interrupt_handler(0xcf, interrupt_207_asm_entry);
|
||||
register_interrupt_handler(0xd0, interrupt_208_asm_entry);
|
||||
register_interrupt_handler(0xd1, interrupt_209_asm_entry);
|
||||
register_interrupt_handler(0xd2, interrupt_210_asm_entry);
|
||||
register_interrupt_handler(0xd3, interrupt_211_asm_entry);
|
||||
register_interrupt_handler(0xd4, interrupt_212_asm_entry);
|
||||
register_interrupt_handler(0xd5, interrupt_213_asm_entry);
|
||||
register_interrupt_handler(0xd6, interrupt_214_asm_entry);
|
||||
register_interrupt_handler(0xd7, interrupt_215_asm_entry);
|
||||
register_interrupt_handler(0xd8, interrupt_216_asm_entry);
|
||||
register_interrupt_handler(0xd9, interrupt_217_asm_entry);
|
||||
register_interrupt_handler(0xda, interrupt_218_asm_entry);
|
||||
register_interrupt_handler(0xdb, interrupt_219_asm_entry);
|
||||
register_interrupt_handler(0xdc, interrupt_220_asm_entry);
|
||||
register_interrupt_handler(0xdd, interrupt_221_asm_entry);
|
||||
register_interrupt_handler(0xde, interrupt_222_asm_entry);
|
||||
register_interrupt_handler(0xdf, interrupt_223_asm_entry);
|
||||
register_interrupt_handler(0xe0, interrupt_224_asm_entry);
|
||||
register_interrupt_handler(0xe1, interrupt_225_asm_entry);
|
||||
register_interrupt_handler(0xe2, interrupt_226_asm_entry);
|
||||
register_interrupt_handler(0xe3, interrupt_227_asm_entry);
|
||||
register_interrupt_handler(0xe4, interrupt_228_asm_entry);
|
||||
register_interrupt_handler(0xe5, interrupt_229_asm_entry);
|
||||
register_interrupt_handler(0xe6, interrupt_230_asm_entry);
|
||||
register_interrupt_handler(0xe7, interrupt_231_asm_entry);
|
||||
register_interrupt_handler(0xe8, interrupt_232_asm_entry);
|
||||
register_interrupt_handler(0xe9, interrupt_233_asm_entry);
|
||||
register_interrupt_handler(0xea, interrupt_234_asm_entry);
|
||||
register_interrupt_handler(0xeb, interrupt_235_asm_entry);
|
||||
register_interrupt_handler(0xec, interrupt_236_asm_entry);
|
||||
register_interrupt_handler(0xed, interrupt_237_asm_entry);
|
||||
register_interrupt_handler(0xee, interrupt_238_asm_entry);
|
||||
register_interrupt_handler(0xef, interrupt_239_asm_entry);
|
||||
register_interrupt_handler(0xf0, interrupt_240_asm_entry);
|
||||
register_interrupt_handler(0xf1, interrupt_241_asm_entry);
|
||||
register_interrupt_handler(0xf2, interrupt_242_asm_entry);
|
||||
register_interrupt_handler(0xf3, interrupt_243_asm_entry);
|
||||
register_interrupt_handler(0xf4, interrupt_244_asm_entry);
|
||||
register_interrupt_handler(0xf5, interrupt_245_asm_entry);
|
||||
register_interrupt_handler(0xf6, interrupt_246_asm_entry);
|
||||
register_interrupt_handler(0xf7, interrupt_247_asm_entry);
|
||||
register_interrupt_handler(0xf8, interrupt_248_asm_entry);
|
||||
register_interrupt_handler(0xf9, interrupt_249_asm_entry);
|
||||
register_interrupt_handler(0xfa, interrupt_250_asm_entry);
|
||||
register_interrupt_handler(0xfb, interrupt_251_asm_entry);
|
||||
register_interrupt_handler(0xfc, interrupt_252_asm_entry);
|
||||
register_interrupt_handler(0xfd, interrupt_253_asm_entry);
|
||||
register_interrupt_handler(0xfe, interrupt_254_asm_entry);
|
||||
register_interrupt_handler(0xff, interrupt_255_asm_entry);
|
||||
|
||||
dbg() << "Installing Unhandled Handlers";
|
||||
|
||||
|
|
@ -688,6 +738,8 @@ FPUState Processor::s_clean_fpu_state;
|
|||
|
||||
static Vector<Processor*>* s_processors;
|
||||
static SpinLock s_processor_lock;
|
||||
volatile u32 Processor::g_total_processors;
|
||||
static volatile bool s_smp_enabled;
|
||||
|
||||
Vector<Processor*>& Processor::processors()
|
||||
{
|
||||
|
|
@ -707,6 +759,12 @@ Processor& Processor::by_id(u32 cpu)
|
|||
return *procs[cpu];
|
||||
}
|
||||
|
||||
[[noreturn]] static inline void halt_this()
|
||||
{
|
||||
for (;;) {
|
||||
asm volatile("cli; hlt");
|
||||
}
|
||||
}
|
||||
|
||||
void Processor::cpu_detect()
|
||||
{
|
||||
|
|
@ -868,13 +926,21 @@ void Processor::early_initialize(u32 cpu)
|
|||
m_invoke_scheduler_async = false;
|
||||
m_scheduler_initialized = false;
|
||||
|
||||
m_message_queue = nullptr;
|
||||
m_idle_thread = nullptr;
|
||||
m_current_thread = nullptr;
|
||||
m_mm_data = nullptr;
|
||||
m_info = nullptr;
|
||||
|
||||
cpu_setup();
|
||||
m_halt_requested = false;
|
||||
if (cpu == 0) {
|
||||
s_smp_enabled = false;
|
||||
atomic_store(&g_total_processors, 1u, AK::MemoryOrder::memory_order_release);
|
||||
} else {
|
||||
atomic_fetch_add(&g_total_processors, 1u, AK::MemoryOrder::memory_order_acq_rel);
|
||||
}
|
||||
|
||||
cpu_setup();
|
||||
gdt_init();
|
||||
ASSERT(¤t() == this); // sanity check
|
||||
}
|
||||
|
|
@ -910,9 +976,9 @@ void Processor::initialize(u32 cpu)
|
|||
if (cpu >= s_processors->size())
|
||||
s_processors->resize(cpu + 1);
|
||||
(*s_processors)[cpu] = this;
|
||||
}
|
||||
|
||||
klog() << "CPU[" << cpu << "]: initialized Processor at " << VirtualAddress(FlatPtr(this));
|
||||
klog() << "CPU[" << cpu << "]: initialized Processor at " << VirtualAddress(FlatPtr(this));
|
||||
}
|
||||
}
|
||||
|
||||
void Processor::write_raw_gdt_entry(u16 selector, u32 low, u32 high)
|
||||
|
|
@ -979,7 +1045,7 @@ bool Processor::get_context_frame_ptr(Thread& thread, u32& frame_ptr, u32& eip)
|
|||
// TODO: If this is the case, the thread is currently running
|
||||
// on another processor. We can't trust the kernel stack as
|
||||
// it may be changing at any time. We need to probably send
|
||||
// an ICI to that processor, have it walk the stack and wait
|
||||
// an IPI to that processor, have it walk the stack and wait
|
||||
// until it returns the data back to us
|
||||
dbg() << "CPU[" << proc.id() << "] getting stack for "
|
||||
<< thread << " on other CPU# " << thread.cpu() << " not yet implemented!";
|
||||
|
|
@ -1288,12 +1354,16 @@ void Processor::initialize_context_switching(Thread& initial_thread)
|
|||
"addl $20, %%ebx \n" // calculate pointer to TrapFrame
|
||||
"pushl %%ebx \n"
|
||||
"cld \n"
|
||||
"pushl %[cpu] \n"
|
||||
"call init_finished \n"
|
||||
"addl $4, %%esp \n"
|
||||
"call enter_trap_no_irq \n"
|
||||
"addl $4, %%esp \n"
|
||||
"lret \n"
|
||||
:: [new_esp] "g" (tss.esp),
|
||||
[new_eip] "a" (tss.eip),
|
||||
[from_to_thread] "b" (&initial_thread)
|
||||
[from_to_thread] "b" (&initial_thread),
|
||||
[cpu] "c" (id())
|
||||
);
|
||||
|
||||
ASSERT_NOT_REACHED();
|
||||
|
|
@ -1312,7 +1382,9 @@ void Processor::exit_trap(TrapFrame& trap)
|
|||
InterruptDisabler disabler;
|
||||
ASSERT(m_in_irq >= trap.prev_irq_level);
|
||||
m_in_irq = trap.prev_irq_level;
|
||||
|
||||
|
||||
smp_process_pending_messages();
|
||||
|
||||
if (!m_in_irq && !m_in_critical)
|
||||
check_invoke_scheduler();
|
||||
}
|
||||
|
|
@ -1327,6 +1399,267 @@ void Processor::check_invoke_scheduler()
|
|||
}
|
||||
}
|
||||
|
||||
void Processor::flush_tlb_local(VirtualAddress vaddr, size_t page_count)
|
||||
{
|
||||
auto ptr = vaddr.as_ptr();
|
||||
while (page_count > 0) {
|
||||
asm volatile("invlpg %0"
|
||||
:
|
||||
: "m"(*(char*)vaddr.get())
|
||||
: "memory");
|
||||
ptr += PAGE_SIZE;
|
||||
page_count--;
|
||||
}
|
||||
}
|
||||
|
||||
void Processor::flush_tlb(VirtualAddress vaddr, size_t page_count)
|
||||
{
|
||||
flush_tlb_local(vaddr, page_count);
|
||||
if (s_smp_enabled)
|
||||
smp_broadcast_flush_tlb(vaddr, page_count);
|
||||
}
|
||||
|
||||
static volatile ProcessorMessage* s_message_pool;
|
||||
|
||||
void Processor::smp_return_to_pool(ProcessorMessage& msg)
|
||||
{
|
||||
ProcessorMessage* next = nullptr;
|
||||
do {
|
||||
msg.next = next;
|
||||
} while (!atomic_compare_exchange_strong(&s_message_pool, next, &msg, AK::MemoryOrder::memory_order_acq_rel));
|
||||
}
|
||||
|
||||
ProcessorMessage& Processor::smp_get_from_pool()
|
||||
{
|
||||
ProcessorMessage* msg;
|
||||
|
||||
// The assumption is that messages are never removed from the pool!
|
||||
for (;;) {
|
||||
msg = atomic_load(&s_message_pool, AK::MemoryOrder::memory_order_consume);
|
||||
if (!msg) {
|
||||
if (!Processor::current().smp_process_pending_messages()) {
|
||||
// TODO: pause for a bit?
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// If another processor were to use this message in the meanwhile,
|
||||
// "msg" is still valid (because it never gets freed). We'd detect
|
||||
// this because the expected value "msg" and pool would
|
||||
// no longer match, and the compare_exchange will fail. But accessing
|
||||
// "msg->next" is always safe here.
|
||||
if (atomic_compare_exchange_strong(&s_message_pool, msg, msg->next, AK::MemoryOrder::memory_order_acq_rel)) {
|
||||
// We successfully "popped" this available message
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT(msg != nullptr);
|
||||
return *msg;
|
||||
}
|
||||
|
||||
void Processor::smp_enable()
|
||||
{
|
||||
size_t msg_pool_size = Processor::count() * 100u;
|
||||
size_t msg_entries_cnt = Processor::count();
|
||||
|
||||
auto msgs = new ProcessorMessage[msg_pool_size];
|
||||
auto msg_entries = new ProcessorMessageEntry[msg_pool_size * msg_entries_cnt];
|
||||
size_t msg_entry_i = 0;
|
||||
for (size_t i = 0; i < msg_pool_size; i++, msg_entry_i += msg_entries_cnt) {
|
||||
auto& msg = msgs[i];
|
||||
msg.next = i < msg_pool_size - 1 ? &msgs[i + 1] : nullptr;
|
||||
msg.per_proc_entries = &msg_entries[msg_entry_i];
|
||||
for (size_t k = 0; k < msg_entries_cnt; k++)
|
||||
msg_entries[msg_entry_i + k].msg = &msg;
|
||||
}
|
||||
|
||||
atomic_store(&s_message_pool, &msgs[0], AK::MemoryOrder::memory_order_release);
|
||||
|
||||
// Start sending IPI messages
|
||||
s_smp_enabled = true;
|
||||
}
|
||||
|
||||
void Processor::smp_cleanup_message(ProcessorMessage& msg)
|
||||
{
|
||||
switch (msg.type) {
|
||||
case ProcessorMessage::CallbackWithData:
|
||||
if (msg.callback_with_data.free)
|
||||
msg.callback_with_data.free(msg.callback_with_data.data);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
bool Processor::smp_process_pending_messages()
|
||||
{
|
||||
bool did_process = false;
|
||||
u32 prev_flags;
|
||||
enter_critical(prev_flags);
|
||||
|
||||
if (auto pending_msgs = atomic_exchange(&m_message_queue, nullptr, AK::MemoryOrder::memory_order_acq_rel))
|
||||
{
|
||||
// We pulled the stack of pending messages in LIFO order, so we need to reverse the list first
|
||||
auto reverse_list =
|
||||
[](ProcessorMessageEntry* list) -> ProcessorMessageEntry*
|
||||
{
|
||||
ProcessorMessageEntry* rev_list = nullptr;
|
||||
while (list) {
|
||||
auto next = list->next;
|
||||
list->next = rev_list;
|
||||
rev_list = list;
|
||||
list = next;
|
||||
}
|
||||
return rev_list;
|
||||
};
|
||||
|
||||
pending_msgs = reverse_list(pending_msgs);
|
||||
|
||||
// now process in the right order
|
||||
ProcessorMessageEntry* next_msg;
|
||||
for (auto cur_msg = pending_msgs; cur_msg; cur_msg = next_msg) {
|
||||
next_msg = cur_msg->next;
|
||||
auto msg = cur_msg->msg;
|
||||
|
||||
#ifdef SMP_DEBUG
|
||||
dbg() << "SMP[" << id() << "]: Processing message " << VirtualAddress(msg);
|
||||
#endif
|
||||
|
||||
switch (msg->type) {
|
||||
case ProcessorMessage::Callback:
|
||||
msg->callback.handler();
|
||||
break;
|
||||
case ProcessorMessage::CallbackWithData:
|
||||
msg->callback_with_data.handler(msg->callback_with_data.data);
|
||||
break;
|
||||
case ProcessorMessage::FlushTlb:
|
||||
flush_tlb_local(VirtualAddress(msg->flush_tlb.ptr), msg->flush_tlb.page_count);
|
||||
break;
|
||||
}
|
||||
|
||||
bool is_async = msg->async; // Need to cache this value *before* dropping the ref count!
|
||||
auto prev_refs = atomic_fetch_sub(&msg->refs, 1u, AK::MemoryOrder::memory_order_acq_rel);
|
||||
ASSERT(prev_refs != 0);
|
||||
if (prev_refs == 1) {
|
||||
// All processors handled this. If this is an async message,
|
||||
// we need to clean it up and return it to the pool
|
||||
if (is_async) {
|
||||
smp_cleanup_message(*msg);
|
||||
smp_return_to_pool(*msg);
|
||||
}
|
||||
}
|
||||
|
||||
if (m_halt_requested)
|
||||
halt_this();
|
||||
}
|
||||
did_process = true;
|
||||
} else if (m_halt_requested) {
|
||||
halt_this();
|
||||
}
|
||||
|
||||
leave_critical(prev_flags);
|
||||
return did_process;
|
||||
}
|
||||
|
||||
bool Processor::smp_queue_message(ProcessorMessage& msg)
|
||||
{
|
||||
// Note that it's quite possible that the other processor may pop
|
||||
// the queue at any given time. We rely on the fact that the messages
|
||||
// are pooled and never get freed!
|
||||
auto& msg_entry = msg.per_proc_entries[id()];
|
||||
ASSERT(msg_entry.msg == &msg);
|
||||
ProcessorMessageEntry* next = nullptr;
|
||||
do {
|
||||
msg_entry.next = next;
|
||||
} while (!atomic_compare_exchange_strong(&m_message_queue, next, &msg_entry, AK::MemoryOrder::memory_order_acq_rel));
|
||||
return next == nullptr;
|
||||
}
|
||||
|
||||
void Processor::smp_broadcast_message(ProcessorMessage& msg, bool async)
|
||||
{
|
||||
auto& cur_proc = Processor::current();
|
||||
msg.async = async;
|
||||
#ifdef SMP_DEBUG
|
||||
dbg() << "SMP[" << cur_proc.id() << "]: Broadcast message " << VirtualAddress(&msg) << " to cpus: " << (count()) << " proc: " << VirtualAddress(&cur_proc);
|
||||
#endif
|
||||
atomic_store(&msg.refs, count() - 1, AK::MemoryOrder::memory_order_release);
|
||||
ASSERT(msg.refs > 0);
|
||||
for_each(
|
||||
[&](Processor& proc) -> IterationDecision
|
||||
{
|
||||
if (&proc != &cur_proc) {
|
||||
if (proc.smp_queue_message(msg)) {
|
||||
// TODO: only send IPI to that CPU if we queued the first
|
||||
}
|
||||
}
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
// Now trigger an IPI on all other APs
|
||||
APIC::the().broadcast_ipi();
|
||||
|
||||
if (!async) {
|
||||
// If synchronous then we must cleanup and return the message back
|
||||
// to the pool. Otherwise, the last processor to complete it will return it
|
||||
while (atomic_load(&msg.refs, AK::MemoryOrder::memory_order_consume) != 0) {
|
||||
// TODO: pause for a bit?
|
||||
}
|
||||
|
||||
smp_cleanup_message(msg);
|
||||
smp_return_to_pool(msg);
|
||||
}
|
||||
}
|
||||
|
||||
void Processor::smp_broadcast(void(*callback)(void*), void* data, void(*free_data)(void*), bool async)
|
||||
{
|
||||
auto& msg = smp_get_from_pool();
|
||||
msg.type = ProcessorMessage::CallbackWithData;
|
||||
msg.callback_with_data.handler = callback;
|
||||
msg.callback_with_data.data = data;
|
||||
msg.callback_with_data.free = free_data;
|
||||
smp_broadcast_message(msg, async);
|
||||
}
|
||||
|
||||
void Processor::smp_broadcast(void(*callback)(), bool async)
|
||||
{
|
||||
auto& msg = smp_get_from_pool();
|
||||
msg.type = ProcessorMessage::CallbackWithData;
|
||||
msg.callback.handler = callback;
|
||||
smp_broadcast_message(msg, async);
|
||||
}
|
||||
|
||||
void Processor::smp_broadcast_flush_tlb(VirtualAddress vaddr, size_t page_count)
|
||||
{
|
||||
auto& msg = smp_get_from_pool();
|
||||
msg.type = ProcessorMessage::FlushTlb;
|
||||
msg.flush_tlb.ptr = vaddr.as_ptr();
|
||||
msg.flush_tlb.page_count = page_count;
|
||||
smp_broadcast_message(msg, false);
|
||||
}
|
||||
|
||||
void Processor::smp_broadcast_halt()
|
||||
{
|
||||
// We don't want to use a message, because this could have been triggered
|
||||
// by being out of memory and we might not be able to get a message
|
||||
for_each(
|
||||
[&](Processor& proc) -> IterationDecision
|
||||
{
|
||||
proc.m_halt_requested = true;
|
||||
return IterationDecision::Continue;
|
||||
});
|
||||
|
||||
// Now trigger an IPI on all other APs
|
||||
APIC::the().broadcast_ipi();
|
||||
}
|
||||
|
||||
void Processor::Processor::halt()
|
||||
{
|
||||
if (s_smp_enabled)
|
||||
smp_broadcast_halt();
|
||||
|
||||
halt_this();
|
||||
}
|
||||
|
||||
void Processor::gdt_init()
|
||||
{
|
||||
m_gdt_length = 0;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue