From 8a258edfd6fed0711d582f48cbc3ce625ef5d792 Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 3 Sep 2021 21:27:57 -0600 Subject: [PATCH] Kernel: Add x2APIC support This allows addressing all cores on more modern processors. For now, we still have a hardcoded limit of 64 due to s_processors being a static array. --- Kernel/ACPI/Definitions.h | 9 +++ Kernel/Arch/x86/Processor.h | 8 +- Kernel/Arch/x86/common/Processor.cpp | 5 ++ Kernel/Interrupts/APIC.cpp | 108 +++++++++++++++++++-------- Kernel/Interrupts/APIC.h | 26 +++---- 5 files changed, 108 insertions(+), 48 deletions(-) diff --git a/Kernel/ACPI/Definitions.h b/Kernel/ACPI/Definitions.h index a35f26603d..748fe3abc2 100644 --- a/Kernel/ACPI/Definitions.h +++ b/Kernel/ACPI/Definitions.h @@ -277,6 +277,15 @@ struct [[gnu::packed]] ProcessorLocalAPIC { u32 flags; }; +// https://uefi.org/specs/ACPI/6.4/05_ACPI_Software_Programming_Model/ACPI_Software_Programming_Model.html#processor-local-x2apic-structure +struct [[gnu::packed]] ProcessorLocalX2APIC { + MADTEntryHeader h; + u16 reserved; + u32 apic_id; + u32 flags; + u32 acpi_processor_id; +}; + // https://uefi.org/specs/ACPI/6.4/05_ACPI_Software_Programming_Model/ACPI_Software_Programming_Model.html#interrupt-source-override-structure struct [[gnu::packed]] InterruptSourceOverride { MADTEntryHeader h; diff --git a/Kernel/Arch/x86/Processor.h b/Kernel/Arch/x86/Processor.h index 1b601b9c58..9ac15711a8 100644 --- a/Kernel/Arch/x86/Processor.h +++ b/Kernel/Arch/x86/Processor.h @@ -102,9 +102,9 @@ struct DeferredCallEntry { }; class Processor; -// Note: We only support 8 processors at most at the moment, -// so allocate 8 slots of inline capacity in the container. -using ProcessorContainer = Array; +// Note: We only support 64 processors at most at the moment, +// so allocate 64 slots of inline capacity in the container. +using ProcessorContainer = Array; class Processor { friend class ProcessorInfo; @@ -190,6 +190,8 @@ public: s_idle_cpu_mask.fetch_and(~(1u << m_cpu), AK::MemoryOrder::memory_order_relaxed); } + static Processor& by_id(u32); + static u32 count() { // NOTE: because this value never changes once all APs are booted, diff --git a/Kernel/Arch/x86/common/Processor.cpp b/Kernel/Arch/x86/common/Processor.cpp index 11affd1bed..dbf892c590 100644 --- a/Kernel/Arch/x86/common/Processor.cpp +++ b/Kernel/Arch/x86/common/Processor.cpp @@ -582,6 +582,11 @@ ProcessorContainer& Processor::processors() return s_processors; } +Processor& Processor::by_id(u32 id) +{ + return *s_processors[id]; +} + void Processor::enter_trap(TrapFrame& trap, bool raise_irq) { VERIFY_INTERRUPTS_DISABLED(); diff --git a/Kernel/Interrupts/APIC.cpp b/Kernel/Interrupts/APIC.cpp index 352e967ca8..b88b30c7df 100644 --- a/Kernel/Interrupts/APIC.cpp +++ b/Kernel/Interrupts/APIC.cpp @@ -34,7 +34,9 @@ #define APIC_ENABLED (1 << 8) #define APIC_BASE_MSR 0x1b +#define APIC_REGS_MSR_BASE 0x800 +#define APIC_REG_ID 0x20 #define APIC_REG_EOI 0xb0 #define APIC_REG_LD 0xd0 #define APIC_REG_DF 0xe0 @@ -145,27 +147,39 @@ PhysicalAddress APIC::get_base() void APIC::set_base(const PhysicalAddress& base) { MSR msr(APIC_BASE_MSR); - msr.set(base.get() | 0x800); + u64 flags = 1 << 11; + if (m_is_x2) + flags |= 1 << 10; + msr.set(base.get() | flags); } void APIC::write_register(u32 offset, u32 value) { - *reinterpret_cast(m_apic_base->vaddr().offset(offset).as_ptr()) = value; + if (m_is_x2) { + MSR msr(APIC_REGS_MSR_BASE + (offset >> 4)); + msr.set(value); + } else { + *reinterpret_cast(m_apic_base->vaddr().offset(offset).as_ptr()) = value; + } } u32 APIC::read_register(u32 offset) { + if (m_is_x2) { + MSR msr(APIC_REGS_MSR_BASE + (offset >> 4)); + return (u32)msr.get(); + } return *reinterpret_cast(m_apic_base->vaddr().offset(offset).as_ptr()); } void APIC::set_lvt(u32 offset, u8 interrupt) { - write_register(offset, (read_register(offset) & 0xffffffff) | interrupt); + write_register(offset, read_register(offset) | interrupt); } void APIC::set_siv(u32 offset, u8 interrupt) { - write_register(offset, (read_register(offset) & 0xffffffff) | interrupt | APIC_ENABLED); + write_register(offset, read_register(offset) | interrupt | APIC_ENABLED); } void APIC::wait_for_pending_icr() @@ -177,8 +191,13 @@ void APIC::wait_for_pending_icr() void APIC::write_icr(const ICRReg& icr) { - write_register(APIC_REG_ICR_HIGH, icr.high()); - write_register(APIC_REG_ICR_LOW, icr.low()); + if (m_is_x2) { + MSR msr(APIC_REGS_MSR_BASE + (APIC_REG_ICR_LOW >> 4)); + msr.set(icr.x2_value()); + } else { + write_register(APIC_REG_ICR_HIGH, icr.x_high()); + write_register(APIC_REG_ICR_LOW, icr.x_low()); + } } #define APIC_LVT_TIMER_ONESHOT 0 @@ -224,15 +243,19 @@ UNMAP_AFTER_INIT bool APIC::init_bsp() CPUID id(1); if ((id.edx() & (1 << 9)) == 0) return false; + if (id.ecx() & (1 << 21)) + m_is_x2 = true; PhysicalAddress apic_base = get_base(); - dbgln_if(APIC_DEBUG, "Initializing APIC, base: {}", apic_base); + dbgln_if(APIC_DEBUG, "Initializing {}APIC, base: {}", m_is_x2 ? "x2" : "x", apic_base); set_base(apic_base); - m_apic_base = MM.allocate_kernel_region(apic_base.page_base(), PAGE_SIZE, {}, Memory::Region::Access::ReadWrite); - if (!m_apic_base) { - dbgln("APIC: Failed to allocate memory for APIC base"); - return false; + if (!m_is_x2) { + m_apic_base = MM.allocate_kernel_region(apic_base.page_base(), PAGE_SIZE, {}, Memory::Region::Access::ReadWrite); + if (!m_apic_base) { + dbgln("APIC: Failed to allocate memory for APIC base"); + return false; + } } auto rsdp = ACPI::StaticParsing::find_rsdp(); @@ -254,10 +277,17 @@ UNMAP_AFTER_INIT bool APIC::init_bsp() size_t entry_length = madt_entry->length; if (madt_entry->type == (u8)ACPI::Structures::MADTEntryType::LocalAPIC) { auto* plapic_entry = (const ACPI::Structures::MADTEntries::ProcessorLocalAPIC*)madt_entry; - dbgln_if(APIC_DEBUG, "APIC: AP found @ MADT entry {}, processor ID: {}, APIC ID: {}, flags: {:#08x}", entry_index, plapic_entry->acpi_processor_id, plapic_entry->apic_id, plapic_entry->flags); + dbgln_if(APIC_DEBUG, "APIC: AP found @ MADT entry {}, processor ID: {}, xAPIC ID: {}, flags: {:#08x}", entry_index, plapic_entry->acpi_processor_id, plapic_entry->apic_id, plapic_entry->flags); m_processor_cnt++; if ((plapic_entry->flags & 0x1) != 0) m_processor_enabled_cnt++; + } else if (madt_entry->type == (u8)ACPI::Structures::MADTEntryType::Local_x2APIC) { + // Only used for APID IDs >= 255 + auto* plx2apic_entry = (const ACPI::Structures::MADTEntries::ProcessorLocalX2APIC*)madt_entry; + dbgln_if(APIC_DEBUG, "APIC: AP found @ MADT entry {}, processor ID: {}, x2APIC ID: {}, flags: {:#08x}", entry_index, plx2apic_entry->acpi_processor_id, plx2apic_entry->apic_id, plx2apic_entry->flags); + m_processor_cnt++; + if ((plx2apic_entry->flags & 0x1) != 0) + m_processor_enabled_cnt++; } madt_entry = (ACPI::Structures::MADTEntryHeader*)(VirtualAddress(madt_entry).offset(entry_length).get()); entries_length -= entry_length; @@ -357,13 +387,13 @@ UNMAP_AFTER_INIT void APIC::do_boot_aps() dbgln_if(APIC_DEBUG, "APIC: Starting {} AP(s)", aps_to_enable); // INIT - write_icr(ICRReg(0, ICRReg::INIT, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf)); + write_icr({ 0, 0, ICRReg::INIT, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf }); IO::delay(10 * 1000); for (int i = 0; i < 2; i++) { // SIPI - write_icr(ICRReg(0x08, ICRReg::StartUp, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf)); // start execution at P8000 + write_icr({ 0x08, 0, ICRReg::StartUp, ICRReg::Physical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf }); // start execution at P8000 IO::delay(200); } @@ -405,17 +435,28 @@ UNMAP_AFTER_INIT void APIC::boot_aps() UNMAP_AFTER_INIT void APIC::enable(u32 cpu) { - if (cpu >= 8) { - // TODO: x2apic support? - PANIC("SMP support is currently limited to 8 CPUs!"); + VERIFY(m_is_x2 || cpu < 8); + + u32 apic_id; + if (m_is_x2) { + dbgln_if(APIC_DEBUG, "Enable x2APIC on CPU #{}", cpu); + + // We need to enable x2 mode on each core independently + set_base(get_base()); + + apic_id = read_register(APIC_REG_ID); + } else { + dbgln_if(APIC_DEBUG, "Setting logical xAPIC ID for CPU #{}", cpu); + + // Use the CPU# as logical apic id + VERIFY(cpu <= 8); + write_register(APIC_REG_LD, (read_register(APIC_REG_LD) & 0x00ffffff) | (cpu << 24)); + + // read it back to make sure it's actually set + apic_id = read_register(APIC_REG_LD) >> 24; } - // Use the CPU# as logical apic id - VERIFY(cpu <= 0xff); - write_register(APIC_REG_LD, (read_register(APIC_REG_LD) & 0x00ffffff) | (cpu << 24)); // TODO: only if not in x2apic mode - - // read it back to make sure it's actually set - auto apic_id = read_register(APIC_REG_LD) >> 24; + dbgln_if(APIC_DEBUG, "CPU #{} apic id: {}", cpu, apic_id); Processor::current().info().set_apic_id(apic_id); dbgln_if(APIC_DEBUG, "Enabling local APIC for CPU #{}, logical APIC ID: {}", cpu, apic_id); @@ -423,20 +464,23 @@ UNMAP_AFTER_INIT void APIC::enable(u32 cpu) if (cpu == 0) { SpuriousInterruptHandler::initialize(IRQ_APIC_SPURIOUS); - // set error interrupt vector - set_lvt(APIC_REG_LVT_ERR, IRQ_APIC_ERR); APICErrInterruptHandler::initialize(IRQ_APIC_ERR); // register IPI interrupt vector APICIPIInterruptHandler::initialize(IRQ_APIC_IPI); } + if (!m_is_x2) { + // local destination mode (flat mode), not supported in x2 mode + write_register(APIC_REG_DF, 0xf0000000); + } + + // set error interrupt vector + set_lvt(APIC_REG_LVT_ERR, IRQ_APIC_ERR); + // set spurious interrupt vector set_siv(APIC_REG_SIV, IRQ_APIC_SPURIOUS); - // local destination mode (flat mode) - write_register(APIC_REG_DF, 0xf0000000); - write_register(APIC_REG_LVT_TIMER, APIC_LVT(0, 0) | APIC_LVT_MASKED); write_register(APIC_REG_LVT_THERMAL, APIC_LVT(0, 0) | APIC_LVT_MASKED); write_register(APIC_REG_LVT_PERFORMANCE_COUNTER, APIC_LVT(0, 0) | APIC_LVT_MASKED); @@ -485,21 +529,21 @@ void APIC::broadcast_ipi() { dbgln_if(APIC_SMP_DEBUG, "SMP: Broadcast IPI from CPU #{}", Processor::current_id()); wait_for_pending_icr(); - write_icr(ICRReg(IRQ_APIC_IPI + IRQ_VECTOR_BASE, ICRReg::Fixed, ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf)); + write_icr({ IRQ_APIC_IPI + IRQ_VECTOR_BASE, 0xffffffff, ICRReg::Fixed, ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::AllExcludingSelf }); } void APIC::send_ipi(u32 cpu) { dbgln_if(APIC_SMP_DEBUG, "SMP: Send IPI from CPU #{} to CPU #{}", Processor::current_id(), cpu); VERIFY(cpu != Processor::current_id()); - VERIFY(cpu < 8); + VERIFY(cpu < Processor::count()); wait_for_pending_icr(); - write_icr(ICRReg(IRQ_APIC_IPI + IRQ_VECTOR_BASE, ICRReg::Fixed, ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::NoShorthand, cpu)); + write_icr({ IRQ_APIC_IPI + IRQ_VECTOR_BASE, m_is_x2 ? Processor::by_id(cpu).info().apic_id() : cpu, ICRReg::Fixed, m_is_x2 ? ICRReg::Physical : ICRReg::Logical, ICRReg::Assert, ICRReg::TriggerMode::Edge, ICRReg::NoShorthand }); } UNMAP_AFTER_INIT APICTimer* APIC::initialize_timers(HardwareTimerBase& calibration_timer) { - if (!m_apic_base) + if (!m_apic_base && !m_is_x2) return nullptr; // We should only initialize and calibrate the APIC timer once on the BSP! diff --git a/Kernel/Interrupts/APIC.h b/Kernel/Interrupts/APIC.h index e59ef33564..60f56333c0 100644 --- a/Kernel/Interrupts/APIC.h +++ b/Kernel/Interrupts/APIC.h @@ -47,11 +47,7 @@ public: u32 get_timer_divisor(); private: - class ICRReg { - u32 m_low { 0 }; - u32 m_high { 0 }; - - public: + struct ICRReg { enum DeliveryMode { Fixed = 0x0, LowPriority = 0x1, @@ -79,14 +75,17 @@ private: AllExcludingSelf = 0x3, }; - ICRReg(u8 vector, DeliveryMode delivery_mode, DestinationMode destination_mode, Level level, TriggerMode trigger_mode, DestinationShorthand destinationShort, u8 destination = 0) - : m_low(vector | (delivery_mode << 8) | (destination_mode << 11) | (level << 14) | (static_cast(trigger_mode) << 15) | (destinationShort << 18)) - , m_high((u32)destination << 24) - { - } + u8 vector { 0 }; + u32 destination { 0 }; + DeliveryMode delivery_mode { DeliveryMode::Fixed }; + DestinationMode destination_mode { DestinationMode::Physical }; + Level level { Level::DeAssert }; + TriggerMode trigger_mode { TriggerMode::Edge }; + DestinationShorthand destination_short { DestinationShorthand::NoShorthand }; - u32 low() const { return m_low; } - u32 high() const { return m_high; } + u32 x_low() const { return (u32)vector | (delivery_mode << 8) | (destination_mode << 11) | (level << 14) | (static_cast(trigger_mode) << 15) | (destination_short << 18); } + u32 x_high() const { return destination << 24; } + u64 x2_value() const { return ((u64)destination << 32) | x_low(); } }; OwnPtr m_apic_base; @@ -97,9 +96,10 @@ private: u32 m_processor_cnt { 0 }; u32 m_processor_enabled_cnt { 0 }; APICTimer* m_apic_timer { nullptr }; + bool m_is_x2 { false }; static PhysicalAddress get_base(); - static void set_base(const PhysicalAddress& base); + void set_base(const PhysicalAddress& base); void write_register(u32 offset, u32 value); u32 read_register(u32 offset); void set_lvt(u32 offset, u8 interrupt);