Kernel: Add MSIx support to NVMe

Add MSIx support to NVMe. Prefer MSIx over pin-based interrupts as they
are more efficient and all modern hardware support them.

Kernel/Storage/NVMe/NVMeController.cpp

@@ -8,6 +8,7 @@
 #include <AK/Format.h>
 #include <AK/Types.h>
 #include <Kernel/Arch/Delay.h>
+#include <Kernel/Arch/Interrupts.h>
 #include <Kernel/Arch/SafeMem.h>
 #include <Kernel/Bus/PCI/API.h>
 #include <Kernel/CommandLine.h>
@@ -52,6 +53,9 @@ UNMAP_AFTER_INIT ErrorOr<void> NVMeController::initialize(bool is_queue_polled)
     m_ready_timeout = Time::from_milliseconds((CAP_TO(caps) + 1) * 500); // CAP.TO is in 500ms units
 
     calculate_doorbell_stride();
+    // IO queues + 1 admin queue
+    m_irq_type = TRY(reserve_irqs(nr_of_queues + 1, true));
+
     TRY(create_admin_queue(queue_type));
     VERIFY(m_admin_queue_ready == true);
 
@@ -281,13 +285,15 @@ UNMAP_AFTER_INIT ErrorOr<void> NVMeController::create_admin_queue(QueueType queu
     m_controller_regs->acq = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(cq_dma_pages.first()->paddr().as_ptr()));
     m_controller_regs->asq = reinterpret_cast<u64>(AK::convert_between_host_and_little_endian(sq_dma_pages.first()->paddr().as_ptr()));
 
+    auto irq = TRY(allocate_irq(0)); // Admin queue always uses the 0th index when using MSIx
+
     maybe_error = start_controller();
     if (maybe_error.is_error()) {
         dmesgln_pci(*this, "Failed to restart the NVMe controller");
         return maybe_error;
     }
     set_admin_queue_ready_flag();
-    m_admin_queue = TRY(NVMeQueue::try_create(*this, 0, device_identifier().interrupt_line().value(), qdepth, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type));
+    m_admin_queue = TRY(NVMeQueue::try_create(*this, 0, irq, qdepth, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type));
 
     dbgln_if(NVME_DEBUG, "NVMe: Admin queue created");
     return {};
@@ -325,9 +331,8 @@ UNMAP_AFTER_INIT ErrorOr<void> NVMeController::create_io_queue(u8 qid, QueueType
         sub.create_cq.qsize = AK::convert_between_host_and_little_endian(IO_QUEUE_SIZE - 1);
         auto flags = (queue_type == QueueType::IRQ) ? QUEUE_IRQ_ENABLED : QUEUE_IRQ_DISABLED;
         flags |= QUEUE_PHY_CONTIGUOUS;
-        // TODO: Eventually move to MSI.
+        // When using MSIx interrupts, qid is used as an index into the interrupt table
-        // For now using pin based interrupts. Clear the first 16 bits
+        sub.create_cq.irq_vector = (m_irq_type == PCI::InterruptType::PIN) ? 0 : qid;
-        // to use pin-based interrupts.
         sub.create_cq.cq_flags = AK::convert_between_host_and_little_endian(flags & 0xFFFF);
         submit_admin_command(sub, true);
     }
@@ -346,8 +351,9 @@ UNMAP_AFTER_INIT ErrorOr<void> NVMeController::create_io_queue(u8 qid, QueueType
 
     auto queue_doorbell_offset = REG_SQ0TDBL_START + ((2 * qid) * (4 << m_dbl_stride));
     auto doorbell_regs = TRY(Memory::map_typed_writable<DoorbellRegister volatile>(PhysicalAddress(m_bar + queue_doorbell_offset)));
+    auto irq = TRY(allocate_irq(qid));
 
-    m_queues.append(TRY(NVMeQueue::try_create(*this, qid, device_identifier().interrupt_line().value(), IO_QUEUE_SIZE, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type)));
+    m_queues.append(TRY(NVMeQueue::try_create(*this, qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), cq_dma_pages, move(sq_dma_region), sq_dma_pages, move(doorbell_regs), queue_type)));
     dbgln_if(NVME_DEBUG, "NVMe: Created IO Queue with QID{}", m_queues.size());
     return {};
 }

Kernel/Storage/NVMe/NVMeController.h

@@ -77,6 +77,7 @@ private:
     AK::Time m_ready_timeout;
     u32 m_bar { 0 };
     u8 m_dbl_stride { 0 };
+    PCI::InterruptType m_irq_type;
     QueueType m_queue_type { QueueType::IRQ };
     static Atomic<u8> s_controller_id;
 };

Kernel/Storage/NVMe/NVMeInterruptQueue.cpp

@@ -11,9 +11,9 @@
 
 namespace Kernel {
 
-UNMAP_AFTER_INIT NVMeInterruptQueue::NVMeInterruptQueue([[maybe_unused]] PCI::Device& device, NonnullOwnPtr<Memory::Region> rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr<Memory::Region> cq_dma_region, Vector<NonnullRefPtr<Memory::PhysicalPage>> cq_dma_page, OwnPtr<Memory::Region> sq_dma_region, Vector<NonnullRefPtr<Memory::PhysicalPage>> sq_dma_page, Memory::TypedMapping<DoorbellRegister volatile> db_regs)
+UNMAP_AFTER_INIT NVMeInterruptQueue::NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr<Memory::Region> rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr<Memory::Region> cq_dma_region, Vector<NonnullRefPtr<Memory::PhysicalPage>> cq_dma_page, OwnPtr<Memory::Region> sq_dma_region, Vector<NonnullRefPtr<Memory::PhysicalPage>> sq_dma_page, Memory::TypedMapping<DoorbellRegister volatile> db_regs)
     : NVMeQueue(move(rw_dma_region), rw_dma_page, qid, q_depth, move(cq_dma_region), cq_dma_page, move(sq_dma_region), sq_dma_page, move(db_regs))
-    , IRQHandler(irq)
+    , PCIIRQHandler(device, irq)
 {
     enable_irq();
 }

Kernel/Storage/NVMe/NVMeInterruptQueue.h

@@ -6,16 +6,18 @@
 
 #pragma once
 
+#include <Kernel/Interrupts/PCIIRQHandler.h>
 #include <Kernel/Storage/NVMe/NVMeQueue.h>
 
 namespace Kernel {
 
 class NVMeInterruptQueue : public NVMeQueue
-    , public IRQHandler {
+    , public PCIIRQHandler {
 public:
     NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr<Memory::Region> rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr<Memory::Region> cq_dma_region, Vector<NonnullRefPtr<Memory::PhysicalPage>> cq_dma_page, OwnPtr<Memory::Region> sq_dma_region, Vector<NonnullRefPtr<Memory::PhysicalPage>> sq_dma_page, Memory::TypedMapping<DoorbellRegister volatile> db_regs);
     void submit_sqe(NVMeSubmission& submission) override;
     virtual ~NVMeInterruptQueue() override {};
+    virtual StringView purpose() const override { return "NVMe"sv; };
 
 private:
     virtual void complete_current_request(u16 cmdid, u16 status) override;