From 5b774f3617297f47e88347b0244f796d4569ed74 Mon Sep 17 00:00:00 2001 From: Pankaj Raghav Date: Wed, 2 Aug 2023 12:37:52 +0200 Subject: [PATCH] NVMe: Add a new struct Doorbell to encapsulate doorbell registers Introduce a new Struct Doorbell that encapsulates the mmio doorbell register. This commit does not introduce any functional changes and it is added in preparation to adding shadow doorbell support. --- Kernel/Devices/Storage/NVMe/NVMeController.cpp | 11 +++++++++-- .../Devices/Storage/NVMe/NVMeInterruptQueue.cpp | 4 ++-- Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.h | 4 ++-- Kernel/Devices/Storage/NVMe/NVMePollQueue.cpp | 4 ++-- Kernel/Devices/Storage/NVMe/NVMePollQueue.h | 4 ++-- Kernel/Devices/Storage/NVMe/NVMeQueue.cpp | 4 ++-- Kernel/Devices/Storage/NVMe/NVMeQueue.h | 15 ++++++++++----- 7 files changed, 29 insertions(+), 17 deletions(-) diff --git a/Kernel/Devices/Storage/NVMe/NVMeController.cpp b/Kernel/Devices/Storage/NVMe/NVMeController.cpp index ca6d104b28..3d7f893c80 100644 --- a/Kernel/Devices/Storage/NVMe/NVMeController.cpp +++ b/Kernel/Devices/Storage/NVMe/NVMeController.cpp @@ -281,6 +281,9 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_admin_queue(QueueType queu sq_dma_region = move(buffer); } auto doorbell_regs = TRY(Memory::map_typed_writable(PhysicalAddress(m_bar + REG_SQ0TDBL_START))); + Doorbell doorbell = { + .mmio_reg = move(doorbell_regs), + }; m_controller_regs->acq = reinterpret_cast(AK::convert_between_host_and_little_endian(cq_dma_pages.first()->paddr().as_ptr())); m_controller_regs->asq = reinterpret_cast(AK::convert_between_host_and_little_endian(sq_dma_pages.first()->paddr().as_ptr())); @@ -293,7 +296,7 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_admin_queue(QueueType queu return maybe_error; } set_admin_queue_ready_flag(); - m_admin_queue = TRY(NVMeQueue::try_create(*this, 0, irq, qdepth, move(cq_dma_region), move(sq_dma_region), move(doorbell_regs), queue_type)); + m_admin_queue = TRY(NVMeQueue::try_create(*this, 0, irq, qdepth, move(cq_dma_region), move(sq_dma_region), move(doorbell), queue_type)); dbgln_if(NVME_DEBUG, "NVMe: Admin queue created"); return {}; @@ -351,9 +354,13 @@ UNMAP_AFTER_INIT ErrorOr NVMeController::create_io_queue(u8 qid, QueueType auto queue_doorbell_offset = REG_SQ0TDBL_START + ((2 * qid) * (4 << m_dbl_stride)); auto doorbell_regs = TRY(Memory::map_typed_writable(PhysicalAddress(m_bar + queue_doorbell_offset))); + Doorbell doorbell = { + .mmio_reg = move(doorbell_regs), + }; + auto irq = TRY(allocate_irq(qid)); - m_queues.append(TRY(NVMeQueue::try_create(*this, qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), move(sq_dma_region), move(doorbell_regs), queue_type))); + m_queues.append(TRY(NVMeQueue::try_create(*this, qid, irq, IO_QUEUE_SIZE, move(cq_dma_region), move(sq_dma_region), move(doorbell), queue_type))); dbgln_if(NVME_DEBUG, "NVMe: Created IO Queue with QID{}", m_queues.size()); return {}; } diff --git a/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.cpp b/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.cpp index 7d9924185f..a6f3870d13 100644 --- a/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.cpp +++ b/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.cpp @@ -11,14 +11,14 @@ namespace Kernel { -ErrorOr> NVMeInterruptQueue::try_create(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs) +ErrorOr> NVMeInterruptQueue::try_create(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs) { auto queue = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) NVMeInterruptQueue(device, move(rw_dma_region), rw_dma_page, qid, irq, q_depth, move(cq_dma_region), move(sq_dma_region), move(db_regs)))); queue->initialize_interrupt_queue(); return queue; } -UNMAP_AFTER_INIT NVMeInterruptQueue::NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs) +UNMAP_AFTER_INIT NVMeInterruptQueue::NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs) : NVMeQueue(move(rw_dma_region), rw_dma_page, qid, q_depth, move(cq_dma_region), move(sq_dma_region), move(db_regs)) , PCIIRQHandler(device, irq) { diff --git a/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.h b/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.h index 2e2d2547ab..fbc1c66671 100644 --- a/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.h +++ b/Kernel/Devices/Storage/NVMe/NVMeInterruptQueue.h @@ -14,14 +14,14 @@ namespace Kernel { class NVMeInterruptQueue : public NVMeQueue , public PCIIRQHandler { public: - static ErrorOr> try_create(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs); + static ErrorOr> try_create(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs); void submit_sqe(NVMeSubmission& submission) override; virtual ~NVMeInterruptQueue() override {}; virtual StringView purpose() const override { return "NVMe"sv; } void initialize_interrupt_queue(); protected: - NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs); + NVMeInterruptQueue(PCI::Device& device, NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs); private: virtual void complete_current_request(u16 cmdid, u16 status) override; diff --git a/Kernel/Devices/Storage/NVMe/NVMePollQueue.cpp b/Kernel/Devices/Storage/NVMe/NVMePollQueue.cpp index 60cb17d843..35b9b7ce41 100644 --- a/Kernel/Devices/Storage/NVMe/NVMePollQueue.cpp +++ b/Kernel/Devices/Storage/NVMe/NVMePollQueue.cpp @@ -11,12 +11,12 @@ namespace Kernel { -ErrorOr> NVMePollQueue::try_create(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs) +ErrorOr> NVMePollQueue::try_create(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs) { return TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) NVMePollQueue(move(rw_dma_region), rw_dma_page, qid, q_depth, move(cq_dma_region), move(sq_dma_region), move(db_regs)))); } -UNMAP_AFTER_INIT NVMePollQueue::NVMePollQueue(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs) +UNMAP_AFTER_INIT NVMePollQueue::NVMePollQueue(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs) : NVMeQueue(move(rw_dma_region), rw_dma_page, qid, q_depth, move(cq_dma_region), move(sq_dma_region), move(db_regs)) { } diff --git a/Kernel/Devices/Storage/NVMe/NVMePollQueue.h b/Kernel/Devices/Storage/NVMe/NVMePollQueue.h index 7329f2df74..f30d614d79 100644 --- a/Kernel/Devices/Storage/NVMe/NVMePollQueue.h +++ b/Kernel/Devices/Storage/NVMe/NVMePollQueue.h @@ -12,12 +12,12 @@ namespace Kernel { class NVMePollQueue : public NVMeQueue { public: - static ErrorOr> try_create(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs); + static ErrorOr> try_create(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs); void submit_sqe(NVMeSubmission& submission) override; virtual ~NVMePollQueue() override {}; protected: - NVMePollQueue(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs); + NVMePollQueue(NonnullOwnPtr rw_dma_region, NonnullRefPtr rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs); private: virtual void complete_current_request(u16 cmdid, u16 status) override; diff --git a/Kernel/Devices/Storage/NVMe/NVMeQueue.cpp b/Kernel/Devices/Storage/NVMe/NVMeQueue.cpp index 9e9cb5bbde..0d7772a7e4 100644 --- a/Kernel/Devices/Storage/NVMe/NVMeQueue.cpp +++ b/Kernel/Devices/Storage/NVMe/NVMeQueue.cpp @@ -12,7 +12,7 @@ #include namespace Kernel { -ErrorOr> NVMeQueue::try_create(NVMeController& device, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs, QueueType queue_type) +ErrorOr> NVMeQueue::try_create(NVMeController& device, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs, QueueType queue_type) { // Note: Allocate DMA region for RW operation. For now the requests don't exceed more than 4096 bytes (Storage device takes care of it) RefPtr rw_dma_page; @@ -30,7 +30,7 @@ ErrorOr> NVMeQueue::try_create(NVMeController& devi return queue; } -UNMAP_AFTER_INIT NVMeQueue::NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs) +UNMAP_AFTER_INIT NVMeQueue::NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs) : m_rw_dma_region(move(rw_dma_region)) , m_qid(qid) , m_admin_queue(qid == 0) diff --git a/Kernel/Devices/Storage/NVMe/NVMeQueue.h b/Kernel/Devices/Storage/NVMe/NVMeQueue.h index 73eb1d621f..df05047684 100644 --- a/Kernel/Devices/Storage/NVMe/NVMeQueue.h +++ b/Kernel/Devices/Storage/NVMe/NVMeQueue.h @@ -26,6 +26,10 @@ struct DoorbellRegister { u32 cq_head; }; +struct Doorbell { + Memory::TypedMapping mmio_reg; +}; + enum class QueueType { Polled, IRQ @@ -48,7 +52,7 @@ struct NVMeIO { class NVMeController; class NVMeQueue : public AtomicRefCounted { public: - static ErrorOr> try_create(NVMeController& device, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs, QueueType queue_type); + static ErrorOr> try_create(NVMeController& device, u16 qid, u8 irq, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs, QueueType queue_type); bool is_admin_queue() { return m_admin_queue; } u16 submit_sync_sqe(NVMeSubmission&); void read(AsyncBlockDeviceRequest& request, u16 nsid, u64 index, u32 count); @@ -60,9 +64,10 @@ protected: u32 process_cq(); void update_sq_doorbell() { - m_db_regs->sq_tail = m_sq_tail; + m_db_regs.mmio_reg->sq_tail = m_sq_tail; } - NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Memory::TypedMapping db_regs); + + NVMeQueue(NonnullOwnPtr rw_dma_region, Memory::PhysicalPage const& rw_dma_page, u16 qid, u32 q_depth, OwnPtr cq_dma_region, OwnPtr sq_dma_region, Doorbell db_regs); [[nodiscard]] u32 get_request_cid() { @@ -83,7 +88,7 @@ private: virtual void complete_current_request(u16 cmdid, u16 status) = 0; void update_cq_doorbell() { - m_db_regs->cq_head = m_cq_head; + m_db_regs.mmio_reg->cq_head = m_cq_head; } protected: @@ -106,7 +111,7 @@ private: OwnPtr m_sq_dma_region; Span m_cqe_array; WaitQueue m_sync_wait_queue; - Memory::TypedMapping m_db_regs; + Doorbell m_db_regs; NonnullRefPtr const m_rw_dma_page; }; }