RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-27 06:47:35 +00:00
Code Issues Activity Actions

Kernel: Move the Storage directory to be a new directory under Devices

Browse source 
The Storage subsystem, like the Audio and HID subsystems, exposes Unix
device files (for example, in the /dev directory). To ensure consistency
across the repository, we should make the Storage subsystem to reside in
the Kernel/Devices directory like the two other mentioned subsystems.
This commit is contained in:
Liav A 2023-03-18 13:32:12 +02:00 • committed by Jelle Raaijmakers
parent f3a58f3a5a
commit 500b7b08d6
59 changed files with 133 additions and 133 deletions
Download patch file Download diff file Expand all files Collapse all files

231
Kernel/Devices/Storage/ATA/AHCI/Controller.cpp Normal file
View file

@ -0,0 +1,231 @@
/*
* Copyright (c) 2021-2022, Liav A. <liavalb@hotmail.co.il>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Atomic.h>
#include <AK/BuiltinWrappers.h>
#include <AK/OwnPtr.h>
#include <AK/Types.h>
#include <Kernel/Arch/Delay.h>
#include <Kernel/Bus/PCI/API.h>
#include <Kernel/CommandLine.h>
#include <Kernel/Devices/Storage/ATA/AHCI/Controller.h>
#include <Kernel/Devices/Storage/ATA/AHCI/InterruptHandler.h>
#include <Kernel/Library/LockRefPtr.h>
#include <Kernel/Memory/MemoryManager.h>
namespace Kernel {
UNMAP_AFTER_INIT ErrorOr<NonnullRefPtr<AHCIController>> AHCIController::initialize(PCI::DeviceIdentifier const& pci_device_identifier)
{
auto controller = adopt_ref_if_nonnull(new (nothrow) AHCIController(pci_device_identifier)).release_nonnull();
TRY(controller->initialize_hba(pci_device_identifier));
return controller;
}
ErrorOr<void> AHCIController::reset()
{
dmesgln_pci(*this, "{}: AHCI controller reset", device_identifier().address());
{
SpinlockLocker locker(m_hba_control_lock);
hba().control_regs.ghc = 1;
dbgln_if(AHCI_DEBUG, "{}: AHCI Controller reset", device_identifier().address());
full_memory_barrier();
size_t retry = 0;
// Note: The HBA is locked or hung if we waited more than 1 second!
while (true) {
if (retry > 1000)
return Error::from_errno(ETIMEDOUT);
if (!(hba().control_regs.ghc & 1))
break;
microseconds_delay(1000);
retry++;
}
// Note: Turn on AHCI HBA and Global HBA Interrupts.
full_memory_barrier();
hba().control_regs.ghc = (1 << 31) | (1 << 1);
full_memory_barrier();
}
// Note: According to the AHCI spec the PI register indicates which ports are exposed by the HBA.
// It is loaded by the BIOS. It indicates which ports that the HBA supports are available for software to use.
// For example, on an HBA that supports 6 ports as indicated in CAP.NP, only ports 1 and 3 could be available,
// with ports 0, 2, 4, and 5 being unavailable.
// Which means that even without clearing the AHCI ports array, we are never able to encounter
// a case that we would have stale left-over ports in there. We still clear the array
// for the sake of clarity and completeness, as it doesn't harm anything anyway.
m_ports.fill({});
auto implemented_ports = AHCI::MaskedBitField((u32 volatile&)(hba().control_regs.pi));
for (auto index : implemented_ports.to_vector()) {
auto port = AHCIPort::create(*this, m_hba_capabilities, static_cast<volatile AHCI::PortRegisters&>(hba().port_regs[index]), index).release_value_but_fixme_should_propagate_errors();
m_ports[index] = port;
port->reset();
}
return {};
}
ErrorOr<void> AHCIController::shutdown()
{
return Error::from_errno(ENOTIMPL);
}
size_t AHCIController::devices_count() const
{
SpinlockLocker locker(m_hba_control_lock);
size_t count = 0;
for (auto port : m_ports) {
if (port && port->connected_device())
count++;
}
return count;
}
void AHCIController::start_request(ATADevice const& device, AsyncBlockDeviceRequest& request)
{
auto port = m_ports[device.ata_address().port];
VERIFY(port);
port->start_request(request);
}
void AHCIController::complete_current_request(AsyncDeviceRequest::RequestResult)
{
VERIFY_NOT_REACHED();
}
volatile AHCI::PortRegisters& AHCIController::port(size_t port_number) const
{
VERIFY(port_number < (size_t)AHCI::Limits::MaxPorts);
return static_cast<volatile AHCI::PortRegisters&>(hba().port_regs[port_number]);
}
volatile AHCI::HBA& AHCIController::hba() const
{
return const_cast<AHCI::HBA&>(*m_hba_mapping);
}
UNMAP_AFTER_INIT AHCIController::AHCIController(PCI::DeviceIdentifier const& pci_device_identifier)
: ATAController()
, PCI::Device(const_cast<PCI::DeviceIdentifier&>(pci_device_identifier))
{
}
UNMAP_AFTER_INIT AHCI::HBADefinedCapabilities AHCIController::capabilities() const
{
u32 capabilities = hba().control_regs.cap;
u32 extended_capabilities = hba().control_regs.cap2;
dbgln_if(AHCI_DEBUG, "{}: AHCI Controller Capabilities = {:#08x}, Extended Capabilities = {:#08x}", device_identifier().address(), capabilities, extended_capabilities);
return (AHCI::HBADefinedCapabilities) {
(capabilities & 0b11111) + 1,
((capabilities >> 8) & 0b11111) + 1,
(u8)((capabilities >> 20) & 0b1111),
(capabilities & (u32)(AHCI::HBACapabilities::SXS)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::EMS)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::CCCS)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::PSC)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SSC)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::PMD)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::FBSS)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SPM)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SAM)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SCLO)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SAL)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SALP)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SSS)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SMPS)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SSNTF)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::SNCQ)) != 0,
(capabilities & (u32)(AHCI::HBACapabilities::S64A)) != 0,
(extended_capabilities & (u32)(AHCI::HBACapabilitiesExtended::BOH)) != 0,
(extended_capabilities & (u32)(AHCI::HBACapabilitiesExtended::NVMP)) != 0,
(extended_capabilities & (u32)(AHCI::HBACapabilitiesExtended::APST)) != 0,
(extended_capabilities & (u32)(AHCI::HBACapabilitiesExtended::SDS)) != 0,
(extended_capabilities & (u32)(AHCI::HBACapabilitiesExtended::SADM)) != 0,
(extended_capabilities & (u32)(AHCI::HBACapabilitiesExtended::DESO)) != 0
};
}
UNMAP_AFTER_INIT ErrorOr<Memory::TypedMapping<AHCI::HBA volatile>> AHCIController::map_default_hba_region(PCI::DeviceIdentifier const& pci_device_identifier)
{
return Memory::map_typed_writable<AHCI::HBA volatile>(PhysicalAddress(PCI::get_BAR5(pci_device_identifier)));
}
AHCIController::~AHCIController() = default;
UNMAP_AFTER_INIT ErrorOr<void> AHCIController::initialize_hba(PCI::DeviceIdentifier const& pci_device_identifier)
{
m_hba_mapping = TRY(map_default_hba_region(pci_device_identifier));
m_hba_capabilities = capabilities();
u32 version = hba().control_regs.version;
hba().control_regs.ghc = 0x80000000; // Ensure that HBA knows we are AHCI aware.
PCI::enable_bus_mastering(device_identifier());
TRY(reserve_irqs(1, true));
auto irq = MUST(allocate_irq(0));
enable_global_interrupts();
auto implemented_ports = AHCI::MaskedBitField((u32 volatile&)(hba().control_regs.pi));
m_irq_handler = AHCIInterruptHandler::create(*this, irq, implemented_ports).release_value_but_fixme_should_propagate_errors();
TRY(reset());
dbgln_if(AHCI_DEBUG, "{}: AHCI Controller Version = {:#08x}", device_identifier().address(), version);
dbgln("{}: AHCI command list entries count - {}", device_identifier().address(), m_hba_capabilities.max_command_list_entries_count);
return {};
}
void AHCIController::handle_interrupt_for_port(Badge<AHCIInterruptHandler>, u32 port_index) const
{
auto port = m_ports[port_index];
VERIFY(port);
port->handle_interrupt();
}
void AHCIController::disable_global_interrupts() const
{
hba().control_regs.ghc = hba().control_regs.ghc & 0xfffffffd;
}
void AHCIController::enable_global_interrupts() const
{
hba().control_regs.ghc = hba().control_regs.ghc | (1 << 1);
}
LockRefPtr<StorageDevice> AHCIController::device_by_port(u32 port_index) const
{
SpinlockLocker locker(m_hba_control_lock);
auto port = m_ports[port_index];
if (!port)
return {};
SpinlockLocker port_hard_locker(port->m_hard_lock);
return port->connected_device();
}
LockRefPtr<StorageDevice> AHCIController::device(u32 index) const
{
Vector<NonnullLockRefPtr<StorageDevice>> connected_devices;
u32 pi = hba().control_regs.pi;
u32 bit = bit_scan_forward(pi);
while (bit) {
dbgln_if(AHCI_DEBUG, "Checking implemented port {}, pi {:b}", bit - 1, pi);
pi &= ~(1u << (bit - 1));
auto checked_device = device_by_port(bit - 1);
bit = bit_scan_forward(pi);
if (checked_device.is_null())
continue;
connected_devices.append(checked_device.release_nonnull());
}
dbgln_if(AHCI_DEBUG, "Connected device count: {}, Index: {}", connected_devices.size(), index);
if (index >= connected_devices.size())
return nullptr;
return connected_devices[index];
}
}

68
Kernel/Devices/Storage/ATA/AHCI/Controller.h Normal file
View file

@ -0,0 +1,68 @@
/*
* Copyright (c) 2021-2022, Liav A. <liavalb@hotmail.co.il>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/OwnPtr.h>
#include <AK/Types.h>
#include <Kernel/Devices/Storage/ATA/AHCI/Definitions.h>
#include <Kernel/Devices/Storage/ATA/ATAController.h>
#include <Kernel/Devices/Storage/StorageDevice.h>
#include <Kernel/Library/LockRefPtr.h>
#include <Kernel/Memory/TypedMapping.h>
#include <Kernel/Sections.h>
namespace Kernel {
class AsyncBlockDeviceRequest;
class AHCIInterruptHandler;
class AHCIPort;
class AHCIController final : public ATAController
, public PCI::Device {
friend class AHCIInterruptHandler;
public:
static ErrorOr<NonnullRefPtr<AHCIController>> initialize(PCI::DeviceIdentifier const& pci_device_identifier);
virtual ~AHCIController() override;
virtual StringView device_name() const override { return "AHCI"sv; }
virtual LockRefPtr<StorageDevice> device(u32 index) const override;
virtual ErrorOr<void> reset() override;
virtual ErrorOr<void> shutdown() override;
virtual size_t devices_count() const override;
virtual void start_request(ATADevice const&, AsyncBlockDeviceRequest&) override;
virtual void complete_current_request(AsyncDeviceRequest::RequestResult) override;
void handle_interrupt_for_port(Badge<AHCIInterruptHandler>, u32 port_index) const;
private:
void disable_global_interrupts() const;
void enable_global_interrupts() const;
explicit AHCIController(PCI::DeviceIdentifier const&);
ErrorOr<void> initialize_hba(PCI::DeviceIdentifier const&);
AHCI::HBADefinedCapabilities capabilities() const;
LockRefPtr<StorageDevice> device_by_port(u32 index) const;
volatile AHCI::PortRegisters& port(size_t port_number) const;
ErrorOr<Memory::TypedMapping<AHCI::HBA volatile>> map_default_hba_region(PCI::DeviceIdentifier const&);
volatile AHCI::HBA& hba() const;
Array<LockRefPtr<AHCIPort>, 32> m_ports;
Memory::TypedMapping<AHCI::HBA volatile> m_hba_mapping;
AHCI::HBADefinedCapabilities m_hba_capabilities;
// FIXME: There could be multiple IRQ (MSI) handlers for AHCI. Find a way to use all of them.
OwnPtr<AHCIInterruptHandler> m_irq_handler;
// Note: This lock is intended to be locked when doing changes to HBA registers
// that affect its core functionality in a manner that controls all attached storage devices
// to the HBA SATA ports.
mutable Spinlock<LockRank::None> m_hba_control_lock {};
};
}

429
Kernel/Devices/Storage/ATA/AHCI/Definitions.h Normal file
View file

@ -0,0 +1,429 @@
/*
* Copyright (c) 2021, Liav A. <liavalb@hotmail.co.il>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Types.h>
namespace Kernel::FIS {
enum class Type : u8 {
RegisterHostToDevice = 0x27,
RegisterDeviceToHost = 0x34,
DMAActivate = 0x39,
DMASetup = 0x41,
Data = 0x46,
BISTActivate = 0x58,
PIOSetup = 0x5F,
SetDeviceBits = 0xA1
};
enum class DwordCount : size_t {
RegisterHostToDevice = 5,
RegisterDeviceToHost = 5,
DMAActivate = 1,
DMASetup = 7,
PIOSetup = 5,
SetDeviceBits = 2
};
enum HeaderAttributes : u8 {
C = (1 << 7), /* Updates Command register */
};
struct [[gnu::packed]] Header {
u8 fis_type;
u8 port_muliplier;
};
}
namespace Kernel::FIS::HostToDevice {
struct [[gnu::packed]] Register {
Header header;
u8 command;
u8 features_low;
u8 lba_low[3];
u8 device;
u8 lba_high[3];
u8 features_high;
u16 count;
u8 icc; /* Isochronous Command Completion */
u8 control;
u32 reserved;
};
};
namespace Kernel::FIS::DeviceToHost {
struct [[gnu::packed]] Register {
Header header;
u8 status;
u8 error;
u8 lba_low[3];
u8 device;
u8 lba_high[3];
u8 reserved;
u16 count;
u8 reserved2[6];
};
struct [[gnu::packed]] SetDeviceBits {
Header header;
u8 status;
u8 error;
u32 protocol_specific;
};
struct [[gnu::packed]] DMAActivate {
Header header;
u16 reserved;
};
struct [[gnu::packed]] PIOSetup {
Header header;
u8 status;
u8 error;
u8 lba_low[3];
u8 device;
u8 lba_high[3];
u8 reserved;
u16 count;
u8 reserved2;
u8 e_status;
u16 transfer_count;
u16 reserved3;
};
}
namespace Kernel::FIS::BiDirectional {
struct [[gnu::packed]] Data {
Header header;
u16 reserved;
u32 data[];
};
struct [[gnu::packed]] BISTActivate {
};
struct [[gnu::packed]] DMASetup {
Header header;
u16 reserved;
u32 dma_buffer_identifier_low;
u32 dma_buffer_identifier_high;
u32 reserved2;
u32 dma_buffer_offset;
u32 dma_transfer_count;
u32 reserved3;
};
}
namespace Kernel::AHCI {
class MaskedBitField {
public:
explicit MaskedBitField(u32 volatile& bitfield_register)
: m_bitfield(bitfield_register)
, m_bit_mask(0xffffffff)
{
}
MaskedBitField(u32 volatile& bitfield_register, u32 bit_mask)
: m_bitfield(bitfield_register)
, m_bit_mask(bit_mask)
{
}
void set_at(u8 index) const
{
VERIFY(((1u << index) & m_bit_mask) != 0);
m_bitfield = m_bitfield | ((1u << index) & m_bit_mask);
}
void set_all() const
{
m_bitfield = m_bitfield | (0xffffffff & m_bit_mask);
}
bool is_set_at(u8 port_index) const
{
return m_bitfield & ((1u << port_index) & m_bit_mask);
}
bool is_zeroed() const
{
return (m_bitfield & m_bit_mask) == 0;
}
Vector<u8> to_vector() const
{
// FIXME: Add a sync mechanism!
Vector<u8> indices;
u32 bitfield = m_bitfield & m_bit_mask;
for (size_t index = 0; index < 32; index++) {
if (bitfield & 1) {
indices.append(index);
}
bitfield >>= 1;
}
return indices;
}
u32 bit_mask() const { return m_bit_mask; };
// Disable default implementations that would use surprising integer promotion.
bool operator==(MaskedBitField const&) const = delete;
bool operator<=(MaskedBitField const&) const = delete;
bool operator>=(MaskedBitField const&) const = delete;
bool operator<(MaskedBitField const&) const = delete;
bool operator>(MaskedBitField const&) const = delete;
private:
u32 volatile& m_bitfield;
const u32 m_bit_mask;
};
enum Limits : u16 {
MaxPorts = 32,
MaxCommands = 32,
MaxMultiplierConnectedPorts = 16,
};
enum CommandHeaderAttributes : u16 {
C = (1 << 10), /* Clear Busy upon R_OK */
P = (1 << 7), /* Prefetchable */
W = (1 << 6), /* Write */
A = (1 << 5), /* ATAPI */
R = (1 << 8) /* Reset */
};
enum HBACapabilities : u32 {
S64A = (u32)1 << 31, /* Supports 64-bit Addressing */
SNCQ = 1 << 30, /* Supports Native Command Queuing */
SSNTF = 1 << 29, /* Supports SNotification Register */
SMPS = 1 << 28, /* Supports Mechanical Presence Switch */
SSS = 1 << 27, /* Supports Staggered Spin-up */
SALP = 1 << 26, /* Supports Aggressive Link Power Management */
SAL = 1 << 25, /* Supports Activity LED */
SCLO = 1 << 24, /* Supports Command List Override */
SAM = 1 << 18, /* Supports AHCI mode only */
SPM = 1 << 17, /* Supports Port Multiplier */
FBSS = 1 << 16, /* FIS-based Switching Supported */
PMD = 1 << 15, /* PIO Multiple DRQ Block */
SSC = 1 << 14, /* Slumber State Capable */
PSC = 1 << 13, /* Partial State Capable */
CCCS = 1 << 7, /* Command Completion Coalescing Supported */
EMS = 1 << 6, /* Enclosure Management Supported */
SXS = 1 << 5 /* Supports External SATA */
};
enum HBACapabilitiesExtended : u32 {
DESO = 1 << 5, /* DevSleep Entrance from Slumber Only */
SADM = 1 << 4, /* Supports Aggressive Device Sleep Management */
SDS = 1 << 3, /* Supports Device Sleep */
APST = 1 << 2, /* Automatic Partial to Slumber Transitions */
NVMP = 1 << 1, /* NVMHCI Present */
BOH = 1 << 0, /* BIOS/OS Handoff */
};
// This structure is not defined by the AHCI spec, but is used within the code
struct [[gnu::packed]] HBADefinedCapabilities {
size_t ports_count { 1 };
size_t max_command_list_entries_count { 1 };
u8 interface_speed_generation { 1 };
bool external_sata_supported : 1 { false };
bool enclosure_management_supported : 1 { false };
bool command_completion_coalescing_supported : 1 { false };
bool partial_state_capable : 1 { false };
bool slumber_state_capable : 1 { false };
bool pio_multiple_drq_block : 1 { false };
bool fis_based_switching_supported : 1 { false };
bool port_multiplier_supported : 1 { false };
bool ahci_mode_only : 1 { true };
bool command_list_override_supported : 1 { false };
bool activity_led_supported : 1 { false };
bool aggressive_link_power_management_supported : 1 { false };
bool staggered_spin_up_supported : 1 { false };
bool mechanical_presence_switch_supported : 1 { false };
bool snotification_register_supported : 1 { false };
bool native_command_queuing_supported : 1 { false };
bool addressing_64_bit_supported : 1 { false };
bool bios_os_handoff : 1 { false };
bool nvmhci_present : 1 { false };
bool automatic_partial_to_slumber_transitions : 1 { false };
bool device_sleep_supported : 1 { false };
bool aggressive_device_sleep_management_supported : 1 { false };
bool devsleep_entrance_from_slumber_only : 1 { false };
};
enum class DeviceDetectionInitialization {
NoActionRequested,
PerformInterfaceInitializationSequence,
DisableInterface
};
enum PortInterruptFlag : u32 {
CPD = (u32)1 << 31, /* Cold Port Detect */
TFE = 1 << 30, /* Task File Error */
HBF = 1 << 29, /* Host Bus Fatal Error */
HBD = 1 << 28, /* Host Bus Data Error */
IF = 1 << 27, /* Interface Fatal Error */
INF = 1 << 26, /* Interface Non-fatal Error */
OF = 1 << 24, /* Overflow */
IPM = 1 << 23, /* Incorrect Port Multiplier */
PRC = 1 << 22, /* PhyRdy Change */
DMP = 1 << 7, /* Device Mechanical Presence */
PC = 1 << 6, /* Port Connect Change */
DP = 1 << 5, /* Descriptor Processed */
UF = 1 << 4, /* Unknown FIS */
SDB = 1 << 3, /* Set Device FIS */
DS = 1 << 2, /* DMA Setup FIS */
PS = 1 << 1, /* PIO Setup FIS */
DHR = 1 << 0 /* Device to Host Register FIS */
};
enum SErr : u32 {
DIAG_X = 1 << 26, /* Exchanged */
DIAG_F = 1 << 25, /* Unknown FIS Type */
DIAG_T = 1 << 24, /* Transport state transition error */
DIAG_S = 1 << 23, /* Link sequence error */
DIAG_H = 1 << 22, /* Handshake error */
DIAG_C = 1 << 21, /* CRC error */
DIAG_D = 1 << 20, /* Disparity error */
DIAG_B = 1 << 19, /* 10B to 8B decode error */
DIAG_W = 1 << 18, /* Comm Wake */
DIAG_I = 1 << 17, /* Phy Internal Error */
DIAG_N = 1 << 16, /* PhyRdy Change */
ERR_E = 1 << 11, /* Internal error */
ERR_P = 1 << 10, /* Protocol error */
ERR_C = 1 << 9, /* Persistent communication or data integrity error */
ERR_T = 1 << 8, /* Transient data integrity error */
ERR_M = 1 << 1, /* Received communications error */
ERR_I = 1 << 0, /* Recovered data integrity error */
};
class PortInterruptStatusBitField {
public:
explicit PortInterruptStatusBitField(u32 volatile& bitfield_register)
: m_bitfield(bitfield_register)
{
}
u32 raw_value() const { return m_bitfield; }
bool is_set(PortInterruptFlag flag) const { return m_bitfield & (u32)flag; }
void clear() { m_bitfield = 0xffffffff; }
// Disable default implementations that would use surprising integer promotion.
bool operator==(MaskedBitField const&) const = delete;
bool operator<=(MaskedBitField const&) const = delete;
bool operator>=(MaskedBitField const&) const = delete;
bool operator<(MaskedBitField const&) const = delete;
bool operator>(MaskedBitField const&) const = delete;
private:
u32 volatile& m_bitfield;
};
class PortInterruptEnableBitField {
public:
explicit PortInterruptEnableBitField(u32 volatile& bitfield_register)
: m_bitfield(bitfield_register)
{
}
u32 raw_value() const { return m_bitfield; }
bool is_set(PortInterruptFlag flag) { return m_bitfield & (u32)flag; }
void set_at(PortInterruptFlag flag) { m_bitfield = m_bitfield | static_cast<u32>(flag); }
void clear() { m_bitfield = 0; }
bool is_cleared() const { return m_bitfield == 0; }
void set_all() { m_bitfield = 0xffffffff; }
// Disable default implementations that would use surprising integer promotion.
bool operator==(MaskedBitField const&) const = delete;
bool operator<=(MaskedBitField const&) const = delete;
bool operator>=(MaskedBitField const&) const = delete;
bool operator<(MaskedBitField const&) const = delete;
bool operator>(MaskedBitField const&) const = delete;
private:
u32 volatile& m_bitfield;
};
struct [[gnu::packed]] PortRegisters {
u32 clb; /* Port x Command List Base Address */
u32 clbu; /* Port x Command List Base Address Upper 32-Bits */
u32 fb; /* Port x FIS Base Address */
u32 fbu; /* Port x FIS Base Address Upper 32-Bits */
u32 is; /* Port x Interrupt Status */
u32 ie; /* Port x Interrupt Enable */
u32 cmd; /* Port x Command and Status */
u32 reserved;
u32 tfd; /* Port x Task File Data */
u32 sig; /* Port x Signature */
u32 ssts; /* Port x Serial ATA Status (SCR0: SStatus) */
u32 sctl; /* Port x Serial ATA Control (SCR2: SControl) */
u32 serr; /* Port x Serial ATA Error (SCR1: SError) */
u32 sact; /* Port x Serial ATA Active (SCR3: SActive) */
u32 ci; /* Port x Command Issue */
u32 sntf; /* Port x Serial ATA Notification (SCR4: SNotification) */
u32 fbs; /* Port x FIS-based Switching Control */
u32 devslp; /* Port x Device Sleep */
u8 reserved2[0x70 - 0x48];
u8 vs[16]; /* Port x Vendor Specific */
};
struct [[gnu::packed]] GenericHostControl {
u32 cap; /* Host Capabilities */
u32 ghc; /* Global Host Control */
u32 is; /* Interrupt Status */
u32 pi; /* Ports Implemented */
u32 version;
u32 ccc_ctl; /* Command Completion Coalescing Control */
u32 ccc_ports; /* Command Completion Coalsecing Ports */
u32 em_loc; /* Enclosure Management Location */
u32 em_ctl; /* Enclosure Management Control */
u32 cap2; /* Host Capabilities Extended */
u32 bohc; /* BIOS/OS Handoff Control and Status */
};
struct [[gnu::packed]] HBA {
GenericHostControl control_regs;
u8 reserved[52];
u8 nvmhci[64];
u8 vendor_specific[96];
PortRegisters port_regs[32];
};
struct [[gnu::packed]] CommandHeader {
u16 attributes;
u16 prdtl; /* Physical Region Descriptor Table Length */
u32 prdbc; /* Physical Region Descriptor Byte Count */
u32 ctba; /* Command Table Descriptor Base Address */
u32 ctbau; /* Command Table Descriptor Base Address Upper 32-bits */
u32 reserved[4];
};
struct [[gnu::packed]] PhysicalRegionDescriptor {
u32 base_low;
u32 base_high;
u32 reserved;
u32 byte_count; /* Bit 31 - Interrupt completion, Bit 0 to 21 - Data Byte Count */
};
struct [[gnu::packed]] CommandTable {
u8 command_fis[64];
u8 atapi_command[32];
u8 reserved[32];
PhysicalRegionDescriptor descriptors[];
};
}

55
Kernel/Devices/Storage/ATA/AHCI/InterruptHandler.cpp Normal file
View file

@ -0,0 +1,55 @@
/*
* Copyright (c) 2021-2022, Liav A. <liavalb@hotmail.co.il>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <Kernel/Devices/Storage/ATA/AHCI/InterruptHandler.h>
namespace Kernel {
UNMAP_AFTER_INIT ErrorOr<NonnullOwnPtr<AHCIInterruptHandler>> AHCIInterruptHandler::create(AHCIController& controller, u8 irq, AHCI::MaskedBitField taken_ports)
{
auto port_handler = TRY(adopt_nonnull_own_or_enomem(new (nothrow) AHCIInterruptHandler(controller, irq, taken_ports)));
port_handler->allocate_resources_and_initialize_ports();
return port_handler;
}
void AHCIInterruptHandler::allocate_resources_and_initialize_ports()
{
// Clear pending interrupts, if there are any!
m_pending_ports_interrupts.set_all();
enable_irq();
}
UNMAP_AFTER_INIT AHCIInterruptHandler::AHCIInterruptHandler(AHCIController& controller, u8 irq, AHCI::MaskedBitField taken_ports)
: PCIIRQHandler(controller, irq)
, m_parent_controller(controller)
, m_taken_ports(taken_ports)
, m_pending_ports_interrupts(create_pending_ports_interrupts_bitfield())
{
dbgln_if(AHCI_DEBUG, "AHCI Port Handler: IRQ {}", irq);
}
AHCI::MaskedBitField AHCIInterruptHandler::create_pending_ports_interrupts_bitfield() const
{
return AHCI::MaskedBitField((u32 volatile&)m_parent_controller->hba().control_regs.is, m_taken_ports.bit_mask());
}
AHCIInterruptHandler::~AHCIInterruptHandler() = default;
bool AHCIInterruptHandler::handle_irq(RegisterState const&)
{
dbgln_if(AHCI_DEBUG, "AHCI Port Handler: IRQ received");
if (m_pending_ports_interrupts.is_zeroed())
return false;
for (auto port_index : m_pending_ports_interrupts.to_vector()) {
dbgln_if(AHCI_DEBUG, "AHCI Port Handler: Handling IRQ for port {}", port_index);
m_parent_controller->handle_interrupt_for_port({}, port_index);
// We do this to clear the pending interrupt after we handled it.
m_pending_ports_interrupts.set_at(port_index);
}
return true;
}
}

59
Kernel/Devices/Storage/ATA/AHCI/InterruptHandler.h Normal file
View file

@ -0,0 +1,59 @@
/*
* Copyright (c) 2021-2022, Liav A. <liavalb@hotmail.co.il>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <Kernel/Devices/Device.h>
#include <Kernel/Devices/Storage/ATA/AHCI/Controller.h>
#include <Kernel/Devices/Storage/ATA/AHCI/Port.h>
#include <Kernel/Devices/Storage/StorageDevice.h>
#include <Kernel/Interrupts/PCIIRQHandler.h>
#include <Kernel/Library/LockRefPtr.h>
#include <Kernel/Locking/Mutex.h>
#include <Kernel/Memory/PhysicalPage.h>
#include <Kernel/PhysicalAddress.h>
#include <Kernel/Random.h>
#include <Kernel/Sections.h>
#include <Kernel/WaitQueue.h>
namespace Kernel {
class AsyncBlockDeviceRequest;
class AHCIController;
class AHCIPort;
class AHCIInterruptHandler final : public PCIIRQHandler {
friend class AHCIController;
public:
static ErrorOr<NonnullOwnPtr<AHCIInterruptHandler>> create(AHCIController&, u8 irq, AHCI::MaskedBitField taken_ports);
virtual ~AHCIInterruptHandler() override;
virtual StringView purpose() const override { return "SATA IRQ Handler"sv; }
bool is_responsible_for_port_index(u32 port_index) const { return m_taken_ports.is_set_at(port_index); }
private:
AHCIInterruptHandler(AHCIController&, u8 irq, AHCI::MaskedBitField taken_ports);
void allocate_resources_and_initialize_ports();
//^ IRQHandler
virtual bool handle_irq(RegisterState const&) override;
enum class Direction : u8 {
Read,
Write,
};
AHCI::MaskedBitField create_pending_ports_interrupts_bitfield() const;
// Data members
NonnullLockRefPtr<AHCIController> m_parent_controller;
AHCI::MaskedBitField m_taken_ports;
AHCI::MaskedBitField m_pending_ports_interrupts;
};
}

815
Kernel/Devices/Storage/ATA/AHCI/Port.cpp Normal file
View file

@ -0,0 +1,815 @@
/*
* Copyright (c) 2021, Liav A. <liavalb@hotmail.co.il>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
// For more information about locking in this code
// please look at Documentation/Kernel/AHCILocking.md
#include <AK/Atomic.h>
#include <Kernel/Arch/Delay.h>
#include <Kernel/Devices/Storage/ATA/AHCI/Port.h>
#include <Kernel/Devices/Storage/ATA/ATADiskDevice.h>
#include <Kernel/Devices/Storage/ATA/Definitions.h>
#include <Kernel/Devices/Storage/StorageManagement.h>
#include <Kernel/Locking/Spinlock.h>
#include <Kernel/Memory/MemoryManager.h>
#include <Kernel/Memory/ScatterGatherList.h>
#include <Kernel/Memory/TypedMapping.h>
#include <Kernel/WorkQueue.h>
namespace Kernel {
UNMAP_AFTER_INIT ErrorOr<NonnullLockRefPtr<AHCIPort>> AHCIPort::create(AHCIController const& controller, AHCI::HBADefinedCapabilities hba_capabilities, volatile AHCI::PortRegisters& registers, u32 port_index)
{
auto identify_buffer_page = MUST(MM.allocate_physical_page());
auto port = TRY(adopt_nonnull_lock_ref_or_enomem(new (nothrow) AHCIPort(controller, move(identify_buffer_page), hba_capabilities, registers, port_index)));
TRY(port->allocate_resources_and_initialize_ports());
return port;
}
ErrorOr<void> AHCIPort::allocate_resources_and_initialize_ports()
{
if (is_interface_disabled()) {
m_disabled_by_firmware = true;
return {};
}
m_fis_receive_page = TRY(MM.allocate_physical_page());
for (size_t index = 0; index < 1; index++) {
auto dma_page = TRY(MM.allocate_physical_page());
m_dma_buffers.append(move(dma_page));
}
for (size_t index = 0; index < 1; index++) {
auto command_table_page = TRY(MM.allocate_physical_page());
m_command_table_pages.append(move(command_table_page));
}
m_command_list_region = TRY(MM.allocate_dma_buffer_page("AHCI Port Command List"sv, Memory::Region::Access::ReadWrite, m_command_list_page));
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Command list page at {}", representative_port_index(), m_command_list_page->paddr());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: FIS receive page at {}", representative_port_index(), m_fis_receive_page->paddr());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Command list region at {}", representative_port_index(), m_command_list_region->vaddr());
return {};
}
UNMAP_AFTER_INIT AHCIPort::AHCIPort(AHCIController const& controller, NonnullRefPtr<Memory::PhysicalPage> identify_buffer_page, AHCI::HBADefinedCapabilities hba_capabilities, volatile AHCI::PortRegisters& registers, u32 port_index)
: m_port_index(port_index)
, m_hba_capabilities(hba_capabilities)
, m_identify_buffer_page(move(identify_buffer_page))
, m_port_registers(registers)
, m_parent_controller(controller)
, m_interrupt_status((u32 volatile&)m_port_registers.is)
, m_interrupt_enable((u32 volatile&)m_port_registers.ie)
{
}
void AHCIPort::clear_sata_error_register() const
{
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Clearing SATA error register.", representative_port_index());
m_port_registers.serr = m_port_registers.serr;
}
void AHCIPort::handle_interrupt()
{
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Interrupt handled, PxIS {}", representative_port_index(), m_interrupt_status.raw_value());
if (m_interrupt_status.raw_value() == 0) {
return;
}
if (m_interrupt_status.is_set(AHCI::PortInterruptFlag::PRC) && m_interrupt_status.is_set(AHCI::PortInterruptFlag::PC)) {
clear_sata_error_register();
if ((m_port_registers.ssts & 0xf) != 3 && m_connected_device) {
m_connected_device->prepare_for_unplug();
StorageManagement::the().remove_device(*m_connected_device);
auto work_item_creation_result = g_io_work->try_queue([this]() {
m_connected_device.clear();
});
if (work_item_creation_result.is_error()) {
auto current_request = m_current_request;
m_current_request.clear();
current_request->complete(AsyncDeviceRequest::OutOfMemory);
}
} else {
auto work_item_creation_result = g_io_work->try_queue([this]() {
reset();
});
if (work_item_creation_result.is_error()) {
auto current_request = m_current_request;
m_current_request.clear();
current_request->complete(AsyncDeviceRequest::OutOfMemory);
}
}
return;
}
if (m_interrupt_status.is_set(AHCI::PortInterruptFlag::PRC)) {
clear_sata_error_register();
}
if (m_interrupt_status.is_set(AHCI::PortInterruptFlag::INF)) {
// We need to defer the reset, because we can receive interrupts when
// resetting the device.
auto work_item_creation_result = g_io_work->try_queue([this]() {
reset();
});
if (work_item_creation_result.is_error()) {
auto current_request = m_current_request;
m_current_request.clear();
current_request->complete(AsyncDeviceRequest::OutOfMemory);
}
return;
}
if (m_interrupt_status.is_set(AHCI::PortInterruptFlag::IF) || m_interrupt_status.is_set(AHCI::PortInterruptFlag::TFE) || m_interrupt_status.is_set(AHCI::PortInterruptFlag::HBD) || m_interrupt_status.is_set(AHCI::PortInterruptFlag::HBF)) {
auto work_item_creation_result = g_io_work->try_queue([this]() {
recover_from_fatal_error();
});
if (work_item_creation_result.is_error()) {
auto current_request = m_current_request;
m_current_request.clear();
current_request->complete(AsyncDeviceRequest::OutOfMemory);
}
return;
}
if (m_interrupt_status.is_set(AHCI::PortInterruptFlag::DHR) || m_interrupt_status.is_set(AHCI::PortInterruptFlag::PS)) {
m_wait_for_completion = false;
// Now schedule reading/writing the buffer as soon as we leave the irq handler.
// This is important so that we can safely access the buffers, which could
// trigger page faults
if (!m_current_request) {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request handled, probably identify request", representative_port_index());
} else {
auto work_item_creation_result = g_io_work->try_queue([this]() {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request handled", representative_port_index());
MutexLocker locker(m_lock);
VERIFY(m_current_request);
VERIFY(m_current_scatter_list);
if (!m_connected_device) {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request success", representative_port_index());
complete_current_request(AsyncDeviceRequest::Failure);
return;
}
if (m_current_request->request_type() == AsyncBlockDeviceRequest::Read) {
if (auto result = m_current_request->write_to_buffer(m_current_request->buffer(), m_current_scatter_list->dma_region().as_ptr(), m_connected_device->block_size() * m_current_request->block_count()); result.is_error()) {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request failure, memory fault occurred when reading in data.", representative_port_index());
m_current_scatter_list = nullptr;
complete_current_request(AsyncDeviceRequest::MemoryFault);
return;
}
}
m_current_scatter_list = nullptr;
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request success", representative_port_index());
complete_current_request(AsyncDeviceRequest::Success);
});
if (work_item_creation_result.is_error()) {
auto current_request = m_current_request;
m_current_request.clear();
current_request->complete(AsyncDeviceRequest::OutOfMemory);
}
}
}
m_interrupt_status.clear();
}
bool AHCIPort::is_interrupts_enabled() const
{
return !m_interrupt_enable.is_cleared();
}
void AHCIPort::recover_from_fatal_error()
{
MutexLocker locker(m_lock);
SpinlockLocker lock(m_hard_lock);
LockRefPtr<AHCIController> controller = m_parent_controller.strong_ref();
if (!controller) {
dmesgln("AHCI Port {}: fatal error, controller not available", representative_port_index());
return;
}
dmesgln("{}: AHCI Port {} fatal error, shutting down!", controller->device_identifier().address(), representative_port_index());
dmesgln("{}: AHCI Port {} fatal error, SError {}", controller->device_identifier().address(), representative_port_index(), (u32)m_port_registers.serr);
stop_command_list_processing();
stop_fis_receiving();
m_interrupt_enable.clear();
}
bool AHCIPort::reset()
{
MutexLocker locker(m_lock);
SpinlockLocker lock(m_hard_lock);
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Resetting", representative_port_index());
if (m_disabled_by_firmware) {
dmesgln("AHCI Port {}: Disabled by firmware ", representative_port_index());
return false;
}
full_memory_barrier();
m_interrupt_enable.clear();
m_interrupt_status.clear();
full_memory_barrier();
start_fis_receiving();
full_memory_barrier();
clear_sata_error_register();
full_memory_barrier();
if (!initiate_sata_reset()) {
return false;
}
return initialize();
}
UNMAP_AFTER_INIT bool AHCIPort::initialize_without_reset()
{
MutexLocker locker(m_lock);
SpinlockLocker lock(m_hard_lock);
dmesgln("AHCI Port {}: {}", representative_port_index(), try_disambiguate_sata_status());
return initialize();
}
bool AHCIPort::initialize()
{
VERIFY(m_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Initialization. Signature = {:#08x}", representative_port_index(), static_cast<u32>(m_port_registers.sig));
if (!is_phy_enabled()) {
// Note: If PHY is not enabled, just clear the interrupt status and enable interrupts, in case
// we are going to hotplug a device later.
m_interrupt_status.clear();
m_interrupt_enable.set_all();
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Bailing initialization, Phy is not enabled.", representative_port_index());
return false;
}
rebase();
power_on();
spin_up();
clear_sata_error_register();
start_fis_receiving();
set_active_state();
m_interrupt_status.clear();
m_interrupt_enable.set_all();
full_memory_barrier();
// This actually enables the port...
start_command_list_processing();
full_memory_barrier();
size_t logical_sector_size = 512;
size_t physical_sector_size = 512;
u64 max_addressable_sector = 0;
if (identify_device()) {
auto identify_block = Memory::map_typed<ATAIdentifyBlock>(m_identify_buffer_page->paddr()).release_value_but_fixme_should_propagate_errors();
// Check if word 106 is valid before using it!
if ((identify_block->physical_sector_size_to_logical_sector_size >> 14) == 1) {
if (identify_block->physical_sector_size_to_logical_sector_size & (1 << 12)) {
VERIFY(identify_block->logical_sector_size != 0);
logical_sector_size = identify_block->logical_sector_size;
}
if (identify_block->physical_sector_size_to_logical_sector_size & (1 << 13)) {
physical_sector_size = logical_sector_size << (identify_block->physical_sector_size_to_logical_sector_size & 0xf);
}
}
// Check if the device supports LBA48 mode
if (identify_block->commands_and_feature_sets_supported[1] & (1 << 10)) {
max_addressable_sector = identify_block->user_addressable_logical_sectors_count;
} else {
max_addressable_sector = identify_block->max_28_bit_addressable_logical_sector;
}
if (is_atapi_attached()) {
m_port_registers.cmd = m_port_registers.cmd | (1 << 24);
}
dmesgln("AHCI Port {}: Device found, Capacity={}, Bytes per logical sector={}, Bytes per physical sector={}", representative_port_index(), max_addressable_sector * logical_sector_size, logical_sector_size, physical_sector_size);
// FIXME: We don't support ATAPI devices yet, so for now we don't "create" them
if (!is_atapi_attached()) {
LockRefPtr<AHCIController> controller = m_parent_controller.strong_ref();
if (!controller) {
dmesgln("AHCI Port {}: Device found, but parent controller is not available, abort.", representative_port_index());
return false;
}
m_connected_device = ATADiskDevice::create(*controller, { m_port_index, 0 }, 0, logical_sector_size, max_addressable_sector);
} else {
dbgln("AHCI Port {}: Ignoring ATAPI devices as we don't support them.", representative_port_index());
}
}
return true;
}
char const* AHCIPort::try_disambiguate_sata_status()
{
switch (m_port_registers.ssts & 0xf) {
case 0:
return "Device not detected, Phy not enabled";
case 1:
return "Device detected, Phy disabled";
case 3:
return "Device detected, Phy enabled";
case 4:
return "interface disabled";
}
VERIFY_NOT_REACHED();
}
void AHCIPort::try_disambiguate_sata_error()
{
dmesgln("AHCI Port {}: SErr breakdown:", representative_port_index());
dmesgln("AHCI Port {}: Diagnostics:", representative_port_index());
constexpr u32 diagnostics_bitfield = 0xFFFF0000;
if ((m_port_registers.serr & diagnostics_bitfield) > 0) {
if (m_port_registers.serr & AHCI::SErr::DIAG_X)
dmesgln("AHCI Port {}: - Exchanged", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_F)
dmesgln("AHCI Port {}: - Unknown FIS Type", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_T)
dmesgln("AHCI Port {}: - Transport state transition error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_S)
dmesgln("AHCI Port {}: - Link sequence error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_H)
dmesgln("AHCI Port {}: - Handshake error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_C)
dmesgln("AHCI Port {}: - CRC error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_D)
dmesgln("AHCI Port {}: - Disparity error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_B)
dmesgln("AHCI Port {}: - 10B to 8B decode error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_W)
dmesgln("AHCI Port {}: - Comm Wake", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_I)
dmesgln("AHCI Port {}: - Phy Internal Error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::DIAG_N)
dmesgln("AHCI Port {}: - PhyRdy Change", representative_port_index());
} else {
dmesgln("AHCI Port {}: - No diagnostic information provided.", representative_port_index());
}
dmesgln("AHCI Port {}: Error(s):", representative_port_index());
constexpr u32 error_bitfield = 0xFFFF;
if ((m_port_registers.serr & error_bitfield) > 0) {
if (m_port_registers.serr & AHCI::SErr::ERR_E)
dmesgln("AHCI Port {}: - Internal error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::ERR_P)
dmesgln("AHCI Port {}: - Protocol error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::ERR_C)
dmesgln("AHCI Port {}: - Persistent communication or data integrity error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::ERR_T)
dmesgln("AHCI Port {}: - Transient data integrity error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::ERR_M)
dmesgln("AHCI Port {}: - Recovered communications error", representative_port_index());
if (m_port_registers.serr & AHCI::SErr::ERR_I)
dmesgln("AHCI Port {}: - Recovered data integrity error", representative_port_index());
} else {
dmesgln("AHCI Port {}: - No error information provided.", representative_port_index());
}
}
void AHCIPort::rebase()
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
VERIFY(!m_command_list_page.is_null() && !m_fis_receive_page.is_null());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Rebasing.", representative_port_index());
full_memory_barrier();
stop_command_list_processing();
stop_fis_receiving();
full_memory_barrier();
// Try to wait 1 second for HBA to clear Command List Running and FIS Receive Running
wait_until_condition_met_or_timeout(1000, 1000, [this]() -> bool {
return !(m_port_registers.cmd & (1 << 15)) && !(m_port_registers.cmd & (1 << 14));
});
full_memory_barrier();
m_port_registers.clbu = 0;
m_port_registers.clb = m_command_list_page->paddr().get();
m_port_registers.fbu = 0;
m_port_registers.fb = m_fis_receive_page->paddr().get();
}
bool AHCIPort::is_operable() const
{
// Note: The definition of "operable" is somewhat ambiguous, but we determine it
// by 3 parameters as shown below.
return (!m_command_list_page.is_null())
&& (!m_fis_receive_page.is_null())
&& ((m_port_registers.cmd & (1 << 14)) != 0);
}
void AHCIPort::set_active_state() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Switching to active state.", representative_port_index());
m_port_registers.cmd = (m_port_registers.cmd & 0x0ffffff) | (1 << 28);
}
void AHCIPort::set_sleep_state() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
m_port_registers.cmd = (m_port_registers.cmd & 0x0ffffff) | (0b1000 << 28);
}
size_t AHCIPort::calculate_descriptors_count(size_t block_count) const
{
VERIFY(m_connected_device);
size_t needed_dma_regions_count = Memory::page_round_up((block_count * m_connected_device->block_size())).value() / PAGE_SIZE;
VERIFY(needed_dma_regions_count <= m_dma_buffers.size());
return needed_dma_regions_count;
}
Optional<AsyncDeviceRequest::RequestResult> AHCIPort::prepare_and_set_scatter_list(AsyncBlockDeviceRequest& request)
{
VERIFY(m_lock.is_locked());
VERIFY(request.block_count() > 0);
Vector<NonnullRefPtr<Memory::PhysicalPage>> allocated_dma_regions;
for (size_t index = 0; index < calculate_descriptors_count(request.block_count()); index++) {
allocated_dma_regions.append(m_dma_buffers.at(index));
}
m_current_scatter_list = Memory::ScatterGatherList::try_create(request, allocated_dma_regions.span(), m_connected_device->block_size(), "AHCI Scattered DMA"sv).release_value_but_fixme_should_propagate_errors();
if (!m_current_scatter_list)
return AsyncDeviceRequest::Failure;
if (request.request_type() == AsyncBlockDeviceRequest::Write) {
if (auto result = request.read_from_buffer(request.buffer(), m_current_scatter_list->dma_region().as_ptr(), m_connected_device->block_size() * request.block_count()); result.is_error()) {
return AsyncDeviceRequest::MemoryFault;
}
}
return {};
}
void AHCIPort::start_request(AsyncBlockDeviceRequest& request)
{
MutexLocker locker(m_lock);
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request start", representative_port_index());
VERIFY(!m_current_request);
VERIFY(!m_current_scatter_list);
m_current_request = request;
auto result = prepare_and_set_scatter_list(request);
if (result.has_value()) {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request failure.", representative_port_index());
locker.unlock();
complete_current_request(result.value());
return;
}
auto success = access_device(request.request_type(), request.block_index(), request.block_count());
if (!success) {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Request failure.", representative_port_index());
locker.unlock();
complete_current_request(AsyncDeviceRequest::Failure);
return;
}
}
void AHCIPort::complete_current_request(AsyncDeviceRequest::RequestResult result)
{
VERIFY(m_current_request);
auto current_request = m_current_request;
m_current_request.clear();
current_request->complete(result);
}
bool AHCIPort::spin_until_ready() const
{
VERIFY(m_lock.is_locked());
size_t spin = 0;
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Spinning until ready.", representative_port_index());
while ((m_port_registers.tfd & (ATA_SR_BSY | ATA_SR_DRQ)) && spin <= 100) {
microseconds_delay(1000);
spin++;
}
if (spin == 100) {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: SPIN exceeded 100 milliseconds threshold", representative_port_index());
return false;
}
return true;
}
bool AHCIPort::access_device(AsyncBlockDeviceRequest::RequestType direction, u64 lba, u8 block_count)
{
VERIFY(m_connected_device);
VERIFY(is_operable());
VERIFY(m_lock.is_locked());
VERIFY(m_current_scatter_list);
SpinlockLocker lock(m_hard_lock);
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Do a {}, lba {}, block count {}", representative_port_index(), direction == AsyncBlockDeviceRequest::RequestType::Write ? "write" : "read", lba, block_count);
if (!spin_until_ready())
return false;
auto unused_command_header = try_to_find_unused_command_header();
VERIFY(unused_command_header.has_value());
auto* command_list_entries = (volatile AHCI::CommandHeader*)m_command_list_region->vaddr().as_ptr();
command_list_entries[unused_command_header.value()].ctba = m_command_table_pages[unused_command_header.value()]->paddr().get();
command_list_entries[unused_command_header.value()].ctbau = 0;
command_list_entries[unused_command_header.value()].prdbc = 0;
command_list_entries[unused_command_header.value()].prdtl = m_current_scatter_list->scatters_count();
// Note: we must set the correct Dword count in this register. Real hardware
// AHCI controllers do care about this field! QEMU doesn't care if we don't
// set the correct CFL field in this register, real hardware will set an
// handshake error bit in PxSERR register if CFL is incorrect.
command_list_entries[unused_command_header.value()].attributes = (size_t)FIS::DwordCount::RegisterHostToDevice | AHCI::CommandHeaderAttributes::P | (is_atapi_attached() ? AHCI::CommandHeaderAttributes::A : 0) | (direction == AsyncBlockDeviceRequest::RequestType::Write ? AHCI::CommandHeaderAttributes::W : 0);
dbgln_if(AHCI_DEBUG, "AHCI Port {}: CLE: ctba={:#08x}, ctbau={:#08x}, prdbc={:#08x}, prdtl={:#04x}, attributes={:#04x}", representative_port_index(), (u32)command_list_entries[unused_command_header.value()].ctba, (u32)command_list_entries[unused_command_header.value()].ctbau, (u32)command_list_entries[unused_command_header.value()].prdbc, (u16)command_list_entries[unused_command_header.value()].prdtl, (u16)command_list_entries[unused_command_header.value()].attributes);
auto command_table_region = MM.allocate_kernel_region(m_command_table_pages[unused_command_header.value()]->paddr().page_base(), Memory::page_round_up(sizeof(AHCI::CommandTable)).value(), "AHCI Command Table"sv, Memory::Region::Access::ReadWrite, Memory::Region::Cacheable::No).release_value();
auto& command_table = *(volatile AHCI::CommandTable*)command_table_region->vaddr().as_ptr();
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Allocated command table at {}", representative_port_index(), command_table_region->vaddr());
memset(const_cast<u8*>(command_table.command_fis), 0, 64);
size_t scatter_entry_index = 0;
size_t data_transfer_count = (block_count * m_connected_device->block_size());
for (auto scatter_page : m_current_scatter_list->vmobject().physical_pages()) {
VERIFY(data_transfer_count != 0);
VERIFY(scatter_page);
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Add a transfer scatter entry @ {}", representative_port_index(), scatter_page->paddr());
command_table.descriptors[scatter_entry_index].base_high = 0;
command_table.descriptors[scatter_entry_index].base_low = scatter_page->paddr().get();
if (data_transfer_count <= PAGE_SIZE) {
command_table.descriptors[scatter_entry_index].byte_count = data_transfer_count - 1;
data_transfer_count = 0;
} else {
command_table.descriptors[scatter_entry_index].byte_count = PAGE_SIZE - 1;
data_transfer_count -= PAGE_SIZE;
}
scatter_entry_index++;
}
command_table.descriptors[scatter_entry_index].byte_count = (PAGE_SIZE - 1) | (1 << 31);
memset(const_cast<u8*>(command_table.atapi_command), 0, 32);
auto& fis = *(volatile FIS::HostToDevice::Register*)command_table.command_fis;
fis.header.fis_type = (u8)FIS::Type::RegisterHostToDevice;
if (is_atapi_attached()) {
fis.command = ATA_CMD_PACKET;
TODO();
} else {
if (direction == AsyncBlockDeviceRequest::RequestType::Write)
fis.command = ATA_CMD_WRITE_DMA_EXT;
else
fis.command = ATA_CMD_READ_DMA_EXT;
}
full_memory_barrier();
fis.device = ATA_USE_LBA_ADDRESSING;
fis.header.port_muliplier = (u8)FIS::HeaderAttributes::C;
fis.lba_high[0] = (lba >> 24) & 0xff;
fis.lba_high[1] = (lba >> 32) & 0xff;
fis.lba_high[2] = (lba >> 40) & 0xff;
fis.lba_low[0] = lba & 0xff;
fis.lba_low[1] = (lba >> 8) & 0xff;
fis.lba_low[2] = (lba >> 16) & 0xff;
fis.count = (block_count);
// The below loop waits until the port is no longer busy before issuing a new command
if (!spin_until_ready())
return false;
full_memory_barrier();
mark_command_header_ready_to_process(unused_command_header.value());
full_memory_barrier();
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Do a {}, lba {}, block count {} @ {}, ended", representative_port_index(), direction == AsyncBlockDeviceRequest::RequestType::Write ? "write" : "read", lba, block_count, m_dma_buffers[0]->paddr());
return true;
}
bool AHCIPort::identify_device()
{
VERIFY(m_lock.is_locked());
VERIFY(is_operable());
if (!spin_until_ready())
return false;
LockRefPtr<AHCIController> controller = m_parent_controller.strong_ref();
if (!controller)
return false;
auto unused_command_header = try_to_find_unused_command_header();
VERIFY(unused_command_header.has_value());
auto* command_list_entries = (volatile AHCI::CommandHeader*)m_command_list_region->vaddr().as_ptr();
command_list_entries[unused_command_header.value()].ctba = m_command_table_pages[unused_command_header.value()]->paddr().get();
command_list_entries[unused_command_header.value()].ctbau = 0;
command_list_entries[unused_command_header.value()].prdbc = 512;
command_list_entries[unused_command_header.value()].prdtl = 1;
// Note: we must set the correct Dword count in this register. Real hardware AHCI controllers do care about this field!
// QEMU doesn't care if we don't set the correct CFL field in this register, real hardware will set an handshake error bit in PxSERR register.
command_list_entries[unused_command_header.value()].attributes = (size_t)FIS::DwordCount::RegisterHostToDevice | AHCI::CommandHeaderAttributes::P;
auto command_table_region = MM.allocate_kernel_region(m_command_table_pages[unused_command_header.value()]->paddr().page_base(), Memory::page_round_up(sizeof(AHCI::CommandTable)).value(), "AHCI Command Table"sv, Memory::Region::Access::ReadWrite).release_value();
auto& command_table = *(volatile AHCI::CommandTable*)command_table_region->vaddr().as_ptr();
memset(const_cast<u8*>(command_table.command_fis), 0, 64);
command_table.descriptors[0].base_high = 0;
command_table.descriptors[0].base_low = m_identify_buffer_page->paddr().get();
command_table.descriptors[0].byte_count = 512 - 1;
auto& fis = *(volatile FIS::HostToDevice::Register*)command_table.command_fis;
fis.header.fis_type = (u8)FIS::Type::RegisterHostToDevice;
fis.command = m_port_registers.sig == ATA::DeviceSignature::ATAPI ? ATA_CMD_IDENTIFY_PACKET : ATA_CMD_IDENTIFY;
fis.device = 0;
fis.header.port_muliplier = fis.header.port_muliplier | (u8)FIS::HeaderAttributes::C;
// The below loop waits until the port is no longer busy before issuing a new command
if (!spin_until_ready())
return false;
// Just in case we have a pending interrupt.
m_interrupt_enable.clear();
m_interrupt_status.clear();
full_memory_barrier();
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Marking command header at index {} as ready to identify device", representative_port_index(), unused_command_header.value());
m_port_registers.ci = 1 << unused_command_header.value();
full_memory_barrier();
size_t time_elapsed = 0;
bool success = false;
while (1) {
// Note: We allow it to spin for 256 milliseconds, which should be enough for a device to respond.
if (time_elapsed >= 256) {
break;
}
if (m_port_registers.serr != 0) {
dbgln("AHCI Port {}: Identify failed, SError {:#08x}", representative_port_index(), (u32)m_port_registers.serr);
try_disambiguate_sata_error();
break;
}
if (!(m_port_registers.ci & (1 << unused_command_header.value()))) {
success = true;
break;
}
microseconds_delay(1000); // delay with 1 milliseconds
time_elapsed++;
}
// Note: We probably ended up triggering an interrupt but we don't really want to handle it,
// so just get rid of it.
// FIXME: Do that in a better way so we don't need to actually remember this every time
// we need to do this.
m_interrupt_status.clear();
m_interrupt_enable.set_all();
return success;
}
void AHCIPort::wait_until_condition_met_or_timeout(size_t delay_in_microseconds, size_t retries, Function<bool(void)> condition_being_met) const
{
size_t retry = 0;
while (retry < retries) {
if (condition_being_met())
break;
microseconds_delay(delay_in_microseconds);
retry++;
}
}
bool AHCIPort::shutdown()
{
MutexLocker locker(m_lock);
SpinlockLocker lock(m_hard_lock);
rebase();
set_interface_state(AHCI::DeviceDetectionInitialization::DisableInterface);
return true;
}
Optional<u8> AHCIPort::try_to_find_unused_command_header()
{
VERIFY(m_lock.is_locked());
u32 commands_issued = m_port_registers.ci;
for (size_t index = 0; index < 32; index++) {
if (!(commands_issued & 1)) {
dbgln_if(AHCI_DEBUG, "AHCI Port {}: unused command header at index {}", representative_port_index(), index);
return index;
}
commands_issued >>= 1;
}
return {};
}
void AHCIPort::start_command_list_processing() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
VERIFY(is_operable());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Starting command list processing.", representative_port_index());
m_port_registers.cmd = m_port_registers.cmd | 1;
}
void AHCIPort::mark_command_header_ready_to_process(u8 command_header_index) const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
VERIFY(is_operable());
VERIFY(!m_wait_for_completion);
m_wait_for_completion = true;
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Marking command header at index {} as ready to process.", representative_port_index(), command_header_index);
m_port_registers.ci = 1 << command_header_index;
}
void AHCIPort::stop_command_list_processing() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Stopping command list processing.", representative_port_index());
m_port_registers.cmd = m_port_registers.cmd & 0xfffffffe;
}
void AHCIPort::start_fis_receiving() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Starting FIS receiving.", representative_port_index());
m_port_registers.cmd = m_port_registers.cmd | (1 << 4);
}
void AHCIPort::power_on() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Power on. Cold presence detection? {}", representative_port_index(), (bool)(m_port_registers.cmd & (1 << 20)));
if (!(m_port_registers.cmd & (1 << 20)))
return;
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Powering on device.", representative_port_index());
m_port_registers.cmd = m_port_registers.cmd | (1 << 2);
}
void AHCIPort::spin_up() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Spin up. Staggered spin up? {}", representative_port_index(), m_hba_capabilities.staggered_spin_up_supported);
if (!m_hba_capabilities.staggered_spin_up_supported)
return;
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Spinning up device.", representative_port_index());
m_port_registers.cmd = m_port_registers.cmd | (1 << 1);
}
void AHCIPort::stop_fis_receiving() const
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Stopping FIS receiving.", representative_port_index());
m_port_registers.cmd = m_port_registers.cmd & 0xFFFFFFEF;
}
bool AHCIPort::initiate_sata_reset()
{
VERIFY(m_lock.is_locked());
VERIFY(m_hard_lock.is_locked());
dbgln_if(AHCI_DEBUG, "AHCI Port {}: Initiate SATA reset", representative_port_index());
stop_command_list_processing();
full_memory_barrier();
// Note: The AHCI specification says to wait now a 500 milliseconds
// Try to wait 1 second for HBA to clear Command List Running
wait_until_condition_met_or_timeout(100, 5000, [this]() -> bool {
return !(m_port_registers.cmd & (1 << 15));
});
full_memory_barrier();
spin_up();
full_memory_barrier();
set_interface_state(AHCI::DeviceDetectionInitialization::PerformInterfaceInitializationSequence);
// The AHCI specification says to wait now a 1 millisecond
microseconds_delay(1000);
full_memory_barrier();
set_interface_state(AHCI::DeviceDetectionInitialization::NoActionRequested);
full_memory_barrier();
wait_until_condition_met_or_timeout(10, 1000, [this]() -> bool {
return is_phy_enabled();
});
dmesgln("AHCI Port {}: {}", representative_port_index(), try_disambiguate_sata_status());
full_memory_barrier();
clear_sata_error_register();
return (m_port_registers.ssts & 0xf) == 3;
}
void AHCIPort::set_interface_state(AHCI::DeviceDetectionInitialization requested_action)
{
switch (requested_action) {
case AHCI::DeviceDetectionInitialization::NoActionRequested:
m_port_registers.sctl = (m_port_registers.sctl & 0xfffffff0);
return;
case AHCI::DeviceDetectionInitialization::PerformInterfaceInitializationSequence:
m_port_registers.sctl = (m_port_registers.sctl & 0xfffffff0) | 1;
return;
case AHCI::DeviceDetectionInitialization::DisableInterface:
m_port_registers.sctl = (m_port_registers.sctl & 0xfffffff0) | 4;
return;
}
VERIFY_NOT_REACHED();
}
}

136
Kernel/Devices/Storage/ATA/AHCI/Port.h Normal file
View file

@ -0,0 +1,136 @@
/*
* Copyright (c) 2021-2022, Liav A. <liavalb@hotmail.co.il>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/OwnPtr.h>
#include <AK/RefPtr.h>
#include <Kernel/Devices/Device.h>
#include <Kernel/Devices/Storage/ATA/AHCI/Definitions.h>
#include <Kernel/Devices/Storage/ATA/AHCI/InterruptHandler.h>
#include <Kernel/Devices/Storage/ATA/ATADevice.h>
#include <Kernel/Devices/Storage/ATA/Definitions.h>
#include <Kernel/Interrupts/IRQHandler.h>
#include <Kernel/Library/LockWeakPtr.h>
#include <Kernel/Library/LockWeakable.h>
#include <Kernel/Locking/Mutex.h>
#include <Kernel/Locking/Spinlock.h>
#include <Kernel/Memory/AnonymousVMObject.h>
#include <Kernel/Memory/PhysicalPage.h>
#include <Kernel/Memory/ScatterGatherList.h>
#include <Kernel/PhysicalAddress.h>
#include <Kernel/Random.h>
#include <Kernel/Sections.h>
#include <Kernel/WaitQueue.h>
namespace Kernel {
class AsyncBlockDeviceRequest;
class AHCIInterruptHandler;
class AHCIPort
: public AtomicRefCounted<AHCIPort>
, public LockWeakable<AHCIPort> {
friend class AHCIController;
public:
static ErrorOr<NonnullLockRefPtr<AHCIPort>> create(AHCIController const&, AHCI::HBADefinedCapabilities, volatile AHCI::PortRegisters&, u32 port_index);
u32 port_index() const { return m_port_index; }
u32 representative_port_index() const { return port_index() + 1; }
bool is_operable() const;
bool is_atapi_attached() const { return m_port_registers.sig == (u32)ATA::DeviceSignature::ATAPI; };
LockRefPtr<StorageDevice> connected_device() const { return m_connected_device; }
bool reset();
bool initialize_without_reset();
void handle_interrupt();
private:
ErrorOr<void> allocate_resources_and_initialize_ports();
bool is_phy_enabled() const { return (m_port_registers.ssts & 0xf) == 3; }
bool initialize();
AHCIPort(AHCIController const&, NonnullRefPtr<Memory::PhysicalPage> identify_buffer_page, AHCI::HBADefinedCapabilities, volatile AHCI::PortRegisters&, u32 port_index);
ALWAYS_INLINE void clear_sata_error_register() const;
char const* try_disambiguate_sata_status();
void try_disambiguate_sata_error();
bool initiate_sata_reset();
void rebase();
void recover_from_fatal_error();
bool shutdown();
ALWAYS_INLINE void spin_up() const;
ALWAYS_INLINE void power_on() const;
void start_request(AsyncBlockDeviceRequest&);
void complete_current_request(AsyncDeviceRequest::RequestResult);
bool access_device(AsyncBlockDeviceRequest::RequestType, u64 lba, u8 block_count);
size_t calculate_descriptors_count(size_t block_count) const;
[[nodiscard]] Optional<AsyncDeviceRequest::RequestResult> prepare_and_set_scatter_list(AsyncBlockDeviceRequest& request);
ALWAYS_INLINE bool is_interrupts_enabled() const;
bool spin_until_ready() const;
bool identify_device();
ALWAYS_INLINE void start_command_list_processing() const;
ALWAYS_INLINE void mark_command_header_ready_to_process(u8 command_header_index) const;
ALWAYS_INLINE void stop_command_list_processing() const;
ALWAYS_INLINE void start_fis_receiving() const;
ALWAYS_INLINE void stop_fis_receiving() const;
ALWAYS_INLINE void set_active_state() const;
ALWAYS_INLINE void set_sleep_state() const;
void set_interface_state(AHCI::DeviceDetectionInitialization);
Optional<u8> try_to_find_unused_command_header();
ALWAYS_INLINE bool is_interface_disabled() const { return (m_port_registers.ssts & 0xf) == 4; };
ALWAYS_INLINE void wait_until_condition_met_or_timeout(size_t delay_in_microseconds, size_t retries, Function<bool(void)> condition_being_met) const;
// Data members
EntropySource m_entropy_source;
LockRefPtr<AsyncBlockDeviceRequest> m_current_request;
Spinlock<LockRank::None> m_hard_lock {};
Mutex m_lock { "AHCIPort"sv };
mutable bool m_wait_for_completion { false };
Vector<NonnullRefPtr<Memory::PhysicalPage>> m_dma_buffers;
Vector<NonnullRefPtr<Memory::PhysicalPage>> m_command_table_pages;
RefPtr<Memory::PhysicalPage> m_command_list_page;
OwnPtr<Memory::Region> m_command_list_region;
RefPtr<Memory::PhysicalPage> m_fis_receive_page;
LockRefPtr<ATADevice> m_connected_device;
u32 m_port_index;
// Note: Ideally the AHCIController should be the only object to hold this data
// but because using the m_parent_controller means we need to take a strong ref,
// it's probably better to just "cache" this here instead.
AHCI::HBADefinedCapabilities const m_hba_capabilities;
NonnullRefPtr<Memory::PhysicalPage> const m_identify_buffer_page;
volatile AHCI::PortRegisters& m_port_registers;
LockWeakPtr<AHCIController> m_parent_controller;
AHCI::PortInterruptStatusBitField m_interrupt_status;
AHCI::PortInterruptEnableBitField m_interrupt_enable;
LockRefPtr<Memory::ScatterGatherList> m_current_scatter_list;
bool m_disabled_by_firmware { false };
};
}