diff --git a/Userland/Libraries/LibAudio/CMakeLists.txt b/Userland/Libraries/LibAudio/CMakeLists.txt index 67777584e5..cd13d42af7 100644 --- a/Userland/Libraries/LibAudio/CMakeLists.txt +++ b/Userland/Libraries/LibAudio/CMakeLists.txt @@ -5,6 +5,8 @@ set(SOURCES FlacLoader.cpp WavWriter.cpp MP3Loader.cpp + QOALoader.cpp + QOATypes.cpp UserSampleQueue.cpp ) diff --git a/Userland/Libraries/LibAudio/Loader.cpp b/Userland/Libraries/LibAudio/Loader.cpp index 12a5279519..e0d5b7e723 100644 --- a/Userland/Libraries/LibAudio/Loader.cpp +++ b/Userland/Libraries/LibAudio/Loader.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021, the SerenityOS developers. + * Copyright (c) 2018-2023, the SerenityOS developers. * * SPDX-License-Identifier: BSD-2-Clause */ @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace Audio { @@ -41,6 +42,12 @@ Result, LoaderError> Loader::create_plugin(StringVie return NonnullOwnPtr(plugin.release_value()); } + { + auto plugin = QOALoaderPlugin::create(path); + if (!plugin.is_error()) + return NonnullOwnPtr(plugin.release_value()); + } + return LoaderError { "No loader plugin available" }; } @@ -64,6 +71,12 @@ Result, LoaderError> Loader::create_plugin(Bytes buf return NonnullOwnPtr(plugin.release_value()); } + { + auto plugin = QOALoaderPlugin::create(buffer); + if (!plugin.is_error()) + return NonnullOwnPtr(plugin.release_value()); + } + return LoaderError { "No loader plugin available" }; } diff --git a/Userland/Libraries/LibAudio/QOALoader.cpp b/Userland/Libraries/LibAudio/QOALoader.cpp new file mode 100644 index 0000000000..7d640f6d2c --- /dev/null +++ b/Userland/Libraries/LibAudio/QOALoader.cpp @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2023, kleines Filmröllchen + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "QOALoader.h" +#include "Loader.h" +#include "LoaderError.h" +#include "QOATypes.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Audio { + +QOALoaderPlugin::QOALoaderPlugin(NonnullOwnPtr stream) + : LoaderPlugin(move(stream)) +{ +} + +Result, LoaderError> QOALoaderPlugin::create(StringView path) +{ + auto stream = LOADER_TRY(Core::BufferedFile::create(LOADER_TRY(Core::File::open(path, Core::File::OpenMode::Read)))); + auto loader = make(move(stream)); + + LOADER_TRY(loader->initialize()); + + return loader; +} + +Result, LoaderError> QOALoaderPlugin::create(Bytes buffer) +{ + auto loader = make(make(buffer)); + + LOADER_TRY(loader->initialize()); + + return loader; +} + +MaybeLoaderError QOALoaderPlugin::initialize() +{ + TRY(parse_header()); + TRY(reset()); + return {}; +} + +MaybeLoaderError QOALoaderPlugin::parse_header() +{ + u32 header_magic = LOADER_TRY(m_stream->read_value>()); + if (header_magic != QOA::magic) + return LoaderError { LoaderError::Category::Format, 0, "QOA header: Magic number must be 'qoaf'" }; + + m_total_samples = LOADER_TRY(m_stream->read_value>()); + + return {}; +} + +MaybeLoaderError QOALoaderPlugin::load_one_frame(Span& target, IsFirstFrame is_first_frame) +{ + QOA::FrameHeader header = LOADER_TRY(m_stream->read_value()); + + if (header.num_channels > 8) + dbgln("QOALoader: Warning: QOA frame at {} has more than 8 channels ({}), this is not supported by the reference implementation.", LOADER_TRY(m_stream->tell()) - sizeof(QOA::FrameHeader), header.num_channels); + if (header.num_channels == 0) + return LoaderError { LoaderError::Category::Format, LOADER_TRY(m_stream->tell()), "QOA frame: Number of channels must be greater than 0" }; + if (header.sample_count > QOA::max_frame_samples) + return LoaderError { LoaderError::Category::Format, LOADER_TRY(m_stream->tell()), "QOA frame: Too many samples in frame" }; + + // We weren't given a large enough buffer; signal that we didn't write anything and return. + if (header.sample_count > target.size()) { + target = target.trim(0); + LOADER_TRY(m_stream->seek(-sizeof(QOA::frame_header_size), AK::SeekMode::FromCurrentPosition)); + return {}; + } + + target = target.trim(header.sample_count); + + auto lms_states = LOADER_TRY(FixedArray::create(header.num_channels)); + for (size_t channel = 0; channel < header.num_channels; ++channel) { + auto history_packed = LOADER_TRY(m_stream->read_value>()); + auto weights_packed = LOADER_TRY(m_stream->read_value>()); + lms_states[channel] = { history_packed, weights_packed }; + } + + // We pre-allocate very large arrays here, but that's the last allocation of the QOA loader! + // Everything else is just shuffling data around. + // (We will also be using all of the arrays in every frame but the last one.) + auto channels = LOADER_TRY((FixedArray>::create(header.num_channels))); + + // There's usually (and at maximum) 256 slices per channel, but less at the very end. + // If the final slice would be partial, we still need to decode it; integer division would tell us that this final slice doesn't exist. + auto const slice_count = static_cast(ceil(static_cast(header.sample_count) / static_cast(QOA::slice_samples))); + VERIFY(slice_count <= QOA::max_slices_per_frame); + + // Observe the loop nesting: Slices are channel-interleaved. + for (size_t slice = 0; slice < slice_count; ++slice) { + for (size_t channel = 0; channel < header.num_channels; ++channel) { + auto slice_samples = channels[channel].span().slice(slice * QOA::slice_samples, QOA::slice_samples); + TRY(read_one_slice(lms_states[channel], slice_samples)); + } + } + + if (is_first_frame == IsFirstFrame::Yes) { + m_num_channels = header.num_channels; + m_sample_rate = header.sample_rate; + } else { + if (m_sample_rate != header.sample_rate) + return LoaderError { LoaderError::Category::Unimplemented, LOADER_TRY(m_stream->tell()), "QOA: Differing sample rate in non-initial frame" }; + if (m_num_channels != header.num_channels) + m_has_uniform_channel_count = false; + } + + switch (header.num_channels) { + case 1: + for (size_t sample = 0; sample < header.sample_count; ++sample) + target[sample] = Sample { static_cast(channels[0][sample]) / static_cast(NumericLimits::max()) }; + break; + // FIXME: Combine surround channels sensibly, FlacLoader has the same simplification at the moment. + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + case 8: + default: + for (size_t sample = 0; sample < header.sample_count; ++sample) { + target[sample] = { + static_cast(channels[0][sample]) / static_cast(NumericLimits::max()), + static_cast(channels[1][sample]) / static_cast(NumericLimits::max()), + }; + } + break; + } + + return {}; +} + +LoaderSamples QOALoaderPlugin::get_more_samples(size_t max_samples_to_read_from_input) +{ + // Because each frame can only have so many inter-channel samples (quite a low number), + // we just load frames until the limit is reached, but at least one. + // This avoids caching samples in the QOA loader, simplifying state management. + + if (max_samples_to_read_from_input < QOA::max_frame_samples) + return LoaderError { LoaderError::Category::Internal, LOADER_TRY(m_stream->tell()), "QOA loader is not capable of reading less than one frame of samples"sv }; + + ssize_t const remaining_samples = static_cast(m_total_samples - m_loaded_samples); + if (remaining_samples <= 0) + return FixedArray {}; + size_t const samples_to_read = min(max_samples_to_read_from_input, remaining_samples); + auto is_first_frame = m_loaded_samples == 0 ? IsFirstFrame::Yes : IsFirstFrame::No; + + auto samples = LOADER_TRY(FixedArray::create(samples_to_read)); + size_t current_loaded_samples = 0; + + while (current_loaded_samples < samples_to_read) { + auto slice_to_load_into = samples.span().slice(current_loaded_samples, min(QOA::max_frame_samples, samples.size() - current_loaded_samples)); + TRY(this->load_one_frame(slice_to_load_into, is_first_frame)); + is_first_frame = IsFirstFrame::No; + VERIFY(slice_to_load_into.size() <= QOA::max_frame_samples); + current_loaded_samples += slice_to_load_into.size(); + // The buffer wasn't large enough for the next frame. + if (slice_to_load_into.size() == 0) + break; + } + m_loaded_samples += current_loaded_samples; + auto trimmed_samples = LOADER_TRY(FixedArray::create(samples.span().trim(current_loaded_samples))); + + return trimmed_samples; +} + +MaybeLoaderError QOALoaderPlugin::reset() +{ + LOADER_TRY(m_stream->seek(QOA::header_size, AK::SeekMode::SetPosition)); + m_loaded_samples = 0; + // Read the first frame, then seek back to the beginning. This is necessary since the first frame contains the sample rate and channel count. + auto frame_samples = LOADER_TRY(FixedArray::create(QOA::max_frame_samples)); + auto span = frame_samples.span(); + LOADER_TRY(load_one_frame(span, IsFirstFrame::Yes)); + + LOADER_TRY(m_stream->seek(QOA::header_size, AK::SeekMode::SetPosition)); + m_loaded_samples = 0; + return {}; +} + +MaybeLoaderError QOALoaderPlugin::seek(int sample_index) +{ + if (sample_index == 0 && m_loaded_samples == 0) + return {}; + // A QOA file consists of 8 bytes header followed by a number of usually fixed-size frames. + // This fixed bitrate allows us to seek in constant time. + if (!m_has_uniform_channel_count) + return LoaderError { LoaderError::Category::Unimplemented, LOADER_TRY(m_stream->tell()), "QOA with non-uniform channel count is currently not seekable"sv }; + /// FIXME: Change the Loader API to use size_t. + VERIFY(sample_index >= 0); + // We seek to the frame "before"; i.e. the frame that contains that sample. + auto const frame_of_sample = static_cast(AK::floor(static_cast(sample_index) / static_cast(QOA::max_frame_samples))); + auto const frame_size = QOA::frame_header_size + m_num_channels * (QOA::lms_state_size + sizeof(QOA::PackedSlice) * QOA::max_slices_per_frame); + auto const byte_index = QOA::header_size + frame_of_sample * frame_size; + LOADER_TRY(m_stream->seek(byte_index, AK::SeekMode::SetPosition)); + m_loaded_samples = frame_of_sample * QOA::max_frame_samples; + return {}; +} + +MaybeLoaderError QOALoaderPlugin::read_one_slice(QOA::LMSState& lms_state, Span& samples) +{ + VERIFY(samples.size() == QOA::slice_samples); + + auto packed_slice = LOADER_TRY(m_stream->read_value>()); + auto unpacked_slice = unpack_slice(packed_slice); + + for (size_t i = 0; i < QOA::slice_samples; ++i) { + auto const residual = unpacked_slice.residuals[i]; + auto const predicted = lms_state.predict(); + auto const dequantized = QOA::dequantization_table[unpacked_slice.scale_factor_index][residual]; + auto const reconstructed = clamp(predicted + dequantized, QOA::sample_minimum, QOA::sample_maximum); + samples[i] = static_cast(reconstructed); + lms_state.update(reconstructed, dequantized); + } + + return {}; +} + +QOA::UnpackedSlice QOALoaderPlugin::unpack_slice(QOA::PackedSlice packed_slice) +{ + size_t const scale_factor_index = (packed_slice >> 60) & 0b1111; + Array residuals = {}; + auto shifted_slice = packed_slice << 4; + + for (size_t i = 0; i < QOA::slice_samples; ++i) { + residuals[i] = static_cast((shifted_slice >> 61) & 0b111); + shifted_slice <<= 3; + } + + return { + .scale_factor_index = scale_factor_index, + .residuals = residuals, + }; +} + +i16 QOALoaderPlugin::qoa_divide(i16 value, i16 scale_factor) +{ + auto const reciprocal = QOA::reciprocal_table[scale_factor]; + auto const n = (value * reciprocal + (1 << 15)) >> 16; + // Rounding away from zero gives better quantization for small values. + auto const n_rounded = n + (static_cast(value > 0) - static_cast(value < 0)) - (static_cast(n > 0) - static_cast(n < 0)); + return static_cast(n_rounded); +} + +} diff --git a/Userland/Libraries/LibAudio/QOALoader.h b/Userland/Libraries/LibAudio/QOALoader.h new file mode 100644 index 0000000000..9d3cbb055e --- /dev/null +++ b/Userland/Libraries/LibAudio/QOALoader.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023, kleines Filmröllchen + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace Audio { + +// Decoder for the Quite Okay Audio (QOA) format. +// NOTE: The QOA format is not finalized yet and this decoder might not be fully spec-compliant as of 2023-02-02. +// +// https://github.com/phoboslab/qoa/blob/master/qoa.h +class QOALoaderPlugin : public LoaderPlugin { +public: + explicit QOALoaderPlugin(NonnullOwnPtr stream); + virtual ~QOALoaderPlugin() override = default; + + static Result, LoaderError> create(StringView path); + static Result, LoaderError> create(Bytes buffer); + + virtual LoaderSamples get_more_samples(size_t max_samples_to_read_from_input = 128 * KiB) override; + + virtual MaybeLoaderError reset() override; + virtual MaybeLoaderError seek(int sample_index) override; + + virtual int loaded_samples() override { return static_cast(m_loaded_samples); } + virtual int total_samples() override { return static_cast(m_total_samples); } + virtual u32 sample_rate() override { return m_sample_rate; } + virtual u16 num_channels() override { return m_num_channels; } + virtual DeprecatedString format_name() override { return "Quite Okay Audio (.qoa)"; } + virtual PcmSampleFormat pcm_format() override { return PcmSampleFormat::Int16; } + +private: + enum class IsFirstFrame : bool { + Yes = true, + No = false, + }; + + MaybeLoaderError initialize(); + MaybeLoaderError parse_header(); + + MaybeLoaderError load_one_frame(Span& target, IsFirstFrame is_first_frame = IsFirstFrame::No); + // Updates predictor values in lms_state so the next slice can reuse the same state. + MaybeLoaderError read_one_slice(QOA::LMSState& lms_state, Span& samples); + static ALWAYS_INLINE QOA::UnpackedSlice unpack_slice(QOA::PackedSlice packed_slice); + + // QOA's division routine for scaling residuals before final quantization. + static ALWAYS_INLINE i16 qoa_divide(i16 value, i16 scale_factor); + + // Because QOA has dynamic sample rate and channel count, we only use the sample rate and channel count from the first frame. + u32 m_sample_rate { 0 }; + u8 m_num_channels { 0 }; + // If this is the case (the reference encoder even enforces it at the moment) + bool m_has_uniform_channel_count { true }; + + size_t m_loaded_samples { 0 }; + size_t m_total_samples { 0 }; +}; + +} diff --git a/Userland/Libraries/LibAudio/QOATypes.cpp b/Userland/Libraries/LibAudio/QOATypes.cpp new file mode 100644 index 0000000000..7ebdac655f --- /dev/null +++ b/Userland/Libraries/LibAudio/QOATypes.cpp @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023, kleines Filmröllchen + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include "QOATypes.h" +#include +#include + +namespace Audio::QOA { + +ErrorOr FrameHeader::read_from_stream(Stream& stream) +{ + FrameHeader header; + header.num_channels = TRY(stream.read_value()); + u8 sample_rate[3]; + // Enforce the order of the reads here, since the order of expression evaluations further down is implementation-defined. + sample_rate[0] = TRY(stream.read_value()); + sample_rate[1] = TRY(stream.read_value()); + sample_rate[2] = TRY(stream.read_value()); + header.sample_rate = (sample_rate[0] << 16) | (sample_rate[1] << 8) | sample_rate[2]; + header.sample_count = TRY(stream.read_value>()); + header.frame_size = TRY(stream.read_value>()); + return header; +} + +LMSState::LMSState(u64 history_packed, u64 weights_packed) +{ + for (size_t i = 0; i < lms_history; ++i) { + // The casts ensure proper sign extension. + history[i] = static_cast(history_packed >> 48); + history_packed <<= 16; + weights[i] = static_cast(weights_packed >> 48); + weights_packed <<= 16; + } +} + +i32 LMSState::predict() const +{ + i32 prediction = 0; + for (size_t i = 0; i < lms_history; ++i) + prediction += history[i] * weights[i]; + return prediction >> 13; +} + +void LMSState::update(i32 sample, i32 residual) +{ + i32 delta = residual >> 4; + for (size_t i = 0; i < lms_history; ++i) + weights[i] += history[i] < 0 ? -delta : delta; + + for (size_t i = 0; i < lms_history - 1; ++i) + history[i] = history[i + 1]; + history[lms_history - 1] = sample; +} + +} diff --git a/Userland/Libraries/LibAudio/QOATypes.h b/Userland/Libraries/LibAudio/QOATypes.h new file mode 100644 index 0000000000..aeefb5f98b --- /dev/null +++ b/Userland/Libraries/LibAudio/QOATypes.h @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2023, kleines Filmröllchen + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace Audio::QOA { + +// 'qoaf' +static constexpr u32 const magic = 0x716f6166; + +static constexpr size_t const header_size = sizeof(u64); + +struct FrameHeader { + u8 num_channels; + u32 sample_rate; // 24 bits + u16 sample_count; + // TODO: might be removed and/or replaced + u16 frame_size; + + static ErrorOr read_from_stream(Stream& stream); +}; + +static constexpr size_t const frame_header_size = sizeof(u64); + +// Least mean squares (LMS) predictor FIR filter size. +static constexpr size_t const lms_history = 4; + +static constexpr size_t const lms_state_size = 2 * lms_history * sizeof(u16); + +// Only used for internal purposes; intermediate LMS states can be beyond 16 bits. +struct LMSState { + i32 history[lms_history] { 0, 0, 0, 0 }; + i32 weights[lms_history] { 0, 0, 0, 0 }; + + LMSState() = default; + LMSState(u64 history_packed, u64 weights_packed); + + i32 predict() const; + void update(i32 sample, i32 residual); +}; + +using PackedSlice = u64; + +// A QOA slice in a more directly readable format, unpacked from the stored 64-bit format. +struct UnpackedSlice { + size_t scale_factor_index; // 4 bits packed + Array residuals; // 3 bits packed +}; + +// Samples within a 64-bit slice. +static constexpr size_t const slice_samples = 20; +static constexpr size_t const max_slices_per_frame = 256; +static constexpr size_t const max_frame_samples = slice_samples * max_slices_per_frame; + +// Defined as clamping limits by the spec. +static constexpr i32 const sample_minimum = -32768; +static constexpr i32 const sample_maximum = 32767; + +// Quantization and scale factor tables computed from formulas given in qoa.h + +constexpr Array generate_scale_factor_table() +{ + Array scalefactor_table; + for (size_t s = 0; s < 17; ++s) + scalefactor_table[s] = static_cast(AK::round(AK::pow(static_cast(s + 1), 2.75))); + + return scalefactor_table; +} + +// FIXME: Get rid of the literal table once Clang understands our constexpr pow() and round() implementations. +#if defined(AK_COMPILER_CLANG) +static constexpr Array scale_factor_table = { + 1, 7, 21, 45, 84, 138, 211, 304, 421, 562, 731, 928, 1157, 1419, 1715, 2048 +}; +#else +static constexpr Array scale_factor_table = generate_scale_factor_table(); +#endif + +constexpr Array generate_reciprocal_table() +{ + Array reciprocal_table; + for (size_t s = 0; s < 17; ++s) { + reciprocal_table[s] = ((1 << 16) + scale_factor_table[s] - 1) / scale_factor_table[s]; + } + return reciprocal_table; +} + +constexpr Array, 16> generate_dequantization_table() +{ + Array, 16> dequantization_table; + constexpr Array float_dequantization_table = { 0.75, -0.75, 2.5, -2.5, 4.5, -4.5, 7, -7 }; + for (size_t scale = 0; scale < 16; ++scale) { + for (size_t quantization = 0; quantization < 8; ++quantization) + dequantization_table[scale][quantization] = static_cast(AK::round( + static_cast(scale_factor_table[scale]) * float_dequantization_table[quantization])); + } + return dequantization_table; +} + +#if defined(AK_COMPILER_CLANG) +static constexpr Array reciprocal_table = { + 65536, 9363, 3121, 1457, 781, 475, 311, 216, 156, 117, 90, 71, 57, 47, 39, 32 +}; +static constexpr Array, 16> dequantization_table = { + Array { 1, -1, 3, -3, 5, -5, 7, -7 }, + { 5, -5, 18, -18, 32, -32, 49, -49 }, + { 16, -16, 53, -53, 95, -95, 147, -147 }, + { 34, -34, 113, -113, 203, -203, 315, -315 }, + { 63, -63, 210, -210, 378, -378, 588, -588 }, + { 104, -104, 345, -345, 621, -621, 966, -966 }, + { 158, -158, 528, -528, 950, -950, 1477, -1477 }, + { 228, -228, 760, -760, 1368, -1368, 2128, -2128 }, + { 316, -316, 1053, -1053, 1895, -1895, 2947, -2947 }, + { 422, -422, 1405, -1405, 2529, -2529, 3934, -3934 }, + { 548, -548, 1828, -1828, 3290, -3290, 5117, -5117 }, + { 696, -696, 2320, -2320, 4176, -4176, 6496, -6496 }, + { 868, -868, 2893, -2893, 5207, -5207, 8099, -8099 }, + { 1064, -1064, 3548, -3548, 6386, -6386, 9933, -9933 }, + { 1286, -1286, 4288, -4288, 7718, -7718, 12005, -12005 }, + { 1536, -1536, 5120, -5120, 9216, -9216, 14336, -14336 }, +}; +#else +static constexpr Array reciprocal_table = generate_reciprocal_table(); +static constexpr Array, 16> dequantization_table = generate_dequantization_table(); +#endif + +static constexpr Array quantization_table = { + 7, 7, 7, 5, 5, 3, 3, 1, // -8 ..-1 + 0, // 0 + 0, 2, 2, 4, 4, 6, 6, 6 // 1 .. 8 +}; + +}