From 9035d9e8451288c240f8f1720ae5eab286bd46d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?kleines=20Filmr=C3=B6llchen?= Date: Wed, 11 May 2022 23:24:46 +0200 Subject: [PATCH] LibDSP+Piano: Convert DSP APIs to accept entire sample ranges This has mainly performance benefits, so that we only need to call into all processors once for every audio buffer segment. It requires adjusting quite some logic in most processors and in Track, as we have to consider a larger collection of notes and samples at each step. There's some cautionary TODOs in the currently unused LibDSP tracks because they don't do things properly yet. --- Userland/Applications/Piano/Track.cpp | 7 ++- Userland/Libraries/LibDSP/Effects.cpp | 30 ++++++----- Userland/Libraries/LibDSP/Effects.h | 4 +- Userland/Libraries/LibDSP/Music.h | 24 +++++++-- Userland/Libraries/LibDSP/Processor.h | 9 ++-- Userland/Libraries/LibDSP/Synthesizers.cpp | 49 +++++++++--------- Userland/Libraries/LibDSP/Synthesizers.h | 2 +- Userland/Libraries/LibDSP/Track.cpp | 58 +++++++++++++--------- Userland/Libraries/LibDSP/Track.h | 15 ++++-- 9 files changed, 123 insertions(+), 75 deletions(-) diff --git a/Userland/Applications/Piano/Track.cpp b/Userland/Applications/Piano/Track.cpp index b9132100de..7d006670d3 100644 --- a/Userland/Applications/Piano/Track.cpp +++ b/Userland/Applications/Piano/Track.cpp @@ -45,8 +45,11 @@ void Track::fill_sample(Sample& sample) m_keyboard_notes[i] = {}; } - auto synthesized_sample = m_synth->process(playing_notes).get(); - auto delayed_sample = m_delay->process(synthesized_sample).get(); + auto synthesized_sample = LibDSP::Signal { FixedArray::must_create_but_fixme_should_propagate_errors(1) }; + m_synth->process(playing_notes, synthesized_sample); + auto delayed_signal = LibDSP::Signal { FixedArray::must_create_but_fixme_should_propagate_errors(1) }; + m_delay->process(synthesized_sample, delayed_signal); + auto delayed_sample = delayed_signal.get>()[0]; // HACK: Convert to old Piano range: 16-bit int delayed_sample *= NumericLimits::max(); diff --git a/Userland/Libraries/LibDSP/Effects.cpp b/Userland/Libraries/LibDSP/Effects.cpp index b62a4b293b..bce940f4a6 100644 --- a/Userland/Libraries/LibDSP/Effects.cpp +++ b/Userland/Libraries/LibDSP/Effects.cpp @@ -5,6 +5,7 @@ */ #include "Effects.h" +#include #include namespace LibDSP::Effects { @@ -32,23 +33,26 @@ void Delay::handle_delay_time_change() } } -Signal Delay::process_impl(Signal const& input_signal) +void Delay::process_impl(Signal const& input_signal, Signal& output_signal) { + // FIXME: This is allocating and needs to happen on a different thread. handle_delay_time_change(); - Sample const& in = input_signal.get(); - Sample out; - out += in.log_multiplied(static_cast(m_dry_gain)); - out += m_delay_buffer[m_delay_index].log_multiplied(m_delay_decay); + auto const& samples = input_signal.get>(); + auto& output = output_signal.get>(); + for (size_t i = 0; i < output.size(); ++i) { + auto& out = output[i]; + auto const& sample = samples[i]; + out += sample.log_multiplied(static_cast(m_dry_gain)); + out += m_delay_buffer[m_delay_index].log_multiplied(m_delay_decay); - // This is also convenient for disabling the delay effect by setting the buffer size to 0 - if (m_delay_buffer.size() >= 1) - m_delay_buffer[m_delay_index++] = out; + // This is also convenient for disabling the delay effect by setting the buffer size to 0 + if (m_delay_buffer.size() >= 1) + m_delay_buffer[m_delay_index++] = out; - if (m_delay_index >= m_delay_buffer.size()) - m_delay_index = 0; - - return Signal(out); + if (m_delay_index >= m_delay_buffer.size()) + m_delay_index = 0; + } } Mastering::Mastering(NonnullRefPtr transport) @@ -56,7 +60,7 @@ Mastering::Mastering(NonnullRefPtr transport) { } -Signal Mastering::process_impl([[maybe_unused]] Signal const& input_signal) +void Mastering::process_impl([[maybe_unused]] Signal const& input_signal, [[maybe_unused]] Signal& output_signal) { TODO(); } diff --git a/Userland/Libraries/LibDSP/Effects.h b/Userland/Libraries/LibDSP/Effects.h index 10ac20ac87..1f35bb2be3 100644 --- a/Userland/Libraries/LibDSP/Effects.h +++ b/Userland/Libraries/LibDSP/Effects.h @@ -20,7 +20,7 @@ public: Delay(NonnullRefPtr); private: - virtual Signal process_impl(Signal const&) override; + virtual void process_impl(Signal const&, Signal&) override; void handle_delay_time_change(); ProcessorRangeParameter m_delay_decay; @@ -38,7 +38,7 @@ public: Mastering(NonnullRefPtr); private: - virtual Signal process_impl(Signal const&) override; + virtual void process_impl(Signal const&, Signal&) override; }; } diff --git a/Userland/Libraries/LibDSP/Music.h b/Userland/Libraries/LibDSP/Music.h index d4b90ca93e..e82a57b789 100644 --- a/Userland/Libraries/LibDSP/Music.h +++ b/Userland/Libraries/LibDSP/Music.h @@ -6,7 +6,9 @@ #pragma once +#include #include +#include #include #include #include @@ -66,13 +68,29 @@ enum class SignalType : u8 { Note }; -using RollNotes = OrderedHashMap; +// Perfect hashing for note (MIDI) values. This just uses the note value as the hash itself. +class PerfectNoteHashTraits : Traits { +public: + static constexpr bool equals(u8 const& a, u8 const& b) { return a == b; } + static constexpr unsigned hash(u8 value) + { + return static_cast(value); + } +}; -struct Signal : public Variant { +using RollNotes = OrderedHashMap; + +struct Signal : public Variant, RollNotes> { using Variant::Variant; + AK_MAKE_NONCOPYABLE(Signal); + +public: + Signal& operator=(Signal&&) = default; + Signal(Signal&&) = default; + ALWAYS_INLINE SignalType type() const { - if (has()) + if (has>()) return SignalType::Sample; if (has()) return SignalType::Note; diff --git a/Userland/Libraries/LibDSP/Processor.h b/Userland/Libraries/LibDSP/Processor.h index c9e26d830d..7df6c9f4ba 100644 --- a/Userland/Libraries/LibDSP/Processor.h +++ b/Userland/Libraries/LibDSP/Processor.h @@ -24,12 +24,11 @@ class Processor : public RefCounted { public: virtual ~Processor() = default; - Signal process(Signal const& input_signal) + void process(Signal const& input_signal, Signal& output_signal) { VERIFY(input_signal.type() == m_input_type); - auto processed = process_impl(input_signal); - VERIFY(processed.type() == m_output_type); - return processed; + process_impl(input_signal, output_signal); + VERIFY(output_signal.type() == m_output_type); } SignalType input_type() const { return m_input_type; } SignalType output_type() const { return m_output_type; } @@ -47,7 +46,7 @@ protected: , m_transport(move(transport)) { } - virtual Signal process_impl(Signal const& input_signal) = 0; + virtual void process_impl(Signal const& input_signal, Signal& output_signal) = 0; NonnullRefPtr m_transport; Vector m_parameters; diff --git a/Userland/Libraries/LibDSP/Synthesizers.cpp b/Userland/Libraries/LibDSP/Synthesizers.cpp index 5cab654a2c..9dc8f4887d 100644 --- a/Userland/Libraries/LibDSP/Synthesizers.cpp +++ b/Userland/Libraries/LibDSP/Synthesizers.cpp @@ -31,38 +31,41 @@ Classic::Classic(NonnullRefPtr transport) m_parameters.append(m_release); } -Signal Classic::process_impl(Signal const& input_signal) +void Classic::process_impl(Signal const& input_signal, [[maybe_unused]] Signal& output_signal) { - auto& in = input_signal.get(); + auto const& in = input_signal.get(); + auto& output_samples = output_signal.get>(); - Sample out; + // Do this for every time step and set the signal accordingly. + for (size_t sample_index = 0; sample_index < output_samples.size(); ++sample_index) { + Sample& out = output_samples[sample_index]; + u32 sample_time = m_transport->time() + sample_index; - SinglyLinkedList playing_envelopes; + SinglyLinkedList playing_envelopes; - // "Press" the necessary notes in the internal representation, - // and "release" all of the others - for (u8 i = 0; i < note_frequencies.size(); ++i) { - if (auto maybe_note = in.get(i); maybe_note.has_value()) - m_playing_notes.set(i, maybe_note.value()); + // "Press" the necessary notes in the internal representation, + // and "release" all of the others + for (u8 i = 0; i < note_frequencies.size(); ++i) { + if (auto maybe_note = in.get(i); maybe_note.has_value()) + m_playing_notes.set(i, maybe_note.value()); - if (m_playing_notes.contains(i)) { - Envelope note_envelope = m_playing_notes.get(i)->to_envelope(m_transport->time(), m_attack * m_transport->ms_sample_rate(), m_decay * m_transport->ms_sample_rate(), m_release * m_transport->ms_sample_rate()); - if (!note_envelope.is_active()) { - m_playing_notes.remove(i); - continue; + if (m_playing_notes.contains(i)) { + Envelope note_envelope = m_playing_notes.get(i)->to_envelope(sample_time, m_attack * m_transport->ms_sample_rate(), m_decay * m_transport->ms_sample_rate(), m_release * m_transport->ms_sample_rate()); + if (!note_envelope.is_active()) { + m_playing_notes.remove(i); + continue; + } + + playing_envelopes.append(PitchedEnvelope { note_envelope, i }); } + } - playing_envelopes.append(PitchedEnvelope { note_envelope, i }); + for (auto envelope : playing_envelopes) { + double volume = volume_from_envelope(envelope); + double wave = wave_position(envelope.note); + out += volume * wave; } } - - for (auto envelope : playing_envelopes) { - double volume = volume_from_envelope(envelope); - double wave = wave_position(envelope.note); - out += volume * wave; - } - - return out; } // Linear ADSR envelope with no peak adjustment. diff --git a/Userland/Libraries/LibDSP/Synthesizers.h b/Userland/Libraries/LibDSP/Synthesizers.h index ee74c4cf9b..9453e97f5c 100644 --- a/Userland/Libraries/LibDSP/Synthesizers.h +++ b/Userland/Libraries/LibDSP/Synthesizers.h @@ -47,7 +47,7 @@ public: Waveform wave() const { return m_waveform.value(); } private: - virtual Signal process_impl(Signal const&) override; + virtual void process_impl(Signal const&, Signal&) override; double volume_from_envelope(Envelope const&) const; double wave_position(u8 note); diff --git a/Userland/Libraries/LibDSP/Track.cpp b/Userland/Libraries/LibDSP/Track.cpp index f04b541b2a..160df426c0 100644 --- a/Userland/Libraries/LibDSP/Track.cpp +++ b/Userland/Libraries/LibDSP/Track.cpp @@ -4,13 +4,16 @@ * SPDX-License-Identifier: BSD-2-Clause */ +#include +#include #include +#include +#include #include +#include #include #include -using namespace std; - namespace LibDSP { bool Track::add_processor(NonnullRefPtr new_processor) @@ -48,20 +51,43 @@ bool NoteTrack::check_processor_chain_valid() const return check_processor_chain_valid_with_initial_type(SignalType::Note); } -Sample Track::current_signal() +ErrorOr Track::resize_internal_buffers_to(size_t buffer_size) { + m_secondary_sample_buffer = TRY(FixedArray::try_create(buffer_size)); + return {}; +} + +void Track::current_signal(FixedArray& output_signal) +{ + // This is real-time code. We must NEVER EVER EVER allocate. + NoAllocationGuard guard; + VERIFY(output_signal.size() == m_secondary_sample_buffer.get>().size()); + compute_current_clips_signal(); - Optional the_signal; + Signal* source_signal = &m_current_signal; + // This provides an audio buffer of the right size. It is not allocated here, but whenever we are informed about a buffer size change. + Signal* target_signal = &m_secondary_sample_buffer; for (auto& processor : m_processor_chain) { - the_signal = processor.process(the_signal.value_or(m_current_signal)); + // Depending on what the processor needs to have as output, we need to place either a pre-allocated note hash map or a pre-allocated sample buffer in the target signal. + if (processor.output_type() == SignalType::Note) + target_signal = &m_secondary_note_buffer; + else + target_signal = &m_secondary_sample_buffer; + processor.process(*source_signal, *target_signal); + swap(source_signal, target_signal); } - VERIFY(the_signal.has_value() && the_signal->type() == SignalType::Sample); - return the_signal->get(); + VERIFY(source_signal->type() == SignalType::Sample); + VERIFY(output_signal.size() == source_signal->get>().size()); + // This is one final unavoidable memcopy. Otherwise we need to special-case the last processor or + AK::TypedTransfer::copy(output_signal.data(), source_signal->get>().data(), output_signal.size()); } void NoteTrack::compute_current_clips_signal() { + // Consider the entire time duration. + TODO(); + u32 time = m_transport->time(); // Find the currently playing clip. NoteClip* playing_clip = nullptr; @@ -91,22 +117,8 @@ void NoteTrack::compute_current_clips_signal() void AudioTrack::compute_current_clips_signal() { - // Find the currently playing clip. - u32 time = m_transport->time(); - AudioClip* playing_clip = nullptr; - for (auto& clip : m_clips) { - if (clip.start() <= time && clip.end() >= time) { - playing_clip = &clip; - break; - } - } - if (playing_clip == nullptr) { - m_current_signal = Signal(static_cast(SAMPLE_OFF)); - } - - // Index into the clip's samples. - u32 effective_sample = time - playing_clip->start(); - m_current_signal = Signal(playing_clip->sample_at(effective_sample)); + // This is quite involved as we need to look at multiple clips and take looping into account. + TODO(); } } diff --git a/Userland/Libraries/LibDSP/Track.h b/Userland/Libraries/LibDSP/Track.h index d3af5b7abc..f7a484e175 100644 --- a/Userland/Libraries/LibDSP/Track.h +++ b/Userland/Libraries/LibDSP/Track.h @@ -23,8 +23,11 @@ public: virtual bool check_processor_chain_valid() const = 0; bool add_processor(NonnullRefPtr new_processor); - // Creates the current signal of the track by processing current note or audio data through the processing chain - Sample current_signal(); + // Creates the current signal of the track by processing current note or audio data through the processing chain. + void current_signal(FixedArray& output_signal); + + // We are informed of an audio buffer size change. This happens off-audio-thread so we can allocate. + ErrorOr resize_internal_buffers_to(size_t buffer_size); NonnullRefPtrVector const& processor_chain() const { return m_processor_chain; } NonnullRefPtr transport() const { return m_transport; } @@ -42,7 +45,13 @@ protected: NonnullRefPtrVector m_processor_chain; NonnullRefPtr m_transport; // The current signal is stored here, to prevent unnecessary reallocation. - Signal m_current_signal { Audio::Sample {} }; + Signal m_current_signal { FixedArray {} }; + + // These are so that we don't have to allocate a secondary buffer in current_signal(). + // A sample buffer possibly used by the processor chain. + Signal m_secondary_sample_buffer { FixedArray {} }; + // A note buffer possibly used by the processor chain. + Signal m_secondary_note_buffer { RollNotes {} }; }; class NoteTrack final : public Track {