From d049626f402f50720a1ccc4452676a56e22debbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?kleines=20Filmr=C3=B6llchen?= Date: Thu, 19 Aug 2021 00:13:26 +0200 Subject: [PATCH] Userland+LibAudio: Make audio applications support dynamic sample rate All audio applications (aplay, Piano, Sound Player) respect the ability of the system to have theoretically any sample rate. Therefore, they resample their own audio into the system sample rate. LibAudio previously had its loaders resample their own audio, even though they expose their sample rate. This is now changed. The loaders output audio data in their file's sample rate, which the user has to query and resample appropriately. Resampling code from Buffer, WavLoader and FlacLoader is removed. Note that these applications only check the sample rate at startup, which is reasonable (the user has to restart applications when changing the sample rate). Fully dynamic adaptation could both lead to errors and will require another IPC interface. This seems to be enough for now. --- .../Applications/Piano/AudioPlayerLoop.cpp | 2 + Userland/Applications/Piano/AudioPlayerLoop.h | 2 + Userland/Applications/Piano/Track.cpp | 7 ++- .../SoundPlayer/PlaybackManager.cpp | 5 +++ .../SoundPlayer/PlaybackManager.h | 1 + Userland/Libraries/LibAudio/Buffer.cpp | 43 +++++++++++-------- Userland/Libraries/LibAudio/Buffer.h | 12 ++++-- Userland/Libraries/LibAudio/FlacLoader.cpp | 6 --- Userland/Libraries/LibAudio/FlacLoader.h | 1 - Userland/Libraries/LibAudio/WavLoader.cpp | 5 --- Userland/Libraries/LibAudio/WavLoader.h | 8 ---- Userland/Utilities/aplay.cpp | 5 +++ 12 files changed, 56 insertions(+), 41 deletions(-) diff --git a/Userland/Applications/Piano/AudioPlayerLoop.cpp b/Userland/Applications/Piano/AudioPlayerLoop.cpp index ea39f00d35..9a0861a0ed 100644 --- a/Userland/Applications/Piano/AudioPlayerLoop.cpp +++ b/Userland/Applications/Piano/AudioPlayerLoop.cpp @@ -31,12 +31,14 @@ AudioPlayerLoop::AudioPlayerLoop(TrackManager& track_manager, bool& need_to_writ (void)buffer_id; enqueue_audio(); }; + m_resampler = Audio::ResampleHelper(Music::sample_rate, m_audio_client->get_sample_rate()); } void AudioPlayerLoop::enqueue_audio() { m_track_manager.fill_buffer(m_buffer); NonnullRefPtr audio_buffer = music_samples_to_buffer(m_buffer); + audio_buffer = Audio::resample_buffer(m_resampler.value(), *audio_buffer); m_audio_client->async_enqueue(audio_buffer); // FIXME: This should be done somewhere else. diff --git a/Userland/Applications/Piano/AudioPlayerLoop.h b/Userland/Applications/Piano/AudioPlayerLoop.h index 41709745c4..0125215298 100644 --- a/Userland/Applications/Piano/AudioPlayerLoop.h +++ b/Userland/Applications/Piano/AudioPlayerLoop.h @@ -8,6 +8,7 @@ #pragma once #include "Music.h" +#include #include #include #include @@ -29,6 +30,7 @@ public: private: TrackManager& m_track_manager; Array m_buffer; + Optional> m_resampler; RefPtr m_audio_client; bool m_should_play_audio = true; diff --git a/Userland/Applications/Piano/Track.cpp b/Userland/Applications/Piano/Track.cpp index 4f8567fe16..9de5b3a1ae 100644 --- a/Userland/Applications/Piano/Track.cpp +++ b/Userland/Applications/Piano/Track.cpp @@ -127,7 +127,12 @@ String Track::set_recorded_sample(const StringView& path) NonnullRefPtr loader = Audio::Loader::create(path); if (loader->has_error()) return String(loader->error_string()); - auto buffer = loader->get_more_samples(60 * sample_rate * sizeof(Sample)); // 1 minute maximum + auto buffer = loader->get_more_samples(60 * loader->sample_rate()); // 1 minute maximum + if (loader->has_error()) + return String(loader->error_string()); + // Resample to Piano's internal sample rate + auto resampler = Audio::ResampleHelper(loader->sample_rate(), sample_rate); + buffer = Audio::resample_buffer(resampler, *buffer); if (!m_recorded_sample.is_empty()) m_recorded_sample.clear(); diff --git a/Userland/Applications/SoundPlayer/PlaybackManager.cpp b/Userland/Applications/SoundPlayer/PlaybackManager.cpp index 9655741211..74a041cab2 100644 --- a/Userland/Applications/SoundPlayer/PlaybackManager.cpp +++ b/Userland/Applications/SoundPlayer/PlaybackManager.cpp @@ -15,6 +15,7 @@ PlaybackManager::PlaybackManager(NonnullRefPtr connecti next_buffer(); }); m_timer->stop(); + m_device_sample_rate = connection->get_sample_rate(); } PlaybackManager::~PlaybackManager() @@ -30,6 +31,7 @@ void PlaybackManager::set_loader(NonnullRefPtr&& loader) m_device_samples_per_buffer = PlaybackManager::buffer_size_ms / 1000.0f * m_device_sample_rate; u32 source_samples_per_buffer = PlaybackManager::buffer_size_ms / 1000.0f * m_loader->sample_rate(); m_source_buffer_size_bytes = source_samples_per_buffer * m_loader->num_channels() * m_loader->bits_per_sample() / 8; + m_resampler = Audio::ResampleHelper(m_loader->sample_rate(), m_device_sample_rate); m_timer->start(); } else { m_timer->stop(); @@ -116,6 +118,9 @@ void PlaybackManager::next_buffer() if (audio_server_remaining_samples < m_device_samples_per_buffer) { m_current_buffer = m_loader->get_more_samples(m_source_buffer_size_bytes); + VERIFY(m_resampler.has_value()); + m_resampler->reset(); + m_current_buffer = Audio::resample_buffer(m_resampler.value(), *m_current_buffer); if (m_current_buffer) m_connection->enqueue(*m_current_buffer); } diff --git a/Userland/Applications/SoundPlayer/PlaybackManager.h b/Userland/Applications/SoundPlayer/PlaybackManager.h index 35afc3fa2c..9d051d2cf8 100644 --- a/Userland/Applications/SoundPlayer/PlaybackManager.h +++ b/Userland/Applications/SoundPlayer/PlaybackManager.h @@ -52,6 +52,7 @@ private: RefPtr m_loader { nullptr }; NonnullRefPtr m_connection; RefPtr m_current_buffer; + Optional> m_resampler; RefPtr m_timer; // Controls the GUI update rate. A smaller value makes the visualizations nicer. diff --git a/Userland/Libraries/LibAudio/Buffer.cpp b/Userland/Libraries/LibAudio/Buffer.cpp index 1039f5bb3f..4f401b4f28 100644 --- a/Userland/Libraries/LibAudio/Buffer.cpp +++ b/Userland/Libraries/LibAudio/Buffer.cpp @@ -45,7 +45,7 @@ i32 Buffer::allocate_id() } template -static void read_samples_from_stream(InputMemoryStream& stream, SampleReader read_sample, Vector& samples, ResampleHelper& resampler, int num_channels) +static void read_samples_from_stream(InputMemoryStream& stream, SampleReader read_sample, Vector& samples, int num_channels) { double norm_l = 0; double norm_r = 0; @@ -53,29 +53,23 @@ static void read_samples_from_stream(InputMemoryStream& stream, SampleReader rea switch (num_channels) { case 1: for (;;) { - while (resampler.read_sample(norm_l, norm_r)) { - samples.append(Frame(norm_l)); - } norm_l = read_sample(stream); + samples.append(Frame(norm_l)); if (stream.handle_any_error()) { break; } - resampler.process_sample(norm_l, norm_r); } break; case 2: for (;;) { - while (resampler.read_sample(norm_l, norm_r)) { - samples.append(Frame(norm_l, norm_r)); - } norm_l = read_sample(stream); norm_r = read_sample(stream); + samples.append(Frame(norm_l, norm_r)); if (stream.handle_any_error()) { break; } - resampler.process_sample(norm_l, norm_r); } break; default: @@ -128,32 +122,32 @@ static double read_norm_sample_8(InputMemoryStream& stream) return double(sample) / NumericLimits::max(); } -RefPtr Buffer::from_pcm_data(ReadonlyBytes data, ResampleHelper& resampler, int num_channels, PcmSampleFormat sample_format) +RefPtr Buffer::from_pcm_data(ReadonlyBytes data, int num_channels, PcmSampleFormat sample_format) { InputMemoryStream stream { data }; - return from_pcm_stream(stream, resampler, num_channels, sample_format, data.size() / (pcm_bits_per_sample(sample_format) / 8)); + return from_pcm_stream(stream, num_channels, sample_format, data.size() / (pcm_bits_per_sample(sample_format) / 8)); } -RefPtr Buffer::from_pcm_stream(InputMemoryStream& stream, ResampleHelper& resampler, int num_channels, PcmSampleFormat sample_format, int num_samples) +RefPtr Buffer::from_pcm_stream(InputMemoryStream& stream, int num_channels, PcmSampleFormat sample_format, int num_samples) { Vector fdata; fdata.ensure_capacity(num_samples); switch (sample_format) { case PcmSampleFormat::Uint8: - read_samples_from_stream(stream, read_norm_sample_8, fdata, resampler, num_channels); + read_samples_from_stream(stream, read_norm_sample_8, fdata, num_channels); break; case PcmSampleFormat::Int16: - read_samples_from_stream(stream, read_norm_sample_16, fdata, resampler, num_channels); + read_samples_from_stream(stream, read_norm_sample_16, fdata, num_channels); break; case PcmSampleFormat::Int24: - read_samples_from_stream(stream, read_norm_sample_24, fdata, resampler, num_channels); + read_samples_from_stream(stream, read_norm_sample_24, fdata, num_channels); break; case PcmSampleFormat::Float32: - read_samples_from_stream(stream, read_float_sample_32, fdata, resampler, num_channels); + read_samples_from_stream(stream, read_float_sample_32, fdata, num_channels); break; case PcmSampleFormat::Float64: - read_samples_from_stream(stream, read_float_sample_64, fdata, resampler, num_channels); + read_samples_from_stream(stream, read_float_sample_64, fdata, num_channels); break; default: VERIFY_NOT_REACHED(); @@ -193,6 +187,21 @@ Vector ResampleHelper::resample(Vector to_re template Vector ResampleHelper::resample(Vector); template Vector ResampleHelper::resample(Vector); +NonnullRefPtr resample_buffer(ResampleHelper& resampler, Buffer const& to_resample) +{ + Vector resampled; + resampled.ensure_capacity(to_resample.sample_count() * ceil_div(resampler.source(), resampler.target())); + for (size_t i = 0; i < static_cast(to_resample.sample_count()); ++i) { + auto sample = to_resample.samples()[i]; + resampler.process_sample(sample.left, sample.right); + + while (resampler.read_sample(sample.left, sample.right)) + resampled.append(sample); + } + + return Buffer::create_with_samples(move(resampled)); +} + template void ResampleHelper::process_sample(SampleType sample_l, SampleType sample_r) { diff --git a/Userland/Libraries/LibAudio/Buffer.h b/Userland/Libraries/LibAudio/Buffer.h index 6bf6377b96..b00a533dc3 100644 --- a/Userland/Libraries/LibAudio/Buffer.h +++ b/Userland/Libraries/LibAudio/Buffer.h @@ -105,6 +105,9 @@ public: void reset(); + u32 source() const { return m_source; } + u32 target() const { return m_target; } + private: const u32 m_source; const u32 m_target; @@ -113,11 +116,11 @@ private: SampleType m_last_sample_r; }; -// A buffer of audio samples, normalized to 44100hz. +// A buffer of audio samples. class Buffer : public RefCounted { public: - static RefPtr from_pcm_data(ReadonlyBytes data, ResampleHelper& resampler, int num_channels, PcmSampleFormat sample_format); - static RefPtr from_pcm_stream(InputMemoryStream& stream, ResampleHelper& resampler, int num_channels, PcmSampleFormat sample_format, int num_samples); + static RefPtr from_pcm_data(ReadonlyBytes data, int num_channels, PcmSampleFormat sample_format); + static RefPtr from_pcm_stream(InputMemoryStream& stream, int num_channels, PcmSampleFormat sample_format, int num_samples); static NonnullRefPtr create_with_samples(Vector&& samples) { return adopt_ref(*new Buffer(move(samples))); @@ -157,4 +160,7 @@ private: const int m_sample_count; }; +// This only works for double resamplers, and therefore cannot be part of the class +NonnullRefPtr resample_buffer(ResampleHelper& resampler, Buffer const& to_resample); + } diff --git a/Userland/Libraries/LibAudio/FlacLoader.cpp b/Userland/Libraries/LibAudio/FlacLoader.cpp index 019633c446..1ac3f62cf7 100644 --- a/Userland/Libraries/LibAudio/FlacLoader.cpp +++ b/Userland/Libraries/LibAudio/FlacLoader.cpp @@ -40,8 +40,6 @@ FlacLoaderPlugin::FlacLoaderPlugin(const StringView& path) reset(); if (!m_valid) return; - - m_resampler = make>(m_sample_rate, 44100); } FlacLoaderPlugin::FlacLoaderPlugin(const ByteBuffer& buffer) @@ -58,8 +56,6 @@ FlacLoaderPlugin::FlacLoaderPlugin(const ByteBuffer& buffer) reset(); if (!m_valid) return; - - m_resampler = make>(m_sample_rate, 44100); } bool FlacLoaderPlugin::sniff() @@ -348,8 +344,6 @@ void FlacLoaderPlugin::next_frame() FlacSubframeHeader new_subframe = next_subframe_header(bit_stream, i); CHECK_ERROR_STRING; Vector subframe_samples = parse_subframe(new_subframe, bit_stream); - m_resampler->reset(); - subframe_samples = m_resampler->resample(subframe_samples); CHECK_ERROR_STRING; current_subframes.append(move(subframe_samples)); } diff --git a/Userland/Libraries/LibAudio/FlacLoader.h b/Userland/Libraries/LibAudio/FlacLoader.h index b593804385..915bd44b5c 100644 --- a/Userland/Libraries/LibAudio/FlacLoader.h +++ b/Userland/Libraries/LibAudio/FlacLoader.h @@ -124,7 +124,6 @@ private: bool m_valid { false }; RefPtr m_file; String m_error_string; - OwnPtr> m_resampler; // Data obtained directly from the FLAC metadata: many values have specific bit counts u32 m_sample_rate { 0 }; // 20 bit diff --git a/Userland/Libraries/LibAudio/WavLoader.cpp b/Userland/Libraries/LibAudio/WavLoader.cpp index 2734ae4109..d0e305fe88 100644 --- a/Userland/Libraries/LibAudio/WavLoader.cpp +++ b/Userland/Libraries/LibAudio/WavLoader.cpp @@ -29,8 +29,6 @@ WavLoaderPlugin::WavLoaderPlugin(const StringView& path) valid = parse_header(); if (!valid) return; - - m_resampler = make>(m_sample_rate, m_device_sample_rate); } WavLoaderPlugin::WavLoaderPlugin(const ByteBuffer& buffer) @@ -45,8 +43,6 @@ WavLoaderPlugin::WavLoaderPlugin(const ByteBuffer& buffer) valid = parse_header(); if (!valid) return; - - m_resampler = make>(m_sample_rate, m_device_sample_rate); } RefPtr WavLoaderPlugin::get_more_samples(size_t max_bytes_to_read_from_input) @@ -81,7 +77,6 @@ RefPtr WavLoaderPlugin::get_more_samples(size_t max_bytes_to_read_from_i RefPtr buffer = Buffer::from_pcm_data( sample_data.bytes(), - *m_resampler, m_num_channels, m_sample_format); diff --git a/Userland/Libraries/LibAudio/WavLoader.h b/Userland/Libraries/LibAudio/WavLoader.h index edef9ed1d3..832d4066d7 100644 --- a/Userland/Libraries/LibAudio/WavLoader.h +++ b/Userland/Libraries/LibAudio/WavLoader.h @@ -67,19 +67,11 @@ private: AK::InputMemoryStream* m_memory_stream; String m_error_string; - // TODO: We should probably move resampling into the audio server. - // - // It would avoid duplicate resampling code and would allow clients - // to be agnostic of the destination audio device's sample rate. - OwnPtr> m_resampler; - u32 m_sample_rate { 0 }; u16 m_num_channels { 0 }; PcmSampleFormat m_sample_format; size_t m_byte_offset_of_data_samples { 0 }; - // FIXME: Get this value from the audio server - int m_device_sample_rate { 44100 }; int m_loaded_samples { 0 }; int m_total_samples { 0 }; }; diff --git a/Userland/Utilities/aplay.cpp b/Userland/Utilities/aplay.cpp index 0dd8c93d4d..92b5ea7f6b 100644 --- a/Userland/Utilities/aplay.cpp +++ b/Userland/Utilities/aplay.cpp @@ -35,12 +35,17 @@ int main(int argc, char** argv) loader->bits_per_sample(), loader->num_channels() == 1 ? "Mono" : "Stereo"); out("\033[34;1mProgress\033[0m: \033[s"); + + auto resampler = Audio::ResampleHelper(loader->sample_rate(), audio_client->get_sample_rate()); + for (;;) { auto samples = loader->get_more_samples(); if (samples) { out("\033[u"); out("{}/{}", loader->loaded_samples(), loader->total_samples()); fflush(stdout); + resampler.reset(); + samples = Audio::resample_buffer(resampler, *samples); audio_client->enqueue(*samples); } else if (loader->has_error()) { outln();