From f94c55437bb9e3ef8aebc044ce291f38a8a68e32 Mon Sep 17 00:00:00 2001 From: "andrew.walker" Date: Wed, 13 Apr 2022 16:13:50 +0100 Subject: [PATCH] Updated to fix some off by 1 errors which was causing major drifts in of itself. --- src/audio/types.cpp | 95 +++++++++++++++++++--------------- src/audio/types.h | 2 + src/rtp/audio_decoders.cpp | 9 ++-- src/video_display/decklink.cpp | 33 +++++------- 4 files changed, 72 insertions(+), 67 deletions(-) diff --git a/src/audio/types.cpp b/src/audio/types.cpp index 13a20d89b..7d28a1d4a 100644 --- a/src/audio/types.cpp +++ b/src/audio/types.cpp @@ -178,6 +178,52 @@ int audio_frame2_resampler::get_resampler_initial_bps() { return this->resample_initial_bps; } +ADD_TO_PARAM("resampler-quality", "* resampler-quality=[0-10]\n" + " Sets audio resampler quality in range 0 (worst) and 10 (best), default " TOSTRING(DEFAULT_RESAMPLE_QUALITY) "\n"); + +bool audio_frame2_resampler::create_resampler(uint32_t original_sample_rate, uint32_t new_sample_rate_num, uint32_t new_sample_rate_den, size_t channel_size, int bps) { +#ifdef HAVE_SPEEXDSP + LOG(LOG_LEVEL_VERBOSE) << "Destroying Resampler\n"; + if (this->resampler) { + speex_resampler_destroy((SpeexResamplerState *) this->resampler); + this->destroy_resampler = false; + } + this->resampler = nullptr; + + int quality = DEFAULT_RESAMPLE_QUALITY; + if (commandline_params.find("resampler-quality") != commandline_params.end()) { + quality = stoi(commandline_params.at("resampler-quality")); + assert(quality >= 0 && quality <= 10); + } + int err = 0; + this->resampler = speex_resampler_init_frac(channel_size, original_sample_rate * new_sample_rate_den, new_sample_rate_num, + original_sample_rate, new_sample_rate_num / new_sample_rate_den, quality, &err); + if (err) { + LOG(LOG_LEVEL_ERROR) << "[audio_frame2_resampler] Cannot initialize resampler: " << speex_resampler_strerror(err) << "\n"; + return false; + } + // Ignore resampler delay. The speex resampler silently adds a delay to the resampler by adding silence at the length + // of the input latency and stored a buffered amount for itself. This is extracted outside of this function on the final + // call before a resampler is marked for destruction. + speex_resampler_skip_zeros((SpeexResamplerState *) this->resampler); + this->resample_from = original_sample_rate; + + // Setup resampler values + this->resample_to_num = new_sample_rate_num; + this->resample_to_den = new_sample_rate_den; + this->resample_ch_count = channel_size; + // Capture the input and output latency. Generally, there is not a difference between the two. + // The input latency is used to calculate leftover audio in the resampler that is collected on the + // audio frame before the resampler is destroyed. + this->resample_input_latency = speex_resampler_get_input_latency((SpeexResamplerState *) this->resampler); + this->resample_output_latency = speex_resampler_get_output_latency((SpeexResamplerState *) this->resampler); + this->resample_initial_bps = bps; + LOG(LOG_LEVEL_ERROR) << "[audio_frame2] Resampler (re)made at " << new_sample_rate_num / new_sample_rate_den << "\n"; + return true; +#endif + return false; +} + /** * @brief Creates empty audio_frame2 */ @@ -500,7 +546,7 @@ void audio_frame2::resample_channel(audio_frame2_resampler* resampler_state, int remainder->append(channel_index, (char *)(in + (in_len * sizeof(int16_t))), in_len_orig - in_len); } // The speex resampler process returns the number of frames written + 1 (so ensure we subtract 1 when setting the length) - new_channel->len = (out_len - 1) * sizeof(int16_t); + new_channel->len = out_len * sizeof(int16_t); #else LOG(LOG_LEVEL_ERROR) << "Audio frame resampler: cannot resample, SpeexDSP was not compiled in!\n"; #endif @@ -530,15 +576,12 @@ void audio_frame2::resample_channel_float(audio_frame2_resampler* resampler_stat remainder->append(channel_index, (char *)(in + (in_len * sizeof(float))), in_len_orig - in_len); } // The speex resampler process returns the number of frames written + 1 (so ensure we subtract 1 when setting the length) - new_channel->len = (out_len - 1) * sizeof(float); + new_channel->len = out_len * sizeof(float); #else LOG(LOG_LEVEL_ERROR) << "Audio frame resampler: cannot resample, SpeexDSP was not compiled in!\n"; #endif } -ADD_TO_PARAM("resampler-quality", "* resampler-quality=[0-10]\n" - " Sets audio resampler quality in range 0 (worst) and 10 (best), default " TOSTRING(DEFAULT_RESAMPLE_QUALITY) "\n"); - tuple audio_frame2::resample_fake([[maybe_unused]] audio_frame2_resampler & resampler_state, int new_sample_rate_num, int new_sample_rate_den) { if (new_sample_rate_num / new_sample_rate_den == sample_rate && new_sample_rate_num % new_sample_rate_den == 0) { @@ -562,43 +605,11 @@ tuple audio_frame2::resample_fake([[maybe_unused]] aud || new_sample_rate_num != resampler_state.resample_to_num || new_sample_rate_den != resampler_state.resample_to_den || channels.size() != resampler_state.resample_ch_count) || resampler_state.resample_initial_bps != this->bps || resampler_state.destroy_resampler) { - if (resampler_state.resampler) { - speex_resampler_destroy((SpeexResamplerState *) resampler_state.resampler); - resampler_state.destroy_resampler = false; + reinitialised_resampler = resampler_state.create_resampler(this->sample_rate, new_sample_rate_num, new_sample_rate_den, this->channels.size(), this->bps); + if(!reinitialised_resampler) { + return {false, false, audio_frame2{}}; } - resampler_state.resampler = nullptr; - - int quality = DEFAULT_RESAMPLE_QUALITY; - if (commandline_params.find("resampler-quality") != commandline_params.end()) { - quality = stoi(commandline_params.at("resampler-quality")); - assert(quality >= 0 && quality <= 10); - } - int err = 0; - resampler_state.resampler = speex_resampler_init_frac(channels.size(), sample_rate * new_sample_rate_den, new_sample_rate_num, - sample_rate, new_sample_rate_num / new_sample_rate_den, quality, &err); - if (err) { - LOG(LOG_LEVEL_ERROR) << "[audio_frame2] Cannot initialize resampler: " << speex_resampler_strerror(err) << "\n"; - return {false, reinitialised_resampler, audio_frame2{}}; - } - // Ignore resampler delay. The speex resampler silently adds a delay to the resampler by adding silence at the length - // of the input latency and stored a buffered amount for itself. This is extracted outside of this function on the final - // call before a resampler is marked for destruction. - speex_resampler_skip_zeros((SpeexResamplerState *) resampler_state.resampler); - resampler_state.resample_from = sample_rate; - - // Setup resampler values - resampler_state.resample_to_num = new_sample_rate_num; - resampler_state.resample_to_den = new_sample_rate_den; - resampler_state.resample_ch_count = channels.size(); - // Capture the input and output latency. Generally, there is not a difference between the two. - // The input latency is used to calculate leftover audio in the resampler that is collected on the - // audio frame before the resampler is destroyed. - resampler_state.resample_input_latency = speex_resampler_get_input_latency((SpeexResamplerState *) resampler_state.resampler); - resampler_state.resample_output_latency = speex_resampler_get_output_latency((SpeexResamplerState *) resampler_state.resampler); - - resampler_state.resample_initial_bps = this->bps; - - reinitialised_resampler = true; + LOG(LOG_LEVEL_ERROR) << "[audio_frame2] Resampler (re)made at " << new_sample_rate_num / new_sample_rate_den << "\n"; } // Initialise the new channels that the resampler is going to write into @@ -643,7 +654,7 @@ tuple audio_frame2::resample_fake([[maybe_unused]] aud std::chrono::high_resolution_clock::time_point resample_end = std::chrono::high_resolution_clock::now(); auto time_diff = std::chrono::duration_cast>(resample_end - resample_begin); - LOG(LOG_LEVEL_DEBUG) << " CALL LENGTH RESAMPLER " << setprecision(30) << time_diff.count() << "\n"; + LOG(LOG_LEVEL_DEBUG) << "CALL LENGTH RESAMPLER " << setprecision(30) << time_diff.count() << "\n"; return {true, reinitialised_resampler, std::move(remainder)}; #else diff --git a/src/audio/types.h b/src/audio/types.h index d8f52e584..f0f7388fa 100644 --- a/src/audio/types.h +++ b/src/audio/types.h @@ -130,6 +130,7 @@ public: int get_resampler_initial_bps(); size_t get_resampler_channel_count(); bool resampler_is_set(); + bool create_resampler(uint32_t original_sample_rate, uint32_t new_sample_rate_num, uint32_t new_sample_rate_den, size_t channel_size, int bps); void resample_set_destroy_flag(bool destroy); private: void *resampler{nullptr}; // type is (SpeexResamplerState *) @@ -141,6 +142,7 @@ private: int resample_initial_bps{0}; size_t resample_ch_count{0}; bool destroy_resampler{false}; + bool initial{}; friend class audio_frame2; }; diff --git a/src/rtp/audio_decoders.cpp b/src/rtp/audio_decoders.cpp index 066f91860..f902d8c14 100644 --- a/src/rtp/audio_decoders.cpp +++ b/src/rtp/audio_decoders.cpp @@ -794,16 +794,17 @@ int decode_audio_frame(struct coded_data *cdata, void *pbuf_data, struct pbuf_st || resample_denominator != decoder->resampler.get_resampler_denominator() || (size_t)decompressed.get_channel_count() != decoder->resampler.get_resampler_channel_count() || decoder->resampler.get_resampler_initial_bps() != decompressed.get_bps())) { - LOG(LOG_LEVEL_DEBUG) << MOD_NAME << " REMOVE removing tail buffer\n"; + LOG(LOG_LEVEL_VERBOSE) << MOD_NAME << " REMOVE removing tail buffer\n"; // The resampler is no longer required. Collect the remaining buffer from the resampler audio_frame2 tail_buffer = audio_frame2(); tail_buffer.init(decompressed.get_channel_count(), decompressed.get_codec(), decoder->resampler.get_resampler_initial_bps(), decompressed.get_sample_rate()); // Generate a buffer the size of the input latency and apply it to all channels - char buffer[decoder->resampler.get_resampler_input_latency() * decoder->resampler.get_resampler_initial_bps()]; - memset(buffer, 0, decoder->resampler.get_resampler_input_latency() * decoder->resampler.get_resampler_initial_bps()); + uint32_t leftover_frames = (decoder->resampler.get_resampler_input_latency() - 1); + char buffer[leftover_frames * decoder->resampler.get_resampler_initial_bps()]; + memset(buffer, 0, leftover_frames * decoder->resampler.get_resampler_initial_bps()); for(size_t i = 0; i < (size_t)tail_buffer.get_channel_count(); i++) { - tail_buffer.append(i, buffer, decoder->resampler.get_resampler_input_latency() * decoder->resampler.get_resampler_initial_bps()); + tail_buffer.append(i, buffer, leftover_frames * decoder->resampler.get_resampler_initial_bps()); } // Extract remaining buffer from resampler by applying a resample the size of the input latency tail_buffer.resample_fake(decoder->resampler, decoder->resampler.get_resampler_numerator(), decoder->resampler.get_resampler_denominator()); diff --git a/src/video_display/decklink.cpp b/src/video_display/decklink.cpp index b1a53cf1c..ff9f9ec57 100644 --- a/src/video_display/decklink.cpp +++ b/src/video_display/decklink.cpp @@ -84,7 +84,7 @@ #endif #define MAX_RESAMPLE_DELTA_DEFAULT 30 -#define MIN_RESAMPLE_DELTA_DEFAULT 5 +#define MIN_RESAMPLE_DELTA_DEFAULT 1 #define TARGET_BUFFER_DEFAULT 2700 static void print_output_modes(IDeckLink *); @@ -403,8 +403,7 @@ public: */ void report() { std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now(); - if(std::chrono::duration_cast(now - this->last_summary).count() > 10) { - + if(std::chrono::duration_cast(now - this->last_summary).count() > 10) { LOG(LOG_LEVEL_INFO) << rang::style::underline << "Decklink stats (cumulative)" << rang::style::reset << " - Total Audio Frames Played: " << rang::style::bold << this->frames_played @@ -421,7 +420,7 @@ public: << rang::style::reset << " / Average Buffer: " << rang::style::bold << this->buffer_average << rang::style::reset << " / Average Added Frames: " - << rang::style::bold << (uint32_t) round(this->avg_added_frames.avg()) + << rang::style::bold << this->avg_added_frames.avg() << rang::style::reset << " / Max time diff audio (ms): " << rang::style::bold << this->audio_time_diff_max << rang::style::reset << " / Min time diff audio (ms): " @@ -540,7 +539,7 @@ private: uint32_t frames_played = 0; // How many times the buffer dropped avg amount of frames being added uint32_t frames_missed = 0; - MovingAverage avg_added_frames{10}; + MovingAverage avg_added_frames{250}; // How many buffer underflows and overflows have occured. uint32_t buffer_underflow = 0; uint32_t buffer_overflow = 0; @@ -664,13 +663,15 @@ public: // Add the amount currently in the buffer to the moving average, and calculate the delta between that and the previous amount // Store the previous buffer count so we can calculate this next frame. this->average_buffer_samples.add((double)buffered_count); - this->average_delta.add((double)buffered_count - previous_buffer); + this->average_delta.add((double)abs((int32_t)buffered_count - (int32_t)previous_buffer)); this->previous_buffer = buffered_count; long long dst_frame_rate = 0; // Calculate the average uint32_t average_buffer_depth = (uint32_t)(this->average_buffer_samples.avg()); + int resample_hz = 0; + // Check to see if our buffered samples has enough to calculate a good average if (this->average_buffer_samples.filled()) { // @todo might be worth trying to make this more dynamic so that certain input values @@ -687,30 +688,20 @@ public: if (average_buffer_depth > target_buffer_fill + this->pos_jitter) { // The buffer is too large, so we need to resample down to remove some frames - int resample_hz = (int)this->scale_buffer_delta(average_buffer_depth - target_buffer_fill + this->pos_jitter); + resample_hz = (int)this->scale_buffer_delta(average_buffer_depth - target_buffer_fill - this->pos_jitter); dst_frame_rate = (bmdAudioSampleRate48kHz - resample_hz) * BASE; - LOG(LOG_LEVEL_VERBOSE) << MOD_NAME << " UPDATE playing speed slow " << average_buffer_depth << " vs " << buffered_count << " " << average_delta.getTotal() << " average_velocity " << resample_hz << " resample_hz\n"; this->audio_summary->increment_resample_low(); } else if(average_buffer_depth < target_buffer_fill - this->neg_jitter) { // The buffer is too small, so we need to resample up to generate some additional frames - int resample_hz = (int)this->scale_buffer_delta(average_buffer_depth - target_buffer_fill - this->neg_jitter); + resample_hz = (int)this->scale_buffer_delta(target_buffer_fill - average_buffer_depth - this->neg_jitter); dst_frame_rate = (bmdAudioSampleRate48kHz + resample_hz) * BASE; - LOG(LOG_LEVEL_VERBOSE) << MOD_NAME << " UPDATE playing speed fast " << average_buffer_depth << " vs " << buffered_count << " " << average_delta.getTotal() << " average_velocity " << resample_hz << " resample_hz\n"; this->audio_summary->increment_resample_high(); } else { - // If there is nothing to do, then set the resample rate to trend towards our target, but at a very - // low sample rate delta. - if(average_buffer_depth > target_buffer_fill) { - dst_frame_rate = (bmdAudioSampleRate48kHz - 1) * BASE; - } - else { - dst_frame_rate = (bmdAudioSampleRate48kHz + 1) * BASE; - } - LOG(LOG_LEVEL_VERBOSE) << MOD_NAME << " UPDATE playing speed normal " << average_buffer_depth << " vs " << buffered_count << " " << average_delta.getTotal() << " average_velocity 1 resample_hz\n"; + dst_frame_rate = (bmdAudioSampleRate48kHz) * BASE; } } - LOG(LOG_LEVEL_DEBUG) << MOD_NAME << " UPDATE2 " << average_buffer_depth << " vs " << buffered_count << " " << dst_frame_rate << " dst_frame_rate "<<"\n"; + LOG(LOG_LEVEL_VERBOSE) << MOD_NAME << " UPDATE playing speed " << average_buffer_depth << " vs " << buffered_count << " " << average_delta.avg() << " average_velocity " << resample_hz << " resample_hz\n"; if (dst_frame_rate != 0) { @@ -807,7 +798,7 @@ struct state_decklink { mutex reconfiguration_lock; ///< for audio and video reconf to be mutually exclusive - AudioDriftFixer audio_drift_fixer{250, 150, 2700, 600, 600}; + AudioDriftFixer audio_drift_fixer{250, 25, 2700, 50, 50}; uint32_t last_buffered_samples; int32_t drift_since_last_correction;