324 lines
13 KiB
C++
324 lines
13 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "DSP/MultichannelLinearResampler.h"
|
|
|
|
#include "DSP/BufferVectorOperations.h"
|
|
#include "DSP/Dsp.h"
|
|
#include "DSP/MultichannelBuffer.h"
|
|
#include "HAL/PlatformMath.h"
|
|
#include "Math/UnrealMathUtility.h"
|
|
|
|
namespace Audio
|
|
{
|
|
const float FMultichannelLinearResampler::MaxFrameRatio = 100.f;
|
|
const float FMultichannelLinearResampler::MinFrameRatio = 0.001f;
|
|
|
|
FMultichannelLinearResampler::FMultichannelLinearResampler(int32 InNumChannels)
|
|
: NumChannels(InNumChannels)
|
|
{
|
|
}
|
|
|
|
void FMultichannelLinearResampler::SetFrameRatio(float InRatio, int32 InDesiredNumFramesToInterpolate)
|
|
{
|
|
if (ensureMsgf((InRatio >= MinFrameRatio) && (InRatio <= MaxFrameRatio), TEXT("The frame ratio (%f) must be between %f and %f."), InRatio, MinFrameRatio, MaxFrameRatio))
|
|
{
|
|
if ((InDesiredNumFramesToInterpolate <= 1) || FMath::IsNearlyEqual(InRatio, CurrentFrameRatio))
|
|
{
|
|
// Set frame ratio immediately.
|
|
CurrentFrameRatio = InRatio;
|
|
TargetFrameRatio = InRatio;
|
|
FrameRatioFrameDelta = 0.f;
|
|
NumFramesToInterpolate = 0;
|
|
}
|
|
else
|
|
{
|
|
// Interpolate frame ratio over output frames.
|
|
TargetFrameRatio = InRatio;
|
|
NumFramesToInterpolate = InDesiredNumFramesToInterpolate;
|
|
FrameRatioFrameDelta = (TargetFrameRatio - CurrentFrameRatio) / (NumFramesToInterpolate);
|
|
|
|
// Frame ratio frame deltas which are very small cause numerical
|
|
// stability issues when mapping between input and output frame
|
|
// indices. If the frame delta is below a threshold, we reduce the
|
|
// number of frames to interpolate over until the frame ratio delta
|
|
// is within an acceptable range.
|
|
while ((NumFramesToInterpolate > 1) && (FMath::Abs(FrameRatioFrameDelta) < MinFrameRatioFrameDelta))
|
|
{
|
|
NumFramesToInterpolate /= 2;
|
|
if (NumFramesToInterpolate > 0)
|
|
{
|
|
FrameRatioFrameDelta = (TargetFrameRatio - CurrentFrameRatio) / (NumFramesToInterpolate);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
float FMultichannelLinearResampler::MapOutputFrameToInputFrame(float InOutputFrameIndex) const
|
|
{
|
|
// Sample index mapping requires a bit of math because the frame
|
|
// ratio is linearly interpolated over some number of output samples. These
|
|
// equations are roughly equivalent to physics equations for position,
|
|
// velocity and acceleration, but differ in respect to using summations
|
|
// as opposed to integrals.
|
|
//
|
|
// To derive equations for mapping, do the following:
|
|
// 1. Define the frame ratio as a function of the output frame index
|
|
//
|
|
// v_o is the initial frame ratio.
|
|
// v_t is the target frame ratio.
|
|
// M is the number of output samples for interpolating between v_o and v_t
|
|
// A[t] is the frame ratio delta per frame.
|
|
// V[t] is the frame ratio at the output sample at index t.
|
|
//
|
|
// D = (v_t - v_o) / M
|
|
// A[t] = D for 0 <= t < M
|
|
// A[t] = 0 for t >= M
|
|
// V[t] = v_o + t * A[t]
|
|
//
|
|
// 2. To map from an output index (y) to an input index (x), take the
|
|
// summation of R(t) from 0 to y.
|
|
//
|
|
// Ti is the input index
|
|
// Ti_o is the initial input index offset
|
|
// To is the output index
|
|
//
|
|
// Ti = Sum(V[t] from t=0 to t=To) + Ti_o
|
|
//
|
|
// Ti = To * v_o + (To * (To + 1) / 2) * D for 0 <= To < M
|
|
// Ti = M * v_o + (M * (M + 1) / 2) * D + (To - M) * v_t for To >= M
|
|
//
|
|
// 3. To map from an input index to and output index, use prior equations
|
|
// and solve for To.
|
|
//
|
|
// P = Ti_o + M * v_o + (M * (M + 1) / 2) * D Final input frame where interpolation is occuring.
|
|
// Qa = D / 2 Temp value for solving quadratic equation
|
|
// Qb = v_o + D / 2 Temp value for solving quadratic equation
|
|
// Qc = Ti_o - Ti Temp value for solving quadratic equation
|
|
//
|
|
//
|
|
// To = (-Qb + sqrt(Qb^2 - 4 * Qa * Qc)) / (2 * Qa) for 0 < Ti < P
|
|
// To = (Ti - Ti_o) - M * v_o - (M * (M + 1) / 2) * D + M * v_t) / v_t for Ti >= P
|
|
//
|
|
checkf(InOutputFrameIndex >= 0.f, TEXT("Frame index mapping function is only value for outputs frames greater than or equal to 0"));
|
|
|
|
float InputFrameIndex = 0.f;
|
|
if (NumFramesToInterpolate > 0)
|
|
{
|
|
if (InOutputFrameIndex < NumFramesToInterpolate)
|
|
{
|
|
// Frame ratio interpolation is still occurring at the output frame index.
|
|
const float AccumulationOfFrameDeltas = FrameRatioFrameDelta * (InOutputFrameIndex * (InOutputFrameIndex + 1.f) / 2.f);
|
|
InputFrameIndex = InOutputFrameIndex * CurrentFrameRatio + AccumulationOfFrameDeltas;
|
|
}
|
|
else
|
|
{
|
|
// Frame ratio interpolation occurred, but has reached the target frame ratio by the output frame index.
|
|
const float AccumulationOfFrameDeltas = FrameRatioFrameDelta * (NumFramesToInterpolate * (NumFramesToInterpolate + 1.f) / 2.f);
|
|
InputFrameIndex = NumFramesToInterpolate * CurrentFrameRatio + AccumulationOfFrameDeltas + (InOutputFrameIndex - NumFramesToInterpolate) * TargetFrameRatio;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// No interpolation is happening. The math is quite a bit simpler.
|
|
InputFrameIndex = TargetFrameRatio * InOutputFrameIndex;
|
|
}
|
|
|
|
// Apply current internal offset.
|
|
InputFrameIndex += CurrentInputFrameIndex;
|
|
return InputFrameIndex;
|
|
}
|
|
|
|
float FMultichannelLinearResampler::MapInputFrameToOutputFrame(float InInputFrameIndex) const
|
|
{
|
|
checkf(InInputFrameIndex >= -1.f, TEXT("Frame index mapping function is only value for inputs frames greater than or equal to -1"));
|
|
|
|
// See comments in "MapOutputFrameToInputFrame(..)" for derivation of these formulas.
|
|
|
|
float OutputFrameIndex = 0.f;
|
|
if (NumFramesToInterpolate > 0)
|
|
{
|
|
const float AccumulationOfFrameDeltas = (NumFramesToInterpolate * (NumFramesToInterpolate + 1.f) / 2.f) * FrameRatioFrameDelta;
|
|
const float NumInputFramesToInterpolate = CurrentInputFrameIndex + NumFramesToInterpolate * CurrentFrameRatio + AccumulationOfFrameDeltas;
|
|
|
|
if (InInputFrameIndex < NumInputFramesToInterpolate)
|
|
{
|
|
// Use double when solving for quadratic to handle numerical stability issues.
|
|
const double QuadA = FrameRatioFrameDelta / 2.;
|
|
const double QuadB = CurrentFrameRatio + FrameRatioFrameDelta / 2.;
|
|
const double QuadC = CurrentInputFrameIndex - InInputFrameIndex;
|
|
OutputFrameIndex = static_cast<float>((-QuadB + FMath::Sqrt(QuadB * QuadB - 4. * QuadA * QuadC)) / (2. * QuadA));
|
|
}
|
|
else
|
|
{
|
|
OutputFrameIndex = InInputFrameIndex - CurrentInputFrameIndex - NumFramesToInterpolate * CurrentFrameRatio - AccumulationOfFrameDeltas + NumFramesToInterpolate * TargetFrameRatio;
|
|
OutputFrameIndex /= TargetFrameRatio;
|
|
}
|
|
}
|
|
else if (TargetFrameRatio > 0.f)
|
|
{
|
|
OutputFrameIndex = (InInputFrameIndex - CurrentInputFrameIndex) / TargetFrameRatio;
|
|
}
|
|
|
|
return OutputFrameIndex;
|
|
}
|
|
|
|
int32 FMultichannelLinearResampler::GetNumInputFramesNeededToProduceOutputFrames(int32 InNumOutputFrames) const
|
|
{
|
|
if (InNumOutputFrames > 0)
|
|
{
|
|
const int32 NumBufferFrames = GetNumBufferFramesToProduceOutputFrames(InNumOutputFrames);
|
|
return FMath::CeilToInt(MapOutputFrameToInputFrame(InNumOutputFrames - 1)) + NumBufferFrames;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int32 FMultichannelLinearResampler::GetNumBufferFramesToProduceOutputFrames(int32 InNumOutputFrames) const
|
|
{
|
|
// buffer frames to deal with numerical accuracy issues when calculating
|
|
// large number of output frames.
|
|
check(InNumOutputFrames > 0);
|
|
return FMath::Max(1, InNumOutputFrames / 100) + 1;
|
|
}
|
|
|
|
template<typename OutputMultichannelBufferType>
|
|
int32 FMultichannelLinearResampler::ProcessAndConsumeAudioInternal(FMultichannelCircularBuffer& InAudio, OutputMultichannelBufferType& OutAudio)
|
|
{
|
|
checkf(InAudio.Num() == OutAudio.Num(), TEXT("Input/output channel count mismatch."));
|
|
checkf(InAudio.Num() == NumChannels, TEXT("Incorrect audio channel count."));
|
|
|
|
int32 NumOutputFrames = GetMultichannelBufferNumFrames(OutAudio);
|
|
const int32 NumAvailableInputFrames = GetMultichannelBufferNumFrames(InAudio);
|
|
int32 NumInputFramesRequired = GetNumInputFramesNeededToProduceOutputFrames(NumOutputFrames);
|
|
|
|
if (NumAvailableInputFrames < NumInputFramesRequired)
|
|
{
|
|
// Update number of frames to generate based on available number of samples
|
|
// When FrameRatioFrameDelta is small (1e-5 or less), MapInputFrameToOutputFrame(...)
|
|
// becomes prone to numerical precision issues.
|
|
// To avoid errors due to precision issues, use the maximum frame rate
|
|
// to determine the number of output frames which can be safely generated
|
|
// from the given number of input frames. .
|
|
|
|
const int32 NumBufferFrames = GetNumBufferFramesToProduceOutputFrames(NumOutputFrames);
|
|
NumOutputFrames = FMath::FloorToInt((NumAvailableInputFrames - NumBufferFrames) / FMath::Max(CurrentFrameRatio, TargetFrameRatio)) - 1;
|
|
NumOutputFrames = FMath::Max(NumOutputFrames, 0);
|
|
NumInputFramesRequired = NumAvailableInputFrames;
|
|
checkf(NumInputFramesRequired > FMath::CeilToInt(MapOutputFrameToInputFrame(NumOutputFrames - 1)), TEXT("Invalid calculation. Required input frames (%d) does not satisfy need for input frames (%f)"), NumInputFramesRequired, MapOutputFrameToInputFrame(NumOutputFrames - 1));
|
|
}
|
|
|
|
if (NumOutputFrames > 0)
|
|
{
|
|
float FinalInputFrameIndex = 0.f;
|
|
int32 NumFramesToPop = 0;
|
|
|
|
// Copy input buffers into work buffer since circular buffers
|
|
// are not ensured to hold entire array contiguously.
|
|
WorkBuffer.SetNumUninitialized(NumInputFramesRequired, EAllowShrinking::No);
|
|
|
|
for (int32 ChannelIndex = 0; ChannelIndex < NumChannels; ChannelIndex++)
|
|
{
|
|
const int32 NumFramesCopied = InAudio[ChannelIndex].Peek(WorkBuffer.GetData(), NumInputFramesRequired);
|
|
check(NumFramesCopied == NumInputFramesRequired);
|
|
|
|
FinalInputFrameIndex = ProcessChannelAudioInternal(WorkBuffer, TArrayView<float>(OutAudio[ChannelIndex].GetData(), NumOutputFrames));
|
|
NumFramesToPop = FMath::Floor(FinalInputFrameIndex);
|
|
|
|
// Remove unneeded input audio.
|
|
InAudio[ChannelIndex].Pop(NumFramesToPop);
|
|
}
|
|
|
|
// Update current frame ratio
|
|
if (NumFramesToInterpolate > 0)
|
|
{
|
|
if (NumFramesToInterpolate > NumOutputFrames)
|
|
{
|
|
CurrentFrameRatio += NumOutputFrames * FrameRatioFrameDelta;
|
|
NumFramesToInterpolate -= NumOutputFrames;
|
|
}
|
|
else
|
|
{
|
|
NumFramesToInterpolate = 0;
|
|
CurrentFrameRatio = TargetFrameRatio;
|
|
}
|
|
}
|
|
|
|
// Increment frame counters.
|
|
CurrentInputFrameIndex = FinalInputFrameIndex - NumFramesToPop;
|
|
NumFramesToInterpolate -= FMath::Min(NumFramesToInterpolate, NumOutputFrames);
|
|
}
|
|
|
|
return NumOutputFrames;
|
|
}
|
|
|
|
int32 FMultichannelLinearResampler::ProcessAndConsumeAudio(FMultichannelCircularBuffer& InAudio, FMultichannelBuffer& OutAudio)
|
|
{
|
|
return ProcessAndConsumeAudioInternal(InAudio, OutAudio);
|
|
}
|
|
|
|
int32 FMultichannelLinearResampler::ProcessAndConsumeAudio(FMultichannelCircularBuffer& InAudio, TArray<TArrayView<float>>& OutAudio)
|
|
{
|
|
return ProcessAndConsumeAudioInternal(InAudio, OutAudio);
|
|
}
|
|
|
|
float FMultichannelLinearResampler::ProcessChannelAudioInternal(TArrayView<const float> InAudio, TArrayView<float> OutAudio)
|
|
{
|
|
const int32 NumOutputFrames = OutAudio.Num();
|
|
|
|
if (NumOutputFrames < 1)
|
|
{
|
|
return 0.f;
|
|
}
|
|
|
|
float* OutAudioData = OutAudio.GetData();
|
|
const float* InAudioData = InAudio.GetData();
|
|
|
|
if ((CurrentFrameRatio == 1.f) && (CurrentInputFrameIndex == 0.f) && (0 == NumFramesToInterpolate))
|
|
{
|
|
// No interpolation is needed. Samples can be copied directly.
|
|
FMemory::Memcpy(OutAudioData, InAudioData, NumOutputFrames * sizeof(float));
|
|
return NumOutputFrames;
|
|
}
|
|
else
|
|
{
|
|
float InputFrameRatio = CurrentFrameRatio;
|
|
float InputFrameIndex = CurrentInputFrameIndex;
|
|
int32 LowerFrameIndex = 0;
|
|
|
|
checkf(InputFrameIndex >= 0.f, TEXT("Input frame index references discarded data"));
|
|
|
|
// Handle frames where samples are interpolated.
|
|
int32 NumFramesComputed = FMath::Min(NumOutputFrames, NumFramesToInterpolate);// +1);
|
|
for (int32 OutputFrameIndex = 0; OutputFrameIndex < NumFramesComputed; OutputFrameIndex++)
|
|
{
|
|
LowerFrameIndex = (int32)InputFrameIndex;
|
|
float Alpha = InputFrameIndex - LowerFrameIndex;
|
|
OutAudioData[OutputFrameIndex] = FMath::Lerp(InAudioData[LowerFrameIndex], InAudioData[LowerFrameIndex + 1], Alpha);
|
|
|
|
InputFrameRatio += FrameRatioFrameDelta;
|
|
InputFrameIndex += InputFrameRatio;
|
|
}
|
|
|
|
// Handle frames where sample rate is constant.
|
|
if (NumFramesComputed < NumOutputFrames)
|
|
{
|
|
InputFrameRatio = TargetFrameRatio;
|
|
for (int32 OutputFrameIndex = NumFramesComputed; OutputFrameIndex < NumOutputFrames; OutputFrameIndex++)
|
|
{
|
|
LowerFrameIndex = (int32)InputFrameIndex;
|
|
float Alpha = InputFrameIndex - LowerFrameIndex;
|
|
OutAudioData[OutputFrameIndex] = FMath::Lerp(InAudioData[LowerFrameIndex], InAudioData[LowerFrameIndex + 1], Alpha);
|
|
InputFrameIndex += InputFrameRatio;
|
|
|
|
}
|
|
}
|
|
|
|
// Check for buffer over run
|
|
checkf((LowerFrameIndex + 1) < InAudio.Num(), TEXT("Buffer overrun in multichannel linear resampler. Attempt to read index %d of array with %d elements. FrameRatio: %f, FrameRatioDelta: %f, NumFramesToInterpolate: %d"), LowerFrameIndex + 1, InAudio.Num(), CurrentFrameRatio, FrameRatioFrameDelta, NumFramesToInterpolate);
|
|
|
|
return InputFrameIndex;
|
|
}
|
|
}
|
|
}
|
|
|