413 lines
13 KiB
C++
413 lines
13 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
|
|
#include "Decoders/OpusAudioInfo.h"
|
|
#include "Interfaces/IAudioFormat.h"
|
|
#include <opus_defines.h>
|
|
#include <opus_types.h>
|
|
|
|
THIRD_PARTY_INCLUDES_START
|
|
#include "opus_multistream.h"
|
|
THIRD_PARTY_INCLUDES_END
|
|
|
|
#define USE_UE_MEM_ALLOC 1
|
|
|
|
DEFINE_LOG_CATEGORY_STATIC(LogOpusAudioDecoder, Log, All);
|
|
|
|
///////////////////////////////////////////////////////////////////////////////////////
|
|
// Followed pattern used in opus_multistream_encoder.c - this will allow us to setup //
|
|
// a multistream decoder without having to save extra information for every asset. //
|
|
///////////////////////////////////////////////////////////////////////////////////////
|
|
struct UnrealChannelLayout{
|
|
int32 NumStreams;
|
|
int32 NumCoupledStreams;
|
|
uint8 Mapping[8];
|
|
};
|
|
|
|
/* Index is NumChannels-1*/
|
|
static const UnrealChannelLayout UnrealMappings[8] = {
|
|
{ 1, 0, { 0 } }, /* 1: mono */
|
|
{ 1, 1, { 0, 1 } }, /* 2: stereo */
|
|
{ 2, 1, { 0, 1, 2 } }, /* 3: 1-d surround */
|
|
{ 2, 2, { 0, 1, 2, 3 } }, /* 4: quadraphonic surround */
|
|
{ 3, 2, { 0, 1, 4, 2, 3 } }, /* 5: 5-channel surround */
|
|
{ 4, 2, { 0, 1, 4, 5, 2, 3 } }, /* 6: 5.1 surround */
|
|
{ 4, 3, { 0, 1, 4, 6, 2, 3, 5 } }, /* 7: 6.1 surround */
|
|
{ 5, 3, { 0, 1, 6, 7, 2, 3, 4, 5 } }, /* 8: 7.1 surround */
|
|
};
|
|
|
|
/*------------------------------------------------------------------------------------
|
|
FOpusDecoderWrapper
|
|
------------------------------------------------------------------------------------*/
|
|
struct FOpusDecoderWrapper
|
|
{
|
|
FOpusDecoderWrapper(uint32 SampleRate, uint8 NumChannels)
|
|
{
|
|
check(NumChannels <= 8);
|
|
const UnrealChannelLayout& Layout = UnrealMappings[NumChannels-1];
|
|
#if USE_UE_MEM_ALLOC
|
|
int32 DecSize = opus_multistream_decoder_get_size(Layout.NumStreams, Layout.NumCoupledStreams);
|
|
Decoder = (OpusMSDecoder*)FMemory::Malloc(DecSize);
|
|
DecError = opus_multistream_decoder_init(Decoder, SampleRate, NumChannels, Layout.NumStreams, Layout.NumCoupledStreams, Layout.Mapping);
|
|
#else
|
|
Decoder = opus_multistream_decoder_create(SampleRate, NumChannels, Layout.NumStreams, Layout.NumCoupledStreams, Layout.Mapping, &DecError);
|
|
#endif
|
|
}
|
|
|
|
~FOpusDecoderWrapper()
|
|
{
|
|
#if USE_UE_MEM_ALLOC
|
|
FMemory::Free(Decoder);
|
|
#else
|
|
opus_multistream_decoder_destroy(Decoder);
|
|
#endif
|
|
}
|
|
|
|
int32 Decode(const uint8* FrameData, uint16 FrameSize, int16* OutPCMData, int32 SampleSize)
|
|
{
|
|
return opus_multistream_decode(Decoder, FrameData, FrameSize, OutPCMData, SampleSize, 0);
|
|
}
|
|
|
|
bool WasInitialisedSuccessfully() const
|
|
{
|
|
return DecError == OPUS_OK;
|
|
}
|
|
|
|
private:
|
|
OpusMSDecoder* Decoder;
|
|
int32 DecError;
|
|
};
|
|
|
|
/*------------------------------------------------------------------------------------
|
|
FOpusAudioInfo.
|
|
------------------------------------------------------------------------------------*/
|
|
FOpusAudioInfo::FOpusAudioInfo()
|
|
: OpusDecoderWrapper(nullptr)
|
|
{
|
|
}
|
|
|
|
FOpusAudioInfo::~FOpusAudioInfo()
|
|
{
|
|
if (OpusDecoderWrapper != nullptr)
|
|
{
|
|
delete OpusDecoderWrapper;
|
|
OpusDecoderWrapper = nullptr;
|
|
}
|
|
}
|
|
|
|
|
|
bool FOpusAudioInfo::ParseHeader(FHeader& OutHeader, uint32& OutNumRead, const uint8* InSrcBufferData, uint32 InSrcBufferDataSize)
|
|
{
|
|
OutNumRead = 0;
|
|
|
|
if ((int32)InSrcBufferDataSize < FHeader::HeaderSize())
|
|
{
|
|
return false;
|
|
}
|
|
|
|
auto Read = [&InSrcBufferData, &OutNumRead](void* To, int32 NumBytes) -> void
|
|
{
|
|
FMemory::Memcpy(To, InSrcBufferData, NumBytes);
|
|
InSrcBufferData += NumBytes;
|
|
OutNumRead += NumBytes;
|
|
};
|
|
|
|
Read(OutHeader.Identifier, 8);
|
|
if (FMemory::Memcmp(OutHeader.Identifier, FHeader::OPUS_ID, 8))
|
|
{
|
|
return false;
|
|
}
|
|
Read(&OutHeader.Version, sizeof(uint8));
|
|
Read(&OutHeader.NumChannels, sizeof(uint8));
|
|
Read(&OutHeader.SampleRate, sizeof(uint32));
|
|
Read(&OutHeader.EncodedSampleRate, sizeof(uint32));
|
|
Read(&OutHeader.ActiveSampleCount, sizeof(uint64));
|
|
Read(&OutHeader.NumEncodedFrames, sizeof(uint32));
|
|
Read(&OutHeader.NumPreSkipSamples, sizeof(int32));
|
|
Read(&OutHeader.NumSilentSamplesAtBeginning, sizeof(int32));
|
|
Read(&OutHeader.NumSilentSamplesAtEnd, sizeof(int32));
|
|
return true;
|
|
}
|
|
|
|
bool FOpusAudioInfo::ParseHeader(const uint8* InSrcBufferData, uint32 InSrcBufferDataSize, struct FSoundQualityInfo* QualityInfo)
|
|
{
|
|
SrcBufferData = InSrcBufferData;
|
|
SrcBufferDataSize = InSrcBufferDataSize;
|
|
SrcBufferOffset = 0;
|
|
CurrentSampleCount = 0;
|
|
|
|
Header.Reset();
|
|
if (!ParseHeader(Header, SrcBufferOffset, InSrcBufferData, InSrcBufferDataSize))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// Store the offset to where the audio data begins
|
|
AudioDataOffset = SrcBufferOffset;
|
|
// Sample counts in the Opus API are always per-channel so we multiply by NumChannels here
|
|
TrueSampleCount = (uint32)Header.ActiveSampleCount * Header.NumChannels;
|
|
NumChannels = Header.NumChannels;
|
|
|
|
// Write out the the header info
|
|
if (QualityInfo)
|
|
{
|
|
QualityInfo->SampleRate = Header.SampleRate;
|
|
QualityInfo->NumChannels = Header.NumChannels;
|
|
QualityInfo->SampleDataSize = (uint32)Header.ActiveSampleCount * QualityInfo->NumChannels * sizeof(int16);
|
|
QualityInfo->Duration = (float)((double)Header.ActiveSampleCount / QualityInfo->SampleRate);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
bool FOpusAudioInfo::CreateDecoder()
|
|
{
|
|
check(OpusDecoderWrapper == nullptr);
|
|
OpusDecoderWrapper = new FOpusDecoderWrapper(Header.EncodedSampleRate, NumChannels);
|
|
if (!OpusDecoderWrapper->WasInitialisedSuccessfully())
|
|
{
|
|
delete OpusDecoderWrapper;
|
|
OpusDecoderWrapper = nullptr;
|
|
return false;
|
|
}
|
|
|
|
NumRemainingSamplesToSkip = Header.NumSilentSamplesAtBeginning + Header.NumPreSkipSamples;
|
|
PreviousDecodedUnusedSamples.Empty();
|
|
return true;
|
|
}
|
|
|
|
int32 FOpusAudioInfo::GetFrameSize()
|
|
{
|
|
// Opus format has variable frame size at the head of each frame...
|
|
// We have to assume that the SrcBufferOffset is at the correct location for the read
|
|
uint16 FrameSize = 0;
|
|
Read(&FrameSize, sizeof(uint16));
|
|
return (int32)FrameSize;
|
|
}
|
|
|
|
uint32 FOpusAudioInfo::GetMaxFrameSizeSamples() const
|
|
{
|
|
// The encoder is using 20ms frame sizes.
|
|
const int32 OPUS_MAX_FRAME_SIZE_MS = 20;
|
|
// There can be at most 2 frames in one packet, so multiply by 2.
|
|
return Header.EncodedSampleRate * OPUS_MAX_FRAME_SIZE_MS * 2 / 1000;
|
|
}
|
|
|
|
FDecodeResult FOpusAudioInfo::Decode(const uint8* CompressedData, const int32 CompressedDataSize, uint8* OutPCMData, const int32 OutputPCMDataSize)
|
|
{
|
|
FDecodeResult Result;
|
|
|
|
if (OpusDecoderWrapper)
|
|
{
|
|
const int32 kFrameBytes = NumChannels * sizeof(int16);
|
|
int32 OutputSizeToGo = OutputPCMDataSize;
|
|
int32 CompressedSizeToGo = CompressedDataSize;
|
|
const uint8* InputDataPtr = CompressedData;
|
|
uint8* OutputDataPtr = OutPCMData;
|
|
|
|
Result.NumAudioFramesProduced = 0;
|
|
while(OutputSizeToGo > 0)
|
|
{
|
|
// Get anything that is left over from the previous call.
|
|
if (PreviousDecodedUnusedSamples.Num())
|
|
{
|
|
check(NumRemainingSamplesToSkip == 0);
|
|
int32 MaxToCopyOut = OutputSizeToGo >= PreviousDecodedUnusedSamples.Num() ? PreviousDecodedUnusedSamples.Num() : OutputSizeToGo;
|
|
FMemory::Memcpy(OutputDataPtr, PreviousDecodedUnusedSamples.GetData(), MaxToCopyOut);
|
|
PreviousDecodedUnusedSamples.RemoveAt(0, MaxToCopyOut);
|
|
OutputSizeToGo -= MaxToCopyOut;
|
|
OutputDataPtr += MaxToCopyOut;
|
|
// If there is still something left over then we are done here.
|
|
Result.NumAudioFramesProduced += MaxToCopyOut / kFrameBytes;
|
|
if (OutputSizeToGo == 0 || PreviousDecodedUnusedSamples.Num())
|
|
{
|
|
Result.NumCompressedBytesConsumed = InputDataPtr - CompressedData;
|
|
Result.NumPcmBytesProduced = Result.NumAudioFramesProduced * kFrameBytes;
|
|
return Result;
|
|
}
|
|
}
|
|
// Something left to decompress?
|
|
if (CompressedSizeToGo <= 0)
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Decode the next chunk.
|
|
const int32 AvailableSampleOutputSize = OutputSizeToGo / kFrameBytes;
|
|
|
|
int32 ChunkToDecodeSize = 0;
|
|
int32 ActualChunkSize = 0;
|
|
// Is this streaming?
|
|
const uint8* ChunkToDecodePtr = InputDataPtr;
|
|
if (!bIsStreaming)
|
|
{
|
|
// When not streaming each input data chunk is prepended with the size of the chunk.
|
|
ChunkToDecodeSize = (int32)InputDataPtr[0] + ((int32)InputDataPtr[1] << 8);
|
|
ActualChunkSize = ChunkToDecodeSize + 2;
|
|
ChunkToDecodePtr += 2;
|
|
}
|
|
else
|
|
{
|
|
// When streaming the chunk size is still prepended to the chunk, but is skipped over
|
|
// by the caller and provided as an argument.
|
|
ChunkToDecodeSize = CompressedDataSize;
|
|
ActualChunkSize = ChunkToDecodeSize;
|
|
}
|
|
|
|
|
|
int32 NumDecodedFrames = OpusDecoderWrapper->Decode(ChunkToDecodePtr, ChunkToDecodeSize, (int16*)OutputDataPtr, AvailableSampleOutputSize);
|
|
if (NumDecodedFrames >= 0)
|
|
{
|
|
CompressedSizeToGo -= ActualChunkSize;
|
|
InputDataPtr += ActualChunkSize;
|
|
if (NumRemainingSamplesToSkip)
|
|
{
|
|
if (NumRemainingSamplesToSkip >= NumDecodedFrames)
|
|
{
|
|
NumRemainingSamplesToSkip -= NumDecodedFrames;
|
|
NumDecodedFrames = 0;
|
|
}
|
|
else
|
|
{
|
|
uint8* FirstUsable = OutputDataPtr + NumRemainingSamplesToSkip * kFrameBytes;
|
|
int32 UsableSize = (NumDecodedFrames - NumRemainingSamplesToSkip) * kFrameBytes;
|
|
FMemory::Memmove(OutputDataPtr, FirstUsable, UsableSize);
|
|
NumDecodedFrames -= NumRemainingSamplesToSkip;
|
|
NumRemainingSamplesToSkip = 0;
|
|
}
|
|
}
|
|
Result.NumAudioFramesProduced += NumDecodedFrames;
|
|
OutputSizeToGo -= NumDecodedFrames * kFrameBytes;
|
|
OutputDataPtr += NumDecodedFrames * kFrameBytes;
|
|
}
|
|
// Was the remaining output buffer too small?
|
|
else if (NumDecodedFrames == OPUS_BUFFER_TOO_SMALL)
|
|
{
|
|
int32 NumSampleSpaceNeeded = GetMaxFrameSizeSamples();
|
|
int32 NumPrevBefore = PreviousDecodedUnusedSamples.Num();
|
|
PreviousDecodedUnusedSamples.AddUninitialized(NumSampleSpaceNeeded * kFrameBytes);
|
|
uint8* TempPtr = PreviousDecodedUnusedSamples.GetData() + NumPrevBefore;
|
|
NumDecodedFrames = OpusDecoderWrapper->Decode(ChunkToDecodePtr, ChunkToDecodeSize, (int16*)TempPtr, NumSampleSpaceNeeded);
|
|
if (NumDecodedFrames >= 0)
|
|
{
|
|
int32 NumPrevNow = NumDecodedFrames * kFrameBytes;
|
|
PreviousDecodedUnusedSamples.SetNum(NumPrevNow);
|
|
CompressedSizeToGo -= ActualChunkSize;
|
|
InputDataPtr += ActualChunkSize;
|
|
if (NumRemainingSamplesToSkip)
|
|
{
|
|
const int32 NumBytesToSkip = NumRemainingSamplesToSkip * kFrameBytes;
|
|
if (NumBytesToSkip >= NumPrevNow)
|
|
{
|
|
PreviousDecodedUnusedSamples.SetNum(0, EAllowShrinking::No);
|
|
NumRemainingSamplesToSkip -= NumPrevNow / kFrameBytes;
|
|
}
|
|
else
|
|
{
|
|
PreviousDecodedUnusedSamples.RemoveAt(0, NumBytesToSkip);
|
|
NumRemainingSamplesToSkip = 0;
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogOpusAudioDecoder, Error, TEXT("opus_multistream_decode() returned %d while decoding into temporary buffer"), NumDecodedFrames);
|
|
return FDecodeResult();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// A decode error of sorts.
|
|
UE_LOG(LogOpusAudioDecoder, Error, TEXT("opus_multistream_decode() returned %d"), NumDecodedFrames);
|
|
return FDecodeResult();
|
|
}
|
|
}
|
|
Result.NumCompressedBytesConsumed = InputDataPtr - CompressedData;
|
|
Result.NumPcmBytesProduced = Result.NumAudioFramesProduced * kFrameBytes;
|
|
}
|
|
return Result;
|
|
}
|
|
|
|
void FOpusAudioInfo::PrepareToLoop()
|
|
{
|
|
IStreamedCompressedInfo::PrepareToLoop();
|
|
NumRemainingSamplesToSkip = Header.NumSilentSamplesAtBeginning + Header.NumPreSkipSamples;
|
|
PreviousDecodedUnusedSamples.Empty();
|
|
}
|
|
|
|
void FOpusAudioInfo::SeekToTime(const float InSeekTime)
|
|
{
|
|
if (GetStreamingSoundWave().IsValid())
|
|
{
|
|
IStreamedCompressedInfo::SeekToTime(InSeekTime);
|
|
}
|
|
else if (SrcBufferData && SrcBufferDataSize)
|
|
{
|
|
uint32 SeekFrameNum = 0;
|
|
if (InSeekTime > 0.0f)
|
|
{
|
|
SeekFrameNum = (uint32)(InSeekTime * Header.SampleRate);
|
|
}
|
|
|
|
SeekToFrame(SeekFrameNum);
|
|
}
|
|
}
|
|
|
|
void FOpusAudioInfo::SeekToFrame(const uint32 InSeekFrame)
|
|
{
|
|
int32 NumSamplesToSkipInChunk = 0;
|
|
if (GetStreamingSoundWave().IsValid())
|
|
{
|
|
IStreamedCompressedInfo::SeekToFrame(InSeekFrame);
|
|
}
|
|
else if (SrcBufferData && SrcBufferDataSize)
|
|
{
|
|
uint32 SeekSampleNum = InSeekFrame * NumChannels;
|
|
const uint8* ChunkPtr = SrcBufferData + AudioDataOffset;
|
|
const uint8* const EndPtr = SrcBufferData + SrcBufferDataSize;
|
|
uint32 CurrentChunkSampleNum = 0;
|
|
while (ChunkPtr < EndPtr)
|
|
{
|
|
uint32 ChunkSize = (uint32)ChunkPtr[0] + ((uint32)ChunkPtr[1] << 8);
|
|
int32 ExpectedFrames = opus_packet_get_nb_frames(ChunkPtr + 2, ChunkSize);
|
|
int32 ExpectedFrameSize = opus_packet_get_samples_per_frame(ChunkPtr + 2, Header.EncodedSampleRate);
|
|
int32 NumExpectedTotal = ExpectedFrames * ExpectedFrameSize * NumChannels;
|
|
|
|
if (SeekSampleNum >= CurrentChunkSampleNum && SeekSampleNum < CurrentChunkSampleNum + NumExpectedTotal)
|
|
{
|
|
CurrentSampleCount = CurrentChunkSampleNum;
|
|
SrcBufferOffset = ChunkPtr - SrcBufferData;
|
|
NumSamplesToSkipInChunk = SeekSampleNum - CurrentChunkSampleNum;
|
|
break;
|
|
}
|
|
CurrentChunkSampleNum += NumExpectedTotal;
|
|
ChunkPtr += ChunkSize + 2;
|
|
}
|
|
// Not found?
|
|
if (ChunkPtr >= EndPtr)
|
|
{
|
|
CurrentSampleCount = SeekSampleNum;
|
|
SrcBufferOffset = SrcBufferDataSize;
|
|
}
|
|
}
|
|
|
|
NumRemainingSamplesToSkip = Header.NumSilentSamplesAtBeginning + Header.NumPreSkipSamples + NumSamplesToSkipInChunk;
|
|
PreviousDecodedUnusedSamples.Empty();
|
|
}
|
|
|
|
class OPUSAUDIODECODER_API FOpusAudioDecoderModule : public IModuleInterface
|
|
{
|
|
public:
|
|
TUniquePtr<IAudioInfoFactory> Factory;
|
|
|
|
virtual void StartupModule() override
|
|
{
|
|
Factory = MakeUnique<FSimpleAudioInfoFactory>([] { return new FOpusAudioInfo(); }, Audio::NAME_OPUS);
|
|
}
|
|
|
|
virtual void ShutdownModule() override {}
|
|
};
|
|
|
|
IMPLEMENT_MODULE(FOpusAudioDecoderModule, OpusAudioDecoder)
|