// Copyright Epic Games, Inc. All Rights Reserved. #include "AudioMixerSourceOutputBuffer.h" #include "SoundFieldRendering.h" #include "AudioMixerSubmix.h" #include "DSP/FloatArrayMath.h" namespace Audio { // Utility function to convert cartesian coordinates to spherical coordinates static void ConvertCartesianToSpherical(const FVector& InVector, float& OutAzimuth, float& OutElevation, float& OutRadius) { // Convert coordinates from unreal cartesian system to left handed spherical coordinates (zenith is positive elevation, right is positive azimuth) const float InX = -InVector.Z; //InVector.Y; const float InY = InVector.X;// -InVector.Z; const float InZ = -InVector.Y; OutElevation = FMath::Atan2(InY, InX); // Note, rather than using arccos(z / radius) here, we use Atan2 to avoid wrapping issues with negative elevation values. OutAzimuth = FMath::Atan2(FMath::Sqrt(InX * InX + InY * InY), InZ); OutRadius = InVector.Size(); } FMixerSourceSubmixOutputBuffer::FMixerSourceSubmixOutputBuffer(FMixerDevice* InMixerDevice, uint32 InNumSourceChannels, uint32 InNumOutputChannels, uint32 InNumFrames) : PreAttenuationSourceBuffer(nullptr) , PostAttenuationSourceBuffer(nullptr) , SourceChannelMap(InNumSourceChannels, InNumOutputChannels) , NumSourceChannels(InNumSourceChannels) , NumFrames(InNumFrames) , NumOutputChannels(InNumOutputChannels) , MixerDevice(InMixerDevice) , bIsInitialDownmix(true) , bIs3D(false) , bIsVorbis(false) { PreAttenuationOutputBuffer.Reset(); PreAttenuationOutputBuffer.AddUninitialized(InNumFrames * InNumOutputChannels); PostAttenuationOutputBuffer.Reset(); PostAttenuationOutputBuffer.AddUninitialized(InNumFrames * InNumOutputChannels); } FMixerSourceSubmixOutputBuffer::~FMixerSourceSubmixOutputBuffer() { } void FMixerSourceSubmixOutputBuffer::Reset(const FMixerSourceSubmixOutputBufferSettings& InInitSettings) { // Whether or not this is a 3D submix output bIs3D = InInitSettings.bIs3D; bIsVorbis = InInitSettings.bIsVorbis; // Reset our record-keeping data NumOutputChannels = InInitSettings.NumOutputChannels; NumSourceChannels = InInitSettings.NumSourceChannels; // Reset the source channel map SourceChannelMap.Reset(NumSourceChannels, NumOutputChannels); // Reset the output buffers PreAttenuationOutputBuffer.Reset(); PreAttenuationOutputBuffer.AddUninitialized(NumFrames * NumOutputChannels); PostAttenuationOutputBuffer.Reset(); PostAttenuationOutputBuffer.AddUninitialized(NumFrames * NumOutputChannels); // Reset the post and pre source buffers PostAttenuationSourceBuffer = nullptr; PreAttenuationSourceBuffer = nullptr; // Reset the soundfield data EncodedSoundfieldDownmixes.Reset(); SoundfieldDecoder.Reset(); bIsInitialDownmix = true; // Make a new soundfield decoder if this sound is a soundfield source if (InInitSettings.bIsSoundfield) { SoundfieldDecoder = CreateDefaultSourceAmbisonicsDecoder(MixerDevice); } // Create data for each of the soundfield submix sends for (const FMixerSubmixPtr& SoundfieldSubmixPtr : InInitSettings.SoundfieldSubmixSends) { // Get the encoding key for the given soundfield submix this source is sending to FSoundfieldEncodingKey EncodingKey = SoundfieldSubmixPtr->GetKeyForSubmixEncoding(); // Create a data bucket based on this encoding key FSoundfieldData& SoundfieldData = EncodedSoundfieldDownmixes.FindOrAdd(EncodingKey); // Get the soundfield submix's factory object, which creates encoders, decoders, transcoders, etc. ISoundfieldFactory* Factory = SoundfieldSubmixPtr->GetSoundfieldFactory(); check(Factory); FAudioPluginInitializationParams PluginInitParams = SoundfieldSubmixPtr->GetInitializationParamsForSoundfieldStream(); PluginInitParams.NumOutputChannels = NumSourceChannels; SoundfieldData.EncoderSettings = SoundfieldSubmixPtr->GetSoundfieldSettings().Duplicate(); // If this source is soundfield, we need to use a transcoder if (InInitSettings.bIsSoundfield) { if (Factory->GetSoundfieldFormatName() == GetUnrealAmbisonicsFormatName()) { SoundfieldData.bIsUnrealAmbisonicsSubmix = true; } else if (Factory->CanTranscodeFromSoundfieldFormat(GetUnrealAmbisonicsFormatName(), GetAmbisonicsSourceDefaultSettings())) { SoundfieldData.SoundfieldTranscoder = Factory->CreateTranscoderStream(GetUnrealAmbisonicsFormatName(), GetAmbisonicsSourceDefaultSettings(), Factory->GetSoundfieldFormatName(), *SoundfieldData.EncoderSettings, PluginInitParams); } } else { check(SoundfieldData.EncoderSettings.IsValid()); SoundfieldData.SoundfieldEncoder = Factory->CreateEncoderStream(PluginInitParams, *SoundfieldData.EncoderSettings); } // Create a blank packet for memory of the encoded packet SoundfieldData.EncodedPacket = Factory->CreateEmptyPacket(); } } void FMixerSourceSubmixOutputBuffer::SetNumOutputChannels(uint32 InNumOutputChannels) { NumOutputChannels = InNumOutputChannels; SourceChannelMap.Reset(NumSourceChannels, NumOutputChannels); PreAttenuationOutputBuffer.Reset(); PreAttenuationOutputBuffer.AddUninitialized(NumFrames * NumOutputChannels); PostAttenuationOutputBuffer.Reset(); PostAttenuationOutputBuffer.AddUninitialized(NumFrames * NumOutputChannels); } bool FMixerSourceSubmixOutputBuffer::SetChannelMap(const FAlignedFloatBuffer& InChannelMap, bool bInIsCenterChannelOnly) { bool bNeedsNewChannelMap = false; // Fix up the channel map in case the device output count changed const uint32 ChannelMapSize = SourceChannelMap.CopySize / sizeof(float); if (InChannelMap.Num() != ChannelMapSize) { FAlignedFloatBuffer NewChannelMap; if (bIs3D) { NewChannelMap.AddZeroed(ChannelMapSize); bNeedsNewChannelMap = true; } else { const uint32 ChannelMapOutputChannels = ChannelMapSize / NumSourceChannels; FMixerDevice::Get2DChannelMap(bIsVorbis, NumSourceChannels, ChannelMapOutputChannels, bInIsCenterChannelOnly, NewChannelMap); } SourceChannelMap.SetChannelMap(NewChannelMap.GetData()); } else { SourceChannelMap.SetChannelMap(InChannelMap.GetData()); } return bNeedsNewChannelMap; } void FMixerSourceSubmixOutputBuffer::SetPreAttenuationSourceBuffer(FAlignedFloatBuffer* InPreAttenuationSourceBuffer) { PreAttenuationSourceBuffer = InPreAttenuationSourceBuffer; } void FMixerSourceSubmixOutputBuffer::SetPostAttenuationSourceBuffer(FAlignedFloatBuffer* InPostAttenuationSourceBuffer) { PostAttenuationSourceBuffer = InPostAttenuationSourceBuffer; } void FMixerSourceSubmixOutputBuffer::ComputeOutput(const FSpatializationParams& InSpatParams) { // No need to compute an output if there is no source buffer available if (!PostAttenuationSourceBuffer && !PreAttenuationSourceBuffer) { return; } // Update our rotational data based off the spat params SoundfieldPositionalData.Rotation = InSpatParams.ListenerOrientation; SoundSourceRotation = InSpatParams.EmitterWorldRotation; if (bIs3D && !bIsInitialDownmix) { ComputeOutput3D(); } else { ComputeOutput2D(); bIsInitialDownmix = false; } // Now check if we need to do any sound field encoding if (EncodedSoundfieldDownmixes.Num()) { EncodeToSoundfieldFormats(InSpatParams); } } void FMixerSourceSubmixOutputBuffer::ComputeOutput3D(FAlignedFloatBuffer& InSourceBuffer, FAlignedFloatBuffer& OutSourceBuffer) { if (SoundfieldDecoder.IsValid()) { FAmbisonicsSoundfieldBuffer SoundfieldBuffer; SoundfieldBuffer.AudioBuffer = MoveTemp(InSourceBuffer); SoundfieldBuffer.NumChannels = NumSourceChannels; SoundfieldBuffer.PreviousRotation = SoundfieldBuffer.Rotation; SoundfieldBuffer.Rotation = SoundSourceRotation; SoundfieldPositionalData.NumChannels = NumOutputChannels; SoundfieldPositionalData.ChannelPositions = MixerDevice->GetDefaultPositionMap(NumOutputChannels); FSoundfieldDecoderInputData SoundfieldDecoderInputData = { SoundfieldBuffer, SoundfieldPositionalData, static_cast(InSourceBuffer.Num() / NumSourceChannels), MixerDevice->GetSampleRate() }; FSoundfieldDecoderOutputData SoundFieldDecoderOutputData = { OutSourceBuffer }; SoundfieldDecoder->Decode(SoundfieldDecoderInputData, SoundFieldDecoderOutputData); InSourceBuffer = MoveTemp(SoundfieldBuffer.AudioBuffer); } else if (NumSourceChannels == 1) { switch (NumOutputChannels) { case 8: MixMonoTo8ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; case 6: MixMonoTo6ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; case 4: MixMonoTo4ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; case 2: MixMonoTo2ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; } } else if (NumSourceChannels == 2) { switch (NumOutputChannels) { case 8: Mix2ChannelsTo8ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; case 6: Mix2ChannelsTo6ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; case 4: Mix2ChannelsTo4ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; case 2: Mix2ChannelsTo2ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); break; } } else { DownmixBuffer(NumSourceChannels, NumOutputChannels, InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelStartGains, SourceChannelMap.ChannelDestinationGains); } } void FMixerSourceSubmixOutputBuffer::ComputeOutput3D() { // Compute the output buffers if there is a corresponding pre or post source buffer set if (PreAttenuationSourceBuffer) { ComputeOutput3D(*PreAttenuationSourceBuffer, PreAttenuationOutputBuffer); } if (PostAttenuationSourceBuffer) { ComputeOutput3D(*PostAttenuationSourceBuffer, PostAttenuationOutputBuffer); } // Do the channel map copy from dest to start (which prevents zippering of dynamic channel mapping for 3d audio) after the data has been computed for both pre- and post- attenuation channel downmixing SourceChannelMap.CopyDestinationToStart(); } void FMixerSourceSubmixOutputBuffer::ComputeOutput2D(FAlignedFloatBuffer& InSourceBuffer, FAlignedFloatBuffer& OutSourceBuffer) { // For 2D sources, we just apply the gain matrix in ChannelDestionationGains with no interpolation. if (SoundfieldDecoder.IsValid()) { FAmbisonicsSoundfieldBuffer SoundfieldBuffer; SoundfieldBuffer.AudioBuffer = MoveTemp(InSourceBuffer); SoundfieldBuffer.NumChannels = NumSourceChannels; SoundfieldBuffer.PreviousRotation = SoundfieldBuffer.Rotation; SoundfieldBuffer.Rotation = SoundSourceRotation; SoundfieldPositionalData.NumChannels = NumOutputChannels; SoundfieldPositionalData.ChannelPositions = MixerDevice->GetDefaultPositionMap(NumOutputChannels); FSoundfieldDecoderInputData SoundfieldDecoderInputData = { SoundfieldBuffer, SoundfieldPositionalData, static_cast(InSourceBuffer.Num() / NumSourceChannels), MixerDevice->GetSampleRate() }; FSoundfieldDecoderOutputData SoundfieldDecoderOutputData = { OutSourceBuffer }; SoundfieldDecoder->Decode(SoundfieldDecoderInputData, SoundfieldDecoderOutputData); // Move the encoded ambisonics source buffer back to PostEffectBuffers to prevent reallocation InSourceBuffer = MoveTemp(SoundfieldBuffer.AudioBuffer); } else if (NumSourceChannels == 1) { switch (NumOutputChannels) { case 8: Audio::MixMonoTo8ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; case 6: Audio::MixMonoTo6ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; case 4: Audio::MixMonoTo4ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; case 2: Audio::MixMonoTo2ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; } } else if (NumSourceChannels == 2) { switch (NumOutputChannels) { case 8: Audio::Mix2ChannelsTo8ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; case 6: Audio::Mix2ChannelsTo6ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; case 4: Audio::Mix2ChannelsTo4ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; case 2: Audio::Mix2ChannelsTo2ChannelsFast(InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); break; } } else { Audio::DownmixBuffer(NumSourceChannels, NumOutputChannels, InSourceBuffer, OutSourceBuffer, SourceChannelMap.ChannelDestinationGains); } } void FMixerSourceSubmixOutputBuffer::ComputeOutput2D() { if (PreAttenuationSourceBuffer) { ComputeOutput2D(*PreAttenuationSourceBuffer, PreAttenuationOutputBuffer); } if (PostAttenuationSourceBuffer) { ComputeOutput2D(*PostAttenuationSourceBuffer, PostAttenuationOutputBuffer); } } void FMixerSourceSubmixOutputBuffer::EncodeToSoundfieldFormats(const FSpatializationParams& InSpatParams) { check(MixerDevice); SoundfieldPositionalData.NumChannels = NumSourceChannels; // Spoof rotation of the source as if it's rotation of the listener when encoding non-sound-field to sound-field SoundfieldPositionalData.Rotation = InSpatParams.EmitterWorldRotation; InputChannelPositions.Reset(); if (bIs3D) { if (NumSourceChannels == 1) { FChannelPositionInfo ChannelPosition; ChannelPosition.Channel = EAudioMixerChannel::FrontCenter; ConvertCartesianToSpherical(InSpatParams.EmitterPosition, ChannelPosition.Azimuth, ChannelPosition.Elevation, ChannelPosition.Radius); ChannelPosition.Radius = InSpatParams.Distance; InputChannelPositions.Add(ChannelPosition); SoundfieldPositionalData.ChannelPositions = &InputChannelPositions; } else if (NumSourceChannels == 2) { FChannelPositionInfo LeftChannelPosition; LeftChannelPosition.Channel = EAudioMixerChannel::FrontLeft; ConvertCartesianToSpherical(InSpatParams.LeftChannelPosition, LeftChannelPosition.Azimuth, LeftChannelPosition.Elevation, LeftChannelPosition.Radius); InputChannelPositions.Add(LeftChannelPosition); FChannelPositionInfo RightChannelPosition; LeftChannelPosition.Channel = EAudioMixerChannel::FrontRight; ConvertCartesianToSpherical(InSpatParams.RightChannelPosition, RightChannelPosition.Azimuth, RightChannelPosition.Elevation, RightChannelPosition.Radius); InputChannelPositions.Add(RightChannelPosition); SoundfieldPositionalData.ChannelPositions = &InputChannelPositions; } } // if 2D or not a supported channel configuration, use default position map if (!InputChannelPositions.Num()) { SoundfieldPositionalData.ChannelPositions = MixerDevice->GetDefaultPositionMap(NumSourceChannels); } // Run the encoders. for (auto& Soundfield : EncodedSoundfieldDownmixes) { FSoundfieldData& SoundfieldData = Soundfield.Value; if (PreAttenuationSourceBuffer) { EncodeSoundfield(SoundfieldData , *PreAttenuationSourceBuffer); } if (PostAttenuationSourceBuffer) { EncodeSoundfield(SoundfieldData, *PostAttenuationSourceBuffer); } } } void FMixerSourceSubmixOutputBuffer::EncodeSoundfield(FSoundfieldData& InSoundfieldData, Audio::FAlignedFloatBuffer& InSourceBuffer) { check(InSoundfieldData.EncoderSettings.IsValid()); check(InSoundfieldData.EncodedPacket.IsValid()); InSoundfieldData.EncodedPacket->Reset(); // We will have an soundfield transcoder if this sound source is a soundfield format if (InSoundfieldData.SoundfieldTranscoder) { FAmbisonicsSoundfieldBuffer SoundfieldBuffer; SoundfieldBuffer.AudioBuffer = MoveTemp(InSourceBuffer); SoundfieldBuffer.NumChannels = NumSourceChannels; SoundfieldBuffer.PreviousRotation = SoundfieldBuffer.Rotation; SoundfieldBuffer.Rotation = SoundSourceRotation; InSoundfieldData.SoundfieldTranscoder->Transcode(SoundfieldBuffer, GetAmbisonicsSourceDefaultSettings(), *InSoundfieldData.EncodedPacket, *InSoundfieldData.EncoderSettings); InSourceBuffer = MoveTemp(SoundfieldBuffer.AudioBuffer); } else if (InSoundfieldData.SoundfieldEncoder) { FSoundfieldEncoderInputData SoundfieldEncoderInputData = { InSourceBuffer, static_cast(NumSourceChannels), *InSoundfieldData.EncoderSettings, SoundfieldPositionalData }; InSoundfieldData.SoundfieldEncoder->Encode(SoundfieldEncoderInputData, *InSoundfieldData.EncodedPacket); } else if (InSoundfieldData.bIsUnrealAmbisonicsSubmix) { FAmbisonicsSoundfieldBuffer& OutputPacket = DowncastSoundfieldRef(*InSoundfieldData.EncodedPacket); // Fixme: This is an array copy. Can we serve InPositionalData directly to this soundfield? OutputPacket.AudioBuffer = InSourceBuffer; OutputPacket.NumChannels = NumSourceChannels; OutputPacket.PreviousRotation = OutputPacket.Rotation; OutputPacket.Rotation = SoundSourceRotation; } } void FMixerSourceSubmixOutputBuffer::MixOutput(float InSendLevel, EMixerSourceSubmixSendStage InSubmixSendStage, FAlignedFloatBuffer& OutMixedBuffer) const { if (InSubmixSendStage == EMixerSourceSubmixSendStage::PostDistanceAttenuation) { Audio::ArrayMixIn(PostAttenuationOutputBuffer, OutMixedBuffer, InSendLevel); } else { Audio::ArrayMixIn(PreAttenuationOutputBuffer, OutMixedBuffer, InSendLevel); } } FQuat FMixerSourceSubmixOutputBuffer::GetListenerRotation() const { return SoundfieldPositionalData.Rotation; } void FMixerSourceSubmixOutputBuffer::CopyReverbPluginOutputData(FAlignedFloatBuffer& InAudioBuffer) { ReverbPluginOutputBuffer.Reset(); ReverbPluginOutputBuffer.Append(InAudioBuffer); } const float* FMixerSourceSubmixOutputBuffer::GetReverbPluginOutputData() const { return ReverbPluginOutputBuffer.GetData(); } const ISoundfieldAudioPacket* FMixerSourceSubmixOutputBuffer::GetSoundfieldPacket(const FSoundfieldEncodingKey& InKey) const { if(EncodedSoundfieldDownmixes.Contains(InKey)) { const FSoundfieldData& SoundfieldData = EncodedSoundfieldDownmixes[InKey]; return SoundfieldData.EncodedPacket.Get(); } return nullptr; } ISoundfieldAudioPacket* FMixerSourceSubmixOutputBuffer::GetSoundFieldPacket(const FSoundfieldEncodingKey& InKey) { FSoundfieldData& SoundfieldData = EncodedSoundfieldDownmixes.FindOrAdd(InKey); return SoundfieldData.EncodedPacket.Get(); } }