// Copyright Epic Games, Inc. All Rights Reserved. #include "EpicRtcStreamer.h" #include "PixelStreaming2PluginSettings.h" #include "PixelStreaming2Delegates.h" #include "Engine/GameEngine.h" #include "Engine/GameViewportClient.h" #include "EpicRtcWebsocket.h" #include "EpicRtcWebsocketFactory.h" #include "TextureResource.h" #include "WebSocketsModule.h" #include "Logging.h" #include "Stats.h" #include "PixelStreaming2StatNames.h" #include "Async/Async.h" #include "Engine/Texture2D.h" #include "RTCStatsCollector.h" #include "Framework/Application/SlateApplication.h" #include "Dom/JsonValue.h" #include "Serialization/JsonWriter.h" #include "Serialization/JsonSerializer.h" #include "Dom/JsonObject.h" #include "PixelStreaming2RTCModule.h" #include "EpicRtcTrack.h" #include "EpicRtcDataTrack.h" #include "IPixelStreaming2InputModule.h" #include "CoreGlobals.h" #include "UtilsAsync.h" #include "UtilsString.h" #include "UtilsCodecs.h" #include "VideoProducerBackBuffer.h" #include "VideoProducerMediaCapture.h" #include "RTCInputHandler.h" #include "PixelStreaming2Common.h" #include "EpicRtcAudioCapturer.h" namespace UE::PixelStreaming2 { FEpicRtcStreamer::FEpicRtcStreamer(const FString& InStreamerId, TRefCountPtr Conference) : StreamerId(InStreamerId) , InputHandler(FRTCInputHandler::Create()) , Participants(new TThreadSafeMap>()) , VideoCapturer(FEpicRtcVideoCapturer::Create()) , VideoSourceGroup(FVideoSourceGroup::Create(VideoCapturer)) , FreezeFrame(FFreezeFrame::Create(Participants, VideoCapturer, InputHandler)) , ReconnectTimer(MakeShared()) , EpicRtcConference(MoveTemp(Conference)) { InputHandler->SetElevatedCheck([this](FString PlayerId) { return GetEnumFromCVar(UPixelStreaming2PluginSettings::CVarInputController) == EInputControllerMode::Any || InputControllingId == INVALID_PLAYER_ID || PlayerId == InputControllingId; }); } FEpicRtcStreamer::~FEpicRtcStreamer() { StopStreaming(); // Pin ref counted objects and clear the session if it is still alive. // Session may still be alive if the manager was destroyed quickly after stopping the stream // and the EpicRtcSessionState::Disconnected event did not have enough time to tick. const TRefCountPtr Session(EpicRtcSession); if (EpicRtcConference.IsValid() && Session.IsValid()) { const FUtf8String Utf8StreamerId(StreamerId); EpicRtcConference->RemoveSession(ToEpicRtcStringView(Utf8StreamerId)); EpicRtcSession = nullptr; } } void FEpicRtcStreamer::Initialize() { TSharedPtr Streamer = AsShared(); TWeakPtr WeakStreamer = Streamer; SessionObserver = MakeRefCount(TObserver(WeakStreamer)); RoomObserver = MakeRefCount(TObserver(WeakStreamer)); AudioTrackObserverFactory = MakeRefCount(TObserver(WeakStreamer)); VideoTrackObserverFactory = MakeRefCount(TObserver(WeakStreamer)); DataTrackObserverFactory = MakeRefCount(TObserver(WeakStreamer)); InputHandler->GetToStreamerProtocol()->OnProtocolUpdated().AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::OnProtocolUpdated); InputHandler->GetFromStreamerProtocol()->OnProtocolUpdated().AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::OnProtocolUpdated); // Set Encoder.MinQP Legacy CVar InputHandler->SetCommandHandler(TEXT("Encoder.MinQP"), [](FString PlayerId, FString Descriptor, FString MinQPString) { int MinQP = FCString::Atoi(*MinQPString); UPixelStreaming2PluginSettings::CVarEncoderMaxQuality->SetWithCurrentPriority(FMath::RoundToFloat(100.0f * (1.0f - (FMath::Clamp(MinQP, 0, 51) / 51.0f)))); }); // Set Encoder.MaxQP Legacy CVar InputHandler->SetCommandHandler(TEXT("Encoder.MaxQP"), [](FString PlayerId, FString Descriptor, FString MaxQPString) { int MaxQP = FCString::Atoi(*MaxQPString); UPixelStreaming2PluginSettings::CVarEncoderMinQuality->SetWithCurrentPriority(FMath::RoundToFloat(100.0f * (1.0f - (FMath::Clamp(MaxQP, 0, 51) / 51.0f)))); }); // Set Encoder.MinQuality CVar InputHandler->SetCommandHandler(TEXT("Encoder.MinQuality"), [](FString PlayerId, FString Descriptor, FString MinQualityString) { int MinQuality = FCString::Atoi(*MinQualityString); UPixelStreaming2PluginSettings::CVarEncoderMinQuality->SetWithCurrentPriority(FMath::Clamp(MinQuality, 0, 100)); }); // Set Encoder.MaxQuality CVar InputHandler->SetCommandHandler(TEXT("Encoder.MaxQuality"), [](FString PlayerId, FString Descriptor, FString MaxQualityString) { int MaxQuality = FCString::Atoi(*MaxQualityString); UPixelStreaming2PluginSettings::CVarEncoderMaxQuality->SetWithCurrentPriority(FMath::Clamp(MaxQuality, 0, 100)); }); // Set WebRTC max FPS InputHandler->SetCommandHandler(TEXT("WebRTC.Fps"), [](FString PlayerId, FString Descriptor, FString FPSString) { int FPS = FCString::Atoi(*FPSString); UPixelStreaming2PluginSettings::CVarWebRTCFps->SetWithCurrentPriority(FPS); }); // Set MinBitrate InputHandler->SetCommandHandler(TEXT("WebRTC.MinBitrate"), [WeakStreamer](FString PlayerId, FString Descriptor, FString MinBitrateString) { // This check pattern is kind of verbose, but as the messages are enqueued on a different thread to where they were added // we need to make sure that both the streamer and input handler are still alive when we process the command TSharedPtr Streamer = WeakStreamer.Pin(); if (!Streamer) { return; } TSharedPtr InputHandler = Streamer->GetInputHandler().Pin(); if (!InputHandler) { return; } if (InputHandler->IsElevated(PlayerId)) { int MinBitrate = FCString::Atoi(*MinBitrateString); UPixelStreaming2PluginSettings::CVarWebRTCMinBitrate->SetWithCurrentPriority(MinBitrate); } }); // Set MaxBitrate InputHandler->SetCommandHandler(TEXT("WebRTC.MaxBitrate"), [WeakStreamer](FString PlayerId, FString Descriptor, FString MaxBitrateString) { TSharedPtr Streamer = WeakStreamer.Pin(); if (!Streamer) { return; } TSharedPtr InputHandler = Streamer->GetInputHandler().Pin(); if (!InputHandler) { return; } if (InputHandler->IsElevated(PlayerId)) { int MaxBitrate = FCString::Atoi(*MaxBitrateString); UPixelStreaming2PluginSettings::CVarWebRTCMaxBitrate->SetWithCurrentPriority(MaxBitrate); } }); InputHandler->RegisterMessageHandler(EPixelStreaming2ToStreamerMessage::UIInteraction, [WeakStreamer](FString PlayerId, FMemoryReader Ar) { TSharedPtr Streamer = WeakStreamer.Pin(); if (!Streamer) { return; } Streamer->OnUIInteraction(Ar); }); // Handle special cases when the InputHandler itself wants to send a message out to all the peers. // Some special cases include when virtual gamepads are connected and a controller id needs to be transmitted. InputHandler->OnSendMessage.AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::OnSendMessage); VideoCapturer->OnFrameCapturerCreated.AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::OnFrameCapturerCreated); if (UPixelStreaming2PluginSettings::FDelegates* Delegates = UPixelStreaming2PluginSettings::Delegates()) { Delegates->OnUseMediaCaptureChanged.AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::OnUseMediaCaptureChanged); } if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { Delegates->OnStatChangedNative.AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::ConsumeStats); Delegates->OnAllConnectionsClosedNative.AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::TriggerMouseLeave); } FPixelStreaming2RTCModule::GetModule()->GetStatsCollector()->OnStatsReady.AddSP(Streamer.ToSharedRef(), &FEpicRtcStreamer::OnStatsReady); } void FEpicRtcStreamer::OnProtocolUpdated() { Participants->Apply([this](FString DataPlayerId, TSharedPtr PlayerContext) { SendProtocol(DataPlayerId); }); } void FEpicRtcStreamer::SetStreamFPS(int32 InFramesPerSecond) { VideoSourceGroup->SetFPS(InFramesPerSecond); } int32 FEpicRtcStreamer::GetStreamFPS() { return VideoSourceGroup->GetFPS(); } void FEpicRtcStreamer::SetCoupleFramerate(bool bCouple) { VideoSourceGroup->SetDecoupleFramerate(!bCouple); } void FEpicRtcStreamer::SetVideoProducer(TSharedPtr Producer) { VideoCapturer->SetVideoProducer(StaticCastSharedPtr(Producer)); } TWeakPtr FEpicRtcStreamer::GetVideoProducer() { return VideoCapturer->GetVideoProducer(); } void FEpicRtcStreamer::SetConnectionURL(const FString& InConnectionURL) { CurrentSignallingServerURL = InConnectionURL; } FString FEpicRtcStreamer::GetConnectionURL() { return CurrentSignallingServerURL; } void FEpicRtcStreamer::StartStreaming() { if (StreamState != EStreamState::Disconnected) { UE_LOGFMT(LogPixelStreaming2RTC, Log, "Streamer is already streaming. Ignoring subsequent call to StartStreaming!"); return; } StreamState = EStreamState::Connecting; if (CurrentSignallingServerURL.IsEmpty()) { UE_LOGFMT(LogPixelStreaming2RTC, Warning, "Attempted to start streamer ({0}) but no signalling server URL has been set. Use Streamer->SetConnectionURL(URL) or -PixelStreamingConnectionURL=", StreamerId); return; } check(EpicRtcConference.IsValid()); ReconnectTimer->Stop(); VideoCapturer->ResetFrameCapturer(); // Broadcast the preconnection event just before we do `TryConnect` OnPreConnection().Broadcast(this); VideoSourceGroup->Start(); FUtf8String Utf8StreamerId(StreamerId); FUtf8String Utf8CurrentSignallingServerURL(CurrentSignallingServerURL); EpicRtcSessionConfig SessionConfig{ ._id = ToEpicRtcStringView(Utf8StreamerId), ._url = ToEpicRtcStringView(Utf8CurrentSignallingServerURL), ._observer = SessionObserver }; if (const EpicRtcErrorCode Result = EpicRtcConference->CreateSession(SessionConfig, EpicRtcSession.GetInitReference()); Result != EpicRtcErrorCode::Ok) { UE_LOGFMT(LogPixelStreaming2RTC, Error, "Failed to create EpicRtc session. CreateSession returned {0}", *ToString(Result)); StreamState = EStreamState::Disconnected; return; } // TODO (william.belcher): This should move to OnSessionStateUpdate(EpicRtcSessionState::New) once EpicRtc starts broadcasting that state if (const EpicRtcErrorCode Result = EpicRtcSession->Connect(); Result != EpicRtcErrorCode::Ok) { UE_LOGFMT(LogPixelStreaming2RTC, Error, "Failed to connect EpicRtcSession. Connect returned {0}", ToString(Result)); StreamState = EStreamState::Disconnected; return; } } void FEpicRtcStreamer::StopStreaming() { if (StreamState == EStreamState::Disconnected || StreamState == EStreamState::Disconnecting) { return; } StreamState = EStreamState::Disconnecting; if (const TRefCountPtr Room = EpicRtcRoom) { Room->Leave(); } VideoSourceGroup->Stop(); TriggerMouseLeave(StreamerId); DeleteAllPlayerSessions(); } void FEpicRtcStreamer::OnStatsReady(const FString& PlayerId, const EpicRtcConnectionStats& ConnectionStats) { TSharedPtr StatsCollector; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { StatsCollector = Participant->StatsCollector; } if (StatsCollector) { StatsCollector->Process(ConnectionStats); } } void FEpicRtcStreamer::OnFrameCapturerCreated() { if (FStats* PSStats = FStats::Get()) { PSStats->RemoveAllApplicationStats(); // Re-add the cvar based application stats so that they're at the top PSStats->StoreApplicationStat(FStat({ .Name = FName(TEXT("PixelStreaming2.Encoder.MinQuality")) }, UPixelStreaming2PluginSettings::CVarEncoderMinQuality.GetValueOnAnyThread(), 0)); PSStats->StoreApplicationStat(FStat({ .Name = FName(TEXT("PixelStreaming2.Encoder.MaxQuality")) }, UPixelStreaming2PluginSettings::CVarEncoderMaxQuality.GetValueOnAnyThread(), 0)); PSStats->StoreApplicationStat(FStat({ .Name = FName(TEXT("PixelStreaming2.Encoder.KeyframeInterval (frames)")) }, UPixelStreaming2PluginSettings::CVarEncoderKeyframeInterval.GetValueOnAnyThread(), 0)); PSStats->StoreApplicationStat(FStat({ .Name = FName(TEXT("PixelStreaming2.WebRTC.Fps")) }, UPixelStreaming2PluginSettings::CVarWebRTCFps.GetValueOnAnyThread(), 0)); PSStats->StoreApplicationStat(FStat({ .Name = FName(TEXT("PixelStreaming2.WebRTC.StartBitrate")) }, UPixelStreaming2PluginSettings::CVarWebRTCStartBitrate.GetValueOnAnyThread(), 0)); PSStats->StoreApplicationStat(FStat({ .Name = FName(TEXT("PixelStreaming2.WebRTC.MinBitrate")) }, UPixelStreaming2PluginSettings::CVarWebRTCMinBitrate.GetValueOnAnyThread(), 0)); PSStats->StoreApplicationStat(FStat({ .Name = FName(TEXT("PixelStreaming2.WebRTC.MaxBitrate")) }, UPixelStreaming2PluginSettings::CVarWebRTCMaxBitrate.GetValueOnAnyThread(), 0)); } } void FEpicRtcStreamer::OnUIInteraction(FMemoryReader Ar) { FString Res; Res.GetCharArray().SetNumUninitialized(Ar.TotalSize() / 2 + 1); Ar.Serialize(Res.GetCharArray().GetData(), Ar.TotalSize()); FString Descriptor = Res.Mid(1); UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer[\"{0}\"]::OnUIInteraction({1})", StreamerId, Descriptor); InputComponents.Apply([Descriptor](uintptr_t Key, UPixelStreaming2Input* Value) { Value->OnInputEvent.Broadcast(Descriptor); }); } void FEpicRtcStreamer::OnSendMessage(FString MessageName, FMemoryReader Ar) { FString Descriptor; Ar << Descriptor; SendAllPlayersMessage(MessageName, Descriptor); } void FEpicRtcStreamer::OnUseMediaCaptureChanged(IConsoleVariable* CVar) { TSharedPtr VideoProducer = VideoCapturer->GetVideoProducer(); if (!VideoProducer) { return; } FString Identifier = VideoProducer->ToString(); // If the identifier is not one of these two types, it means it was user created so we don't want to // override it if (Identifier != VideoProducerIdentifiers::FVideoProducerMediaCapture && Identifier != VideoProducerIdentifiers::FVideoProducerBackBuffer) { return; } if (CVar->GetBool()) { SetVideoProducer(FVideoProducerMediaCapture::CreateActiveViewportCapture()); } else { SetVideoProducer(FVideoProducerBackBuffer::Create()); } } EpicRtcBitrate FEpicRtcStreamer::GetBitrates() const { const int MinBitrate = UPixelStreaming2PluginSettings::CVarWebRTCMinBitrate.GetValueOnAnyThread(); const int MaxBitrate = UPixelStreaming2PluginSettings::CVarWebRTCMaxBitrate.GetValueOnAnyThread(); const int StartBitrate = FMath::Clamp(UPixelStreaming2PluginSettings::CVarWebRTCStartBitrate.GetValueOnAnyThread(), MinBitrate, MaxBitrate); EpicRtcBitrate Bitrates = { ._minBitrateBps = MinBitrate, ._hasMinBitrateBps = true, ._maxBitrateBps = MaxBitrate, ._hasMaxBitrateBps = true, ._startBitrateBps = StartBitrate, ._hasStartBitrateBps = true }; return Bitrates; } IPixelStreaming2Streamer::FPreConnectionEvent& FEpicRtcStreamer::OnPreConnection() { return StreamingPreConnectionEvent; } IPixelStreaming2Streamer::FStreamingStartedEvent& FEpicRtcStreamer::OnStreamingStarted() { return StreamingStartedEvent; } IPixelStreaming2Streamer::FStreamingStoppedEvent& FEpicRtcStreamer::OnStreamingStopped() { return StreamingStoppedEvent; } void FEpicRtcStreamer::ForceKeyFrame() { VideoSourceGroup->ForceKeyFrame(); } void FEpicRtcStreamer::FreezeStream(UTexture2D* Texture) { FreezeFrame->StartFreeze(Texture); } void FEpicRtcStreamer::UnfreezeStream() { // Force a keyframe so when stream unfreezes if player has never received a frame before they can still connect. ForceKeyFrame(); FreezeFrame->StopFreeze(); } void FEpicRtcStreamer::SendAllPlayersMessage(FString MessageType, const FString& Descriptor) { Participants->Apply([&MessageType, &Descriptor](FString PlayerId, TSharedPtr Participant) { if (Participant->DataTrack && !IsSFU(PlayerId)) { Participant->DataTrack->SendMessage(MessageType, Descriptor); } }); } void FEpicRtcStreamer::SendPlayerMessage(FString PlayerId, FString MessageType, const FString& Descriptor) { if (IsSFU(PlayerId)) { return; } TSharedPtr DataTrack; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { DataTrack = Participant->DataTrack; } if (DataTrack) { DataTrack->SendMessage(MessageType, Descriptor); } } void FEpicRtcStreamer::SendFileData(const TArray64& ByteData, FString& MimeType, FString& FileExtension) { // TODO this should be dispatched as an async task, but because we lock when we visit the data // channels it might be a bad idea. At some point it would be good to take a snapshot of the // keys in the map when we start, then one by one get the channel and send the data Participants->Apply([&ByteData, &MimeType, &FileExtension](FString PlayerId, TSharedPtr Participant) { if (!Participant->DataTrack) { return; } // Send the mime type first Participant->DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::FileMimeType, MimeType); // Send the extension next Participant->DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::FileExtension, FileExtension); // Send the contents of the file. Note to callers: consider running this on its own thread, it can take a while if the file is big. Participant->DataTrack->SendArbitraryData(EPixelStreaming2FromStreamerMessage::FileContents, ByteData); }); } void FEpicRtcStreamer::KickPlayer(FString PlayerId) { TRefCountPtr ParticipantInterface; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { ParticipantInterface = Participant->ParticipantInterface; } ParticipantInterface->Kick(); } TArray FEpicRtcStreamer::GetConnectedPlayers() { TSet ConnectedParticipantIds; Participants->GetKeys(ConnectedParticipantIds); return ConnectedParticipantIds.Array(); } TWeakPtr FEpicRtcStreamer::GetPeerAudioSink(FString PlayerId) { TWeakPtr Result = nullptr; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { Result = Participant->AudioSink ? Participant->AudioSink : nullptr; } return Result; } TWeakPtr FEpicRtcStreamer::GetUnlistenedAudioSink() { TWeakPtr Result = nullptr; Participants->ApplyUntil([&Result](FString PlayerId, TSharedPtr Participant) { if (Participant->AudioSink) { if (!Participant->AudioSink->HasAudioConsumers()) { Result = Participant->AudioSink; return true; } } return false; }); return Result; } TWeakPtr FEpicRtcStreamer::GetPeerVideoSink(FString PlayerId) { TWeakPtr Result = nullptr; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { Result = Participant->VideoSink ? Participant->VideoSink : nullptr; } return Result; } TWeakPtr FEpicRtcStreamer::GetUnwatchedVideoSink() { TWeakPtr Result = nullptr; Participants->ApplyUntil([&Result](FString PlayerId, TSharedPtr Participant) { if (Participant->VideoSink) { if (!Participant->VideoSink->HasVideoConsumers()) { Result = Participant->VideoSink; return true; } } return false; }); return Result; } void FEpicRtcStreamer::SetConfigOption(const FName& OptionName, const FString& Value) { if (Value.IsEmpty()) { ConfigOptions.Remove(OptionName); } else { ConfigOptions.Add(OptionName, Value); } } bool FEpicRtcStreamer::GetConfigOption(const FName& OptionName, FString& OutValue) { FString* OptionValue = ConfigOptions.Find(OptionName); if (OptionValue) { OutValue = *OptionValue; return true; } else { return false; } } void FEpicRtcStreamer::PlayerRequestsBitrate(FString PlayerId, int MinBitrate, int MaxBitrate) { UPixelStreaming2PluginSettings::CVarWebRTCMinBitrate.AsVariable()->SetWithCurrentPriority(MinBitrate); UPixelStreaming2PluginSettings::CVarWebRTCMaxBitrate.AsVariable()->SetWithCurrentPriority(MaxBitrate); } void FEpicRtcStreamer::RefreshStreamBitrate() { Participants->Apply([this](FString PlayerId, TSharedPtr Participant) { if (!Participant->ParticipantInterface) { return; } TRefCountPtr ConnectionInterface = Participant->ParticipantInterface->GetConnection(); if (!ConnectionInterface) { return; } ConnectionInterface->SetConnectionRates(GetBitrates()); }); } void FEpicRtcStreamer::ForEachPlayer(const TFunction)>& Func) { Participants->Apply(Func); } void FEpicRtcStreamer::ConsumeStats(FString PlayerId, FName StatName, float StatValue) { if (IsSFU(PlayerId)) { return; } if (StatName != PixelStreaming2StatNames::MeanQPPerSecond) { return; } TSharedPtr DataTrack; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { DataTrack = Participant->DataTrack; } if (!DataTrack) { return; } DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::VideoEncoderAvgQP, FString::FromInt(static_cast(StatValue))); } void FEpicRtcStreamer::DeletePlayerSession(FString PlayerId) { // We need to keep a copy of the participant to delete as when destroying the ParticipantInterface, // webrtc will finish tasks in the other threads which can execute FindRef and end up deadlocking // the map. // TL:DR Participant deletion needs to happen outside the TThreadSafeMap scope lock TSharedPtr ParticipantToDelete; if (!Participants->RemoveAndCopyValue(PlayerId, ParticipantToDelete) || !ParticipantToDelete.IsValid()) { return; } UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::DeletePlayerSession(Participant [{0}])", PlayerId); OnDataTrackClosed(PlayerId); // FIXME (RTCP-7928): EpicRtc currently isn't broadcasting a stopped track state for remote track OnAudioTrackClosed(PlayerId, true); OnVideoTrackClosed(PlayerId, true); if (ParticipantToDelete->VideoSource) { VideoSourceGroup->RemoveVideoSource(ParticipantToDelete->VideoSource.Get()); } DoOnGameThread([StreamerId = this->StreamerId, PlayerId, bIsEmpty = Participants->IsEmpty()]() { if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { Delegates->OnClosedConnection.Broadcast(StreamerId, PlayerId); Delegates->OnClosedConnectionNative.Broadcast(StreamerId, PlayerId); if (bIsEmpty) { Delegates->OnAllConnectionsClosed.Broadcast(StreamerId); Delegates->OnAllConnectionsClosedNative.Broadcast(StreamerId); } } }); if (FStats* PSStats = FStats::Get()) { PSStats->RemovePeerStats(PlayerId); } } void FEpicRtcStreamer::DeleteAllPlayerSessions() { if (FStats* PSStats = FStats::Get()) { PSStats->RemoveAllPeerStats(); } TSet PlayerIds; Participants->GetKeys(PlayerIds); // We have to iterate the keys separately as OnDataTrackClosed also loops through the player map // and we will deadlock for (const FString& PlayerId : PlayerIds) { DeletePlayerSession(PlayerId); } // Further cleanup VideoSourceGroup->RemoveAllVideoSources(); AudioTrackPlayerIdMap.Empty(); VideoTrackPlayerIdMap.Empty(); Participants->Empty(); InputControllingId = INVALID_PLAYER_ID; } void FEpicRtcStreamer::OnDataTrackOpen(FString PlayerId) { UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnDataTrackOpen(Participant [{0}])", PlayerId); // Only time we automatically make a new peer the input controlling host is if they are the first peer (and not the SFU). const bool HostControlsInput = GetEnumFromCVar(UPixelStreaming2PluginSettings::CVarInputController) == EInputControllerMode::Host; if (HostControlsInput && !IsSFU(PlayerId) && InputControllingId == INVALID_PLAYER_ID) { InputControllingId = PlayerId; } DoOnGameThread([StreamerId = this->StreamerId, PlayerId]() { if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { Delegates->OnDataTrackOpen.Broadcast(StreamerId, PlayerId); Delegates->OnDataTrackOpenNative.Broadcast(StreamerId, PlayerId); } }); // When data channel is open SendProtocol(PlayerId); // Try to send cached freeze frame (if we have one) FreezeFrame->SendCachedFreezeFrameTo(PlayerId); SendInitialSettings(PlayerId); SendPeerControllerMessages(PlayerId); } void FEpicRtcStreamer::OnDataTrackClosed(FString PlayerId) { UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnDataTrackClosed(Participant [{0}])", PlayerId); if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { Participant->DataTrack = nullptr; } if (InputControllingId == PlayerId) { InputControllingId = INVALID_PLAYER_ID; // just get the first channel we have and give it input control. Participants->ApplyUntil([this](FString PlayerId, TSharedPtr Participant) { if (!Participant->DataTrack) { return false; } if (IsSFU(PlayerId)) { return false; } InputControllingId = PlayerId; Participant->DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::InputControlOwnership, 1 /* ControlsInput */); return true; }); } DoOnGameThread([StreamerId = this->StreamerId, PlayerId]() { if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { Delegates->OnDataTrackClosed.Broadcast(StreamerId, PlayerId); Delegates->OnDataTrackClosedNative.Broadcast(StreamerId, PlayerId); } }); } void FEpicRtcStreamer::OnAudioTrackOpen(FString PlayerId, bool bIsRemote) { UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnAudioTrackOpen(Participant [{0}], IsRemote [{1}])", PlayerId, bIsRemote); if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { // NOTE: Native delegates aren't required to be broadcast on game thread Delegates->OnAudioTrackOpenNative.Broadcast(StreamerId, PlayerId, bIsRemote); } } void FEpicRtcStreamer::OnAudioTrackClosed(FString PlayerId, bool bIsRemote) { UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnAudioTrackClosed(Participant [{0}], IsRemote [{1}])", PlayerId, bIsRemote); if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { if (bIsRemote) { Participant->AudioSink = nullptr; } else { Participant->AudioSource = nullptr; } } if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { // NOTE: Native delegates aren't required to be broadcast on game thread Delegates->OnAudioTrackClosedNative.Broadcast(StreamerId, PlayerId, bIsRemote); } } void FEpicRtcStreamer::OnVideoTrackOpen(FString PlayerId, bool bIsRemote) { UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnVideoTrackOpen(Participant [{0}], IsRemote [{1}])", PlayerId, bIsRemote); if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { // NOTE: Native delegates aren't required to be broadcast on game thread Delegates->OnVideoTrackOpenNative.Broadcast(StreamerId, PlayerId, bIsRemote); } } void FEpicRtcStreamer::OnVideoTrackClosed(FString PlayerId, bool bIsRemote) { UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnVideoTrackClosed(Participant [{0}], IsRemote [{1}])", PlayerId, bIsRemote); if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { if (bIsRemote) { Participant->VideoSink = nullptr; } else { Participant->VideoSource = nullptr; } } if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { // NOTE: Native delegates aren't required to be broadcast on game thread Delegates->OnVideoTrackClosedNative.Broadcast(StreamerId, PlayerId, bIsRemote); } } void FEpicRtcStreamer::SendInitialSettings(FString PlayerId) const { const FString PixelStreaming2Payload = FString::Printf(TEXT("{ \"AllowPixelStreamingCommands\": %s, \"DisableLatencyTest\": %s }"), UPixelStreaming2PluginSettings::CVarInputAllowConsoleCommands.GetValueOnAnyThread() ? TEXT("true") : TEXT("false"), UPixelStreaming2PluginSettings::CVarDisableLatencyTester.GetValueOnAnyThread() ? TEXT("true") : TEXT("false")); const FString WebRTCPayload = FString::Printf(TEXT("{ \"FPS\": %d, \"MinBitrate\": %d, \"MaxBitrate\": %d }"), UPixelStreaming2PluginSettings::CVarWebRTCFps.GetValueOnAnyThread(), UPixelStreaming2PluginSettings::CVarWebRTCMinBitrate.GetValueOnAnyThread(), UPixelStreaming2PluginSettings::CVarWebRTCMaxBitrate.GetValueOnAnyThread()); const FString EncoderPayload = FString::Printf(TEXT("{ \"TargetBitrate\": %d, \"MinQuality\": %d, \"MaxQuality\": %d }"), UPixelStreaming2PluginSettings::CVarEncoderTargetBitrate.GetValueOnAnyThread(), UPixelStreaming2PluginSettings::CVarEncoderMinQuality.GetValueOnAnyThread(), UPixelStreaming2PluginSettings::CVarEncoderMaxQuality.GetValueOnAnyThread()); FString ConfigPayload = TEXT("{ "); bool bComma = false; // Simplest way to avoid complaints from pedantic JSON parsers for (const TPair& Option : ConfigOptions) { if (bComma) { ConfigPayload.Append(TEXT(", ")); } ConfigPayload.Append(FString::Printf(TEXT("\"%s\": \"%s\""), *Option.Key.ToString(), *Option.Value)); bComma = true; } ConfigPayload.Append(TEXT("}")); const FString FullPayload = FString::Printf(TEXT("{ \"PixelStreaming\": %s, \"Encoder\": %s, \"WebRTC\": %s, \"ConfigOptions\": %s }"), *PixelStreaming2Payload, *EncoderPayload, *WebRTCPayload, *ConfigPayload); TSharedPtr DataTrack; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { DataTrack = Participant->DataTrack; } if (!DataTrack) { return; } DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::InitialSettings, FullPayload); } void FEpicRtcStreamer::SendProtocol(FString PlayerId) const { const TArray> Protocols = { InputHandler->GetToStreamerProtocol(), InputHandler->GetFromStreamerProtocol() }; for (TSharedPtr Protocol : Protocols) { TSharedPtr ProtocolJson = Protocol->ToJson(); FString Body; TSharedRef> JsonWriter = TJsonWriterFactory<>::Create(&Body); if (!ensure(FJsonSerializer::Serialize(ProtocolJson.ToSharedRef(), JsonWriter))) { UE_LOG(LogPixelStreaming2RTC, Warning, TEXT("Cannot serialize protocol json object")); return; } TSharedPtr DataTrack; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { DataTrack = Participant->DataTrack; } if (!DataTrack) { return; } DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::Protocol, Body); } } void FEpicRtcStreamer::SendPeerControllerMessages(FString PlayerId) const { TSharedPtr DataTrack; { TSharedPtr Participant = Participants->FindRef(PlayerId); if (!Participant) { return; } DataTrack = Participant->DataTrack; } if (!DataTrack) { return; } const uint8 ControlsInput = (GetEnumFromCVar(UPixelStreaming2PluginSettings::CVarInputController) == EInputControllerMode::Host) ? (PlayerId == InputControllingId) : 1; // Even though the QualityController feature is removed we send it for backwards compatibility with older frontends (can probably remove 2 versions after 5.5) DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::InputControlOwnership, ControlsInput); DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::QualityControlOwnership, 1 /* True */); } void FEpicRtcStreamer::SendLatencyReport(FString PlayerId) const { if (UPixelStreaming2PluginSettings::CVarDisableLatencyTester.GetValueOnAnyThread()) { return; } double ReceiptTimeMs = FPlatformTime::ToMilliseconds64(FPlatformTime::Cycles64()); DoOnGameThread([this, PlayerId, ReceiptTimeMs]() { FString ReportToTransmitJSON; if (!UPixelStreaming2PluginSettings::CVarWebRTCDisableStats.GetValueOnAnyThread()) { double EncodeMs = -1.0; double CaptureToSendMs = 0.0; FStats* Stats = FStats::Get(); if (Stats) { Stats->QueryPeerStat(PlayerId, FName(*RTCStatCategories::LocalVideoTrack), PixelStreaming2StatNames::MeanEncodeTime, EncodeMs); Stats->QueryPeerStat(PlayerId, FName(*RTCStatCategories::LocalVideoTrack), PixelStreaming2StatNames::MeanSendDelay, CaptureToSendMs); } double TransmissionTimeMs = FPlatformTime::ToMilliseconds64(FPlatformTime::Cycles64()); ReportToTransmitJSON = FString::Printf( TEXT("{ \"ReceiptTimeMs\": %.2f, \"EncodeMs\": %.2f, \"CaptureToSendMs\": %.2f, \"TransmissionTimeMs\": %.2f }"), ReceiptTimeMs, EncodeMs, CaptureToSendMs, TransmissionTimeMs); } else { double TransmissionTimeMs = FPlatformTime::ToMilliseconds64(FPlatformTime::Cycles64()); ReportToTransmitJSON = FString::Printf( TEXT("{ \"ReceiptTimeMs\": %.2f, \"EncodeMs\": \"Pixel Streaming stats are disabled\", \"CaptureToSendMs\": \"Pixel Streaming stats are disabled\", \"TransmissionTimeMs\": %.2f }"), ReceiptTimeMs, TransmissionTimeMs); } TSharedPtr DataTrack; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { DataTrack = Participant->DataTrack; } if (!DataTrack) { return; } DataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::LatencyTest, ReportToTransmitJSON); }); } void FEpicRtcStreamer::HandleRelayStatusMessage(const uint8_t* Data, uint32_t Size, EpicRtcDataTrackInterface* DataTrack) { // skip type Data++; Size--; FString PlayerId = ReadString(Data, Size); checkf(Size > 0, TEXT("Malformed relay status message!")) bool bIsOn = static_cast(Data[0]); FString DataTrackId = ToString(DataTrack->GetId()); if (bIsOn) { UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::HandleRelayStatusMessage() Adding new PlayerId [%s] with DataTrackId [%s]"), *PlayerId, *DataTrackId); FString SFUId; if (FindPlayerFromTrack(DataTrack, SFUId)) { TSharedPtr SFUDataTrack; if (TSharedPtr SFUParticipant = Participants->FindRef(SFUId); SFUParticipant.IsValid()) { SFUDataTrack = SFUParticipant->DataTrack; } if (SFUDataTrack) { TSharedPtr& Participant = Participants->FindOrAdd(PlayerId); Participant = MakeShared(); Participant->DataTrack = FEpicRtcMutliplexDataTrack::Create(SFUDataTrack, InputHandler->GetFromStreamerProtocol(), PlayerId); OnDataTrackOpen(PlayerId); } } else { UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::HandleRelayStatusMessage() Failed to find SFU PlayerContext")); } } else { UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::HandleRelayStatusMessage() Removing PlayerId [%s] with DataTrackId [%s]"), *PlayerId, *DataTrackId); OnDataTrackClosed(PlayerId); Participants->Remove(PlayerId); } } void FEpicRtcStreamer::TriggerMouseLeave(FString InStreamerId) { if (!IsEngineExitRequested() && StreamerId == InStreamerId) { TSharedPtr SharedInputHandler = InputHandler; // Force a MouseLeave event. This prevents the PixelStreaming2ApplicationWrapper from // still wrapping the base FSlateApplication after we stop streaming const auto MouseLeaveFunction = [SharedInputHandler]() { if (SharedInputHandler.IsValid()) { TArray EmptyArray; TFunction MouseLeaveHandler = SharedInputHandler->FindMessageHandler("MouseLeave"); MouseLeaveHandler("", FMemoryReader(EmptyArray)); } }; if (IsInGameThread()) { MouseLeaveFunction(); } else { DoOnGameThread([MouseLeaveFunction]() { MouseLeaveFunction(); }); } } } void FEpicRtcStreamer::OnSessionStateUpdate(const EpicRtcSessionState State) { switch (State) { case EpicRtcSessionState::New: // Indicates newly created session. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionStateUpdate State=New"); break; } case EpicRtcSessionState::Pending: // Indicates connection is in progress. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionStateUpdate State=Pending"); break; } case EpicRtcSessionState::Connected: // Indicates session is connected to signalling server. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionStateUpdate State=Connected"); DoOnGameThread([StreamerId = this->StreamerId]() { if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { Delegates->OnConnectedToSignallingServer.Broadcast(StreamerId); Delegates->OnConnectedToSignallingServerNative.Broadcast(StreamerId); } }); const EpicRtcBitrate Bitrate = GetBitrates(); EpicRtcPortAllocator PortAllocator = { ._minPort = UPixelStreaming2PluginSettings::CVarWebRTCMinPort.GetValueOnAnyThread(), ._hasMinPort = true, ._maxPort = UPixelStreaming2PluginSettings::CVarWebRTCMaxPort.GetValueOnAnyThread(), ._hasMaxPort = true, ._portAllocation = static_cast(UPixelStreaming2PluginSettings::GetPortAllocationFlags()) }; EpicRtcConnectionConfig ConnectionConfig = { ._iceServers = { ._ptr = nullptr, ._size = 0 }, // This can stay empty because EpicRtc handles the ice servers internally ._portAllocator = PortAllocator, ._bitrate = Bitrate, ._iceConnectionPolicy = EpicRtcIcePolicy::All, ._disableTcpCandidates = false }; FUtf8String Utf8StreamerId(StreamerId); EpicRtcRoomConfig RoomConfig = { ._id = ToEpicRtcStringView(Utf8StreamerId), ._connectionConfig = ConnectionConfig, ._ticket = EpicRtcStringView{ ._ptr = nullptr, ._length = 0 }, ._observer = RoomObserver, ._audioTrackObserverFactory = AudioTrackObserverFactory, ._dataTrackObserverFactory = DataTrackObserverFactory, ._videoTrackObserverFactory = VideoTrackObserverFactory }; EpicRtcErrorCode Result = EpicRtcSession->CreateRoom(RoomConfig, EpicRtcRoom.GetInitReference()); if (Result != EpicRtcErrorCode::Ok) { UE_LOGFMT(LogPixelStreaming2RTC, Error, "Failed to create EpicRtc room. CreateRoom returned {0}", ToString(Result)); break; } // TODO (william.belcher): This should move to OnRoomStateUpdate(EpicRtcRoomState::New) once EpicRtc starts broadcasting that state EpicRtcRoom->Join(); break; } case EpicRtcSessionState::Disconnected: // Indicates session is disconnected from the signalling server. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionStateUpdate State=Disconnected"); // If the session unexpectedly disconnects (ie signalling server goes away), we should try and reconnect if (StreamState == EStreamState::Connected) { // Call stop streaming first. This will remove the room StopStreaming(); ReconnectTimer->Start(AsShared()); } DoOnGameThread([StreamerId = this->StreamerId]() { if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { Delegates->OnDisconnectedFromSignallingServer.Broadcast(StreamerId); Delegates->OnDisconnectedFromSignallingServerNative.Broadcast(StreamerId); } }); OnStreamingStopped().Broadcast(this); // The session has been disconnected (either through a call to StopStreaming or by an error) // so remove it if (EpicRtcConference.IsValid()) { FUtf8String Utf8StreamerId(StreamerId); EpicRtcConference->RemoveSession(ToEpicRtcStringView(Utf8StreamerId)); EpicRtcSession = nullptr; } // We are fully disconnect at this point so we update the state so we can StartStreaming again StreamState = EStreamState::Disconnected; break; } case EpicRtcSessionState::Failed: // Indicates session failed and is unusable. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionStateUpdate State=Failed"); break; } case EpicRtcSessionState::Exiting: // Indicates session has terminated without a result as a response to the application exiting. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionStateUpdate State=Exiting"); break; } default: { UE_LOGFMT(LogPixelStreaming2RTC, Error, "FEpicRtcStreamer::OnSessionStateUpdate An unhandled session state was encountered. This switch might be missing a case."); checkNoEntry(); // All cases should be handled break; } } } void FEpicRtcStreamer::OnSessionErrorUpdate(const EpicRtcErrorCode ErrorUpdate) { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionErrorUpdate does nothing"); } void FEpicRtcStreamer::OnSessionRoomsAvailableUpdate(EpicRtcStringArrayInterface* RoomsList) { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnSessionRoomsAvailableUpdate does nothing"); } void FEpicRtcStreamer::OnRoomStateUpdate(const EpicRtcRoomState State) { switch (State) { case EpicRtcRoomState::New: // Indicates newly created EpicRtcRoomInterface. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnRoomStateUpdate State=New"); break; } case EpicRtcRoomState::Pending: // Indicates join of the local participant is in progress. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnRoomStateUpdate State=Pending"); break; } case EpicRtcRoomState::Joined: // Indicates local participant (this streamer) is joined. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnRoomStateUpdate State=Joined"); StreamState = EStreamState::Connected; OnStreamingStarted().Broadcast(this); break; } case EpicRtcRoomState::Left: // Indicates local participant (this streamer) has left this EpicRtcRoomInterface. Room is not usable once in this state. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnRoomStateUpdate State=Left"); TRefCountPtr PinnedSession = EpicRtcSession; if (!PinnedSession) { break; } FUtf8String Utf8StreamerId(StreamerId); PinnedSession->RemoveRoom(ToEpicRtcStringView(Utf8StreamerId)); EpicRtcRoom = nullptr; const EpicRtcErrorCode Result = PinnedSession->Disconnect(ToEpicRtcStringView("Streaming Session Removed")); if (Result == EpicRtcErrorCode::SessionDisconnected) { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "Session disconnected cleanly."); } else if (Result != EpicRtcErrorCode::Ok) { UE_LOGFMT(LogPixelStreaming2RTC, Error, "Failed to disconnect EpicRtcSession. Disconnect returned {0}", *ToString(Result)); } break; } case EpicRtcRoomState::Failed: // Indicates EpicRtcRoomInterface failed and is unusable. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnRoomStateUpdate State=Failed"); break; } case EpicRtcRoomState::Exiting: // Indicates EpicRtcRoomInterface has terminated without a result as a response to the application exiting. { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnRoomStateUpdate State=Exiting"); break; } default: { UE_LOGFMT(LogPixelStreaming2RTC, Error, "FEpicRtcStreamer::OnRoomStateUpdate An unhandled room state was encountered. This switch might be missing a case."); checkNoEntry(); // All cases should be handled break; } } } FUtf8String GetAudioStreamID() { const bool bSyncVideoAndAudio = !UPixelStreaming2PluginSettings::CVarWebRTCDisableAudioSync.GetValueOnAnyThread(); return bSyncVideoAndAudio ? "pixelstreaming_av_stream_id" : "pixelstreaming_audio_stream_id"; } FUtf8String GetVideoStreamID() { const bool bSyncVideoAndAudio = !UPixelStreaming2PluginSettings::CVarWebRTCDisableAudioSync.GetValueOnAnyThread(); return bSyncVideoAndAudio ? "pixelstreaming_av_stream_id" : "pixelstreaming_video_stream_id"; } void FEpicRtcStreamer::OnRoomJoinedUpdate(EpicRtcParticipantInterface* ParticipantInterface) { const FString ParticipantId = ToString(ParticipantInterface->GetId()); UE_LOG(LogPixelStreaming2RTC, Log, TEXT("Player (%s) joined"), *ParticipantId); if (ParticipantId == StreamerId) { return; } DoOnGameThread([StreamerId = this->StreamerId, ParticipantId]() { if (UPixelStreaming2Delegates* Delegates = UPixelStreaming2Delegates::Get()) { Delegates->OnNewConnection.Broadcast(StreamerId, ParticipantId); Delegates->OnNewConnectionNative.Broadcast(StreamerId, ParticipantId); } }); TSharedPtr& Participant = Participants->FindOrAdd(ParticipantId); Participant = MakeShared(); Participant->ParticipantInterface = ParticipantInterface; Participant->StatsCollector = FRTCStatsCollector::Create(ParticipantId); TRefCountPtr ParticipantConnection = ParticipantInterface->GetConnection(); ParticipantConnection->SetManualNegotiation(true); const EVideoCodec SelectedCodec = GetEnumFromCVar(UPixelStreaming2PluginSettings::CVarEncoderCodec); const bool bNegotiateCodecs = UPixelStreaming2PluginSettings::CVarWebRTCNegotiateCodecs.GetValueOnAnyThread(); const bool bTransmitUEVideo = !UPixelStreaming2PluginSettings::CVarWebRTCDisableTransmitVideo.GetValueOnAnyThread(); bool bReceiveBrowserVideo = !UPixelStreaming2PluginSettings::CVarWebRTCDisableReceiveVideo.GetValueOnAnyThread(); // Check if the user has selected only H.264 on a AMD gpu and disable receiving video. // WebRTC does not support using SendRecv if the encoding and decoding do not support the same codec. // AMD GPUs currently have decoding disabled so WebRTC fails to create SDP codecs with SendRecv. // TODO (Eden.Harris) RTCP-8039: This workaround won't be needed once H.264 decoding is enabled with AMD GPUs. if (IsRHIDeviceAMD() && (bNegotiateCodecs || (!bNegotiateCodecs && SelectedCodec == EVideoCodec::H264))) { if (bReceiveBrowserVideo) { bReceiveBrowserVideo = false; UE_LOGFMT(LogPixelStreaming2RTC, Warning, "AMD GPUs do not support receiving H.264 video."); } } if (bTransmitUEVideo || bReceiveBrowserVideo) { TArray VideoEncodingConfigs; // We need ensure the Rids have the same lifetime as the VideoEncodingConfigs // to ensure the contents don't get deleted before we can call AddVideoSource TArray Rids; int MaxFramerate = UPixelStreaming2PluginSettings::CVarWebRTCFps.GetValueOnAnyThread(); // WebRTC cannot set the bitrate outside the first initial biterate set by the VideoEncodingConfig. // By setting a high value here, the real value can be set by SetConnectionRates which can fit within this range. // Without this, changing the max bitrate at runtime will be capped at the initial max bitrate. // SetConnectionRates below will set the real max bitrate. constexpr uint32 InitialMinBitrate = 1'000; constexpr uint32 InitialMaxBitrate = 1'000'000'000; TArray SimulcastParams = UE::PixelStreaming2::GetSimulcastParameters(); if (IsSFU(ParticipantId) && SimulcastParams.Num() > 1) { for (int i = 0; i < SimulcastParams.Num(); ++i) { const FPixelStreaming2SimulcastLayer& SpatialLayer = SimulcastParams[i]; Rids.Add(FUtf8String("simulcast") + FUtf8String::FromInt(SimulcastParams.Num() - i)); EpicRtcVideoEncodingConfig VideoEncodingConfig = { // clang-format off ._rid = EpicRtcStringView{._ptr = (const char*)(*(Rids[i])), ._length = static_cast(Rids[i].Len()) }, // clang-format on ._scaleResolutionDownBy = SpatialLayer.Scaling, ._scalabilityMode = static_cast(GetEnumFromCVar(UPixelStreaming2PluginSettings::CVarEncoderScalabilityMode)), // HACK if the Enums become un-aligned ._minBitrate = static_cast(SpatialLayer.MinBitrate), ._maxBitrate = static_cast(SpatialLayer.MaxBitrate), ._maxFrameRate = static_cast(MaxFramerate) }; VideoEncodingConfigs.Add(VideoEncodingConfig); } } else { // Default video config for P2P or if simulcast disabled EpicRtcVideoEncodingConfig VideoEncodingConfig = { // clang-format off // TODO (Migration): RTCP-7027 Maybe bug in EpicRtc? Setting an rid if there's only one config results in no video ._rid = EpicRtcStringView{._ptr = nullptr, ._length = 0 }, // clang-format on ._scaleResolutionDownBy = 1.f, ._scalabilityMode = static_cast(GetEnumFromCVar(UPixelStreaming2PluginSettings::CVarEncoderScalabilityMode)), // HACK if the Enums become un-aligned ._minBitrate = InitialMinBitrate, ._maxBitrate = InitialMaxBitrate, ._maxFrameRate = static_cast(MaxFramerate) }; VideoEncodingConfigs.Add(VideoEncodingConfig); } EpicRtcVideoEncodingConfigSpan VideoEncodingConfigSpan = { ._ptr = VideoEncodingConfigs.GetData(), ._size = (uint64_t)VideoEncodingConfigs.Num() }; EpicRtcMediaSourceDirection VideoDirection; if (bTransmitUEVideo && bReceiveBrowserVideo) { VideoDirection = EpicRtcMediaSourceDirection::SendRecv; } else if (bTransmitUEVideo) { VideoDirection = EpicRtcMediaSourceDirection::SendOnly; } else if (bReceiveBrowserVideo) { VideoDirection = EpicRtcMediaSourceDirection::RecvOnly; } else { VideoDirection = EpicRtcMediaSourceDirection::RecvOnly; } FUtf8String VideoStreamID = GetVideoStreamID(); EpicRtcVideoSource VideoSource = { ._streamId = ToEpicRtcStringView(VideoStreamID), ._encodings = VideoEncodingConfigSpan, ._direction = VideoDirection }; ParticipantConnection->AddVideoSource(VideoSource); } const bool bTransmitUEAudio = !UPixelStreaming2PluginSettings::CVarWebRTCDisableTransmitAudio.GetValueOnAnyThread(); const bool bReceiveBrowserAudio = !UPixelStreaming2PluginSettings::CVarWebRTCDisableReceiveAudio.GetValueOnAnyThread(); if (bTransmitUEAudio || bReceiveBrowserAudio) { EpicRtcMediaSourceDirection AudioDirection; if (bTransmitUEAudio && bReceiveBrowserAudio) { AudioDirection = EpicRtcMediaSourceDirection::SendRecv; } else if (bTransmitUEAudio) { AudioDirection = EpicRtcMediaSourceDirection::SendOnly; } else if (bReceiveBrowserAudio) { AudioDirection = EpicRtcMediaSourceDirection::RecvOnly; } else { AudioDirection = EpicRtcMediaSourceDirection::RecvOnly; } FUtf8String AudioStreamID = GetAudioStreamID(); EpicRtcAudioSource AudioSource = { ._streamId = ToEpicRtcStringView(AudioStreamID), ._bitrate = 510000, ._channels = 2, ._direction = AudioDirection }; ParticipantConnection->AddAudioSource(AudioSource); } if (IsSFU(ParticipantId)) { FString RecvLabel(TEXT("recv-datachannel")); FUtf8String Utf8RecvLabel = *RecvLabel; EpicRtcDataSource RecvDataSource = { ._label = ToEpicRtcStringView(Utf8RecvLabel), ._maxRetransmitTime = 0, ._maxRetransmits = 0, ._isOrdered = true, ._protocol = EpicRtcDataSourceProtocol::Sctp, ._negotiated = true, ._transportChannelId = 1 }; ParticipantConnection->AddDataSource(RecvDataSource); FString SendLabel(TEXT("send-datachannel")); FUtf8String Utf8SendLabel = *SendLabel; EpicRtcDataSource SendDataSource = { ._label = ToEpicRtcStringView(Utf8RecvLabel), ._maxRetransmitTime = 0, ._maxRetransmits = 0, ._isOrdered = true, ._protocol = EpicRtcDataSourceProtocol::Sctp, ._negotiated = true, ._transportChannelId = 0 }; ParticipantConnection->AddDataSource(SendDataSource); } else { EpicRtcDataSource DataSource = { ._label = ParticipantInterface->GetId(), ._maxRetransmitTime = 0, ._maxRetransmits = 0, ._isOrdered = true, ._protocol = EpicRtcDataSourceProtocol::Sctp }; ParticipantConnection->AddDataSource(DataSource); } ParticipantConnection->SetConnectionRates(GetBitrates()); ParticipantConnection->StartNegotiation(); } void FEpicRtcStreamer::OnRoomLeftUpdate(const EpicRtcStringView Participant) { FString ParticipantId = ToString(Participant); UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnRoomLeftUpdate(Participant [%s] left the room.)"), *ParticipantId); // Remove the player DeletePlayerSession(ParticipantId); } void FEpicRtcStreamer::OnAudioTrackUpdate(EpicRtcParticipantInterface* ParticipantInterface, EpicRtcAudioTrackInterface* AudioTrack) { const FString ParticipantId = ToString(ParticipantInterface->GetId()); const FString AudioTrackId = ToString(AudioTrack->GetId()); const bool bIsRemote = static_cast(AudioTrack->IsRemote()); UE_LOGFMT(LogPixelStreaming2RTC, Log, "FEpicRtcStreamer::OnAudioTrackUpdate(Participant [{0}], AudioTrack [{1}, IsRemote [{2}]])", ParticipantId, AudioTrackId, bIsRemote); AudioTrackPlayerIdMap.Add(reinterpret_cast(AudioTrack), ParticipantId); if (TSharedPtr Participant = Participants->FindRef(ParticipantId); Participant.IsValid()) { if (bIsRemote) { Participant->AudioSink = FEpicRtcAudioSink::Create(AudioTrack); } else { Participant->AudioSource = FEpicRtcAudioSource::Create(AudioTrack, FPixelStreaming2RTCModule::GetModule()->GetAudioCapturer()); } } OnAudioTrackOpen(ParticipantId, bIsRemote); } void FEpicRtcStreamer::OnVideoTrackUpdate(EpicRtcParticipantInterface* ParticipantInterface, EpicRtcVideoTrackInterface* VideoTrack) { const FString ParticipantId = ToString(ParticipantInterface->GetId()); const FString VideoTrackId = ToString(VideoTrack->GetId()); const bool bIsRemote = static_cast(VideoTrack->IsRemote()); UE_LOGFMT(LogPixelStreaming2RTC, Log, "FEpicRtcStreamer::OnVideoTrackUpdate(Participant [{0}], VideoTrack [{1}], IsRemote[{2}])", ParticipantId, VideoTrackId, bIsRemote); VideoTrackPlayerIdMap.Add(reinterpret_cast(VideoTrack), ParticipantId); if (TSharedPtr Participant = Participants->FindRef(ParticipantId); Participant.IsValid()) { if (bIsRemote) { Participant->VideoSink = FEpicRtcVideoSink::Create(VideoTrack); } else { Participant->VideoSource = FEpicRtcVideoSource::Create(VideoTrack, VideoCapturer, VideoSourceGroup); } } OnVideoTrackOpen(ParticipantId, bIsRemote); } void FEpicRtcStreamer::OnDataTrackUpdate(EpicRtcParticipantInterface* ParticipantInterface, EpicRtcDataTrackInterface* DataTrack) { FString ParticipantId = ToString(ParticipantInterface->GetId()); FString DataTrackId = ToString(DataTrack->GetId()); UE_LOGFMT(LogPixelStreaming2RTC, Log, "FEpicRtcStreamer::OnDataTrackUpdate(Participant [{0}], DataTrack [{1}])", ParticipantId, DataTrackId); if (TSharedPtr Participant = Participants->FindRef(ParticipantId); Participant.IsValid()) { if (!Participant->DataTrack) { Participant->DataTrack = FEpicRtcDataTrack::Create(DataTrack, InputHandler->GetFromStreamerProtocol()); } else { Participant->DataTrack->SetSendTrack(DataTrack); } } } [[nodiscard]] EpicRtcSdpInterface* FEpicRtcStreamer::OnLocalSdpUpdate(EpicRtcParticipantInterface* Participant, EpicRtcSdpInterface* Sdp) { FString ParticipantId = ToString(Participant->GetId()); FString SdpType = TEXT(""); switch (Sdp->GetType()) { case EpicRtcSdpType::Offer: SdpType = TEXT("Offer"); break; case EpicRtcSdpType::Answer: SdpType = TEXT("Answer"); break; } UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnLocalSdpUpdate(Participant [%s], Type [%s])"), *ParticipantId, *SdpType); return nullptr; } [[nodiscard]] EpicRtcSdpInterface* FEpicRtcStreamer::OnRemoteSdpUpdate(EpicRtcParticipantInterface* Participant, EpicRtcSdpInterface* Sdp) { FString ParticipantId = ToString(Participant->GetId()); FString SdpType = TEXT(""); switch (Sdp->GetType()) { case EpicRtcSdpType::Offer: SdpType = TEXT("Offer"); break; case EpicRtcSdpType::Answer: SdpType = TEXT("Answer"); break; } UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnRemoteSdpUpdate(Participant [%s], Type [%s])"), *ParticipantId, *SdpType); return nullptr; } void FEpicRtcStreamer::OnRoomErrorUpdate(const EpicRtcErrorCode Error) { UE_LOGFMT(LogPixelStreaming2RTC, VeryVerbose, "FEpicRtcStreamer::OnRoomErrorUpdate does nothing"); } void FEpicRtcStreamer::OnAudioTrackMuted(EpicRtcAudioTrackInterface* AudioTrack, EpicRtcBool bIsMuted) { FString PlayerId; bool bFoundPlayer = FindPlayerFromTrack(AudioTrack, PlayerId); FString AudioTrackId = ToString(AudioTrack->GetId()); UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnAudioTrackMuted(AudioTrack [%s], bIsMuted[%s], PlayerId[%s])"), *AudioTrackId, bIsMuted ? TEXT("True") : TEXT("False"), *PlayerId); if (!bFoundPlayer) { UE_LOG(LogPixelStreaming2RTC, Warning, TEXT("FEpicRtcStreamer::OnAudioTrackMuted(Failed to find a player for audio track [%s])"), *AudioTrackId); return; } if (AudioTrack->IsRemote()) { TSharedPtr AudioSink; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { AudioSink = Participant->AudioSink; } if (AudioSink) { AudioSink->SetMuted(static_cast(bIsMuted)); } } else { TSharedPtr AudioSource; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { AudioSource = Participant->AudioSource; } if (AudioSource) { AudioSource->SetMuted(static_cast(bIsMuted)); } } } void FEpicRtcStreamer::OnAudioTrackFrame(EpicRtcAudioTrackInterface* AudioTrack, const EpicRtcAudioFrame& Frame) { FString PlayerId; bool bFoundPlayer = FindPlayerFromTrack(AudioTrack, PlayerId); FString AudioTrackId = ToString(AudioTrack->GetId()); if (!bFoundPlayer) { UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnAudioTrackFrame(Failed to find a player for audio track [%s])"), *AudioTrackId); return; } TSharedPtr AudioSink; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { AudioSink = Participant->AudioSink; } if (AudioSink) { AudioSink->OnAudioData(Frame._data, Frame._length, Frame._format._numChannels, Frame._format._sampleRate); } } void FEpicRtcStreamer::OnAudioTrackRemoved(EpicRtcAudioTrackInterface* AudioTrack) { // TODO (Eden.Harris) OnAudioTrackRemoved is not hooked up in EpicRtc. UE_LOGFMT(LogPixelStreaming2RTC, Warning, "FEpicRtcStreamer::OnAudioTrackRemoved was called. Use OnAudioTrackState with State [EpicRtcTrackState::Stopped]"); } void FEpicRtcStreamer::OnAudioTrackState(EpicRtcAudioTrackInterface* AudioTrack, const EpicRtcTrackState State) { FString PlayerId; const bool bFoundPlayer = FindPlayerFromTrack(AudioTrack, PlayerId); const bool bIsRemote = static_cast(AudioTrack->IsRemote()); const FString AudioTrackId = ToString(AudioTrack->GetId()); if (!bFoundPlayer) { // Not finding a player is expected as OnAudioTrackState will happen for OnAudioTrackUpdate during track addition UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnAudioTrackState(Cannot to find a player for audio track [{0}])", AudioTrackId); return; } UE_LOGFMT(LogPixelStreaming2RTC, Log, "FEpicRtcStreamer::OnAudioTrackState(AudioTrack=[{0}], Player=[{1}], State=[{2}])", AudioTrackId, PlayerId, ToString(State)); if (State == EpicRtcTrackState::Stopped) { AudioTrackPlayerIdMap.Remove(reinterpret_cast(AudioTrack)); OnAudioTrackClosed(PlayerId, bIsRemote); } } void FEpicRtcStreamer::OnVideoTrackMuted(EpicRtcVideoTrackInterface* VideoTrack, EpicRtcBool bIsMuted) { FString PlayerId; bool bFoundPlayer = FindPlayerFromTrack(VideoTrack, PlayerId); FString VideoTrackId = ToString(VideoTrack->GetId()); UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnVideoTrackMuted(VideoTrack [%s], bIsMuted[%s], PlayerId[%s])"), *VideoTrackId, bIsMuted ? TEXT("True") : TEXT("False"), *PlayerId); if (!bFoundPlayer) { UE_LOG(LogPixelStreaming2RTC, Warning, TEXT("FEpicRtcStreamer::OnVideoTrackMuted(Failed to find a player for video track [%s])"), *VideoTrackId); return; } if (VideoTrack->IsRemote()) { TSharedPtr VideoSink; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { VideoSink = Participant->VideoSink; } if (VideoSink) { VideoSink->SetMuted(static_cast(bIsMuted)); } } else { TSharedPtr VideoSource; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { VideoSource = Participant->VideoSource; } if (VideoSource) { VideoSource->SetMuted(static_cast(bIsMuted)); } } } void FEpicRtcStreamer::OnVideoTrackFrame(EpicRtcVideoTrackInterface* VideoTrack, const EpicRtcVideoFrame& Frame) { FString PlayerId; bool bFoundPlayer = FindPlayerFromTrack(VideoTrack, PlayerId); FString VideoTrackId = ToString(VideoTrack->GetId()); if (!bFoundPlayer) { UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnVideoTrackFrame(Failed to find a player for video track [%s])"), *VideoTrackId); return; } TSharedPtr VideoSink; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { VideoSink = Participant->VideoSink; } if (VideoSink) { VideoSink->OnEpicRtcFrame(Frame); } } void FEpicRtcStreamer::OnVideoTrackRemoved(EpicRtcVideoTrackInterface* VideoTrack) { // TODO (Eden.Harris) OnVideoTrackRemoved is not hooked up in EpicRtc. UE_LOGFMT(LogPixelStreaming2RTC, Warning, "FEpicRtcStreamer::OnVideoTrackRemoved was called. Use OnVideoTrackState with State [EpicRtcTrackState::Stopped]"); } void FEpicRtcStreamer::OnVideoTrackState(EpicRtcVideoTrackInterface* VideoTrack, const EpicRtcTrackState State) { FString PlayerId; const bool bFoundPlayer = FindPlayerFromTrack(VideoTrack, PlayerId); const bool bIsRemote = static_cast(VideoTrack->IsRemote()); const FString VideoTrackId = ToString(VideoTrack->GetId()); if (!bFoundPlayer) { // Not finding a player is expected as OnVideoTrackState will happen for OnVideoTrackUpdate during track addition UE_LOGFMT(LogPixelStreaming2RTC, Verbose, "FEpicRtcStreamer::OnVideoTrackState(Cannot to find a player for video track [{0}])", VideoTrackId); return; } UE_LOGFMT(LogPixelStreaming2RTC, Log, "FEpicRtcStreamer::OnVideoTrackState(VideoTrack=[{0}], Player=[{1}], State=[{2}])", VideoTrackId, PlayerId, ToString(State)); if (State == EpicRtcTrackState::Stopped) { VideoTrackPlayerIdMap.Remove(reinterpret_cast(VideoTrack)); OnVideoTrackClosed(PlayerId, bIsRemote); } } void FEpicRtcStreamer::OnVideoTrackEncodedFrame(EpicRtcVideoTrackInterface* VideoTrack, const EpicRtcEncodedVideoFrame& EncodedFrame) { } EpicRtcBool FEpicRtcStreamer::Enabled() const { return true; } void FEpicRtcStreamer::OnDataTrackState(EpicRtcDataTrackInterface* DataTrack, const EpicRtcTrackState State) { FString PlayerId; bool bFoundPlayer = FindPlayerFromTrack(DataTrack, PlayerId); FString DataTrackId = ToString(DataTrack->GetId()); if (!bFoundPlayer) { UE_LOGFMT(LogPixelStreaming2RTC, Warning, "FEpicRtcStreamer::OnDataTrackState(Failed to find a player for data track [{0}])", DataTrackId); return; } UE_LOGFMT(LogPixelStreaming2RTC, Log, "FEpicRtcStreamer::OnDataTrackState(DataTrack=[{0}], Player=[{1}], State=[{2}])", DataTrackId, PlayerId, ToString(State)); if (State == EpicRtcTrackState::Active) { OnDataTrackOpen(PlayerId); } else if (State == EpicRtcTrackState::Stopped) { OnDataTrackClosed(PlayerId); } } void FEpicRtcStreamer::OnDataTrackMessage(EpicRtcDataTrackInterface* DataTrack) { FString DataTrackId = ToString(DataTrack->GetId()); TRefCountPtr DataFrame; if (!DataTrack->PopFrame(DataFrame.GetInitReference())) { UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnDataTrackMessage(Failed to PopFrame [%s])"), *DataTrackId); return; } FString PlayerId; const uint8_t* Data = DataFrame->Data(); uint32_t DataSize = DataFrame->Size(); uint8 Type = Data[0]; TSharedPtr ToStreamerProtocol = InputHandler->GetToStreamerProtocol(); if (Type == ToStreamerProtocol->Find(EPixelStreaming2ToStreamerMessage::Multiplexed)->GetID()) { // skip type Data++; DataSize--; PlayerId = ReadString(Data, DataSize); Type = Data[0]; UE_LOG(LogPixelStreaming2RTC, VeryVerbose, TEXT("FEpicRtcStreamer::OnDataTrackMessage(Received multiplexed message of type [%d] with PlayerId [%s])"), Type, *PlayerId); } else if (Type == ToStreamerProtocol->Find(EPixelStreaming2ToStreamerMessage::ChannelRelayStatus)->GetID()) { HandleRelayStatusMessage(Data, DataSize, DataTrack); return; } else if (!FindPlayerFromTrack(DataTrack, PlayerId)) { UE_LOG(LogPixelStreaming2RTC, Log, TEXT("FEpicRtcStreamer::OnDataTrackMessage(Failed to find a player for data track [%s])"), *DataTrackId); return; } if (Type == ToStreamerProtocol->Find(EPixelStreaming2ToStreamerMessage::LatencyTest)->GetID()) { SendLatencyReport(PlayerId); } else if (Type == ToStreamerProtocol->Find(EPixelStreaming2ToStreamerMessage::RequestInitialSettings)->GetID()) { SendInitialSettings(PlayerId); } else if (Type == ToStreamerProtocol->Find(EPixelStreaming2ToStreamerMessage::IFrameRequest)->GetID()) { ForceKeyFrame(); } else if (Type == ToStreamerProtocol->Find(EPixelStreaming2ToStreamerMessage::TestEcho)->GetID()) { TSharedPtr ParticipantDataTrack; if (TSharedPtr Participant = Participants->FindRef(PlayerId); Participant.IsValid()) { ParticipantDataTrack = Participant->DataTrack; } if (ParticipantDataTrack) { const size_t DescriptorSize = (DataSize - 1) / sizeof(TCHAR); const TCHAR* DescPtr = reinterpret_cast(Data + 1); const FString Message(DescriptorSize, DescPtr); ParticipantDataTrack->SendMessage(EPixelStreaming2FromStreamerMessage::TestEcho, Message); } } else if (!IsEngineExitRequested()) { // If we are in "Host" mode and the current peer is not the host, then discard this input. if (GetEnumFromCVar(UPixelStreaming2PluginSettings::CVarInputController) == EInputControllerMode::Host && InputControllingId != PlayerId) { return; } TArray MessageData(Data, DataSize); if (InputHandler) { InputHandler->OnMessage(PlayerId, MessageData); } } } void FEpicRtcStreamer::OnDataTrackError(EpicRtcDataTrackInterface* DataTrack, const EpicRtcErrorCode Error) { } FString FRTCStreamerFactory::GetStreamType() { return RTC_STREAM_TYPE; } FRTCStreamerFactory::FRTCStreamerFactory(TRefCountPtr Conference) : EpicRtcConference(MoveTemp(Conference)) { } TSharedPtr FRTCStreamerFactory::CreateNewStreamer(const FString& StreamerId) { TSharedPtr NewStreamer = MakeShared(StreamerId, EpicRtcConference); // default to the scene viewport if we have a game engine if (UGameEngine* GameEngine = Cast(GEngine)) { TSharedPtr TargetWindow = GameEngine->GameViewport->GetWindow(); TSharedPtr InputHandler = NewStreamer->GetInputHandler().Pin(); if (TargetWindow.IsValid() && InputHandler.IsValid()) { InputHandler->SetTargetWindow(TargetWindow); } else { UE_LOG(LogPixelStreaming2RTC, Error, TEXT("Cannot set target window - target window is not valid.")); } } // If the user hasn't specified a connection url on the command line or in the ini, don't set // the video producer in order to not tax their GPU unnecessarily if (!UPixelStreaming2PluginSettings::CVarConnectionURL.GetValueOnAnyThread().IsEmpty()) { if (UPixelStreaming2PluginSettings::CVarUseMediaCapture.GetValueOnAnyThread()) { NewStreamer->SetVideoProducer(FVideoProducerMediaCapture::CreateActiveViewportCapture()); } else { NewStreamer->SetVideoProducer(FVideoProducerBackBuffer::Create()); } } return NewStreamer; } } // namespace UE::PixelStreaming2