// Copyright Epic Games, Inc. All Rights Reserved. #include "EpicRtcVideoSink.h" #include "Async/Async.h" #include "ColorConversion.h" #include "EpicRtcVideoBufferI420.h" #include "EpicRtcVideoBufferRHI.h" #include "IPixelCaptureOutputFrame.h" #include "Logging.h" #include "PixelCaptureInputFrameI420.h" #include "PixelCaptureInputFrameRHI.h" #include "PixelCaptureOutputFrameRHI.h" #include "PixelStreaming2Trace.h" #include "RenderTargetPool.h" #include "Stats.h" namespace UE::PixelStreaming2 { TSharedPtr FEpicRtcVideoSink::Create(TRefCountPtr InTrack) { TSharedPtr VideoSink = MakeShareable(new FEpicRtcVideoSink(InTrack)); VideoSink->VideoCapturer->OnFrameCaptured.AddSP(VideoSink.ToSharedRef(), &FEpicRtcVideoSink::OnFrameCaptured); return VideoSink; } FEpicRtcVideoSink::FEpicRtcVideoSink(TRefCountPtr InTrack) : TEpicRtcTrack(InTrack) , VideoCapturer(FVideoCapturer::Create(nullptr, true)) { } void FEpicRtcVideoSink::OnEpicRtcFrame(const EpicRtcVideoFrame& Frame) { if (!HasVideoConsumers() || bIsMuted || IsEngineExitRequested()) { return; } TRACE_CPUPROFILER_EVENT_SCOPE_ON_CHANNEL_STR("FEpicRtcVideoSink::OnEpicRtcFrame", PixelStreaming2Channel); int32_t Width = Frame._buffer->GetWidth(); int32_t Height = Frame._buffer->GetHeight(); if (Frame._buffer->GetFormat() != EpicRtcPixelFormat::Native) { UE_LOGFMT(LogPixelStreaming2RTC, Error, "Received an EpicRtcVideoFrame that doesn't have a native buffer!"); return; } FEpicRtcVideoBuffer* const BaseFrameBuffer = static_cast(Frame._buffer); if (!BaseFrameBuffer) { return; } if (BaseFrameBuffer->GetBufferFormat() == PixelCaptureBufferFormat::FORMAT_RHI) { FEpicRtcVideoBufferRHI* const FrameBuffer = static_cast(Frame._buffer); if (FrameBuffer == nullptr) { return; } TSharedPtr VideoResource = FrameBuffer->GetVideoResource(); if (VideoResource->GetFormat() != EVideoFormat::BGRA) { VideoResource = VideoResource->TransformResource(FVideoDescriptor(EVideoFormat::BGRA, Width, Height)); } TWeakPtr WeakSink = AsWeak(); ENQUEUE_RENDER_COMMAND(CaptureDecodedFrameCommand) ([WeakSink, VideoResource](FRHICommandList& RHICmdList) { if (TSharedPtr PinnedSink = WeakSink.Pin()) { auto& Raw = VideoResource->GetRaw(); PinnedSink->VideoCapturer->OnFrame(FPixelCaptureInputFrameRHI(Raw.Texture)); } }); } else if (BaseFrameBuffer->GetBufferFormat() == PixelCaptureBufferFormat::FORMAT_I420) { FEpicRtcVideoBufferI420* const FrameBuffer = static_cast(Frame._buffer); if (FrameBuffer == nullptr) { return; } TSharedPtr I420Buffer = FrameBuffer->GetBuffer(); TWeakPtr WeakSink = AsWeak(); ENQUEUE_RENDER_COMMAND(CaptureDecodedFrameCommand) ([WeakSink, I420Buffer](FRHICommandList& RHICmdList) { if (TSharedPtr PinnedSink = WeakSink.Pin()) { PinnedSink->VideoCapturer->OnFrame(FPixelCaptureInputFrameI420(I420Buffer)); } }); } } void FEpicRtcVideoSink::OnFrameCaptured() { TWeakPtr WeakSink = AsWeak(); ENQUEUE_RENDER_COMMAND(DisplayCapturedFrameCommand) ([WeakSink](FRHICommandList& RHICmdList) { if (TSharedPtr PinnedSink = WeakSink.Pin()) { TSharedPtr OutputFrame = PinnedSink->VideoCapturer->RequestFormat(PixelCaptureBufferFormat::FORMAT_RHI); if (!OutputFrame) { return; } TSharedPtr RHIFrame = StaticCastSharedPtr(OutputFrame); if (!RHIFrame->GetFrameTexture()) { return; } OutputFrame->Metadata.UseCount++; if (OutputFrame->Metadata.UseCount == 1) { OutputFrame->Metadata.ProcessName = FString::Printf(TEXT("VideoSink %s"), *(OutputFrame->Metadata.ProcessName)); } FStats::Get()->AddFrameTimingStats(OutputFrame->Metadata); PinnedSink->OnVideoData(RHIFrame->GetFrameTexture()); } }); } } // namespace UE::PixelStreaming2