// Copyright Epic Games, Inc. All Rights Reserved. #include "MediaRecorder.h" #include "Async/Async.h" #include "Containers/UnrealString.h" #include "HAL/FileManager.h" #include "IImageWrapper.h" #include "IImageWrapperModule.h" #include "ImagePixelData.h" #include "ImageWriteQueue.h" #include "IMediaClock.h" #include "IMediaClockSink.h" #include "IMediaModule.h" #include "Math/UnrealMathUtility.h" #include "MediaUtilsPrivate.h" #include "Modules/ModuleManager.h" #include "Serialization/Archive.h" #include "MediaPlayerFacade.h" /* Local helpers *****************************************************************************/ /** * Media clock sink for media textures. */ class FMediaRecorderClockSink : public IMediaClockSink { public: FMediaRecorderClockSink(FMediaRecorder& InOwner) : Owner(InOwner) { } virtual ~FMediaRecorderClockSink() { } public: virtual void TickOutput(FTimespan DeltaTime, FTimespan Timecode) override { Owner.TickRecording(); } private: FMediaRecorder& Owner; }; /** * ImagePixelData for TextureSample. * Can only be used when Stride == Dim.X*"Number of channels" */ struct FMediaImagePixelData : FImagePixelData { FMediaImagePixelData(TSharedPtr& InSample , const FIntPoint& InSize , EImagePixelType InPixelType , ERGBFormat InPixelLayout , uint8 InBitDepth , uint8 InNumChannels) : FImagePixelData(InSize, InPixelType, InPixelLayout, InBitDepth, InNumChannels, nullptr) , Sample(InSample) { } TSharedPtr Sample; virtual TUniquePtr Move() override { return MakeUnique(MoveTemp(*this)); } virtual TUniquePtr Copy() const override { return MakeUnique(*this); } virtual void RetrieveData(const void*& OutDataPtr, int64& OutSizeBytes) const override { OutDataPtr = Sample->GetBuffer(); OutSizeBytes = int64(Sample->GetStride()) * int64(Sample->GetDim().Y); } }; /* MediaRecorderHelpers namespace *****************************************************************************/ namespace MediaRecorderHelpers { template TUniquePtr> CreatePixelData(TSharedPtr InSample, const FIntPoint InSize, int32 InNumChannels) { const int64 NumberOfTexel = InSize.Y * InSize.X; TUniquePtr> PixelData = MakeUnique>(InSize); PixelData->Pixels.Reset(NumberOfTexel); const void* Buffer = InSample->GetBuffer(); const uint64 Stride = InSample->GetStride(); if (NumberOfTexel == (Stride *InSize.Y) / InNumChannels) { PixelData->Pixels.Append(reinterpret_cast(Buffer), NumberOfTexel); } else { for (int64 IndexY = 0; IndexY < InSize.Y; ++IndexY) { PixelData->Pixels.Append(reinterpret_cast(reinterpret_cast(Buffer) + (Stride*IndexY)), InSize.X); } } return MoveTemp(PixelData); }; } /* FMediaRecorder implementation *****************************************************************************/ FMediaRecorder::FMediaRecorder() : bRecording(false) , bUnsupportedWarningShowed(false) , FrameCount(0) , NumerationStyle(EMediaRecorderNumerationStyle::AppendSampleTime) , bSetAlpha(false) , CompressionQuality(0) , ImageWriteQueue(nullptr) { } void FMediaRecorder::StartRecording(const FMediaRecorderData& InRecoderData) { if (bRecording) { StopRecording(); } IMediaModule* MediaModule = FModuleManager::LoadModulePtr("Media"); if (MediaModule == nullptr) { return; } BaseFilename = InRecoderData.BaseFilename; NumerationStyle = InRecoderData.NumerationStyle; TargetImageFormat = InRecoderData.TargetImageFormat; bSetAlpha = InRecoderData.bResetAlpha; CompressionQuality = InRecoderData.CompressionQuality; SampleQueue = MakeShared(); InRecoderData.PlayerFacade->AddVideoSampleSink(SampleQueue.ToSharedRef()); ClockSink = MakeShared(*this); MediaModule->GetClock().AddSink(ClockSink.ToSharedRef()); ImageWriteQueue = &FModuleManager::LoadModuleChecked("ImageWriteQueue").GetWriteQueue(); bRecording = true; } void FMediaRecorder::StopRecording() { if (bRecording && ImageWriteQueue) { CompletedFence = ImageWriteQueue->CreateFence(); } bRecording = false; ImageWriteQueue = nullptr; ClockSink.Reset(); SampleQueue.Reset(); } bool FMediaRecorder::WaitPendingTasks(const FTimespan& InDuration) { bool bResult = true; if (CompletedFence.IsValid()) { bResult = CompletedFence.WaitFor(InDuration); } CompletedFence = TFuture(); return bResult; } void FMediaRecorder::TickRecording() { if (!bRecording) { return; // not recording } check(SampleQueue.IsValid()); if (ImageWriteQueue == nullptr) { while (SampleQueue->Pop()); StopRecording(); } else { TSharedPtr Sample; while (SampleQueue->Dequeue(Sample) && Sample.IsValid()) { const void* SampleBuffer = Sample->GetBuffer(); if (SampleBuffer == nullptr) { continue; // only raw samples supported right now } if (Sample->GetDim().GetMin() == 0) { continue; // nothing to save } if (Sample->GetFormat() != EMediaTextureSampleFormat::CharBGRA && Sample->GetFormat() != EMediaTextureSampleFormat::FloatRGBA) { if (!bUnsupportedWarningShowed) { UE_LOG(LogMediaUtils, Warning, TEXT("Media Source type cannot be recorded by Sequence Recorder. Texture Sample Format '%s' is not supported by Media Recorder."), MediaTextureSampleFormat::EnumToString(Sample->GetFormat())); bUnsupportedWarningShowed = true; } continue; } TUniquePtr ImageTask = MakeUnique(); // Set PixelData { const FIntPoint Size = Sample->GetDim(); EImagePixelType PixelType = EImagePixelType::Color; ERGBFormat PixelLayout = ERGBFormat::BGRA; uint8 BitDepth = 8; uint8 NumChannels = 4; if (Sample->GetFormat() == EMediaTextureSampleFormat::FloatRGBA) { PixelType = EImagePixelType::Float16; PixelLayout = ERGBFormat::RGBAF; BitDepth = 16; } // Should we move the color buffer into a raw image data container. bool bUseFMediaImagePixelData = bSetAlpha || (Sample->GetStride() != Size.X * NumChannels * (BitDepth/8)); if (bUseFMediaImagePixelData) { const int32 NumberOfTexel = Size.Y * Size.X; if (Sample->GetFormat() == EMediaTextureSampleFormat::FloatRGBA) { ImageTask->PixelData = MediaRecorderHelpers::CreatePixelData(Sample, Size, NumChannels); } else { check(Sample->GetFormat() == EMediaTextureSampleFormat::CharBGRA); ImageTask->PixelData = MediaRecorderHelpers::CreatePixelData(Sample, Size, NumChannels); } } else { // Use the MediaSample to save memory ImageTask->PixelData = MakeUnique(Sample, Size, PixelType, PixelLayout, BitDepth, NumChannels); } if (bSetAlpha) { check(bUseFMediaImagePixelData); ImageTask->AddPreProcessorToSetAlphaOpaque(); } } if (Sample->GetFormat() == EMediaTextureSampleFormat::CharBGRA) { // IsOutputSrgb almost always true ImageTask->PixelData->SetSRGB( Sample->IsOutputSrgb() ); } ImageTask->Format = TargetImageFormat; ImageTask->CompressionQuality = CompressionQuality; ImageTask->bOverwriteFile = true; if (NumerationStyle == EMediaRecorderNumerationStyle::AppendFrameNumber) { ImageTask->Filename = FString::Printf(TEXT("%s_%08d"), *BaseFilename, FrameCount); } else { ImageTask->Filename = FString::Printf(TEXT("%s_%.16" INT64_FMT), *BaseFilename, Sample->GetTime().Time.GetTicks()); } ImageWriteQueue->Enqueue(MoveTemp(ImageTask), false); ++FrameCount; } } }