// Copyright Epic Games, Inc. All Rights Reserved. #include "PromotedFrameUtils.h" #include "CaptureData.h" #include "LandmarkConfigIdentityHelper.h" #include "MetaHumanIdentity.h" #include "MetaHumanIdentityPose.h" #include "MetaHumanIdentityPromotedFrames.h" #include "MetaHumanContourDataVersion.h" #include "CameraCalibration.h" #include "IImageWrapper.h" #include "IImageWrapperModule.h" #include "ImgMediaSource.h" #include "ImageSequenceUtils.h" #include "UObject/UObjectGlobals.h" #include "Modules/ModuleManager.h" #include "Misc/FileHelper.h" #include "Engine/Texture2D.h" #include "TextureResource.h" // TODO: This is a copy-extract from Identity toolkit footage flow. Unify as part of scripting mesh flow bool UPromotedFrameUtils::InitializeContourDataForFootageFrame(UMetaHumanIdentityPose* InPose, UMetaHumanIdentityFootageFrame* InFootageFrame) { bool bInitalized = false; if (UFootageCaptureData* FootageCaptureData = Cast(InPose->GetCaptureData())) { if (InPose->GetIsFrameValid(InFootageFrame->FrameNumber) == UMetaHumanIdentityPose::ECurrentFrameValid::Valid) { EIdentityPoseType PoseType = InPose->PoseType; FLandmarkConfigIdentityHelper ConfigHelper; ECurvePresetType CurvePreset = ConfigHelper.GetCurvePresetFromIdentityPose(PoseType); const FIntPoint TextureResolution = FootageCaptureData->GetFootageColorResolution(); FFrameTrackingContourData ContourData = ConfigHelper.GetDefaultContourDataFromConfig(FVector2D(TextureResolution.X, TextureResolution.Y), CurvePreset); if (!ContourData.TrackingContours.IsEmpty()) { const FString ConfigVersion = FMetaHumanContourDataVersion::GetContourDataVersionString(); InFootageFrame->InitializeMarkersFromParsedConfig(ContourData, ConfigVersion); bInitalized = true; } } } return bInitalized; } bool UPromotedFrameUtils::GetPromotedFrameAsPixelArrayFromDisk(const FString& InImagePath, FIntPoint& OutImageSize, TArray& OutLocalSamples) { if (UTexture2D* LoadedTex = GetBGRATextureFromFile(InImagePath)) { FTexture2DMipMap& Mip0 = LoadedTex->GetPlatformData()->Mips[0]; if (FColor* TextureData = (FColor*)Mip0.BulkData.Lock(LOCK_READ_ONLY)) { for (int32 Index = 0; Index < LoadedTex->GetSizeX() * LoadedTex->GetSizeY(); ++Index) { OutLocalSamples.Add(TextureData[Index]); } Mip0.BulkData.Unlock(); OutImageSize = FIntPoint{ LoadedTex->GetSizeX(), LoadedTex->GetSizeY() }; return true; } } return false; } UTexture2D* UPromotedFrameUtils::GetBGRATextureFromFile(const FString& InFilePath) { TArray FileRawData; if (FFileHelper::LoadFileToArray(FileRawData, *InFilePath)) { IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked(FName("ImageWrapper")); for (EImageFormat ImageFormat : {EImageFormat::PNG, EImageFormat::JPEG}) { TSharedPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper(ImageFormat); if (ImageWrapper.IsValid() && ImageWrapper->SetCompressed(FileRawData.GetData(), FileRawData.Num())) { TArray ImageWrapperData; // GetRaw will return the data in the format we request it in if (ImageWrapper->GetRaw(ERGBFormat::BGRA, 8, ImageWrapperData)) { UTexture2D* TransientTex = UTexture2D::CreateTransient(ImageWrapper->GetWidth(), ImageWrapper->GetHeight()); FTexture2DMipMap& Mip0 = TransientTex->GetPlatformData()->Mips[0]; if (FColor* TextureData = (FColor*)Mip0.BulkData.Lock(LOCK_READ_WRITE)) { FMemory::Memcpy(TextureData, ImageWrapperData.GetData(), ImageWrapperData.Num()); Mip0.BulkData.Unlock(); TransientTex->UpdateResource(); return TransientTex; } } } } } return nullptr; } UTexture2D* UPromotedFrameUtils::GetDepthTextureFromFile(const FString& InFilePath) { TArray RawFileData; if (FFileHelper::LoadFileToArray(RawFileData, *InFilePath)) { IImageWrapperModule& ImageWrapperModule = FModuleManager::LoadModuleChecked(FName("ImageWrapper")); TSharedPtr ImageWrapper = ImageWrapperModule.CreateImageWrapper(EImageFormat::EXR); if (ImageWrapper.IsValid() && ImageWrapper->SetCompressed(RawFileData.GetData(), RawFileData.Num())) { if (ImageWrapper->GetBitDepth() == 32 && ImageWrapper->GetFormat() == ERGBFormat::GrayF) { TArray IntData; if (ImageWrapper->GetRaw(ERGBFormat::GrayF, 32, IntData)) { TArray DepthData; DepthData.SetNumUninitialized(ImageWrapper->GetWidth() * ImageWrapper->GetHeight()); const float* FloatData = static_cast((void*)IntData.GetData()); FMemory::Memcpy(&DepthData[0], FloatData, ImageWrapper->GetWidth() * ImageWrapper->GetHeight() * sizeof(float)); UTexture2D* Texture = UTexture2D::CreateTransient(ImageWrapper->GetWidth(), ImageWrapper->GetHeight(), EPixelFormat::PF_R32_FLOAT); FTexture2DMipMap& DepthMip0 = Texture->GetPlatformData()->Mips[0]; if (float* DepthTextureData = (float*)DepthMip0.BulkData.Lock(LOCK_READ_WRITE)) { FMemory::Memcpy(DepthTextureData, DepthData.GetData(), DepthData.Num() * sizeof(float)); DepthMip0.BulkData.Unlock(); Texture->UpdateResource(); return Texture; } } } } } return nullptr; } FString UPromotedFrameUtils::GetImagePathForFrame(const UFootageCaptureData* InFootageCaptureData, const FString& InCamera, const int32 InFrameId, bool bInIsImageSequence, ETimecodeAlignment InAlignment) { check(InFootageCaptureData && InFootageCaptureData->IsInitialized(UCaptureData::EInitializedCheck::Full)); int32 ViewIndex = -1; if (InFootageCaptureData) { ViewIndex = InFootageCaptureData->GetViewIndexByCameraName(InCamera); } check(ViewIndex >= 0 && ViewIndex < InFootageCaptureData->ImageSequences.Num() && ViewIndex < InFootageCaptureData->DepthSequences.Num()); FString CurrentImagePath; UImgMediaSource* MediaSequence = bInIsImageSequence ? InFootageCaptureData->ImageSequences[ViewIndex] : InFootageCaptureData->DepthSequences[ViewIndex]; TArray FrameImageNames; FImageSequenceUtils::GetImageSequenceFilesFromPath(MediaSequence->GetFullPath(), FrameImageNames); const int32 FrameNumber = IdentityFrameNumberToImageSequenceFrameNumber(InFootageCaptureData, InCamera, InFrameId, bInIsImageSequence, InAlignment); if (FrameImageNames.IsValidIndex(FrameNumber)) { CurrentImagePath = MediaSequence->GetFullPath() / FrameImageNames[FrameNumber]; } return CurrentImagePath; } int32 UPromotedFrameUtils::IdentityFrameNumberToImageSequenceFrameNumber(const UFootageCaptureData* InFootageCaptureData, const FString& InCamera, const int32 InFrameId, bool bInIsImageSequence, ETimecodeAlignment InAlignment) { check(InFootageCaptureData && InFootageCaptureData->IsInitialized(UCaptureData::EInitializedCheck::Full)); int32 ViewIndex = -1; if (InFootageCaptureData) { ViewIndex = InFootageCaptureData->GetViewIndexByCameraName(InCamera); } check(ViewIndex >= 0 && ViewIndex < InFootageCaptureData->ImageSequences.Num() && ViewIndex < InFootageCaptureData->DepthSequences.Num()); UImgMediaSource* MediaSequence = bInIsImageSequence ? InFootageCaptureData->ImageSequences[ViewIndex] : InFootageCaptureData->DepthSequences[ViewIndex]; TMap, TRange> MediaFrameRanges; TRange ProcessingLimitFrameRange, MaxFrameRange; InFootageCaptureData->GetFrameRanges(MediaSequence->FrameRateOverride, InAlignment, false, MediaFrameRanges, ProcessingLimitFrameRange, MaxFrameRange); return InFrameId - MediaFrameRanges[MediaSequence].GetLowerBoundValue().Value; }