// Copyright Epic Games, Inc. All Rights Reserved. #include "Generation/ChunkDeltaOptimiser.h" #include "Containers/List.h" #include "Containers/Ticker.h" #include "Containers/Queue.h" #include "HAL/ThreadSafeBool.h" #include "Async/Async.h" #include "Misc/ConfigCacheIni.h" #include "Misc/FileHelper.h" #include "Misc/OutputDeviceRedirector.h" #include "Policies/CondensedJsonPrintPolicy.h" #include "Serialization/JsonWriter.h" #include "HttpModule.h" #include "Core/MeanValue.h" #include "Core/Platform.h" #include "Core/ProcessTimer.h" #include "Common/ChunkDataSizeProvider.h" #include "Common/SpeedRecorder.h" #include "Common/FileSystem.h" #include "Common/HttpManager.h" #include "Common/StatsCollector.h" #include "Generation/BuildStreamer.h" #include "Generation/ChunkMatchProcessor.h" #include "Generation/ChunkWriter.h" #include "Generation/ChunkSearch.h" #include "Generation/DataScanner.h" #include "Generation/DeltaEnumeration.h" #include "Installer/InstallerAnalytics.h" #include "Installer/Statistics/DownloadServiceStatistics.h" #include "Installer/MemoryChunkStore.h" #include "Installer/ChunkEvictionPolicy.h" #include "Installer/MessagePump.h" #include "Installer/InstallerError.h" #include "Installer/Statistics/CloudChunkSourceStatistics.h" #include "BuildPatchHash.h" #include "BuildPatchUtil.h" #include "IBuildManifestSet.h" #include "Misc/CommandLine.h" DECLARE_LOG_CATEGORY_CLASS(LogChunkDeltaOptimiser, Log, All); // For the output file we'll use pretty json in debug, otherwise condensed. #if UE_BUILD_DEBUG typedef TJsonWriter> FDeltaJsonWriter; typedef TJsonWriterFactory> FDeltaJsonWriterFactory; #else typedef TJsonWriter> FDeltaJsonWriter; typedef TJsonWriterFactory> FDeltaJsonWriterFactory; #endif //UE_BUILD_DEBUG namespace DeltaOptimiseHelpers { using namespace BuildPatchServices; FSHAHash GetShaForDataSet(const uint8* Data, uint32 Size) { FSHAHash SHAHash; FSHA1::HashBuffer(Data, Size, SHAHash.Hash); return SHAHash; } FSHAHash GetShaForDataSet(const TArray& Data) { return GetShaForDataSet(Data.GetData(), Data.Num()); } int32 GetMaxScannerBacklogCount() { int32 MaxScannerBacklogCount = 75; GConfig->GetInt(TEXT("BuildPatchServices"), TEXT("MaxScannerBacklog"), MaxScannerBacklogCount, GEngineIni); MaxScannerBacklogCount = FMath::Clamp(MaxScannerBacklogCount, 5, 500); return MaxScannerBacklogCount; } bool HasUnusedCpu() { static const int32 NumThreadsAvailable = GThreadPool->GetNumThreads(); const bool bHasUnusedCpu = NumThreadsAvailable > FDataScannerCounter::GetNumRunningScanners(); #if UE_BUILD_DEBUG static const bool bSingleScannerThread = FParse::Param(FCommandLine::Get(), TEXT("singlescanneronly")); return bSingleScannerThread ? false : bHasUnusedCpu; #else return bHasUnusedCpu; #endif } template bool BacklogIsFull(const TArray& Scanners) { static int32 MaxScannerBacklogCount = GetMaxScannerBacklogCount(); return Scanners.Num() >= MaxScannerBacklogCount; } template bool ScannerArrayFull(const TArray& Scanners) { const bool bScannerArrayFull = (FDataScannerCounter::GetNumIncompleteScanners() > FDataScannerCounter::GetNumRunningScanners()) || BacklogIsFull(Scanners); #if UE_BUILD_DEBUG static const bool bSingleScannerThread = FParse::Param(FCommandLine::Get(), TEXT("singlescanneronly")); return bSingleScannerThread ? (FDataScannerCounter::GetNumIncompleteScanners() + FDataScannerCounter::GetNumRunningScanners()) > 0 : bScannerArrayFull; #else return bScannerArrayFull; #endif } FChunkPart SelectBytes(const FChunkPart& FullPart, uint32 LeftChop, uint32 Size) { FChunkPart Selected = FullPart; Selected.Offset += LeftChop; Selected.Size = Size; return Selected; } void StompChunkPart(const FChunkPart& NewMatchPart, const FBlockStructure& NewMatchBlocks, FChunkSearcher& ChunkSearcher, TSet& UpdatedFiles) { uint64 NewMatchPartStart = 0; ChunkSearcher.ForEachOverlap(NewMatchBlocks, [&](const FBlockRange& OverlapRange, FChunkSearcher::FFileDListNode* File, FChunkSearcher::FChunkDListNode* Chunk) { FChunkSearcher::FChunkNode& ChunkNode = Chunk->GetValue(); UpdatedFiles.Add(&File->GetValue()); const FChunkPart NewMatchPartBlock = DeltaOptimiseHelpers::SelectBytes(NewMatchPart, NewMatchPartStart, OverlapRange.GetSize()); NewMatchPartStart += OverlapRange.GetSize(); // If we fully replace this part. if (OverlapRange == ChunkNode.BuildRange) { ChunkNode.ChunkPart = NewMatchPartBlock; } // If we insert before this part, left chopping it. else if (OverlapRange.GetFirst() == ChunkNode.BuildRange.GetFirst()) { // Make the new node. const FChunkSearcher::FChunkNode NewMatchChunkNode(NewMatchPartBlock, FBlockRange::FromFirstAndSize(ChunkNode.BuildRange.GetFirst(), OverlapRange.GetSize())); // Left chop current node. ChunkNode.ChunkPart.Offset += OverlapRange.GetSize(); ChunkNode.ChunkPart.Size -= OverlapRange.GetSize(); ChunkNode.BuildRange = FBlockRange::FromFirstAndLast(NewMatchChunkNode.BuildRange.GetLast() + 1, ChunkNode.BuildRange.GetLast()); // Insert new node before current node. ListHelpers::InsertBefore(NewMatchChunkNode, File->GetValue().ChunkParts, Chunk); } // If we insert after this part, right chopping it. else if (OverlapRange.GetLast() == ChunkNode.BuildRange.GetLast()) { // Right chop current node. ChunkNode.ChunkPart.Size -= OverlapRange.GetSize(); ChunkNode.BuildRange = FBlockRange::FromFirstAndSize(ChunkNode.BuildRange.GetFirst(), ChunkNode.ChunkPart.Size); // Make the new node. const FChunkSearcher::FChunkNode NewMatchChunkNode(NewMatchPartBlock, FBlockRange::FromFirstAndSize(ChunkNode.BuildRange.GetLast() + 1, OverlapRange.GetSize())); // Insert chunk part after. ListHelpers::InsertAfter(NewMatchChunkNode, File->GetValue().ChunkParts, Chunk); } // If we insert inside this part. else { // Make the right side. const uint32 LeftChopSize = (OverlapRange.GetLast() - ChunkNode.BuildRange.GetFirst()) + 1; const uint32 RightSideSize = ChunkNode.BuildRange.GetSize() - LeftChopSize; const FChunkPart RightSide = DeltaOptimiseHelpers::SelectBytes(ChunkNode.ChunkPart, LeftChopSize, RightSideSize); const FChunkSearcher::FChunkNode RightSideChunkNode(RightSide, FBlockRange::FromFirstAndSize(OverlapRange.GetLast() + 1, RightSideSize)); // Make the middle piece. const FChunkSearcher::FChunkNode MiddleChunkNode(NewMatchPartBlock, OverlapRange); // Right chop current node. ChunkNode.ChunkPart.Size = OverlapRange.GetFirst() - ChunkNode.BuildRange.GetFirst(); ChunkNode.BuildRange = FBlockRange::FromFirstAndSize(ChunkNode.BuildRange.GetFirst(), ChunkNode.ChunkPart.Size); check(OverlapRange == FBlockRange::FromFirstAndLast(ChunkNode.BuildRange.GetLast() + 1, RightSideChunkNode.BuildRange.GetFirst() - 1)); // Insert right side part after current. ListHelpers::InsertAfter(RightSideChunkNode, File->GetValue().ChunkParts, Chunk); // Insert middle part after current (thus before right side). ListHelpers::InsertAfter(MiddleChunkNode, File->GetValue().ChunkParts, Chunk); } }); } void MakeScannerLocalList(FChunkSearcher& ChunkSearcher, IDeltaChunkEnumeration* Enumeration, const FBlockStructure& BuildStructure, FScannerFilesList& Result) { uint64 FirstByte = 0; ChunkSearcher.ForEachOverlap(BuildStructure, [&](const FBlockRange& OverlapRange, FChunkSearcher::FFileDListNode* File, FChunkSearcher::FChunkDListNode* Chunk) { const FFilenameId FilenameId = Enumeration->MakeFilenameId(File->GetValue().Manifest->Filename); const TArray& FileTagset = File->GetValue().Manifest->InstallTags; const FBlockRange& FileRange = File->GetValue().BuildRange; const uint64 FileOffset = OverlapRange.GetFirst() - FileRange.GetFirst(); Result.AddTail(FScannerFileElement{FBlockRange::FromFirstAndSize(FirstByte, OverlapRange.GetSize()), FilenameId, FileTagset, FileOffset}); FirstByte += OverlapRange.GetSize(); }); check(BlockStructureHelpers::CountSize(BuildStructure) == FirstByte); } } namespace DeltaStats { class FNoMemoryChunkStoreStat : public BuildPatchServices::IMemoryChunkStoreStat { public: FNoMemoryChunkStoreStat() { } ~FNoMemoryChunkStoreStat() { } // IMemoryChunkStoreStat interface begin. virtual void OnChunkStored(const FGuid& ChunkId) override { } virtual void OnChunkReleased(const FGuid& ChunkId) override { } virtual void OnChunkBooted(const FGuid& ChunkId) override { } virtual void OnStoreUseUpdated(int32 ChunkCount) override { } virtual void OnStoreSizeUpdated(int32 Size) override { } // IMemoryChunkStoreStat interface end. }; class FNoCloudChunkSourceStat : public BuildPatchServices::ICloudChunkSourceStat { public: FNoCloudChunkSourceStat() { } ~FNoCloudChunkSourceStat() { } // ICloudChunkSourceStat interface begin. virtual void OnDownloadRequested(const FGuid& ChunkId) override { } virtual void OnDownloadSuccess(const FGuid& ChunkId) override { } virtual void OnDownloadFailed(const FGuid& ChunkId, const FString& Url) override { } virtual void OnDownloadCorrupt(const FGuid& ChunkId, const FString& Url, BuildPatchServices::EChunkLoadResult LoadResult) override { } virtual void OnDownloadAborted(const FGuid& ChunkId, const FString& Url, double DownloadTimeMean, double DownloadTimeStd, double DownloadTime, double BreakingPoint) override { } virtual void OnReceivedDataUpdated(int64 TotalBytes) override { } virtual void OnRequiredDataUpdated(int64 TotalBytes) override { } virtual void OnDownloadHealthUpdated(EBuildPatchDownloadHealth DownloadHealth) override { } virtual void OnSuccessRateUpdated(float SuccessRate) override { } virtual void OnActiveRequestCountUpdated(uint32 RequestCount) override { } virtual void OnAcceptedNewRequirements(const TSet& ChunkIds) override { } // ICloudChunkSourceStat interface end. }; } namespace DeltaFactories { using namespace BuildPatchServices; class FChunkReferenceTrackerFactory : public IManifestBuildStreamer::IChunkReferenceTrackerFactory { public: FChunkReferenceTrackerFactory() { } virtual ~FChunkReferenceTrackerFactory() { } // IManifestBuildStreamer::IChunkReferenceTrackerFactory interface begin. virtual IChunkReferenceTracker* Create(IManifestBuildStreamer::FCustomChunkReferences CustomChunkReferences) override { return BuildPatchServices::FChunkReferenceTrackerFactory::Create(MoveTemp(CustomChunkReferences)); } // IManifestBuildStreamer::IChunkReferenceTrackerFactory interface end. }; struct FCloudChunkSourceFactoryShared { public: IFileSystem* FileSystem; IDownloadService* DownloadService; IChunkDataSerialization* ChunkDataSerialization; IMessagePump* MessagePump; IBuildManifestSet* ManifestSet; }; class FCloudChunkSourceFactory : public IManifestBuildStreamer::ICloudChunkSourceFactory { private: struct FInstanceDependancies { TUniquePtr MemoryEvictionPolicy; TUniquePtr CloudChunkStore; TUniquePtr ConnectionCount; }; public: FCloudChunkSourceFactory(const FString& CloudDir, FCloudChunkSourceFactoryShared InShared) : Shared(MoveTemp(InShared)) , CloudSourceConfig({CloudDir}) , Platform(FPlatformFactory::Create()) , MemoryChunkStoreStat(new DeltaStats::FNoMemoryChunkStoreStat()) , InstallerError(FInstallerErrorFactory::Create()) , CloudChunkSourceStat(new DeltaStats::FNoCloudChunkSourceStat()) { CloudSourceConfig.bBeginDownloadsOnFirstGet = false; CloudSourceConfig.MaxRetryCount = 30; } virtual ~FCloudChunkSourceFactory() { } // IManifestBuildStreamer::ICloudChunkSourceFactory interface begin. virtual ICloudChunkSource* Create(IChunkReferenceTracker* ChunkReferenceTracker) override { InstanceDependancies.AddDefaulted(); FInstanceDependancies& Dependancies = InstanceDependancies.Last(); Dependancies.MemoryEvictionPolicy.Reset(FChunkEvictionPolicyFactory::Create(ChunkReferenceTracker)); Dependancies.CloudChunkStore.Reset(FMemoryChunkStoreFactory::Create( 100, Dependancies.MemoryEvictionPolicy.Get(), nullptr, MemoryChunkStoreStat.Get(), nullptr)); Dependancies.ConnectionCount.Reset(BuildPatchServices::FDownloadConnectionCountFactory::Create(ConnectionCountConfig, nullptr)); ICloudChunkSource* CloudChunkSource = BuildPatchServices::FCloudChunkSourceFactory::Create( CloudSourceConfig, Platform.Get(), Dependancies.CloudChunkStore.Get(), Shared.DownloadService, ChunkReferenceTracker, Shared.ChunkDataSerialization, Shared.MessagePump, InstallerError.Get(), Dependancies.ConnectionCount.Get(), CloudChunkSourceStat.Get(), Shared.ManifestSet, ChunkReferenceTracker->GetReferencedChunks()); TFunction LostChunkCallback = [CloudChunkSource](const FGuid& LostChunk) { CloudChunkSource->AddRepeatRequirement(LostChunk); }; Dependancies.CloudChunkStore->SetLostChunkCallback(LostChunkCallback); return CloudChunkSource; } // IManifestBuildStreamer::ICloudChunkSourceFactory interface end. private: FCloudChunkSourceFactoryShared Shared; FCloudSourceConfig CloudSourceConfig; FDownloadConnectionCountConfig ConnectionCountConfig; TUniquePtr Platform; TUniquePtr MemoryChunkStoreStat; TUniquePtr InstallerError; TUniquePtr CloudChunkSourceStat; TArray InstanceDependancies; }; } namespace BuildPatchServices { class FChunkMatchStomper { public: typedef TTuple, FBlockStructure> FNewMatch; typedef TQueue FNewMatchQueue; FChunkMatchStomper(const FBuildPatchAppManifest& InManifestA, const FBuildPatchAppManifest& InManifestB) : ManifestA(InManifestA) , ManifestB(InManifestB) , BuildAFiles(ListHelpers::GetFileList(ManifestA)) , BuildBFiles(ListHelpers::GetFileList(ManifestB)) , bExpectsMoreData(true) , ThreadTrigger(FPlatformProcess::GetSynchEventFromPool(true)) { FileManifestListFuture = Async(EAsyncExecution::Thread, [this]() { return AsyncRun(); }); } ~FChunkMatchStomper() { // Ensures the thread work completes. bExpectsMoreData = false; ThreadTrigger->Trigger(); FileManifestListFuture.Wait(); FPlatformProcess::ReturnSynchEventToPool(ThreadTrigger); } FFileManifestList AsyncRun() { FChunkSearcher SearcherB(ManifestB); TSet UpdatedFiles; // Start with searcher B invalidating unknown chunks. FChunkSearcher::FFileDListNode* FileBNode = SearcherB.GetHead(); while (FileBNode) { FChunkSearcher::FChunkDListNode* ChunkBNode = FileBNode->GetValue().ChunkParts.GetHead(); while (ChunkBNode) { if (ManifestA.GetChunkInfo(ChunkBNode->GetValue().ChunkPart.Guid) == nullptr) { ChunkBNode->GetValue().ChunkPart.Guid.Invalidate(); } ChunkBNode = ChunkBNode->GetNextNode(); } FileBNode = FileBNode->GetNextNode(); } bool bHasNewMatch = false; FNewMatch NewMatch; while ((bHasNewMatch = NewMatchQueue.Dequeue(NewMatch), bHasNewMatch) || bExpectsMoreData) { if (bHasNewMatch) { const TArray& NewChunkParts = NewMatch.Get<0>(); const FBlockStructure& BuildBStructure = NewMatch.Get<1>(); uint64 ByteCount = 0; for (const FChunkPart& NewChunkPart : NewChunkParts) { FBlockStructure PartStructure; BuildBStructure.SelectSerialBytes(ByteCount, NewChunkPart.Size, PartStructure); DeltaOptimiseHelpers::StompChunkPart(NewChunkPart, PartStructure, SearcherB, UpdatedFiles); ByteCount += NewChunkPart.Size; } } else { ThreadTrigger->Wait(1000); ThreadTrigger->Reset(); } } // Ensure priority to original matches? ClobberAllKnownChunks(SearcherB, UpdatedFiles); // Collapse all adjacent chunkparts. FileBNode = SearcherB.GetHead(); while (FileBNode) { MergeAdjacentChunkParts(FileBNode->GetValue().ChunkParts); FileBNode = FileBNode->GetNextNode(); } return SearcherB.BuildNewFileManifestList(); } void ReplaceChunkReferences(const TArray& NewChunkReferences, const FBlockStructure& BuildBStructure) { checkf(bExpectsMoreData, TEXT("You can't provide more data after collecting the result.")); NewMatchQueue.Enqueue(FNewMatch{NewChunkReferences, BuildBStructure}); ThreadTrigger->Trigger(); } FFileManifestList GetNewFileManifests() { bExpectsMoreData = false; ThreadTrigger->Trigger(); return FileManifestListFuture.Get(); } private: void ClobberAllKnownChunks(FChunkSearcher& ChunkSearcher, TSet& UpdatedFiles) { uint64 BuildFileFirst = 0; uint64 ChunkPartFirst = 0; for (const FString& BuildFilename : BuildBFiles) { const FFileManifest* FileManifest = ManifestB.GetFileManifest(BuildFilename); check(FileManifest != nullptr); const FBlockRange FileRange = FBlockRange::FromFirstAndSize(BuildFileFirst, FileManifest->FileSize); if (FileRange.GetSize() > 0) { ChunkPartFirst = FileRange.GetFirst(); for (const FChunkPart& ChunkPart : FileManifest->ChunkParts) { const FBlockRange ChunkPartRange = FBlockRange::FromFirstAndSize(ChunkPartFirst, ChunkPart.Size); if (ManifestA.GetChunkInfo(ChunkPart.Guid) != nullptr) { DeltaOptimiseHelpers::StompChunkPart(ChunkPart, FBlockStructure(ChunkPartRange.GetFirst(), ChunkPartRange.GetSize()), ChunkSearcher, UpdatedFiles); } ChunkPartFirst += ChunkPartRange.GetSize(); } } check(ChunkPartFirst == (BuildFileFirst + FileRange.GetSize())); BuildFileFirst += FileRange.GetSize(); } } void MergeAdjacentChunkParts(FChunkSearcher::FChunkDList& ChunkParts) { FChunkSearcher::FChunkDListNode* ChunkNode = ChunkParts.GetHead(); while (ChunkNode) { FChunkSearcher::FChunkDListNode* NextChunkNode = ChunkNode->GetNextNode(); while (NextChunkNode) { // Assert if we skipped build data check((ChunkNode->GetValue().BuildRange.GetLast() + 1) == NextChunkNode->GetValue().BuildRange.GetFirst()); FChunkPart& ThisChunkPart = ChunkNode->GetValue().ChunkPart; FChunkPart& NextChunkPart = NextChunkNode->GetValue().ChunkPart; const FBlockRange LastMatchPartRange = FBlockRange::FromFirstAndSize(ThisChunkPart.Offset, ThisChunkPart.Size); const FBlockRange ThisMatchPartRange = FBlockRange::FromFirstAndSize(NextChunkPart.Offset, NextChunkPart.Size); const bool bBothInvalid = !NextChunkPart.Guid.IsValid() && !ThisChunkPart.Guid.IsValid(); const bool bBothSamePadding = ThisChunkPart.IsPadding() && NextChunkPart.IsPadding() && ThisChunkPart.GetPaddingByte() == NextChunkPart.GetPaddingByte(); const bool bSameChunk = NextChunkPart.Guid == ThisChunkPart.Guid; const bool bAdjacentData = (LastMatchPartRange.GetLast() + 1) == ThisMatchPartRange.GetFirst(); bool bMerged = false; FChunkSearcher::FChunkDListNode* NextNextChunkNode = NextChunkNode->GetNextNode(); if (bBothInvalid) { const uint64 TotalSize = NextChunkNode->GetValue().BuildRange.GetSize() + ChunkNode->GetValue().BuildRange.GetSize(); if (TotalSize < TNumericLimits::Max()) { ThisChunkPart.Size = TotalSize; ChunkNode->GetValue().BuildRange = FBlockRange::FromFirstAndSize(ChunkNode->GetValue().BuildRange.GetFirst(), TotalSize); ChunkParts.RemoveNode(NextChunkNode); bMerged = true; } } else if (bBothSamePadding) { const uint64 TotalSize = NextChunkNode->GetValue().BuildRange.GetSize() + ChunkNode->GetValue().BuildRange.GetSize(); if (TotalSize < PaddingChunk::ChunkSize) { ThisChunkPart.Offset = 0; ThisChunkPart.Size = TotalSize; ChunkNode->GetValue().BuildRange = FBlockRange::FromFirstAndSize(ChunkNode->GetValue().BuildRange.GetFirst(), TotalSize); ChunkParts.RemoveNode(NextChunkNode); bMerged = true; } } else if (bSameChunk && bAdjacentData) { const FBlockRange MergedPartRange = FBlockRange::FromMerge(ThisMatchPartRange, LastMatchPartRange); ThisChunkPart.Offset = MergedPartRange.GetFirst(); ThisChunkPart.Size = MergedPartRange.GetSize(); ChunkNode->GetValue().BuildRange = FBlockRange::FromMerge(ChunkNode->GetValue().BuildRange, NextChunkNode->GetValue().BuildRange); ChunkParts.RemoveNode(NextChunkNode); bMerged = true; } if (!bMerged) { ChunkNode = NextChunkNode; } NextChunkNode = NextNextChunkNode; } ChunkNode = NextChunkNode; } } private: const FBuildPatchAppManifest& ManifestA; const FBuildPatchAppManifest& ManifestB; const TArray BuildAFiles; const TArray BuildBFiles; FThreadSafeBool bExpectsMoreData; FEvent* ThreadTrigger; TFuture FileManifestListFuture; FNewMatchQueue NewMatchQueue; }; } namespace BuildPatchServices { struct FDeltaScannerEntry { public: FDeltaScannerEntry() : bIsFinalScanner(false) , bWasFork(false) , Offset(0) { } public: TArray Data; FScannerFilesList FilesList; TUniquePtr Scanner; bool bIsFinalScanner; bool bWasFork; uint64 Offset; }; class FChunkDeltaOptimiser : public IChunkDeltaOptimiser { public: FChunkDeltaOptimiser(const FChunkDeltaOptimiserConfiguration& InConfiguration); ~FChunkDeltaOptimiser(); // IChunkDeltaOptimiser interface begin. virtual bool Run() override; // IChunkDeltaOptimiser interface end. private: TArray AsyncRun(); void HandleDownloadComplete(int32 RequestId, const FDownloadRef& Download); FBlockStructure GetDesiredBytes(const FBuildPatchAppManifestPtr& Manifest, const TSet& Chunks); private: const FChunkDeltaOptimiserConfiguration Configuration; FTSTicker& CoreTicker; FDownloadCompleteDelegate DownloadCompleteDelegate; FDownloadProgressDelegate DownloadProgressDelegate; TUniquePtr FileSystem; TUniquePtr HttpManager; TUniquePtr ChunkDataSizeProvider; TUniquePtr DownloadSpeedRecorder; TUniquePtr InstallerAnalytics; TUniquePtr DownloadServiceStatistics; TUniquePtr DownloadService; TUniquePtr MessagePump; TUniquePtr StatsCollector; FThreadSafeBool bShouldRun; FThreadSafeBool bSuccess; // Manifest downloading int32 RequestIdManifestA; int32 RequestIdManifestB; TPromise PromiseManifestA; TPromise PromiseManifestB; TFuture FutureManifestA; TFuture FutureManifestB; }; FChunkDeltaOptimiser::FChunkDeltaOptimiser(const FChunkDeltaOptimiserConfiguration& InConfiguration) : Configuration(InConfiguration) , CoreTicker(FTSTicker::GetCoreTicker()) , DownloadCompleteDelegate(FDownloadCompleteDelegate::CreateRaw(this, &FChunkDeltaOptimiser::HandleDownloadComplete)) , DownloadProgressDelegate() , FileSystem(FFileSystemFactory::Create()) , HttpManager(FHttpManagerFactory::Create()) , ChunkDataSizeProvider(FChunkDataSizeProviderFactory::Create()) , DownloadSpeedRecorder(FSpeedRecorderFactory::Create()) , InstallerAnalytics(FInstallerAnalyticsFactory::Create(nullptr)) , DownloadServiceStatistics(FDownloadServiceStatisticsFactory::Create(DownloadSpeedRecorder.Get(), ChunkDataSizeProvider.Get(), InstallerAnalytics.Get())) , DownloadService(FDownloadServiceFactory::Create(HttpManager.Get(), FileSystem.Get(), DownloadServiceStatistics.Get(), InstallerAnalytics.Get())) , MessagePump(FMessagePumpFactory::Create()) , StatsCollector(FStatsCollectorFactory::Create()) , bShouldRun(true) , RequestIdManifestA(INDEX_NONE) , RequestIdManifestB(INDEX_NONE) , PromiseManifestA() , PromiseManifestB() , FutureManifestA(PromiseManifestA.GetFuture()) , FutureManifestB(PromiseManifestB.GetFuture()) { } FChunkDeltaOptimiser::~FChunkDeltaOptimiser() { } bool FChunkDeltaOptimiser::Run() { // Run any core initialisation required. FHttpModule::Get(); // Setup Generation stats. volatile int64* StatTotalTime = StatsCollector->CreateStat(TEXT("Generation: Total Time"), EStatFormat::Timer); volatile int64* StatStreamSpeed = StatsCollector->CreateStat(TEXT("Generation: Data stream speed"), EStatFormat::DataSpeed); const uint64 StartTime = FStatsCollector::GetCycles(); const float SpeedStatOverTime = TNumericLimits::Max(); // Kick off Manifest downloads. RequestIdManifestA = DownloadService->RequestFile(Configuration.ManifestAUri, DownloadCompleteDelegate, DownloadProgressDelegate); RequestIdManifestB = DownloadService->RequestFile(Configuration.ManifestBUri, DownloadCompleteDelegate, DownloadProgressDelegate); // Start the generation thread. TFuture> Thread = Async(EAsyncExecution::Thread, [this](){ return AsyncRun(); }); // Main timers. double DeltaTime = 0.0; double LastTime = FPlatformTime::Seconds(); // Setup desired frame times. float MainsFramerate = 100.0f; const float MainsFrameTime = 1.0f / MainsFramerate; // Load settings from config. float StatsLoggerTimeSeconds = 10.0f; GConfig->GetFloat(TEXT("BuildPatchServices"), TEXT("StatsLoggerTimeSeconds"), StatsLoggerTimeSeconds, GEngineIni); StatsLoggerTimeSeconds = FMath::Clamp(StatsLoggerTimeSeconds, 1.0f, 60.0f); // Run the main loop. while (bShouldRun) { // Increment global frame counter once for each app tick. GFrameCounter++; // Application tick. FTaskGraphInterface::Get().ProcessThreadUntilIdle(ENamedThreads::GameThread); FTSTicker::GetCoreTicker().Tick(DeltaTime); // Message pump. MessagePump->PumpMessages(); // Log collected stats. GLog->FlushThreadedLogs(); FStatsCollector::Set(StatTotalTime, FStatsCollector::GetCycles() - StartTime); FStatsCollector::Set(StatStreamSpeed, DownloadSpeedRecorder->GetAverageSpeed(SpeedStatOverTime)); StatsCollector->LogStats(StatsLoggerTimeSeconds); // Control frame rate. FPlatformProcess::Sleep(FMath::Max(0.0f, MainsFrameTime - (FPlatformTime::Seconds() - LastTime))); // Calculate deltas. const double AppTime = FPlatformTime::Seconds(); DeltaTime = AppTime - LastTime; LastTime = AppTime; } // Log collected stats. TArray FinalStatLogs = Thread.Get(); GLog->FlushThreadedLogs(); FStatsCollector::Set(StatTotalTime, FStatsCollector::GetCycles() - StartTime); FStatsCollector::Set(StatStreamSpeed, DownloadSpeedRecorder->GetAverageSpeed(SpeedStatOverTime)); StatsCollector->LogStats(); for (const FString& LogLine : FinalStatLogs) { UE_LOG(LogChunkDeltaOptimiser, Display, TEXT("%s"), *LogLine); } // Return thread success. return bSuccess; } TArray FChunkDeltaOptimiser::AsyncRun() { const FNumberFormattingOptions PercentFormat = FNumberFormattingOptions().SetMaximumFractionalDigits(1).SetMinimumFractionalDigits(1).SetRoundingMode(ERoundingMode::ToZero); FBuildPatchAppManifestPtr ManifestA = FutureManifestA.Get(); FBuildPatchAppManifestPtr ManifestB = FutureManifestB.Get(); TArray FinalStatLogs; bSuccess = true; if (ManifestA.IsValid() == false) { UE_LOG(LogChunkDeltaOptimiser, Error, TEXT("Could not download ManifestA from %s."), *Configuration.ManifestAUri); bSuccess = false; } if (ManifestB.IsValid() == false) { UE_LOG(LogChunkDeltaOptimiser, Error, TEXT("Could not download ManifestB from %s."), *Configuration.ManifestBUri); bSuccess = false; } if (bSuccess) { UE_LOG(LogChunkDeltaOptimiser, Display, TEXT("Running optimisation for patching %s -> %s"), *ManifestA->GetVersionString(), *ManifestB->GetVersionString()); FProcessTimer ProcessTimer; FProcessTimer ChunkingTimer; FProcessTimer ScanningTimer; ProcessTimer.Start(); TSet ChunksA; TSet ChunksB; ManifestA->GetDataList(ChunksA); ManifestB->GetDataList(ChunksB); // Check for ManifestA -> ManifestB compatibility. We don't yet support downgrading chunk version, only upgrading. const TCHAR* ManifestAChunkSubdir = ManifestVersionHelpers::GetChunkSubdir(ManifestA->ManifestMeta.FeatureLevel); const TCHAR* ManifestBChunkSubdir = ManifestVersionHelpers::GetChunkSubdir(ManifestB->ManifestMeta.FeatureLevel); const bool bUsingDifferentChunkSubdir = ManifestAChunkSubdir != ManifestBChunkSubdir; const bool bIsDowngrade = ManifestB->ManifestMeta.FeatureLevel < ManifestA->ManifestMeta.FeatureLevel; if (bUsingDifferentChunkSubdir && bIsDowngrade) { UE_LOG(LogChunkDeltaOptimiser, Error, TEXT("Destination manifest does not support source manifest's FeatureLevel (%s [%d] -> %s [%d])."), FeatureLevelToString(ManifestA->ManifestMeta.FeatureLevel), (int32)ManifestA->ManifestMeta.FeatureLevel, FeatureLevelToString(ManifestB->ManifestMeta.FeatureLevel), (int32)ManifestB->ManifestMeta.FeatureLevel); bSuccess = false; } // Check for output chunk size compatibility changes. const uint32 OutputChunkSize = ManifestB->ManifestMeta.FeatureLevel >= EFeatureLevel::VariableSizeChunks ? Configuration.OutputChunkSize : 1024*1024; if (Configuration.OutputChunkSize != OutputChunkSize) { UE_LOG(LogChunkDeltaOptimiser, Log, TEXT("Destination manifest does not support EFeatureLevel::VariableSizeChunks, reverting OutputChunkSize to %u."), OutputChunkSize); } // Check that an optimisation does not already exist, and skip long process if so. FBuildPatchAppManifest DeltaManifest; const FString OutputDeltaFilename = Configuration.CloudDirectory / FBuildPatchUtils::GetChunkDeltaFilename(*ManifestA.Get(), *ManifestB.Get()); const bool bDeltaPreviouslyCompleted = FileSystem->FileExists(*OutputDeltaFilename); if (bDeltaPreviouslyCompleted) { if (DeltaManifest.LoadFromFile(OutputDeltaFilename)) { FinalStatLogs.Add(FString::Printf(TEXT("** Chunk delta optimisation already completed for provided manifests. **"))); FinalStatLogs.Add(FString::Printf(TEXT("Loaded optimised delta file %s"), *OutputDeltaFilename)); } else { UE_LOG(LogChunkDeltaOptimiser, Error, TEXT("Optimised delta completed previously but could not be loaded %s."), *OutputDeltaFilename); bSuccess = false; } } // Check for aborting if original delta is over provided threshold. TSet TagsA, TagsB; ManifestA->GetFileTagList(TagsA); ManifestB->GetFileTagList(TagsB); const uint64 OriginalUnknownCompressedBytes = (uint64)ManifestB->GetDeltaDownloadSize(TagsB, ManifestA.ToSharedRef(), TagsA); const bool bOverAbortThreshold = OriginalUnknownCompressedBytes >= Configuration.DiffAbortThreshold; if (!bDeltaPreviouslyCompleted && bOverAbortThreshold) { FinalStatLogs.Add(FString::Printf(TEXT("** Aborting delta optimisation due to original delta over threshold. **"))); FinalStatLogs.Add(FString::Printf(TEXT("%llu >= %llu"), OriginalUnknownCompressedBytes, Configuration.DiffAbortThreshold)); } const bool bRunProcess = bSuccess && !bDeltaPreviouslyCompleted && !bOverAbortThreshold; if (bRunProcess) { // Runtime composition. TUniquePtr ChunkDataSerializationReader(FChunkDataSerializationFactory::Create(FileSystem.Get())); TUniquePtr ChunkReferenceTrackerFactory(new DeltaFactories::FChunkReferenceTrackerFactory()); TUniquePtr SetA(FBuildManifestSetFactory::Create({ FInstallerAction::MakeInstall(ManifestA.ToSharedRef()) })); TUniquePtr SetB(FBuildManifestSetFactory::Create({ FInstallerAction::MakeInstall(ManifestB.ToSharedRef()) })); DeltaFactories::FCloudChunkSourceFactoryShared CloudChunkSourceFactorySharedA; CloudChunkSourceFactorySharedA.FileSystem = FileSystem.Get(); CloudChunkSourceFactorySharedA.DownloadService = DownloadService.Get(); CloudChunkSourceFactorySharedA.ChunkDataSerialization = ChunkDataSerializationReader.Get(); CloudChunkSourceFactorySharedA.MessagePump = MessagePump.Get(); CloudChunkSourceFactorySharedA.ManifestSet = SetA.Get(); DeltaFactories::FCloudChunkSourceFactoryShared CloudChunkSourceFactorySharedB; CloudChunkSourceFactorySharedB.FileSystem = FileSystem.Get(); CloudChunkSourceFactorySharedB.DownloadService = DownloadService.Get(); CloudChunkSourceFactorySharedB.ChunkDataSerialization = ChunkDataSerializationReader.Get(); CloudChunkSourceFactorySharedB.MessagePump = MessagePump.Get(); CloudChunkSourceFactorySharedB.ManifestSet = SetB.Get(); TUniquePtr CloudChunkSourceFactoryA(new DeltaFactories::FCloudChunkSourceFactory(Configuration.CloudDirectory, CloudChunkSourceFactorySharedA)); TUniquePtr CloudChunkSourceFactoryB(new DeltaFactories::FCloudChunkSourceFactory(Configuration.CloudDirectory, CloudChunkSourceFactorySharedB)); // Buffer for data streaming. const EAllowShrinking AllowShrinking = EAllowShrinking::No; const uint32 StreamBufferReadSize = Configuration.ScanWindowSize * 32; const uint32 ScannerDataSize = StreamBufferReadSize; TArray StreamBuffer; StreamBuffer.Reserve(StreamBufferReadSize + Configuration.ScanWindowSize); // Calculate the desired bytes for manifest streams. FBlockStructure ManifestADesiredBytes = GetDesiredBytes(ManifestA, ChunksA.Difference(ChunksB)); FBlockStructure ManifestBDesiredBytes = GetDesiredBytes(ManifestB, ChunksB.Difference(ChunksA)); const uint64 ManifestBStreamSize = BlockStructureHelpers::CountSize(ManifestBDesiredBytes); // Start the ManifestA stream and chunk enumeration. FManifestBuildStreamerConfig ManifestAStreamConfig({Configuration.CloudDirectory, ManifestADesiredBytes}); FManifestBuildStreamerDependencies ManifestAStreamDependencies({ChunkReferenceTrackerFactory.Get(), CloudChunkSourceFactoryA.Get(), StatsCollector.Get(), ManifestA.Get()}); TUniquePtr ManifestAStream(FBuildStreamerFactory::Create(MoveTemp(ManifestAStreamConfig), MoveTemp(ManifestAStreamDependencies))); // First we re-chunk prev unknown build parts into the scanner window size. TUniquePtr DeltaChunkEnumeration(FDeltaChunkEnumerationFactory::Create(ManifestAStream.Get(), StatsCollector.Get(), *ManifestA.Get(), Configuration.ScanWindowSize)); ChunkingTimer.Start(); DeltaChunkEnumeration->Run(); ChunkingTimer.Stop(); // Setup scanning stats. volatile int64* StatScannerBacklog = StatsCollector->CreateStat(TEXT("BuildB: Scanner backlog"), EStatFormat::Value); volatile int64* StatScannerForks = StatsCollector->CreateStat(TEXT("BuildB: Scanner forks"), EStatFormat::Value); volatile int64* StatScanningTime = StatsCollector->CreateStat(TEXT("BuildB: Scanning time"), EStatFormat::Timer); volatile int64* StatScanningCompleted = StatsCollector->CreateStat(TEXT("BuildB: Progress"), EStatFormat::Percentage); // Start the ManifestB stream. FManifestBuildStreamerConfig ManifestBStreamConfig({Configuration.CloudDirectory, ManifestBDesiredBytes}); FManifestBuildStreamerDependencies ManifestBStreamDependencies({ChunkReferenceTrackerFactory.Get(), CloudChunkSourceFactoryB.Get(), StatsCollector.Get(), ManifestB.Get()}); TUniquePtr ManifestBStream(FBuildStreamerFactory::Create(MoveTemp(ManifestBStreamConfig), MoveTemp(ManifestBStreamDependencies))); // Our second loop which finds matching chunks in the new build. ScanningTimer.Start(); FChunkSearcher FileListSearcher(*ManifestB.Get()); TUniquePtr ChunkMatchStomper(new FChunkMatchStomper(*ManifestA.Get(), *ManifestB.Get())); const uint32 ScannerOverlapSize = Configuration.ScanWindowSize - 1; TUniquePtr ChunkMatchProcessor(FChunkMatchProcessorFactory::Create()); TArray> DataScanners; int32 NumScannersCreated = 0; int32 NumScannersRequired = ManifestBStreamSize / (ScannerDataSize - ScannerOverlapSize); FMeanValue MeanScannerTime(5); int32 ConsumedBufferData = 0; uint64 StreamStartPosition = 0; StreamBuffer.SetNumUninitialized(0, AllowShrinking); const TMap& ChunkBuildReferences = DeltaChunkEnumeration->GetChunkBuildReferences(); uint64 BuildBScanTimer; FStatsCollector::AccumulateTimeBegin(BuildBScanTimer); while (ManifestBStream->IsEndOfData() == false || DataScanners.Num() > 0) { // Grab new stream data. check(StreamBuffer.Num() >= ConsumedBufferData); uint32 BufferDataSize = StreamBuffer.Num() - ConsumedBufferData; if (!ManifestBStream->IsEndOfData() && (BufferDataSize < ScannerDataSize)) { // Move unconsumed data to the beginning. if (BufferDataSize > 0) { uint8* const CopyTo = StreamBuffer.GetData(); const uint8* const CopyFrom = &StreamBuffer[ConsumedBufferData]; FMemory::Memcpy(CopyTo, CopyFrom, BufferDataSize); } StreamStartPosition += ConsumedBufferData; ConsumedBufferData = 0; // Fill the rest of the buffer. StreamBuffer.SetNumUninitialized(BufferDataSize + StreamBufferReadSize, AllowShrinking); const uint32 SizeRead = ManifestBStream->DequeueData(StreamBuffer.GetData() + BufferDataSize, StreamBufferReadSize); StreamBuffer.SetNumUninitialized(BufferDataSize + SizeRead, AllowShrinking); BufferDataSize = StreamBuffer.Num(); } // Grab a scanner result. if (DataScanners.Num() > 0 && DataScanners[0]->Scanner->IsComplete()) { FDeltaScannerEntry& ScannerDetails = *DataScanners[0]; if (!ScannerDetails.bWasFork) { MeanScannerTime.AddSample(ScannerDetails.Scanner->GetTimeRunning()); } TArray ChunkMatches = ScannerDetails.Scanner->GetResultWhenComplete(); for (FChunkMatch& ChunkMatch : ChunkMatches) { FBlockStructure ChunkCBuildBStructure; ChunkMatch.DataOffset += ScannerDetails.Offset; ManifestBDesiredBytes.SelectSerialBytes(ChunkMatch.DataOffset, ChunkMatch.WindowSize, ChunkCBuildBStructure); ChunkMatchProcessor->ProcessMatch(0, ChunkMatch, MoveTemp(ChunkCBuildBStructure)); } const FBlockRange ScannerRange = FBlockRange::FromFirstAndSize(ScannerDetails.Offset, ScannerDetails.Data.Num()); const uint64 SafeFlushSize = ScannerDetails.bIsFinalScanner ? ScannerRange.GetLast() + 1 : ScannerRange.GetFirst(); if (SafeFlushSize > 0) { ChunkMatchProcessor->FlushLayer(0, SafeFlushSize); } DataScanners.RemoveAt(0); } // Handle extra matches accepted. TArray AcceptedChunkMatches; const FBlockRange CollectionRange = ChunkMatchProcessor->CollectLayer(0, AcceptedChunkMatches); if (CollectionRange.GetSize() > 0) { for (FMatchEntry& AcceptedChunkMatch : AcceptedChunkMatches) { const FChunkMatch& ChunkCMatch = AcceptedChunkMatch.ChunkMatch; const TArray& NewChunkReferences = ChunkBuildReferences[ChunkCMatch.ChunkGuid].Get<0>(); const FBlockStructure& ChunkCBuildBStructure = AcceptedChunkMatch.BlockStructure; ChunkMatchStomper->ReplaceChunkReferences(NewChunkReferences, ChunkCBuildBStructure); } } // Create new scanner. uint32 SizeToScan = FMath::Min(ScannerDataSize, BufferDataSize); const bool bHasData = SizeToScan == ScannerDataSize || (ManifestBStream->IsEndOfData() && BufferDataSize > 0); if (bHasData && !DeltaOptimiseHelpers::ScannerArrayFull(DataScanners)) { FDeltaScannerEntry* NewScanner = new FDeltaScannerEntry(); NewScanner->Data.Append(StreamBuffer.GetData() + ConsumedBufferData, SizeToScan); NewScanner->Offset = StreamStartPosition + ConsumedBufferData; FBlockStructure ScannerBuildStructure; ManifestBDesiredBytes.SelectSerialBytes(NewScanner->Offset, SizeToScan, ScannerBuildStructure); DeltaOptimiseHelpers::MakeScannerLocalList(FileListSearcher, DeltaChunkEnumeration.Get(), ScannerBuildStructure, NewScanner->FilesList); NewScanner->Scanner.Reset(FDeltaScannerFactory::Create(Configuration.ScanWindowSize, NewScanner->Data, NewScanner->FilesList, DeltaChunkEnumeration.Get(), StatsCollector.Get())); ConsumedBufferData += SizeToScan; NewScanner->bIsFinalScanner = ManifestBStream->IsEndOfData() && ConsumedBufferData >= StreamBuffer.Num(); if (!NewScanner->bIsFinalScanner) { ConsumedBufferData -= ScannerOverlapSize; } DataScanners.Emplace(NewScanner); ++NumScannersCreated; } // Fork a scanner with too much work? if (DataScanners.Num() > 0 && MeanScannerTime.IsReliable() && DeltaOptimiseHelpers::HasUnusedCpu()) { FDeltaScannerEntry& DataScannerEntry = *DataScanners[0]; const double TopScannerTime = DataScannerEntry.Scanner->GetTimeRunning(); double DownloadTimeMean; double DownloadTimeStd; MeanScannerTime.GetValues(DownloadTimeMean, DownloadTimeStd); const double BreakingPoint = FMath::Max(0.25, DownloadTimeMean + DownloadTimeStd); if (TopScannerTime > BreakingPoint && DataScannerEntry.Scanner->SupportsFork()) { DataScannerEntry.bWasFork = true; FStatsCollector::Accumulate(StatScannerForks, 1); const FBlockRange UnscannedRange = DataScannerEntry.Scanner->Fork(); const uint64 ForkSize = (UnscannedRange.GetSize() / 2) + 1; if (ForkSize < UnscannedRange.GetSize()) { // Insert the right fork first. const FBlockRange RightFork = FBlockRange::FromFirstAndLast(UnscannedRange.GetLast() - ForkSize, UnscannedRange.GetLast()); FDeltaScannerEntry* NewScanner = new FDeltaScannerEntry(); NewScanner->Data.Append(DataScannerEntry.Data.GetData() + RightFork.GetFirst(), RightFork.GetSize()); NewScanner->Offset = DataScannerEntry.Offset + RightFork.GetFirst(); FBlockStructure ScannerBuildStructure; ManifestBDesiredBytes.SelectSerialBytes(NewScanner->Offset, RightFork.GetSize(), ScannerBuildStructure); DeltaOptimiseHelpers::MakeScannerLocalList(FileListSearcher, DeltaChunkEnumeration.Get(), ScannerBuildStructure, NewScanner->FilesList); NewScanner->Scanner.Reset(FDeltaScannerFactory::Create(Configuration.ScanWindowSize, NewScanner->Data, NewScanner->FilesList, DeltaChunkEnumeration.Get(), StatsCollector.Get())); NewScanner->bIsFinalScanner = DataScannerEntry.bIsFinalScanner; NewScanner->bWasFork = true; DataScanners.EmplaceAt(1, NewScanner); // Insert the left fork. const FBlockRange LeftFork = FBlockRange::FromFirstAndLast(UnscannedRange.GetFirst(), UnscannedRange.GetFirst() + ForkSize); NewScanner = new FDeltaScannerEntry(); NewScanner->Data.Append(DataScannerEntry.Data.GetData() + LeftFork.GetFirst(), LeftFork.GetSize()); NewScanner->Offset = DataScannerEntry.Offset + LeftFork.GetFirst(); ScannerBuildStructure.Empty(); ManifestBDesiredBytes.SelectSerialBytes(NewScanner->Offset, LeftFork.GetSize(), ScannerBuildStructure); DeltaOptimiseHelpers::MakeScannerLocalList(FileListSearcher, DeltaChunkEnumeration.Get(), ScannerBuildStructure, NewScanner->FilesList); NewScanner->Scanner.Reset(FDeltaScannerFactory::Create(Configuration.ScanWindowSize, NewScanner->Data, NewScanner->FilesList, DeltaChunkEnumeration.Get(), StatsCollector.Get())); NewScanner->bIsFinalScanner = false; NewScanner->bWasFork = true; DataScanners.EmplaceAt(1, NewScanner); // Adjust original meta. DataScannerEntry.bIsFinalScanner = false; DataScannerEntry.Data.SetNumUninitialized(UnscannedRange.GetFirst(), EAllowShrinking::No); } else { // Something has gone wrong with the size calculation, this is fatal. check(ForkSize < UnscannedRange.GetSize()); } } } const double PercentScanned = (double)(NumScannersCreated - DataScanners.Num()) / (double)NumScannersRequired; FStatsCollector::SetAsPercentage(StatScanningCompleted, PercentScanned); FStatsCollector::Set(StatScannerBacklog, DataScanners.Num()); FStatsCollector::AccumulateTimeEnd(StatScanningTime, BuildBScanTimer); FStatsCollector::AccumulateTimeBegin(BuildBScanTimer); } FStatsCollector::AccumulateTimeEnd(StatScanningTime, BuildBScanTimer); FStatsCollector::SetAsPercentage(StatScanningCompleted, 1.0); ScanningTimer.Stop(); // Grab the new manifest data. FFileManifestList FileManifestList = ChunkMatchStomper->GetNewFileManifests(); // For all unknown data we need to re-chunk it out and fill in the gaps we have. FBlockStructure NewStreamBlocks; TArray> NewChunks; uint64 ByteLocation = 0; for (const FFileManifest& FileManifest : FileManifestList.FileList) { for (const FChunkPart& ChunkPart : FileManifest.ChunkParts) { if (ChunkPart.Guid.IsValid() == false) { uint64 PartByteLocation = ByteLocation; uint32 PartSizeRemaining = ChunkPart.Size; while (PartSizeRemaining > 0) { // Start new chunk? if (NewChunks.Num() == 0 || NewChunks.Last().Get<1>().Size >= OutputChunkSize) { NewChunks.AddDefaulted_GetRef().Get<1>().Guid = FGuid::NewGuid(); } TTuple* LastChunkDetail = &NewChunks.Last(); const uint32 NewTotalSize = LastChunkDetail->Get<1>().Size + PartSizeRemaining; const uint32 ChunkPartConsume = NewTotalSize > OutputChunkSize ? PartSizeRemaining - (NewTotalSize - OutputChunkSize) : PartSizeRemaining; check(PartSizeRemaining >= ChunkPartConsume); NewStreamBlocks.Add(PartByteLocation, ChunkPartConsume, ESearchDir::FromEnd); LastChunkDetail->Get<0>().Add(PartByteLocation, ChunkPartConsume, ESearchDir::FromEnd); LastChunkDetail->Get<1>().Size += ChunkPartConsume; PartByteLocation += ChunkPartConsume; PartSizeRemaining -= ChunkPartConsume; } } ByteLocation += ChunkPart.Size; } } // Save out all new chunk data. TMap NewChunkWindowSizes; TSet UpdatedFiles; FChunkSearcher ManifestSearcher(FileManifestList); FManifestBuildStreamerConfig UnknownDataStreamConfig({Configuration.CloudDirectory, NewStreamBlocks}); FManifestBuildStreamerDependencies UnknownDataStreamDependencies({ChunkReferenceTrackerFactory.Get(), CloudChunkSourceFactoryB.Get(), StatsCollector.Get(), ManifestB.Get()}); TUniquePtr UnknownDataStream(FBuildStreamerFactory::Create(MoveTemp(UnknownDataStreamConfig), MoveTemp(UnknownDataStreamDependencies))); TUniquePtr ChunkDataSerializationWriter(FChunkDataSerializationFactory::Create(FileSystem.Get(), ManifestB->ManifestMeta.FeatureLevel)); FParallelChunkWriterConfig ChunkWriterConfig = FParallelChunkWriterConfig({5, 5, 50, 8, Configuration.CloudDirectory, ManifestB->ManifestMeta.FeatureLevel}); TUniquePtr ChunkWriter(FParallelChunkWriterFactory::Create(ChunkWriterConfig, FileSystem.Get(), ChunkDataSerializationWriter.Get(), StatsCollector.Get())); StreamBuffer.SetNumUninitialized(0, AllowShrinking); for (const TTuple& NewChunk : NewChunks) { const FBlockStructure& NewChunkStructure = NewChunk.Get<0>(); const FChunkPart& NewChunkPart = NewChunk.Get<1>(); NewChunkWindowSizes.Add(NewChunkPart.Guid, NewChunkPart.Size); DeltaOptimiseHelpers::StompChunkPart(NewChunkPart, NewChunkStructure, ManifestSearcher, UpdatedFiles); // Collect all the chunk data. const FBlockEntry* NewChunkBlock = NewChunkStructure.GetHead(); StreamBuffer.SetNumUninitialized(NewChunkPart.Size, AllowShrinking); uint32 ChunkLocationOffset = 0; while (NewChunkBlock) { const uint32 SizeRead = UnknownDataStream->DequeueData(StreamBuffer.GetData() + ChunkLocationOffset, NewChunkBlock->GetSize()); check(SizeRead == NewChunkBlock->GetSize()); ChunkLocationOffset += NewChunkBlock->GetSize(); NewChunkBlock = NewChunkBlock->GetNext(); } check(ChunkLocationOffset == StreamBuffer.Num()); // Ensure padding if necessary. StreamBuffer.SetNumZeroed(OutputChunkSize, AllowShrinking); // Save out new chunk. const uint64 NewChunkHash = FRollingHash::GetHashForDataSet(StreamBuffer.GetData(), StreamBuffer.Num()); const FSHAHash NewChunkSha = DeltaOptimiseHelpers::GetShaForDataSet(StreamBuffer.GetData(), StreamBuffer.Num()); // Save it out. ChunkWriter->AddChunkData(StreamBuffer, NewChunkPart.Guid, NewChunkHash, NewChunkSha); } // We also need to potentially upgrade chunks from ManifestA into ManifestB feature level. if (bUsingDifferentChunkSubdir) { // Enumerate all chunks referenced from ManifestA. TSet SourceChunkReferences; FChunkSearcher::FFileDListNode* FileNode = ManifestSearcher.GetHead(); while (FileNode) { FChunkSearcher::FChunkDListNode* ChunkNode = FileNode->GetValue().ChunkParts.GetHead(); while (ChunkNode) { if (ManifestA->GetChunkInfo(ChunkNode->GetValue().ChunkPart.Guid) != nullptr) { SourceChunkReferences.Add(ChunkNode->GetValue().ChunkPart.Guid); } ChunkNode = ChunkNode->GetNextNode(); } FileNode = FileNode->GetNextNode(); } // Load these chunks and save them out in the new format. TUniquePtr UpgradeChunkReferenceTracker(ChunkReferenceTrackerFactory->Create(SourceChunkReferences.Array())); TUniquePtr UpgradeCloudChunkSource(CloudChunkSourceFactoryA->Create(UpgradeChunkReferenceTracker.Get())); for (const FGuid& UpgradeChunk : SourceChunkReferences) { const FChunkInfo* UpgradeChunkInfo = ManifestA->GetChunkInfo(UpgradeChunk); IChunkDataAccess* UpgradeChunkDataAccess = UpgradeCloudChunkSource->Get(UpgradeChunk); checkf(UpgradeChunkDataAccess != nullptr, TEXT("Failed to download chunk from source %s."), *FBuildPatchUtils::GetDataFilename(*ManifestA.Get(), UpgradeChunk)); FScopeLockedChunkData LockedChunkData(UpgradeChunkDataAccess); TArray ChunkDataArray(LockedChunkData.GetData(), LockedChunkData.GetHeader()->DataSizeUncompressed); ChunkWriter->AddChunkData(MoveTemp(ChunkDataArray), UpgradeChunk, UpgradeChunkInfo->Hash, UpgradeChunkInfo->ShaHash); UpgradeChunkReferenceTracker->PopReference(UpgradeChunk); } } // We always make sure padding chunks are saved out, so a legacy client could actually grab it. FSHAHash PaddingChunkSha; FGuid PaddingChunkId = PaddingChunk::MakePaddingGuid(0); NewChunkWindowSizes.Add(PaddingChunkId, PaddingChunk::ChunkSize); StreamBuffer.SetNumUninitialized(PaddingChunk::ChunkSize); FMemory::Memset(StreamBuffer.GetData(), PaddingChunkId.D, PaddingChunk::ChunkSize); FSHA1::HashBuffer(StreamBuffer.GetData(), PaddingChunk::ChunkSize, PaddingChunkSha.Hash); ChunkWriter->AddChunkData(StreamBuffer, PaddingChunkId, FRollingHash::GetHashForDataSet(StreamBuffer.GetData(), PaddingChunk::ChunkSize), PaddingChunkSha); for (uint32 LoopIdx = 1; LoopIdx <= 255; ++LoopIdx) { const uint8 Byte = LoopIdx & 0xFF; PaddingChunkId.D = Byte; NewChunkWindowSizes.Add(PaddingChunkId, PaddingChunk::ChunkSize); FMemory::Memset(StreamBuffer.GetData(), PaddingChunkId.D, PaddingChunk::ChunkSize); FSHA1::HashBuffer(StreamBuffer.GetData(), PaddingChunk::ChunkSize, PaddingChunkSha.Hash); ChunkWriter->AddChunkData(StreamBuffer, PaddingChunkId, FRollingHash::GetHashForDataSet(StreamBuffer.GetData(), PaddingChunk::ChunkSize), PaddingChunkSha); } // Complete chunk writer. FParallelChunkWriterSummaries ChunkWriterSummaries = ChunkWriter->OnProcessComplete(); // Produce the new stomped file manifests, but remove any that we no longer need if they don't actually change with the delta. FileManifestList = ManifestSearcher.BuildNewFileManifestList(); for (auto FileListIterator = FileManifestList.FileList.CreateIterator(); FileListIterator; ++FileListIterator) { if (ManifestB->IsFileOutdated(ManifestA.ToSharedRef(), (*FileListIterator).Filename) == false) { FileListIterator.RemoveCurrent(); } } // Save out the delta file. DeltaManifest.ManifestMeta = ManifestB->ManifestMeta; DeltaManifest.CustomFields = ManifestB->CustomFields; DeltaManifest.FileManifestList = MoveTemp(FileManifestList); TSet AddedChunkInfos; for (const FFileManifest& FileManifest : DeltaManifest.FileManifestList.FileList) { for (const FChunkPart& ChunkPart : FileManifest.ChunkParts) { bool bWasAlreadyInSet = false; AddedChunkInfos.Add(ChunkPart.Guid, &bWasAlreadyInSet); if (!bWasAlreadyInSet) { const FChunkInfo* OldChunkInfo = ManifestB->GetChunkInfo(ChunkPart.Guid); if (OldChunkInfo != nullptr) { DeltaManifest.ChunkDataList.ChunkList.Add(*OldChunkInfo); } else { OldChunkInfo = ManifestA->GetChunkInfo(ChunkPart.Guid); if (OldChunkInfo != nullptr) { DeltaManifest.ChunkDataList.ChunkList.Add(*OldChunkInfo); } else { FChunkInfo& NewChunkInfo = DeltaManifest.ChunkDataList.ChunkList.AddDefaulted_GetRef(); NewChunkInfo.Guid = ChunkPart.Guid; NewChunkInfo.Hash = ChunkWriterSummaries.ChunkOutputHashes[ChunkPart.Guid]; NewChunkInfo.ShaHash = ChunkWriterSummaries.ChunkOutputShas[ChunkPart.Guid]; NewChunkInfo.GroupNumber = FCrc::MemCrc32(&ChunkPart.Guid, sizeof(FGuid)) % 100; NewChunkInfo.WindowSize = NewChunkWindowSizes[ChunkPart.Guid]; NewChunkInfo.FileSize = ChunkWriterSummaries.ChunkOutputSizes[ChunkPart.Guid]; } } } } } DeltaManifest.InitLookups(); // For save format, we'll pick the newest of two input manifests, or EFeatureLevel::FirstOptimisedDelta if manifests are older. EFeatureLevel DeltaOutputFormat = FMath::Max3(EFeatureLevel::FirstOptimisedDelta, ManifestA->GetFeatureLevel(), ManifestB->GetFeatureLevel()); const FString TmpOutputDeltaFilename = OutputDeltaFilename + TEXT("tmp"); DeltaManifest.SaveToFile(TmpOutputDeltaFilename, DeltaOutputFormat); FileSystem->MoveFile(*OutputDeltaFilename, *TmpOutputDeltaFilename); FinalStatLogs.Add(FString::Printf(TEXT("Saved new optimised delta file %s"), *OutputDeltaFilename)); } if (bSuccess) { // Count stats? TSet ChunksUnknown = ChunksB.Difference(ChunksA); uint64 OriginalUnknownBytes = 0; for (const FString& ManifestBFile : ListHelpers::GetFileList(*ManifestB)) { for (const FChunkPart& ChunkPart : ManifestB->GetFileManifest(ManifestBFile)->ChunkParts) { if (ChunksUnknown.Contains(ChunkPart.Guid)) { OriginalUnknownBytes += ChunkPart.Size; } } } uint64 FinalUnknownBytes = 0; for (const FFileManifest& FileManifest : DeltaManifest.FileManifestList.FileList) { for (const FChunkPart& ChunkPart : FileManifest.ChunkParts) { const bool bDeltaUniqueChunk = ManifestA->GetChunkInfo(ChunkPart.Guid) == nullptr && ManifestB->GetChunkInfo(ChunkPart.Guid) == nullptr; if (bDeltaUniqueChunk) { FinalUnknownBytes += ChunkPart.Size; } } } uint64 FinalUnknownCompressedBytes = 0; TSet TempTest; for (const FChunkInfo& DeltaChunkInfo : DeltaManifest.ChunkDataList.ChunkList) { const bool bDeltaUniqueChunk = ManifestA->GetChunkInfo(DeltaChunkInfo.Guid) == nullptr && ManifestB->GetChunkInfo(DeltaChunkInfo.Guid) == nullptr; if (bDeltaUniqueChunk) { FinalUnknownCompressedBytes += DeltaChunkInfo.FileSize; check(TempTest.Contains(DeltaChunkInfo.Guid) == false); TempTest.Add(DeltaChunkInfo.Guid); } } int64 DeltaFileSize = 0; if (bRunProcess && !FileSystem->GetFileSize(*OutputDeltaFilename, DeltaFileSize)) { UE_LOG(LogChunkDeltaOptimiser, Error, TEXT("Could not save output to %s"), *OutputDeltaFilename); bSuccess = false; DeltaFileSize = 0; } ProcessTimer.Stop(); // Final improvement stat logs. if (bRunProcess || bDeltaPreviouslyCompleted) { FinalUnknownCompressedBytes += DeltaFileSize; FinalStatLogs.Add(FString::Printf(TEXT("Final unknown compressed bytes, plus meta %llu"), FinalUnknownCompressedBytes)); FinalStatLogs.Add(FString::Printf(TEXT("Original unknown compressed bytes %llu"), OriginalUnknownCompressedBytes)); if (OriginalUnknownCompressedBytes > FinalUnknownCompressedBytes) { FinalStatLogs.Add(FString::Printf(TEXT("Improvement: %s"), *FText::AsPercent(1.0 - ((double)FinalUnknownCompressedBytes / (double)OriginalUnknownCompressedBytes), &PercentFormat).ToString())); } } if (bRunProcess) { const FString TempMetaFilename = OutputDeltaFilename.Replace(TEXT("Deltas/"), TEXT("DeltaMetas/")).Replace(TEXT(".delta"), TEXT(".json")); FString JsonOutput; TSharedRef Writer = FDeltaJsonWriterFactory::Create(&JsonOutput); Writer->WriteObjectStart(); { Writer->WriteValue(TEXT("SourceBuildVersion"), ManifestA->GetVersionString()); Writer->WriteValue(TEXT("DestinationBuildVersion"), ManifestB->GetVersionString()); Writer->WriteValue(TEXT("OriginalUnknownBuildBytes"), (int64)OriginalUnknownBytes); Writer->WriteValue(TEXT("FinalUnknownBuildBytes"), (int64)FinalUnknownBytes); Writer->WriteValue(TEXT("OriginalUnknownCompressedBytes"), (int64)OriginalUnknownCompressedBytes); Writer->WriteValue(TEXT("FinalUnknownCompressedBytes"), (int64)FinalUnknownCompressedBytes); Writer->WriteValue(TEXT("ChunkBuildATime"), ChunkingTimer.GetSeconds()); Writer->WriteValue(TEXT("ScanBuildBTime"), ScanningTimer.GetSeconds()); Writer->WriteValue(TEXT("TotalProcessTime"), ProcessTimer.GetSeconds()); } Writer->WriteObjectEnd(); Writer->Close(); if (!FFileHelper::SaveStringToFile(JsonOutput, *TempMetaFilename)) { UE_LOG(LogChunkDeltaOptimiser, Error, TEXT("Could not save output to %s"), *TempMetaFilename); bSuccess = false; } } } } bShouldRun = false; return FinalStatLogs; } void FChunkDeltaOptimiser::HandleDownloadComplete(int32 RequestId, const FDownloadRef& Download) { TPromise* RelevantPromisePtr = RequestId == RequestIdManifestA ? &PromiseManifestA : RequestId == RequestIdManifestB ? &PromiseManifestB : nullptr; if (RelevantPromisePtr != nullptr) { if (Download->ResponseSuccessful()) { Async(EAsyncExecution::ThreadPool, [Download, RelevantPromisePtr]() { FBuildPatchAppManifestPtr Manifest = MakeShareable(new FBuildPatchAppManifest()); if (!Manifest->DeserializeFromData(Download->GetData())) { Manifest.Reset(); } RelevantPromisePtr->SetValue(Manifest); }); } else { RelevantPromisePtr->SetValue(FBuildPatchAppManifestPtr()); } } } FBlockStructure FChunkDeltaOptimiser::GetDesiredBytes(const FBuildPatchAppManifestPtr& Manifest, const TSet& UnknownChunks) { uint64 UnknownCount = 0; FBlockStructure DesiredBytes; uint64 ChunkPartCount = 0; for (const FString& BuildFile : ListHelpers::GetFileList(*Manifest.Get())) { const FFileManifest* FileManifest = Manifest->GetFileManifest(BuildFile); for (const FChunkPart& ChunkPart : FileManifest->ChunkParts) { if (UnknownChunks.Contains(ChunkPart.Guid)) { DesiredBytes.Add(ChunkPartCount, ChunkPart.Size, ESearchDir::FromEnd); UnknownCount += ChunkPart.Size; } ChunkPartCount += ChunkPart.Size; } } check(UnknownCount == BlockStructureHelpers::CountSize(DesiredBytes)); return DesiredBytes; } IChunkDeltaOptimiser* FChunkDeltaOptimiserFactory::Create(const FChunkDeltaOptimiserConfiguration& Configuration) { return new FChunkDeltaOptimiser(Configuration); } }