769 lines
37 KiB
C++
769 lines
37 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "BuildPatchGeneration.h"
|
|
#include "Templates/UniquePtr.h"
|
|
#include "Templates/Greater.h"
|
|
#include "Templates/Tuple.h"
|
|
#include "Algo/Sort.h"
|
|
#include "Misc/ConfigCacheIni.h"
|
|
#include "Misc/SecureHash.h"
|
|
#include "Misc/OutputDeviceRedirector.h"
|
|
#include "Misc/CommandLine.h"
|
|
#include "Containers/Ticker.h"
|
|
#include "Containers/Queue.h"
|
|
#include "Async/Future.h"
|
|
#include "Async/Async.h"
|
|
#include "BuildPatchManifest.h"
|
|
#include "BuildPatchServicesModule.h"
|
|
#include "BuildPatchHash.h"
|
|
#include "Core/BlockStructure.h"
|
|
#include "Core/BlockData.h"
|
|
#include "Common/FileSystem.h"
|
|
#include "Data/ChunkData.h"
|
|
#include "Generation/DataScanner.h"
|
|
#include "Generation/BuildStreamer.h"
|
|
#include "Generation/CloudEnumeration.h"
|
|
#include "Generation/ManifestBuilder.h"
|
|
#include "Generation/FileAttributesParser.h"
|
|
#include "Generation/ChunkWriter.h"
|
|
#include "Generation/ChunkMatchProcessor.h"
|
|
#include "BuildPatchUtil.h"
|
|
|
|
using namespace BuildPatchServices;
|
|
|
|
DECLARE_LOG_CATEGORY_EXTERN(LogPatchGeneration, Log, All);
|
|
DEFINE_LOG_CATEGORY(LogPatchGeneration);
|
|
|
|
namespace BuildPatchServices
|
|
{
|
|
struct FScannerDetails
|
|
{
|
|
public:
|
|
FScannerDetails(int32 InLayer, uint64 InLayerOffset, bool bInIsFinalScanner, uint64 InPaddingSize, TArray<uint8> InData, FBlockStructure InStructure, const TArray<uint32>& ChunkWindowSizes, const ICloudEnumeration* CloudEnumeration, FStatsCollector* StatsCollector)
|
|
: Layer(InLayer)
|
|
, LayerOffset(InLayerOffset)
|
|
, bIsFinalScanner(bInIsFinalScanner)
|
|
, PaddingSize(InPaddingSize)
|
|
, Data(MoveTemp(InData))
|
|
, Structure(MoveTemp(InStructure))
|
|
, Scanner(FDataScannerFactory::Create(ChunkWindowSizes, Data, CloudEnumeration, StatsCollector))
|
|
{}
|
|
|
|
public:
|
|
const int32 Layer;
|
|
const uint64 LayerOffset;
|
|
const bool bIsFinalScanner;
|
|
const uint64 PaddingSize;
|
|
const TArray<uint8> Data;
|
|
const FBlockStructure Structure;
|
|
const TUniquePtr<IDataScanner> Scanner;
|
|
};
|
|
}
|
|
|
|
namespace PatchGenerationHelpers
|
|
{
|
|
int32 GetMaxScannerBacklogCount()
|
|
{
|
|
int32 MaxScannerBacklogCount = 75;
|
|
GConfig->GetInt(TEXT("BuildPatchServices"), TEXT("MaxScannerBacklog"), MaxScannerBacklogCount, GEngineIni);
|
|
MaxScannerBacklogCount = FMath::Clamp<int32>(MaxScannerBacklogCount, 5, 500);
|
|
return MaxScannerBacklogCount;
|
|
}
|
|
|
|
bool ScannerArrayFull(const TArray<TUniquePtr<FScannerDetails>>& Scanners)
|
|
{
|
|
static int32 MaxScannerBacklogCount = GetMaxScannerBacklogCount();
|
|
return (FDataScannerCounter::GetNumIncompleteScanners() > FDataScannerCounter::GetNumRunningScanners()) || (Scanners.Num() >= MaxScannerBacklogCount);
|
|
}
|
|
|
|
FSHAHash GetShaForDataSet(const uint8* DataSet, uint32 DataSize)
|
|
{
|
|
FSHAHash SHAHash;
|
|
FSHA1::HashBuffer(DataSet, DataSize, SHAHash.Hash);
|
|
return SHAHash;
|
|
}
|
|
|
|
}
|
|
|
|
bool FBuildDataGenerator::ChunkBuildDirectory(const BuildPatchServices::FChunkBuildConfiguration& Settings)
|
|
{
|
|
const uint64 StartTime = FStatsCollector::GetCycles();
|
|
bool bIsGameThread = IsInGameThread();
|
|
|
|
// Check for the required output filename.
|
|
if (Settings.OutputFilename.IsEmpty())
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Manifest OutputFilename was not provided"));
|
|
return false;
|
|
}
|
|
|
|
// Ensure that cloud directory exists, and create it if not.
|
|
IFileManager::Get().MakeDirectory(*Settings.CloudDirectory, true);
|
|
if (!IFileManager::Get().DirectoryExists(*Settings.CloudDirectory))
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Unable to create specified cloud directory %s"), *Settings.CloudDirectory);
|
|
return false;
|
|
}
|
|
|
|
// Output to log for builder info.
|
|
UE_LOG(LogPatchGeneration, Log, TEXT("Running NEW Chunks Patch Generation for: %u:%s %s"), Settings.AppId, *Settings.AppName, *Settings.BuildVersion);
|
|
|
|
// The last time we logged out data processed.
|
|
double LastProgressLog = FPlatformTime::Seconds();
|
|
const double TimeGenStarted = LastProgressLog;
|
|
|
|
// Load settings from config.
|
|
float GenerationScannerSizeMegabytes = 32.5f;
|
|
float StatsLoggerTimeSeconds = 10.0f;
|
|
GConfig->GetFloat(TEXT("BuildPatchServices"), TEXT("GenerationScannerSizeMegabytes"), GenerationScannerSizeMegabytes, GEngineIni);
|
|
GConfig->GetFloat(TEXT("BuildPatchServices"), TEXT("StatsLoggerTimeSeconds"), StatsLoggerTimeSeconds, GEngineIni);
|
|
GenerationScannerSizeMegabytes = FMath::Clamp<float>(GenerationScannerSizeMegabytes, 10.0f, 500.0f);
|
|
StatsLoggerTimeSeconds = FMath::Clamp<float>(StatsLoggerTimeSeconds, 1.0f, 60.0f);
|
|
const uint64 ScannerDataSize = GenerationScannerSizeMegabytes * 1048576;
|
|
|
|
// Create stat collector.
|
|
TUniquePtr<FStatsCollector> StatsCollector(FStatsCollectorFactory::Create());
|
|
|
|
// Setup Generation stats.
|
|
volatile int64* StatTotalTime = StatsCollector->CreateStat(TEXT("Generation: Total Time"), EStatFormat::Timer);
|
|
volatile int64* StatLayers = StatsCollector->CreateStat(TEXT("Generation: Layers"), EStatFormat::Value);
|
|
volatile int64* StatNumScanners = StatsCollector->CreateStat(TEXT("Generation: Scanner Backlog"), EStatFormat::Value);
|
|
volatile int64* StatUnknownDataAlloc = StatsCollector->CreateStat(TEXT("Generation: Unmatched Buffers Allocation"), EStatFormat::DataSize);
|
|
volatile int64* StatUnknownDataNum = StatsCollector->CreateStat(TEXT("Generation: Unmatched Buffers Use"), EStatFormat::DataSize);
|
|
int64 MaxLayer = 0;
|
|
|
|
// Create a chunk writer.
|
|
TUniquePtr<IFileSystem> FileSystem(FFileSystemFactory::Create());
|
|
TUniquePtr<IChunkDataSerialization> ChunkDataSerialization(FChunkDataSerializationFactory::Create(FileSystem.Get(), Settings.FeatureLevel));
|
|
TUniquePtr<IParallelChunkWriter> ChunkWriter(FParallelChunkWriterFactory::Create({5, 5, 50, 8, Settings.CloudDirectory, Settings.FeatureLevel}, FileSystem.Get(), ChunkDataSerialization.Get(), StatsCollector.Get()));
|
|
|
|
// Create a manifest details.
|
|
FManifestDetails ManifestDetails;
|
|
ManifestDetails.FeatureLevel = Settings.FeatureLevel;
|
|
ManifestDetails.AppId = Settings.AppId;
|
|
ManifestDetails.AppName = Settings.AppName;
|
|
ManifestDetails.BuildVersion = Settings.BuildVersion;
|
|
ManifestDetails.LaunchExe = Settings.LaunchExe;
|
|
ManifestDetails.LaunchCommand = Settings.LaunchCommand;
|
|
ManifestDetails.PrereqIds = Settings.PrereqIds;
|
|
ManifestDetails.PrereqName = Settings.PrereqName;
|
|
ManifestDetails.PrereqPath = Settings.PrereqPath;
|
|
ManifestDetails.PrereqArgs = Settings.PrereqArgs;
|
|
ManifestDetails.CustomFields = Settings.CustomFields;
|
|
|
|
// Load the required file attributes.
|
|
if (!Settings.AttributeListFile.IsEmpty())
|
|
{
|
|
FFileAttributesParserRef FileAttributesParser = FFileAttributesParserFactory::Create();
|
|
if (!FileAttributesParser->ParseFileAttributes(Settings.AttributeListFile, ManifestDetails.FileAttributesMap))
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Attributes list file did not parse %s"), *Settings.AttributeListFile);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// Enumerate Chunks.
|
|
const FDateTime Cutoff = Settings.bShouldHonorReuseThreshold ? FDateTime::UtcNow() - FTimespan::FromDays(Settings.DataAgeThreshold) : FDateTime::MinValue();
|
|
TUniquePtr<ICloudEnumeration> CloudEnumeration(FCloudEnumerationFactory::Create(Settings.CloudDirectory, Cutoff, Settings.FeatureLevel, StatsCollector.Get()));
|
|
|
|
// Start the build stream.
|
|
FDirectoryBuildStreamerConfig BuildStreamConfig({Settings.RootDirectory, Settings.InputListFile, Settings.IgnoreListFile});
|
|
FDirectoryBuildStreamerDependencies BuildStreamDependencies({StatsCollector.Get(), FileSystem.Get()});
|
|
TUniquePtr<IDirectoryBuildStreamer> BuildStream(FBuildStreamerFactory::Create(MoveTemp(BuildStreamConfig), MoveTemp(BuildStreamDependencies)));
|
|
TArray<FString> EnumeratedFiles = BuildStream->GetAllFilenames();
|
|
|
|
// Check existence of launch exe, if specified.
|
|
if (!Settings.LaunchExe.IsEmpty() && !EnumeratedFiles.Contains(FPaths::Combine(Settings.RootDirectory, Settings.LaunchExe)))
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Provided launch executable file was not found within the build root. %s"), *Settings.LaunchExe);
|
|
return false;
|
|
}
|
|
|
|
// Check existence of prereq exe, if specified.
|
|
if (!Settings.PrereqPath.IsEmpty() && !EnumeratedFiles.Contains(FPaths::Combine(Settings.RootDirectory, Settings.PrereqPath)))
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Provided prerequisite executable file was not found within the build root. %s"), *Settings.PrereqPath);
|
|
return false;
|
|
}
|
|
|
|
// We've got to wait for enumeration to complete as that shares a thread pool.
|
|
while (CloudEnumeration->IsComplete() == false)
|
|
{
|
|
// Log collected stats.
|
|
if (bIsGameThread)
|
|
{
|
|
GLog->FlushThreadedLogs();
|
|
}
|
|
FStatsCollector::Set(StatTotalTime, FStatsCollector::GetCycles() - StartTime);
|
|
StatsCollector->LogStats(StatsLoggerTimeSeconds);
|
|
|
|
// Sleep to allow other threads.
|
|
FPlatformProcess::Sleep(0.01f);
|
|
}
|
|
|
|
// Grab the window sizes we are trying to match against.
|
|
TArray<uint32> WindowSizes;
|
|
if (Settings.bShouldMatchAnyWindowSize)
|
|
{
|
|
WindowSizes.Append(CloudEnumeration->GetUniqueWindowSizes().Array());
|
|
}
|
|
else
|
|
{
|
|
WindowSizes.Add(Settings.OutputChunkWindowSize);
|
|
}
|
|
Algo::Sort(WindowSizes, TGreater<uint32>());
|
|
const uint32 LargestWindowSize = FMath::Max<uint32>(Settings.OutputChunkWindowSize, WindowSizes.Num() > 0 ? WindowSizes[0] : 0);
|
|
const uint64 ScannerOverlapSize = LargestWindowSize - 1;
|
|
|
|
// Construct the chunk match processor.
|
|
TUniquePtr<IChunkMatchProcessor> ChunkMatchProcessor(FChunkMatchProcessorFactory::Create());
|
|
|
|
// Keep a record of the new chunk inventory.
|
|
TMap<uint64, TSet<FGuid>> ChunkInventory;
|
|
TMap<FGuid, FSHAHash> ChunkShaHashes;
|
|
|
|
// Tracking info per layer for rescanning.
|
|
TMap<int32, uint64> LayerToScannerCount;
|
|
TMap<int32, FBlockStructure> LayerToBuildSpaceStructure;
|
|
TMap<int32, uint64> LayerToCreatedScannerOffset;
|
|
TMap<int32, uint64> LayerToScannedSize;
|
|
TMap<int32, uint64> LayerToTotalDataSize;
|
|
TMap<int32, FBlockStructure> LayerToUnknownLayerSpaceStructure;
|
|
TMap<int32, FBlockStructure> LayerToUnknownBuildSpaceStructure;
|
|
TMap<int32, TBlockData<uint8>> LayerToLayerSpaceBlockData;
|
|
|
|
// This is a blatant hack :(
|
|
TMap<FGuid, uint32> OriginalWindowSizes;
|
|
|
|
// Create the manifest builder.
|
|
FManifestBuilderConfig ManifestBuilderConfig;
|
|
ManifestBuilderConfig.bAllowEmptyBuilds = Settings.bAllowEmptyBuild;
|
|
IManifestBuilderRef ManifestBuilder = FManifestBuilderFactory::Create(FileSystem.Get(), ManifestBuilderConfig, ManifestDetails);
|
|
|
|
FBlockStructure AcceptedBuildSpaceMatches;
|
|
FBlockStructure CreatedBuildSpaceMatches;
|
|
TSet<FGuid> NewCreatedChunks;
|
|
TMap<int32, FBlockStructure> LayerCreatingScannersTest;
|
|
TMap<int32, FBlockStructure> LayerCreatingScannersLayerSpaceTest;
|
|
TMap<int32, FBlockStructure> LayerCreatingChunksTest;
|
|
TMap<int32, bool> LayerCreatingFinalTest;
|
|
|
|
// Run the main loop.
|
|
TArray<uint8> DataBuffer;
|
|
uint64 DataBufferFirstIdx = 0;
|
|
uint32 ReadLen = 0;
|
|
TArray<TUniquePtr<FScannerDetails>> Scanners;
|
|
bool bHasUnknownData = true;
|
|
while (!BuildStream->HasAborted() && (!BuildStream->IsEndOfData() || Scanners.Num() > 0 || bHasUnknownData))
|
|
{
|
|
// Grab a scanner result.
|
|
if (Scanners.Num() > 0 && Scanners[0]->Scanner->IsComplete())
|
|
{
|
|
FScannerDetails& ScannerDetails = *Scanners[0];
|
|
TArray<FChunkMatch> ChunkMatches = ScannerDetails.Scanner->GetResultWhenComplete();
|
|
for (FChunkMatch& ChunkMatch : ChunkMatches)
|
|
{
|
|
// Translate to build space.
|
|
FBlockStructure BuildSpaceChunkStructure;
|
|
const uint64 BytesFound = ScannerDetails.Structure.SelectSerialBytes(ChunkMatch.DataOffset, ChunkMatch.WindowSize, BuildSpaceChunkStructure);
|
|
const bool bFoundOk = ScannerDetails.bIsFinalScanner || BytesFound == ChunkMatch.WindowSize;
|
|
if (!bFoundOk)
|
|
{
|
|
// Fatal error if the scanner returned a matched range that doesn't fit inside it's data.
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Chunk match was not within scanner's data structure."));
|
|
return false;
|
|
}
|
|
|
|
ChunkMatch.DataOffset += ScannerDetails.LayerOffset;
|
|
if (ChunkMatch.WindowSize != BytesFound)
|
|
{
|
|
OriginalWindowSizes.Add(ChunkMatch.ChunkGuid, ChunkMatch.WindowSize);
|
|
}
|
|
ChunkMatch.WindowSize = BytesFound;
|
|
|
|
ChunkMatchProcessor->ProcessMatch(ScannerDetails.Layer, ChunkMatch, MoveTemp(BuildSpaceChunkStructure));
|
|
}
|
|
|
|
FBlockStructure OverlapStructure = LayerToBuildSpaceStructure.FindOrAdd(ScannerDetails.Layer).Intersect(ScannerDetails.Structure);
|
|
uint64 OverlapBytes = BlockStructureHelpers::CountSize(OverlapStructure);
|
|
check(OverlapBytes == ScannerOverlapSize || ScannerDetails.LayerOffset == 0);
|
|
|
|
// Store the layer build space.
|
|
LayerToBuildSpaceStructure.FindOrAdd(ScannerDetails.Layer).Add(ScannerDetails.Structure);
|
|
|
|
// Add to layer space block data. We include padding that comes at the end of any layer as that may be included.
|
|
const uint64 LayerDataStartIndex = ScannerDetails.LayerOffset == 0 ? 0 : ScannerOverlapSize;
|
|
const uint64 LayerDataSize = ScannerDetails.LayerOffset == 0 ? ScannerDetails.Data.Num() : ScannerDetails.Data.Num() - ScannerOverlapSize;
|
|
TBlockData<uint8>& LayerSpaceBlockData = LayerToLayerSpaceBlockData.FindOrAdd(ScannerDetails.Layer);
|
|
LayerSpaceBlockData.AddData(FBlockStructure(ScannerDetails.LayerOffset + LayerDataStartIndex, LayerDataSize), ScannerDetails.Data.GetData() + LayerDataStartIndex, LayerDataSize);
|
|
|
|
// Give some flush time to the processor.
|
|
check(ScannerDetails.PaddingSize == 0 || ScannerDetails.bIsFinalScanner);
|
|
const FBlockRange ScannerRange = FBlockRange::FromFirstAndSize(ScannerDetails.LayerOffset, ScannerDetails.Data.Num() - ScannerDetails.PaddingSize);
|
|
const uint64 SafeFlushSize = ScannerDetails.bIsFinalScanner ? ScannerRange.GetLast() + 1 : ScannerRange.GetLast() - ScannerOverlapSize;
|
|
ChunkMatchProcessor->FlushLayer(ScannerDetails.Layer, SafeFlushSize);
|
|
if (ScannerDetails.bIsFinalScanner)
|
|
{
|
|
LayerToTotalDataSize.Add(ScannerDetails.Layer, SafeFlushSize);
|
|
}
|
|
|
|
// Remove scanner from list.
|
|
LayerToScannerCount.FindOrAdd(ScannerDetails.Layer)--;
|
|
Scanners.RemoveAt(0);
|
|
}
|
|
|
|
// Handle accepted chunk matches, and unknown data tracking.
|
|
for (int32 LayerIdx = 0; LayerIdx <= MaxLayer; ++LayerIdx)
|
|
{
|
|
TArray<FMatchEntry> AcceptedChunkMatches;
|
|
const FBlockStructure& LayerBuildSpaceStructure = LayerToBuildSpaceStructure.FindOrAdd(LayerIdx);
|
|
uint64& LayerScannedSize = LayerToScannedSize.FindOrAdd(LayerIdx);
|
|
const FBlockRange CollectionRange = ChunkMatchProcessor->CollectLayer(LayerIdx, AcceptedChunkMatches);
|
|
if (CollectionRange.GetSize() > 0)
|
|
{
|
|
// Add new chunk matches to the manifest builder, and track new unknown data.
|
|
TBlockData<uint8>& LayerSpaceBlockData = LayerToLayerSpaceBlockData.FindOrAdd(LayerIdx);
|
|
FBlockStructure BlockDataToRemove;
|
|
FBlockStructure NewUnknownLayerSpaceStructure(CollectionRange.GetFirst(), CollectionRange.GetSize());
|
|
FBlockStructure NewUnknownBuildSpaceStructure;
|
|
const uint64 BytesFound = LayerBuildSpaceStructure.SelectSerialBytes(CollectionRange.GetFirst(), CollectionRange.GetSize(), NewUnknownBuildSpaceStructure);
|
|
checkSlow(BytesFound == CollectionRange.GetSize());
|
|
const uint64 BeforeDataCount = LayerSpaceBlockData.GetDataCount();
|
|
const uint64 BeforeStructureCount = CollectionRange.GetSize();
|
|
for (FMatchEntry& AcceptedChunkMatch : AcceptedChunkMatches)
|
|
{
|
|
const FChunkMatch& ChunkMatch = AcceptedChunkMatch.ChunkMatch;
|
|
const FBlockStructure& BlockStructure = AcceptedChunkMatch.BlockStructure;
|
|
const FBlockStructure LayerSpaceStructure(ChunkMatch.DataOffset, ChunkMatch.WindowSize);
|
|
|
|
check(BlockStructureHelpers::CountSize(LayerSpaceStructure.Intersect(NewUnknownLayerSpaceStructure)) == ChunkMatch.WindowSize);
|
|
check(BlockStructureHelpers::CountSize(LayerSpaceStructure.Intersect(BlockDataToRemove)) == 0);
|
|
check(BlockStructureHelpers::CountSize(NewUnknownBuildSpaceStructure.Intersect(BlockStructure)) == ChunkMatch.WindowSize);
|
|
checkf(CreatedBuildSpaceMatches.Intersect(BlockStructure).GetHead() == nullptr, TEXT("ACCEPTEDCHUNK Overlap %llu bytes with created struct!"), BlockStructureHelpers::CountSize(CreatedBuildSpaceMatches.Intersect(BlockStructure)));
|
|
checkf(AcceptedBuildSpaceMatches.Intersect(BlockStructure).GetHead() == nullptr, TEXT("ACCEPTEDCHUNK Overlap %llu bytes with accepted struct!"), BlockStructureHelpers::CountSize(AcceptedBuildSpaceMatches.Intersect(BlockStructure)));
|
|
|
|
AcceptedBuildSpaceMatches.Add(BlockStructure);
|
|
NewUnknownBuildSpaceStructure.Remove(BlockStructure);
|
|
ManifestBuilder->AddChunkMatch(ChunkMatch.ChunkGuid, BlockStructure);
|
|
// Do we need to re-save the chunk at current feature level?
|
|
if (!CloudEnumeration->IsChunkFeatureLevelMatch(ChunkMatch.ChunkGuid))
|
|
{
|
|
// Grab the data.
|
|
TArray<uint8> ChunkDataArray;
|
|
LayerSpaceBlockData.CopyTo(ChunkDataArray, LayerSpaceStructure);
|
|
check(ChunkDataArray.Num() == ChunkMatch.WindowSize);
|
|
// Ensure padding if necessary.
|
|
const uint32 TrueWindowSize = OriginalWindowSizes.Contains(ChunkMatch.ChunkGuid) ? OriginalWindowSizes[ChunkMatch.ChunkGuid] : ChunkMatch.WindowSize;
|
|
ChunkDataArray.SetNumZeroed(TrueWindowSize);
|
|
|
|
// Save it out.
|
|
const uint64& ChunkHash = CloudEnumeration->GetChunkHash(ChunkMatch.ChunkGuid);
|
|
const FSHAHash& ChunkSha = CloudEnumeration->GetChunkShaHash(ChunkMatch.ChunkGuid);
|
|
check(ChunkHash == FRollingHash::GetHashForDataSet(ChunkDataArray.GetData(), TrueWindowSize));
|
|
check(ChunkSha == PatchGenerationHelpers::GetShaForDataSet(ChunkDataArray.GetData(), TrueWindowSize));
|
|
ChunkWriter->AddChunkData(MoveTemp(ChunkDataArray), ChunkMatch.ChunkGuid, ChunkHash, ChunkSha);
|
|
}
|
|
NewUnknownLayerSpaceStructure.Remove(LayerSpaceStructure);
|
|
BlockDataToRemove.Add(LayerSpaceStructure);
|
|
}
|
|
const uint64 BlockDataToRemoveSize = BlockStructureHelpers::CountSize(BlockDataToRemove);
|
|
LayerSpaceBlockData.RemoveData(BlockDataToRemove);
|
|
const uint64 AfterDataCount = LayerSpaceBlockData.GetDataCount();
|
|
const uint64 AfterStructureCount = BlockStructureHelpers::CountSize(NewUnknownLayerSpaceStructure);
|
|
const uint64 RemovedDataCount = BeforeDataCount - AfterDataCount;
|
|
const uint64 RemovedStructureCount = BeforeStructureCount - AfterStructureCount;
|
|
check(BeforeDataCount >= AfterDataCount);
|
|
check(BeforeStructureCount >= AfterStructureCount);
|
|
check(RemovedDataCount == RemovedStructureCount);
|
|
check(RemovedDataCount == BlockDataToRemoveSize);
|
|
|
|
check(BlockStructureHelpers::CountSize(NewUnknownLayerSpaceStructure) == BlockStructureHelpers::CountSize(NewUnknownBuildSpaceStructure));
|
|
|
|
// Grab layer tracking.
|
|
FBlockStructure& UnknownLayerSpaceStructure = LayerToUnknownLayerSpaceStructure.FindOrAdd(LayerIdx);
|
|
FBlockStructure& UnknownBuildSpaceStructure = LayerToUnknownBuildSpaceStructure.FindOrAdd(LayerIdx);
|
|
|
|
// Expect to never get overlap with this new system.
|
|
check(!BlockStructureHelpers::HasIntersection(UnknownLayerSpaceStructure, NewUnknownLayerSpaceStructure));
|
|
check(!BlockStructureHelpers::HasIntersection(UnknownBuildSpaceStructure, NewUnknownBuildSpaceStructure));
|
|
|
|
// Add unknown tracking to the structures.
|
|
UnknownLayerSpaceStructure.Add(NewUnknownLayerSpaceStructure);
|
|
UnknownBuildSpaceStructure.Add(NewUnknownBuildSpaceStructure);
|
|
check(BlockStructureHelpers::CountSize(UnknownLayerSpaceStructure) == BlockStructureHelpers::CountSize(UnknownBuildSpaceStructure));
|
|
|
|
// Count processed data
|
|
LayerScannedSize = CollectionRange.GetLast() + 1;
|
|
}
|
|
}
|
|
|
|
// Collect unknown data into new chunks.
|
|
for (int32 LayerIdx = 0; LayerIdx <= MaxLayer; ++LayerIdx)
|
|
{
|
|
FBlockStructure& UnknownLayerSpaceStructure = LayerToUnknownLayerSpaceStructure.FindOrAdd(LayerIdx);
|
|
FBlockStructure& UnknownBuildSpaceStructure = LayerToUnknownBuildSpaceStructure.FindOrAdd(LayerIdx);
|
|
|
|
check(BlockStructureHelpers::CountSize(UnknownLayerSpaceStructure) == BlockStructureHelpers::CountSize(UnknownBuildSpaceStructure));
|
|
|
|
const uint64& LayerScannedSize = LayerToScannedSize.FindOrAdd(LayerIdx);
|
|
const bool bLayerComplete = LayerToTotalDataSize.Contains(LayerIdx) && (LayerScannedSize >= LayerToTotalDataSize[LayerIdx]);
|
|
|
|
FBlockStructure ChunkedLayerSpaceStructure;
|
|
FBlockStructure ChunkedBuildSpaceStructure;
|
|
const FBlockEntry* UnknownLayerBlock = UnknownLayerSpaceStructure.GetHead();
|
|
const bool bIsFinalSingleBlock = bLayerComplete && UnknownLayerBlock != nullptr && UnknownLayerBlock == UnknownLayerSpaceStructure.GetTail();
|
|
uint64 UnknownBlockByteCount = 0;
|
|
|
|
if (UnknownLayerBlock != nullptr)
|
|
{
|
|
UE_LOG(LogPatchGeneration, VeryVerbose, TEXT("Unknown layer[%d] data at %llu bytes"), LayerIdx, BlockStructureHelpers::CountSize(UnknownLayerSpaceStructure));
|
|
}
|
|
|
|
TBlockData<uint8>& LayerSpaceBlockData = LayerToLayerSpaceBlockData.FindOrAdd(LayerIdx);
|
|
FBlockStructure BlockDataToRemove;
|
|
while (UnknownLayerBlock != nullptr)
|
|
{
|
|
uint64 UnknownBlockOffset = UnknownLayerBlock->GetOffset();
|
|
uint64 UnknownBlockSize = UnknownLayerBlock->GetSize();
|
|
bool bFinalLayerChunk = false;
|
|
while (UnknownBlockSize >= LargestWindowSize || (bIsFinalSingleBlock && !bFinalLayerChunk))
|
|
{
|
|
// Copy out the chunk data.
|
|
FBlockStructure NewChunkLayerSpace(UnknownBlockOffset, FMath::Min<uint64>(Settings.OutputChunkWindowSize, UnknownBlockSize));
|
|
checkf((BlockStructureHelpers::CountSize(NewChunkLayerSpace) == Settings.OutputChunkWindowSize) || bIsFinalSingleBlock, TEXT("Expected correct chunk size! (%llu == %u) || %d"), BlockStructureHelpers::CountSize(NewChunkLayerSpace), Settings.OutputChunkWindowSize, bIsFinalSingleBlock);
|
|
TArray<uint8> NewChunkDataArray;
|
|
LayerSpaceBlockData.CopyTo(NewChunkDataArray, NewChunkLayerSpace);
|
|
check( bIsFinalSingleBlock || NewChunkDataArray.Num() == Settings.OutputChunkWindowSize);
|
|
check(!bIsFinalSingleBlock || NewChunkDataArray.Num() == FMath::Min<uint64>(Settings.OutputChunkWindowSize, UnknownBlockSize));
|
|
// Ensure padding if necessary.
|
|
NewChunkDataArray.SetNumZeroed(Settings.OutputChunkWindowSize);
|
|
|
|
// Create data for new chunk.
|
|
const FGuid NewChunkGuid = FGuid::NewGuid();
|
|
const uint64 NewChunkHash = FRollingHash::GetHashForDataSet(NewChunkDataArray.GetData(), Settings.OutputChunkWindowSize);
|
|
const FSHAHash NewChunkSha = PatchGenerationHelpers::GetShaForDataSet(NewChunkDataArray.GetData(), Settings.OutputChunkWindowSize);
|
|
|
|
// Save it out.
|
|
ChunkWriter->AddChunkData(MoveTemp(NewChunkDataArray), NewChunkGuid, NewChunkHash, NewChunkSha);
|
|
ChunkShaHashes.Add(NewChunkGuid, NewChunkSha);
|
|
ChunkInventory.FindOrAdd(NewChunkHash).Add(NewChunkGuid);
|
|
BlockDataToRemove.Add(NewChunkLayerSpace);
|
|
|
|
UE_LOG(LogPatchGeneration, Verbose, TEXT("Created layer[%d] chunk @ %llu for %d out of %llu"), LayerIdx, UnknownBlockOffset, Settings.OutputChunkWindowSize, UnknownBlockSize);
|
|
|
|
// Add to manifest builder.
|
|
FBlockStructure BuildSpaceChunkStructure;
|
|
const uint64 ChunkBuildSize = UnknownBuildSpaceStructure.SelectSerialBytes(UnknownBlockByteCount, Settings.OutputChunkWindowSize, BuildSpaceChunkStructure);
|
|
bFinalLayerChunk = bIsFinalSingleBlock && (UnknownBlockSize == ChunkBuildSize);
|
|
|
|
// Chunk build space should either be window size, or size minus any padding if the final piece.
|
|
check( bIsFinalSingleBlock || ChunkBuildSize == Settings.OutputChunkWindowSize);
|
|
check(!bIsFinalSingleBlock || ChunkBuildSize == FMath::Min<uint64>(Settings.OutputChunkWindowSize, UnknownBlockSize));
|
|
|
|
// This new chunk must not overlap any previous chunks.
|
|
check(CreatedBuildSpaceMatches.Intersect(BuildSpaceChunkStructure).GetHead() == nullptr);
|
|
check(AcceptedBuildSpaceMatches.Intersect(BuildSpaceChunkStructure).GetHead() == nullptr);
|
|
|
|
CreatedBuildSpaceMatches.Add(BuildSpaceChunkStructure);
|
|
NewCreatedChunks.Add(NewChunkGuid);
|
|
|
|
LayerCreatingChunksTest.FindOrAdd(LayerIdx).Add(BuildSpaceChunkStructure);
|
|
ManifestBuilder->AddChunkMatch(NewChunkGuid, BuildSpaceChunkStructure);
|
|
|
|
// Track data selected.
|
|
ChunkedLayerSpaceStructure.Add(UnknownBlockOffset, ChunkBuildSize);
|
|
ChunkedBuildSpaceStructure.Add(BuildSpaceChunkStructure);
|
|
|
|
check(BlockStructureHelpers::CountSize(ChunkedLayerSpaceStructure) == BlockStructureHelpers::CountSize(ChunkedBuildSpaceStructure));
|
|
|
|
UnknownBlockOffset += ChunkBuildSize;
|
|
UnknownBlockSize -= ChunkBuildSize;
|
|
UnknownBlockByteCount += ChunkBuildSize;
|
|
check(!bFinalLayerChunk || UnknownBlockSize == 0);
|
|
}
|
|
UnknownBlockByteCount += UnknownBlockSize;
|
|
UnknownLayerBlock = UnknownLayerBlock->GetNext();
|
|
check(!bFinalLayerChunk || UnknownLayerBlock == nullptr);
|
|
}
|
|
UnknownLayerSpaceStructure.Remove(ChunkedLayerSpaceStructure);
|
|
UnknownBuildSpaceStructure.Remove(ChunkedBuildSpaceStructure);
|
|
LayerSpaceBlockData.RemoveData(BlockDataToRemove);
|
|
}
|
|
|
|
// Create new scanners from unknown data.
|
|
while (!PatchGenerationHelpers::ScannerArrayFull(Scanners))
|
|
{
|
|
bool bScannerCreated = false;
|
|
for (int32 LayerIdx = 0; LayerIdx <= MaxLayer; ++LayerIdx)
|
|
{
|
|
// Check that we have enough slack space in the data array to be queuing up more scanners on the next layer.
|
|
const int32 NextLayer = LayerIdx + 1;
|
|
const uint64 NextLayerSpaceBlockCount = LayerToLayerSpaceBlockData.FindOrAdd(NextLayer).GetDataCount();
|
|
const int32 OneGigabyte = 1073741824;
|
|
const bool bQueuedDataFull = NextLayerSpaceBlockCount > OneGigabyte;
|
|
if (bQueuedDataFull)
|
|
{
|
|
UE_LOG(LogPatchGeneration, Verbose, TEXT("Not making new scanners on next layer %d due to current backlog %llu bytes"), NextLayer, NextLayerSpaceBlockCount);
|
|
break;
|
|
}
|
|
|
|
FBlockStructure& UnknownLayerSpaceStructure = LayerToUnknownLayerSpaceStructure.FindOrAdd(LayerIdx);
|
|
FBlockStructure& UnknownBuildSpaceStructure = LayerToUnknownBuildSpaceStructure.FindOrAdd(LayerIdx);
|
|
TBlockData<uint8>& LayerSpaceBlockData = LayerToLayerSpaceBlockData.FindOrAdd(LayerIdx);
|
|
const uint64& LayerScannedSize = LayerToScannedSize.FindOrAdd(LayerIdx);
|
|
const bool bLayerComplete = LayerToTotalDataSize.Contains(LayerIdx) && (LayerScannedSize >= LayerToTotalDataSize[LayerIdx]);
|
|
|
|
FBlockStructure NewScannerBuildSpaceStructure;
|
|
FBlockStructure NewScannerLayerSpaceStructure;
|
|
const uint64 SelectedBuildSpaceSize = UnknownBuildSpaceStructure.SelectSerialBytes(0, ScannerDataSize, NewScannerBuildSpaceStructure);
|
|
const uint64 UnknownDataSize = BlockStructureHelpers::CountSize(UnknownBuildSpaceStructure);
|
|
|
|
// Make sure there are enough bytes available for a scanner, plus a chunk, so that we know no more chunks will get
|
|
// made from this sequential unknown data.
|
|
const uint64 RequiredScannerBytes = ScannerDataSize + LargestWindowSize;
|
|
const bool bHasEnoughData = bLayerComplete || UnknownDataSize > RequiredScannerBytes;
|
|
|
|
if (bHasEnoughData && (SelectedBuildSpaceSize == ScannerDataSize || (bLayerComplete && SelectedBuildSpaceSize > 0)))
|
|
{
|
|
check(bHasEnoughData || bLayerComplete);
|
|
|
|
const uint64 SelectedLayerSpaceSize = UnknownLayerSpaceStructure.SelectSerialBytes(0, ScannerDataSize, NewScannerLayerSpaceStructure);
|
|
check(SelectedBuildSpaceSize == SelectedLayerSpaceSize);
|
|
bScannerCreated = true;
|
|
LayerToScannerCount.FindOrAdd(NextLayer)++;
|
|
MaxLayer = FMath::Max<int64>(MaxLayer, NextLayer);
|
|
FStatsCollector::Set(StatLayers, MaxLayer);
|
|
|
|
uint64& NextLayerScannerOffset = LayerToCreatedScannerOffset.FindOrAdd(NextLayer);
|
|
TArray<uint8> ScannerData;
|
|
LayerSpaceBlockData.CopyTo(ScannerData, NewScannerLayerSpaceStructure);
|
|
check(ScannerData.Num() == SelectedLayerSpaceSize);
|
|
|
|
const bool bIsFinalScanner = bLayerComplete && UnknownDataSize <= SelectedBuildSpaceSize;
|
|
const uint64 PadSize = bIsFinalScanner ? ScannerOverlapSize : 0;
|
|
ScannerData.AddZeroed(PadSize);
|
|
|
|
// Test overlaps.
|
|
FBlockStructure OverlapStructure = LayerCreatingScannersTest.FindOrAdd(NextLayer).Intersect(NewScannerBuildSpaceStructure);
|
|
uint64 OverlapBytes = BlockStructureHelpers::CountSize(OverlapStructure);
|
|
FBlockStructure OverlapLayerSpaceStructure = LayerCreatingScannersLayerSpaceTest.FindOrAdd(NextLayer).Intersect(NewScannerLayerSpaceStructure);
|
|
uint64 OverlapLayerSpaceBytes = BlockStructureHelpers::CountSize(OverlapLayerSpaceStructure);
|
|
check(OverlapLayerSpaceBytes == ScannerOverlapSize || NextLayerScannerOffset == 0);
|
|
check(OverlapBytes == ScannerOverlapSize || NextLayerScannerOffset == 0);
|
|
LayerCreatingScannersTest.FindOrAdd(NextLayer).Add(NewScannerBuildSpaceStructure);
|
|
LayerCreatingScannersLayerSpaceTest.FindOrAdd(NextLayer).Add(NewScannerLayerSpaceStructure);
|
|
|
|
// Check only one final scanner.
|
|
if (bIsFinalScanner)
|
|
{
|
|
check(LayerCreatingFinalTest.Contains(NextLayer) == false);
|
|
LayerCreatingFinalTest.Add(NextLayer, true);
|
|
}
|
|
|
|
UE_LOG(LogPatchGeneration, Verbose, TEXT("Creating scanner on layer %d at %llu. IsFinal:%d. Mapping:%s, BuildMapping:%s"), NextLayer, NextLayerScannerOffset, bIsFinalScanner, *NewScannerLayerSpaceStructure.ToString(), *NewScannerBuildSpaceStructure.ToString());
|
|
Scanners.Emplace(new FScannerDetails(NextLayer, NextLayerScannerOffset, bIsFinalScanner, PadSize, MoveTemp(ScannerData), NewScannerBuildSpaceStructure, WindowSizes, CloudEnumeration.Get(), StatsCollector.Get()));
|
|
NextLayerScannerOffset += ScannerDataSize - ScannerOverlapSize;
|
|
|
|
// Remove blocks from structures.
|
|
NewScannerBuildSpaceStructure.Empty();
|
|
NewScannerLayerSpaceStructure.Empty();
|
|
const uint64 SerialBytesToSelect = bIsFinalScanner ? ScannerDataSize : ScannerDataSize - ScannerOverlapSize;
|
|
const uint64 SizeBuildRemoving = UnknownBuildSpaceStructure.SelectSerialBytes(0, SerialBytesToSelect, NewScannerBuildSpaceStructure);
|
|
const uint64 SizeLayerRemoving = UnknownLayerSpaceStructure.SelectSerialBytes(0, SerialBytesToSelect, NewScannerLayerSpaceStructure);
|
|
UnknownBuildSpaceStructure.Remove(NewScannerBuildSpaceStructure);
|
|
UnknownLayerSpaceStructure.Remove(NewScannerLayerSpaceStructure);
|
|
LayerSpaceBlockData.RemoveData(NewScannerLayerSpaceStructure);
|
|
check(SizeBuildRemoving == SizeLayerRemoving);
|
|
check(SizeBuildRemoving == SerialBytesToSelect || SizeBuildRemoving == UnknownDataSize);
|
|
check(!bIsFinalScanner || BlockStructureHelpers::CountSize(UnknownBuildSpaceStructure) == 0);
|
|
check(!bIsFinalScanner || BlockStructureHelpers::CountSize(UnknownLayerSpaceStructure) == 0);
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogPatchGeneration, Verbose, TEXT("Not making Layer[%d] unknown data scanners.. RequiredScannerBytes:%llu UnknownDataSize:%llu"), LayerIdx, RequiredScannerBytes, UnknownDataSize);
|
|
}
|
|
}
|
|
// Stop when we cannot make scanners anymore.
|
|
if (!bScannerCreated)
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Stream some build data.
|
|
if (!PatchGenerationHelpers::ScannerArrayFull(Scanners))
|
|
{
|
|
// Check that we have enough slack space in the data array to be queuing up more scanners on layer 0.
|
|
const uint64 BottomLayerSpaceBlockCount = LayerToLayerSpaceBlockData.FindOrAdd(0).GetDataCount();
|
|
const int32 OneGigabyte = 1073741824;
|
|
const bool bQueuedDataFull = BottomLayerSpaceBlockCount > OneGigabyte;
|
|
if (bQueuedDataFull)
|
|
{
|
|
UE_LOG(LogPatchGeneration, Verbose, TEXT("Not making new scanners on layer 0 due to current backlog %llu bytes."), BottomLayerSpaceBlockCount);
|
|
}
|
|
else
|
|
{
|
|
// Create a scanner from new build data?
|
|
if (!BuildStream->IsEndOfData())
|
|
{
|
|
// Keep the overlap data from previous scanner.
|
|
int32 PreviousSize = DataBuffer.Num();
|
|
if (PreviousSize > 0)
|
|
{
|
|
check(PreviousSize > ScannerOverlapSize);
|
|
uint8* CopyTo = DataBuffer.GetData();
|
|
uint8* CopyFrom = CopyTo + (PreviousSize - ScannerOverlapSize);
|
|
FMemory::Memcpy(CopyTo, CopyFrom, ScannerOverlapSize);
|
|
DataBuffer.SetNum(ScannerOverlapSize, EAllowShrinking::No);
|
|
DataBufferFirstIdx += (PreviousSize - ScannerOverlapSize);
|
|
}
|
|
|
|
// Grab some data from the build stream.
|
|
PreviousSize = DataBuffer.Num();
|
|
DataBuffer.SetNumUninitialized(ScannerDataSize);
|
|
const bool bWaitForData = true;
|
|
ReadLen = BuildStream->DequeueData(DataBuffer.GetData() + PreviousSize, ScannerDataSize - PreviousSize, bWaitForData);
|
|
DataBuffer.SetNum(PreviousSize + ReadLen, EAllowShrinking::No);
|
|
|
|
// Only make a scanner if we are getting new data.
|
|
if (ReadLen > 0)
|
|
{
|
|
// Pad scanner data if end of build
|
|
uint64 PadSize = BuildStream->IsEndOfData() ? ScannerOverlapSize : 0;
|
|
DataBuffer.AddZeroed(PadSize);
|
|
|
|
// Create data scanner.
|
|
const bool bIsFinalScanner = BuildStream->IsEndOfData();
|
|
FBlockStructure Structure;
|
|
Structure.Add(DataBufferFirstIdx, DataBuffer.Num() - PadSize);
|
|
|
|
// Test overlaps.
|
|
FBlockStructure OverlapStructure = LayerCreatingScannersTest.FindOrAdd(0).Intersect(Structure);
|
|
FBlockStructure OverlapLayerSpaceStructure = LayerCreatingScannersLayerSpaceTest.FindOrAdd(0).Intersect(Structure);
|
|
uint64 OverlapBytes = BlockStructureHelpers::CountSize(OverlapStructure);
|
|
uint64 OverlapLayerSpaceBytes = BlockStructureHelpers::CountSize(OverlapLayerSpaceStructure);
|
|
check(OverlapBytes == ScannerOverlapSize || DataBufferFirstIdx == 0);
|
|
check(OverlapLayerSpaceBytes == ScannerOverlapSize || DataBufferFirstIdx == 0);
|
|
LayerCreatingScannersTest.FindOrAdd(0).Add(Structure);
|
|
LayerCreatingScannersLayerSpaceTest.FindOrAdd(0).Add(Structure);
|
|
|
|
// Check only one final scanner.
|
|
if (bIsFinalScanner)
|
|
{
|
|
check(LayerCreatingFinalTest.Contains(0) == false);
|
|
LayerCreatingFinalTest.Add(0, true);
|
|
}
|
|
|
|
UE_LOG(LogPatchGeneration, Verbose, TEXT("Creating scanner on layer 0 at %llu. IsFinal:%d. Mapping:%s"), DataBufferFirstIdx, BuildStream->IsEndOfData(), *Structure.ToString());
|
|
Scanners.Emplace(new FScannerDetails(0, DataBufferFirstIdx, BuildStream->IsEndOfData(), PadSize, DataBuffer, MoveTemp(Structure), WindowSizes, CloudEnumeration.Get(), StatsCollector.Get()));
|
|
LayerToScannerCount.FindOrAdd(0)++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Did we run out of unknown data?
|
|
bHasUnknownData = false;
|
|
for (const TPair<int32, FBlockStructure>& UnknownBuildSpaceStructurePair : LayerToUnknownBuildSpaceStructure)
|
|
{
|
|
if (UnknownBuildSpaceStructurePair.Value.GetHead() != nullptr)
|
|
{
|
|
bHasUnknownData = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Update some stats.
|
|
int64 UnknownDataAlloc = 0;
|
|
int64 UnknownDataNum = 0;
|
|
for (const TPair<int32, TBlockData<uint8>>& LayerToLayerSpaceBlockDataPair : LayerToLayerSpaceBlockData)
|
|
{
|
|
UnknownDataNum += LayerToLayerSpaceBlockDataPair.Value.GetDataCount();
|
|
UnknownDataAlloc += LayerToLayerSpaceBlockDataPair.Value.GetAllocatedSize();
|
|
}
|
|
FStatsCollector::Set(StatUnknownDataAlloc, UnknownDataAlloc);
|
|
FStatsCollector::Set(StatUnknownDataNum, UnknownDataNum);
|
|
FStatsCollector::Set(StatNumScanners, Scanners.Num());
|
|
|
|
// Log collected stats.
|
|
if (bIsGameThread)
|
|
{
|
|
GLog->FlushThreadedLogs();
|
|
}
|
|
FStatsCollector::Set(StatTotalTime, FStatsCollector::GetCycles() - StartTime);
|
|
StatsCollector->LogStats(StatsLoggerTimeSeconds);
|
|
|
|
// Sleep to allow other threads.
|
|
FPlatformProcess::Sleep(0.01f);
|
|
}
|
|
|
|
if (BuildStream->HasAborted())
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Directory Build Streamer aborted"));
|
|
return false;
|
|
}
|
|
|
|
// Complete chunk writer.
|
|
FParallelChunkWriterSummaries ChunkWriterSummaries = ChunkWriter->OnProcessComplete();
|
|
|
|
// Produce final stats log.
|
|
const uint64 EndTime = FStatsCollector::GetCycles();
|
|
FStatsCollector::Set(StatTotalTime, EndTime - StartTime);
|
|
StatsCollector->LogStats();
|
|
|
|
// Collect chunk info for the manifest builder.
|
|
TMap<FGuid, FChunkInfo> ChunkInfoMap;
|
|
TMap<FGuid, int64> ChunkFileSizes = CloudEnumeration->GetChunkFileSizes();
|
|
TMap<FGuid, uint32> ChunkWindowSizes = CloudEnumeration->GetChunkWindowSizes();
|
|
ChunkFileSizes.Append(ChunkWriterSummaries.ChunkOutputSizes);
|
|
for (const TPair<uint64, TSet<FGuid>>& ChunkInventoryPair : CloudEnumeration->GetChunkInventory())
|
|
{
|
|
TSet<FGuid>& ChunkSet = ChunkInventory.FindOrAdd(ChunkInventoryPair.Key);
|
|
ChunkSet = ChunkSet.Union(ChunkInventoryPair.Value);
|
|
}
|
|
ChunkShaHashes.Append(CloudEnumeration->GetChunkShaHashes());
|
|
for (const TPair<uint64, TSet<FGuid>>& ChunkInventoryPair : ChunkInventory)
|
|
{
|
|
for (const FGuid& ChunkGuid : ChunkInventoryPair.Value)
|
|
{
|
|
if (ChunkShaHashes.Contains(ChunkGuid) && ChunkFileSizes.Contains(ChunkGuid))
|
|
{
|
|
FChunkInfo& ChunkInfo = ChunkInfoMap.FindOrAdd(ChunkGuid);
|
|
ChunkInfo.Guid = ChunkGuid;
|
|
ChunkInfo.Hash = ChunkInventoryPair.Key;
|
|
FMemory::Memcpy(ChunkInfo.ShaHash.Hash, ChunkShaHashes[ChunkGuid].Hash, FSHA1::DigestSize);
|
|
ChunkInfo.FileSize = ChunkFileSizes[ChunkGuid];
|
|
ChunkInfo.GroupNumber = FCrc::MemCrc32(&ChunkGuid, sizeof(FGuid)) % 100;
|
|
ChunkInfo.WindowSize = ChunkWindowSizes.Contains(ChunkGuid) ? ChunkWindowSizes[ChunkGuid] : Settings.OutputChunkWindowSize;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Finalize the manifest data.
|
|
TArray<FChunkInfo> ChunkInfoList;
|
|
ChunkInfoMap.GenerateValueArray(ChunkInfoList);
|
|
if (ManifestBuilder->FinalizeData(BuildStream->GetAllFiles(), MoveTemp(ChunkInfoList)) == false)
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Finalizing manifest failed."));
|
|
return false;
|
|
}
|
|
uint64 NewChunkBytes = 0;
|
|
for (const FGuid& NewChunk : NewCreatedChunks)
|
|
{
|
|
NewChunkBytes += ChunkWriterSummaries.ChunkOutputSizes[NewChunk];
|
|
}
|
|
UE_LOG(LogPatchGeneration, Log, TEXT("Created %d chunks (%llu build bytes) (%llu compressed bytes)"), NewCreatedChunks.Num(), BlockStructureHelpers::CountSize(CreatedBuildSpaceMatches), NewChunkBytes);
|
|
UE_LOG(LogPatchGeneration, Log, TEXT("Completed in %s."), *FPlatformTime::PrettyTime(FStatsCollector::CyclesToSeconds(*StatTotalTime)));
|
|
|
|
// Save manifest out to the cloud directory.
|
|
const FString OutputFilename = Settings.CloudDirectory / Settings.OutputFilename;
|
|
if (ManifestBuilder->SaveToFile(OutputFilename) == false)
|
|
{
|
|
UE_LOG(LogPatchGeneration, Error, TEXT("Saving manifest failed."));
|
|
return false;
|
|
}
|
|
UE_LOG(LogPatchGeneration, Log, TEXT("Saved manifest to %s."), *OutputFilename);
|
|
|
|
return true;
|
|
}
|