1461 lines
46 KiB
C++
1461 lines
46 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "BuildPatchManifest.h"
|
|
#include "HAL/FileManager.h"
|
|
#include "Misc/FileHelper.h"
|
|
#include "Misc/Paths.h"
|
|
#include "Serialization/MemoryWriter.h"
|
|
#include "Serialization/MemoryReader.h"
|
|
#include "Policies/PrettyJsonPrintPolicy.h"
|
|
#include "Policies/CondensedJsonPrintPolicy.h"
|
|
#include "Serialization/JsonTypes.h"
|
|
#include "Serialization/JsonReader.h"
|
|
#include "Serialization/JsonSerializer.h"
|
|
#include "Algo/Sort.h"
|
|
#include "Algo/Accumulate.h"
|
|
#include "Algo/Transform.h"
|
|
#include "Core/BlockStructure.h"
|
|
#include "Data/ChunkData.h"
|
|
#include "Data/ManifestData.h"
|
|
#include "BuildPatchUtil.h"
|
|
|
|
using namespace BuildPatchServices;
|
|
|
|
#define LOCTEXT_NAMESPACE "BuildPatchManifest"
|
|
|
|
/**
|
|
* Helper functions that convert generic types to and from string blobs for use with JSON parsing.
|
|
* It's kind of horrible but guarantees no loss of data as the JSON reader/writer only supports float functionality
|
|
* which would result in data loss with high int32 values, and we'll be using uint64.
|
|
*/
|
|
template< typename DataType >
|
|
bool FromStringBlob( const FString& StringBlob, DataType& ValueOut )
|
|
{
|
|
void* AsBuffer = &ValueOut;
|
|
return FString::ToBlob( StringBlob, static_cast< uint8* >( AsBuffer ), sizeof( DataType ) );
|
|
}
|
|
template< typename DataType >
|
|
FString ToStringBlob( const DataType& DataVal )
|
|
{
|
|
const void* AsBuffer = &DataVal;
|
|
return FString::FromBlob( static_cast<const uint8*>( AsBuffer ), sizeof( DataType ) );
|
|
}
|
|
template< typename DataType >
|
|
bool FromHexString(const FString& HexString, DataType& ValueOut)
|
|
{
|
|
void* AsBuffer = &ValueOut;
|
|
if (HexString.Len() == (sizeof(DataType)* 2))
|
|
{
|
|
HexToBytes(HexString, static_cast<uint8*>(AsBuffer));
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
template< typename DataType >
|
|
FString ToHexString(const DataType& DataVal)
|
|
{
|
|
const void* AsBuffer = &DataVal;
|
|
return BytesToHex(static_cast<const uint8*>(AsBuffer), sizeof(DataType));
|
|
}
|
|
|
|
/**
|
|
* Helper functions to decide whether the passed in data is a JSON string we expect to deserialize a manifest from
|
|
*/
|
|
bool BufferIsJsonManifest(const TArray<uint8>& DataInput)
|
|
{
|
|
// The best we can do is look for the mandatory first character open curly brace,
|
|
// it will be within the first 4 characters (may have BOM)
|
|
for (int32 idx = 0; idx < 4 && idx < DataInput.Num(); ++idx)
|
|
{
|
|
if (DataInput[idx] == TEXT('{'))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/* FBuildPatchCustomField implementation
|
|
*****************************************************************************/
|
|
FBuildPatchCustomField::FBuildPatchCustomField(const FString& Value)
|
|
: CustomValue(Value)
|
|
{
|
|
}
|
|
|
|
FString FBuildPatchCustomField::AsString() const
|
|
{
|
|
return CustomValue;
|
|
}
|
|
|
|
double FBuildPatchCustomField::AsDouble() const
|
|
{
|
|
// The Json parser currently only supports float so we have to decode string blob instead
|
|
double Rtn;
|
|
if( FromStringBlob( CustomValue, Rtn ) )
|
|
{
|
|
return Rtn;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int64 FBuildPatchCustomField::AsInteger() const
|
|
{
|
|
// The Json parser currently only supports float so we have to decode string blob instead
|
|
int64 Rtn;
|
|
if( FromStringBlob( CustomValue, Rtn ) )
|
|
{
|
|
return Rtn;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/* FBuildPatchAppManifest implementation
|
|
*****************************************************************************/
|
|
|
|
FBuildPatchAppManifest::FBuildPatchAppManifest()
|
|
: TotalBuildSize(INDEX_NONE)
|
|
, TotalDownloadSize(INDEX_NONE)
|
|
, bNeedsResaving(false)
|
|
{
|
|
}
|
|
|
|
FBuildPatchAppManifest::FBuildPatchAppManifest(const uint32& InAppID, const FString& InAppName)
|
|
: FBuildPatchAppManifest()
|
|
{
|
|
ManifestMeta.AppID = InAppID;
|
|
ManifestMeta.AppName = InAppName;
|
|
}
|
|
|
|
FBuildPatchAppManifest::FBuildPatchAppManifest(const FBuildPatchAppManifest& Other)
|
|
: ManifestMeta(Other.ManifestMeta)
|
|
, ChunkDataList(Other.ChunkDataList)
|
|
, FileManifestList(Other.FileManifestList)
|
|
, CustomFields(Other.CustomFields)
|
|
, TotalBuildSize(Other.TotalBuildSize)
|
|
, TotalDownloadSize(Other.TotalDownloadSize)
|
|
, bNeedsResaving(Other.bNeedsResaving)
|
|
{
|
|
InitLookups();
|
|
}
|
|
|
|
FBuildPatchAppManifest::~FBuildPatchAppManifest()
|
|
{
|
|
DestroyData();
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::SaveToFile(const FString& Filename, BuildPatchServices::EFeatureLevel SaveFormat)
|
|
{
|
|
bool bSuccess = SaveFormat >= GetFeatureLevel();
|
|
if (bSuccess)
|
|
{
|
|
TUniquePtr<FArchive> FileOut(IFileManager::Get().CreateFileWriter(*Filename));
|
|
bSuccess = FileOut.IsValid();
|
|
if (bSuccess)
|
|
{
|
|
if (SaveFormat >= BuildPatchServices::EFeatureLevel::StoredAsBinaryData)
|
|
{
|
|
bSuccess = FManifestData::Serialize(*FileOut, *this, SaveFormat);
|
|
}
|
|
else
|
|
{
|
|
FString JSONOutput;
|
|
SerializeToJSON(JSONOutput);
|
|
FTCHARToUTF8 JsonUTF8(*JSONOutput);
|
|
FileOut->Serialize((UTF8CHAR*)JsonUTF8.Get(), JsonUTF8.Length() * sizeof(UTF8CHAR));
|
|
}
|
|
bSuccess = FileOut->Close() && bSuccess;
|
|
}
|
|
}
|
|
return bSuccess;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::LoadFromFile(const FString& Filename)
|
|
{
|
|
TArray<uint8> FileData;
|
|
if (FFileHelper::LoadFileToArray(FileData, *Filename))
|
|
{
|
|
return DeserializeFromData(FileData);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::DeserializeFromData(const TArray<uint8>& DataInput)
|
|
{
|
|
if (DataInput.Num())
|
|
{
|
|
if (BufferIsJsonManifest(DataInput))
|
|
{
|
|
FString JsonManifest;
|
|
FFileHelper::BufferToString(JsonManifest, DataInput.GetData(), DataInput.Num());
|
|
return DeserializeFromJSON(JsonManifest);
|
|
}
|
|
else
|
|
{
|
|
FMemoryReader MemoryReader(DataInput);
|
|
return FManifestData::Serialize(MemoryReader, *this);
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void FBuildPatchAppManifest::DestroyData()
|
|
{
|
|
// Clear Manifest data
|
|
ManifestMeta = FManifestMeta();
|
|
ChunkDataList = FChunkDataList();
|
|
FileManifestList = FFileManifestList();
|
|
CustomFields = FCustomFields();
|
|
FileNameLookup.Empty();
|
|
FileManifestLookup.Empty();
|
|
TaggedFilesLookup.Empty();
|
|
ChunkInfoLookup.Empty();
|
|
TotalBuildSize = INDEX_NONE;
|
|
TotalDownloadSize = INDEX_NONE;
|
|
bNeedsResaving = false;
|
|
}
|
|
|
|
void FBuildPatchAppManifest::InitLookups()
|
|
{
|
|
// Create file lookups.
|
|
const int32 NumFiles = FileManifestList.FileList.Num();
|
|
FileNameLookup.Empty(ManifestMeta.bIsFileData ? NumFiles : 0);
|
|
FileManifestLookup.Empty(NumFiles);
|
|
TaggedFilesLookup.Empty();
|
|
for (const FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
FileManifestLookup.Add(FileManifest.Filename, &FileManifest);
|
|
if (ManifestMeta.bIsFileData)
|
|
{
|
|
FileNameLookup.Add(FileManifest.ChunkParts[0].Guid, &FileManifest.Filename);
|
|
}
|
|
if (FileManifest.InstallTags.Num() == 0)
|
|
{
|
|
TaggedFilesLookup.FindOrAdd(TEXT("")).Add(&FileManifest);
|
|
}
|
|
else
|
|
{
|
|
for (const FString& FileTag : FileManifest.InstallTags)
|
|
{
|
|
TaggedFilesLookup.FindOrAdd(FileTag).Add(&FileManifest);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Create chunk lookup.
|
|
const int32 NumChunks = ChunkDataList.ChunkList.Num();
|
|
ChunkInfoLookup.Empty(NumChunks);
|
|
for (const FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
ChunkInfoLookup.Add(ChunkInfo.Guid, &ChunkInfo);
|
|
}
|
|
|
|
// Calculate build sizes.
|
|
TotalBuildSize = 0;
|
|
TotalDownloadSize = 0;
|
|
for (const FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
TotalBuildSize += FileManifest.FileSize;
|
|
}
|
|
for (const FChunkInfo& Chunk : ChunkDataList.ChunkList)
|
|
{
|
|
TotalDownloadSize += Chunk.FileSize;
|
|
}
|
|
}
|
|
|
|
void FBuildPatchAppManifest::SerializeToJSON(FString& JSONOutput)
|
|
{
|
|
using namespace BuildPatchServices;
|
|
#if UE_BUILD_DEBUG // We'll use this to switch between human readable JSON
|
|
TSharedRef< TJsonWriter< TCHAR, TPrettyJsonPrintPolicy< TCHAR > > > Writer = TJsonWriterFactory< TCHAR, TPrettyJsonPrintPolicy< TCHAR > >::Create(&JSONOutput);
|
|
#else
|
|
TSharedRef< TJsonWriter< TCHAR, TCondensedJsonPrintPolicy< TCHAR > > > Writer = TJsonWriterFactory< TCHAR, TCondensedJsonPrintPolicy< TCHAR > >::Create(&JSONOutput);
|
|
#endif //ALLOW_DEBUG_FILES
|
|
|
|
Writer->WriteObjectStart();
|
|
{
|
|
// Write general data
|
|
Writer->WriteValue(TEXT("ManifestFileVersion"), ToStringBlob(static_cast<int32>(ManifestMeta.FeatureLevel)));
|
|
Writer->WriteValue(TEXT("bIsFileData"), ManifestMeta.bIsFileData);
|
|
Writer->WriteValue(TEXT("AppID"), ToStringBlob(ManifestMeta.AppID));
|
|
Writer->WriteValue(TEXT("AppNameString"), ManifestMeta.AppName);
|
|
Writer->WriteValue(TEXT("BuildVersionString"), ManifestMeta.BuildVersion);
|
|
Writer->WriteValue(TEXT("LaunchExeString"), ManifestMeta.LaunchExe);
|
|
Writer->WriteValue(TEXT("LaunchCommand"), ManifestMeta.LaunchCommand);
|
|
Writer->WriteArrayStart(TEXT("PrereqIds"));
|
|
for (const FString& PrereqId : ManifestMeta.PrereqIds)
|
|
{
|
|
Writer->WriteValue(PrereqId);
|
|
}
|
|
Writer->WriteArrayEnd();
|
|
Writer->WriteValue(TEXT("PrereqName"), ManifestMeta.PrereqName);
|
|
Writer->WriteValue(TEXT("PrereqPath"), ManifestMeta.PrereqPath);
|
|
Writer->WriteValue(TEXT("PrereqArgs"), ManifestMeta.PrereqArgs);
|
|
// Write file manifest data
|
|
Writer->WriteArrayStart(TEXT("FileManifestList"));
|
|
for (const FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
Writer->WriteObjectStart();
|
|
{
|
|
Writer->WriteValue(TEXT("Filename"), FileManifest.Filename);
|
|
Writer->WriteValue(TEXT("FileHash"), FString::FromBlob(FileManifest.FileHash.Hash, FSHA1::DigestSize));
|
|
if (EnumHasAllFlags(FileManifest.FileMetaFlags, EFileMetaFlags::UnixExecutable))
|
|
{
|
|
Writer->WriteValue(TEXT("bIsUnixExecutable"), true);
|
|
}
|
|
if (EnumHasAllFlags(FileManifest.FileMetaFlags, EFileMetaFlags::ReadOnly))
|
|
{
|
|
Writer->WriteValue(TEXT("bIsReadOnly"), true);
|
|
}
|
|
if (EnumHasAllFlags(FileManifest.FileMetaFlags, EFileMetaFlags::Compressed))
|
|
{
|
|
Writer->WriteValue(TEXT("bIsCompressed"), true);
|
|
}
|
|
const bool bIsSymlink = !FileManifest.SymlinkTarget.IsEmpty();
|
|
if (bIsSymlink)
|
|
{
|
|
Writer->WriteValue(TEXT("SymlinkTarget"), FileManifest.SymlinkTarget);
|
|
}
|
|
else
|
|
{
|
|
Writer->WriteArrayStart(TEXT("FileChunkParts"));
|
|
{
|
|
for (const FChunkPart& ChunkPart : FileManifest.ChunkParts)
|
|
{
|
|
Writer->WriteObjectStart();
|
|
{
|
|
Writer->WriteValue(TEXT("Guid"), ChunkPart.Guid.ToString());
|
|
Writer->WriteValue(TEXT("Offset"), ToStringBlob(ChunkPart.Offset));
|
|
Writer->WriteValue(TEXT("Size"), ToStringBlob(ChunkPart.Size));
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
}
|
|
}
|
|
Writer->WriteArrayEnd();
|
|
}
|
|
if (FileManifest.InstallTags.Num() > 0)
|
|
{
|
|
Writer->WriteArrayStart(TEXT("InstallTags"));
|
|
{
|
|
for (const FString& InstallTag : FileManifest.InstallTags)
|
|
{
|
|
Writer->WriteValue(InstallTag);
|
|
}
|
|
}
|
|
Writer->WriteArrayEnd();
|
|
}
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
}
|
|
Writer->WriteArrayEnd();
|
|
// Write chunk hash list
|
|
Writer->WriteObjectStart(TEXT("ChunkHashList"));
|
|
for (const FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
const FGuid& ChunkGuid = ChunkInfo.Guid;
|
|
const uint64& ChunkHash = ChunkInfo.Hash;
|
|
Writer->WriteValue(ChunkGuid.ToString(), ToStringBlob(ChunkHash));
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
// Write chunk sha list
|
|
Writer->WriteObjectStart(TEXT("ChunkShaList"));
|
|
for (const FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
const FGuid& ChunkGuid = ChunkInfo.Guid;
|
|
const FSHAHash& ChunkSha = ChunkInfo.ShaHash;
|
|
Writer->WriteValue(ChunkGuid.ToString(), ToHexString(ChunkSha));
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
// Write data group list
|
|
Writer->WriteObjectStart(TEXT("DataGroupList"));
|
|
for (const FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
const FGuid& DataGuid = ChunkInfo.Guid;
|
|
const uint8& DataGroup = ChunkInfo.GroupNumber;
|
|
Writer->WriteValue(DataGuid.ToString(), ToStringBlob(DataGroup));
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
// Write chunk size list
|
|
Writer->WriteObjectStart(TEXT("ChunkFilesizeList"));
|
|
for (const FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
const FGuid& ChunkGuid = ChunkInfo.Guid;
|
|
const int64& ChunkSize = ChunkInfo.FileSize;
|
|
Writer->WriteValue(ChunkGuid.ToString(), ToStringBlob(ChunkSize));
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
// Write custom fields
|
|
Writer->WriteObjectStart(TEXT("CustomFields"));
|
|
for (const TPair<FString, FString>& CustomField : CustomFields.Fields)
|
|
{
|
|
Writer->WriteValue(CustomField.Key, CustomField.Value);
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
}
|
|
Writer->WriteObjectEnd();
|
|
|
|
Writer->Close();
|
|
}
|
|
|
|
// @TODO LSwift: Perhaps replace FromBlob and ToBlob usage with hexadecimal notation instead
|
|
bool FBuildPatchAppManifest::DeserializeFromJSON( const FString& JSONInput )
|
|
{
|
|
bool bSuccess = true;
|
|
TSharedPtr<FJsonObject> JSONManifestObject;
|
|
TSharedRef<TJsonReader<TCHAR>> Reader = TJsonReaderFactory<TCHAR>::Create(JSONInput);
|
|
|
|
// Clear current data
|
|
DestroyData();
|
|
|
|
// Attempt to deserialize JSON
|
|
if (!FJsonSerializer::Deserialize(Reader, JSONManifestObject) || !JSONManifestObject.IsValid())
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// Store a list of all data GUID for later use
|
|
TSet<FGuid> AllDataGuids;
|
|
|
|
// Get the values map
|
|
TMap<FString, TSharedPtr<FJsonValue>>& JsonValueMap = JSONManifestObject->Values;
|
|
|
|
// Feature Level did not always exist
|
|
int32 FeatureLevelInt = 0;
|
|
TSharedPtr<FJsonValue> JsonFeatureLevel = JsonValueMap.FindRef(TEXT("ManifestFileVersion"));
|
|
if (JsonFeatureLevel.IsValid() && FromStringBlob(JsonFeatureLevel->AsString(), FeatureLevelInt))
|
|
{
|
|
ManifestMeta.FeatureLevel = static_cast<EFeatureLevel>(FeatureLevelInt);
|
|
}
|
|
else
|
|
{
|
|
// Then we presume version just before we started outputting the version
|
|
ManifestMeta.FeatureLevel = EFeatureLevel::CustomFields;
|
|
}
|
|
|
|
// If we loaded the default version number, we know it was saved is a CL range that was bugged,
|
|
// when the correct version should have been StoresChunkFileSizes.
|
|
if (ManifestMeta.FeatureLevel == EFeatureLevel::BrokenJsonVersion)
|
|
{
|
|
ManifestMeta.FeatureLevel = EFeatureLevel::StoresChunkFileSizes;
|
|
}
|
|
|
|
// Get the app and version strings
|
|
TSharedPtr<FJsonValue> JsonAppID = JsonValueMap.FindRef(TEXT("AppID"));
|
|
TSharedPtr<FJsonValue> JsonAppNameString = JsonValueMap.FindRef(TEXT("AppNameString"));
|
|
TSharedPtr<FJsonValue> JsonBuildVersionString = JsonValueMap.FindRef(TEXT("BuildVersionString"));
|
|
TSharedPtr<FJsonValue> JsonLaunchExe = JsonValueMap.FindRef(TEXT("LaunchExeString"));
|
|
TSharedPtr<FJsonValue> JsonLaunchCommand = JsonValueMap.FindRef(TEXT("LaunchCommand"));
|
|
TSharedPtr<FJsonValue> JsonPrereqName = JsonValueMap.FindRef(TEXT("PrereqName"));
|
|
TSharedPtr<FJsonValue> JsonPrereqPath = JsonValueMap.FindRef(TEXT("PrereqPath"));
|
|
TSharedPtr<FJsonValue> JsonPrereqArgs = JsonValueMap.FindRef(TEXT("PrereqArgs"));
|
|
bSuccess = bSuccess && JsonAppID.IsValid();
|
|
if( bSuccess )
|
|
{
|
|
bSuccess = bSuccess && FromStringBlob( JsonAppID->AsString(), ManifestMeta.AppID );
|
|
}
|
|
bSuccess = bSuccess && JsonAppNameString.IsValid();
|
|
if( bSuccess )
|
|
{
|
|
ManifestMeta.AppName = JsonAppNameString->AsString();
|
|
}
|
|
bSuccess = bSuccess && JsonBuildVersionString.IsValid();
|
|
if( bSuccess )
|
|
{
|
|
ManifestMeta.BuildVersion = JsonBuildVersionString->AsString();
|
|
}
|
|
bSuccess = bSuccess && JsonLaunchExe.IsValid();
|
|
if( bSuccess )
|
|
{
|
|
ManifestMeta.LaunchExe = JsonLaunchExe->AsString();
|
|
}
|
|
bSuccess = bSuccess && JsonLaunchCommand.IsValid();
|
|
if( bSuccess )
|
|
{
|
|
ManifestMeta.LaunchCommand = JsonLaunchCommand->AsString();
|
|
}
|
|
|
|
// Get the prerequisites installer info. These are optional entries.
|
|
ManifestMeta.PrereqName = JsonPrereqName.IsValid() ? JsonPrereqName->AsString() : FString();
|
|
ManifestMeta.PrereqPath = JsonPrereqPath.IsValid() ? JsonPrereqPath->AsString() : FString();
|
|
ManifestMeta.PrereqArgs = JsonPrereqArgs.IsValid() ? JsonPrereqArgs->AsString() : FString();
|
|
|
|
// Get the FileManifestList
|
|
TSharedPtr<FJsonValue> JsonFileManifestList = JsonValueMap.FindRef(TEXT("FileManifestList"));
|
|
bSuccess = bSuccess && JsonFileManifestList.IsValid();
|
|
if( bSuccess )
|
|
{
|
|
TArray<TSharedPtr<FJsonValue>> JsonFileManifestArray = JsonFileManifestList->AsArray();
|
|
for (auto JsonFileManifestIt = JsonFileManifestArray.CreateConstIterator(); JsonFileManifestIt && bSuccess; ++JsonFileManifestIt)
|
|
{
|
|
TSharedPtr<FJsonObject> JsonFileManifest = (*JsonFileManifestIt)->AsObject();
|
|
|
|
const int32 FileIndex = FileManifestList.FileList.Add(FFileManifest());
|
|
FFileManifest& FileManifest = FileManifestList.FileList[FileIndex];
|
|
FileManifest.Filename = JsonFileManifest->GetStringField(TEXT("Filename"));
|
|
bSuccess = bSuccess && FString::ToBlob(JsonFileManifest->GetStringField(TEXT("FileHash")), FileManifest.FileHash.Hash, FSHA1::DigestSize);
|
|
TArray<TSharedPtr<FJsonValue>> JsonChunkPartArray = JsonFileManifest->GetArrayField(TEXT("FileChunkParts"));
|
|
for (auto JsonChunkPartIt = JsonChunkPartArray.CreateConstIterator(); JsonChunkPartIt && bSuccess; ++JsonChunkPartIt)
|
|
{
|
|
const int32 ChunkIndex = FileManifest.ChunkParts.Add(FChunkPart());
|
|
FChunkPart& FileChunkPart = FileManifest.ChunkParts[ChunkIndex];
|
|
TSharedPtr<FJsonObject> JsonChunkPart = (*JsonChunkPartIt)->AsObject();
|
|
bSuccess = bSuccess && FGuid::Parse(JsonChunkPart->GetStringField(TEXT("Guid")), FileChunkPart.Guid);
|
|
bSuccess = bSuccess && FromStringBlob(JsonChunkPart->GetStringField(TEXT("Offset")), FileChunkPart.Offset);
|
|
bSuccess = bSuccess && FromStringBlob(JsonChunkPart->GetStringField(TEXT("Size")), FileChunkPart.Size);
|
|
AllDataGuids.Add(FileChunkPart.Guid);
|
|
}
|
|
if (JsonFileManifest->HasTypedField<EJson::Array>(TEXT("InstallTags")))
|
|
{
|
|
TArray<TSharedPtr<FJsonValue>> JsonInstallTagsArray = JsonFileManifest->GetArrayField(TEXT("InstallTags"));
|
|
for (auto JsonInstallTagIt = JsonInstallTagsArray.CreateConstIterator(); JsonInstallTagIt && bSuccess; ++JsonInstallTagIt)
|
|
{
|
|
FileManifest.InstallTags.Add((*JsonInstallTagIt)->AsString());
|
|
}
|
|
}
|
|
if (JsonFileManifest->HasField(TEXT("bIsUnixExecutable")) && JsonFileManifest->GetBoolField(TEXT("bIsUnixExecutable")))
|
|
{
|
|
FileManifest.FileMetaFlags |= EFileMetaFlags::UnixExecutable;
|
|
}
|
|
if (JsonFileManifest->HasField(TEXT("bIsReadOnly")) && JsonFileManifest->GetBoolField(TEXT("bIsReadOnly")))
|
|
{
|
|
FileManifest.FileMetaFlags |= EFileMetaFlags::ReadOnly;
|
|
}
|
|
if (JsonFileManifest->HasField(TEXT("bIsCompressed")) && JsonFileManifest->GetBoolField(TEXT("bIsCompressed")))
|
|
{
|
|
FileManifest.FileMetaFlags |= EFileMetaFlags::Compressed;
|
|
}
|
|
FileManifest.SymlinkTarget = JsonFileManifest->HasField(TEXT("SymlinkTarget")) ? JsonFileManifest->GetStringField(TEXT("SymlinkTarget")) : TEXT("");
|
|
}
|
|
}
|
|
|
|
for (FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
FileManifestLookup.Add(FileManifest.Filename, &FileManifest);
|
|
}
|
|
|
|
// For each chunk setup its info
|
|
for (const FGuid& DataGuid : AllDataGuids)
|
|
{
|
|
int32 ChunkIndex = ChunkDataList.ChunkList.Add(FChunkInfo());
|
|
ChunkDataList.ChunkList[ChunkIndex].Guid = DataGuid;
|
|
}
|
|
|
|
// Create a lookup table for chunks to speed up parsing
|
|
TMap<FGuid,FChunkInfo*> MutableChunkInfoLookup;
|
|
for (FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
MutableChunkInfoLookup.Add(ChunkInfo.Guid, &ChunkInfo);
|
|
}
|
|
|
|
// Get the ChunkHashList
|
|
bool bHasChunkHashList = false;
|
|
TSharedPtr<FJsonValue> JsonChunkHashList = JsonValueMap.FindRef(TEXT("ChunkHashList"));
|
|
bSuccess = bSuccess && JsonChunkHashList.IsValid();
|
|
if (bSuccess)
|
|
{
|
|
TSharedPtr<FJsonObject> JsonChunkHashListObj = JsonChunkHashList->AsObject();
|
|
for (auto ChunkHashIt = JsonChunkHashListObj->Values.CreateConstIterator(); ChunkHashIt && bSuccess; ++ChunkHashIt)
|
|
{
|
|
FGuid ChunkGuid;
|
|
uint64 ChunkHash = 0;
|
|
bSuccess = bSuccess && FGuid::Parse(ChunkHashIt.Key(), ChunkGuid);
|
|
bSuccess = bSuccess && FromStringBlob(ChunkHashIt.Value()->AsString(), ChunkHash);
|
|
if (bSuccess && MutableChunkInfoLookup.Contains(ChunkGuid))
|
|
{
|
|
FChunkInfo* ChunkInfoData = MutableChunkInfoLookup[ChunkGuid];
|
|
ChunkInfoData->Hash = ChunkHash;
|
|
bHasChunkHashList = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Get the ChunkShaList (optional)
|
|
TSharedPtr<FJsonValue> JsonChunkShaList = JsonValueMap.FindRef(TEXT("ChunkShaList"));
|
|
if (JsonChunkShaList.IsValid())
|
|
{
|
|
TSharedPtr<FJsonObject> JsonChunkHashListObj = JsonChunkShaList->AsObject();
|
|
for (auto ChunkHashIt = JsonChunkHashListObj->Values.CreateConstIterator(); ChunkHashIt && bSuccess; ++ChunkHashIt)
|
|
{
|
|
FGuid ChunkGuid;
|
|
FSHAHash ChunkSha;
|
|
bSuccess = bSuccess && FGuid::Parse(ChunkHashIt.Key(), ChunkGuid);
|
|
bSuccess = bSuccess && FromHexString(ChunkHashIt.Value()->AsString(), ChunkSha);
|
|
if (bSuccess && MutableChunkInfoLookup.Contains(ChunkGuid))
|
|
{
|
|
FChunkInfo* ChunkInfoData = MutableChunkInfoLookup[ChunkGuid];
|
|
ChunkInfoData->ShaHash = ChunkSha;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Get the PrereqIds (optional)
|
|
TSharedPtr<FJsonValue> JsonPrereqIds = JsonValueMap.FindRef(TEXT("PrereqIds"));
|
|
if (bSuccess && JsonPrereqIds.IsValid())
|
|
{
|
|
TArray<TSharedPtr<FJsonValue>> JsonPrereqIdsArray = JsonPrereqIds->AsArray();
|
|
for (TSharedPtr<FJsonValue> JsonPrereqId : JsonPrereqIdsArray)
|
|
{
|
|
ManifestMeta.PrereqIds.Add(JsonPrereqId->AsString());
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// We fall back to using the hash of the prereq exe if we have no prereq ids specified
|
|
FString PrereqFilename = ManifestMeta.PrereqPath;
|
|
PrereqFilename.ReplaceInline(TEXT("\\"), TEXT("/"));
|
|
const FFileManifest* const * FoundFileManifest = FileManifestLookup.Find(PrereqFilename);
|
|
if (FoundFileManifest)
|
|
{
|
|
FSHAHash PrereqHash;
|
|
FMemory::Memcpy(PrereqHash.Hash, (*FoundFileManifest)->FileHash.Hash, FSHA1::DigestSize);
|
|
ManifestMeta.PrereqIds.Add(PrereqHash.ToString());
|
|
}
|
|
}
|
|
|
|
// Get the DataGroupList
|
|
TSharedPtr<FJsonValue> JsonDataGroupList = JsonValueMap.FindRef(TEXT("DataGroupList"));
|
|
if (JsonDataGroupList.IsValid())
|
|
{
|
|
TSharedPtr<FJsonObject> JsonDataGroupListObj = JsonDataGroupList->AsObject();
|
|
for (auto DataGroupIt = JsonDataGroupListObj->Values.CreateConstIterator(); DataGroupIt && bSuccess; ++DataGroupIt)
|
|
{
|
|
FGuid DataGuid;
|
|
uint8 DataGroup = INDEX_NONE;
|
|
// If the list exists, we must be able to parse it ok otherwise error
|
|
bSuccess = bSuccess && FGuid::Parse(DataGroupIt.Key(), DataGuid);
|
|
bSuccess = bSuccess && FromStringBlob(DataGroupIt.Value()->AsString(), DataGroup);
|
|
if (bSuccess && MutableChunkInfoLookup.Contains(DataGuid))
|
|
{
|
|
FChunkInfo* ChunkInfoData = MutableChunkInfoLookup[DataGuid];
|
|
ChunkInfoData->GroupNumber = DataGroup;
|
|
}
|
|
}
|
|
}
|
|
else if (bSuccess)
|
|
{
|
|
// If the list did not exist in the manifest then the grouping is the deprecated crc functionality, as long
|
|
// as there are no previous parsing errors we can build the group list from the Guids.
|
|
for (FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
ChunkInfo.GroupNumber = FCrc::MemCrc_DEPRECATED(&ChunkInfo.Guid, sizeof(FGuid)) % 100;
|
|
}
|
|
}
|
|
|
|
// Get the ChunkFilesizeList
|
|
bool bHasChunkFilesizeList = false;
|
|
TSharedPtr< FJsonValue > JsonChunkFilesizeList = JsonValueMap.FindRef(TEXT("ChunkFilesizeList"));
|
|
if (JsonChunkFilesizeList.IsValid())
|
|
{
|
|
TSharedPtr< FJsonObject > JsonChunkFilesizeListObj = JsonChunkFilesizeList->AsObject();
|
|
for (auto ChunkFilesizeIt = JsonChunkFilesizeListObj->Values.CreateConstIterator(); ChunkFilesizeIt; ++ChunkFilesizeIt)
|
|
{
|
|
FGuid ChunkGuid;
|
|
int64 ChunkSize = 0;
|
|
if (FGuid::Parse(ChunkFilesizeIt.Key(), ChunkGuid))
|
|
{
|
|
FromStringBlob(ChunkFilesizeIt.Value()->AsString(), ChunkSize);
|
|
if (MutableChunkInfoLookup.Contains(ChunkGuid))
|
|
{
|
|
FChunkInfo* ChunkInfoData = MutableChunkInfoLookup[ChunkGuid];
|
|
ChunkInfoData->FileSize = ChunkSize;
|
|
bHasChunkFilesizeList = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (bHasChunkFilesizeList == false)
|
|
{
|
|
// Missing chunk list, version before we saved them compressed. Assume original fixed chunk size of 1 MiB.
|
|
for (FChunkInfo& ChunkInfo : ChunkDataList.ChunkList)
|
|
{
|
|
ChunkInfo.FileSize = 1048576;
|
|
}
|
|
}
|
|
|
|
// Get the bIsFileData value. The variable will exist in versions of StoresIfChunkOrFileData or later, otherwise the previous method is to check
|
|
// if ChunkHashList is empty.
|
|
TSharedPtr<FJsonValue> JsonIsFileData = JsonValueMap.FindRef(TEXT("bIsFileData"));
|
|
if (JsonIsFileData.IsValid() && JsonIsFileData->Type == EJson::Boolean)
|
|
{
|
|
ManifestMeta.bIsFileData = JsonIsFileData->AsBool();
|
|
}
|
|
else
|
|
{
|
|
ManifestMeta.bIsFileData = !bHasChunkHashList;
|
|
}
|
|
|
|
// Get the custom fields. This is optional, and should not fail if it does not exist
|
|
TSharedPtr< FJsonValue > JsonCustomFields = JsonValueMap.FindRef( TEXT( "CustomFields" ) );
|
|
if( JsonCustomFields.IsValid() )
|
|
{
|
|
TSharedPtr< FJsonObject > JsonCustomFieldsObj = JsonCustomFields->AsObject();
|
|
for( auto CustomFieldIt = JsonCustomFieldsObj->Values.CreateConstIterator(); CustomFieldIt && bSuccess; ++CustomFieldIt )
|
|
{
|
|
CustomFields.Fields.Add(CustomFieldIt.Key(), CustomFieldIt.Value()->AsString());
|
|
}
|
|
}
|
|
|
|
// If this is file data, fill out the guid to filename lookup, and chunk file size and SHA.
|
|
if (ManifestMeta.bIsFileData)
|
|
{
|
|
for (FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
if (FileManifest.ChunkParts.Num() == 1)
|
|
{
|
|
const FGuid& Guid = FileManifest.ChunkParts[0].Guid;
|
|
FileNameLookup.Add(Guid, &FileManifest.Filename);
|
|
if (MutableChunkInfoLookup.Contains(Guid))
|
|
{
|
|
FChunkInfo* ChunkInfoData = MutableChunkInfoLookup[Guid];
|
|
ChunkInfoData->FileSize = FileManifest.FileSize;
|
|
ChunkInfoData->ShaHash = FileManifest.FileHash;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
bSuccess = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Setup build id from backwards compat route
|
|
ManifestMeta.BuildId = FBuildPatchUtils::GetBackwardsCompatibleBuildId(ManifestMeta);
|
|
|
|
// Call OnPostLoad for the file manifest list
|
|
FileManifestList.OnPostLoad();
|
|
|
|
// Mark as should be re-saved, client that stores manifests should start using binary
|
|
bNeedsResaving = true;
|
|
|
|
// Setup internal lookups
|
|
InitLookups();
|
|
|
|
// Make sure we don't have any half loaded data
|
|
if( !bSuccess )
|
|
{
|
|
DestroyData();
|
|
}
|
|
|
|
return bSuccess;
|
|
}
|
|
|
|
EFeatureLevel FBuildPatchAppManifest::GetFeatureLevel() const
|
|
{
|
|
return ManifestMeta.FeatureLevel;
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetChunksRequiredForFiles(const TSet<FString>& Filenames, TSet<FGuid>& RequiredChunks) const
|
|
{
|
|
for (const FString& Filename : Filenames)
|
|
{
|
|
const FFileManifest* FileManifest = GetFileManifest(Filename);
|
|
if (FileManifest != nullptr)
|
|
{
|
|
for (const FChunkPart& ChunkPart : FileManifest->ChunkParts)
|
|
{
|
|
RequiredChunks.Add(ChunkPart.Guid);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetDownloadSize() const
|
|
{
|
|
return TotalDownloadSize;
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetDownloadSize(const TSet<FString>& Tags) const
|
|
{
|
|
// For each tag we iterate the files and for each new chunk we find we add the download size for it.
|
|
TSet<FGuid> RequiredChunks;
|
|
int64 TotalSize = 0;
|
|
for (const FString& Tag : Tags)
|
|
{
|
|
const TArray<const FFileManifest*>* Files = TaggedFilesLookup.Find(Tag);
|
|
if (Files != nullptr)
|
|
{
|
|
for (const FFileManifest* File : *Files)
|
|
{
|
|
for (const FChunkPart& ChunkPart : File->ChunkParts)
|
|
{
|
|
bool bAlreadyInSet;
|
|
RequiredChunks.Add(ChunkPart.Guid, &bAlreadyInSet);
|
|
if (!bAlreadyInSet)
|
|
{
|
|
const FChunkInfo * const * ChunkInfo = ChunkInfoLookup.Find(ChunkPart.Guid);
|
|
if (ChunkInfo != nullptr)
|
|
{
|
|
TotalSize += (*ChunkInfo)->FileSize;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return TotalSize;
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetDeltaDownloadSize(const TSet<FString>& Tags, const IBuildManifestRef& PreviousVersion) const
|
|
{
|
|
return GetDeltaDownloadSize(Tags, PreviousVersion, Tags);
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetDeltaDownloadSize(const TSet<FString>& InTags, const IBuildManifestRef& InPreviousVersion, const TSet<FString>& InPreviousTags) const
|
|
{
|
|
TSet<FString> Tags = InTags;
|
|
FBuildPatchAppManifestRef PreviousVersion = StaticCastSharedRef< FBuildPatchAppManifest >(InPreviousVersion);
|
|
TSet<FString> PreviousTags = InPreviousTags;
|
|
if (Tags.Num() == 0)
|
|
{
|
|
GetFileTagList(Tags);
|
|
}
|
|
if (PreviousTags.Num() == 0)
|
|
{
|
|
PreviousVersion->GetFileTagList(PreviousTags);
|
|
}
|
|
|
|
// Enumerate what is available.
|
|
TSet<FString> FilesInstalled;
|
|
TSet<FGuid> ChunksRequired;
|
|
TSet<FGuid> ChunksInstalled;
|
|
PreviousVersion->GetTaggedFileList(PreviousTags, FilesInstalled);
|
|
PreviousVersion->GetChunksRequiredForFiles(FilesInstalled, ChunksRequired);
|
|
PreviousVersion->EnumerateProducibleChunks(PreviousTags, ChunksRequired, ChunksInstalled);
|
|
|
|
// Enumerate what has changed.
|
|
FString DummyString;
|
|
TSet<FString> OutdatedFiles;
|
|
GetOutdatedFiles(PreviousVersion, DummyString, OutdatedFiles);
|
|
|
|
// Enumerate what is needed for the update.
|
|
TSet<FString> FilesNeeded;
|
|
TSet<FGuid> ChunksNeeded;
|
|
GetTaggedFileList(Tags, FilesNeeded);
|
|
FilesNeeded = OutdatedFiles.Intersect(FilesNeeded);
|
|
GetChunksRequiredForFiles(FilesNeeded, ChunksNeeded);
|
|
ChunksNeeded = ChunksNeeded.Difference(ChunksInstalled);
|
|
|
|
// Return download size of required chunks.
|
|
return GetDataSize(ChunksNeeded);
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetBuildSize() const
|
|
{
|
|
return TotalBuildSize;
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetBuildSize(const TSet<FString>& Tags) const
|
|
{
|
|
// For each tag we iterate the files and for each new file we find we add the size for it.
|
|
TSet<const FFileManifest*> RequiredFiles;
|
|
int64 TotalSize = 0;
|
|
for (const FString& Tag : Tags)
|
|
{
|
|
const TArray<const FFileManifest*>* Files = TaggedFilesLookup.Find(Tag);
|
|
if (Files != nullptr)
|
|
{
|
|
for (const FFileManifest* File : *Files)
|
|
{
|
|
bool bAlreadyInSet;
|
|
RequiredFiles.Add(File, &bAlreadyInSet);
|
|
if (!bAlreadyInSet)
|
|
{
|
|
TotalSize += File->FileSize;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return TotalSize;
|
|
}
|
|
|
|
TArray<FString> FBuildPatchAppManifest::GetBuildFileList() const
|
|
{
|
|
TArray<FString> Filenames;
|
|
GetFileList(Filenames);
|
|
return Filenames;
|
|
}
|
|
|
|
TArray<FStringView> FBuildPatchAppManifest::GetBuildFileListView() const
|
|
{
|
|
TArray<FStringView> Filenames;
|
|
GetFileList(Filenames);
|
|
return Filenames;
|
|
}
|
|
|
|
TArray<FString> FBuildPatchAppManifest::GetBuildFileList(const TSet<FString>& Tags) const
|
|
{
|
|
TArray<FString> Filenames;
|
|
GetTaggedFileList(Tags, Filenames);
|
|
return Filenames;
|
|
}
|
|
|
|
TArray<FStringView> FBuildPatchAppManifest::GetBuildFileListView(const TSet<FString>& Tags) const
|
|
{
|
|
TArray<FStringView> Filenames;
|
|
GetTaggedFileList(Tags, Filenames);
|
|
return Filenames;
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetFileSize(const TArray<FString>& Filenames) const
|
|
{
|
|
return Algo::Accumulate<int64>(Filenames, 0, [this](int64 Size, const FString& Filename){ return Size + GetFileSize(Filename); });
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetFileSize(const TSet<FString>& Filenames) const
|
|
{
|
|
return Algo::Accumulate<int64>(Filenames, 0, [this](int64 Size, const FString& Filename){ return Size + GetFileSize(Filename); });
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetFileSize(FStringView Filename) const
|
|
{
|
|
const FFileManifest *const *const FileManifest = FileManifestLookup.Find(Filename);
|
|
if (FileManifest)
|
|
{
|
|
return (*FileManifest)->FileSize;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetDataSize(const FGuid& DataGuid) const
|
|
{
|
|
if (ChunkInfoLookup.Contains(DataGuid))
|
|
{
|
|
// Chunk file sizes are stored in the info
|
|
return ChunkInfoLookup[DataGuid]->FileSize;
|
|
}
|
|
else if (ManifestMeta.bIsFileData)
|
|
{
|
|
// For file data, the file must exist in the list
|
|
check(FileNameLookup.Contains(DataGuid));
|
|
return GetFileSize(*FileNameLookup[DataGuid]);
|
|
}
|
|
else
|
|
{
|
|
// Default chunk size to be the original fixed data size of 1 MiB. Inaccurate, but represents original behavior.
|
|
return 1024 * 1024;
|
|
}
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetDataSize(const TArray<FGuid>& DataGuids) const
|
|
{
|
|
return Algo::Accumulate<int64>(DataGuids, 0, [this](int64 Size, const FGuid& DataGuid){ return Size + GetDataSize(DataGuid); });
|
|
}
|
|
|
|
int64 FBuildPatchAppManifest::GetDataSize(const TSet<FGuid>& DataGuids) const
|
|
{
|
|
return Algo::Accumulate<int64>(DataGuids, 0, [this](int64 Size, const FGuid& DataGuid){ return Size + GetDataSize(DataGuid); });
|
|
}
|
|
|
|
uint32 FBuildPatchAppManifest::GetNumFiles() const
|
|
{
|
|
return FileManifestList.FileList.Num();
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetFileList(TArray<FString>& Filenames) const
|
|
{
|
|
Algo::Transform(FileManifestList.FileList, Filenames, &FFileManifest::Filename);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetFileList(TArray<FStringView>& Filenames) const
|
|
{
|
|
Algo::Transform(FileManifestList.FileList, Filenames, &FFileManifest::Filename);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetFileList(TSet<FString>& Filenames) const
|
|
{
|
|
Algo::Transform(FileManifestList.FileList, Filenames, &FFileManifest::Filename);
|
|
}
|
|
|
|
TSet<FString> FBuildPatchAppManifest::GetFileTagList() const
|
|
{
|
|
TSet<FString> TagSet;
|
|
GetFileTagList(TagSet);
|
|
return TagSet;
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetFileTagList(TSet<FString>& Tags) const
|
|
{
|
|
Algo::Transform(TaggedFilesLookup, Tags, &TPair<FString, TArray<const BuildPatchServices::FFileManifest*>>::Key);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetTaggedFileList(const TSet<FString>& Tags, TArray<FString>& TaggedFiles) const
|
|
{
|
|
for (const FString& Tag : Tags)
|
|
{
|
|
const TArray<const FFileManifest*> *const Files = TaggedFilesLookup.Find(Tag);
|
|
if (Files != nullptr)
|
|
{
|
|
for (const FFileManifest* File : *Files)
|
|
{
|
|
TaggedFiles.Add(File->Filename);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetTaggedFileList(const TSet<FString>& Tags, TArray<FStringView>& TaggedFiles) const
|
|
{
|
|
for (const FString& Tag : Tags)
|
|
{
|
|
const TArray<const FFileManifest*>* const Files = TaggedFilesLookup.Find(Tag);
|
|
if (Files != nullptr)
|
|
{
|
|
for (const FFileManifest* File : *Files)
|
|
{
|
|
TaggedFiles.Add(File->Filename);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetTaggedFileList(const TSet<FString>& Tags, TSet<FString>& TaggedFiles) const
|
|
{
|
|
for (const FString& Tag : Tags)
|
|
{
|
|
const TArray<const FFileManifest*> *const Files = TaggedFilesLookup.Find(Tag);
|
|
if (Files != nullptr)
|
|
{
|
|
for (const FFileManifest* File : *Files)
|
|
{
|
|
TaggedFiles.Add(File->Filename);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetDataList(TArray<FGuid>& DataGuids) const
|
|
{
|
|
Algo::Transform(ChunkDataList.ChunkList, DataGuids, &FChunkInfo::Guid);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetDataList(TSet<FGuid>& DataGuids) const
|
|
{
|
|
Algo::Transform(ChunkDataList.ChunkList, DataGuids, &FChunkInfo::Guid);
|
|
}
|
|
|
|
const FFileManifest* FBuildPatchAppManifest::GetFileManifest(const FString& Filename) const
|
|
{
|
|
const FFileManifest* const * FileManifest = FileManifestLookup.Find(Filename);
|
|
return (FileManifest) ? (*FileManifest) : nullptr;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::IsFileDataManifest() const
|
|
{
|
|
return ManifestMeta.bIsFileData;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::GetChunkHash(const FGuid& ChunkGuid, uint64& OutHash) const
|
|
{
|
|
const FChunkInfo* const * ChunkInfo = ChunkInfoLookup.Find(ChunkGuid);
|
|
if (ChunkInfo)
|
|
{
|
|
OutHash = (*ChunkInfo)->Hash;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::GetChunkShaHash(const FGuid& ChunkGuid, FSHAHash& OutHash) const
|
|
{
|
|
static const uint8 Zero[FSHA1::DigestSize] = {0};
|
|
const FChunkInfo* const * ChunkInfo = ChunkInfoLookup.Find(ChunkGuid);
|
|
if (ChunkInfo != nullptr)
|
|
{
|
|
OutHash = (*ChunkInfo)->ShaHash;
|
|
return FMemory::Memcmp(OutHash.Hash, Zero, FSHA1::DigestSize) != 0;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
const BuildPatchServices::FChunkInfo* FBuildPatchAppManifest::GetChunkInfo(const FGuid& ChunkGuid) const
|
|
{
|
|
const BuildPatchServices::FChunkInfo* const * const ChunkInfoPtrPtr = ChunkInfoLookup.Find(ChunkGuid);
|
|
const BuildPatchServices::FChunkInfo* const ChunkInfoPtr = ChunkInfoPtrPtr == nullptr ? nullptr : *ChunkInfoPtrPtr;
|
|
return ChunkInfoPtr;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::GetFileHash(const FGuid& FileGuid, FSHAHash& OutHash) const
|
|
{
|
|
const FString* const * FoundFilename = FileNameLookup.Find(FileGuid);
|
|
if (FoundFilename)
|
|
{
|
|
return GetFileHash(**FoundFilename, OutHash);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::GetFileHash(const FString& Filename, FSHAHash& OutHash) const
|
|
{
|
|
const FFileManifest* const * FoundFileManifest = FileManifestLookup.Find(Filename);
|
|
if (FoundFileManifest)
|
|
{
|
|
FMemory::Memcpy(OutHash.Hash, (*FoundFileManifest)->FileHash.Hash, FSHA1::DigestSize);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::GetFilePartHash(const FGuid& FilePartGuid, uint64& OutHash) const
|
|
{
|
|
const FChunkInfo* const * FilePartInfo = ChunkInfoLookup.Find(FilePartGuid);
|
|
if (FilePartInfo)
|
|
{
|
|
OutHash = (*FilePartInfo)->Hash;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
uint32 FBuildPatchAppManifest::GetAppID() const
|
|
{
|
|
return ManifestMeta.AppID;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetAppName() const
|
|
{
|
|
return ManifestMeta.AppName;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetVersionString() const
|
|
{
|
|
return ManifestMeta.BuildVersion;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetLaunchExe() const
|
|
{
|
|
return ManifestMeta.LaunchExe;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetLaunchCommand() const
|
|
{
|
|
return ManifestMeta.LaunchCommand;
|
|
}
|
|
|
|
const TSet<FString>& FBuildPatchAppManifest::GetPrereqIds() const
|
|
{
|
|
return ManifestMeta.PrereqIds;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetPrereqName() const
|
|
{
|
|
return ManifestMeta.PrereqName;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetPrereqPath() const
|
|
{
|
|
return ManifestMeta.PrereqPath;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetPrereqArgs() const
|
|
{
|
|
return ManifestMeta.PrereqArgs;
|
|
}
|
|
|
|
const FString& FBuildPatchAppManifest::GetBuildId() const
|
|
{
|
|
return ManifestMeta.BuildId;
|
|
}
|
|
|
|
IBuildManifestRef FBuildPatchAppManifest::Duplicate() const
|
|
{
|
|
return MakeShareable(new FBuildPatchAppManifest(*this));
|
|
}
|
|
|
|
void FBuildPatchAppManifest::CopyCustomFields(const IBuildManifestRef& InOther, bool bClobber)
|
|
{
|
|
// Cast manifest parameters
|
|
FBuildPatchAppManifestRef Other = StaticCastSharedRef< FBuildPatchAppManifest >(InOther);
|
|
|
|
for (const TPair<FString, FString>& CustomField : Other->CustomFields.Fields)
|
|
{
|
|
if (bClobber || !CustomFields.Fields.Contains(CustomField.Key))
|
|
{
|
|
CustomFields.Fields.Add(CustomField.Key, CustomField.Value);
|
|
}
|
|
}
|
|
}
|
|
|
|
const IManifestFieldPtr FBuildPatchAppManifest::GetCustomField(const FString& FieldName) const
|
|
{
|
|
IManifestFieldPtr Return;
|
|
if (CustomFields.Fields.Contains(FieldName))
|
|
{
|
|
Return = MakeShareable(new FBuildPatchCustomField(CustomFields.Fields[FieldName]));
|
|
}
|
|
return Return;
|
|
}
|
|
|
|
const IManifestFieldPtr FBuildPatchAppManifest::SetCustomField(const FString& FieldName, const FString& Value)
|
|
{
|
|
CustomFields.Fields.Add(FieldName, Value);
|
|
return GetCustomField(FieldName);
|
|
}
|
|
|
|
const IManifestFieldPtr FBuildPatchAppManifest::SetCustomField(const FString& FieldName, const double& Value)
|
|
{
|
|
return SetCustomField(FieldName, ToStringBlob(Value));
|
|
}
|
|
|
|
const IManifestFieldPtr FBuildPatchAppManifest::SetCustomField(const FString& FieldName, const int64& Value)
|
|
{
|
|
return SetCustomField(FieldName, ToStringBlob(Value));
|
|
}
|
|
|
|
void FBuildPatchAppManifest::RemoveCustomField(const FString& FieldName)
|
|
{
|
|
CustomFields.Fields.Remove(FieldName);
|
|
}
|
|
|
|
int32 FBuildPatchAppManifest::EnumerateProducibleChunks(const FString& InstallDirectory, const TSet<FGuid>& ChunksRequired, TSet<FGuid>& ChunksAvailable) const
|
|
{
|
|
TMap<FString, int64> InstallationFileSizes;
|
|
return EnumerateProducibleChunks_Internal([&](const FString& Filename)
|
|
{
|
|
if (InstallationFileSizes.Contains(Filename) == false)
|
|
{
|
|
InstallationFileSizes.Add(Filename, IFileManager::Get().FileSize(*(InstallDirectory / Filename)));
|
|
}
|
|
return GetFileSize(Filename) == InstallationFileSizes[Filename];
|
|
}, ChunksRequired, ChunksAvailable);
|
|
}
|
|
|
|
int32 FBuildPatchAppManifest::EnumerateProducibleChunks(const TSet<FString>& TagSet, const TSet<FGuid>& ChunksRequired, TSet<FGuid>& ChunksAvailable) const
|
|
{
|
|
TSet<FString> AvailableFiles;
|
|
GetTaggedFileList(TagSet, AvailableFiles);
|
|
return EnumerateProducibleChunks_Internal([&](const FString& Filename) { return AvailableFiles.Contains(Filename); }, ChunksRequired, ChunksAvailable);
|
|
}
|
|
|
|
int32 FBuildPatchAppManifest::EnumerateProducibleChunks_Internal(const TFunction<bool(const FString&)>& FileAccessChecker, const TSet<FGuid>& ChunksRequired, TSet<FGuid>& ChunksAvailable) const
|
|
{
|
|
int32 Count = 0;
|
|
TMap<FGuid, FBlockStructure> ChunkBlockStructures;
|
|
ChunkBlockStructures.Reserve(ChunksRequired.Num());
|
|
for (const FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
const bool bHasFile = FileAccessChecker(FileManifest.Filename);
|
|
if (bHasFile)
|
|
{
|
|
for (const FChunkPart& ChunkPart : FileManifest.ChunkParts)
|
|
{
|
|
if (ChunksRequired.Contains(ChunkPart.Guid))
|
|
{
|
|
FBlockStructure& FoundChunkParts = ChunkBlockStructures.FindOrAdd(ChunkPart.Guid);
|
|
FoundChunkParts.Add(ChunkPart.Offset, ChunkPart.Size, ESearchDir::FromEnd);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for (const TPair<FGuid, FBlockStructure>& ChunkBlockStructurePair : ChunkBlockStructures)
|
|
{
|
|
const FGuid& ChunkBlockId = ChunkBlockStructurePair.Key;
|
|
const FBlockStructure& ChunkBlocks = ChunkBlockStructurePair.Value;
|
|
const FChunkInfo* ChunkInfoPtr = ChunkInfoLookup[ChunkBlockId];
|
|
if (ChunkBlocks.GetHead() == ChunkBlocks.GetTail() && ChunkBlocks.GetHead() && ChunkBlocks.GetHead()->GetSize() == ChunkInfoPtr->WindowSize)
|
|
{
|
|
ChunksAvailable.Add(ChunkBlockId);
|
|
++Count;
|
|
}
|
|
}
|
|
return Count;
|
|
}
|
|
|
|
TArray<FFileChunkPart> FBuildPatchAppManifest::GetFilePartsForChunk(const FGuid& ChunkId) const
|
|
{
|
|
TArray<FFileChunkPart> FileParts;
|
|
FBlockStructure FoundParts;
|
|
for (const FFileManifest& FileManifest: FileManifestList.FileList)
|
|
{
|
|
uint64 FileOffset = 0;
|
|
for (const FChunkPart& ChunkPart : FileManifest.ChunkParts)
|
|
{
|
|
if (ChunkId == ChunkPart.Guid)
|
|
{
|
|
FFileChunkPart FileChunkPart;
|
|
FileChunkPart.Filename = FileManifest.Filename;
|
|
FileChunkPart.ChunkPart = ChunkPart;
|
|
FileChunkPart.FileOffset = FileOffset;
|
|
FileParts.Add(FileChunkPart);
|
|
FoundParts.Add(ChunkPart.Offset, ChunkPart.Size, ESearchDir::FromEnd);
|
|
}
|
|
FileOffset += ChunkPart.Size;
|
|
}
|
|
}
|
|
|
|
// If the structure is not a single complete block, then the chunk is not recoverable.
|
|
if (FoundParts.GetHead() == nullptr || FoundParts.GetHead() != FoundParts.GetTail() /* || @TODO Possible to check FoundParts.GetHead()->GetSize() != UncompressedDataSize here? */)
|
|
{
|
|
FileParts.Empty();
|
|
}
|
|
return FileParts;
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::HasFileAttributes() const
|
|
{
|
|
for (const FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
if (FileManifest.FileMetaFlags != EFileMetaFlags::None)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetRemovableFiles(const IBuildManifestRef& InOldManifest, TArray<FString>& RemovableFiles) const
|
|
{
|
|
// Cast manifest parameters
|
|
const FBuildPatchAppManifestRef OldManifest = StaticCastSharedRef<FBuildPatchAppManifest>(InOldManifest);
|
|
GetRemovableFiles(OldManifest.Get(), RemovableFiles);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetRemovableFiles(const FBuildPatchAppManifest& OldManifest, TArray<FString>& RemovableFiles) const
|
|
{
|
|
// Simply put, any files that exist in the OldManifest file list, but do not in this manifest's file list, are assumed
|
|
// to be files no longer required by the build
|
|
for (const FFileManifest& OldFile : OldManifest.FileManifestList.FileList)
|
|
{
|
|
if (!FileManifestLookup.Contains(OldFile.Filename))
|
|
{
|
|
RemovableFiles.Add(OldFile.Filename);
|
|
}
|
|
}
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetRemovableFiles(const TCHAR* InInstallPath, TArray<FString>& RemovableFiles) const
|
|
{
|
|
// For the logic below our InstallPath must be collapsed, normalised and end with a /
|
|
// Normalising a directory removes trailing slashes, and we need the slash back on before calling CollapseRelativeDirectories.
|
|
FString InstallPath = InInstallPath;
|
|
FPaths::NormalizeDirectoryName(InstallPath);
|
|
InstallPath += TEXT("/");
|
|
FPaths::CollapseRelativeDirectories(InstallPath);
|
|
|
|
TArray<FString> AllFiles;
|
|
IFileManager::Get().FindFilesRecursive(AllFiles, *InstallPath, TEXT("*"), true, false);
|
|
|
|
#if PLATFORM_MAC
|
|
// On Mac paths in manifest start with app bundle name
|
|
if (InstallPath.EndsWith(".app/"))
|
|
{
|
|
InstallPath = FPaths::GetPath(InstallPath.LeftChop(1)) + "/";
|
|
}
|
|
#endif
|
|
|
|
for (int32 FileIndex = 0; FileIndex < AllFiles.Num(); ++FileIndex)
|
|
{
|
|
const FString& Filename = AllFiles[FileIndex].RightChop(InstallPath.Len());
|
|
if (!FileManifestLookup.Contains(Filename))
|
|
{
|
|
RemovableFiles.Add(AllFiles[FileIndex]);
|
|
}
|
|
}
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::NeedsResaving() const
|
|
{
|
|
// The bool is marked during file load if we load an old version that should be upgraded
|
|
return bNeedsResaving;
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetOutdatedFiles(const IBuildManifestRef& InOldManifest, TSet<FString>& OutdatedFiles) const
|
|
{
|
|
GetOutdatedFiles(FBuildPatchAppManifestPtr(StaticCastSharedRef<FBuildPatchAppManifest>(InOldManifest)), TEXT(""), OutdatedFiles);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetOutdatedFiles(const FBuildPatchAppManifestPtr& OldManifest, const FString& InstallDirectory, TSet<FString>& OutDatedFiles) const
|
|
{
|
|
GetOutdatedFiles(OldManifest.Get(), InstallDirectory, OutDatedFiles);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetOutdatedFiles(const FBuildPatchAppManifest* OldManifest, const FString& InstallDirectory, TSet<FString>& OutDatedFiles) const
|
|
{
|
|
TSet<FString> FilesToCheck;
|
|
GetFileList(FilesToCheck);
|
|
GetOutdatedFiles(OldManifest, InstallDirectory, FilesToCheck, OutDatedFiles);
|
|
}
|
|
|
|
void FBuildPatchAppManifest::GetOutdatedFiles(const FBuildPatchAppManifest* OldManifest, const FString& InstallDirectory, const TSet<FString>& FilesToCheck, TSet<FString>& OutDatedFiles) const
|
|
{
|
|
const bool bCheckExistingFile = InstallDirectory.IsEmpty() == false;
|
|
if (nullptr == OldManifest)
|
|
{
|
|
// All files are outdated if no OldManifest
|
|
TSet<FString> AllFiles;
|
|
GetFileList(AllFiles);
|
|
OutDatedFiles.Append(AllFiles.Intersect(FilesToCheck));
|
|
}
|
|
else
|
|
{
|
|
// Enumerate files in the this file list, that do not exist, or have different hashes in the OldManifest
|
|
// to be files no longer required by the build
|
|
for (const FString& FileToCheck : FilesToCheck)
|
|
{
|
|
const FFileManifest* NewFile = GetFileManifest(FileToCheck);
|
|
if (NewFile != nullptr)
|
|
{
|
|
// Check changed
|
|
if (IsFileOutdated(*OldManifest, NewFile->Filename))
|
|
{
|
|
OutDatedFiles.Add(NewFile->Filename);
|
|
}
|
|
// Double check an unchanged file is not missing (size will be -1) or is incorrect size
|
|
else if (bCheckExistingFile)
|
|
{
|
|
const int64 ExistingFileSize = IFileManager::Get().FileSize(*(InstallDirectory / NewFile->Filename));
|
|
if ((ExistingFileSize < 0) || (ExistingFileSize != NewFile->FileSize))
|
|
{
|
|
OutDatedFiles.Add(NewFile->Filename);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::IsFileOutdated(const FBuildPatchAppManifestRef& OldManifest, const FString& Filename) const
|
|
{
|
|
return IsFileOutdated(OldManifest.Get(), Filename);
|
|
}
|
|
|
|
bool FBuildPatchAppManifest::IsFileOutdated(const FBuildPatchAppManifest& OldManifest, const FString& Filename) const
|
|
{
|
|
// If both app manifests are the same, return false as only repair would touch the file.
|
|
if (&OldManifest == this)
|
|
{
|
|
return false;
|
|
}
|
|
// Get file manifests
|
|
const FFileManifest* OldFile = OldManifest.GetFileManifest(Filename);
|
|
const FFileManifest* NewFile = GetFileManifest(Filename);
|
|
// Out of date if not in either manifest
|
|
if (!OldFile || !NewFile)
|
|
{
|
|
return true;
|
|
}
|
|
// Different hash means different file
|
|
if (OldFile->FileHash != NewFile->FileHash)
|
|
{
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
uint32 FBuildPatchAppManifest::GetNumberOfChunkReferences(const FGuid& ChunkGuid) const
|
|
{
|
|
uint32 RefCount = 0;
|
|
// For each file in the manifest, check if it has references to this chunk
|
|
for (const FFileManifest& FileManifest : FileManifestList.FileList)
|
|
{
|
|
for (const FChunkPart& ChunkPart : FileManifest.ChunkParts)
|
|
{
|
|
if (ChunkPart.Guid == ChunkGuid)
|
|
{
|
|
++RefCount;
|
|
}
|
|
}
|
|
}
|
|
return RefCount;
|
|
}
|
|
|
|
#undef LOCTEXT_NAMESPACE
|