Files
UnrealEngine/Engine/Source/Editor/UnrealEd/Private/Cooker/LooseCookedPackageWriter.cpp
2025-05-18 13:04:45 +08:00

750 lines
25 KiB
C++

// Copyright Epic Games, Inc. All Rights Reserved.
#include "Cooker/LooseCookedPackageWriter.h"
#include "AssetRegistry/AssetRegistryState.h"
#include "AssetRegistry/IAssetRegistry.h"
#include "Async/ParallelFor.h"
#include "Cooker/AsyncIODelete.h"
#include "Cooker/CompactBinaryTCP.h"
#include "Cooker/CookSandbox.h"
#include "HAL/FileManager.h"
#include "HAL/PlatformFileManager.h"
#include "Interfaces/ITargetPlatform.h"
#include "LooseFilesCookArtifactReader.h"
#include "Misc/FileHelper.h"
#include "Misc/Paths.h"
#include "Misc/PathViews.h"
#include "PackageStoreOptimizer.h"
#include "Serialization/ArchiveStackTrace.h"
#include "Serialization/CompactBinarySerialization.h"
#include "Serialization/CompactBinaryWriter.h"
#include "Serialization/FilePackageWriterUtil.h"
#include "Serialization/LargeMemoryWriter.h"
#include "UObject/SavePackage.h"
LLM_DEFINE_TAG(Cooker_PackageStoreManifest);
FLooseCookedPackageWriter::FLooseCookedPackageWriter(const FString& InOutputPath,
const FString& InMetadataDirectoryPath, const ITargetPlatform* InTargetPlatform, FAsyncIODelete& InAsyncIODelete,
UE::Cook::FCookSandbox& InSandboxFile, FBeginCacheCallback&& InBeginCacheCallback,
FRegisterDeterminismHelperCallback&& InRegisterDeterminismHelperCallback,
TSharedRef<FLooseFilesCookArtifactReader> InCookArtifactReader)
: CookArtifactReader(InCookArtifactReader)
, OutputPath(InOutputPath)
, MetadataDirectoryPath(InMetadataDirectoryPath)
, TargetPlatform(*InTargetPlatform)
, SandboxFile(InSandboxFile)
, AsyncIODelete(InAsyncIODelete)
, BeginCacheCallback(MoveTemp(InBeginCacheCallback))
, RegisterDeterminismHelperCallback(MoveTemp(InRegisterDeterminismHelperCallback))
{
}
FLooseCookedPackageWriter::~FLooseCookedPackageWriter()
{
}
void FLooseCookedPackageWriter::BeginPackage(const FBeginPackageInfo& Info)
{
Super::BeginPackage(Info);
LLM_SCOPE_BYTAG(Cooker_PackageStoreManifest);
FScopeLock Lock(&OplogLock);
FOplogPackageInfo& PackageInfo = Oplog.FindOrAdd(Info.PackageName);
PackageInfo.PackageName = Info.PackageName;
PackageInfo.PackageDataChunks.Reset();
PackageInfo.BulkDataChunks.Reset();
}
void FLooseCookedPackageWriter::CommitPackageInternal(FPackageWriterRecords::FPackage&& BaseRecord, const FCommitPackageInfo& Info)
{
{
FFilePackageWriterUtil::FRecord& Record = static_cast<FFilePackageWriterUtil::FRecord&>(BaseRecord);
FFilePackageWriterUtil::FWritePackageParameters Parameters(Record, Info, &AllPackageHashes, &PackageHashesLock, bProvidePerPackageResults);
FFilePackageWriterUtil::WritePackage(Parameters);
}
if (Info.Status != IPackageWriter::ECommitStatus::Canceled && Info.Status != IPackageWriter::ECommitStatus::NotCommitted)
{
FPackageWriterRecords::FPackage& Record = static_cast<FPackageWriterRecords::FPackage&>(BaseRecord);
UpdateManifest(Record);
}
}
void FLooseCookedPackageWriter::CompleteExportsArchiveForDiff(FPackageInfo& Info, FLargeMemoryWriter& ExportsArchive)
{
FPackageWriterRecords::FPackage& BaseRecord = Records.FindRecordChecked(Info.PackageName);
FFilePackageWriterUtil::FRecord& Record = static_cast<FFilePackageWriterUtil::FRecord&>(BaseRecord);
Record.bCompletedExportsArchiveForDiff = true;
// Add on all the attachments which are usually added on during Commit. The order must match AsyncSave.
for (FBulkDataRecord& BulkRecord : Record.BulkDatas)
{
if (BulkRecord.Info.BulkDataType == FBulkDataInfo::AppendToExports && BulkRecord.Info.MultiOutputIndex == Info.MultiOutputIndex)
{
ExportsArchive.Serialize(const_cast<void*>(BulkRecord.Buffer.GetData()),
BulkRecord.Buffer.GetSize());
}
}
for (FLinkerAdditionalDataRecord& AdditionalRecord : Record.LinkerAdditionalDatas)
{
if (AdditionalRecord.Info.MultiOutputIndex == Info.MultiOutputIndex)
{
ExportsArchive.Serialize(const_cast<void*>(AdditionalRecord.Buffer.GetData()),
AdditionalRecord.Buffer.GetSize());
}
}
uint32 FooterData = PACKAGE_FILE_TAG;
ExportsArchive << FooterData;
for (FPackageTrailerRecord& PackageTrailer : Record.PackageTrailers)
{
if (PackageTrailer.Info.MultiOutputIndex == Info.MultiOutputIndex)
{
ExportsArchive.Serialize(const_cast<void*>(PackageTrailer.Buffer.GetData()),
PackageTrailer.Buffer.GetSize());
}
}
}
int64 FLooseCookedPackageWriter::GetExportsFooterSize()
{
return sizeof(uint32);
}
FPackageWriterRecords::FPackage* FLooseCookedPackageWriter::ConstructRecord()
{
//Watch out : if you change the type of record, you need to go through this file to make sure there is not a static cast that would not work anymore.
return new FFilePackageWriterUtil::FRecord();
}
void FLooseCookedPackageWriter::UpdateManifest(const FPackageWriterRecords::FPackage& Record)
{
LLM_SCOPE_BYTAG(Cooker_PackageStoreManifest);
FScopeLock Lock(&OplogLock);
for (const FPackageWriterRecords::FWritePackage& Package : Record.Packages)
{
FOplogPackageInfo* PackageInfo = Oplog.Find(Package.Info.PackageName);
check(PackageInfo);
FOplogChunkInfo& ChunkInfo = PackageInfo->PackageDataChunks.AddDefaulted_GetRef();
ChunkInfo.ChunkId = Package.Info.ChunkId;
FStringView RelativePathView;
FPathViews::TryMakeChildPathRelativeTo(Package.Info.LooseFilePath, OutputPath, RelativePathView);
ChunkInfo.RelativeFileName = RelativePathView;
}
for (const FPackageWriterRecords::FBulkData& BulkData : Record.BulkDatas)
{
FOplogPackageInfo* PackageInfo = Oplog.Find(BulkData.Info.PackageName);
check(PackageInfo);
FOplogChunkInfo& ChunkInfo = PackageInfo->BulkDataChunks.AddDefaulted_GetRef();
ChunkInfo.ChunkId = BulkData.Info.ChunkId;
FStringView RelativePathView;
FPathViews::TryMakeChildPathRelativeTo(BulkData.Info.LooseFilePath, OutputPath, RelativePathView);
ChunkInfo.RelativeFileName = RelativePathView;
}
}
bool FLooseCookedPackageWriter::GetPreviousCookedBytes(const FPackageInfo& Info, FPreviousCookedBytesData& OutData)
{
UE::ArchiveStackTrace::FPackageData ExistingPackageData;
TUniquePtr<uint8, UE::ArchiveStackTrace::FDeleteByFree> Bytes;
UE::ArchiveStackTrace::LoadPackageIntoMemory(*Info.LooseFilePath, ExistingPackageData, Bytes);
OutData.Size = ExistingPackageData.Size;
OutData.HeaderSize = ExistingPackageData.HeaderSize;
OutData.StartOffset = ExistingPackageData.StartOffset;
OutData.Data.Reset(Bytes.Release());
return OutData.Data.IsValid();
}
FDateTime FLooseCookedPackageWriter::GetPreviousCookTime() const
{
const FString PreviousAssetRegistry = FPaths::Combine(MetadataDirectoryPath, GetDevelopmentAssetRegistryFilename());
return IFileManager::Get().GetTimeStamp(*PreviousAssetRegistry);
}
void FLooseCookedPackageWriter::RegisterDeterminismHelper(UObject* SourceObject,
const TRefCountPtr<UE::Cook::IDeterminismHelper>& DeterminismHelper)
{
if (RegisterDeterminismHelperCallback)
{
RegisterDeterminismHelperCallback(SourceObject, DeterminismHelper);
}
}
void FLooseCookedPackageWriter::Initialize(const FCookInfo& Info)
{
bLegacyIterativeSharedBuild = Info.bLegacyIterativeSharedBuild;
if (Info.bFullBuild && !Info.bWorkerOnSharedSandbox)
{
DeleteSandboxDirectory();
}
if (!Info.bWorkerOnSharedSandbox)
{
TRACE_CPUPROFILER_EVENT_SCOPE(SaveScriptObjects);
FPackageStoreOptimizer PackageStoreOptimizer;
PackageStoreOptimizer.Initialize();
FIoBuffer ScriptObjectsBuffer = PackageStoreOptimizer.CreateScriptObjectsBuffer();
FFileHelper::SaveArrayToFile(
MakeArrayView(ScriptObjectsBuffer.Data(), ScriptObjectsBuffer.DataSize()),
*(MetadataDirectoryPath / TEXT("scriptobjects.bin")));
}
}
EPackageWriterResult FLooseCookedPackageWriter::BeginCacheForCookedPlatformData(
FBeginCacheForCookedPlatformDataInfo& Info)
{
return BeginCacheCallback(Info);
}
void FLooseCookedPackageWriter::WriteOplogEntry(FCbWriter& Writer, const FOplogPackageInfo& PackageInfo)
{
Writer.BeginObject();
Writer.BeginObject("packagestoreentry");
Writer << "packagename" << PackageInfo.PackageName;
Writer.EndObject();
Writer.BeginArray("packagedata");
for (const FOplogChunkInfo& ChunkInfo : PackageInfo.PackageDataChunks)
{
Writer.BeginObject();
Writer << "id" << ChunkInfo.ChunkId;
Writer << "filename" << ChunkInfo.RelativeFileName;
Writer.EndObject();
}
Writer.EndArray();
Writer.BeginArray("bulkdata");
for (const FOplogChunkInfo& ChunkInfo : PackageInfo.BulkDataChunks)
{
Writer.BeginObject();
Writer << "id" << ChunkInfo.ChunkId;
Writer << "filename" << ChunkInfo.RelativeFileName;
Writer.EndObject();
}
Writer.EndArray();
Writer.EndObject();
}
bool FLooseCookedPackageWriter::ReadOplogEntry(FOplogPackageInfo& PackageInfo, const FCbFieldView& Field)
{
FCbObjectView PackageStoreEntryObjectView = Field["packagestoreentry"].AsObjectView();
if (!PackageStoreEntryObjectView)
{
return false;
}
PackageInfo.PackageName = FName(PackageStoreEntryObjectView["packagename"].AsString());
FCbArrayView PackageDataView = Field["packagedata"].AsArrayView();
PackageInfo.PackageDataChunks.Reset();
PackageInfo.PackageDataChunks.Reserve(PackageDataView.Num());
for (const FCbFieldView& ChunkEntry : PackageDataView)
{
FOplogChunkInfo& ChunkInfo = PackageInfo.PackageDataChunks.AddDefaulted_GetRef();
ChunkInfo.ChunkId.Set(ChunkEntry["id"].AsObjectId().GetView());
ChunkInfo.RelativeFileName = FString(ChunkEntry["filename"].AsString());
}
FCbArrayView BulkDataView = Field["bulkdata"].AsArrayView();
PackageInfo.BulkDataChunks.Reset();
PackageInfo.BulkDataChunks.Reserve(BulkDataView.Num());
for (const FCbFieldView& ChunkEntry : BulkDataView)
{
FOplogChunkInfo& ChunkInfo = PackageInfo.BulkDataChunks.AddDefaulted_GetRef();
ChunkInfo.ChunkId.Set(ChunkEntry["id"].AsObjectId().GetView());
ChunkInfo.RelativeFileName = FString(ChunkEntry["filename"].AsString());
}
return true;
}
TMap<FName, TRefCountPtr<FPackageHashes>>& FLooseCookedPackageWriter::GetPackageHashes()
{
return AllPackageHashes;
}
void FLooseCookedPackageWriter::BeginCook(const FCookInfo& Info)
{
if (!Info.bWorkerOnSharedSandbox)
{
TRACE_CPUPROFILER_EVENT_SCOPE(LoadPackageStoreManifest);
FString PackageStoreManifestFilePath = FPaths::Combine(MetadataDirectoryPath, TEXT("packagestore.manifest"));
TUniquePtr<FArchive> Ar(IFileManager::Get().CreateFileReader(*PackageStoreManifestFilePath));
if (Ar)
{
FCbField ManifestField = LoadCompactBinary(*Ar);
FCbField OplogField = ManifestField["oplog"];
if (OplogField)
{
FCbArray EntriesArray = OplogField["entries"].AsArray();
FScopeLock Lock(&OplogLock);
Oplog.Reserve(EntriesArray.Num());
for (const FCbField& EntryField : EntriesArray)
{
FOplogPackageInfo PackageInfo;
ReadOplogEntry(PackageInfo, EntryField);
Oplog.Add(PackageInfo.PackageName, MoveTemp(PackageInfo));
}
}
}
}
else
{
bProvidePerPackageResults = true;
}
AllPackageHashes.Empty();
}
void FLooseCookedPackageWriter::EndCook(const FCookInfo& Info)
{
if (!Info.bWorkerOnSharedSandbox)
{
TRACE_CPUPROFILER_EVENT_SCOPE(SavePackageStoreManifest);
FScopeLock Lock(&OplogLock);
// Sort packages for determinism
TArray<const FOplogPackageInfo*> SortedPackages;
SortedPackages.Reserve(Oplog.Num());
for (const auto& KV : Oplog)
{
SortedPackages.Add(&KV.Value);
}
Algo::Sort(SortedPackages, [](const FOplogPackageInfo* A, const FOplogPackageInfo* B)
{
return A->PackageName.LexicalLess(B->PackageName);
});
FCbWriter ManifestWriter;
ManifestWriter.BeginObject();
ManifestWriter.BeginObject("oplog");
ManifestWriter.BeginArray("entries");
for (const FOplogPackageInfo* Package : SortedPackages)
{
WriteOplogEntry(ManifestWriter, *Package);
}
ManifestWriter.EndArray();
ManifestWriter.EndObject();
ManifestWriter.EndObject();
FString PackageStoreManifestFilePath = FPaths::Combine(MetadataDirectoryPath, TEXT("packagestore.manifest"));
TUniquePtr<FArchive> Ar(IFileManager::Get().CreateFileWriter(*PackageStoreManifestFilePath));
if (Ar)
{
SaveCompactBinary(*Ar, ManifestWriter.Save());
}
else
{
UE_LOG(LogSavePackage, Error, TEXT("Failed saving package store manifest file '%s'"), *PackageStoreManifestFilePath);
}
}
}
TUniquePtr<FAssetRegistryState> FLooseCookedPackageWriter::LoadPreviousAssetRegistry()
{
// Report files from the shared build if the option is set
FString PreviousAssetRegistryFile;
if (bLegacyIterativeSharedBuild)
{
// clean the sandbox
DeleteSandboxDirectory();
PreviousAssetRegistryFile = FPaths::Combine(*FPaths::ProjectSavedDir(), TEXT("SharedIterativeBuild"),
*TargetPlatform.PlatformName(), TEXT("Metadata"), GetDevelopmentAssetRegistryFilename());
}
else
{
PreviousAssetRegistryFile = FPaths::Combine(MetadataDirectoryPath, GetDevelopmentAssetRegistryFilename());
}
PackageNameToCookedFiles.Reset();
TUniquePtr<FArchive> Reader(CookArtifactReader->CreateFileReader(*PreviousAssetRegistryFile));
if (!Reader)
{
RemoveCookedPackages();
return TUniquePtr<FAssetRegistryState>();
}
TUniquePtr<FAssetRegistryState> PreviousState = MakeUnique<FAssetRegistryState>();
PreviousState->Load(*Reader);
// If we are legacyiterative from a shared build the cooked files do not exist in the local cooked directory;
// we assume they are packaged in the pak file (which we don't want to extract to confirm) and keep them all.
if (!bLegacyIterativeSharedBuild)
{
// For regular iteration, remove each SuccessfulSave cooked file from the PreviousState if it no longer exists
// in the cooked directory. Keep the FailedSave previous cook packages; we don't expect them to exist on disk.
// Also, remove from the registry and from disk the cooked files that no longer exist in the editor.
GetAllCookedFiles();
TSet<FName> RemoveFromRegistry;
TSet<FName> RemoveFromDisk;
RemoveFromDisk.Reserve(PackageNameToCookedFiles.Num());
for (TPair<FName, TArray<FString>>& Pair : PackageNameToCookedFiles)
{
RemoveFromDisk.Add(Pair.Key);
}
IAssetRegistry& AssetRegistry = IAssetRegistry::GetChecked();
for (const TPair<FName, const FAssetPackageData*>& Pair : PreviousState->GetAssetPackageDataMap())
{
FName PackageName = Pair.Key;
bool bCurrentPackageExists = AssetRegistry.DoesPackageExistOnDisk(PackageName);
bool bNoLongerExistsInEditor = false;
bool bIsScriptPackage = FPackageName::IsScriptPackage(WriteToString<256>(PackageName));
if (!bCurrentPackageExists)
{
// Script and generated packages do not exist uncooked
// Check that the package is not an exception before removing from cooked
bool bIsCookedOnly = bIsScriptPackage;
if (!bIsCookedOnly)
{
PreviousState->EnumerateAssetsByPackageName(PackageName, [&bIsCookedOnly](const FAssetData* AssetData)
{
bIsCookedOnly |= !!(AssetData->PackageFlags & PKG_CookGenerated);
return true; // Keep iterating
});
}
bNoLongerExistsInEditor = !bIsCookedOnly;
}
if (bNoLongerExistsInEditor)
{
// Remove package from both disk and registry
// Keep its RemoveFromDisk entry
RemoveFromRegistry.Add(PackageName); // Add a RemoveFromRegistry entry
}
else
{
// Keep package on disk if it exists. Keep package in registry if it exists on disk or was a FailedSave.
bool bExistsOnDisk = (RemoveFromDisk.Remove(PackageName) == 1); // Remove its RemoveFromDisk entry
const FAssetPackageData* PackageData = Pair.Value;
if (!bExistsOnDisk && PackageData->DiskSize >= 0 && !bIsScriptPackage)
{
// Add RemoveFromRegistry entry if it didn't exist on disk and is a SuccessfulSave package
RemoveFromRegistry.Add(PackageName);
}
}
}
if (RemoveFromRegistry.Num())
{
PreviousState->PruneAssetData(TSet<FName>(), RemoveFromRegistry, FAssetRegistrySerializationOptions());
}
if (RemoveFromDisk.Num())
{
RemoveCookedPackagesByPackageName(RemoveFromDisk.Array(), true /* bRemoveRecords */);
}
}
return PreviousState;
}
FCbObject FLooseCookedPackageWriter::GetOplogAttachment(FName PackageName, FUtf8StringView AttachmentKey)
{
/** Oplog attachments are not implemented by FLooseCookedPackageWriter */
return FCbObject();
}
void FLooseCookedPackageWriter::GetOplogAttachments(TArrayView<FName> PackageNames,
TArrayView<FUtf8StringView> AttachmentKeys,
TUniqueFunction<void(FName PackageName, FUtf8StringView AttachmentKey, FCbObject&& Attachment)>&& Callback)
{
/** Oplog attachments are not implemented by FLooseCookedPackageWriter */
for (FName PackageName: PackageNames)
{
for (FUtf8StringView AttachmentKey: AttachmentKeys)
{
Callback(PackageName, AttachmentKey, FCbObject());
}
}
}
IPackageWriter::ECommitStatus FLooseCookedPackageWriter::GetCommitStatus(FName PackageName)
{
// GetCommitStatus is not implemented by FLooseCookedPackageWriter
return ECommitStatus::NotCommitted;
}
void FLooseCookedPackageWriter::RemoveCookedPackages(TArrayView<const FName> PackageNamesToRemove)
{
if (PackageNameToCookedFiles.IsEmpty())
{
FindAndDeleteCookedFilesForPackages(PackageNamesToRemove);
return;
}
// if we are going to clear the cooked packages it is conceivable that we will recook the packages which we just cooked
// that means it's also conceivable that we will recook the same package which currently has an outstanding async write request
UPackage::WaitForAsyncFileWrites();
RemoveCookedPackagesByPackageName(PackageNamesToRemove, false /* bRemoveRecords */);
// PackageNameToCookedFiles is no longer used after the RemoveCookedPackages call at the beginning of the cook.
PackageNameToCookedFiles.Empty();
}
void FLooseCookedPackageWriter::RemoveCookedPackagesByPackageName(TArrayView<const FName> PackageNamesToRemove, bool bRemoveRecords)
{
auto DeletePackageLambda = [&PackageNamesToRemove, this](int32 PackageIndex)
{
FName PackageName = PackageNamesToRemove[PackageIndex];
TArray<FString>* CookedFileNames = PackageNameToCookedFiles.Find(PackageName);
if (CookedFileNames)
{
for (const FString& CookedFileName : *CookedFileNames)
{
IFileManager::Get().Delete(*CookedFileName, true, true, true);
}
}
};
ParallelFor(PackageNamesToRemove.Num(), DeletePackageLambda);
if (bRemoveRecords)
{
for (FName PackageName : PackageNamesToRemove)
{
PackageNameToCookedFiles.Remove(PackageName);
}
}
}
void FLooseCookedPackageWriter::UpdatePackageModificationStatus(FName PackageName,
bool bIncrementallyUnmodified, bool& bInOutShouldIncrementallySkip)
{
}
TFuture<FCbObject> FLooseCookedPackageWriter::WriteMPCookMessageForPackage(FName PackageName)
{
FCbFieldIterator OplogEntryField;
{
FOplogPackageInfo PackageInfo;
bool bValid = false;
{
FScopeLock Lock(&OplogLock);
bValid = Oplog.RemoveAndCopyValue(PackageName, PackageInfo);
}
if (bValid)
{
check(PackageName == PackageInfo.PackageName);
FCbWriter OplogEntryWriter;
WriteOplogEntry(OplogEntryWriter, PackageInfo);
OplogEntryField = OplogEntryWriter.Save();
}
}
TRefCountPtr<FPackageHashes> PackageHashes;
{
FScopeLock PackageHashesScopeLock(&PackageHashesLock);
AllPackageHashes.RemoveAndCopyValue(PackageName, PackageHashes);
}
auto ComposeMessage = [OplogEntryField = MoveTemp(OplogEntryField)](FPackageHashes* PackageHashes)
{
FCbWriter Writer;
Writer.BeginObject();
if (OplogEntryField.HasValue())
{
Writer << "OplogEntry" << OplogEntryField;
}
if (PackageHashes)
{
Writer << "PackageHash" << PackageHashes->PackageHash;
Writer << "ChunkHashes" << PackageHashes->ChunkHashes;
}
Writer.EndObject();
return Writer.Save().AsObject();
};
if (PackageHashes && PackageHashes->CompletionFuture.IsValid())
{
TUniquePtr<TPromise<FCbObject>> Promise(new TPromise<FCbObject>());
TFuture<FCbObject> ResultFuture = Promise->GetFuture();
PackageHashes->CompletionFuture.Next(
[PackageHashes, Promise = MoveTemp(Promise), ComposeMessage = MoveTemp(ComposeMessage)]
(int)
{
Promise->SetValue(ComposeMessage(PackageHashes.GetReference()));
});
return ResultFuture;
}
else
{
TPromise<FCbObject> Promise;
Promise.SetValue(ComposeMessage(PackageHashes.GetReference()));
return Promise.GetFuture();
}
}
bool FLooseCookedPackageWriter::TryReadMPCookMessageForPackage(FName PackageName, FCbObjectView Message)
{
FOplogPackageInfo PackageInfo;
FCbFieldView OplogEntryField(Message["OplogEntry"]);
if (ReadOplogEntry(PackageInfo, OplogEntryField))
{
check(PackageName == PackageInfo.PackageName);
FScopeLock Lock(&OplogLock);
Oplog.Add(PackageName, MoveTemp(PackageInfo));
}
bool bOk = true;
TRefCountPtr<FPackageHashes> ThisPackageHashes(new FPackageHashes());
if (LoadFromCompactBinary(Message["PackageHash"], ThisPackageHashes->PackageHash))
{
bOk = LoadFromCompactBinary(Message["ChunkHashes"], ThisPackageHashes->ChunkHashes) & bOk;
if (bOk)
{
bool bAlreadyExisted = false;
{
FScopeLock PackageHashesScopeLock(&PackageHashesLock);
TRefCountPtr<FPackageHashes>& ExistingPackageHashes = AllPackageHashes.FindOrAdd(PackageName);
bAlreadyExisted = ExistingPackageHashes.IsValid();
ExistingPackageHashes = ThisPackageHashes;
}
if (bAlreadyExisted)
{
UE_LOG(LogSavePackage, Error, TEXT("FLooseCookedPackageWriter encountered the same package twice in a cook! (%s)"),
*PackageName.ToString());
}
}
}
return bOk;
}
void FLooseCookedPackageWriter::RemoveCookedPackages()
{
DeleteSandboxDirectory();
}
void FLooseCookedPackageWriter::DeleteSandboxDirectory()
{
// if we are going to clear the cooked packages it is conceivable that we will recook the packages which we just cooked
// that means it's also conceivable that we will recook the same package which currently has an outstanding async write request
UPackage::WaitForAsyncFileWrites();
FString SandboxDirectory = OutputPath;
FPaths::NormalizeDirectoryName(SandboxDirectory);
AsyncIODelete.DeleteDirectory(SandboxDirectory);
}
class FPackageSearchVisitor : public IPlatformFile::FDirectoryVisitor
{
TArray<FString>& FoundFiles;
public:
FPackageSearchVisitor(TArray<FString>& InFoundFiles)
: FoundFiles(InFoundFiles)
{}
virtual bool Visit(const TCHAR* FilenameOrDirectory, bool bIsDirectory)
{
if (bIsDirectory == false)
{
FString Filename(FilenameOrDirectory);
FStringView Extension(FPathViews::GetExtension(Filename, true /* bIncludeDot */));
if (!Extension.IsEmpty())
{
EPackageExtension ExtensionEnum = FPackagePath::ParseExtension(Extension);
if (ExtensionEnum != EPackageExtension::Unspecified && ExtensionEnum != EPackageExtension::Custom)
{
FoundFiles.Add(Filename);
}
}
}
return true;
}
};
void FLooseCookedPackageWriter::GetAllCookedFiles()
{
TRACE_CPUPROFILER_EVENT_SCOPE(FLooseCookedPackageWriter::GetAllCookedFiles);
const FString& SandboxRootDir = OutputPath;
TArray<FString> CookedFiles;
{
IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();
FPackageSearchVisitor PackageSearch(CookedFiles);
PlatformFile.IterateDirectoryRecursively(*SandboxRootDir, PackageSearch);
}
const FString SandboxProjectDir = FPaths::Combine(OutputPath, FApp::GetProjectName()) + TEXT("/");
UE::Cook::FCookSandboxConvertCookedPathToPackageNameContext Context;
Context.SandboxRootDir = SandboxRootDir;
Context.SandboxProjectDir = SandboxProjectDir;
SandboxFile.FillContext(Context);
for (const FString& CookedFile : CookedFiles)
{
const FName PackageName = SandboxFile.ConvertCookedPathToPackageName(CookedFile, Context);
if (!PackageName.IsNone())
{
PackageNameToCookedFiles.FindOrAdd(PackageName).Add(CookedFile);
}
}
}
void FLooseCookedPackageWriter::FindAndDeleteCookedFilesForPackages(TConstArrayView<FName> PackageNames)
{
const FString& SandboxRootDir = OutputPath;
const FString SandboxProjectDir = FPaths::Combine(OutputPath, FApp::GetProjectName()) + TEXT("/");
UE::Cook::FCookSandboxConvertCookedPathToPackageNameContext Context;
Context.SandboxRootDir = SandboxRootDir;
Context.SandboxProjectDir = SandboxProjectDir;
SandboxFile.FillContext(Context);
for (FName PackageName : PackageNames)
{
FString CookedFileName = SandboxFile.ConvertPackageNameToCookedPath(WriteToString<256>(PackageName), Context);
if (CookedFileName.IsEmpty())
{
continue;
}
FStringView ParentDir;
FStringView BaseName;
FStringView Extension;
FPathViews::Split(CookedFileName, ParentDir, BaseName, Extension);
IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();
TArray<FString, TInlineAllocator<3>> FilesToRemove;
PlatformFile.IterateDirectory(*WriteToString<1024>(ParentDir),
[BaseName, ParentDir, &FilesToRemove](const TCHAR* FoundFullPath, bool bDirectory)
{
FStringView FoundParentDir;
FStringView FoundBaseName;
FStringView FoundExtension;
FPathViews::Split(FoundFullPath, FoundParentDir, FoundBaseName, FoundExtension);
if (FoundBaseName == BaseName)
{
if (FoundParentDir.IsEmpty())
{
FilesToRemove.Add(FPaths::ConvertRelativePathToFull(FString(ParentDir), FString(FoundFullPath)));
}
else
{
FilesToRemove.Add(FString(FoundFullPath));
}
}
return true;
});
for (const FString& FileName : FilesToRemove)
{
PlatformFile.DeleteFile(*FileName);
}
}
}
EPackageExtension FLooseCookedPackageWriter::BulkDataTypeToExtension(FBulkDataInfo::EType BulkDataType)
{
switch (BulkDataType)
{
case FBulkDataInfo::AppendToExports:
return EPackageExtension::Exports;
case FBulkDataInfo::BulkSegment:
return EPackageExtension::BulkDataDefault;
case FBulkDataInfo::Mmap:
return EPackageExtension::BulkDataMemoryMapped;
case FBulkDataInfo::Optional:
return EPackageExtension::BulkDataOptional;
default:
checkNoEntry();
return EPackageExtension::Unspecified;
}
}