440 lines
15 KiB
C++
440 lines
15 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "CookDiagnostics.h"
|
|
|
|
#include "AssetRegistry/IAssetRegistry.h"
|
|
#include "Containers/Array.h"
|
|
#include "Containers/RingBuffer.h"
|
|
#include "Cooker/CookGenerationHelper.h"
|
|
#include "Cooker/CookPackageData.h"
|
|
#include "Cooker/CookRequestCluster.h"
|
|
#include "Cooker/CookTypes.h"
|
|
#include "CookOnTheSide/CookLog.h"
|
|
#include "CookOnTheSide/CookOnTheFlyServer.h"
|
|
#include "Logging/LogMacros.h"
|
|
#include "Misc/Optional.h"
|
|
#include "Misc/StringBuilder.h"
|
|
#include "Serialization/ArchiveSerializedPropertyChain.h"
|
|
#include "UObject/ICookInfo.h"
|
|
#include "UObject/LinkerLoad.h"
|
|
#include "UObject/Package.h"
|
|
#include "UObject/SoftObjectPath.h"
|
|
#include "UObject/UObjectGlobals.h"
|
|
|
|
class ITargetPlatform;
|
|
|
|
DEFINE_LOG_CATEGORY_STATIC(LogHiddenDependencies, Log, All);
|
|
|
|
namespace UE::Cook
|
|
{
|
|
|
|
struct FPackageReferenceAndPropertyChain
|
|
{
|
|
FName PackageName;
|
|
TArray<FProperty*, TInlineAllocator<1>> Properties;
|
|
};
|
|
/**
|
|
* From a passed in export, collects referenced exports and imports, and stores the property chain
|
|
* that references each import.
|
|
*/
|
|
class FWhyImportedCollector : public FArchiveUObject
|
|
{
|
|
public:
|
|
explicit FWhyImportedCollector(UPackage* InRootPackage)
|
|
: RootPackage(InRootPackage)
|
|
{
|
|
ArIsObjectReferenceCollector = true;
|
|
SetIsSaving(true);
|
|
SetIsPersistent(true);
|
|
}
|
|
|
|
void Reset()
|
|
{
|
|
Exports.Reset();
|
|
Imports.Reset();
|
|
}
|
|
virtual FArchive& operator<<(UObject*& Obj) override
|
|
{
|
|
if (Obj)
|
|
{
|
|
UPackage* Package = Obj->GetPackage();
|
|
if (Package == RootPackage)
|
|
{
|
|
Exports.Add(Obj);
|
|
}
|
|
else
|
|
{
|
|
Imports.Add(FPackageReferenceAndPropertyChain{ Package->GetFName(), ConvertSerializedPropertyChain() });
|
|
}
|
|
}
|
|
return *this;
|
|
}
|
|
virtual FArchive& operator<<(FSoftObjectPath& Value) override
|
|
{
|
|
FName Package = Value.GetLongPackageFName();
|
|
if (Package != RootPackage->GetFName() && !Package.IsNone())
|
|
{
|
|
Imports.Add(FPackageReferenceAndPropertyChain{ Package, ConvertSerializedPropertyChain() });
|
|
}
|
|
return *this;
|
|
}
|
|
|
|
TArray<FProperty*, TInlineAllocator<1>> ConvertSerializedPropertyChain()
|
|
{
|
|
TArray<FProperty*, TInlineAllocator<1>> Result;
|
|
const FArchiveSerializedPropertyChain* Chain = GetSerializedPropertyChain();
|
|
if (Chain && Chain->GetNumProperties())
|
|
{
|
|
Result.Reserve(Chain->GetNumProperties());
|
|
for (int32 Index = 0; Index < Chain->GetNumProperties(); ++Index)
|
|
{
|
|
Result.Add(Chain->GetPropertyFromRoot(Index));
|
|
}
|
|
}
|
|
return Result;
|
|
}
|
|
|
|
public:
|
|
UPackage* RootPackage;
|
|
TArray<UObject*> Exports;
|
|
TArray<FPackageReferenceAndPropertyChain> Imports;
|
|
};
|
|
|
|
void FDiagnostics::AnalyzeHiddenDependencies(UCookOnTheFlyServer& COTFS, FPackageData& PackageData,
|
|
TMap<FPackageData*, EInstigator>* DiscoveredDependencies, TMap<FPackageData*, EInstigator>& SaveReferences,
|
|
TConstArrayView<const ITargetPlatform*> ReachablePlatforms, bool bOnlyEditorOnlyDebug,
|
|
bool bHiddenDependenciesDebug)
|
|
{
|
|
TOptional<TArray<FName>> ExpectedDependencies;
|
|
TOptional<TMap<FName, TMap<UObject*, TArray<FProperty*, TInlineAllocator<1>>>>> ExportsUsingPackageName;
|
|
bool bHasResaved = false;
|
|
FMultiPackageReaderResults Resave;
|
|
UPackage* Package = PackageData.GetPackage();
|
|
check(Package); // We are called from SavePackage so the PackageData still has a pointer to it
|
|
|
|
// Unsolicited packages are added to the cook when not using OnlyEditorOnly.
|
|
// When using OnlyEditorOnly, only packages referenced by the cooked SavePackage are added.
|
|
// For the period where we are transitioning the engine to always run OnlyEditorOnly, there will be
|
|
// some false negatives: OnlyEditorOnly will fail to add a package that is necessary.
|
|
// This function investigates each unsolicited package that was not added and writes information about
|
|
// why it was not added. Each of these needs to be investigated, and either fixed somehow so that it is
|
|
// referenced from the cooked SavePackage, or marked as an expected correct difference using FCookLoadScope.
|
|
if (!DiscoveredDependencies)
|
|
{
|
|
return;
|
|
}
|
|
for (const TPair<FPackageData*, EInstigator>& UnsolicitedPair : *DiscoveredDependencies)
|
|
{
|
|
const FPackageData* Unsolicited = UnsolicitedPair.Key;
|
|
EInstigator Instigator = UnsolicitedPair.Value;
|
|
if (!bHiddenDependenciesDebug && SaveReferences.Contains(Unsolicited))
|
|
{
|
|
// This package is included by OnlyEditorOnly as well. Remove it from our list to investigate.
|
|
continue;
|
|
}
|
|
if (!bOnlyEditorOnlyDebug && Instigator != EInstigator::Unsolicited)
|
|
{
|
|
// Dependencies are only hidden if they're detected as Unsolicited; other instigator types
|
|
// are reported only for comparing SkipOnlyEditorOnly to legacy WhatGetsCooked rules.
|
|
continue;
|
|
}
|
|
FName UnsolicitedPackageName = Unsolicited->GetPackageName();
|
|
|
|
// If the package is not cookable on any of the platforms we are cooking, then even though it was loaded
|
|
// it will not be cooked by LegacyWhatShouldBeCooked. Remove it from our list to investigate.
|
|
UPackage* UnsolicitedUPackage = FindPackage(nullptr, *WriteToString<256>(UnsolicitedPackageName));
|
|
bool bSuppressedOnAllPlatforms = true;
|
|
for (const ITargetPlatform* ReachablePlatform : ReachablePlatforms)
|
|
{
|
|
if (UnsolicitedUPackage && FCoreUObjectDelegates::ShouldCookPackageForPlatform.IsBound())
|
|
{
|
|
if (!FCoreUObjectDelegates::ShouldCookPackageForPlatform.Execute(UnsolicitedUPackage, ReachablePlatform))
|
|
{
|
|
continue;
|
|
}
|
|
}
|
|
bool bCookable;
|
|
bool bExplorable;
|
|
ESuppressCookReason Reason;
|
|
FRequestCluster::IsRequestCookable(ReachablePlatform, *Unsolicited, COTFS, Reason, bCookable, bExplorable);
|
|
if (!bCookable)
|
|
{
|
|
continue;
|
|
}
|
|
bSuppressedOnAllPlatforms = false;
|
|
break;
|
|
}
|
|
if (bSuppressedOnAllPlatforms)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (PackageData.GetGenerationHelper())
|
|
{
|
|
// TODO: Collect SaveReferences for all generated packages and collect Unsolicited from all externalactor packages
|
|
// and run the missing solicited test when the GeneratorPackage finishes saving all generated
|
|
continue;
|
|
}
|
|
|
|
// If the referenced package was one of the declared dependencies of the source package, then it is not unsolicited.
|
|
// We only need to log unsolicited dependencies. Declared dependencies are either added already, or they were editoronly
|
|
// and were intentionally not added to the cook. They would still be cooked by LegacyWhatShouldBeCooked, and that is a difference,
|
|
// but we will trust the system that declared them as editoronly and therefore not log them as needs investigation.
|
|
if (!ExpectedDependencies)
|
|
{
|
|
ExpectedDependencies.Emplace();
|
|
COTFS.AssetRegistry->GetDependencies(PackageData.GetPackageName(), *ExpectedDependencies,
|
|
UE::AssetRegistry::EDependencyCategory::Package, UE::AssetRegistry::EDependencyQuery::NoRequirements);
|
|
}
|
|
if (ExpectedDependencies->Contains(Unsolicited->GetPackageName()))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
// Maybe the unsolicited package was a used by a previous version the source package, but postload steps
|
|
// removed it. Or maybe it was previously undeclared as a dependency but new code in savepackage declares it.
|
|
// Try resaving the source package, and if the unsolicited package was an import but is no longer, or it is
|
|
// now an editor-only import, then it would no longer be unsolicited at head version and it is correct that
|
|
// OnlyEditorOnly removes it.
|
|
FLinkerLoad* LinkerLoad = Package->GetLinker();
|
|
FName ClassNameOfUPackage = UPackage::StaticClass()->GetFName();
|
|
FName ClassPackageOfUPackage = UPackage::StaticClass()->GetOuter()->GetFName();
|
|
bool bIsImportOfOriginalPackageLoad = false;
|
|
if (LinkerLoad)
|
|
{
|
|
for (FObjectImport& Import : LinkerLoad->ImportMap)
|
|
{
|
|
if (Import.ClassName == ClassNameOfUPackage && Import.ClassPackage == ClassPackageOfUPackage &&
|
|
Import.OuterIndex.IsNull() && Import.ObjectName == UnsolicitedPackageName)
|
|
{
|
|
bIsImportOfOriginalPackageLoad = true;
|
|
break;
|
|
}
|
|
}
|
|
for (FName SoftPackageReference : LinkerLoad->SoftPackageReferenceList)
|
|
{
|
|
if (SoftPackageReference == UnsolicitedPackageName)
|
|
{
|
|
bIsImportOfOriginalPackageLoad = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!bHasResaved)
|
|
{
|
|
bHasResaved = true;
|
|
FSavePackageArgs SaveArgs;
|
|
SaveArgs.TopLevelFlags = RF_Standalone;
|
|
SaveArgs.SaveFlags = SAVE_NoError // Do not crash the SaveServer on an error
|
|
| SAVE_BulkDataByReference // EditorDomain saves reference bulkdata from the WorkspaceDomain rather than duplicating it
|
|
| SAVE_Async // SavePackage support for PackageWriter is only implemented with SAVE_Async
|
|
;
|
|
SaveArgs.bSlowTask = false;
|
|
Resave = GetSaveExportsAndImports(Package, nullptr, SaveArgs);
|
|
}
|
|
|
|
if (Resave.Realms[0].bValid)
|
|
{
|
|
FPackageReaderResults& Realm = Resave.Realms[0];
|
|
FSoftObjectPath PackagePath(FTopLevelAssetPath(UnsolicitedPackageName, NAME_None), FString());
|
|
FSoftObjectPath PackageClassPath(UPackage::StaticClass());
|
|
bool bIsImportOfResavedPackage = false;
|
|
bool bIsEditorOnlyImportOfResavedPackage = false;
|
|
for (TPair<FSoftObjectPath, FPackageReader::FObjectData>& Pair : Realm.Imports)
|
|
{
|
|
if (Pair.Value.ClassPath == PackageClassPath && Pair.Key == PackagePath)
|
|
{
|
|
bIsImportOfResavedPackage = true;
|
|
bIsEditorOnlyImportOfResavedPackage = Pair.Value.bUsedInGame;
|
|
break;
|
|
}
|
|
}
|
|
for (TPair<FName, bool>& Pair : Realm.SoftPackageReferences)
|
|
{
|
|
if (Pair.Key == UnsolicitedPackageName)
|
|
{
|
|
bIsImportOfResavedPackage = true;
|
|
bIsEditorOnlyImportOfResavedPackage = !Pair.Value;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (bIsImportOfOriginalPackageLoad && !bIsImportOfResavedPackage)
|
|
{
|
|
// An import that was removed by upgrade steps; ignore it
|
|
continue;
|
|
}
|
|
|
|
if (bIsEditorOnlyImportOfResavedPackage)
|
|
{
|
|
// An import that the new save code identifies as editoronly; ignore it
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// No more filtering, we're going to log this as an unexpected difference.
|
|
// Try to find out how it was loaded so we can give information about what code needs to
|
|
// be modified to support OnlyEditorOnly.
|
|
|
|
if (bOnlyEditorOnlyDebug)
|
|
{
|
|
// Serialize all referenced exports in the source package and record all the packages each one
|
|
// references. If any of those packages is the unsolicited package, print out the export and property
|
|
// that refers to it.
|
|
if (!ExportsUsingPackageName)
|
|
{
|
|
ExportsUsingPackageName.Emplace();
|
|
TSet<UObject*> Exports;
|
|
TRingBuffer<UObject*> ExportsQueue;
|
|
ForEachObjectWithPackage(Package, [&Exports, &ExportsQueue](UObject* Object)
|
|
{
|
|
if (Object->HasAnyFlags(RF_Public))
|
|
{
|
|
bool bAlreadyExists;
|
|
Exports.Add(Object, &bAlreadyExists);
|
|
if (!bAlreadyExists)
|
|
{
|
|
ExportsQueue.Add(Object);
|
|
}
|
|
}
|
|
return true;
|
|
});
|
|
FWhyImportedCollector Collector(Package);
|
|
while (!ExportsQueue.IsEmpty())
|
|
{
|
|
UObject* Export = ExportsQueue.PopFrontValue();
|
|
Export->Serialize(Collector);
|
|
for (UObject* Dependency : Collector.Exports)
|
|
{
|
|
bool bAlreadyExists;
|
|
Exports.Add(Dependency, &bAlreadyExists);
|
|
if (!bAlreadyExists)
|
|
{
|
|
ExportsQueue.Add(Dependency);
|
|
}
|
|
}
|
|
for (FPackageReferenceAndPropertyChain& ImportChain : Collector.Imports)
|
|
{
|
|
TArray<FProperty*, TInlineAllocator<1>>& ExistingChain = ExportsUsingPackageName->FindOrAdd(ImportChain.PackageName).FindOrAdd(Export);
|
|
if (ExistingChain.IsEmpty())
|
|
{
|
|
ExistingChain = MoveTemp(ImportChain.Properties);
|
|
}
|
|
}
|
|
Collector.Reset();
|
|
}
|
|
}
|
|
TMap<UObject*, TArray<FProperty*, TInlineAllocator<1>>>* ExportsUsingUnsolicited = ExportsUsingPackageName->Find(UnsolicitedPackageName);
|
|
TStringBuilder<256> WhyReferenced;
|
|
|
|
int32 PackageNameLen = Package->GetName().Len();
|
|
if (ExportsUsingUnsolicited)
|
|
{
|
|
int32 Count = 0;
|
|
constexpr int32 MaxCount = 3;
|
|
WhyReferenced << TEXT("\n\tReferencers:");
|
|
for (TPair<UObject*, TArray<FProperty*, TInlineAllocator<1>>>&Pair : *ExportsUsingUnsolicited)
|
|
{
|
|
UObject* Export = Pair.Key;
|
|
WhyReferenced << TEXT("\n\t\t");
|
|
if (Count++ == MaxCount)
|
|
{
|
|
WhyReferenced << TEXT("...");
|
|
break;
|
|
}
|
|
WhyReferenced << Export->GetClass()->GetPathName() << TEXT(" ");
|
|
WhyReferenced << Export->GetPathName().RightChop(PackageNameLen + 1);
|
|
WhyReferenced << TEXT("#");
|
|
if (Pair.Value.IsEmpty())
|
|
{
|
|
WhyReferenced << TEXT("<UnknownProperty>");
|
|
}
|
|
else
|
|
{
|
|
int32 NumProperties = Pair.Value.Num();
|
|
bool bAddedSeparator = false;
|
|
for (int32 PropertyIndex = 0; PropertyIndex < NumProperties; ++PropertyIndex)
|
|
{
|
|
FProperty* Property = Pair.Value[PropertyIndex];
|
|
if (PropertyIndex < NumProperties - 1 && Property->GetFName() == Pair.Value[PropertyIndex + 1]->GetFName() &&
|
|
Property->IsA(FArrayProperty::StaticClass()))
|
|
{
|
|
// Omit adding a duplicate entry for inner property of an array which has the same name as its array property
|
|
continue;
|
|
}
|
|
WhyReferenced << Property->GetName() << TEXT("#");
|
|
bAddedSeparator = true;
|
|
}
|
|
if (bAddedSeparator)
|
|
{
|
|
WhyReferenced.RemoveSuffix(1);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// If we didn't find any exports referring to the unsolicited package, then just print out the list of class types
|
|
// in the package to show which code needs to be investigated.
|
|
enum class EClassPriority
|
|
{
|
|
PrimaryUAsset,
|
|
Asset,
|
|
Public,
|
|
Private,
|
|
Count,
|
|
};
|
|
TMap<UClass*, EClassPriority> ObjectClasses;
|
|
FName PackageLeafName = FName(*FPaths::GetBaseFilename(Package->GetName(), true /* bRemovePath */));
|
|
ForEachObjectWithPackage(Package, [&ObjectClasses, PackageLeafName](UObject* Object)
|
|
{
|
|
UClass* Class = Object->GetClass();
|
|
EClassPriority NewPriority = EClassPriority::Private;
|
|
if (Object->IsAsset())
|
|
{
|
|
NewPriority = Object->GetFName() == PackageLeafName ? EClassPriority::PrimaryUAsset : EClassPriority::Asset;
|
|
}
|
|
else
|
|
{
|
|
NewPriority = Object->HasAnyFlags(RF_Public) ? EClassPriority::Public : EClassPriority::Private;
|
|
}
|
|
EClassPriority& OldPriority = ObjectClasses.FindOrAdd(Class, NewPriority);
|
|
OldPriority = static_cast<EClassPriority>(FMath::Min(static_cast<int32>(OldPriority), static_cast<int32>(NewPriority)));
|
|
return true;
|
|
});
|
|
WhyReferenced << TEXT("\n\tNo exports found referencing the dependency. Classes in the referencer package:");
|
|
int32 ClassCount = 0;
|
|
constexpr int32 MaxClassCount = 10;
|
|
ObjectClasses.ValueSort([](EClassPriority A, EClassPriority B)
|
|
{
|
|
if (A != B)
|
|
{
|
|
return (int32)A < (int32)B;
|
|
}
|
|
return false;
|
|
});
|
|
for (TPair<UClass*, EClassPriority>& ClassPair : ObjectClasses)
|
|
{
|
|
if (ClassCount++ >= MaxClassCount)
|
|
{
|
|
break;
|
|
}
|
|
WhyReferenced << TEXT("\n\t\t") << ClassPair.Key->GetPathName();
|
|
}
|
|
}
|
|
|
|
UE_LOG(LogHiddenDependencies, Display, TEXT("Skipped adding %s -> (%s) %s%s"),
|
|
*WriteToString<256>(PackageData.GetPackageName()), LexToString(Instigator),
|
|
*WriteToString<256>(Unsolicited->GetPackageName()), *WhyReferenced);
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogHiddenDependencies, Display, TEXT("HiddenDependency %s -> (%s) %s"),
|
|
*WriteToString<256>(PackageData.GetPackageName()), LexToString(Instigator),
|
|
*WriteToString<256>(Unsolicited->GetPackageName()));
|
|
}
|
|
}
|
|
}
|
|
|
|
} |