// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using EpicGames.Core;
using Microsoft.Extensions.Logging;
using OpenTracing.Util;
using UnrealBuildBase;
using UnrealBuildTool.Artifacts;
namespace UnrealBuildTool
{
///
/// Options controlling how a target is built
///
[Flags]
enum BuildOptions
{
///
/// Default options
///
None = 0,
///
/// Don't build anything, just do target setup and terminate
///
SkipBuild = 1,
///
/// Just output a list of XGE actions; don't build anything
///
XGEExport = 2,
///
/// Fail if any engine files would be modified by the build
///
NoEngineChanges = 4,
}
///
/// Builds a target
///
[ToolMode("Build", ToolModeOptions.XmlConfig | ToolModeOptions.BuildPlatforms | ToolModeOptions.SingleInstance | ToolModeOptions.StartPrefetchingEngine | ToolModeOptions.ShowExecutionTime | ToolModeOptions.UseStartupTraceListener)]
class BuildMode : ToolMode
{
///
/// Specifies the file to use for logging.
///
[XmlConfigFile(Category = "BuildConfiguration")]
public string? BaseLogFileName;
///
/// Whether to skip checking for files identified by the junk manifest.
///
[XmlConfigFile]
[CommandLine("-IgnoreJunk")]
public bool bIgnoreJunk = false;
///
/// Skip building; just do setup and terminate.
///
[CommandLine("-SkipBuild")]
public bool bSkipBuild = false;
///
/// Skip pre build targets; just do the main target.
///
[CommandLine("-SkipPreBuildTargets")]
public bool bSkipPreBuildTargets = false;
///
/// Whether we should just export the XGE XML and pretend it succeeded
///
[CommandLine("-XGEExport")]
public bool bXGEExport = false;
///
/// Do not allow any engine files to be output (used by compile on startup functionality)
///
[CommandLine("-NoEngineChanges")]
public bool bNoEngineChanges = false;
///
/// Whether we should just export the outdated actions list
///
[CommandLine("-WriteOutdatedActions=")]
public FileReference? WriteOutdatedActionsFile = null;
///
/// An optional directory to copy crash dump files into
///
[CommandLine("-SaveCrashDumps=")]
public DirectoryReference? SaveCrashDumpDirectory = null;
///
/// If specified, then only this type of action will execute
///
[CommandLine("-ActionTypeFilter=")]
public string? ActionTypeFilter = null;
///
/// Main entry point
///
/// Command-line arguments
/// One of the values of ECompilationResult
///
public override async Task ExecuteAsync(CommandLineArguments Arguments, ILogger Logger)
{
Arguments.ApplyTo(this);
// Write the command line
Logger.LogDebug("Command line: {EnvironmentCommandLine}", Environment.CommandLine);
// Grab the environment.
UnrealBuildTool.InitialEnvironment = Environment.GetEnvironmentVariables();
if (UnrealBuildTool.InitialEnvironment.Count < 1)
{
throw new BuildException("Environment could not be read");
}
// Read the XML configuration files
XmlConfig.ApplyTo(this);
// Apply to architecture configs that need to read commandline arguments and didn't have the Arguments passed in during construction
foreach (UnrealArchitectureConfig Config in UnrealArchitectureConfig.AllConfigs())
{
Arguments.ApplyTo(Config);
}
// Fixup the log path if it wasn't overridden by a config file
BaseLogFileName ??= FileReference.Combine(Unreal.EngineProgramSavedDirectory, "UnrealBuildTool", "Log.txt").FullName;
// Create the log file, and flush the startup listener to it
if (!Arguments.HasOption("-NoLog") && !Log.HasFileWriter())
{
Log.AddFileWriter("DefaultLogTraceListener", FileReference.FromString(BaseLogFileName));
}
else
{
Log.RemoveStartupTraceListener();
}
// Create the build configuration object, and read the settings
BuildConfiguration BuildConfiguration = new BuildConfiguration();
XmlConfig.ApplyTo(BuildConfiguration);
Arguments.ApplyTo(BuildConfiguration);
// Check the root path length isn't too long
if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Win64 && Unreal.RootDirectory.FullName.Length > BuildConfiguration.MaxRootPathLength)
{
Logger.LogWarning("Running from a path with a long directory name (\"{Path}\" = {NumChars} characters). Root paths shorter than {MaxChars} characters are recommended to avoid exceeding maximum path lengths on Windows.", Unreal.RootDirectory, Unreal.RootDirectory.FullName.Length, BuildConfiguration.MaxRootPathLength);
}
// now that we know the available platforms, we can delete other platforms' junk. if we're only building specific modules from the editor, don't touch anything else (it may be in use).
if (!bIgnoreJunk && !Unreal.IsEngineInstalled())
{
using (GlobalTracer.Instance.BuildSpan("DeleteJunk()").StartActive())
{
JunkDeleter.DeleteJunk(Logger);
}
}
// Parse and build the targets
try
{
List TargetDescriptors;
// Parse all the target descriptors
using (GlobalTracer.Instance.BuildSpan("TargetDescriptor.ParseCommandLine()").StartActive())
{
TargetDescriptors = TargetDescriptor.ParseCommandLine(Arguments, BuildConfiguration, Logger);
// Handle BuildConfiguration arguments nested inside -Target= args
TargetDescriptors.ForEach(x => x.AdditionalArguments.ApplyTo(BuildConfiguration));
}
// Hack for specific files compile; don't build the ShaderCompileWorker target that's added to the command line for generated project files
if (TargetDescriptors.Count > 1)
{
TargetDescriptors.RemoveAll(x => (x.Name == "ShaderCompileWorker" || x.Name == "LiveCodingConsole" || x.Name == "InterchangeWorker") && x.SpecificFilesToCompile.Count > 0);
}
// Clean any target that wanted to be cleaned before being rebuilt
if (TargetDescriptors.Any(D => D.bRebuild))
{
CleanMode CleanMode = new CleanMode();
CleanMode.bSkipPreBuildTargets = bSkipPreBuildTargets;
CleanMode.Clean(TargetDescriptors.Where(D => D.bRebuild).ToList(), BuildConfiguration, Logger);
}
// Handle remote builds
for (int Idx = 0; Idx < TargetDescriptors.Count; ++Idx)
{
TargetDescriptor TargetDesc = TargetDescriptors[Idx];
if (RemoteMac.HandlesTargetPlatform(TargetDesc.Platform))
{
FileReference BaseLogFile = Log.OutputFile ?? new FileReference(BaseLogFileName);
FileReference RemoteLogFile = FileReference.Combine(BaseLogFile.Directory, BaseLogFile.GetFileNameWithoutExtension() + "_Remote.txt");
int RemoteIniIndex = TargetDesc.AdditionalArguments.FindIndex(x => x.ToLower().StartsWith("-remoteini="));
string? RemoteIni = RemoteIniIndex == -1 ? null : new string(TargetDesc.AdditionalArguments[RemoteIniIndex].Substring(11));
RemoteMac RemoteMac = new RemoteMac(TargetDesc.ProjectFile, Logger, RemoteIni);
if (!RemoteMac.Build(TargetDesc, RemoteLogFile, bSkipPreBuildTargets, Logger))
{
return (int)CompilationResult.Unknown;
}
TargetDescriptors.RemoveAt(Idx--);
}
}
// Handle local builds
if (TargetDescriptors.Count > 0)
{
// Get a set of all the project directories
HashSet ProjectDirs = new HashSet();
foreach (TargetDescriptor TargetDesc in TargetDescriptors)
{
if (TargetDesc.ProjectFile != null)
{
DirectoryReference ProjectDirectory = TargetDesc.ProjectFile.Directory;
FileMetadataPrefetch.QueueProjectDirectory(ProjectDirectory);
ProjectDirs.Add(ProjectDirectory);
}
// print out SDK info for only the platforms that are being compiled
UEBuildPlatformSDK.GetSDKForPlatform(TargetDesc.Platform.ToString())?.PrintSDKInfoAndReturnValidity();
}
// Get all the build options
BuildOptions Options = BuildOptions.None;
if (bSkipBuild)
{
Options |= BuildOptions.SkipBuild;
}
if (bXGEExport)
{
Options |= BuildOptions.XGEExport;
}
if (bNoEngineChanges)
{
Options |= BuildOptions.NoEngineChanges;
}
// Create the working set provider per group.
using (ISourceFileWorkingSet WorkingSet = SourceFileWorkingSet.Create(Unreal.RootDirectory, ProjectDirs, Logger))
{
await BuildAsync(TargetDescriptors, BuildConfiguration, WorkingSet, Options, WriteOutdatedActionsFile, Logger, bSkipPreBuildTargets, ActionTypeFilter:ActionTypeFilter);
}
}
}
finally
{
// Check if anything failed during our run, and act accordingly.
ProcessCoreDumps(SaveCrashDumpDirectory, Logger);
// Save all the caches
using (Timeline.ScopeEvent("Saving caches"))
{
SourceFileMetadataCache.SaveAll();
CppDependencyCache.SaveAll();
}
}
return 0;
}
///
/// Creates scripts for executing the init scripts (only supported on project descriptors)
///
/// The current target
/// List of created script files
public static FileReference[] CreateInitScripts(TargetDescriptor TargetDescriptor)
{
if (TargetDescriptor.ProjectFile == null)
{
return Array.Empty();
}
ProjectDescriptor ProjectDescriptor = ProjectDescriptor.FromFile(TargetDescriptor.ProjectFile);
List InitCommandBatches = new List();
if (ProjectDescriptor != null && ProjectDescriptor.InitSteps != null)
{
if (ProjectDescriptor.InitSteps.TryGetCommands(BuildHostPlatform.Current.Platform, out string[]? Commands))
{
InitCommandBatches.Add(Commands);
}
}
if (InitCommandBatches.Count == 0)
{
return Array.Empty();
}
DirectoryReference ProjectDirectory = DirectoryReference.FromFile(TargetDescriptor.ProjectFile);
string PlatformIntermediateFolder = UEBuildTarget.GetPlatformIntermediateFolder(TargetDescriptor.Platform, TargetDescriptor.Architectures, false);
DirectoryReference ProjectIntermediateDirectory = DirectoryReference.Combine(ProjectDirectory, PlatformIntermediateFolder, TargetDescriptor.Name, TargetDescriptor.Configuration.ToString());
return WriteInitScripts(TargetDescriptor, BuildHostPlatform.Current.Platform, ProjectIntermediateDirectory, "Init", InitCommandBatches);
}
///
/// Write scripts containing the custom build steps for the given host platform
///
/// The current target
/// The current host platform
/// The output directory for the scripts
/// Bare prefix for all the created script files
/// List of custom build steps
/// List of created script files
private static FileReference[] WriteInitScripts(TargetDescriptor TargetDescriptor, UnrealTargetPlatform HostPlatform, DirectoryReference Directory, string FilePrefix, List CommandBatches)
{
List ScriptFiles = new List();
foreach (string[] CommandBatch in CommandBatches)
{
// Find all the standard variables
Dictionary Variables = GetTargetVariables(TargetDescriptor);
// Get the output path to the script
string ScriptExtension = OperatingSystem.IsWindows() ? ".bat" : ".sh";
FileReference ScriptFile = FileReference.Combine(Directory, String.Format("{0}-{1}{2}", FilePrefix, ScriptFiles.Count + 1, ScriptExtension));
// Write it to disk
List Contents = new List();
if (OperatingSystem.IsWindows())
{
Contents.Insert(0, "@echo off");
}
foreach (string Command in CommandBatch)
{
Contents.Add(Utils.ExpandVariables(Command, Variables));
}
if (!DirectoryReference.Exists(ScriptFile.Directory))
{
DirectoryReference.CreateDirectory(ScriptFile.Directory);
}
FileReference.WriteAllLines(ScriptFile, Contents);
// Add the output file to the list of generated scripts
ScriptFiles.Add(ScriptFile);
}
return ScriptFiles.ToArray();
}
///
/// Gets a list of variables that can be expanded in paths referenced by this target
///
/// The current target
/// Map of variable names to values
private static Dictionary GetTargetVariables(TargetDescriptor TargetDescriptor)
{
Dictionary Variables = new Dictionary();
Variables.Add("RootDir", Unreal.RootDirectory.FullName);
Variables.Add("EngineDir", Unreal.EngineDirectory.FullName);
Variables.Add("TargetName", TargetDescriptor.Name);
Variables.Add("TargetPlatform", TargetDescriptor.Platform.ToString());
Variables.Add("TargetConfiguration", TargetDescriptor.Configuration.ToString());
if (TargetDescriptor.ProjectFile != null)
{
Variables.Add("ProjectDir", TargetDescriptor.ProjectFile.Directory.FullName);
Variables.Add("ProjectFile", TargetDescriptor.ProjectFile.FullName);
}
if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out BuildVersion? Version))
{
Variables.Add("EngineVersion", String.Format("{0}.{1}.{2}", Version.MajorVersion, Version.MinorVersion, Version.PatchVersion));
}
return Variables;
}
///
/// Build a list of targets
///
/// Target descriptors
/// Current build configuration
/// The source file working set
/// Additional options for the build
/// Files to write the list of outdated actions to (rather than building them)
/// Logger for output
/// If true then only the current target descriptors will be built.
/// If specified, only actions of this type will be run
/// Result from the compilation
public static async Task BuildAsync(List TargetDescriptors, BuildConfiguration BuildConfiguration, ISourceFileWorkingSet WorkingSet, BuildOptions Options, FileReference? WriteOutdatedActionsFile, ILogger Logger, bool bSkipPreBuildTargets = false, string? ActionTypeFilter = null)
{
List TargetMakefiles = new List();
for (int Idx = 0; Idx < TargetDescriptors.Count; ++Idx)
{
using ITimelineEvent SetupScope = Timeline.ScopeEvent($"Creating MakeFile for {TargetDescriptors[Idx].Name}");
// Create and execute the init scripts
FileReference[] InitScripts = CreateInitScripts(TargetDescriptors[Idx]);
if (InitScripts.Length > 0)
{
Utils.ExecuteCustomBuildSteps(InitScripts, Logger);
}
TargetMakefile NewMakefile = await CreateMakefileAsync(BuildConfiguration, TargetDescriptors[Idx], WorkingSet, Logger);
TargetMakefiles.Add(NewMakefile);
if (!bSkipPreBuildTargets)
{
foreach (TargetInfo PreBuildTarget in NewMakefile.PreBuildTargets)
{
TargetDescriptor NewTarget = TargetDescriptor.FromTargetInfo(PreBuildTarget);
if (!TargetDescriptors.Contains(NewTarget))
{
TargetDescriptors.Add(NewTarget);
}
}
}
}
await BuildAsync(TargetMakefiles.ToArray(), TargetDescriptors, BuildConfiguration, Options, WriteOutdatedActionsFile, Logger, ActionTypeFilter);
}
static Lazy s_memoryMappedFileCache = new Lazy(() => new MemoryMappedFileCache());
///
/// Build a list of targets with a given set of makefiles.
///
/// Makefiles created with CreateMakefiles
/// Target descriptors
/// Current build configuration
/// Additional options for the build
/// Files to write the list of outdated actions to (rather than building them)
/// Logger for output
/// If specified, only actions of this type will be run
/// Result from the compilation
internal static async Task BuildAsync(TargetMakefile[] Makefiles, List TargetDescriptors, BuildConfiguration BuildConfiguration, BuildOptions Options, FileReference? WriteOutdatedActionsFile, ILogger Logger, string? ActionTypeFilter=null)
{
// Execute the build
if ((Options & BuildOptions.SkipBuild) == 0)
{
using ITimelineEvent ActionGraphTimer = Timeline.ScopeEvent($"Preparing ActionGraph");
// Make sure that none of the actions conflict with any other (producing output files differently, etc...)
using (GlobalTracer.Instance.BuildSpan("ActionGraph.CheckForConflicts").StartActive())
{
// TODO: Skipping conflicts for WriteMetadata is a hack to maintain parity with the BuildGraph executor which allowed
// exporting mulitple conflicing WriteMetadata actions.
// This is technically buggy behavior and should be properly fixed in UEBuildTarget.cs
IEnumerable CheckActions = Makefiles.SelectMany(x => x.Actions).Where(x => !x.IgnoreConflicts());
ActionGraph.CheckForConflicts(CheckActions, Logger);
}
// Check we don't exceed the nominal max path length
using (GlobalTracer.Instance.BuildSpan("ActionGraph.CheckPathLengths").StartActive())
{
ActionGraph.CheckPathLengths(BuildConfiguration, Makefiles.SelectMany(x => x.Actions), Logger);
}
HashSet MergedOutputItems = new HashSet();
// Create a QueuedAction instance for each action in the makefiles
List[] QueuedActions = new List[Makefiles.Length];
for (int Idx = 0; Idx < Makefiles.Length; Idx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[Idx];
TargetMakefile Makefile = Makefiles[Idx];
QueuedActions[Idx] = Makefile.Actions.ConvertAll(x => new LinkedAction(x, TargetDescriptors[Idx]));
if (TargetDescriptor.SpecificFilesToCompile.Count != 0)
{
// Filter out SpecificFilesToCompile so only source files and headers included in the target are considered
HashSet TargetFiles = Makefile.SourceAndHeaderFiles.Select(x => x.Location).ToHashSet();
List FilesToBuild = TargetDescriptor.SpecificFilesToCompile
.Where(x => TargetFiles.Contains(x))
.ToList();
// If we have specific files to build we need to put those as MergedOutputItems (and create special single file actions if needed)
if (FilesToBuild.Count != 0)
{
// If there are headers in the list, expand the FilesToBuild to also include all files that include those headers
if (TargetDescriptor.bSingleFileBuildDependents)
{
FilesToBuild = GetAllSourceFilesIncludingHeader(FilesToBuild, TargetDescriptor.ProjectFile, Makefile.Actions, Logger);
}
// We have specific files to compile so we will only queue up those files.
List ProducedItems = CreateLinkedActionsFromFileList(TargetDescriptor, BuildConfiguration, FilesToBuild, QueuedActions[Idx], Logger);
MergedOutputItems.UnionWith(ProducedItems);
}
}
}
// Clean up any previous hot reload runs, and reapply the current state if it's already active
Dictionary? InitialPatchedOldLocationToNewLocation = null;
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
InitialPatchedOldLocationToNewLocation = HotReload.Setup(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], QueuedActions[TargetIdx], BuildConfiguration, Logger);
}
// Merge the action graphs together
List MergedActions;
if (TargetDescriptors.Count == 1)
{
MergedActions = QueuedActions[0];
}
else
{
MergedActions = MergeActionGraphs(TargetDescriptors, QueuedActions);
}
// Gather all the prerequisite actions that are part of the targets
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
GatherOutputItems(TargetDescriptors[TargetIdx], Makefiles[TargetIdx], MergedOutputItems);
}
// Link all the actions together (No need sorting, we'll do that later again)
ActionGraph.Link(MergedActions, Logger, Sort: false);
// Get all the actions that are prerequisites for these targets. This forms the list of actions that we want executed.
List PrerequisiteActions = ActionGraph.GatherPrerequisiteActions(MergedActions, MergedOutputItems);
// Create the action history
ActionHistory History = new ActionHistory();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
using (GlobalTracer.Instance.BuildSpan("Reading action history").StartActive())
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
if (TargetDescriptor.ProjectFile != null)
{
History.Mount(TargetDescriptor.ProjectFile.Directory);
}
}
}
// Create the C++ dependency cache
CppDependencyCache CppDependencies = new CppDependencyCache();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
using (GlobalTracer.Instance.BuildSpan("Reading dependency cache").StartActive())
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
CppDependencies.Mount(TargetDescriptor, Makefiles[TargetIdx].TargetType, Logger);
}
}
// Now that we have the dependencies object, optionally create the artifact cache
IActionArtifactCache? actionArtifactCache = null;
if (!String.IsNullOrEmpty(BuildConfiguration.ArtifactDirectory) && (BuildConfiguration.bArtifactRead || BuildConfiguration.bArtifactWrites))
{
DirectoryReference artifactDirectory = new(BuildConfiguration.ArtifactDirectory);
DirectoryReference.CreateDirectory(artifactDirectory);
string mode = (BuildConfiguration.bArtifactRead, BuildConfiguration.bArtifactWrites) switch
{
(false, true) => "write only",
(true, false) => "read only",
_ => "read/write",
};
Logger.LogInformation("Experimental artifact system using '{Directory}' in {Mode} mode", artifactDirectory.FullName, mode);
actionArtifactCache = ActionArtifactCache.CreateHordeFileCache(artifactDirectory, CppDependencies, s_memoryMappedFileCache.Value, Logger);
actionArtifactCache.EnableReads = BuildConfiguration.bArtifactRead;
actionArtifactCache.EnableWrites = BuildConfiguration.bArtifactWrites;
actionArtifactCache.LogCacheMisses = BuildConfiguration.bLogArtifactCacheMisses;
actionArtifactCache.EngineRoot = Unreal.EngineDirectory;
}
// Pre-process module interfaces to generate dependency files
List ModuleDependencyActions = PrerequisiteActions.Where(x => x.ActionType == ActionType.GatherModuleDependencies).ToList();
if (ModuleDependencyActions.Count > 0)
{
ConcurrentDictionary PreprocessActionToOutdatedFlag = new ConcurrentDictionary();
ActionGraph.GatherAllOutdatedActions(ModuleDependencyActions, History, PreprocessActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries, Logger);
List PreprocessActions = PreprocessActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList();
if (PreprocessActions.Count > 0)
{
Logger.LogInformation("Updating module dependencies...");
await ActionGraph.ExecuteActionsAsync(BuildConfiguration, PreprocessActions, TargetDescriptors, Logger);
foreach (FileItem ProducedItem in PreprocessActions.SelectMany(x => x.ProducedItems))
{
ProducedItem.ResetCachedInfo();
}
}
}
// Figure out which actions need to be built
ConcurrentDictionary ActionToOutdatedFlag = new ConcurrentDictionary();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
TargetDescriptor TargetDescriptor = TargetDescriptors[TargetIdx];
// Update the module dependencies
Dictionary ModuleOutputs = new Dictionary(StringComparer.Ordinal);
if (ModuleDependencyActions.Count > 0)
{
Dictionary> ModuleImports = new Dictionary>();
List CompiledModuleInterfaces = new List();
foreach (LinkedAction ModuleDependencyAction in ModuleDependencyActions)
{
ICppCompileAction? CppModulesAction = ModuleDependencyAction.Inner as ICppCompileAction;
if (CppModulesAction != null && CppModulesAction.CompiledModuleInterfaceFile != null)
{
string? ProducedModule;
if (CppDependencies.TryGetProducedModule(ModuleDependencyAction.DependencyListFile!, Logger, out ProducedModule))
{
ModuleOutputs[ProducedModule] = CppModulesAction.CompiledModuleInterfaceFile;
}
List<(string Name, string BMI)>? ImportedModules;
if (CppDependencies.TryGetImportedModules(ModuleDependencyAction.DependencyListFile!, Logger, out ImportedModules))
{
ModuleImports[CppModulesAction.CompiledModuleInterfaceFile] = ImportedModules.Select(x => x.Name).ToList();
}
CompiledModuleInterfaces.Add(CppModulesAction.CompiledModuleInterfaceFile);
}
}
Dictionary ModuleInputs = ModuleOutputs.ToDictionary(x => x.Value, x => x.Key);
foreach (LinkedAction PrerequisiteAction in PrerequisiteActions)
{
if (PrerequisiteAction.ActionType == ActionType.CompileModuleInterface)
{
ICppCompileAction CppModulesAction = (ICppCompileAction)PrerequisiteAction.Inner;
List? ImportedModules;
if (ModuleImports.TryGetValue(CppModulesAction.CompiledModuleInterfaceFile!, out ImportedModules))
{
foreach (string ImportedModule in ImportedModules)
{
FileItem? ModuleOutput;
if (ModuleOutputs.TryGetValue(ImportedModule, out ModuleOutput))
{
Action NewAction = new Action(PrerequisiteAction.Inner);
NewAction.PrerequisiteItems.Add(ModuleOutput);
NewAction.CommandArguments += String.Format(" /reference \"{0}={1}\"", ImportedModule, ModuleOutput.FullName);
PrerequisiteAction.Inner = NewAction;
}
else
{
throw new BuildException("Unable to find interface for module '{0}'", ImportedModule);
}
}
}
}
else if (PrerequisiteAction.ActionType == ActionType.Compile)
{
foreach (FileItem PrerequisiteItem in PrerequisiteAction.PrerequisiteItems)
{
string? ModuleName;
if (ModuleInputs.TryGetValue(PrerequisiteItem, out ModuleName))
{
Action NewAction = new Action(PrerequisiteAction.Inner);
NewAction.CommandArguments += String.Format(" /reference \"{0}={1}\"", ModuleName, PrerequisiteItem.AbsolutePath);
PrerequisiteAction.Inner = NewAction;
}
}
}
else
{
if (PrerequisiteAction.ActionType != ActionType.GatherModuleDependencies)
{
Action NewAction = new Action(PrerequisiteAction.Inner);
NewAction.PrerequisiteItems.UnionWith(CompiledModuleInterfaces);
PrerequisiteAction.Inner = NewAction;
}
}
}
}
// Plan the actions to execute for the build. For single file compiles, always rebuild the source file regardless of whether it's out of date.
ActionGraph.GatherAllOutdatedActions(PrerequisiteActions, History, ActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries, Logger);
}
// Link the action graph again to sort it
List MergedActionsToExecute = ActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList();
// if we want to only run one type of action, filter it now
if (ActionTypeFilter != null)
{
ActionType Filter;
if (Enum.TryParse(ActionTypeFilter, out Filter))
{
MergedActionsToExecute = MergedActionsToExecute.Where(x => x.ActionType == Filter).ToList();
}
else
{
throw new BuildException($"Unknown ActionType for ActionTypeFilter: {ActionTypeFilter}");
}
}
ActionGraph.Link(MergedActionsToExecute, Logger);
// Allow hot reload to override the actions
int HotReloadTargetIdx = -1;
for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
{
if (TargetDescriptors[Idx].HotReloadMode != HotReloadMode.Disabled)
{
if (HotReloadTargetIdx != -1)
{
throw new BuildException("Unable to perform hot reload with multiple targets.");
}
else
{
MergedActionsToExecute = HotReload.PatchActionsForTarget(BuildConfiguration, TargetDescriptors[Idx], Makefiles[Idx], PrerequisiteActions, MergedActionsToExecute, InitialPatchedOldLocationToNewLocation, Logger);
}
HotReloadTargetIdx = Idx;
}
else if (MergedActionsToExecute.Count > 0)
{
HotReload.CheckForLiveCodingSessionActive(TargetDescriptors[Idx], Makefiles[Idx], BuildConfiguration, Logger);
}
}
if (HotReloadTargetIdx != -1)
{
Logger.LogDebug("Re-evaluating action graph");
// Re-check the graph to remove any LiveCoding actions added by PatchActionsForTarget() that are already up to date.
ConcurrentDictionary LiveActionToOutdatedFlag = new ConcurrentDictionary(Environment.ProcessorCount, MergedActionsToExecute.Count);
ActionGraph.GatherAllOutdatedActions(MergedActionsToExecute, History, LiveActionToOutdatedFlag, CppDependencies, BuildConfiguration.bIgnoreOutdatedImportLibraries, Logger);
List LiveCodingActionsToExecute = LiveActionToOutdatedFlag.Where(x => x.Value).Select(x => x.Key).ToList();
ActionGraph.Link(LiveCodingActionsToExecute, Logger);
MergedActionsToExecute = LiveCodingActionsToExecute;
}
// Make sure we're not modifying any engine files
if ((Options & BuildOptions.NoEngineChanges) != 0)
{
List EngineChanges = MergedActionsToExecute.SelectMany(x => x.ProducedItems).Where(x => x.Location.IsUnderDirectory(Unreal.EngineDirectory)).Distinct().OrderBy(x => x.FullName).ToList();
if (EngineChanges.Count > 0)
{
StringBuilder Result = new StringBuilder("Building would modify the following engine files:\n");
foreach (FileItem EngineChange in EngineChanges)
{
Result.AppendFormat("\n{0}", EngineChange.FullName);
}
Result.Append("\n\nPlease rebuild from an IDE instead.");
Logger.LogError("{Result}", Result.ToString());
throw new CompilationResultException(CompilationResult.FailedDueToEngineChange);
}
}
// Make sure the appropriate executor is selected
foreach (TargetDescriptor TargetDescriptor in TargetDescriptors)
{
UEBuildPlatform BuildPlatform = UEBuildPlatform.GetBuildPlatform(TargetDescriptor.Platform);
BuildConfiguration.bAllowXGE &= BuildPlatform.CanUseXGE();
BuildConfiguration.bAllowFASTBuild &= BuildPlatform.CanUseFASTBuild();
BuildConfiguration.bAllowSNDBS &= BuildPlatform.CanUseSNDBS();
}
// Delete produced items that are outdated.
ActionGraph.DeleteOutdatedProducedItems(MergedActionsToExecute, Logger);
// Save all the action histories now that files have been removed. We have to do this after deleting produced items to ensure that any
// items created during the build don't have the wrong command line.
History.Save();
// Create directories for the outdated produced items.
ActionGraph.CreateDirectoriesForProducedItems(MergedActionsToExecute);
ActionGraphTimer.Finish();
// Execute the actions
if ((Options & BuildOptions.XGEExport) != 0)
{
OutputToolchainInfo(TargetDescriptors, Makefiles, Logger);
// Just export to an XML file
using (GlobalTracer.Instance.BuildSpan("XGE.ExportActions()").StartActive())
{
using ITimelineEvent XGETimer = Timeline.ScopeEvent("XGEExport");
XGE.ExportActions(MergedActionsToExecute, Logger);
}
}
else if (WriteOutdatedActionsFile != null)
{
OutputToolchainInfo(TargetDescriptors, Makefiles, Logger);
// Write actions to an output file
using (GlobalTracer.Instance.BuildSpan("ActionGraph.WriteActions").StartActive())
{
using ITimelineEvent WriteActionsTimer = Timeline.ScopeEvent("Writing actions");
ActionGraph.ExportJson(MergedActionsToExecute, WriteOutdatedActionsFile);
Logger.LogInformation("Exported build actions to {WriteOutdatedActionsFile}", WriteOutdatedActionsFile);
}
}
else
{
// Execute the actions
if (MergedActionsToExecute.Count == 0)
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
if (TargetDescriptors.Count == 1)
{
Logger.LogInformation("Target is up to date");
}
else
{
Logger.LogInformation("Targets are up to date");
}
}
}
else
{
if (TargetDescriptors.Any(x => !x.bQuiet))
{
Logger.LogInformation("Building {Targets}...", StringUtils.FormatList(TargetDescriptors.Select(x => x.Name).Distinct()));
}
OutputToolchainInfo(TargetDescriptors, Makefiles, Logger);
ActionExecutor.SetMemoryPerActionOverride(Makefiles.Select(x => x.MemoryPerActionGB).Max());
using ITimelineEvent ExcuteTimer = Timeline.ScopeEvent("Executing ActionGraph");
using (GlobalTracer.Instance.BuildSpan("ActionGraph.ExecuteActions()").StartActive())
{
// We need to wait for the cache to be ready. If we fail to fetch from the cache for PCH files, they are
// non-deterministic and will cause rippled cache misses. Changing the system to assume a PCH file is the same
// if the inputs are the same might be a good way to avoid this issue.
if (actionArtifactCache != null && actionArtifactCache.ArtifactCache != null)
{
await actionArtifactCache.ArtifactCache.WaitForReadyAsync();
}
await ActionGraph.ExecuteActionsAsync(BuildConfiguration, MergedActionsToExecute, TargetDescriptors, Logger, actionArtifactCache);
if (actionArtifactCache != null)
{
await actionArtifactCache.FlushChangesAsync(default); //ETSTODO
}
}
// these are parsed by external tools wishing to open this file directly
foreach (LinkedAction BuildAction in MergedActionsToExecute.Where(BuildAction => BuildAction.ActionType == ActionType.Compile && (BuildAction.Inner as VCCompileAction)?.bIsAnalyzing != true))
{
FileItem? PreprocessedFile = BuildAction.ProducedItems.FirstOrDefault(x => x.HasExtension(".i"));
if (PreprocessedFile != null)
{
Logger.LogInformation("PreProcessPath: {File}", PreprocessedFile);
}
FileItem? AssemblyPath = BuildAction.ProducedItems.FirstOrDefault(x => x.HasExtension(".asm"));
if (AssemblyPath != null)
{
Logger.LogInformation("AssemblyPath: {File}", AssemblyPath);
}
}
}
// Run the deployment steps
using ITimelineEvent DeployTimer = Timeline.ScopeEvent("Deploying targets");
foreach (TargetMakefile Makefile in Makefiles)
{
// Receipt file may not exist when compiling specific files
if (Makefile.bDeployAfterCompile && FileReference.Exists(Makefile.ReceiptFile))
{
TargetReceipt Receipt = TargetReceipt.Read(Makefile.ReceiptFile);
Logger.LogInformation("Deploying {ReceiptTargetName} {ReceiptPlatform} {ReceiptConfiguration}...", Receipt.TargetName, Receipt.Platform, Receipt.Configuration);
UEBuildPlatform.GetBuildPlatform(Receipt.Platform).Deploy(Receipt);
}
}
}
}
else
{
using ITimelineEvent MetadataTimer = Timeline.ScopeEvent("Writing Target Metadata");
foreach (TargetMakefile Makefile in Makefiles)
{
FileReference TargetInfoFile = FileReference.Combine(Makefile.ProjectIntermediateDirectory, "TargetMetadata.dat");
if (FileReference.Exists(TargetInfoFile))
{
List ArgumentsList = new List()
{
$"-Input={TargetInfoFile}",
$"-Version={WriteMetadataMode.CurrentVersionNumber}"
};
CommandLineArguments Arguments = new CommandLineArguments(ArgumentsList.ToArray());
WriteMetadataMode MetadataMode = new WriteMetadataMode();
await MetadataMode.ExecuteAsync(Arguments, Logger);
}
}
}
}
internal static List CreateLinkedActionsFromFileList(TargetDescriptor Target, BuildConfiguration BuildConfiguration, List FileList, List Actions, ILogger Logger)
{
// We have specific files to compile so we will only queue up those files.
// We will also add them to the MergedOutputItems to make sure they are not skipped
List ProducedItems = new();
// First, find all the SpecificFileActions.
// These are used to create a custom action for the source file in case it is inside a unity or normally not part of the build (headers for example)
Dictionary SpecificFileActions = new();
foreach (ISpecificFileAction SpecificFileAction in Actions.Select(x => x.Inner).OfType())
{
SpecificFileActions.TryAdd(SpecificFileAction.RootDirectory, SpecificFileAction);
}
// Now traverse all specific files and make sure we find or create actions for them.
foreach (FileItem SourceFile in FileList.Select(x => FileItem.GetItemByFileReference(x)).Where(x => x.Exists))
{
ISpecificFileAction? SpecificFileAction = null;
// Traverse upwards from file directory to try to find a SpecificFileAction.
DirectoryItem? Dir = SourceFile.Directory;
while (Dir != null)
{
if (SpecificFileActions.TryGetValue(Dir.Location, out SpecificFileAction))
{
break;
}
Dir = Dir.GetParentDirectoryItem();
}
// We found an action to take care of this file.
if (SpecificFileAction != null)
{
IExternalAction? NewAction = SpecificFileAction.CreateAction(SourceFile, Logger);
if (NewAction != null)
{
Actions.Add(new LinkedAction(NewAction, Target));
ProducedItems.AddRange(NewAction.ProducedItems);
continue;
}
}
// There is no special action for this file.. let's look for an action that has this exact file as input and use that (for example ispc files)
LinkedAction? Action = Actions.Find(x => x.PrerequisiteItems.Contains(SourceFile));
if (Action != null)
{
ProducedItems.AddRange(Action.ProducedItems);
}
else if (!BuildConfiguration.bIgnoreInvalidFiles)
{
Logger.LogError("ERROR: Failed to find an Action that can be used to build \"{Path}\" (does target use this file?)", SourceFile);
}
}
return ProducedItems;
}
///
/// If there are headers in the specific list, this function will expand the list to include all files that include those headers
///
/// List of files to compile
/// Project file if there is one
/// Actions to filter out expansion of files
/// Logger for output
internal static List GetAllSourceFilesIncludingHeader(List SpecificFilesToCompile, FileReference? ProjectFile, List Actions, ILogger Logger)
{
Func IsCode = x => IsHeader(x) || x.HasExtension(".cpp") || x.HasExtension(".c");
IEnumerable SpecificHeaderFiles = SpecificFilesToCompile.Where(IsHeader);
if (!SpecificHeaderFiles.Any())
{
return SpecificFilesToCompile;
}
Logger.LogInformation("Building dependency cache for specified headers");
foreach (FileReference HeaderFile in SpecificHeaderFiles)
{
Logger.LogDebug(" {HeaderFile}", HeaderFile);
}
// TODO: This code below works very similar to the code before this changelist but is not a great solution for figuring out which files to compile.
// Suggested approach is to:
// 1. Write out a file with additional include paths per module.
// 2. Use the special single file actions to figure out if the specific headers are included by the source files using additional include paths
// 3. Fix so CppIncludeLookup is including headers... Current code does not handle this: A.h <- B.h <- C.cpp... C.cpp is not added.
// 4. Cache which files unity files include to reduce number of files needed to be read
List BaseDirs = [Unreal.EngineDirectory];
DirectoryReference CacheDir = Unreal.EngineDirectory;
if (ProjectFile != null)
{
BaseDirs.Add(ProjectFile.Directory);
CacheDir = ProjectFile.Directory;
}
IEnumerable CodeFiles = Actions.SelectMany(x => x.PrerequisiteItems).Distinct().Where(x => IsCode(x.Location) && !x.Name.StartsWith(Unity.ModulePrefix));
FileReference IncludeCache = FileReference.Combine(CacheDir, "Intermediate", "HeaderLookup.dat");
CppIncludeLookup IncludeLookup = new CppIncludeLookup(IncludeCache);
IncludeLookup.Load();
IncludeLookup.Update(BaseDirs, CodeFiles);
IncludeLookup.Save();
// Find all files that can be part of this build
IEnumerable UsedFiles = CodeFiles.Select(x => x.Location);
IEnumerable FoundFiles = IncludeLookup.FindFiles(SpecificFilesToCompile.Select(x => x.GetFileName()).Distinct()).Select(x => x.Location);
// Found files can be a lot (Commandline.h gives 130.000+ files). And we only want to build the ones that are in existing actions
IEnumerable NewFiles = FoundFiles.Intersect(UsedFiles).Distinct();
int genFileCount = NewFiles.Where(x => x.GetFileName().EndsWith(".gen.cpp", StringComparison.OrdinalIgnoreCase)).Count();
Logger.LogInformation("Found {NewFilesCount} source files", NewFiles.Count() - genFileCount);
if (genFileCount > 0)
{
Logger.LogInformation("Found {NewFilesCount} generated source files", genFileCount);
}
foreach (FileReference NewFile in NewFiles.Order())
{
Logger.LogDebug(" {NewFile}", NewFile);
}
return NewFiles.Concat(SpecificFilesToCompile).Distinct().Order().ToList();
}
///
/// Outputs the toolchain used to build each target
///
/// List of targets being built
/// Matching array of makefiles for each target
/// Logger for output
static void OutputToolchainInfo(List TargetDescriptors, TargetMakefile[] Makefiles, ILogger Logger)
{
List OutputIndices = new List();
for (int Idx = 0; Idx < TargetDescriptors.Count; Idx++)
{
if (!TargetDescriptors[Idx].bQuiet)
{
OutputIndices.Add(Idx);
}
}
if (OutputIndices.Count == 1)
{
foreach (string Diagnostic in Makefiles[OutputIndices[0]].Diagnostics)
{
Logger.LogInformation("{Diagnostic}", Diagnostic);
}
}
else
{
foreach (int OutputIndex in OutputIndices)
{
foreach (string Diagnostic in Makefiles[OutputIndex].Diagnostics)
{
Logger.LogInformation("{Name}: {Diagnostic}", TargetDescriptors[OutputIndex].Name, Diagnostic);
}
}
}
}
///
/// Returns true if File is header
///
/// File to check
internal static bool IsHeader(FileReference File) { return File.HasExtension(".h") || File.HasExtension(".hpp") || File.HasExtension(".hxx"); }
///
/// Creates the makefile for a target. If an existing, valid makefile already exists on disk, loads that instead.
///
/// The build configuration
/// Target being built
/// Set of source files which are part of the working set
/// Logger for output
/// Makefile for the given target
internal static async Task CreateMakefileAsync(BuildConfiguration BuildConfiguration, TargetDescriptor TargetDescriptor, ISourceFileWorkingSet WorkingSet, ILogger Logger)
{
// Get the path to the makefile for this target
FileReference? MakefileLocation = null;
if (BuildConfiguration.bUseUBTMakefiles)
{
MakefileLocation = TargetMakefile.GetLocation(TargetDescriptor.ProjectFile, TargetDescriptor.Name, TargetDescriptor.Platform, TargetDescriptor.Architectures, TargetDescriptor.Configuration, TargetDescriptor.IntermediateEnvironment);
}
// Try to load an existing makefile
TargetMakefile? Makefile = null;
if (MakefileLocation != null)
{
using (GlobalTracer.Instance.BuildSpan("TargetMakefile.Load()").StartActive())
{
string? ReasonNotLoaded;
Makefile = TargetMakefile.Load(MakefileLocation, TargetDescriptor.ProjectFile, TargetDescriptor.Platform, TargetDescriptor.AdditionalArguments.GetRawArray(), Logger, out ReasonNotLoaded);
if (Makefile == null)
{
Logger.LogInformation("Creating makefile for {TargetDescriptorName} ({ReasonNotLoaded})", TargetDescriptor.Name, ReasonNotLoaded);
}
}
}
// If we have a makefile, execute the pre-build steps and check it's still valid
bool bHasRunPreBuildScripts = false;
if (Makefile != null)
{
// Execute the scripts.
Utils.ExecuteCustomBuildSteps(Makefile.PreBuildScripts, Logger);
// Don't run the pre-build steps again, even if we invalidate the makefile.
bHasRunPreBuildScripts = true;
// Check that the makefile is still valid
string? Reason;
if (!TargetMakefile.IsValidForSourceFiles(Makefile, TargetDescriptor.ProjectFile, TargetDescriptor.Platform, WorkingSet, Logger, out Reason))
{
Logger.LogInformation("Invalidating makefile for {TargetDescriptorName} ({Reason})", TargetDescriptor.Name, Reason);
Makefile = null;
}
}
// If we couldn't load a makefile, create a new one
if (Makefile == null)
{
// Create the target
UEBuildTarget Target;
using (GlobalTracer.Instance.BuildSpan("UEBuildTarget.Create()").StartActive())
{
Target = UEBuildTarget.Create(TargetDescriptor, BuildConfiguration, Logger);
}
// Create the pre-build scripts
FileReference[] PreBuildScripts = Target.CreatePreBuildScripts();
// Execute the pre-build scripts
if (!bHasRunPreBuildScripts)
{
Utils.ExecuteCustomBuildSteps(PreBuildScripts, Logger);
bHasRunPreBuildScripts = true;
}
// Build the target
using (GlobalTracer.Instance.BuildSpan("UEBuildTarget.Build()").StartActive())
{
Makefile = await Target.BuildAsync(BuildConfiguration, WorkingSet, TargetDescriptor, Logger);
}
Makefile.MemoryPerActionGB = Target.Rules.MemoryPerActionGB;
// Save the pre-build scripts onto the makefile
Makefile.PreBuildScripts = PreBuildScripts;
// Save the additional command line arguments
Makefile.AdditionalArguments = TargetDescriptor.AdditionalArguments.GetRawArray();
// Save the environment variables
foreach (System.Collections.DictionaryEntry? EnvironmentVariable in Environment.GetEnvironmentVariables())
{
if (EnvironmentVariable != null)
{
Makefile.EnvironmentVariables.Add(Tuple.Create((string)EnvironmentVariable.Value.Key, (string)EnvironmentVariable.Value.Value!));
}
}
// Save the makefile for next time
if (MakefileLocation != null)
{
using (GlobalTracer.Instance.BuildSpan("TargetMakefile.Save()").StartActive())
{
Makefile.Save(MakefileLocation);
}
}
}
else
{
// Restore the environment variables
foreach (Tuple EnvironmentVariable in Makefile.EnvironmentVariables)
{
Environment.SetEnvironmentVariable(EnvironmentVariable.Item1, EnvironmentVariable.Item2);
}
#if __VPROJECT_AVAILABLE__
Task VNITask = Task.CompletedTask;
// Same for VNI
if (Makefile.VNIModules.Count > 0)
{
VNITask = Task.Run(() => VNIExecution.ExecuteVNITool(Makefile, TargetDescriptor, Logger));
}
#endif
// If the target needs UHT to be run, we'll go ahead and do that now
if (Makefile.UObjectModules.Count > 0)
{
await ExternalExecution.ExecuteHeaderToolIfNecessaryAsync(BuildConfiguration, TargetDescriptor.ProjectFile, Makefile, TargetDescriptor.Name, WorkingSet, Logger);
}
#if __VPROJECT_AVAILABLE__
VNITask.Wait();
#endif
}
return Makefile;
}
///
/// Determines all the actions that should be executed for a target (filtering for single module/file, etc..)
///
/// The target being built
/// Makefile for the target
/// Set of all output items
/// List of actions that need to be executed
static void GatherOutputItems(TargetDescriptor TargetDescriptor, TargetMakefile Makefile, HashSet OutputItems)
{
if (TargetDescriptor.SpecificFilesToCompile.Count > 0)
{
}
else if (TargetDescriptor.OnlyModuleNames.Count > 0)
{
// Find the output items for this module
foreach (string OnlyModuleName in TargetDescriptor.OnlyModuleNames)
{
FileItem[]? OutputItemsForModule;
if (!Makefile.ModuleNameToOutputItems.TryGetValue(OnlyModuleName, out OutputItemsForModule))
{
throw new BuildException("Unable to find output items for module '{0}'", OnlyModuleName);
}
OutputItems.UnionWith(OutputItemsForModule);
}
}
else
{
// Use all the output items from the target
OutputItems.UnionWith(Makefile.OutputItems);
}
}
///
/// Merge action graphs for multiple targets into a single set of actions. Sets group names on merged actions to indicate which target they belong to.
///
/// List of target descriptors
/// Array of actions for each target
/// List of merged actions
static List MergeActionGraphs(List TargetDescriptors, List[] TargetActions)
{
// Set of all output items. Knowing that there are no conflicts in produced items, we use this to eliminate duplicate actions.
Dictionary OutputItemToProducingAction = new Dictionary();
HashSet IgnoreConflictActions = new HashSet();
for (int TargetIdx = 0; TargetIdx < TargetDescriptors.Count; TargetIdx++)
{
string GroupPrefix = String.Format("{0}-{1}-{2}", TargetDescriptors[TargetIdx].Name, TargetDescriptors[TargetIdx].Platform, TargetDescriptors[TargetIdx].Configuration);
foreach (LinkedAction TargetAction in TargetActions[TargetIdx])
{
if (!TargetAction.ProducedItems.Any())
{
continue;
}
if (TargetAction.IgnoreConflicts())
{
IgnoreConflictActions.Add(TargetAction);
TargetAction.GroupNames.Add(GroupPrefix);
continue;
}
FileItem ProducedItem = TargetAction.ProducedItems.First();
LinkedAction? ExistingAction;
if (!OutputItemToProducingAction.TryGetValue(ProducedItem, out ExistingAction))
{
ExistingAction = new LinkedAction(TargetAction, TargetDescriptors[TargetIdx]);
OutputItemToProducingAction[ProducedItem] = ExistingAction;
}
ExistingAction.GroupNames.Add(GroupPrefix);
}
}
List Results = new List(OutputItemToProducingAction.Values);
Results.AddRange(IgnoreConflictActions);
return Results;
}
void ProcessCoreDumps(DirectoryReference? SaveCrashDumpDirectory, ILogger Logger)
{
if (SaveCrashDumpDirectory == null)
{
return;
}
if (!OperatingSystem.IsWindows())
{
return;
}
// set to true to have this code create some files in expected locations to report
bool bTestDumpDetection = false;
using OpenTracing.IScope Scope = GlobalTracer.Instance.BuildSpan("Processing crash dump files").StartActive();
DateTime UBTStartTime = Process.GetCurrentProcess().StartTime;
List FoundCrashDumps = new List();
// examine the contents of %LOCALAPPDATA%\CrashDumps
string? LocalAppData = Environment.GetEnvironmentVariable("LOCALAPPDATA");
if (LocalAppData != null)
{
DirectoryReference CrashDumpsDirectory = DirectoryReference.Combine(DirectoryReference.FromString(LocalAppData)!, "CrashDumps");
if (DirectoryReference.Exists(CrashDumpsDirectory))
{
if (bTestDumpDetection)
{
System.IO.File.Create(System.IO.Path.Combine(CrashDumpsDirectory.FullName, Guid.NewGuid().ToString() + ".dmp")).Close();
}
foreach (FileReference CrashDump in DirectoryReference.EnumerateFiles(CrashDumpsDirectory))
{
if (FileReference.GetLastWriteTime(CrashDump) > UBTStartTime)
{
Logger.LogWarning("Crash dump {CrashDump} was created duing UnrealBuildTool execution", CrashDump);
FoundCrashDumps.Add(CrashDump);
}
}
}
}
// examine the contents of %TMP% (on CI agents, this should not be unreasonably slow as the tmp dir gets cleaned between builds
string? TMP = Environment.GetEnvironmentVariable("TMP");
if (TMP != null)
{
DirectoryReference TmpDir = DirectoryReference.FromString(TMP)!;
if (bTestDumpDetection)
{
System.IO.File.Create(System.IO.Path.Combine(TmpDir.FullName, Guid.NewGuid().ToString() + ".dmp")); //.Close(); Intentionally not closed, to trigger catch() block below
}
List AccessibleTmpDirectories = new List();
AccessibleTmpDirectories.Add(TmpDir);
for (int I = 0; I < AccessibleTmpDirectories.Count; ++I)
{
try
{
// Manual recursion to avoid the case where an inaccessible file prevents us from iterating reachable parts of the dir
AccessibleTmpDirectories.AddRange(DirectoryReference.EnumerateDirectories(AccessibleTmpDirectories[I]));
foreach (FileReference TmpFile in DirectoryReference.EnumerateFiles(AccessibleTmpDirectories[I]))
{
if (TmpFile.HasExtension(".dmp") && FileReference.GetLastWriteTime(TmpFile) > UBTStartTime)
{
Logger.LogWarning("Crash dump {TmpFile} was created duing UnrealBuildTool execution", TmpFile);
FoundCrashDumps.Add(TmpFile);
}
}
}
catch
{
// silently ignore inaccessible directories
}
}
}
if (FoundCrashDumps.Count > 0)
{
DirectoryReference.CreateDirectory(SaveCrashDumpDirectory);
foreach (FileReference CrashDump in FoundCrashDumps)
{
Logger.LogInformation("Copying {CrashDump} to {SaveCrashDumpDirectory}", CrashDump, SaveCrashDumpDirectory);
try
{
FileReference.Copy(CrashDump, FileReference.Combine(SaveCrashDumpDirectory, CrashDump.GetFileName()));
}
catch (Exception Ex)
{
// don't stop if there was a problem copying one of the files
Logger.LogWarning("Failed to copy crash dump {CrashDump}: {Ex}", CrashDump, ExceptionUtils.FormatException(Ex));
}
}
}
}
}
}