// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using System.Xml; using EpicGames.Core; using OpenTracing; using UnrealBuildTool; namespace AutomationTool.Tasks { /// /// Parameters for a compile task /// public class CompileTaskParameters { /// /// The target to compile. /// [TaskParameter(Optional = true)] public string Target { get; set; } /// /// The configuration to compile. /// [TaskParameter] public UnrealTargetConfiguration Configuration { get; set; } /// /// The platform to compile for. /// [TaskParameter] public UnrealTargetPlatform Platform { get; set; } /// /// The project to compile with. /// [TaskParameter(Optional = true, ValidationType = TaskParameterValidationType.FileSpec)] public string Project { get; set; } /// /// Additional arguments for UnrealBuildTool. /// [TaskParameter(Optional = true)] public string Arguments { get; set; } /// /// Whether to allow using XGE for compilation. /// [TaskParameter(Optional = true)] public bool AllowXGE { get; set; } = true; /// /// No longer necessary as UnrealBuildTool is run to compile targets. /// [TaskParameter(Optional = true)] [Obsolete("This setting is no longer used")] public bool AllowParallelExecutor { get; set; } = true; /// /// Whether to allow UBT to use all available cores, when AllowXGE is disabled. /// [TaskParameter(Optional = true)] public bool AllowAllCores { get; set; } = false; /// /// Whether to allow cleaning this target. If unspecified, targets are cleaned if the -Clean argument is passed on the command line. /// [TaskParameter(Optional = true)] public bool? Clean { get; set; } = null; /// /// Global flag passed to UBT that can be used to generate target files without fully compiling. /// [TaskParameter(Optional = true)] public bool SkipBuild { get; set; } = false; /// /// Tag to be applied to build products of this task. /// [TaskParameter(Optional = true, ValidationType = TaskParameterValidationType.TagList)] public string Tag { get; set; } } /// /// Executor for compile tasks, which can compile multiple tasks simultaneously /// public class CompileTaskExecutor : ITaskExecutor { /// /// List of targets to compile. As well as the target specifically added for this task, additional compile tasks may be merged with it. /// readonly List _targets = new List(); /// /// Mapping of receipt filename to its corresponding tag name /// readonly Dictionary _targetToTagName = new Dictionary(); /// /// Whether to allow using XGE for this job /// bool _allowXge = true; /// /// Whether to allow using all available cores for this job, when bAllowXGE is false /// bool _allowAllCores = false; /// /// Should SkipBuild be passed to UBT so that only .target files are generated. /// bool _skipBuild = false; /// /// Constructor /// /// Initial task to execute public CompileTaskExecutor(CompileTask task) { Add(task); } /// /// Adds another task to this executor /// /// Task to add /// True if the task could be added, false otherwise public bool Add(BgTaskImpl task) { CompileTask compileTask = task as CompileTask; if (compileTask == null) { return false; } CompileTaskParameters parameters = compileTask.Parameters; if (_targets.Count > 0) { if (_allowXge != parameters.AllowXGE || _skipBuild != parameters.SkipBuild) { return false; } } else { _allowXge = parameters.AllowXGE; _allowAllCores = parameters.AllowAllCores; _skipBuild = parameters.SkipBuild; } _allowXge &= parameters.AllowXGE; _allowAllCores &= parameters.AllowAllCores; UnrealBuild.BuildTarget target = new UnrealBuild.BuildTarget { TargetName = parameters.Target, Platform = parameters.Platform, Config = parameters.Configuration, UprojectPath = compileTask.FindProjectFile(), UBTArgs = (parameters.Arguments ?? ""), Clean = parameters.Clean }; if (!String.IsNullOrEmpty(parameters.Tag)) { _targetToTagName.Add(target, parameters.Tag); } _targets.Add(target); return true; } /// /// ExecuteAsync all the tasks added to this executor. /// /// Information about the current job /// Set of build products produced by this node. /// Mapping from tag names to the set of files they include /// Whether the task succeeded or not. Exiting with an exception will be caught and treated as a failure. public Task ExecuteAsync(JobContext job, HashSet buildProducts, Dictionary> tagNameToFileSet) { // Create the agenda UnrealBuild.BuildAgenda agenda = new UnrealBuild.BuildAgenda(); agenda.Targets.AddRange(_targets); // Build everything Dictionary targetToManifest = new Dictionary(); UnrealBuild builder = new UnrealBuild(job.OwnerCommand); bool allCores = (CommandUtils.IsBuildMachine || _allowAllCores); // Enable using all cores if this is a build agent or the flag was passed in to the task and XGE is disabled. builder.Build(agenda, InDeleteBuildProducts: null, InUpdateVersionFiles: false, InForceNoXGE: !_allowXge, InAllCores: allCores, InTargetToManifest: targetToManifest, InSkipBuild: _skipBuild); UnrealBuild.CheckBuildProducts(builder.BuildProductFiles); // Tag all the outputs foreach (KeyValuePair targetTagName in _targetToTagName) { BuildManifest manifest; if (!targetToManifest.TryGetValue(targetTagName.Key, out manifest)) { throw new AutomationException("Missing manifest for target {0} {1} {2}", targetTagName.Key.TargetName, targetTagName.Key.Platform, targetTagName.Key.Config); } HashSet manifestBuildProducts = manifest.BuildProducts.Select(x => new FileReference(x)).ToHashSet(); // when we make a Mac/IOS build, Xcode will finalize the .app directory, adding files that UBT has no idea about, so now we recursively add any files in the .app // as BuildProducts. look for any .apps that we have any files as BuildProducts, and expand to include all files in the .app if (BuildHostPlatform.Current.Platform == UnrealTargetPlatform.Mac) { HashSet appBundleLocations = new(); foreach (FileReference file in manifestBuildProducts) { // look for a ".app/" portion and chop off anything after it int appLocation = file.FullName.IndexOf(".app/", StringComparison.InvariantCultureIgnoreCase); if (appLocation > 0) { appBundleLocations.Add(file.FullName.Substring(0, appLocation + 4)); } } // now with a unique set of app bundles, add all files in them foreach (string appBundleLocation in appBundleLocations) { manifestBuildProducts.UnionWith(DirectoryReference.EnumerateFiles(new DirectoryReference(appBundleLocation), "*", System.IO.SearchOption.AllDirectories)); } } foreach (string tagName in CustomTask.SplitDelimitedList(targetTagName.Value)) { HashSet fileSet = CustomTask.FindOrAddTagSet(tagNameToFileSet, tagName); fileSet.UnionWith(manifestBuildProducts); } } // Add everything to the list of build products buildProducts.UnionWith(builder.BuildProductFiles.Select(x => new FileReference(x))); return Task.CompletedTask; } } /// /// Compiles a target with UnrealBuildTool. /// [TaskElement("Compile", typeof(CompileTaskParameters))] public class CompileTask : BgTaskImpl { /// /// Parameters for this task /// public CompileTaskParameters Parameters { get; set; } /// /// Resolved path to Project file /// public FileReference ProjectFile { get; set; } = null; /// /// Construct a compile task /// /// Parameters for this task public CompileTask(CompileTaskParameters parameters) { Parameters = parameters; } /// /// Resolve the path to the project file /// public FileReference FindProjectFile() { FileReference projectFile = null; // Resolve the full path to the project file if (!String.IsNullOrEmpty(Parameters.Project)) { if (Parameters.Project.EndsWith(".uproject", StringComparison.OrdinalIgnoreCase)) { projectFile = CustomTask.ResolveFile(Parameters.Project); } else { projectFile = NativeProjects.EnumerateProjectFiles(Log.Logger).FirstOrDefault(x => x.GetFileNameWithoutExtension().Equals(Parameters.Project, StringComparison.OrdinalIgnoreCase)); } if (projectFile == null || !FileReference.Exists(projectFile)) { throw new BuildException("Unable to resolve project '{0}'", Parameters.Project); } } return projectFile; } /// /// ExecuteAsync the task. /// /// Information about the current job /// Set of build products produced by this node. /// Mapping from tag names to the set of files they include public override Task ExecuteAsync(JobContext job, HashSet buildProducts, Dictionary> tagNameToFileSet) { // // Don't do any logic here. You have to do it in the ctor or a getter // otherwise you break the ITaskExecutor pathway, which doesn't call this function! // return GetExecutor().ExecuteAsync(job, buildProducts, tagNameToFileSet); } /// /// /// /// public override ITaskExecutor GetExecutor() { return new CompileTaskExecutor(this); } /// /// Get properties to include in tracing info /// /// The span to add metadata to /// Prefix for all metadata keys public override void GetTraceMetadata(ITraceSpan span, string prefix) { base.GetTraceMetadata(span, prefix); span.AddMetadata(prefix + "target.name", Parameters.Target); span.AddMetadata(prefix + "target.config", Parameters.Configuration.ToString()); span.AddMetadata(prefix + "target.platform", Parameters.Platform.ToString()); if (Parameters.Project != null) { span.AddMetadata(prefix + "target.project", Parameters.Project); } } /// /// Get properties to include in tracing info /// /// The span to add metadata to /// Prefix for all metadata keys public override void GetTraceMetadata(ISpan span, string prefix) { base.GetTraceMetadata(span, prefix); span.SetTag(prefix + "target.name", Parameters.Target); span.SetTag(prefix + "target.config", Parameters.Configuration.ToString()); span.SetTag(prefix + "target.platform", Parameters.Platform.ToString()); if (Parameters.Project != null) { span.SetTag(prefix + "target.project", Parameters.Project); } } /// /// Output this task out to an XML writer. /// public override void Write(XmlWriter writer) { Write(writer, Parameters); } /// /// Find all the tags which are used as inputs to this task /// /// The tag names which are read by this task public override IEnumerable FindConsumedTagNames() { yield break; } /// /// Find all the tags which are modified by this task /// /// The tag names which are modified by this task public override IEnumerable FindProducedTagNames() { return FindTagNamesFromList(Parameters.Tag); } } public static partial class StandardTasks { /// /// Compiles a target /// /// The target to compile /// The configuration to compile /// The platform to compile for /// The project to compile with /// Additional arguments for UnrealBuildTool /// Whether to allow using XGE for compilation /// Whether to allow cleaning this target. If unspecified, targets are cleaned if the -Clean argument is passed on the command line /// Build products from the compile public static async Task CompileAsync(string target, UnrealTargetPlatform platform, UnrealTargetConfiguration configuration, FileReference project = null, string arguments = null, bool allowXge = true, bool? clean = null) { CompileTaskParameters parameters = new CompileTaskParameters(); parameters.Target = target; parameters.Platform = platform; parameters.Configuration = configuration; parameters.Project = project?.FullName; parameters.Arguments = arguments; parameters.AllowXGE = allowXge; parameters.Clean = clean; return await ExecuteAsync(new CompileTask(parameters)); } } }