// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.ComponentModel; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Threading.Tasks; using EpicGames.BuildGraph; using EpicGames.BuildGraph.Expressions; using EpicGames.Core; using Microsoft.Extensions.Logging; using OpenTracing; using OpenTracing.Util; using UnrealBuildBase; using UnrealBuildTool; #nullable enable namespace AutomationTool { /// /// Base class for binding and executing nodes /// abstract class BgNodeExecutor { public abstract Task Execute(JobContext job, Dictionary> tagNameToFileSet); } class BgBytecodeNodeExecutor : BgNodeExecutor { class BgContextImpl : BgContext { public BgContextImpl(JobContext jobContext, Dictionary> tagNameToFileSet) : base(tagNameToFileSet.ToDictionary(x => x.Key, x => FileSet.FromFiles(Unreal.RootDirectory, x.Value))) { _ = jobContext; } public override string Stream => CommandUtils.P4Enabled ? CommandUtils.P4Env.Branch : ""; public override int Change => CommandUtils.P4Enabled ? CommandUtils.P4Env.Changelist : 0; public override int CodeChange => CommandUtils.P4Enabled ? CommandUtils.P4Env.CodeChangelist : 0; public override (int Major, int Minor, int Patch) EngineVersion { get { ReadOnlyBuildVersion current = ReadOnlyBuildVersion.Current; return (current.MajorVersion, current.MinorVersion, current.PatchVersion); } } public override bool IsBuildMachine => CommandUtils.IsBuildMachine; } readonly BgNodeDef _node; public BgBytecodeNodeExecutor(BgNodeDef node) { _node = node; } public static bool Bind(ILogger logger) { _ = logger; return true; } /// /// ExecuteAsync the method given in the /// /// /// /// public override async Task Execute(JobContext job, Dictionary> tagNameToFileSet) { BgThunkDef thunk = _node.Thunk!; MethodInfo method = thunk.Method; HashSet buildProducts = tagNameToFileSet[_node.DefaultOutput.TagName]; BgContextImpl context = new BgContextImpl(job, tagNameToFileSet); ParameterInfo[] parameters = method.GetParameters(); object?[] arguments = new object[parameters.Length]; for (int idx = 0; idx < parameters.Length; idx++) { Type parameterType = parameters[idx].ParameterType; if (parameterType == typeof(BgContext)) { arguments[idx] = context; } else { arguments[idx] = thunk.Arguments[idx]; } } Task task = (Task)method.Invoke(null, arguments)!; await task; if (_node.Outputs.Count > 0) { object? result = null; if (method.ReturnType.IsGenericType && method.ReturnType.GetGenericTypeDefinition() == typeof(Task<>)) { Type taskType = task.GetType(); #pragma warning disable CA1849 // Task.Result synchronously blocks PropertyInfo property = taskType.GetProperty(nameof(Task.Result))!; #pragma warning restore CA1849 result = property!.GetValue(task); } object?[] outputValues; if (result is ITuple tuple) { outputValues = Enumerable.Range(0, tuple.Length).Select(x => tuple[x]).ToArray(); } else { outputValues = new[] { result }; } for (int idx = 0; idx < outputValues.Length; idx++) { if (outputValues[idx] is BgFileSetOutputExpr fileSet) { string tagName = _node.Outputs[idx + 1].TagName; tagNameToFileSet[tagName] = new HashSet(fileSet.Value.Flatten().Values); buildProducts.UnionWith(tagNameToFileSet[tagName]); } } } return true; } } /// /// Implementation of for graphs defined through XML syntax /// class BgScriptNodeExecutor : BgNodeExecutor { /// /// The script node /// public BgScriptNode Node { get; } /// /// List of bound task implementations /// readonly List _boundTasks = new List(); /// /// Constructor /// public BgScriptNodeExecutor(BgScriptNode node) { Node = node; } public async ValueTask BindAsync(Dictionary nameToTask, Dictionary tagNameToNodeOutput, ILogger logger) { bool result = true; foreach (BgTask taskInfo in Node.Tasks) { BgTaskImpl? boundTask = await BindTaskAsync(taskInfo, nameToTask, tagNameToNodeOutput, logger); if (boundTask == null) { result = false; } else { _boundTasks.Add(boundTask); } } return result; } async ValueTask BindTaskAsync(BgTask taskInfo, Dictionary nameToTask, IReadOnlyDictionary tagNameToNodeOutput, ILogger logger) { // Get the reflection info for this element ScriptTaskBinding? task; if (!nameToTask.TryGetValue(taskInfo.Name, out task)) { logger.LogScriptError(taskInfo.Location, "Unknown task '{TaskName}'", taskInfo.Name); return null; } // Check all the required parameters are present bool hasRequiredAttributes = true; foreach (ScriptTaskParameterBinding parameter in task.NameToParameter.Values) { if (!parameter.Optional && !taskInfo.Arguments.ContainsKey(parameter.Name)) { logger.LogScriptError(taskInfo.Location, "Missing required attribute - {AttrName}", parameter.Name); hasRequiredAttributes = false; } } // Create a context for evaluating conditions BgConditionContext conditionContext = new BgConditionContext(Unreal.RootDirectory); // Read all the attributes into a parameters object for this task object parametersObject = Activator.CreateInstance(task.ParametersClass)!; foreach ((string name, string value) in taskInfo.Arguments) { // Get the field that this attribute should be written to in the parameters object ScriptTaskParameterBinding? parameter; if (!task.NameToParameter.TryGetValue(name, out parameter)) { logger.LogScriptError(taskInfo.Location, "Unknown attribute '{AttrName}'", name); continue; } // If it's a collection type, split it into separate values try { if (parameter.CollectionType == null) { // Parse it and assign it to the parameters object object? fieldValue = await ParseValueAsync(value, parameter.ValueType, conditionContext); if (fieldValue != null) { parameter.SetValue(parametersObject, fieldValue); } else if (!parameter.Optional) { logger.LogScriptError(taskInfo.Location, "Empty value for parameter '{AttrName}' is not allowed.", name); } } else { // Get the collection, or create one if necessary object? collectionValue = parameter.GetValue(parametersObject); if (collectionValue == null) { collectionValue = Activator.CreateInstance(parameter.ParameterType)!; parameter.SetValue(parametersObject, collectionValue); } // Parse the values and add them to the collection List valueStrings = BgTaskImpl.SplitDelimitedList(value); foreach (string valueString in valueStrings) { object? elementValue = await ParseValueAsync(valueString, parameter.ValueType, conditionContext); if (elementValue != null) { parameter.CollectionType.InvokeMember("Add", BindingFlags.InvokeMethod | BindingFlags.Instance | BindingFlags.Public, null, collectionValue, new object[] { elementValue }); } } } } catch (Exception ex) { logger.LogScriptError(taskInfo.Location, "Unable to parse argument {Name} from {Value}", name, value); logger.LogDebug(ex, "Exception while parsing argument {Name}", name); } } // Construct the task if (!hasRequiredAttributes) { return null; } // Add it to the list BgTaskImpl newTask = (BgTaskImpl)Activator.CreateInstance(task.TaskClass, parametersObject)!; // Set up the source location for diagnostics newTask.SourceLocation = taskInfo.Location; // Make sure all the read tags are local or listed as a dependency foreach (string readTagName in newTask.FindConsumedTagNames()) { BgNodeOutput? output; if (tagNameToNodeOutput.TryGetValue(readTagName, out output)) { if (output != null && output.ProducingNode != Node && !Node.Inputs.Contains(output)) { logger.LogScriptError(taskInfo.Location, "The tag '{TagName}' is not a dependency of node '{Node}'", readTagName, Node.Name); } } } // Make sure all the written tags are local or listed as an output foreach (string modifiedTagName in newTask.FindProducedTagNames()) { BgNodeOutput? output; if (tagNameToNodeOutput.TryGetValue(modifiedTagName, out output)) { if (output != null && !Node.Outputs.Contains(output)) { logger.LogScriptError(taskInfo.Location, "The tag '{TagName}' is created by '{Node}', and cannot be modified downstream", output.TagName, output.ProducingNode.Name); } } } return newTask; } /// /// Parse a value of the given type /// /// The text to parse /// Type of the value to parse /// Context for evaluating boolean expressions /// Value that was parsed static async ValueTask ParseValueAsync(string valueText, Type valueType, BgConditionContext context) { // Parse it and assign it to the parameters object if (valueType.IsEnum) { return Enum.Parse(valueType, valueText); } else if (valueType == typeof(bool)) { return await BgCondition.EvaluateAsync(valueText, context); } else if (valueType == typeof(FileReference)) { if (String.IsNullOrEmpty(valueText)) { return null; } else { return BgTaskImpl.ResolveFile(valueText); } } else if (valueType == typeof(DirectoryReference)) { if (String.IsNullOrEmpty(valueText)) { return null; } else { return BgTaskImpl.ResolveDirectory(valueText); } } TypeConverter converter = TypeDescriptor.GetConverter(valueType); if (converter.CanConvertFrom(typeof(string))) { return converter.ConvertFromString(valueText); } else { return Convert.ChangeType(valueText, valueType); } } /// /// Build all the tasks for this node /// /// Information about the current job /// Mapping from tag names to the set of files they include. Should be set to contain the node inputs on entry. /// Whether the task succeeded or not. Exiting with an exception will be caught and treated as a failure. public override async Task Execute(JobContext job, Dictionary> tagNameToFileSet) { // Run each of the tasks in order HashSet buildProducts = tagNameToFileSet[Node.DefaultOutput.TagName]; for (int idx = 0; idx < _boundTasks.Count; idx++) { using (IScope scope = GlobalTracer.Instance.BuildSpan("Task").WithTag("resource", _boundTasks[idx].GetTraceName()).StartActive()) { ITaskExecutor? executor = _boundTasks[idx].GetExecutor(); if (executor == null) { // ExecuteAsync this task directly try { _boundTasks[idx].GetTraceMetadata(scope.Span, ""); await _boundTasks[idx].ExecuteAsync(job, buildProducts, tagNameToFileSet); } catch (Exception ex) { ExceptionUtils.AddContext(ex, "while executing task {0}", _boundTasks[idx].GetTraceString()); BgScriptLocation? sourceLocation = _boundTasks[idx].SourceLocation; if (sourceLocation != null) { ExceptionUtils.AddContext(ex, "at {0}({1})", sourceLocation.File, sourceLocation.LineNumber); } throw; } } else { _boundTasks[idx].GetTraceMetadata(scope.Span, "1."); // The task has a custom executor, which may be able to execute several tasks simultaneously. Try to add the following tasks. int firstIdx = idx; while (idx + 1 < Node.Tasks.Count && executor.Add(_boundTasks[idx + 1])) { idx++; _boundTasks[idx].GetTraceMetadata(scope.Span, String.Format("{0}.", 1 + idx - firstIdx)); } try { await executor.ExecuteAsync(job, buildProducts, tagNameToFileSet); } catch (Exception ex) { for (int taskIdx = firstIdx; taskIdx <= idx; taskIdx++) { ExceptionUtils.AddContext(ex, "while executing {0}", _boundTasks[taskIdx].GetTraceString()); } BgScriptLocation? sourceLocation = _boundTasks[firstIdx].SourceLocation; if (sourceLocation != null) { ExceptionUtils.AddContext(ex, "at {0}({1})", sourceLocation.File, sourceLocation.LineNumber); } throw; } } } } // Remove anything that doesn't exist, since these files weren't explicitly tagged buildProducts.RemoveWhere(x => !FileReference.Exists(x)); return true; } } }