// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; using System.Text; using EpicGames.Core; using Microsoft.Extensions.Logging; [assembly: InternalsVisibleTo("EpicGames.BuildGraph.Tests")] namespace EpicGames.BuildGraph { /// /// Options for how the graph should be printed /// [Flags] public enum GraphPrintOptions { /// /// No options specified /// None = 0, /// /// Includes a list of the graph options /// ShowCommandLineOptions = 0x1, /// /// Includes the list of dependencies for each node /// ShowDependencies = 0x2, /// /// Includes the list of notifiers for each node /// ShowNotifications = 0x4, } /// /// Definition of a graph. /// public class BgGraphDef { /// /// List of options, in the order they were specified /// public List Options { get; } = new List(); /// /// List of agents containing nodes to execute /// public List Agents { get; } = new List(); /// /// Mapping from name to agent /// public Dictionary NameToAgent { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of names to the corresponding node. /// public Dictionary NameToNode { get; set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of names to the corresponding report. /// public Dictionary NameToReport { get; private set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of names to their corresponding node output. /// public Dictionary TagNameToNodeOutput { get; private set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of aggregate names to their respective nodes /// public Dictionary NameToAggregate { get; private set; } = new Dictionary(StringComparer.OrdinalIgnoreCase); /// /// Mapping of artifact names to their definitions. Artifacts will be produced from matching tag names. /// public List Artifacts { get; } = new List(); /// /// List of badges that can be displayed for this build /// public List Badges { get; } = new List(); /// /// List of labels that can be displayed for this build /// public List Labels { get; } = new List(); /// /// Diagnostics at graph scope /// public List Diagnostics { get; } = new List(); /// /// Default constructor /// public BgGraphDef() { } /// /// Checks whether a given name already exists /// /// The name to check. /// True if the name exists, false otherwise. public bool ContainsName(string name) { return NameToNode.ContainsKey(name) || NameToReport.ContainsKey(name) || NameToAggregate.ContainsKey(name); } /// /// Gets diagnostics from all graph structures /// /// List of diagnostics public List GetAllDiagnostics() { List diagnostics = new List(Diagnostics); foreach (BgAgentDef agent in Agents) { diagnostics.AddRange(agent.Diagnostics); foreach (BgNodeDef node in agent.Nodes) { diagnostics.AddRange(node.Diagnostics); } } return diagnostics; } /// /// Tries to resolve the given name to one or more nodes. Checks for aggregates, and actual nodes. /// /// The name to search for /// If the name is a match, receives an array of nodes and their output names /// True if the name was found, false otherwise. public bool TryResolveReference(string name, [NotNullWhen(true)] out BgNodeDef[]? outNodes) { // Check if it's a tag reference or node reference if (name.StartsWith("#", StringComparison.Ordinal)) { // Check if it's a regular node or output name BgNodeOutput? output; if (TagNameToNodeOutput.TryGetValue(name, out output)) { outNodes = new BgNodeDef[] { output.ProducingNode }; return true; } } else { // Check if it's a regular node or output name BgNodeDef? node; if (NameToNode.TryGetValue(name, out node)) { outNodes = new BgNodeDef[] { node }; return true; } // Check if it's an aggregate name BgAggregateDef? aggregate; if (NameToAggregate.TryGetValue(name, out aggregate)) { outNodes = aggregate.RequiredNodes.ToArray(); return true; } // Check if it's a group name BgAgentDef? agent; if (NameToAgent.TryGetValue(name, out agent)) { outNodes = agent.Nodes.ToArray(); return true; } } // Otherwise fail outNodes = null; return false; } /// /// Tries to resolve the given name to one or more node outputs. Checks for aggregates, and actual nodes. /// /// The name to search for /// If the name is a match, receives an array of nodes and their output names /// True if the name was found, false otherwise. public bool TryResolveInputReference(string name, [NotNullWhen(true)] out BgNodeOutput[]? outOutputs) { // Check if it's a tag reference or node reference if (name.StartsWith("#", StringComparison.Ordinal)) { // Check if it's a regular node or output name BgNodeOutput? output; if (TagNameToNodeOutput.TryGetValue(name, out output)) { outOutputs = new BgNodeOutput[] { output }; return true; } } else { // Check if it's a regular node or output name BgNodeDef? node; if (NameToNode.TryGetValue(name, out node)) { outOutputs = node.Outputs.Union(node.Inputs).ToArray(); return true; } // Check if it's an aggregate name BgAggregateDef? aggregate; if (NameToAggregate.TryGetValue(name, out aggregate)) { outOutputs = aggregate.RequiredNodes.SelectMany(x => x.Outputs.Union(x.Inputs)).Distinct().ToArray(); return true; } } // Otherwise fail outOutputs = null; return false; } static void AddDependencies(BgNodeDef node, HashSet retainNodes) { if (retainNodes.Add(node)) { foreach (BgNodeDef inputDependency in node.InputDependencies) { AddDependencies(inputDependency, retainNodes); } } } /// /// Cull the graph to only include the given nodes and their dependencies /// /// A set of target nodes to build public void Select(IEnumerable targetNodes) { // Find this node and all its dependencies HashSet retainNodes = new HashSet(); foreach (BgNodeDef targetNode in targetNodes) { AddDependencies(targetNode, retainNodes); } // Remove all the nodes which are not marked to be kept foreach (BgAgentDef agent in Agents) { agent.Nodes = agent.Nodes.Where(x => retainNodes.Contains(x)).ToList(); } // Remove all the empty agents Agents.RemoveAll(x => x.Nodes.Count == 0); // Trim down the list of nodes for each report to the ones that are being built foreach (BgReport report in NameToReport.Values) { report.Nodes.RemoveWhere(x => !retainNodes.Contains(x)); } // Remove all the empty reports NameToReport = NameToReport.Where(x => x.Value.Nodes.Count > 0).ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.InvariantCultureIgnoreCase); // Remove all the order dependencies which are no longer part of the graph. Since we don't need to build them, we don't need to wait for them foreach (BgNodeDef node in retainNodes) { node.OrderDependencies.RemoveAll(x => !retainNodes.Contains(x)); } // Create a new list of aggregates for everything that's left Dictionary newNameToAggregate = new Dictionary(NameToAggregate.Comparer); foreach (BgAggregateDef aggregate in NameToAggregate.Values) { if (aggregate.RequiredNodes.All(x => retainNodes.Contains(x))) { newNameToAggregate[aggregate.Name] = aggregate; } } NameToAggregate = newNameToAggregate; // Remove any labels that are no longer valid foreach (BgLabelDef label in Labels) { label.RequiredNodes.RemoveWhere(x => !retainNodes.Contains(x)); label.IncludedNodes.RemoveWhere(x => !retainNodes.Contains(x)); } Labels.RemoveAll(x => x.RequiredNodes.Count == 0); // Remove any badges which do not have all their dependencies Badges.RemoveAll(x => x.Nodes.Any(y => !retainNodes.Contains(y))); // Rebuild the tag name to output dictionary Dictionary newTagNameToNodeOutput = new Dictionary(TagNameToNodeOutput.Comparer); foreach (BgNodeOutput output in Agents.SelectMany(x => x.Nodes).SelectMany(x => x.Outputs)) { newTagNameToNodeOutput.Add(output.TagName, output); } TagNameToNodeOutput = newTagNameToNodeOutput; // Remove any artifacts whose outputs are not produced Artifacts.RemoveAll(x => x.TagName != null && !TagNameToNodeOutput.ContainsKey(x.TagName)); // Remove any artifacts whose nodes are no longer run HashSet newNodeNames = Agents.SelectMany(x => x.Nodes).Select(x => x.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); Artifacts.RemoveAll(x => x.NodeName != null && !newNodeNames.Contains(x.NodeName)); } /// /// Export the build graph to a Json file, for parallel execution by the build system /// /// Output file to write /// Set of nodes which have been completed public void Export(FileReference file, HashSet completedNodes) { // Find all the nodes which we're actually going to execute. We'll use this to filter the graph. HashSet nodesToExecute = new HashSet(); foreach (BgNodeDef node in Agents.SelectMany(x => x.Nodes)) { if (!completedNodes.Contains(node)) { nodesToExecute.Add(node); } } // Open the output file using (JsonWriter jsonWriter = new JsonWriter(file.FullName)) { jsonWriter.WriteObjectStart(); // Write all the agents jsonWriter.WriteArrayStart("Groups"); foreach (BgAgentDef agent in Agents) { BgNodeDef[] nodes = agent.Nodes.Where(x => nodesToExecute.Contains(x)).ToArray(); if (nodes.Length > 0) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", agent.Name); jsonWriter.WriteArrayStart("Agent Types"); foreach (string agentType in agent.PossibleTypes) { jsonWriter.WriteValue(agentType); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Nodes"); foreach (BgNodeDef node in nodes) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", node.Name); jsonWriter.WriteValue("DependsOn", String.Join(";", node.GetDirectOrderDependencies().Where(x => nodesToExecute.Contains(x)))); jsonWriter.WriteValue("RunEarly", node.RunEarly); jsonWriter.WriteObjectStart("Notify"); jsonWriter.WriteValue("Default", String.Join(";", node.NotifyUsers)); jsonWriter.WriteValue("Submitters", String.Join(";", node.NotifySubmitters)); jsonWriter.WriteValue("Warnings", node.NotifyOnWarnings); jsonWriter.WriteObjectEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); // Write all the badges jsonWriter.WriteArrayStart("Badges"); foreach (BgBadgeDef badge in Badges) { BgNodeDef[] dependencies = badge.Nodes.Where(x => nodesToExecute.Contains(x)).ToArray(); if (dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet directDependencies = new HashSet(dependencies); foreach (BgNodeDef dependency in dependencies) { directDependencies.ExceptWith(dependency.OrderDependencies); } jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", badge.Name); if (!String.IsNullOrEmpty(badge.Project)) { jsonWriter.WriteValue("Project", badge.Project); } if (badge.Change != 0) { jsonWriter.WriteValue("Change", badge.Change); } jsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => dependencies.Contains(x)).Select(x => x.Name))); jsonWriter.WriteValue("DirectDependencies", String.Join(";", directDependencies.Select(x => x.Name))); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); // Write all the triggers and reports. jsonWriter.WriteArrayStart("Reports"); foreach (BgReport report in NameToReport.Values) { BgNodeDef[] dependencies = report.Nodes.Where(x => nodesToExecute.Contains(x)).ToArray(); if (dependencies.Length > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet directDependencies = new HashSet(dependencies); foreach (BgNodeDef dependency in dependencies) { directDependencies.ExceptWith(dependency.OrderDependencies); } jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", report.Name); jsonWriter.WriteValue("AllDependencies", String.Join(";", Agents.SelectMany(x => x.Nodes).Where(x => dependencies.Contains(x)).Select(x => x.Name))); jsonWriter.WriteValue("DirectDependencies", String.Join(";", directDependencies.Select(x => x.Name))); jsonWriter.WriteValue("Notify", String.Join(";", report.NotifyUsers)); jsonWriter.WriteValue("IsTrigger", false); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } } /// /// Export the build graph to a Json file for parsing by Horde /// /// Output file to write public void ExportForHorde(FileReference file) { DirectoryReference.CreateDirectory(file.Directory); using (JsonWriter jsonWriter = new JsonWriter(file.FullName)) { jsonWriter.WriteObjectStart(); jsonWriter.WriteArrayStart("Artifacts"); foreach (BgArtifactDef artifact in Artifacts) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", artifact.Name); if (!String.IsNullOrEmpty(artifact.Type)) { jsonWriter.WriteValue("Type", artifact.Type); } if (!String.IsNullOrEmpty(artifact.Description)) { jsonWriter.WriteValue("Description", artifact.Description); } if (!String.IsNullOrEmpty(artifact.BasePath)) { jsonWriter.WriteValue("BasePath", artifact.BasePath); } if (artifact.Keys.Count > 0) { jsonWriter.WriteArrayStart("Keys"); foreach (string key in artifact.Keys) { jsonWriter.WriteValue(key); } jsonWriter.WriteArrayEnd(); } if (artifact.Metadata.Count > 0) { jsonWriter.WriteArrayStart("Metadata"); foreach (string metadata in artifact.Metadata) { jsonWriter.WriteValue(metadata); } jsonWriter.WriteArrayEnd(); } if (!String.IsNullOrEmpty(artifact.NodeName)) { jsonWriter.WriteValue("NodeName", artifact.NodeName); } if (!String.IsNullOrEmpty(artifact.TagName)) { jsonWriter.WriteValue("OutputName", artifact.TagName); } jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Groups"); foreach (BgAgentDef agent in Agents) { jsonWriter.WriteObjectStart(); jsonWriter.WriteArrayStart("Types"); foreach (string possibleType in agent.PossibleTypes) { jsonWriter.WriteValue(possibleType); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Nodes"); foreach (BgNodeDef node in agent.Nodes) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", node.Name); jsonWriter.WriteValue("RunEarly", node.RunEarly); jsonWriter.WriteValue("Warnings", node.NotifyOnWarnings); jsonWriter.WriteArrayStart("Inputs"); foreach (BgNodeOutput input in node.Inputs) { jsonWriter.WriteValue(input.TagName); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Outputs"); foreach (BgNodeOutput output in node.Outputs) { jsonWriter.WriteValue(output.TagName); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("InputDependencies"); foreach (string inputDependency in node.GetDirectInputDependencies().Select(x => x.Name)) { jsonWriter.WriteValue(inputDependency); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("OrderDependencies"); foreach (string orderDependency in node.GetDirectOrderDependencies().Select(x => x.Name)) { jsonWriter.WriteValue(orderDependency); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectStart("Annotations"); foreach ((string key, string value) in node.Annotations) { jsonWriter.WriteValue(key, value); } jsonWriter.WriteObjectEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Aggregates"); foreach (BgAggregateDef aggregate in NameToAggregate.Values) { jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", aggregate.Name); jsonWriter.WriteArrayStart("Nodes"); foreach (BgNodeDef requiredNode in aggregate.RequiredNodes.OrderBy(x => x.Name)) { jsonWriter.WriteValue(requiredNode.Name); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Labels"); foreach (BgLabelDef label in Labels) { jsonWriter.WriteObjectStart(); if (!String.IsNullOrEmpty(label.DashboardName)) { jsonWriter.WriteValue("Name", label.DashboardName); } if (!String.IsNullOrEmpty(label.DashboardCategory)) { jsonWriter.WriteValue("Category", label.DashboardCategory); } if (!String.IsNullOrEmpty(label.UgsBadge)) { jsonWriter.WriteValue("UgsBadge", label.UgsBadge); } if (!String.IsNullOrEmpty(label.UgsProject)) { jsonWriter.WriteValue("UgsProject", label.UgsProject); } if (label.Change != BgLabelChange.Current) { jsonWriter.WriteValue("Change", label.Change.ToString()); } jsonWriter.WriteArrayStart("RequiredNodes"); foreach (BgNodeDef requiredNode in label.RequiredNodes.OrderBy(x => x.Name)) { jsonWriter.WriteValue(requiredNode.Name); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("IncludedNodes"); foreach (BgNodeDef includedNode in label.IncludedNodes.OrderBy(x => x.Name)) { jsonWriter.WriteValue(includedNode.Name); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } jsonWriter.WriteArrayEnd(); jsonWriter.WriteArrayStart("Badges"); foreach (BgBadgeDef badge in Badges) { HashSet dependencies = badge.Nodes; if (dependencies.Count > 0) { // Reduce that list to the smallest subset of direct dependencies HashSet directDependencies = new HashSet(dependencies); foreach (BgNodeDef dependency in dependencies) { directDependencies.ExceptWith(dependency.OrderDependencies); } jsonWriter.WriteObjectStart(); jsonWriter.WriteValue("Name", badge.Name); if (!String.IsNullOrEmpty(badge.Project)) { jsonWriter.WriteValue("Project", badge.Project); } if (badge.Change != 0) { jsonWriter.WriteValue("Change", badge.Change); } jsonWriter.WriteValue("Dependencies", String.Join(";", directDependencies.Select(x => x.Name))); jsonWriter.WriteObjectEnd(); } } jsonWriter.WriteArrayEnd(); jsonWriter.WriteObjectEnd(); } } /// /// Print the contents of the graph /// /// Set of nodes which are already complete /// Options for how to print the graph /// public void Print(HashSet completedNodes, GraphPrintOptions printOptions, ILogger logger) { // Print the options if ((printOptions & GraphPrintOptions.ShowCommandLineOptions) != 0) { // Get the list of messages List> parameters = new List>(); foreach (BgOptionDef option in Options) { string name = String.Format("-set:{0}=...", option.Name); StringBuilder description = new StringBuilder(option.Description); string? defaultValue = null; if (option is BgBoolOptionDef boolOption) { defaultValue = boolOption.DefaultValue.ToString(); } else if (option is BgIntOptionDef intOption) { defaultValue = intOption.DefaultValue.ToString(); } else if (option is BgStringOptionDef stringOption) { defaultValue = stringOption.DefaultValue; } if (!String.IsNullOrEmpty(defaultValue)) { description.AppendFormat(" (Default: {0})", defaultValue); } parameters.Add(new KeyValuePair(name, description.ToString())); } // Format them to the log if (parameters.Count > 0) { logger.LogInformation(""); logger.LogInformation("Options:"); logger.LogInformation(""); List lines = new List(); HelpUtils.FormatTable(parameters, 4, 24, HelpUtils.WindowWidth - 20, lines); foreach (string line in lines) { logger.Log(LogLevel.Information, "{Line}", line); } } } // Output all the triggers in order logger.LogInformation(""); logger.LogInformation("Graph:"); foreach (BgAgentDef agent in Agents) { logger.LogInformation(" Agent: {Name} ({Types})", agent.Name, String.Join(";", agent.PossibleTypes)); foreach (BgNodeDef node in agent.Nodes) { logger.LogInformation(" Node: {Name}{Type}", node.Name, completedNodes.Contains(node) ? " (completed)" : node.RunEarly ? " (early)" : ""); if (printOptions.HasFlag(GraphPrintOptions.ShowDependencies)) { HashSet inputDependencies = new HashSet(node.GetDirectInputDependencies()); foreach (BgNodeDef inputDependency in inputDependencies) { logger.LogInformation(" input> {Name}", inputDependency.Name); } HashSet orderDependencies = new HashSet(node.GetDirectOrderDependencies()); foreach (BgNodeDef orderDependency in orderDependencies.Except(inputDependencies)) { logger.LogInformation(" after> {Name}", orderDependency.Name); } } if (printOptions.HasFlag(GraphPrintOptions.ShowNotifications)) { string label = node.NotifyOnWarnings ? "warnings" : "errors"; foreach (string user in node.NotifyUsers) { logger.LogInformation(" {Name}> {User}", label, user); } foreach (string submitter in node.NotifySubmitters) { logger.LogInformation(" {Name}> submitters to {Submitter}", label, submitter); } } } } logger.LogInformation(""); // Print out all the non-empty aggregates BgAggregateDef[] aggregates = NameToAggregate.Values.OrderBy(x => x.Name).ToArray(); if (aggregates.Length > 0) { logger.LogInformation("Aggregates:"); foreach (string aggregateName in aggregates.Select(x => x.Name)) { logger.LogInformation(" {Aggregate}", aggregateName); } logger.LogInformation(""); } // Print all the produced artifacts BgArtifactDef[] artifacts = Artifacts.OrderBy(x => x.Name).ToArray(); if (artifacts.Length > 0) { logger.LogInformation("Artifacts:"); foreach (BgArtifactDef artifact in artifacts.OrderBy(x => x.Description)) { logger.LogInformation(" {Name}", artifact.Name); } logger.LogInformation(""); } } } /// /// Definition of a graph from bytecode. Can be converted to regular graph definition. /// public class BgGraphExpressionDef { /// /// Nodes for the graph /// public List Nodes { get; } = new List(); /// /// Aggregates for the graph /// public List Aggregates { get; } = new List(); /// /// Creates a graph definition from this template /// /// public BgGraphDef ToGraphDef() { List nodes = new List(Nodes); BgGraphDef graph = new BgGraphDef(); foreach (BgAggregateExpressionDef aggregate in Aggregates) { graph.NameToAggregate[aggregate.Name] = aggregate.ToAggregateDef(); nodes.AddRange(aggregate.RequiredNodes.Select(x => (BgNodeExpressionDef)x)); } HashSet uniqueNodes = new HashSet(); HashSet uniqueAgents = new HashSet(); foreach (BgNodeExpressionDef node in nodes) { RegisterNode(graph, node, uniqueNodes, uniqueAgents); } HashSet labels = new HashSet(); foreach (BgNodeExpressionDef node in nodes) { foreach (BgLabelDef label in node.Labels) { label.RequiredNodes.Add(node); label.IncludedNodes.Add(node); labels.Add(label); } } foreach (BgAggregateExpressionDef aggregate in Aggregates) { if (aggregate.Label != null) { aggregate.Label.RequiredNodes.UnionWith(aggregate.RequiredNodes); aggregate.Label.IncludedNodes.UnionWith(aggregate.RequiredNodes); labels.Add(aggregate.Label); } } graph.Labels.AddRange(labels); return graph; } static void RegisterNode(BgGraphDef graph, BgNodeExpressionDef node, HashSet uniqueNodes, HashSet uniqueAgents) { if (uniqueNodes.Add(node)) { foreach (BgNodeExpressionDef inputNode in node.InputDependencies.OfType()) { RegisterNode(graph, inputNode, uniqueNodes, uniqueAgents); } BgAgentDef agent = node.Agent; if (uniqueAgents.Add(agent)) { graph.Agents.Add(agent); graph.NameToAgent.Add(agent.Name, agent); } agent.Nodes.Add(node); graph.NameToNode.Add(node.Name, node); } } } }