// Copyright Epic Games, Inc. All Rights Reserved. // This software is provided "as-is," without any express or implied warranty. // In no event shall the author, nor Epic Games, Inc. be held liable for any damages arising from the use of this software. // This software will not be supported. // Use at your own risk. using System; using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Diagnostics; using System.Reflection; using EpicGames.Core; using System.IO; using System.Collections.Generic; using UnrealBuildBase; using Microsoft.Extensions.Logging; namespace AutomationToolDriver { /// /// Main entry point /// public partial class Program { /// /// Parses command line parameter. /// /// Parameter /// Recently parsed command /// Logger for output /// True if the parameter has been successfully parsed. private static void ParseParam(string CurrentParam, CommandInfo CurrentCommand, ILogger Logger) { if (AutomationToolCommandLine.IsParameterIgnored(CurrentParam)) { return; } bool bGlobalParam = AutomationToolCommandLine.TrySetGlobal(CurrentParam); // Global value parameters, handled explicitly string Option_ScriptsForProject = "-ScriptsForProject"; string Option_ScriptDir = "-ScriptDir"; string Option_Telemetry = "-Telemetry"; string Option_WaitForStdStreams = "-WaitForStdStreams"; // The parameter was not found in the list of global parameters, continue looking... if (CurrentParam.StartsWith(Option_ScriptsForProject + "=", StringComparison.InvariantCultureIgnoreCase)) { if (AutomationToolCommandLine.IsSetUnchecked(Option_ScriptsForProject)) { throw new Exception("The -ScriptsForProject argument may only be specified once"); } string ProjectFileName = CurrentParam.Substring(CurrentParam.IndexOf('=') + 1).Replace("\"", ""); FileReference ProjectReference = NativeProjectsBase.FindProjectFile(ProjectFileName, Logger); if (ProjectReference != null) { AutomationToolCommandLine.SetUnchecked(Option_ScriptsForProject, ProjectReference.FullName); Logger.LogDebug("Found project file: {0}", ProjectReference.FullName); } else if (Path.IsPathFullyQualified(ProjectFileName)) { throw new Exception($"Project '{ProjectFileName}' does not exist"); } else { throw new Exception($"Project '{ProjectFileName}' does not exist relative to any entries in *.uprojectdirs"); } } else if (CurrentParam.StartsWith(Option_ScriptDir + "=", StringComparison.InvariantCultureIgnoreCase)) { string ScriptDir = CurrentParam.Substring(CurrentParam.IndexOf('=') + 1); if (Directory.Exists(ScriptDir)) { List OutAdditionalScriptDirectories = (List)AutomationToolCommandLine.GetValueUnchecked(Option_ScriptDir) ?? new List(); OutAdditionalScriptDirectories.Add(Path.GetFullPath(ScriptDir)); AutomationToolCommandLine.SetUnchecked(Option_ScriptDir, OutAdditionalScriptDirectories); Logger.LogDebug("Found additional script dir: {0}", Path.GetFullPath(ScriptDir)); } else { DirectoryReference ScriptDirReference = NativeProjectsBase.FindRelativeDirectoryReference(ScriptDir, Logger); if (ScriptDirReference != null) { List OutAdditionalScriptDirectories = (List)AutomationToolCommandLine.GetValueUnchecked(Option_ScriptDir) ?? new List(); OutAdditionalScriptDirectories.Add(ScriptDirReference.FullName); AutomationToolCommandLine.SetUnchecked(Option_ScriptDir, OutAdditionalScriptDirectories); Logger.LogDebug("Found additional script dir: {0}", ScriptDirReference.FullName); } else if (Path.IsPathFullyQualified(ScriptDir)) { throw new Exception($"Specified ScriptDir doesn't exist: {ScriptDir}"); } else { throw new Exception($"Specified ScriptDir doesn't exist relative to any entries in *.uprojectdirs: {ScriptDir}"); } } } else if (CurrentParam.StartsWith(Option_Telemetry + "=", StringComparison.InvariantCultureIgnoreCase)) { string TelemetryPath = CurrentParam.Substring(CurrentParam.IndexOf('=') + 1); AutomationToolCommandLine.SetUnchecked(Option_Telemetry, TelemetryPath); } else if (CurrentParam.StartsWith(Option_WaitForStdStreams + "=", StringComparison.InvariantCultureIgnoreCase)) { string WaitTime = CurrentParam.Substring(CurrentParam.IndexOf('=') + 1); AutomationToolCommandLine.SetUnchecked(Option_WaitForStdStreams, WaitTime); } else if (CurrentParam.StartsWith("-")) { if (CurrentCommand != null) { CurrentCommand.Arguments.Add(CurrentParam.Substring(1)); } else if (!bGlobalParam) { throw new Exception($"Unknown parameter {CurrentParam} in the command line that does not belong to any command."); } } else if (CurrentParam.Contains("=")) { // Environment variable int ValueStartIndex = CurrentParam.IndexOf('=') + 1; string EnvVarName = CurrentParam.Substring(0, ValueStartIndex - 1); if (String.IsNullOrEmpty(EnvVarName)) { throw new Exception($"Unable to parse environment variable that has no name. Error when parsing command line param {CurrentParam}"); } string EnvVarValue = CurrentParam.Substring(ValueStartIndex); Logger.LogDebug($"SetEnvVar {EnvVarName}={EnvVarValue}"); Environment.SetEnvironmentVariable(EnvVarName, EnvVarValue); } } private static string ParseString(string Key, string Value) { if (!String.IsNullOrEmpty(Key)) { if (Value == "true" || Value == "false") { return "-" + Key; } else { string param = "-" + Key + "="; if (Value.Contains(" ")) { param += "\"" + Value + "\""; } else { param += Value; } return param; } } else { return Value; } } private static string ParseList(string Key, List Value) { string param = "-" + Key + "="; bool bStart = true; foreach (var Val in Value) { if (!bStart) { param += "+"; } param += Val as string; bStart = false; } return param; } private static void ParseDictionary(Dictionary Value, List Arguments) { foreach (var Pair in Value) { if ((Pair.Value as string) != null && !string.IsNullOrEmpty(Pair.Value as string)) { Arguments.Add(ParseString(Pair.Key, Pair.Value as string)); } else if (Pair.Value.GetType() == typeof(bool)) { if ((bool)Pair.Value) { Arguments.Add("-" + Pair.Key); } } else if ((Pair.Value as List) != null) { Arguments.Add(ParseList(Pair.Key, Pair.Value as List)); } else if ((Pair.Value as Dictionary) != null) { string param = "-" + Pair.Key + "=\""; List Args = new List(); ParseDictionary(Pair.Value as Dictionary, Args); bool bStart = true; foreach (var Arg in Args) { if (!bStart) { param += " "; } param += Arg.Replace("\"", "\'"); bStart = false; } param += "\""; Arguments.Add(param); } } } private static void ParseProfile(ref string[] CommandLine) { // find if there is a profile file to read string Profile = ""; List Arguments = new List(); for (int Index = 0; Index < CommandLine.Length; ++Index) { if (CommandLine[Index].StartsWith("-profile=")) { Profile = CommandLine[Index].Substring(CommandLine[Index].IndexOf('=') + 1); } else { Arguments.Add(CommandLine[Index]); } } if (!string.IsNullOrEmpty(Profile)) { if (File.Exists(Profile)) { // find if the command has been specified var text = File.ReadAllText(Profile); var RawObject = fastJSON.JSON.Instance.Parse(text) as Dictionary; var Params = RawObject["scripts"] as List; foreach (var Script in Params) { string ScriptName = (Script as Dictionary)["script"] as string; if (!string.IsNullOrEmpty(ScriptName) && !Arguments.Contains(ScriptName)) { Arguments.Add(ScriptName); } (Script as Dictionary).Remove("script"); ParseDictionary((Script as Dictionary), Arguments); } } } CommandLine = Arguments.ToArray(); } /// /// Parse the command line and create a list of commands to execute. /// /// Command line /// Logger for output public static void ParseCommandLine(string[] Arguments, ILogger Logger) { AutomationToolCommandLine = new ParsedCommandLine( new Dictionary(StringComparer.InvariantCultureIgnoreCase) { {"-Verbose", "Enables verbose logging"}, {"-VeryVerbose", "Enables very verbose logging"}, {"-TimeStamps", ""}, {"-Submit", "Allows UAT command to submit changes"}, {"-NoSubmit", "Prevents any submit attempts"}, {"-NoP4", "Disables Perforce functionality {default if not run on a build machine}"}, {"-P4", "Enables Perforce functionality {default if run on a build machine}"}, {"-IgnoreDependencies", ""}, {"-Help", "Displays help"}, {"-List", "Lists all available commands"}, {"-NoKill", "Does not kill any spawned processes on exit"}, {"-UTF8Output", ""}, {"-AllowStdOutLogVerbosity", ""}, {"-NoAutoSDK", ""}, {"-Compile", "Force all script modules to be compiled"}, {"-NoCompile", "Do not attempt to compile any script modules - attempts to run with whatever is up to date" }, {"-IgnoreBuildRecords", "Ignore build records (Intermediate/ScriptModule/ProjectName.json) files when determining if script modules are up to date" }, {"-UseLocalBuildStorage", @"Allows you to use local storage for your root build storage dir {default of P:\Builds {on PC} is changed to Engine\Saved\LocalBuilds}. Used for local testing."}, {"-WaitForDebugger", "Waits for a debugger to be attached, and breaks once debugger successfully attached."}, {"-BuildMachine", "" }, {"-WaitForUATMutex", "" }, {"-WaitForStdStreams", "Time in milliseconds to wait for std streams to close in child processes." } }, new HashSet(StringComparer.InvariantCultureIgnoreCase) { "-msbuild-verbose", "-NoCompileUAT" } ); ParseProfile(ref Arguments); Logger.LogInformation("Parsing command line: {CommandLine}", CommandLine.FormatCommandLine(Arguments)); CommandInfo CurrentCommand = null; for (int Index = 0; Index < Arguments.Length; ++Index) { // Guard against empty arguments passed as "" on the command line string Param = Arguments[Index]; if(Param.Length > 0) { if (Param.StartsWith("-") || Param.Contains("=")) { ParseParam(Arguments[Index], CurrentCommand, Logger); } else { CurrentCommand = new CommandInfo(Arguments[Index]); AutomationToolCommandLine.CommandsToExecute.Add(CurrentCommand); } } } // Validate var Result = AutomationToolCommandLine.CommandsToExecute.Count > 0 || AutomationToolCommandLine.IsSetGlobal("-Help") || AutomationToolCommandLine.IsSetGlobal("-List"); if (AutomationToolCommandLine.CommandsToExecute.Count > 0) { Logger.LogDebug("Found {NumScripts} scripts to execute:", AutomationToolCommandLine.CommandsToExecute.Count); foreach (CommandInfo Command in AutomationToolCommandLine.CommandsToExecute) { Logger.LogDebug(" {Command}", Command.ToString()); } } else if (!Result) { throw new Exception("Failed to find scripts to execute in the command line params."); } if (AutomationToolCommandLine.IsSetGlobal("-NoP4") && AutomationToolCommandLine.IsSetGlobal("-P4")) { throw new Exception("'-NoP4' and '-P4' can't be set simultaneously."); } if (AutomationToolCommandLine.IsSetGlobal("-NoSubmit") && AutomationToolCommandLine.IsSetGlobal("-Submit")) { throw new Exception("'-NoSubmit' and '-Submit' can't be set simultaneously."); } } static ParsedCommandLine AutomationToolCommandLine; static StartupTraceListener StartupListener = new StartupTraceListener(); /// /// Main entry point /// /// Do not add [STAThread] here. It will cause deadlocks in platform automation code. public static async Task Main(string[] Arguments) { ILogger Logger = Log.Logger; // Initialize the log system, buffering the output until we can create the log file Log.AddTraceListener(StartupListener); Logger.LogInformation("Starting AutomationTool..."); // Populate AutomationToolCommandLine and CommandsToExecute try { ParseCommandLine(Arguments, Logger); } catch (Exception Ex) { Logger.LogError(Ex, "ERROR: " + Ex.Message); return (int)ExitCode.Error_Arguments; } // Wait for a debugger to be attached if (AutomationToolCommandLine.IsSetGlobal("-WaitForDebugger")) { Console.WriteLine("Waiting for debugger to be attached..."); while (Debugger.IsAttached == false) { Thread.Sleep(100); } Debugger.Break(); } Stopwatch Timer = Stopwatch.StartNew(); // Ensure UTF8Output flag is respected, since we are initializing logging early in the program. if (AutomationToolCommandLine.IsSetGlobal("-UTF8Output")) { Console.OutputEncoding = new System.Text.UTF8Encoding(false, false); } // Parse the log level argument if (AutomationToolCommandLine.IsSetGlobal("-Verbose")) { Log.OutputLevel = LogEventType.Verbose; } if (AutomationToolCommandLine.IsSetGlobal("-VeryVerbose")) { Log.OutputLevel = LogEventType.VeryVerbose; } // Configure log timestamps Log.IncludeTimestamps = AutomationToolCommandLine.IsSetGlobal("-Timestamps"); // Configure the structured logging event parser with matchers from UBT Assembly UnrealBuildToolAssembly = typeof(UnrealBuildTool.BuildVersion).Assembly; Log.EventParser.AddMatchersFromAssembly(UnrealBuildToolAssembly); // when running frmo RunUAT.sh (Mac/Linux) we need to install a Ctrl-C handler, or hitting Ctrl-C from a terminal // can leave dotnet process in a zombie state (some order of process destruction is failing) // by putting this in, the Ctrl-C may not be handled immediately, but it shouldn't leave a zombie process if (OperatingSystem.IsMacOS() || OperatingSystem.IsLinux()) { Console.CancelKeyPress += delegate { Console.WriteLine("AutomationTool: Ctrl-C pressed. Exiting..."); }; } // Enter the main program section ExitCode ReturnCode = ExitCode.Error_Unknown; try { // Set the working directory to the Unreal root directory DirectoryReference.SetCurrentDirectory(Unreal.RootDirectory); // Ensure we can resolve any external assemblies as necessary. DirectoryReference AutomationToolLocation = new DirectoryReference(Path.GetDirectoryName(Assembly.GetEntryAssembly().GetOriginalLocation())); AssemblyUtils.InstallAssemblyResolver(AutomationToolLocation.FullName); AssemblyUtils.InstallRecursiveAssemblyResolver(AutomationToolLocation.FullName); // Log the operating environment. Since we usually compile to AnyCPU, we may be executed using different system paths under WOW64. Logger.LogDebug("Running on {Platform} as a {Bitness}-bit process.", RuntimePlatform.Current.ToString(), Environment.Is64BitProcess ? 64 : 32); // Log if we're running from the launcher string ExecutingAssemblyLocation = Assembly.GetExecutingAssembly().Location; if (string.Compare(ExecutingAssemblyLocation, Assembly.GetEntryAssembly().GetOriginalLocation(), StringComparison.OrdinalIgnoreCase) != 0) { Logger.LogDebug("Executed from AutomationToolLauncher ({Location})", ExecutingAssemblyLocation); } Logger.LogDebug("CWD={Cwd}", Environment.CurrentDirectory); // Log the application version FileVersionInfo Version = AssemblyUtils.ExecutableVersion; Logger.LogDebug("{ProductName} ver. {ProductVersion}", Version.ProductName, Version.ProductVersion); bool bWaitForUATMutex = AutomationToolCommandLine.IsSetGlobal("-WaitForUATMutex"); // Don't allow simultaneous execution of AT (in the same branch) ReturnCode = await ProcessSingleton.RunSingleInstanceAsync(MainProc, bWaitForUATMutex, Log.Logger); } catch (Exception Ex) { Logger.LogError(Ex, "Unhandled exception: {Message}", ExceptionUtils.FormatException(Ex)); } finally { // If there was an error before a log writer was attached, do so now if (!Log.HasFileWriter() && ReturnCode != ExitCode.Success) { FileReference errorLogFile = FileReference.Combine(Unreal.WritableEngineDirectory, "Programs/AutomationTool/Saved/Logs/ErrorLog.txt"); Log.AddFileWriterWithoutBackup("UnhandledFileLogListener", errorLogFile); Logger.LogError("AutomationTool encountered an error before a log file was set, see '{File}' for more details", errorLogFile); } // Write the exit code Logger.LogInformation("AutomationTool executed for {Time}", Timer.Elapsed.ToString("h'h 'm'm 's's'")); Logger.LogInformation("AutomationTool exiting with ExitCode={ExitCode} ({ExitReason})", (int)ReturnCode, ReturnCode); // Can't use NoThrow here because the code logs exceptions. We're shutting down logging! Trace.Close(); } return (int)ReturnCode; } static async Task MainProc() { ILogger Logger = Log.Logger; Logger.LogInformation("Initializing script modules..."); var StartTime = DateTime.UtcNow; string ScriptsForProject = (string)AutomationToolCommandLine.GetValueUnchecked("-ScriptsForProject"); List AdditionalScriptDirs = (List) AutomationToolCommandLine.GetValueUnchecked("-ScriptDir"); bool bForceCompile = AutomationToolCommandLine.IsSetGlobal("-Compile"); bool bNoCompile = AutomationToolCommandLine.IsSetGlobal("-NoCompile"); bool bUseBuildRecords = !AutomationToolCommandLine.IsSetGlobal("-IgnoreBuildRecords"); List Commands = AutomationToolCommandLine.IsSetGlobal("-List") ? null : AutomationToolCommandLine.CommandsToExecute; bool bBuildSuccess; HashSet ScriptModuleAssemblyPaths = CompileScriptModule.InitializeScriptModules( Rules.RulesFileType.AutomationModule, ScriptsForProject, AdditionalScriptDirs, bForceCompile, bNoCompile, bUseBuildRecords, out bBuildSuccess, (int Count) => { Logger.LogInformation("Building {Count} projects...", Count); }, Log.Logger); if (!bBuildSuccess) { return ExitCode.Error_Unknown; } // when the engine is installed, or UAT is invoked with -NoCompile, we expect to find at least one script module (AutomationUtils is a necessity) if (ScriptModuleAssemblyPaths.Count == 0) { throw new Exception("Found no script module records."); } // Load AutomationUtils.Automation.dll FileReference AssemblyPath = ScriptModuleAssemblyPaths.FirstOrDefault(x => x.GetFileNameWithoutExtension().Contains("AutomationUtils.Automation")); AssemblyUtils.InstallRecursiveAssemblyResolver(AssemblyPath.Directory.FullName); Assembly AutomationUtilsAssembly = AssemblyPath != null ? Assembly.LoadFrom(AssemblyPath.FullName) : null; if (AutomationUtilsAssembly == null) { throw new Exception("Did not find an AutomationUtils.Automation.dll"); } // Call into AutomationTool.Automation.Process() Type AutomationTools_Automation = AutomationUtilsAssembly.GetType("AutomationTool.Automation"); MethodInfo Automation_Process = AutomationTools_Automation.GetMethod("ProcessAsync"); Logger.LogInformation("Total script module initialization time: {InitTime:0.00} s.", (DateTime.UtcNow - StartTime).TotalMilliseconds / 1000); return await (Task) Automation_Process.Invoke(null, new object[] {AutomationToolCommandLine, StartupListener, ScriptModuleAssemblyPaths}); } } }