// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using AutomationTool; using UnrealBuildTool; using System.Linq; using EpicGames.Core; namespace Gauntlet { public class TestRequest { public string TestName { get; set; } public List Platforms = new List(); public Params TestParams { get; set; } public override string ToString() { return string.Format("{0}({1})", TestName, string.Join(",", Platforms.Aggregate(new List(), (L, P) => { L.Add(P.Argument); return L; }))); } static public TestRequest CreateRequest(string InName) { return new TestRequest() { TestName = InName, TestParams = new Params(new string[0]) }; } } /// /// Represents options that can be configured on the test context. These options are considered /// global, e.g they apply to all tests executed in a given session and are not exposed to /// invidiual tests to modify /// public class UnrealTestOptions : TestExecutorOptions, IAutoParamNotifiable { /// /// Params for this test run /// public Params Params { get; protected set; } /// /// Name or path to this this project. This will be normalized to the project name /// and the path to the project file available via the ProjectPath argument, /// [AutoParam] public string Project = ""; /// /// Returns the path to the project file. Created based on the Project argument /// public FileReference ProjectPath { get; set; } /// /// Reference to the build that is being tested /// [AutoParamWithNames("Build", "Builds")] public string Build = ""; /// /// Location of the editor build /// [AutoParam] public string EditorDir = ""; /// /// Does this project use 'Game' or Client/Server? /// [AutoParam(true)] public bool UsesSharedBuildType = true; // todo - remove this and pass in BuildSource public Type BuildSourceType; /// /// Configuration to perform tests on /// [AutoParam(UnrealTargetConfiguration.Development)] public UnrealTargetConfiguration Configuration; /// /// Platforms to perform tests on and their params /// public List PlatformList = new List(); /// /// Optional set of platform overrides for a given role /// public Dictionary RolePlatformOverrides = new Dictionary(); /// /// Tests to run and their params. /// public List TestList = new List(); /// /// List of devices to use for tests /// public List DeviceList = new List(); /// /// Service URL of devices to use for tests /// [AutoParam("")] public string DeviceURL; /// /// Maximum number of available local(host platform) devices /// [AutoParam(10)] public int MaxLocalDevices; /// /// Maximum number of available virtual devices of each type (supported on the host platform) /// [AutoParam(2)] public int MaxVirtualDevices; /// /// Details for current job (example: link to CIS, etc) /// [AutoParam("")] public string JobDetails; /// /// Comma-separated list of namespaces to check for tests. E.g. /// with test 'Foo' and namespace 'Bar' then Bar.Foo and Foo will /// be checked for. /// [AutoParam("")] public string Namespaces; public IEnumerable SearchPaths; /// /// Directory to store temp files (including builds that need to be run locally) /// [AutoParam("")] public string Sandbox { get; set; } /// /// Skip any check or copying of builds. Mostly useful when debugging /// [AutoParam(false)] public bool Attended { get; set; } /// /// Skip any check or copying of builds. Mostly useful when debugging /// [AutoParam(false)] public bool SkipCopy { get; set; } /// /// 'Dev' - turns on a few options, most useful of which is that local exes will be used if newer /// [AutoParam(false)] public bool Dev { get; set; } /// /// 'LogPSO' - turns on logpso on the client and uploads the files to P: /// [AutoParam(false)] public bool LogPSO { get; set; } /// /// Use a nullrhi /// [AutoParam(false)] public bool NullRHI { get; set; } /// /// Location to store temporary files. Defaults to GauntletTemp in the root of the /// current drive /// [AutoParam("")] public string TempDir; /// /// Location to store log files. Defaults to TEmpDir/Logs /// [AutoParam("")] public string LogDir; /// /// Custom name for the directory under LogDir where artifacts are stored. /// [AutoParam("")] public string ArtifactName; /// /// Psotfix to apply to the artifact folder (e.g. don't replace the whole thing like ArtifactName, just append this string) /// [AutoParam("")] public string ArtifactPostfix; /// /// Add custom module name/role pair to identify custom project target executable /// Format is :+: /// [AutoParam("")] public string CustomModuleRoles { get; set; } /// /// Set the target name of the module name to identify a custom project target executable /// [AutoParam("")] public string Target { get; set; } /// /// Less logging /// [AutoParamWithNames(false, "Verbose", "Gauntlet.Verbose")] public bool Verbose; /// /// Less logging /// [AutoParam(false)] [AutoParamWithNames(false, "VeryVerbose", "Gauntlet.VeryVerbose")] public bool VeryVerbose; public UnrealTestOptions() { // todo, make platform default to the current os Configuration = UnrealTargetConfiguration.Development; SearchPaths = new string[0]; BuildSourceType = typeof(UnrealBuildSource); } /// /// Called after command line params are applied. Perform any checks / fixup /// /// public virtual void ParametersWereApplied(string[] InParams) { // save params this.Params = new Params(InParams); if (string.IsNullOrEmpty(TempDir)) { TempDir = Globals.TempDir; } else { Globals.TempDir = TempDir; } if (string.IsNullOrEmpty(LogDir)) { LogDir = Globals.LogDir; } else { Globals.LogDir = LogDir; } // Normalize paths LogDir = Path.GetFullPath(LogDir); TempDir = Path.GetFullPath(TempDir); // normalize the project name and get the path if (File.Exists(Project)) { ProjectPath = new FileReference(Project); Project = ProjectPath.GetFileNameWithoutExtension(); } else { if (!string.IsNullOrEmpty(Project)) { ProjectPath = ProjectUtils.FindProjectFileFromName(Project); if (ProjectPath == null) { throw new AutomationException("Could not find project file for {0}", Project); } Project = ProjectPath.GetFileNameWithoutExtension(); } } if (string.IsNullOrEmpty(Sandbox)) { Sandbox = Project; } // parse platforms. These will be in the format of Win64(param1,param2='foo,bar') etc // check for old-style -platform=Win64(params) List PlatformArgStrings = Params.ParseValues("Platform="); // check for convenience flags of -Win64(params) (TODO - need to think about this..) /*foreach (UnrealTargetPlatform Plat in UnrealTargetPlatform.GetValidPlatforms()) { IEnumerable RawPlatformArgs = InParams.Where(P => P.ToLower().StartsWith(Plat.ToString().ToLower())); PlatformArgStrings.AddRange(RawPlatformArgs); }*/ // now turn the Plat(param1,param2) etc into an argument/parm pair PlatformList = PlatformArgStrings.SelectMany(P => ArgumentWithParams.CreateFromString(P)).ToList(); List TestArgStrings = Params.ParseValues("test="); // clear all tests incase this is called multiple times TestList.Clear(); if (TestArgStrings.Count == 0) { TestArgStrings = Params.ParseValues("tests="); } foreach (string TestArg in TestArgStrings) { foreach (ArgumentWithParams TestWithParms in ArgumentWithParams.CreateFromString(TestArg)) { TestRequest Test = new TestRequest() { TestName = TestWithParms.Argument, TestParams = new Params(TestWithParms.AllArguments) }; // parse any specified platforms foreach (string PlatformArg in Test.TestParams.ParseValues("Platform=")) { List PlatParams = ArgumentWithParams.CreateFromString(PlatformArg); Test.Platforms.AddRange(PlatParams); // register platform in test options PlatParams.ForEach(TestPlat => { if (PlatformList.Where(Plat => Plat.Argument == TestPlat.Argument).Count() == 0) PlatformList.Add(TestPlat); }); } TestList.Add(Test); } } if (PlatformList.Count == 0) { // Default to local platform PlatformList.Add(new ArgumentWithParams(BuildHostPlatform.Current.Platform.ToString(), new string[0])); } // do we have any tests? Need to check the global test list bool HaveTests = TestList.Count > 0 || PlatformList.Where(Plat => Plat.ParseValues("test").Count() > 0).Count() > 0; List DeviceArgStrings = Params.ParseValues("device="); if (DeviceArgStrings.Count == 0) { DeviceArgStrings = Params.ParseValues("devices="); } DeviceList = DeviceArgStrings.SelectMany(D => ArgumentWithParams.CreateFromString(D)).ToList(); if (DeviceList.Count == 0) { // Add the default test DeviceList.Add(new ArgumentWithParams("default", new string[0])); } // remote all -test and -platform arguments from our params. Nothing else should be looking at these now... string[] CleanArgs = Params.AllArguments .Where(Arg => !Arg.StartsWith("test=", StringComparison.OrdinalIgnoreCase) && !Arg.StartsWith("platform=", StringComparison.OrdinalIgnoreCase) && !Arg.StartsWith("device=", StringComparison.OrdinalIgnoreCase)) .ToArray(); Params = new Params(CleanArgs); // Custom Module name/role if (!string.IsNullOrEmpty(CustomModuleRoles)) { foreach (var Pair in CustomModuleRoles.Split("+")) { var SplittedPair = Pair.Split(":"); if (SplittedPair.Length > 1) { string Name = SplittedPair[0]; UnrealTargetRole Role = UnrealTargetRole.Unknown; if (Enum.TryParse(SplittedPair[1], out Role)) { UnrealHelpers.AddCustomModuleName(Name, Role); } else { throw new AutomationException(string.Format("Target Role '{0}' for Custom Module '{1}' is unknown.", SplittedPair[1], Name)); } } else { Gauntlet.Log.Warning("CustomModuleRoles is poorly formatted. Expected : pair. Got '{0}'", Pair); } } } if(!string.IsNullOrEmpty(Target) && Target != Project) { bool IsEditor = Build.Equals("Editor", StringComparison.InvariantCultureIgnoreCase) || Globals.Params.ParseParam("editor"); UnrealTargetRole Role = IsEditor? UnrealTargetRole.Editor : UnrealTargetRole.Client; UnrealHelpers.AddCustomModuleName(IsEditor? string.Format("{0}Editor", Target): Target, Role); } } } /// /// This class describes the "context" that is used for a given role. E.g. if the test specifies it needs a Client role /// then this class determines what that actually means, and even whether it's really a client or a client emulated /// by running the editor with -game /// /// This mappinf of role->context is performed by the TestContext so this structure holds the absolute truth of how /// a client is configured /// /// public class UnrealTestRoleContext : ICloneable { /// /// Role of this role :) /// public UnrealTargetRole Type; /// /// Platform that this role is run under /// public UnrealTargetPlatform Platform; /// /// Configuration that this role is run under /// public UnrealTargetConfiguration Configuration; /// /// Extra args to use for this role /// public string ExtraArgs; /// /// Are these roles actually skipped during tests? Useful if you need to manually start /// a role when debugging etc /// public bool Skip; public object Clone() { // no deep copy needed return this.MemberwiseClone(); } public override string ToString() { string Description = string.Format("{0} {1} {2}", Platform, Configuration, Type); return Description; } }; /// /// This class contains the Context - build and global options - that one or more tests are going to /// be executed under. /// public class UnrealTestContext : ITestContext, ICloneable { // Begin ITestContext implementation public UnrealBuildSource BuildInfo { get; private set; } // Worker Job ID (generates unique node results and logs in parallel runs) public string WorkerJobID; /// /// Global options for this test /// public UnrealTestOptions Options { get; set; } public Params TestParams { get; set; } public UnrealTestRoleContext GetRoleContext(UnrealTargetRole Role) { return RoleContext[Role]; } public Dictionary RoleContext { get; set; } /// /// Target constraint that this test is run under /// public UnrealDeviceTargetConstraint Constraint; public UnrealTestContext(UnrealBuildSource InBuildInfo, Dictionary InRoleContexts, UnrealTestOptions InOptions) { //BuildRepository = InBuildRepository; BuildInfo = InBuildInfo; Options = InOptions; TestParams = new Params(new string[0]); RoleContext = InRoleContexts; } public object Clone() { UnrealTestContext Copy = (UnrealTestContext)this.MemberwiseClone(); // todo - what else shou;d be unique? Copy.RoleContext = this.RoleContext.ToDictionary(entry => entry.Key, entry => (UnrealTestRoleContext)entry.Value.Clone()); return Copy; } public override string ToString() { return ToString(false); } public string ToString(bool bWithServerType = false) { string Description = string.Format("{0}", RoleContext[UnrealTargetRole.Client]); if (WorkerJobID != null) { Description += " " + WorkerJobID; } if (bWithServerType) { Description += ", " + RoleContext[UnrealTargetRole.Server].ToString(); } return Description; } } }