// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.Linq; using EpicGame; using UnrealBuildBase; using Gauntlet; using Microsoft.Extensions.Logging; using Log = EpicGames.Core.Log; namespace AutomatedPerfTest { public class AutomatedPerfTestConfigBase : UnrealTestConfiguration { /// /// Used to override the test controller portion of the data source name /// Will be appended to Automation.ProjectName to construct the full data source name for the test /// can be overridden with AutomatedPerfTest.DataSourceNameOverride /// [AutoParamWithNames("AutomatedPerfTest.DataSourceTypeOverride")] public string DataSourceTypeOverride = ""; /// /// Fully override the data source name /// [AutoParamWithNames("AutomatedPerfTest.DataSourceNameOverride")] public string DataSourceNameOverride = ""; /// /// Name of the test, useful for identifying it later /// [AutoParamWithNames("AutomatedPerfTest.TestID")] public string TestID = ""; /// /// If set, will prepend platform name and use this device profile instead of the default /// [AutoParamWithNames("AutomatedPerfTest.DeviceProfileOverride")] public string DeviceProfileOverride = ""; /// /// If data from the test should be gathered with Unreal Insights /// [AutoParamWithNames(false, "AutomatedPerfTest.DoInsightsTrace")] public bool DoInsightsTrace; /// /// Which trace channels to test with /// [AutoParamWithNames("AutomatedPerfTest.TraceChannels")] public string TraceChannels = "default,screenshot,stats"; /// /// If data from the test should be gathered with the CSV Profiler /// [AutoParamWithNames(false, "AutomatedPerfTest.DoCSVProfiler")] public bool DoCSVProfiler; /// /// If data from the test should be gathered with FPS Charts /// [AutoParamWithNames(false, "AutomatedPerfTest.DoFPSChart")] public bool DoFPSChart; /// /// Whether to run the test with -llm /// [AutoParamWithNames(false, "AutomatedPerfTest.DoLLM")] public bool DoLLM; /// /// Whether to run GPU perf test, that is, dynres is fixed, while GPU async work gets reduced, serializing passes to be timed separately /// [AutoParamWithNames(false, "AutomatedPerfTest.DoGPUPerf")] public bool DoGPUPerf; /// /// Whether to run GPU Reshape's instrumentation on the target /// [AutoParamWithNames(false, "AutomatedPerfTest.DoGPUReshape")] public bool DoGPUReshape; /// /// The assigned GPU Reshape workspace /// [AutoParamWithNames("BasicWorkspace", "AutomatedPerfTest.GPUReshapeWorkspace")] public string GPUReshapeWorkspace; /// /// Whether to lock dynamic resolution. This is applied automatically when GPUPerf test is run. /// [AutoParamWithNames(false,"AutomatedPerfTest.LockDynamicRes")] public bool LockDynamicRes; /// /// Path where Perf Test artifacts (Logs, CSVs, Reports, Cache) will be stored. /// [AutoParamWithNames("", "AutomatedPerfTest.PerfOutputPath")] public string PerfOutputPath; /// /// Let BuildGraph tell us where we should output the Insights trace after running a test so that we know where to grab it from when we're done /// [AutoParamWithNames("", "AutomatedPerfTest.PerfCacheRoot")] public string PerfCacheRoot; /// /// Path to a JSON file with ignored issues (ensures, warnings, errros). Can be used to suppress hard-to-fix issues, on a per-branch basis /// [AutoParamWithNames("", "AutomatedPerfTest.IgnoredIssuesConfigAbsPath")] public string IgnoredIssuesConfigAbsPath; /// /// If we should trigger a video capture /// [AutoParamWithNames(false, "AutomatedPerfTest.DoVideoCapture")] public bool DoVideoCapture; /// /// If true, will look for the shipping configuration of Unreal Insights in order to parse Insights trace files /// This should be True unless you need to test an issue in parsing the Insights file itself. /// [AutoParamWithNames(false, "AutomatedPerfTest.UseShippingInsights")] public bool UseShippingInsights; /// /// Used to find the report type and report graph xmls for generating local reports /// Defaults to those provided in the AutomatedPerfTesting plugin /// [AutoParamWithNames("", "AutomatedPerfTest.ReportConfigDir")] public string ReportConfigDir; /// /// Used to specify where performance reports should be saved /// [AutoParamWithNames("", "AutomatedPerfTest.ReportPath")] public string ReportPath; /// /// Specify whether this test requires additional debug memory /// on target device. The allocation of this memory is entirely /// dependent on the target device and may not be guaranteed. /// [AutoParamWithNames(false, "AutomatedPerfTest.UsePlatformExtraDebugMemory")] public bool UsePlatformExtraDebugMemory; /// /// If true, logs will not be considered when passing or failing /// a test in Test builds. Useful if logging is disabled in Test /// builds and we do not want to fail the test if no logs are /// found. /// [AutoParamWithNames(false, "AutomatedPerfTest.IgnoreTestBuildLogging")] public bool IgnoreTestBuildLogging; /// /// Used to specify a different test name in the CSV meta data, primarily used to distinguish between RHIs /// [AutoParamWithNames("", "AutomatedPerfTest.TestConfigName")] public string TestConfigName = ""; public string DataSourceName; /// /// Call this in the test node's GetConfiguration function to set the DataSourceName used for later data processing /// /// Name of the project /// Which test controller is being used to generate the data /// the properly formatted data source name public string GetDataSourceName(string ProjectName, string DataSourceType="") { // if the name has been fully overridden by the calling process, just use that if (!string.IsNullOrEmpty(DataSourceNameOverride)) { return DataSourceNameOverride; } // otherwise, if the data source type has been override, use either that or the one passed into this function if (string.IsNullOrEmpty(DataSourceType) && string.IsNullOrEmpty(DataSourceTypeOverride)) { Log.Logger.LogError("No DataSourceType or DataSourceTypeOverride has been provided."); return null; } string dataSourceType = string.IsNullOrEmpty(DataSourceTypeOverride) ? DataSourceType : DataSourceTypeOverride; return $"Automation.{ProjectName}.{dataSourceType}"; } public override void ApplyToConfig(UnrealAppConfig AppConfig, UnrealSessionRole ConfigRole, IEnumerable OtherRoles) { base.ApplyToConfig(AppConfig, ConfigRole, OtherRoles); // Let the app know if we need to use extra debug memory before we // instantiate it. AppConfig.UsePlatformExtraDebugMemory = UsePlatformExtraDebugMemory; } } public class AutomatedSequencePerfTestConfig : AutomatedPerfTestConfigBase { /// /// Which map to run the test on /// [AutoParamWithNames("", "AutomatedPerfTest.SequencePerfTest.MapSequenceName")] public string MapSequenceComboName; } public class AutomatedReplayPerfTestConfig : AutomatedPerfTestConfigBase { /// /// Which replay to run the test on /// [AutoParamWithNames("", "AutomatedPerfTest.ReplayPerfTest.ReplayName")] public string ReplayName; } public class AutomatedStaticCameraPerfTestConfig : AutomatedPerfTestConfigBase { /// /// Which map to run the test on /// [AutoParamWithNames("", "AutomatedPerfTest.StaticCameraPerfTest.MapName")] public string MapName; } public class AutomatedMaterialPerfTestConfig : AutomatedPerfTestConfigBase { } }