// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using EpicGame;
using UnrealBuildBase;
using Gauntlet;
using Microsoft.Extensions.Logging;
using Log = EpicGames.Core.Log;
namespace AutomatedPerfTest
{
public class AutomatedPerfTestConfigBase : UnrealTestConfiguration
{
///
/// Used to override the test controller portion of the data source name
/// Will be appended to Automation.ProjectName to construct the full data source name for the test
/// can be overridden with AutomatedPerfTest.DataSourceNameOverride
///
[AutoParamWithNames("AutomatedPerfTest.DataSourceTypeOverride")]
public string DataSourceTypeOverride = "";
///
/// Fully override the data source name
///
[AutoParamWithNames("AutomatedPerfTest.DataSourceNameOverride")]
public string DataSourceNameOverride = "";
///
/// Name of the test, useful for identifying it later
///
[AutoParamWithNames("AutomatedPerfTest.TestID")]
public string TestID = "";
///
/// If set, will prepend platform name and use this device profile instead of the default
///
[AutoParamWithNames("AutomatedPerfTest.DeviceProfileOverride")]
public string DeviceProfileOverride = "";
///
/// If data from the test should be gathered with Unreal Insights
///
[AutoParamWithNames(false, "AutomatedPerfTest.DoInsightsTrace")]
public bool DoInsightsTrace;
///
/// Which trace channels to test with
///
[AutoParamWithNames("AutomatedPerfTest.TraceChannels")]
public string TraceChannels = "default,screenshot,stats";
///
/// If data from the test should be gathered with the CSV Profiler
///
[AutoParamWithNames(false, "AutomatedPerfTest.DoCSVProfiler")]
public bool DoCSVProfiler;
///
/// If data from the test should be gathered with FPS Charts
///
[AutoParamWithNames(false, "AutomatedPerfTest.DoFPSChart")]
public bool DoFPSChart;
///
/// Whether to run the test with -llm
///
[AutoParamWithNames(false, "AutomatedPerfTest.DoLLM")]
public bool DoLLM;
///
/// Whether to run GPU perf test, that is, dynres is fixed, while GPU async work gets reduced, serializing passes to be timed separately
///
[AutoParamWithNames(false, "AutomatedPerfTest.DoGPUPerf")]
public bool DoGPUPerf;
///
/// Whether to run GPU Reshape's instrumentation on the target
///
[AutoParamWithNames(false, "AutomatedPerfTest.DoGPUReshape")]
public bool DoGPUReshape;
///
/// The assigned GPU Reshape workspace
///
[AutoParamWithNames("BasicWorkspace", "AutomatedPerfTest.GPUReshapeWorkspace")]
public string GPUReshapeWorkspace;
///
/// Whether to lock dynamic resolution. This is applied automatically when GPUPerf test is run.
///
[AutoParamWithNames(false,"AutomatedPerfTest.LockDynamicRes")]
public bool LockDynamicRes;
///
/// Path where Perf Test artifacts (Logs, CSVs, Reports, Cache) will be stored.
///
[AutoParamWithNames("", "AutomatedPerfTest.PerfOutputPath")]
public string PerfOutputPath;
///
/// Let BuildGraph tell us where we should output the Insights trace after running a test so that we know where to grab it from when we're done
///
[AutoParamWithNames("", "AutomatedPerfTest.PerfCacheRoot")]
public string PerfCacheRoot;
///
/// Path to a JSON file with ignored issues (ensures, warnings, errros). Can be used to suppress hard-to-fix issues, on a per-branch basis
///
[AutoParamWithNames("", "AutomatedPerfTest.IgnoredIssuesConfigAbsPath")]
public string IgnoredIssuesConfigAbsPath;
///
/// If we should trigger a video capture
///
[AutoParamWithNames(false, "AutomatedPerfTest.DoVideoCapture")]
public bool DoVideoCapture;
///
/// If true, will look for the shipping configuration of Unreal Insights in order to parse Insights trace files
/// This should be True unless you need to test an issue in parsing the Insights file itself.
///
[AutoParamWithNames(false, "AutomatedPerfTest.UseShippingInsights")]
public bool UseShippingInsights;
///
/// Used to find the report type and report graph xmls for generating local reports
/// Defaults to those provided in the AutomatedPerfTesting plugin
///
[AutoParamWithNames("", "AutomatedPerfTest.ReportConfigDir")]
public string ReportConfigDir;
///
/// Used to specify where performance reports should be saved
///
[AutoParamWithNames("", "AutomatedPerfTest.ReportPath")]
public string ReportPath;
///
/// Specify whether this test requires additional debug memory
/// on target device. The allocation of this memory is entirely
/// dependent on the target device and may not be guaranteed.
///
[AutoParamWithNames(false, "AutomatedPerfTest.UsePlatformExtraDebugMemory")]
public bool UsePlatformExtraDebugMemory;
///
/// If true, logs will not be considered when passing or failing
/// a test in Test builds. Useful if logging is disabled in Test
/// builds and we do not want to fail the test if no logs are
/// found.
///
[AutoParamWithNames(false, "AutomatedPerfTest.IgnoreTestBuildLogging")]
public bool IgnoreTestBuildLogging;
///
/// Used to specify a different test name in the CSV meta data, primarily used to distinguish between RHIs
///
[AutoParamWithNames("", "AutomatedPerfTest.TestConfigName")]
public string TestConfigName = "";
public string DataSourceName;
///
/// Call this in the test node's GetConfiguration function to set the DataSourceName used for later data processing
///
/// Name of the project
/// Which test controller is being used to generate the data
/// the properly formatted data source name
public string GetDataSourceName(string ProjectName, string DataSourceType="")
{
// if the name has been fully overridden by the calling process, just use that
if (!string.IsNullOrEmpty(DataSourceNameOverride))
{
return DataSourceNameOverride;
}
// otherwise, if the data source type has been override, use either that or the one passed into this function
if (string.IsNullOrEmpty(DataSourceType) && string.IsNullOrEmpty(DataSourceTypeOverride))
{
Log.Logger.LogError("No DataSourceType or DataSourceTypeOverride has been provided.");
return null;
}
string dataSourceType = string.IsNullOrEmpty(DataSourceTypeOverride) ? DataSourceType : DataSourceTypeOverride;
return $"Automation.{ProjectName}.{dataSourceType}";
}
public override void ApplyToConfig(UnrealAppConfig AppConfig, UnrealSessionRole ConfigRole, IEnumerable OtherRoles)
{
base.ApplyToConfig(AppConfig, ConfigRole, OtherRoles);
// Let the app know if we need to use extra debug memory before we
// instantiate it.
AppConfig.UsePlatformExtraDebugMemory = UsePlatformExtraDebugMemory;
}
}
public class AutomatedSequencePerfTestConfig : AutomatedPerfTestConfigBase
{
///
/// Which map to run the test on
///
[AutoParamWithNames("", "AutomatedPerfTest.SequencePerfTest.MapSequenceName")]
public string MapSequenceComboName;
}
public class AutomatedReplayPerfTestConfig : AutomatedPerfTestConfigBase
{
///
/// Which replay to run the test on
///
[AutoParamWithNames("", "AutomatedPerfTest.ReplayPerfTest.ReplayName")]
public string ReplayName;
}
public class AutomatedStaticCameraPerfTestConfig : AutomatedPerfTestConfigBase
{
///
/// Which map to run the test on
///
[AutoParamWithNames("", "AutomatedPerfTest.StaticCameraPerfTest.MapName")]
public string MapName;
}
public class AutomatedMaterialPerfTestConfig : AutomatedPerfTestConfigBase
{
}
}