2072 lines
65 KiB
C#
2072 lines
65 KiB
C#
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
using AutomationTool;
|
|
using EpicGames.Core;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading;
|
|
using UnrealBuildTool;
|
|
using Gauntlet.Utils;
|
|
using System.Net;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace Gauntlet
|
|
{
|
|
public enum ERoleModifier
|
|
{
|
|
None,
|
|
Dummy,
|
|
Null
|
|
};
|
|
|
|
/// <summary>
|
|
/// Represents a role that will be performed in an Unreal Session
|
|
/// </summary>
|
|
public class UnrealSessionRole
|
|
{
|
|
|
|
/// <summary>
|
|
/// Type of role
|
|
/// </summary>
|
|
public UnrealTargetRole RoleType;
|
|
|
|
/// <summary>
|
|
/// Platform this role uses
|
|
/// </summary>
|
|
public UnrealTargetPlatform? Platform;
|
|
|
|
/// <summary>
|
|
/// Configuration this role runs in
|
|
/// </summary>
|
|
public UnrealTargetConfiguration Configuration;
|
|
|
|
/// <summary>
|
|
/// Constraints this role runs under
|
|
/// </summary>
|
|
public UnrealDeviceTargetConstraint Constraint;
|
|
|
|
/// <summary>
|
|
/// Options that this role needs
|
|
/// </summary>
|
|
public IConfigOption<UnrealAppConfig> Options;
|
|
|
|
|
|
/// <summary>
|
|
/// Unique tag for logging purposes
|
|
/// </summary>
|
|
public string Tag;
|
|
|
|
/// <summary>
|
|
/// Command line that this role will use
|
|
/// </summary>
|
|
public string CommandLine
|
|
{
|
|
get
|
|
{
|
|
if (CommandLineParams == null)
|
|
{
|
|
CommandLineParams = new GauntletCommandLine();
|
|
}
|
|
return CommandLineParams.GenerateFullCommandLine();
|
|
}
|
|
set
|
|
{
|
|
if (CommandLineParams == null)
|
|
{
|
|
CommandLineParams = new GauntletCommandLine();
|
|
}
|
|
|
|
CommandLineParams.ClearCommandLine();
|
|
|
|
CommandLineParams.AddRawCommandline(value, false);
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// Dictionary of commandline arguments that are turned into a commandline at the end.
|
|
/// For flags, leave the value set to null. Generated from the Test Role's commandline object
|
|
/// and modified as needed. Then passed through to the AppConfig in UnrealBuildSource.cs
|
|
/// </summary>
|
|
public GauntletCommandLine CommandLineParams { get; set; }
|
|
|
|
|
|
/// <summary>
|
|
/// Map override to use on a server in case we don't want them all running the same map.
|
|
/// </summary>
|
|
public string MapOverride;
|
|
|
|
/// <summary>
|
|
/// List of files to copy to the device.
|
|
/// </summary>
|
|
public List<UnrealFileToCopy> FilesToCopy;
|
|
|
|
/// <summary>
|
|
/// Additional UE directories to copy from when saving artifacts
|
|
/// </summary>
|
|
public List<EIntendedBaseCopyDirectory> AdditionalArtifactDirectories;
|
|
|
|
/// <summary>
|
|
/// Role device configuration
|
|
/// </summary>
|
|
public ConfigureDeviceHandler ConfigureDevice;
|
|
|
|
/// <summary>
|
|
/// Properties we require our build to have
|
|
/// </summary>
|
|
public BuildFlags RequiredBuildFlags;
|
|
|
|
/// <summary>
|
|
/// Flavor of the build
|
|
/// </summary>
|
|
public string RequiredFlavor;
|
|
|
|
/// <summary>
|
|
/// Should be represented by a null device?
|
|
/// </summary>
|
|
public ERoleModifier RoleModifier;
|
|
|
|
/// <summary>
|
|
/// Is this a dummy executable?
|
|
/// </summary>
|
|
public bool IsDummy() { return RoleModifier == ERoleModifier.Dummy; }
|
|
|
|
/// <summary>
|
|
/// Whether this role should be responsible only for installing the build and not monitoring a process.
|
|
/// </summary>
|
|
public bool InstallOnly { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether this role will launched by the test node at a later time, typically during TickTest(). By default, all roles are launched immediately.
|
|
/// </summary>
|
|
public bool DeferredLaunch { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether this role will compress screenshots produced as an artifact into a jpeg format
|
|
/// </summary>
|
|
public bool CompressScreenshots { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether this role will create a gif from screenshots produced as artifacts
|
|
/// </summary>
|
|
public bool CreateGifFromScreenshots { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether or not this role should skip the cleaning of its device's artifact between each test run
|
|
/// </summary>
|
|
public bool SkipCleanDeviceArtifacts { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether or not this role will save artifacts when running with -dev.
|
|
/// These can be huge, so be cautious with using this.
|
|
/// </summary>
|
|
public bool ArchiveDevArtifacts { get; set; }
|
|
|
|
/// <summary>
|
|
/// Is this role Null?
|
|
/// </summary>
|
|
public bool IsNullRole() { return RoleModifier == ERoleModifier.Null; }
|
|
|
|
|
|
/// <summary>
|
|
/// Constructor taking limited params
|
|
/// </summary>
|
|
/// <param name="InType"></param>
|
|
/// <param name="InPlatform"></param>
|
|
/// <param name="InConfiguration"></param>
|
|
/// <param name="InOptions"></param>
|
|
public UnrealSessionRole(UnrealTargetRole InType, UnrealTargetPlatform? InPlatform, UnrealTargetConfiguration InConfiguration, IConfigOption<UnrealAppConfig> InOptions)
|
|
: this(InType, InPlatform, InConfiguration, null, InOptions)
|
|
{
|
|
}
|
|
|
|
/// <summary>
|
|
/// Constructor taking optional params
|
|
/// </summary>
|
|
/// <param name="InType"></param>
|
|
/// <param name="InPlatform"></param>
|
|
/// <param name="InConfiguration"></param>
|
|
/// <param name="InCommandLine"></param>
|
|
/// <param name="InOptions"></param>
|
|
public UnrealSessionRole(UnrealTargetRole InType, UnrealTargetPlatform? InPlatform, UnrealTargetConfiguration InConfiguration, string InCommandLine = null, IConfigOption<UnrealAppConfig> InOptions = null, bool CompressScreenshots = true, bool CreateGifFromScreenshots = true)
|
|
{
|
|
RoleType = InType;
|
|
|
|
Platform = InPlatform;
|
|
Configuration = InConfiguration;
|
|
MapOverride = string.Empty;
|
|
|
|
if (string.IsNullOrEmpty(InCommandLine))
|
|
{
|
|
CommandLine = string.Empty;
|
|
}
|
|
else
|
|
{
|
|
CommandLine = InCommandLine;
|
|
}
|
|
|
|
RequiredBuildFlags = BuildFlags.None;
|
|
|
|
if (Globals.IsRunningDev && !RoleType.UsesEditor())
|
|
{
|
|
RequiredBuildFlags |= BuildFlags.CanReplaceExecutable;
|
|
}
|
|
|
|
// Enforce build flags for the platform build that support it
|
|
IDeviceBuildSupport TargetBuildSupport = InterfaceHelpers.FindImplementations<IDeviceBuildSupport>().Where(B => B.CanSupportPlatform(InPlatform)).FirstOrDefault();
|
|
if (TargetBuildSupport != null)
|
|
{
|
|
if (Globals.Params.ParseParam("bulk") && TargetBuildSupport.CanSupportBuildType(BuildFlags.Bulk))
|
|
{
|
|
RequiredBuildFlags |= BuildFlags.Bulk;
|
|
}
|
|
else if (TargetBuildSupport.CanSupportBuildType(BuildFlags.NotBulk))
|
|
{
|
|
RequiredBuildFlags |= BuildFlags.NotBulk;
|
|
}
|
|
|
|
if (Globals.Params.ParseParam("packaged") && TargetBuildSupport.CanSupportBuildType(BuildFlags.Packaged))
|
|
{
|
|
RequiredBuildFlags |= BuildFlags.Packaged;
|
|
}
|
|
else if (Globals.Params.ParseParam("staged") && TargetBuildSupport.CanSupportBuildType(BuildFlags.Loose))
|
|
{
|
|
RequiredBuildFlags |= BuildFlags.Loose;
|
|
}
|
|
}
|
|
|
|
string RoleName = RoleType.ToString().ToLower();
|
|
RequiredFlavor = Globals.Params.ParseValue(RoleName + "flavor", "");
|
|
|
|
InstallOnly = false;
|
|
DeferredLaunch = false;
|
|
Options = InOptions;
|
|
FilesToCopy = new List<UnrealFileToCopy>();
|
|
CommandLineParams = new GauntletCommandLine();
|
|
RoleModifier = ERoleModifier.None;
|
|
this.CompressScreenshots = CompressScreenshots;
|
|
this.CreateGifFromScreenshots = CreateGifFromScreenshots;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Debugging aid
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public override string ToString()
|
|
{
|
|
return string.Format("{0} {1} {2} {3}", Platform, Configuration, RoleType, RequiredFlavor);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Represents an instance of a running an Unreal session. Basically an aggregate of all processes for
|
|
/// all roles (clients, server, etc
|
|
///
|
|
/// TODO - combine this into UnrealSession
|
|
/// </summary>
|
|
public class UnrealSessionInstance : IDisposable
|
|
{
|
|
/// <summary>
|
|
/// Represents an running role in our session
|
|
/// </summary>
|
|
public class RoleInstance
|
|
{
|
|
public RoleInstance(UnrealSessionRole InRole, IAppInstance InInstance)
|
|
{
|
|
Role = InRole;
|
|
AppInstance = InInstance;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Role that is being performed in this session
|
|
/// </summary>
|
|
public UnrealSessionRole Role { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Underlying AppInstance that is running the role
|
|
/// </summary>
|
|
public IAppInstance AppInstance { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Debugging aid
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public override string ToString()
|
|
{
|
|
return Role.ToString();
|
|
}
|
|
};
|
|
|
|
/// <summary>
|
|
/// All roles
|
|
/// </summary>
|
|
public RoleInstance[] AllRoles { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// All running roles
|
|
/// </summary>
|
|
public IEnumerable<RoleInstance> RunningRoles { get { return AllRoles.Where( X => X.AppInstance != null ); } }
|
|
|
|
/// <summary>
|
|
/// All deferred roles
|
|
/// </summary>
|
|
public IEnumerable<RoleInstance> DeferredRoles { get { return DeferredRoleToAppInstall.Keys; } }
|
|
|
|
/// <summary>
|
|
/// All deferred roles and their associated IAppInstall
|
|
/// </summary>
|
|
public Dictionary<RoleInstance, IAppInstall> DeferredRoleToAppInstall { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Helper for accessing all client processes. May return an empty array if no clients are involved
|
|
/// </summary>
|
|
public IAppInstance[] ClientApps
|
|
{
|
|
get
|
|
{
|
|
return RunningRoles.Where(R => R.Role.RoleType.IsClient()).Select(R => R.AppInstance).ToArray();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper for accessing server process. May return null if no server is involved
|
|
/// </summary>
|
|
public IAppInstance ServerApp
|
|
{
|
|
get
|
|
{
|
|
return RunningRoles.Where(R => R.Role.RoleType.IsServer()).Select(R => R.AppInstance).FirstOrDefault();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper for accessing editor process. May return null if no editor is involved
|
|
/// </summary>
|
|
public IAppInstance EditorApp
|
|
{
|
|
get
|
|
{
|
|
return RunningRoles.Where(R => R.Role.RoleType.IsEditor()).Select(R => R.AppInstance).FirstOrDefault();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper that returns true if clients are currently running
|
|
/// </summary>
|
|
public bool ClientsRunning
|
|
{
|
|
get
|
|
{
|
|
return ClientApps != null && ClientApps.Where(C => C.HasExited).Count() == 0;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Helper that returns true if there's a running server
|
|
/// </summary>
|
|
public bool ServerRunning
|
|
{
|
|
get
|
|
{
|
|
return ServerApp != null && ServerApp.HasExited == false;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns true if any of our roles are still running
|
|
/// </summary>
|
|
public bool IsRunningRoles
|
|
{
|
|
get
|
|
{
|
|
return RunningRoles.Any(R => R.AppInstance.HasExited == false);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Constructor. Roles must be passed in
|
|
/// </summary>
|
|
/// <param name="InAllRoles"></param>
|
|
/// <param name="InDeferredRoleToAppInstall"></param>
|
|
public UnrealSessionInstance(RoleInstance[] InAllRoles, Dictionary<RoleInstance, IAppInstall> InDeferredRoleToAppInstall = null)
|
|
{
|
|
AllRoles = InAllRoles;
|
|
DeferredRoleToAppInstall = InDeferredRoleToAppInstall;
|
|
}
|
|
|
|
~UnrealSessionInstance()
|
|
{
|
|
Dispose(false);
|
|
}
|
|
|
|
#region IDisposable Support
|
|
private bool disposedValue = false; // To detect redundant calls
|
|
|
|
protected virtual void Dispose(bool disposing)
|
|
{
|
|
if (!disposedValue)
|
|
{
|
|
if (disposing)
|
|
{
|
|
// TODO: dispose managed state (managed objects).
|
|
}
|
|
|
|
Shutdown();
|
|
disposedValue = true;
|
|
}
|
|
}
|
|
|
|
// This code added to correctly implement the disposable pattern.
|
|
public void Dispose()
|
|
{
|
|
// Do not change this code. Put cleanup code in Dispose(bool disposing) above.
|
|
Dispose(true);
|
|
}
|
|
#endregion
|
|
|
|
|
|
/// <summary>
|
|
/// Returns the app install for a given role, where the role was marked as 'DeferredLaunch'
|
|
/// </summary>
|
|
/// <param name="Role"></param>
|
|
public IAppInstall FindInstallForDeferredRole( UnrealSessionRole Role )
|
|
{
|
|
IAppInstall AppInstall = DeferredRoleToAppInstall
|
|
.Where(X => X.Key.Role == Role)
|
|
.Select(X => X.Value)
|
|
.FirstOrDefault();
|
|
|
|
if (AppInstall == null)
|
|
{
|
|
Log.Error("Cannot find derferred role {0}", Role.ToString());
|
|
}
|
|
|
|
return AppInstall;
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Launches a role that was previously flagged as 'DeferredLaunch'. Note that this does not handle device failure, marking problem devices etc.
|
|
/// </summary>
|
|
/// <param name="Role"></param>
|
|
/// <returns></returns>
|
|
public bool LaunchDeferredRole( UnrealSessionRole Role )
|
|
{
|
|
if (Role.DeferredLaunch == false)
|
|
{
|
|
Log.Error("Cannot start deferred role {0} because it is not marked 'DeferredLaunch'", Role.ToString());
|
|
return false;
|
|
}
|
|
|
|
int CurrentRoleInstanceIndex = AllRoles.FindIndex( X => X.Role == Role );
|
|
if (CurrentRoleInstanceIndex < 0)
|
|
{
|
|
Log.Error("Cannot start dereferred role {0} because it cannot be found among the 'all roles' list", Role.ToString());
|
|
return false;
|
|
}
|
|
|
|
IAppInstall CurrentInstall = FindInstallForDeferredRole(Role);
|
|
|
|
bool Success = false;
|
|
|
|
try
|
|
{
|
|
Log.Info("Starting deferred {0} on {1}", Role, CurrentInstall.Device);
|
|
IAppInstance Instance = CurrentInstall.Run();
|
|
IDeviceUsageReporter.RecordStart(Instance.Device.Name, Instance.Device.Platform, IDeviceUsageReporter.EventType.Test);
|
|
|
|
if (Instance != null || Globals.CancelSignalled)
|
|
{
|
|
// remove deferred role and update to a running role instance
|
|
DeferredRoleToAppInstall.Remove(AllRoles[CurrentRoleInstanceIndex]);
|
|
AllRoles[CurrentRoleInstanceIndex] = new RoleInstance(Role, Instance);
|
|
}
|
|
|
|
Success = true;
|
|
}
|
|
catch (DeviceException Ex)
|
|
{
|
|
Log.Error("Cannot start deferred role {0} because device {Name} threw an exception during launch. \nException={Exception}", Role.ToString(), CurrentInstall.Device, Ex.Message);
|
|
Success = false;
|
|
}
|
|
|
|
return Success;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Shutdown the session by killing any remaining processes.
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public void Shutdown(bool GenerateDumpOnKill = false)
|
|
{
|
|
// Kill any remaining client processes
|
|
if (ClientApps != null)
|
|
{
|
|
List<IAppInstance> RunningApps = ClientApps.Where(App =>( App != null && App.HasExited == false)).ToList();
|
|
|
|
if (RunningApps.Count > 0)
|
|
{
|
|
Log.Info("Shutting down {0} clients", RunningApps.Count);
|
|
RunningApps.ForEach(App =>
|
|
{
|
|
App.Kill(GenerateDumpOnKill);
|
|
// Apps that are still running have timed out => fail
|
|
IDeviceUsageReporter.RecordEnd(App.Device.Name, App.Device.Platform, IDeviceUsageReporter.EventType.Test, IDeviceUsageReporter.EventState.Success);
|
|
});
|
|
}
|
|
|
|
List<IAppInstance> ClosedApps = ClientApps.Where(App => (App != null && App.HasExited == true)).ToList();
|
|
if(ClosedApps.Count > 0)
|
|
{
|
|
ClosedApps.ForEach(App =>
|
|
{
|
|
// Apps that have already exited have 'succeeded'
|
|
IDeviceUsageReporter.RecordEnd(App.Device.Name, App.Device.Platform, IDeviceUsageReporter.EventType.Test, IDeviceUsageReporter.EventState.Failure);
|
|
});
|
|
}
|
|
}
|
|
|
|
if (ServerApp != null)
|
|
{
|
|
if (ServerApp.HasExited == false)
|
|
{
|
|
Log.Info("Shutting down server");
|
|
ServerApp.Kill(GenerateDumpOnKill);
|
|
}
|
|
}
|
|
|
|
// kill anything that's left
|
|
RunningRoles.Where(R => !R.Role.InstallOnly && R.AppInstance.HasExited == false).ToList().ForEach(R => R.AppInstance.Kill(GenerateDumpOnKill));
|
|
|
|
// Wait for it all to end
|
|
RunningRoles.Where(R => !R.Role.InstallOnly).ToList().ForEach(R => R.AppInstance.WaitForExit());
|
|
|
|
Thread.Sleep(3000);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Represents the set of available artifacts available after an UnrealSessionRole has completed
|
|
/// </summary>
|
|
public class UnrealRoleArtifacts
|
|
{
|
|
/// <summary>
|
|
/// Session role info that created these artifacts
|
|
/// </summary>
|
|
public UnrealSessionRole SessionRole { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// AppInstance that was used to run this role
|
|
/// </summary>
|
|
public IAppInstance AppInstance { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Path to artifacts from this role (these are local and were retried from the device).
|
|
/// </summary>
|
|
public string ArtifactPath { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Path to Log from this role
|
|
/// </summary>
|
|
public string LogPath { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Constructor, all values must be provided
|
|
/// </summary>
|
|
/// <param name="InSessionRole"></param>
|
|
/// <param name="InAppInstance"></param>
|
|
/// <param name="InArtifactPath"></param>
|
|
/// <param name="InLogPath"></param>
|
|
public UnrealRoleArtifacts(UnrealSessionRole InSessionRole, IAppInstance InAppInstance, string InArtifactPath, string InLogPath)
|
|
{
|
|
SessionRole = InSessionRole;
|
|
AppInstance = InAppInstance;
|
|
ArtifactPath = InArtifactPath;
|
|
LogPath = InLogPath;
|
|
}
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Helper class that understands how to launch/monitor/stop an Unreal test (clients + server) based on params contained in the test context and config
|
|
/// </summary>
|
|
public class UnrealSession : IDisposable
|
|
{
|
|
/// <summary>
|
|
/// Device reservation instance of this session
|
|
/// </summary>
|
|
public UnrealDeviceReservation UnrealDeviceReservation { get; private set; }
|
|
|
|
/// <summary>
|
|
/// Source of the build that will be launched
|
|
/// </summary>
|
|
protected UnrealBuildSource BuildSource { get; set; }
|
|
|
|
/// <summary>
|
|
/// Roles that will be performed by this session
|
|
/// </summary>
|
|
protected UnrealSessionRole[] SessionRoles { get; set; }
|
|
|
|
/// <summary>
|
|
/// Running instance of this session
|
|
/// </summary>
|
|
public UnrealSessionInstance SessionInstance { get; protected set; }
|
|
|
|
/// <summary>
|
|
/// Sandbox for installed apps
|
|
/// </summary>
|
|
public string Sandbox { get; set; }
|
|
|
|
/// <summary>
|
|
/// Whether or not devices should be retained between each test iteration
|
|
/// </summary>
|
|
public bool ShouldRetainDevices { get; set; }
|
|
|
|
[AutoParam(false)]
|
|
public bool ReinstallPerPass { get; set; }
|
|
|
|
/// <summary>
|
|
/// Number of attempts when launching a session.
|
|
/// Failed installs and runs will trigger a re-try
|
|
/// </summary>
|
|
public int LaunchSessionAttempts { get; set; }
|
|
|
|
/// <summary>
|
|
/// Number of attempts when trying to reserve devices
|
|
/// </summary>
|
|
public int DeviceReservationAttempts { get; set; }
|
|
|
|
/// <summary>
|
|
/// Number of seconds to wait between failed device reservation attempts
|
|
/// </summary>
|
|
public int DeviceReservationRetryTime { get; set; }
|
|
|
|
/// <summary>
|
|
/// Record of each ITargetDevice assigned to a given role
|
|
/// </summary>
|
|
public Dictionary<UnrealSessionRole, ITargetDevice> RolesToDevices { get; private set; }
|
|
|
|
/// <summary>
|
|
/// Record of each UnrealAppConfig created for each role
|
|
/// </summary>
|
|
public Dictionary<UnrealSessionRole, UnrealAppConfig> RolesToConfigs { get; private set; }
|
|
|
|
/// <summary>
|
|
/// Record of our installations in case we want to re-use them in a later pass
|
|
/// </summary>
|
|
public Dictionary<UnrealSessionRole, IAppInstall> RolesToInstalls { get; private set; }
|
|
|
|
/// <summary>
|
|
/// Constructor that takes a build source and a number of roles
|
|
/// </summary>
|
|
/// <param name="InSource"></param>
|
|
/// <param name="InSessionRoles"></param>
|
|
public UnrealSession(UnrealBuildSource InSource, IEnumerable<UnrealSessionRole> InSessionRoles)
|
|
{
|
|
AutoParam.ApplyParamsAndDefaults(this, Globals.Params.AllArguments);
|
|
|
|
BuildSource = InSource;
|
|
SessionRoles = InSessionRoles.ToArray();
|
|
ShouldRetainDevices = !Globals.Params.ParseParam("ReacquireDevicesPerPass");
|
|
|
|
RolesToDevices = new Dictionary<UnrealSessionRole, ITargetDevice>();
|
|
RolesToConfigs = new Dictionary<UnrealSessionRole, UnrealAppConfig>();
|
|
RolesToInstalls = new Dictionary<UnrealSessionRole, IAppInstall>();
|
|
|
|
LaunchSessionAttempts = 3;
|
|
DeviceReservationAttempts = 5;
|
|
DeviceReservationRetryTime = 120;
|
|
|
|
if (SessionRoles.Length == 0)
|
|
{
|
|
throw new AutomationException("No roles specified for Unreal session");
|
|
}
|
|
|
|
List<string> ValidationIssues = new List<string>();
|
|
|
|
if (!CheckRolesArePossible(ref ValidationIssues))
|
|
{
|
|
ValidationIssues.ForEach(S => Log.Error(KnownLogEvents.Gauntlet_BuildDropEvent, S));
|
|
throw new AutomationException("One or more issues occurred when validating build {0} against requested roles", InSource.BuildName);
|
|
}
|
|
|
|
UnrealDeviceReservation = new UnrealDeviceReservation();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Destructor, terminates any running session
|
|
/// </summary>
|
|
~UnrealSession()
|
|
{
|
|
Dispose(false);
|
|
}
|
|
|
|
#region IDisposable Support
|
|
private bool disposedValue = false; // To detect redundant calls
|
|
|
|
protected virtual void Dispose(bool disposing)
|
|
{
|
|
if (!disposedValue)
|
|
{
|
|
if (disposing)
|
|
{
|
|
// TODO: dispose managed state (managed objects).
|
|
}
|
|
|
|
ShutdownSession();
|
|
disposedValue = true;
|
|
}
|
|
}
|
|
|
|
// This code added to correctly implement the disposable pattern.
|
|
public void Dispose()
|
|
{
|
|
// Do not change this code. Put cleanup code in Dispose(bool disposing) above.
|
|
Dispose(true);
|
|
}
|
|
#endregion
|
|
|
|
/// <summary>
|
|
/// Helper that reserves and returns a list of available devices based on the passed in roles
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public bool TryReserveDevices()
|
|
{
|
|
if (ShouldRetainDevices && HasAcquiredDevices())
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Figure out how many of each device we need
|
|
Dictionary<UnrealDeviceTargetConstraint, int> RequiredDeviceTypes = new Dictionary<UnrealDeviceTargetConstraint, int>();
|
|
IEnumerable<UnrealSessionRole> RolesThatRequireDevice = SessionRoles
|
|
.Where(Role => !Role.IsNullRole())
|
|
.Where(RoleNeedsDevice);
|
|
|
|
// Get a count of the number of devices required for each platform
|
|
foreach (UnrealSessionRole Role in RolesThatRequireDevice)
|
|
{
|
|
if (RequiredDeviceTypes.ContainsKey(Role.Constraint))
|
|
{
|
|
++RequiredDeviceTypes[Role.Constraint];
|
|
}
|
|
else
|
|
{
|
|
RequiredDeviceTypes.Add(Role.Constraint, 1);
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
return UnrealDeviceReservation.TryReserveDevices(RequiredDeviceTypes, RolesThatRequireDevice.Count(), true);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.InnerException != null && ex.InnerException is WebException
|
|
&& (ex.InnerException as WebException).Response != null
|
|
&& ((ex.InnerException as WebException).Response is HttpWebResponse WebResponse)
|
|
&& WebResponse.StatusCode == HttpStatusCode.Conflict)
|
|
{
|
|
// Reservation failed, usually for lack of devices
|
|
// Return false to let attempt loop to kick in
|
|
Log.Info(ex.Message);
|
|
return false;
|
|
}
|
|
|
|
throw;
|
|
}
|
|
}
|
|
|
|
public bool TryReserveDevices(int Attempts)
|
|
{
|
|
for (int RemainingAttempts = Attempts; RemainingAttempts > 0; --RemainingAttempts)
|
|
{
|
|
if(Globals.CancelSignalled)
|
|
{
|
|
ReleaseSessionDevices();
|
|
return false;
|
|
}
|
|
|
|
if(TryReserveDevices())
|
|
{
|
|
return true;
|
|
}
|
|
else
|
|
{
|
|
Thread.Sleep(1000 * DeviceReservationRetryTime);
|
|
}
|
|
}
|
|
|
|
Log.Error("Failed to reserve devices after {Attempts} attempts", Attempts);
|
|
ReleaseSessionDevices();
|
|
return false;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Check that all the current roles can be performed by our build source
|
|
/// </summary>
|
|
/// <param name="Issues"></param>
|
|
/// <returns></returns>
|
|
bool CheckRolesArePossible(ref List<string> Issues)
|
|
{
|
|
bool Success = true;
|
|
foreach (var Role in SessionRoles)
|
|
{
|
|
if (!BuildSource.CanSupportRole(Role, ref Issues))
|
|
{
|
|
Success = false;
|
|
}
|
|
}
|
|
|
|
return Success;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Installs and launches all of our roles and returns an UnrealSessonInstance that represents the aggregate
|
|
/// of all of these processes. Will perform retries if errors with devices are encountered so this is a "best
|
|
/// attempt" at running with the devices provided
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public UnrealSessionInstance LaunchSession()
|
|
{
|
|
if(!Globals.UseExperimentalFeatures)
|
|
{
|
|
return Legacy_LaunchSession();
|
|
}
|
|
|
|
// Clear any existing session from a previous iteration
|
|
SessionInstance = null;
|
|
|
|
// When launching, issues with devices may be encountered.
|
|
// When these issues occur, those devices will be marked as problem devices and returned to the pool.
|
|
// A new set of devices will then be reserved and another attempt at launching the processes will be made.
|
|
// If LaunchSession() fails LaunchSessionAttempts amount of times, an exception is thrown.
|
|
for (int RemainingAttempts = LaunchSessionAttempts; RemainingAttempts > 0; --RemainingAttempts)
|
|
{
|
|
// Reserve devices, if needed
|
|
if (!TryReserveDevices(DeviceReservationAttempts))
|
|
{
|
|
if(Globals.CancelSignalled)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
// If device reservation fails, the device pool cannot support this launch.
|
|
throw new AutomationException("Failed to acquire all devices for launch. See above for details.");
|
|
}
|
|
|
|
if(Globals.CancelSignalled)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
if(!TryAssignDevicesToRoles())
|
|
{
|
|
// If device assignment fails, reservations were likely deleted at an unexpected time.
|
|
ReleaseSessionDevices();
|
|
continue;
|
|
}
|
|
|
|
// All roles should now be assigned a device.
|
|
// Install necessary builds, clear stale test artifacts, and copy any additional files
|
|
try
|
|
{
|
|
ReadyDevicesForSession();
|
|
}
|
|
catch (Exception Ex)
|
|
{
|
|
if(IsOutOfSpaceException(Ex) || IsOverlayException(Ex) || Ex is not DeviceException)
|
|
{
|
|
RemainingAttempts = 0;
|
|
ReleaseSessionDevices();
|
|
continue;
|
|
}
|
|
else if (RemainingAttempts > 1)
|
|
{
|
|
Log.Info("A new device will be selected and another attempt at launching session will be made.");
|
|
}
|
|
|
|
ReleaseProblemDevices();
|
|
continue;
|
|
}
|
|
|
|
if (Globals.CancelSignalled)
|
|
{
|
|
ReleaseSessionDevices();
|
|
return null;
|
|
}
|
|
|
|
// All roles should now be assigned device with an associated IAppInstall.
|
|
// Launch all the processes!
|
|
try
|
|
{
|
|
SessionInstance = LaunchProcesses();
|
|
}
|
|
catch
|
|
{
|
|
if (RemainingAttempts > 1)
|
|
{
|
|
Log.Info("A new device will be selected and another attempt at launching session will be made.");
|
|
}
|
|
|
|
ReleaseProblemDevices();
|
|
continue;
|
|
}
|
|
|
|
if (Globals.CancelSignalled)
|
|
{
|
|
ReleaseSessionDevices();
|
|
return null;
|
|
}
|
|
|
|
return SessionInstance;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Restarts the current session (if any)
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public UnrealSessionInstance RestartSession()
|
|
{
|
|
ShutdownSession();
|
|
|
|
// AG-TODO - want to preserve device reservations here...
|
|
|
|
return LaunchSession();
|
|
}
|
|
|
|
///<summary>
|
|
/// Shuts down any running apps
|
|
/// </summary>
|
|
public void ShutdownInstance()
|
|
{
|
|
if (SessionInstance != null)
|
|
{
|
|
SessionInstance.Dispose();
|
|
SessionInstance = null;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Shuts down the current session (if any)
|
|
/// </summary>
|
|
/// <returns></returns>
|
|
public void ShutdownSession()
|
|
{
|
|
ShutdownInstance();
|
|
|
|
if (!ShouldRetainDevices)
|
|
{
|
|
ReleaseSessionDevices();
|
|
}
|
|
}
|
|
|
|
public void ReleaseSessionDevices()
|
|
{
|
|
// Terminate any running apps
|
|
if (SessionInstance != null && SessionInstance.RunningRoles != null)
|
|
{
|
|
foreach (UnrealSessionInstance.RoleInstance RunningRole in SessionInstance.RunningRoles)
|
|
{
|
|
Log.Info("Shutting down {0}", RunningRole.AppInstance.Device);
|
|
RunningRole.AppInstance.Kill();
|
|
RunningRole.AppInstance.Device.Disconnect();
|
|
}
|
|
}
|
|
|
|
if (UnrealDeviceReservation == null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
UnrealDeviceReservation.ReleaseDevices();
|
|
|
|
RolesToDevices.Clear();
|
|
RolesToConfigs.Clear();
|
|
RolesToInstalls.Clear();
|
|
}
|
|
|
|
public void ReleaseProblemDevices()
|
|
{
|
|
// Terminate any running apps
|
|
if (SessionInstance != null && SessionInstance.RunningRoles != null)
|
|
{
|
|
foreach (UnrealSessionInstance.RoleInstance RunningRole in SessionInstance.RunningRoles.Where(Role => Role.AppInstance != null))
|
|
{
|
|
Log.Info("Shutting down {0}", RunningRole.AppInstance.Device);
|
|
RunningRole.AppInstance.Kill();
|
|
RunningRole.AppInstance.Device.Disconnect();
|
|
}
|
|
}
|
|
|
|
if (UnrealDeviceReservation == null)
|
|
{
|
|
return;
|
|
}
|
|
|
|
IEnumerable<ITargetDevice> ProblemDevices = UnrealDeviceReservation.ReleaseProblemDevices();
|
|
IEnumerable<UnrealSessionRole> RolesToClear = RolesToDevices.Keys.Where(Role => ProblemDevices.Contains(RolesToDevices[Role]));
|
|
|
|
Log.Info("Released problem devices...");
|
|
foreach(UnrealSessionRole Role in RolesToClear)
|
|
{
|
|
Log.Info("\t {DeviceName}", RolesToDevices[Role].Name);
|
|
RolesToDevices.Remove(Role);
|
|
RolesToConfigs.Remove(Role);
|
|
RolesToInstalls.Remove(Role);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieves and saves all artifacts from the provided session role. Artifacts are saved to the destination path
|
|
/// </summary>
|
|
/// <param name="InContext"></param>
|
|
/// <param name="InRunningRole"></param>
|
|
/// <param name="DestinationArtifactPath"></param>
|
|
/// <returns></returns>
|
|
public UnrealRoleArtifacts SaveRoleArtifacts(UnrealTestContext InContext, UnrealSessionInstance.RoleInstance InRunningRole, string DestinationArtifactPath)
|
|
{
|
|
DirectoryInfo SourceDirectory = new DirectoryInfo(InRunningRole.AppInstance.ArtifactPath);
|
|
DirectoryInfo DestinationDirectory = new DirectoryInfo(DestinationArtifactPath);
|
|
DestinationDirectory.Create();
|
|
|
|
// Whether this is a Dummy, Client, Server, Editor, etc
|
|
string RoleName = (InRunningRole.Role.IsDummy() ? "Dummy" : "") + InRunningRole.Role.RoleType.ToString();
|
|
|
|
// Move over any additional artifacts that a role requested (before they are removed potentially later)
|
|
foreach (EIntendedBaseCopyDirectory AdditionalDirectory in InRunningRole.Role.AdditionalArtifactDirectories)
|
|
{
|
|
Dictionary<EIntendedBaseCopyDirectory, string> PlatformMappings = InRunningRole.AppInstance.Device.GetPlatformDirectoryMappings();
|
|
if (PlatformMappings.ContainsKey(AdditionalDirectory))
|
|
{
|
|
DirectoryInfo AdditionalSourceDirectory = new DirectoryInfo(PlatformMappings[AdditionalDirectory]);
|
|
if (AdditionalSourceDirectory.Exists)
|
|
{
|
|
string TargetDirectory = Path.Combine(DestinationDirectory.FullName, AdditionalSourceDirectory.Name);
|
|
|
|
try
|
|
{
|
|
SystemHelpers.CopyDirectory(AdditionalSourceDirectory.FullName, TargetDirectory);
|
|
}
|
|
catch (Exception Exception)
|
|
{
|
|
Log.Warning("Encountered an {Exception} when trying to copy additional artifact directory {BaseCopyDirectory}" +
|
|
" from {AdditionalSourceDirectory} to {TargetDirectory}.",
|
|
Exception, AdditionalDirectory, AdditionalDirectory, TargetDirectory);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// We only want to move any other artifacts for editor data if there was a crash on a buildmachine.
|
|
// Also, don't move artifacts in dev mode, because peoples saved data could be huuuuuuuge!
|
|
bool IsEditorBuild = InRunningRole.Role.RoleType.UsesEditor();
|
|
bool IsBuildMachine = CommandUtils.IsBuildMachine;
|
|
bool SkipArchivingAssets = (Globals.IsRunningDev && !InRunningRole.Role.ArchiveDevArtifacts) || (IsEditorBuild && (IsBuildMachine == false || InRunningRole.AppInstance.ExitCode == 0));
|
|
bool bRetainArtifacts = InContext.TestParams.ParseParam("RetainDeviceArtifacts");
|
|
|
|
// Check if we should copy artifacts
|
|
if (!SkipArchivingAssets)
|
|
{
|
|
if (SourceDirectory.Exists)
|
|
{
|
|
// If a PersistentDownloadDirectory exists in the saved folder, delete it.
|
|
foreach (DirectoryInfo SubDirectory in SourceDirectory.EnumerateDirectories("*", SearchOption.AllDirectories))
|
|
{
|
|
if (SubDirectory.Name.Equals(EIntendedBaseCopyDirectory.PersistentDownloadDir.ToString(), StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
try
|
|
{
|
|
SubDirectory.Delete(true);
|
|
}
|
|
catch (Exception Exception)
|
|
{
|
|
Log.Info("Encountered a {Exception} when attempting to delete PersistentDownloadDirectory {Directory}. The PDD will be present in artifacts", Exception, SubDirectory);
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Perform the copy
|
|
try
|
|
{
|
|
// Save screenshots and create a gif when not running a server
|
|
if (!InRunningRole.Role.RoleType.IsServer())
|
|
{
|
|
SaveScreenshots(RoleName, SourceDirectory.FullName, DestinationDirectory.FullName, InRunningRole.Role.CompressScreenshots, InRunningRole.Role.CreateGifFromScreenshots);
|
|
}
|
|
|
|
// Copy remaining artifacts
|
|
SystemHelpers.CopyDirectory(SourceDirectory.FullName, DestinationDirectory.FullName, SystemHelpers.CopyOptions.Default, TruncateLongPathFilter);
|
|
}
|
|
catch (Exception Exception)
|
|
{
|
|
bRetainArtifacts = true;
|
|
Log.Warning("Encountered an {Exception} when copying saved artifacts from {SourceDirectory} to {DestinationDirectory}. " +
|
|
"Artifacts will not be saved locally, but the source artifacts will not be deleted.", Exception, SourceDirectory, DestinationDirectory);
|
|
}
|
|
|
|
// By default we delete the source artifacts, but we keep them if requested
|
|
if (!bRetainArtifacts)
|
|
{
|
|
// Account for any read-only files.
|
|
void SetAttributesNormal(DirectoryInfo Directory)
|
|
{
|
|
foreach (FileInfo File in Directory.GetFiles())
|
|
{
|
|
File.Attributes = FileAttributes.Normal;
|
|
}
|
|
foreach (DirectoryInfo SubDirectory in Directory.GetDirectories())
|
|
{
|
|
SetAttributesNormal(SubDirectory);
|
|
}
|
|
};
|
|
try
|
|
{
|
|
SetAttributesNormal(SourceDirectory);
|
|
}
|
|
catch
|
|
{
|
|
Log.Info("Could not remove the read-only attribute from {SourceDirectory}.", SourceDirectory);
|
|
}
|
|
try
|
|
{
|
|
SourceDirectory.Delete(true);
|
|
}
|
|
catch (Exception Exception)
|
|
{
|
|
Log.Info("Encountered an exception when deleting source artifacts at {SourceDirectory}. Artifacts will remain on the device.\n\tError message: {Exception}", SourceDirectory, Exception.Message);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Log.Info("Unable to find source artifact directory {SourceDirectory}", SourceDirectory.FullName);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (IsEditorBuild)
|
|
{
|
|
Log.Info("Skipping archival of assets for editor {Role}", RoleName);
|
|
}
|
|
else if (Globals.IsRunningDev)
|
|
{
|
|
Log.Info("Skipping archival of assets for dev build");
|
|
}
|
|
}
|
|
|
|
// Now write the role's log file
|
|
string ArtifactLogFilePath = Path.GetFullPath(Path.Combine(DestinationDirectory.FullName, RoleName + "Output.log"));
|
|
try
|
|
{
|
|
if (!InRunningRole.AppInstance.WriteOutputToFile(ArtifactLogFilePath))
|
|
{
|
|
ArtifactLogFilePath = string.Empty;
|
|
}
|
|
}
|
|
catch (Exception Ex)
|
|
{
|
|
string Message = "Encountered an {0} when attempting to write the {1} process log. The log will not be present in the artifacts.\n {2}";
|
|
Log.Warning(Message, Ex.GetType().Name, RoleName, Ex.Message);
|
|
ArtifactLogFilePath = string.Empty;
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(ArtifactLogFilePath))
|
|
{
|
|
Log.Info($"Wrote {RoleName} Log to {ArtifactLogFilePath}");
|
|
|
|
// On build machines, copy all role logs to Horde.
|
|
if (IsBuildMachine && Horde.IsHordeJob)
|
|
{
|
|
// Extract the log path portion that includes the Gauntlet test name. ie: UE.BootTest(Win64_Test_Client)\Client\ClientOutput.log
|
|
// That is to handle situation where multiple tests are run within the same Gauntlet Session and logs get overwritten.
|
|
string LogName = ArtifactLogFilePath.Replace(Path.GetFullPath(InContext.Options.LogDir), "").TrimStart(Path.DirectorySeparatorChar);
|
|
if (Path.IsPathFullyQualified(LogName))
|
|
{
|
|
// The path was expected to be relative to LogDir, however it appeared to be an absolute path.
|
|
// So we revert to default behavior and save the log directly in UAT log folder.
|
|
LogName = RoleName + "Output.log";
|
|
}
|
|
string HordeLogFilePath = Path.GetFullPath(Path.Combine(CommandUtils.CmdEnv.LogFolder, LogName));
|
|
Log.Verbose($"Copy log for Horde to {HordeLogFilePath}");
|
|
string TargetDirectry = Path.GetDirectoryName(HordeLogFilePath);
|
|
if (!Directory.Exists(TargetDirectry)) { Directory.CreateDirectory(TargetDirectry); }
|
|
File.Copy(ArtifactLogFilePath, HordeLogFilePath, true);
|
|
}
|
|
}
|
|
|
|
// TODO REMOVEME- this should go elsewhere, likely a utile that can be called or inserted by relevant test nodes.
|
|
SavePSOs(InContext, InRunningRole, DestinationDirectory.FullName);
|
|
// END REMOVEME
|
|
|
|
// Save the Artifact filepath
|
|
return new UnrealRoleArtifacts(InRunningRole.Role, InRunningRole.AppInstance, DestinationDirectory.FullName, ArtifactLogFilePath);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Saves all artifacts from the provided session to the specified output path.
|
|
/// </summary>
|
|
/// <param name="Context"></param>
|
|
/// <param name="TestInstance"></param>
|
|
/// <param name="OutputPath"></param>
|
|
/// <returns></returns>
|
|
public IEnumerable<UnrealRoleArtifacts> SaveRoleArtifacts(UnrealTestContext Context, UnrealSessionInstance TestInstance, string OutputPath)
|
|
{
|
|
int DummyClientCount = 0;
|
|
Dictionary<UnrealTargetRole, int> RoleCounts = new Dictionary<UnrealTargetRole, int>();
|
|
List<UnrealRoleArtifacts> AllArtifacts = new List<UnrealRoleArtifacts>();
|
|
|
|
foreach (UnrealSessionInstance.RoleInstance App in TestInstance.RunningRoles)
|
|
{
|
|
string RoleName = (App.Role.IsDummy() ? "Dummy" : "") + App.Role.RoleType.ToString();
|
|
string FolderName = RoleName;
|
|
|
|
int RoleCount = 1;
|
|
|
|
if (App.Role.IsDummy())
|
|
{
|
|
DummyClientCount++;
|
|
RoleCount = DummyClientCount;
|
|
}
|
|
else
|
|
{
|
|
if (!RoleCounts.ContainsKey(App.Role.RoleType))
|
|
{
|
|
RoleCounts.Add(App.Role.RoleType, 1);
|
|
}
|
|
else
|
|
{
|
|
RoleCounts[App.Role.RoleType]++;
|
|
}
|
|
|
|
RoleCount = RoleCounts[App.Role.RoleType];
|
|
}
|
|
|
|
if (RoleCount > 1)
|
|
{
|
|
FolderName += string.Format("_{0:00}", RoleCount);
|
|
}
|
|
|
|
string DestPath = Path.Combine(OutputPath, FolderName);
|
|
|
|
if (!App.Role.IsNullRole() && !App.Role.InstallOnly)
|
|
{
|
|
Log.VeryVerbose("Calling SaveRoleArtifacts, Role: {0} Artifact Path: {1}", App.ToString(), App.AppInstance.ArtifactPath);
|
|
UnrealRoleArtifacts Artifacts = null;
|
|
ITargetDevice device = App.AppInstance.Device;
|
|
IDeviceUsageReporter.RecordStart(device.Name, device.Platform, IDeviceUsageReporter.EventType.SavingArtifacts);
|
|
try
|
|
{
|
|
Artifacts = SaveRoleArtifacts(Context, App, DestPath);
|
|
}
|
|
catch (Exception SaveArtifactsException)
|
|
{
|
|
// Caught an exception -> report failure
|
|
IDeviceUsageReporter.RecordEnd(device.Name, device.Platform, IDeviceUsageReporter.EventType.SavingArtifacts, IDeviceUsageReporter.EventState.Failure);
|
|
|
|
// Retry once only, after rebooting device, if artifacts couldn't be saved.
|
|
if (SaveArtifactsException.Message.Contains("A retry should be performed"))
|
|
{
|
|
Log.Info("Rebooting device and retrying save role artifacts once.");
|
|
App.AppInstance.Device.Reboot();
|
|
Artifacts = SaveRoleArtifacts(Context, App, DestPath);
|
|
}
|
|
else
|
|
{
|
|
// Pass exception to the surrounding try/catch in UnrealTestNode while preserving the original callstack
|
|
throw;
|
|
}
|
|
}
|
|
// Did not catch -> successful reporting
|
|
IDeviceUsageReporter.RecordEnd(device.Name, device.Platform, IDeviceUsageReporter.EventType.SavingArtifacts, IDeviceUsageReporter.EventState.Success);
|
|
|
|
if (Artifacts != null)
|
|
{
|
|
AllArtifacts.Add(Artifacts);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Log.Verbose("Skipping SaveRoleArtifacts for Null Role: {0}", App.ToString());
|
|
}
|
|
}
|
|
|
|
return AllArtifacts;
|
|
}
|
|
|
|
private UnrealSessionInstance Legacy_LaunchSession()
|
|
{
|
|
SessionInstance = null;
|
|
|
|
// The number of retries when launching session to avoid an endless loop if package can't be installed, network timeouts, etc
|
|
int SessionRetries = 2;
|
|
|
|
// tries to find devices and launch our session. Will loop until we succeed, we run out of devices/retries, or
|
|
// something fatal occurs..
|
|
while (SessionInstance == null && Globals.CancelSignalled == false)
|
|
{
|
|
int ReservationRetries = 5;
|
|
int ReservationRetryWait = 120;
|
|
|
|
IEnumerable<UnrealSessionRole> RolesNeedingDevices = SessionRoles.Where(R => R.IsNullRole() == false);
|
|
|
|
while (UnrealDeviceReservation.ReservedDevices.Count() < RolesNeedingDevices.Count())
|
|
{
|
|
// get devices
|
|
TryReserveDevices();
|
|
|
|
if (Globals.CancelSignalled)
|
|
{
|
|
break;
|
|
}
|
|
|
|
// if we failed to get enough devices, show a message and wait
|
|
if (UnrealDeviceReservation.ReservedDevices.Count() != SessionRoles.Count())
|
|
{
|
|
if (ReservationRetries == 0)
|
|
{
|
|
throw new AutomationException("Unable to acquire all devices for test.");
|
|
}
|
|
Log.Info("\nUnable to find enough device(s). Waiting {0} secs (retries left={1})\n", ReservationRetryWait, --ReservationRetries);
|
|
Thread.Sleep(ReservationRetryWait * 1000);
|
|
}
|
|
}
|
|
|
|
if (Globals.CancelSignalled)
|
|
{
|
|
return null;
|
|
}
|
|
|
|
Dictionary<IAppInstall, UnrealSessionRole> InstallsToRoles = new Dictionary<IAppInstall, UnrealSessionRole>();
|
|
Dictionary<IAppInstall, UnrealAppConfig> InstallsToConfig = new Dictionary<IAppInstall, UnrealAppConfig>();
|
|
|
|
// create a copy of our list
|
|
IEnumerable<ITargetDevice> DevicesToInstallOn = UnrealDeviceReservation.ReservedDevices.ToArray();
|
|
|
|
bool InstallSuccess = true;
|
|
|
|
// sort by constraints, so that we pick constrained devices first
|
|
List<UnrealSessionRole> SortedRoles = SessionRoles.OrderBy(R => R.Constraint.IsIdentity() ? 1 : 0).ToList();
|
|
|
|
// first install all roles on these devices
|
|
foreach (UnrealSessionRole Role in SortedRoles)
|
|
{
|
|
ITargetDevice Device = null;
|
|
|
|
if (Role.IsNullRole() == false)
|
|
{
|
|
Device = DevicesToInstallOn.Where(D => D.IsConnected && D.Platform == Role.Platform
|
|
&& (Role.Constraint.IsIdentity() || DevicePool.Instance.GetConstraint(D) == Role.Constraint)).First();
|
|
|
|
DevicesToInstallOn = DevicesToInstallOn.Where(D => D != Device);
|
|
}
|
|
else
|
|
{
|
|
Device = new TargetDeviceNull(string.Format("Null{0}", Role.RoleType));
|
|
}
|
|
|
|
IEnumerable<UnrealSessionRole> OtherRoles = SortedRoles.Where(R => R != Role);
|
|
|
|
// create a config from the build source (this also applies the role options)
|
|
UnrealAppConfig AppConfig = BuildSource.CreateConfiguration(Role, OtherRoles);
|
|
|
|
// todo - should this be elsewhere?
|
|
AppConfig.Sandbox = Sandbox;
|
|
|
|
IAppInstall Install = null;
|
|
if (RolesToInstalls == null || !RolesToInstalls.ContainsKey(Role) || ReinstallPerPass)
|
|
{
|
|
// Tag the device for report result
|
|
if (BuildHostPlatform.Current.Platform != Device.Platform)
|
|
{
|
|
AppConfig.CommandLineParams.Add("DeviceTag", Device.Name);
|
|
}
|
|
|
|
IDeviceUsageReporter.RecordStart(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Device, IDeviceUsageReporter.EventState.Success);
|
|
IDeviceUsageReporter.RecordStart(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Install, IDeviceUsageReporter.EventState.Success, BuildSource.BuildName);
|
|
try
|
|
{
|
|
// Make sure the artifact paths are clean before installation
|
|
if(!Role.SkipCleanDeviceArtifacts)
|
|
{
|
|
Device.CleanArtifacts(AppConfig);
|
|
}
|
|
|
|
Install = Device.InstallApplication(AppConfig);
|
|
IDeviceUsageReporter.RecordEnd(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Install, IDeviceUsageReporter.EventState.Success);
|
|
}
|
|
catch (Exception Ex)
|
|
{
|
|
InstallSuccess = false;
|
|
string ErrorMessage = string.Format("Encountered error setting up device {0} for role {1}. {2}", Device, Role, Ex);
|
|
if (ErrorMessage.Contains("not enough space") && Device.Platform == BuildHostPlatform.Current.Platform)
|
|
{
|
|
Log.Error(ErrorMessage);
|
|
// If on desktop platform, we are not retrying.
|
|
// It is unlikely that space is going to be made and InstallBuildParallel has marked the build path as problematic.
|
|
SessionRetries = 0;
|
|
break;
|
|
}
|
|
|
|
// Warn, ignore the device, and do not continue
|
|
if (Ex is DeviceException)
|
|
{
|
|
UnrealDeviceReservation.MarkProblemDevice(Device, ErrorMessage);
|
|
}
|
|
else
|
|
{
|
|
Log.Warning(KnownLogEvents.Gauntlet_DeviceEvent, ErrorMessage);
|
|
}
|
|
|
|
IDeviceUsageReporter.RecordEnd(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Install, IDeviceUsageReporter.EventState.Failure);
|
|
break;
|
|
}
|
|
|
|
|
|
if (Globals.CancelSignalled)
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Device has app installed, give role a chance to configure device
|
|
Role.ConfigureDevice?.Invoke(Device);
|
|
|
|
InstallsToRoles[Install] = Role;
|
|
InstallsToConfig[Install] = AppConfig;
|
|
|
|
if (ReinstallPerPass)
|
|
{
|
|
RolesToInstalls[Role] = Install;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Install = RolesToInstalls[Role];
|
|
InstallsToRoles[Install] = Role;
|
|
InstallsToConfig[Install] = AppConfig;
|
|
Log.Info("Using previous install of {0} on {1}", Install.Name, Install.Device.Name);
|
|
}
|
|
}
|
|
|
|
if (InstallSuccess == false)
|
|
{
|
|
ReleaseSessionDevices();
|
|
|
|
if (SessionRetries == 0)
|
|
{
|
|
throw new AutomationException("Unable to install application for session.");
|
|
}
|
|
|
|
Log.Info("\nUnable to install application for session (retries left={0})\n", --SessionRetries);
|
|
}
|
|
|
|
if (InstallSuccess && Globals.CancelSignalled == false)
|
|
{
|
|
List<UnrealSessionInstance.RoleInstance> AllRoles = new List<UnrealSessionInstance.RoleInstance>();
|
|
Dictionary<UnrealSessionInstance.RoleInstance, IAppInstall> DeferredRoleToAppInstall = new Dictionary<UnrealSessionInstance.RoleInstance, IAppInstall>();
|
|
|
|
// Now try to run all installs on their devices
|
|
foreach (var InstallRoleKV in InstallsToRoles)
|
|
{
|
|
IAppInstall CurrentInstall = InstallRoleKV.Key;
|
|
if (InstallRoleKV.Value.InstallOnly)
|
|
{
|
|
AllRoles.Add(new UnrealSessionInstance.RoleInstance(InstallRoleKV.Value, null));
|
|
continue;
|
|
}
|
|
if (InstallRoleKV.Value.DeferredLaunch)
|
|
{
|
|
UnrealSessionInstance.RoleInstance DeferredRoleInstance = new UnrealSessionInstance.RoleInstance(InstallRoleKV.Value, null);
|
|
|
|
DeferredRoleToAppInstall.Add(DeferredRoleInstance, CurrentInstall);
|
|
AllRoles.Add(DeferredRoleInstance);
|
|
continue;
|
|
}
|
|
bool Success = false;
|
|
|
|
try
|
|
{
|
|
Log.Info("Starting {0} on {1}", InstallRoleKV.Value, CurrentInstall.Device);
|
|
IAppInstance Instance = CurrentInstall.Run();
|
|
IDeviceUsageReporter.RecordStart(Instance.Device.Name, Instance.Device.Platform, IDeviceUsageReporter.EventType.Test);
|
|
|
|
if (Instance != null || Globals.CancelSignalled)
|
|
{
|
|
AllRoles.Add(new UnrealSessionInstance.RoleInstance(InstallRoleKV.Value, Instance));
|
|
}
|
|
|
|
Success = true;
|
|
}
|
|
catch (DeviceException Ex)
|
|
{
|
|
// mark that device as a problem
|
|
Log.Info(KnownLogEvents.Gauntlet_DeviceEvent, "Failed to start build on {DeviceName}. Marking as problem device and retrying with new set", CurrentInstall.Device.Name);
|
|
UnrealDeviceReservation.MarkProblemDevice(CurrentInstall.Device, $"Device threw an exception during launch.\nException={Ex.Message}");
|
|
Success = false;
|
|
}
|
|
|
|
if (Success == false)
|
|
{
|
|
// terminate anything that's running
|
|
foreach (UnrealSessionInstance.RoleInstance RunningRole in AllRoles.Where(X => X.AppInstance != null))
|
|
{
|
|
Log.Info("Shutting down {DeviceName}", RunningRole.AppInstance.Device.Name);
|
|
RunningRole.AppInstance.Kill();
|
|
RunningRole.AppInstance.Device.Disconnect();
|
|
}
|
|
|
|
ReleaseSessionDevices();
|
|
|
|
if (SessionRetries == 0)
|
|
{
|
|
throw new AutomationException("Unable to start application for session, see log for details.");
|
|
}
|
|
|
|
Log.Info("\nUnable to start application for session (retries left={0})\n", --SessionRetries);
|
|
|
|
break; // do not continue loop
|
|
}
|
|
}
|
|
|
|
if (AllRoles.Count() == SessionRoles.Count())
|
|
{
|
|
SessionInstance = new UnrealSessionInstance(AllRoles.ToArray(), DeferredRoleToAppInstall);
|
|
}
|
|
}
|
|
}
|
|
|
|
return SessionInstance;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns true if the number of reserved devices matches the number on non-null session roles
|
|
/// </summary>
|
|
private bool HasAcquiredDevices()
|
|
{
|
|
IEnumerable<UnrealSessionRole> RolesThatRequireDevice = SessionRoles.Where(Role => !Role.IsNullRole());
|
|
return UnrealDeviceReservation != null
|
|
&& UnrealDeviceReservation.ReservedDevices != null
|
|
&& UnrealDeviceReservation.ReservedDevices.Count() == RolesThatRequireDevice.Count();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns true if the provided session role has not yet been assigned a device
|
|
/// </summary>
|
|
private bool RoleNeedsDevice(UnrealSessionRole Role)
|
|
{
|
|
return !(RolesToDevices.ContainsKey(Role) && RolesToDevices[Role] != null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns true if the provided session role has not yet had an install performed on it's assigned device
|
|
/// </summary>
|
|
private bool RoleNeedsInstall(UnrealSessionRole Role)
|
|
{
|
|
return ReinstallPerPass
|
|
|| !(RolesToConfigs.ContainsKey(Role) && RolesToConfigs[Role] != null)
|
|
|| !(RolesToInstalls.ContainsKey(Role) && RolesToInstalls[Role] != null);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns true if the provided target device matches the constraint requested by the session role
|
|
/// </summary>
|
|
private bool DeviceMatchesRoleConstraint(UnrealSessionRole Role, ITargetDevice Device)
|
|
{
|
|
bool bRoleMatchesConstraint = Role.Constraint.Equals(DevicePool.Instance.GetConstraint(Device));
|
|
|
|
return Device.IsConnected
|
|
&& Device.Platform == Role.Platform
|
|
&& Role.Constraint.IsIdentity() || bRoleMatchesConstraint;
|
|
}
|
|
|
|
/// <summary>
|
|
/// From the existing reserved device pool, assign each role a device that matches it's requested constraint
|
|
/// This will cache a map of each role and the target device it's using in this UnrealSession
|
|
/// </summary>
|
|
private bool TryAssignDevicesToRoles()
|
|
{
|
|
// Order by constraint. This ensures roles with constraints have their devices selected first.
|
|
IEnumerable<UnrealSessionRole> RolesSortedByConstraint = SessionRoles.OrderBy(R => R.Constraint.IsIdentity() ? 1 : 0);
|
|
IEnumerable<ITargetDevice> ReservedDevices = UnrealDeviceReservation.ReservedDevices;
|
|
|
|
foreach (UnrealSessionRole Role in RolesSortedByConstraint)
|
|
{
|
|
if (RoleNeedsDevice(Role))
|
|
{
|
|
ITargetDevice DeviceToAssign = null;
|
|
|
|
if (Role.IsNullRole())
|
|
{
|
|
DeviceToAssign = new TargetDeviceNull($"Null{Role.RoleType}");
|
|
}
|
|
else
|
|
{
|
|
try
|
|
{
|
|
DeviceToAssign = ReservedDevices.Where(Device => DeviceMatchesRoleConstraint(Role, Device)).First();
|
|
IDeviceUsageReporter.RecordStart(DeviceToAssign.Name, DeviceToAssign.Platform, IDeviceUsageReporter.EventType.Device, IDeviceUsageReporter.EventState.Success);
|
|
}
|
|
catch (Exception Ex)
|
|
{
|
|
Log.Warning("Failed to assign a reserved device to role {Role}. " +
|
|
"This usually means devices were unexpectedly released mid session\n{Exception}", Role, Ex);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
RolesToDevices.Add(Role, DeviceToAssign);
|
|
ReservedDevices = ReservedDevices.Except(Enumerable.Repeat(DeviceToAssign, 1));
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Prepares each device for launch by performing the following
|
|
/// - Install builds
|
|
/// - Clean up old artifacts
|
|
/// - Copy additional files requested by a test
|
|
/// - Specific role configurations
|
|
/// This will create and cache both an UnrealAppConfig and an IAppInstall for future reference
|
|
/// </summary>
|
|
private void ReadyDevicesForSession()
|
|
{
|
|
foreach(UnrealSessionRole Role in SessionRoles)
|
|
{
|
|
UnrealAppConfig AppConfig = null;
|
|
ITargetDevice Device = RolesToDevices[Role];
|
|
|
|
if (Globals.CancelSignalled)
|
|
{
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
if (RoleNeedsInstall(Role))
|
|
{
|
|
RolesToConfigs.Remove(Role);
|
|
RolesToInstalls.Remove(Role);
|
|
|
|
// Create the app config
|
|
IEnumerable<UnrealSessionRole> OtherRoles = SessionRoles.Where(Other => Other != Role);
|
|
AppConfig = BuildSource.CreateConfiguration(Role, OtherRoles);
|
|
AppConfig.Sandbox = Sandbox;
|
|
AppConfig.CommandLineParams.AddUnique("DeviceTag", Device.Name);
|
|
RolesToConfigs.Add(Role, AppConfig);
|
|
|
|
if (AppConfig.FullClean)
|
|
{
|
|
Log.Info("Fully cleaning device before install...");
|
|
Device.FullClean();
|
|
}
|
|
|
|
if (!Role.SkipCleanDeviceArtifacts)
|
|
{
|
|
Device.CleanArtifacts(AppConfig);
|
|
}
|
|
|
|
if (AppConfig.SkipInstall)
|
|
{
|
|
Log.Info("Skipping install due to SkipInstall");
|
|
}
|
|
else
|
|
{
|
|
// Telemetry
|
|
DateTimeStopwatch Stopwatch = DateTimeStopwatch.Start();
|
|
Log.Info("Installing {BuildName} of type {BuildType} to {DeviceName}...", BuildSource.BuildName, AppConfig.Build.GetType().Name, Device.Name);
|
|
IDeviceUsageReporter.RecordStart(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Install, IDeviceUsageReporter.EventState.Success, BuildSource.BuildName);
|
|
|
|
try
|
|
{
|
|
Device.InstallBuild(AppConfig);
|
|
IDeviceUsageReporter.RecordEnd(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Install, IDeviceUsageReporter.EventState.Success);
|
|
Log.Info("Installation completed in {InstallTime}", GetInstallTime(Stopwatch.ElapsedTime));
|
|
}
|
|
catch
|
|
{
|
|
IDeviceUsageReporter.RecordEnd(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Install, IDeviceUsageReporter.EventState.Failure);
|
|
throw;
|
|
}
|
|
}
|
|
|
|
IAppInstall Install = Device.CreateAppInstall(AppConfig);
|
|
RolesToInstalls.Add(Role, Install);
|
|
}
|
|
else
|
|
{
|
|
AppConfig = RolesToConfigs[Role];
|
|
if (!Role.SkipCleanDeviceArtifacts)
|
|
{
|
|
Device.CleanArtifacts(AppConfig);
|
|
}
|
|
}
|
|
|
|
if (RolesToConfigs[Role].Build.SupportsAdditionalFileCopy)
|
|
{
|
|
Device.CopyAdditionalFiles(RolesToConfigs[Role].FilesToCopy);
|
|
}
|
|
|
|
Role.ConfigureDevice?.Invoke(Device);
|
|
}
|
|
catch(Exception Ex)
|
|
{
|
|
string Message = $"Encountered {Ex.GetType()} when creating installation.\n{Ex.Message}";
|
|
|
|
bool IsHostPlatform = Device.Platform == BuildHostPlatform.Current.Platform;
|
|
EventId EventType = IsHostPlatform ? KnownLogEvents.Gauntlet : KnownLogEvents.Gauntlet_DeviceEvent;
|
|
|
|
if (IsOutOfSpaceException(Ex) && IsHostPlatform)
|
|
{
|
|
// If on desktop platform, we are not retrying.
|
|
// It is unlikely that space is going to be made and InstallBuildParallel has marked the build path as problematic.
|
|
Log.Error(EventType, Message);
|
|
throw;
|
|
}
|
|
else if (IsOverlayException(Ex))
|
|
{
|
|
// Errors with Overlay executables are caused by missing files or improper setup, inform user and exit early
|
|
Log.Error(EventType, "Device {DeviceName} {Message}", Device.Name, Message);
|
|
throw;
|
|
}
|
|
else if (Ex is DeviceException && !IsHostPlatform)
|
|
{
|
|
UnrealDeviceReservation.MarkProblemDevice(Device, Message);
|
|
}
|
|
else
|
|
{
|
|
Log.Warning(EventType, Message);
|
|
}
|
|
throw; // Can consider not throwing here - this would let every device complete setup before releasing the problem devices
|
|
}
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Launches the Unreal Engine processes for each role requested by this session.
|
|
/// Roles marked InstallOnly will not have a process launched.
|
|
/// Roles marked DeferredLaunch will not have a process launched.
|
|
/// Deferred roles can be launched at anytime (usually in TickTest()) by calling UnrealSessionInstance.LaunchDeferredRole
|
|
/// </summary>
|
|
private UnrealSessionInstance LaunchProcesses()
|
|
{
|
|
List<UnrealSessionInstance.RoleInstance> RoleInstances = new();
|
|
Dictionary<UnrealSessionInstance.RoleInstance, IAppInstall> DeferredRolesToInstalls = new();
|
|
|
|
foreach (KeyValuePair<UnrealSessionRole, IAppInstall> RoleInstall in RolesToInstalls)
|
|
{
|
|
UnrealSessionRole Role = RoleInstall.Key;
|
|
IAppInstall Install = RoleInstall.Value;
|
|
UnrealAppConfig Config = RolesToConfigs[Role];
|
|
|
|
// InstallOnly roles don't execute a process
|
|
if (Role.InstallOnly)
|
|
{
|
|
RoleInstances.Add(new UnrealSessionInstance.RoleInstance(Role, null));
|
|
continue;
|
|
}
|
|
|
|
// DeferredLaunch roles don't immediately execute a process.
|
|
// We cache deferred roles so users can launch the process at the desired time.
|
|
else if (Role.DeferredLaunch)
|
|
{
|
|
UnrealSessionInstance.RoleInstance DeferredRoleInstance = new(Role, null);
|
|
RoleInstances.Add(DeferredRoleInstance);
|
|
DeferredRolesToInstalls.Add(DeferredRoleInstance, Install);
|
|
continue;
|
|
}
|
|
|
|
try
|
|
{
|
|
Log.Info("Launching {Install} on {DeviceName}", Install, RolesToDevices[Role].Name);
|
|
IAppInstance AppInstance = Install.Run();
|
|
|
|
if (AppInstance == null)
|
|
{
|
|
throw new AutomationException("Failed to create an IAppInstance after attempting Run() on {Install}", Install);
|
|
}
|
|
else
|
|
{
|
|
RoleInstances.Add(new UnrealSessionInstance.RoleInstance(Role, AppInstance));
|
|
}
|
|
}
|
|
catch(Exception Ex)
|
|
{
|
|
// Kill any processes that were started
|
|
foreach (UnrealSessionInstance.RoleInstance Instance in RoleInstances.Where(Role => Role.AppInstance != null))
|
|
{
|
|
Log.Info("Shutting down {AppInstance}", Instance.AppInstance);
|
|
Instance.AppInstance.Kill();
|
|
}
|
|
|
|
string WarningMessage = $"Encountered {Ex.GetType()} when attempting to run install {Ex.Message}";
|
|
if (Ex is DeviceException)
|
|
{
|
|
UnrealDeviceReservation.MarkProblemDevice(Install.Device, WarningMessage);
|
|
}
|
|
else
|
|
{
|
|
Log.Warning(WarningMessage);
|
|
}
|
|
throw;
|
|
}
|
|
}
|
|
|
|
return new UnrealSessionInstance(RoleInstances.ToArray(), DeferredRolesToInstalls);
|
|
}
|
|
|
|
private void SaveScreenshots(string RoleName, string SourceDirectory, string DestinationDirectory, bool bCompressImages, bool bCreateGif)
|
|
{
|
|
try
|
|
{
|
|
string ScreenshotPath = PathUtils.FindRelevantPath(BasePath: SourceDirectory, "Screenshots");
|
|
|
|
if (string.IsNullOrEmpty(ScreenshotPath) || !Directory.Exists(ScreenshotPath))
|
|
{
|
|
Log.Verbose("Could not locate Screenshots subdirectory in artifact path {ArtifactPath}. Skipping GIF creation", SourceDirectory);
|
|
return;
|
|
}
|
|
|
|
// Image transformation relies on the image being directly on the host PC
|
|
// This means it can't be on a remote device, or in a network storage.
|
|
// The screenshots directory will be copied to a temp directory for the transformations, and then moved to the final artifact destination.
|
|
DirectoryInfo ScreenshotDirectory = new(ScreenshotPath);
|
|
DirectoryInfo ImageStage = new(Path.Combine(Path.GetTempPath(), "ImageStage", RoleName));
|
|
if (ImageStage.Exists)
|
|
{
|
|
ImageStage.Delete(true);
|
|
}
|
|
|
|
// Copy the images to a the staging directory
|
|
SystemHelpers.CopyDirectory(ScreenshotPath, ImageStage.FullName, SystemHelpers.CopyOptions.Mirror);
|
|
|
|
// Creates an enumerable containing both the root directory and any sub directories
|
|
IEnumerable<DirectoryInfo> ImageDirectories = ImageStage
|
|
.EnumerateDirectories()
|
|
.Concat(Enumerable.Repeat(ImageStage, 1));
|
|
|
|
// Compress the images and convert them to a gif
|
|
foreach (DirectoryInfo ImageDirectory in ImageDirectories)
|
|
{
|
|
FileInfo[] Files = ImageDirectory.GetFiles();
|
|
if (Files.Any())
|
|
{
|
|
if (bCompressImages)
|
|
{
|
|
Image.ConvertImages(ImageDirectory.FullName, ImageDirectory.FullName, "jpg", true);
|
|
}
|
|
|
|
string GifPath = GenerateNotTakenFilePath(Path.Combine(DestinationDirectory, RoleName + "Test.gif"));
|
|
if (bCreateGif)
|
|
{
|
|
if (Image.SaveImagesAsGif(ImageDirectory.FullName, GifPath))
|
|
{
|
|
Log.Info("Saved gif to {0}", GifPath);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Now, copy the compressed images in the temp staged directory to the desired artifact destination
|
|
string RelativeScreenshotDirectory = ScreenshotPath.Replace(SourceDirectory, string.Empty).Trim('\\').Trim('/');
|
|
string ScreenshotDestination = Path.Combine(DestinationDirectory, RelativeScreenshotDirectory);
|
|
SystemHelpers.CopyDirectory(ImageStage.FullName, ScreenshotDestination, SystemHelpers.CopyOptions.Mirror);
|
|
|
|
// Delete the source screenshot directory so they aren't duplicated in the following artifact copy
|
|
ScreenshotDirectory.Delete(true);
|
|
ImageStage.Delete(true);
|
|
}
|
|
catch (Exception Ex)
|
|
{
|
|
Log.Info("Failed to downsize and gif-ify images! {0}", Ex.Message);
|
|
}
|
|
}
|
|
|
|
private void SavePSOs(UnrealTestContext InContext, UnrealSessionInstance.RoleInstance InRunningRole, string DestSavedDir)
|
|
{
|
|
if (InRunningRole.Role.RoleType.IsServer())
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (!InContext.Options.LogPSO)
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (!Directory.Exists(DestSavedDir))
|
|
{
|
|
try
|
|
{
|
|
Directory.CreateDirectory(DestSavedDir);
|
|
}
|
|
catch (Exception Ex)
|
|
{
|
|
Log.Info($"Archive path '{DestSavedDir}' was not found!\n{Ex.Message}");
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Copy over PSOs
|
|
try
|
|
{
|
|
foreach (string ThisFile in CommandUtils.FindFiles_NoExceptions(true, "*.rec.upipelinecache", true, DestSavedDir))
|
|
{
|
|
bool Copied = false;
|
|
string JustFile = Path.GetFileName(ThisFile);
|
|
if (!JustFile.StartsWith("++"))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string[] Parts = JustFile.Split(new Char[] { '+', '-' }).Where(A => A != "").ToArray();
|
|
if (Parts.Count() >= 2)
|
|
{
|
|
string ProjectName = Parts[0].ToString();
|
|
string BuildRoot = CommandUtils.CombinePaths(CommandUtils.RootBuildStorageDirectory());
|
|
|
|
string SrcBuildPath = CommandUtils.CombinePaths(BuildRoot, ProjectName);
|
|
string SrcBuildPath2 = CommandUtils.CombinePaths(BuildRoot, ProjectName.Replace("Game", "").Replace("game", ""));
|
|
|
|
if (!CommandUtils.DirectoryExists(SrcBuildPath))
|
|
{
|
|
SrcBuildPath = SrcBuildPath2;
|
|
}
|
|
if (CommandUtils.DirectoryExists(SrcBuildPath))
|
|
{
|
|
var JustBuildFolder = JustFile.Replace("-" + Parts.Last(), "");
|
|
|
|
string PlatformStr = InRunningRole.Role.Platform.ToString();
|
|
string SrcCLMetaPath = CommandUtils.CombinePaths(SrcBuildPath, JustBuildFolder, PlatformStr, "MetaData");
|
|
if (CommandUtils.DirectoryExists(SrcCLMetaPath))
|
|
{
|
|
string SrcCLMetaPathCollected = CommandUtils.CombinePaths(SrcCLMetaPath, "CollectedPSOs");
|
|
if (!CommandUtils.DirectoryExists(SrcCLMetaPathCollected))
|
|
{
|
|
Log.Info("Creating Directory {0}", SrcCLMetaPathCollected);
|
|
CommandUtils.CreateDirectory(SrcCLMetaPathCollected);
|
|
}
|
|
if (CommandUtils.DirectoryExists(SrcCLMetaPathCollected))
|
|
{
|
|
string DestFile = CommandUtils.CombinePaths(SrcCLMetaPathCollected, JustFile);
|
|
CommandUtils.CopyFile_NoExceptions(ThisFile, DestFile, true);
|
|
if (CommandUtils.FileExists(true, DestFile))
|
|
{
|
|
Log.Info("Deleting local file, copied to {0}", DestFile);
|
|
CommandUtils.DeleteFile_NoExceptions(ThisFile, true);
|
|
Copied = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if (!Copied)
|
|
{
|
|
Log.Warning("Could not find anywhere to put this file {0}", JustFile);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception Ex)
|
|
{
|
|
Log.Info("Failed to copy upipelinecaches to the network {0}", Ex);
|
|
}
|
|
}
|
|
|
|
private string GenerateNotTakenFilePath(string DesiredPath)
|
|
{
|
|
string ResultPath = null;
|
|
|
|
FileInfo PotentialPathFileInfo = new FileInfo(DesiredPath);
|
|
|
|
for (int NumericPostfix = 0; (string.IsNullOrEmpty(ResultPath)) && (NumericPostfix < int.MaxValue); NumericPostfix++)
|
|
{
|
|
string PotentialPath = DesiredPath;
|
|
|
|
if (NumericPostfix > 0)
|
|
{
|
|
PotentialPath = Path.Combine(
|
|
PotentialPathFileInfo.DirectoryName,
|
|
string.Format("{0}_{1}", Path.GetFileNameWithoutExtension(PotentialPathFileInfo.Name), NumericPostfix));
|
|
|
|
if (!string.IsNullOrEmpty(PotentialPathFileInfo.Extension))
|
|
{
|
|
PotentialPath += PotentialPathFileInfo.Extension;
|
|
}
|
|
}
|
|
|
|
bool PathIsTaken = File.Exists(PotentialPath);
|
|
if (PathIsTaken)
|
|
{
|
|
Log.VeryVerbose("File already exists at {0}", PotentialPath);
|
|
}
|
|
else
|
|
{
|
|
ResultPath = PotentialPath;
|
|
}
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(ResultPath))
|
|
{
|
|
throw new AutomationException("Cannot generate not taken file path for the path {0}", DesiredPath);
|
|
}
|
|
|
|
return ResultPath;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Filter that Truncate long file paths such as CrashReporter files. //UECC-Windows-F0DD9BB04C3C9250FAF39D8AB4A88556//
|
|
/// These are particularly problematic with testflights which append a long random name to the destination folder,
|
|
/// easily pushing past 260 chars.
|
|
/// </summary>
|
|
/// <param name="LongFilePath"></param>
|
|
/// <returns></returns>
|
|
private string TruncateLongPathFilter(string LongFilePath)
|
|
{
|
|
Dictionary<string, string> LongCrashReporterStringToIndex = new Dictionary<string, string>();
|
|
|
|
Match RegexMatch = Regex.Match(LongFilePath, @"((?i)UECC)-.+-([\dA-Fa-f]+)");
|
|
|
|
if (RegexMatch.Success)
|
|
{
|
|
string LongString = RegexMatch.Groups[2].ToString();
|
|
|
|
if (!LongCrashReporterStringToIndex.ContainsKey(LongString))
|
|
{
|
|
LongCrashReporterStringToIndex[LongString] = LongCrashReporterStringToIndex.Keys.Count.ToString("D2");
|
|
}
|
|
|
|
string ShortSring = LongCrashReporterStringToIndex[LongString];
|
|
LongFilePath = LongFilePath.Replace(LongString, ShortSring);
|
|
}
|
|
|
|
return LongFilePath;
|
|
}
|
|
|
|
private string GetInstallTime(TimeSpan Time)
|
|
{
|
|
string Hours = Time.Hours > 0 ? string.Format("{0} hrs, ", Time.Hours) : string.Empty;
|
|
string Minutes = Time.Minutes > 0 ? string.Format("{0} mins, ", Time.Minutes) : string.Empty;
|
|
string Seconds = string.Format("{0} secs", Time.Seconds);
|
|
|
|
return Hours + Minutes + Seconds;
|
|
}
|
|
|
|
private bool IsOutOfSpaceException(Exception Ex)
|
|
{
|
|
return Ex.Message.Contains("not enough space", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
private bool IsOverlayException(Exception Ex)
|
|
{
|
|
return Ex.Message.Contains("Overlay Error", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
}
|
|
|
|
}
|