// Copyright Epic Games, Inc. All Rights Reserved. using System.Collections.Generic; using System.Linq; using UnrealBuildTool; namespace Gauntlet { /// /// Device reservation utility class. /// public class UnrealDeviceReservation { public List ReservedDevices { get; protected set; } = new List(); protected List ProblemDevices { get; private set; } = new List(); private static object LockObject = new object(); public bool TryReserveDevices(Dictionary RequiredDeviceTypes, int ExpectedNumberOfDevices, bool bAllowPartialReservations = false) { // First, determine if this is a partial reservation. In the event a device is considered a problem device, don't discard all devices. // Instead, only discard the devices that have a problem. if (bAllowPartialReservations && ReservedDevices.Count > 0) { RequiredDeviceTypes = GetPartialReservationTypes(RequiredDeviceTypes); if (RequiredDeviceTypes.Count == 0) { // We've already reserved all the requested devices return true; } int NewExpectedCount = 0; foreach (var Pair in RequiredDeviceTypes) { NewExpectedCount += Pair.Value; } ExpectedNumberOfDevices = NewExpectedCount; } else { ReleaseDevices(); ProblemDevices.Clear(); } lock (LockObject) { // Ensure the device pool can support the request. This will reserve devices from services if it needs to if (!DevicePool.Instance.CheckAvailableDevices(RequiredDeviceTypes, ProblemDevices)) { return false; } List AcquiredDevices = new List(); List SkippedDevices = new List(); // The pool now contains enough devices to support this reservation. // For each platform, enumerate and select from the available devices as long as they match the predicate. // This will discard any devices that are considered unavailable or marked as a problem foreach (var PlatformReqKP in RequiredDeviceTypes) { UnrealDeviceTargetConstraint Constraint = PlatformReqKP.Key; UnrealTargetPlatform? Platform = Constraint.Platform; int NeedOfThisType = RequiredDeviceTypes[Constraint]; DevicePool.Instance.EnumerateDevices(Constraint, Device => { int HaveOfThisType = AcquiredDevices.Where(D => D.Platform == Device.Platform && Constraint.Check(Device)).Count(); bool WeWant = NeedOfThisType > HaveOfThisType; if (WeWant) { bool Available = Device.IsAvailable; bool Have = AcquiredDevices.Contains(Device); bool Problem = ProblemDevices.Where(D => D.Name == Device.Name && D.Platform == Device.Platform).Count() > 0; Log.Verbose("Device {0}: Available:{1}, Have:{2}, HasProblem:{3}", Device.Name, Available, Have, Problem); if (Available && Have == false && Problem == false) { Log.Info("Acquiring device {0}", Device.Name); AcquiredDevices.Add(Device); HaveOfThisType++; } else { Log.Info("Skipping device {0}", Device.Name); SkippedDevices.Add(Device); } } // continue if we need more of this platform type return HaveOfThisType < NeedOfThisType; }); } // Release any devices that were skipped DevicePool.Instance.ReleaseDevices(SkippedDevices); // Obtain connections to any required devices. EstablishDeviceConnections(AcquiredDevices); // Mark any devices we can't connect to as problems and release them. // Could be something grabbed them before us, could be that they are unresponsive in some way List LostDevices = AcquiredDevices.Where(Device => !Device.IsConnected).ToList(); if (LostDevices.Count > 0) { LostDevices.ForEach(Device => MarkProblemDevice(Device, "Agent lost connection to device.")); AcquiredDevices = AcquiredDevices.Except(LostDevices).ToList(); ReleaseProblemDevices(); } // If we failed to acquire every device we needed, but support partial reservations, hold onto any devices we did manage to acquire if (AcquiredDevices.Count < ExpectedNumberOfDevices && AcquiredDevices.Count > 0) { if (bAllowPartialReservations) { if (ClaimDevices(AcquiredDevices)) { ReservedDevices.AddRange(AcquiredDevices); } } else { DevicePool.Instance.ReleaseDevices(AcquiredDevices); } return false; } if (!ClaimDevices(AcquiredDevices)) { return false; } ReservedDevices.AddRange(AcquiredDevices); } return true; } public void ReleaseDevices() { if ((ReservedDevices != null) && (ReservedDevices.Count() > 0)) { foreach (ITargetDevice Device in ReservedDevices) { IDeviceUsageReporter.RecordEnd(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Device); } DevicePool.Instance.ReleaseDevices(ReservedDevices); ReservedDevices.Clear(); } } public IEnumerable ReleaseProblemDevices() { List Devices = new(); bool bReservedDevices = ReservedDevices != null && ReservedDevices.Any(); bool bProblemDevices = ProblemDevices != null && ProblemDevices.Any(); if(bReservedDevices && bProblemDevices) { foreach(ProblemDevice Problem in ProblemDevices) { foreach(ITargetDevice Device in ReservedDevices) { if (Problem.Name == Device.Name && Problem.Platform == Device.Platform) { Devices.Add(Device); } } } DevicePool.Instance.ReleaseDevices(Devices); ProblemDevices.Clear(); foreach (ITargetDevice Device in Devices) { ReservedDevices.Remove(Device); IDeviceUsageReporter.RecordEnd(Device.Name, Device.Platform, IDeviceUsageReporter.EventType.Device); } } return Devices; } public void MarkProblemDevice(ITargetDevice Device, string ErrorMessage) { if (ProblemDevices.Where(D => D.Name == Device.Name && D.Platform == Device.Platform).Count() > 0) { return; } // report device has a problem to the pool DevicePool.Instance.ReportDeviceError(Device, ErrorMessage); if (Device.Platform != null) { ProblemDevices.Add(new ProblemDevice(Device.Name, Device.Platform.Value)); } } /// /// Converts a set of required devices into only what is necessary by analyzing the list of already reserved devices /// /// Request list /// The difference in constraints/values between the request types and the currently reserved devices private Dictionary GetPartialReservationTypes(Dictionary DeviceTypes) { Dictionary CurrentlyAvailableTypes = new(); foreach (ITargetDevice Device in ReservedDevices) { UnrealDeviceTargetConstraint Constraint = DevicePool.Instance.GetConstraint(Device); if (CurrentlyAvailableTypes.ContainsKey(Constraint)) { ++CurrentlyAvailableTypes[Constraint]; } else { CurrentlyAvailableTypes.Add(Constraint, 1); } } Dictionary NewSet = new(); foreach (var Pair in DeviceTypes) { if (CurrentlyAvailableTypes.ContainsKey(Pair.Key)) { // If we have more devices already reserved than what is requested we don't need to reserve devices if (CurrentlyAvailableTypes[Pair.Key] < Pair.Value) { // We have at least 1 device with this constraint, but not enough to fufill the request. Request only what is needed for this constraint NewSet.Add(Pair.Key, Pair.Value - CurrentlyAvailableTypes[Pair.Key]); } } else { // No existing constraint, we don't have these devices NewSet.Add(Pair.Key, Pair.Value); } } return NewSet; } private void EstablishDeviceConnections(IEnumerable Devices) { foreach (ITargetDevice Device in Devices) { if (Device.IsOn == false) { Log.Info("Powering on {0}", Device); Device.PowerOn(); } else if (Globals.Params.ParseParam("reboot")) { Log.Info("Rebooting {0}", Device); Device.Reboot(); // Force a re-login after a reboot to be sure it is ready. if (Globals.Params.ParseParam("VerifyLogin") && Device is IOnlineServiceLogin DeviceLogin) { Log.Info("Revalidate device login after reboot..."); DeviceLogin.VerifyLogin(); } } if (Device.IsConnected == false) { Log.Verbose("Connecting to {0}", Device); Device.Connect(); } } } private bool ClaimDevices(IEnumerable Devices) { bool bSuccess = DevicePool.Instance.ClaimDevices(Devices); if (!bSuccess) { Log.Warning("Attempted to claim the following devices which are already marked as claimed:\n{Devices}", string.Join("\n\t", Devices.Select(Device => Device.Name))); } return bSuccess; } } }