// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.Threading; using System.Reflection; using System.Linq; using System.Threading.Tasks; using AutomationTool; using UnrealBuildTool; using EpicGames.Core; using System.Text.RegularExpressions; using UnrealBuildBase; using Microsoft.Extensions.Logging; using static AutomationTool.CommandUtils; public class SharedCookedBuild { private const string SyncedBuildFileName = "SyncedBuild.txt"; /// /// Types of shared cook base builds /// public enum SharedCookType { /// /// Only allow shared cook build of version identical to local sync /// Exact, /// /// Allow any previous version that is only a content change from local sync /// Content, /// /// Closest previous version, regardless of code/content changes /// Any, } public enum SharedCookSource { Manifest, LooseFiles, } private FileReference ProjectFile { get; set; } private DirectoryReference InstallPath { get; set; } private HashSet TargetPlatforms { get; set; } private SharedCookType BuildType { get; set; } private List CandidateBuilds { get; set; } private BuildVersion LocalSync { get; set; } public SharedCookedBuild(ProjectParams Params) { List Platforms = new List(); foreach (TargetPlatformDescriptor ClientPlatform in Params.ClientTargetPlatforms) { TargetPlatformDescriptor DataPlatformDesc = Params.GetCookedDataPlatformForClientTarget(ClientPlatform); Platforms.Add(Platform.Platforms[DataPlatformDesc].GetCookPlatform(false, Params.Client)); } SharedCookType BuildType = (SharedCookType)Enum.Parse(typeof(SharedCookType), Params.IterateSharedCookedBuild, true); SharedCookedBuildConstructor(Params.RawProjectPath, Platforms, BuildType); } public SharedCookedBuild(FileReference ProjectFile, IEnumerable TargetPlatforms, SharedCookType BuildType) { SharedCookedBuildConstructor(ProjectFile, TargetPlatforms, BuildType); } private void SharedCookedBuildConstructor(FileReference ProjectFile, IEnumerable Platforms, SharedCookType BuildType) { TargetPlatforms = new HashSet(); CandidateBuilds = new List(); this.ProjectFile = ProjectFile; this.BuildType = BuildType; InstallPath = DirectoryReference.Combine(ProjectFile.Directory, "Saved", "SharedIterativeBuild"); LocalSync = GetLocalSync(); foreach (string Platform in Platforms) { TargetPlatforms.Add(Platform); } } public void CopySharedCookedBuilds() { CandidateBuilds = FindBestBuilds(); if (CandidateBuilds.Count == 0) { throw new AutomationException("No valid shared cooked builds available"); } foreach (string TargetPlatform in TargetPlatforms) { // Prefer existing sync if listed IEnumerable LocalSync = CandidateBuilds.Where(x => x.GetType() == typeof(ExistingSharedCookedBuild) && x.Platform.Equals(TargetPlatform, StringComparison.InvariantCultureIgnoreCase)); if (LocalSync.Count() > 0) { LocalSync.First().CopyBuild(InstallPath); continue; } IEnumerable PlatformBuilds = CandidateBuilds.Where(x => x.Platform.Equals(TargetPlatform, StringComparison.InvariantCultureIgnoreCase)); if (PlatformBuilds.Count() > 0) { ISharedCookedBuild Build = PlatformBuilds.First(); if (!Build.CopyBuild(InstallPath)) { throw new AutomationException("Failed to copy shared build for {0} {1}", Build.Platform, Build.CL); } } } } public static BuildVersion GetLocalSync() { BuildVersion P4Version = new BuildVersion(); if (CommandUtils.P4Enabled) { P4Version.BranchName = CommandUtils.P4Env.Branch.Replace("/", "+"); P4Version.Changelist = CommandUtils.P4Env.Changelist; P4Version.CompatibleChangelist = CommandUtils.P4Env.CodeChangelist; } BuildVersion UGSVersion; if (BuildVersion.TryRead(BuildVersion.GetDefaultFileName(), out UGSVersion)) { return UGSVersion; } if (!CommandUtils.P4Enabled) { throw new AutomationException("Cannot determine local sync"); } return P4Version; } public List FindBestBuilds() { // Attempt manifest searching first ConfigHierarchy Hierarchy = ConfigCache.ReadHierarchy(ConfigHierarchyType.Engine, DirectoryReference.FromFile(ProjectFile), UnrealTargetPlatform.Win64); IReadOnlyList RawSharedCookedSources = null; Hierarchy.TryGetValues("SharedCookedBuildSettings", "SharedCookedSources", out RawSharedCookedSources); if (RawSharedCookedSources == null) { throw new AutomationException("Unable to locate shared cooked builds. SharedCookedSources not set in Engine.ini [SharedCookedBuildSettings]"); } List> ParsedSharedCookSources = new List>(); foreach (string RawConfig in RawSharedCookedSources) { Dictionary ParsedSource = null; if (ConfigHierarchy.TryParse(RawConfig, out ParsedSource)) { ParsedSharedCookSources.Add(ParsedSource); } } List CandidateBuilds = new List(); // If existing sync is present, stick to it. Read version out of sync file foreach (string Platform in TargetPlatforms) { FileReference SyncedBuildFile = new FileReference(CommandUtils.CombinePaths(InstallPath.FullName, Platform, SyncedBuildFileName)); if (FileReference.Exists(SyncedBuildFile)) { string[] SyncedBuildInfo = FileReference.ReadAllLines(SyncedBuildFile); int SyncedCL = int.Parse(SyncedBuildInfo[0]); if (IsValidCL(SyncedCL, BuildType, LocalSync)) { CandidateBuilds.Add(new ExistingSharedCookedBuild() { CL = SyncedCL, Platform = Platform }); } } } foreach (Dictionary Source in ParsedSharedCookSources) { SharedCookSource SourceType = (SharedCookSource)Enum.Parse(typeof(SharedCookSource), Source["Type"], true); foreach (string Platform in TargetPlatforms) { if (SourceType == SharedCookSource.Manifest) { CandidateBuilds.AddRange(FindValidManifestBuilds(Source["Path"], Platform)); } else if (SourceType == SharedCookSource.LooseFiles) { CandidateBuilds.AddRange(FindValidLooseBuilds(Source["Path"], Platform)); } } } // Strip all failed searches CandidateBuilds.RemoveAll(x => x == null); // Make sure we have a matching CL for all target platforms, regardless of source List OrderedDistinctCLs = CandidateBuilds.Select(x => x.CL).Distinct().OrderByDescending(i => i).ToList(); int BestCL = -1; foreach (int CL in OrderedDistinctCLs) { // Ensure we have a platform for each HashSet CLPlatforms = new HashSet(CandidateBuilds.Where(x => x.CL == CL).Select(x => x.Platform).ToList()); if (CLPlatforms.SetEquals(TargetPlatforms)) { BestCL = CL; break; } } if (BestCL < 0) { Logger.LogError("Could not locate valid shared cooked build for all target platforms"); Logger.LogError("Current CL: {Arg0}, Current Code CL: {Arg1}", LocalSync.Changelist, LocalSync.CompatibleChangelist); } return CandidateBuilds.Where(x => x.CL == BestCL).ToList(); } public List FindValidManifestBuilds(string Path, string TargetPlatform) { List ValidBuilds = new List(); Tuple SplitPath = SplitOnFixedPrefix(Path); Regex Pattern = RegexFromWildcards(SplitPath.Item2, LocalSync, TargetPlatform); DirectoryReference SearchDir = new DirectoryReference(SplitPath.Item1); if (DirectoryReference.Exists(SearchDir)) { foreach (FileReference File in DirectoryReference.EnumerateFiles(SearchDir)) { Match Match = Pattern.Match(File.FullName); if (Match.Success) { int MatchCL = int.Parse(Match.Result("${CL}")); if (IsValidCL(MatchCL, BuildType, LocalSync)) { ValidBuilds.Add(new ManifestSharedCookedBuild { CL = MatchCL, Manifest = File, Platform = TargetPlatform }); } } } } return ValidBuilds; } public List FindValidLooseBuilds(string Path, string TargetPlatform) { List ValidBuilds = new List(); Tuple SplitPath = SplitOnFixedPrefix(Path); Regex Pattern = RegexFromWildcards(SplitPath.Item2, LocalSync, TargetPlatform); // Search for all available builds const string MetaDataFilename = "\\Metadata\\DevelopmentAssetRegistry.bin"; string BuildRule = SplitPath.Item2 + MetaDataFilename; BuildRule = BuildRule.Replace("[BRANCHNAME]", LocalSync.BranchName); BuildRule = BuildRule.Replace("[PLATFORM]", TargetPlatform); string IncludeRule = BuildRule.Replace("[CL]", "*"); string ExcludeRule = BuildRule.Replace("[CL]", "*-PF-*"); // Exclude preflights FileFilter BuildSearch = new FileFilter(); BuildSearch.AddRule(IncludeRule); BuildSearch.AddRule(ExcludeRule, FileFilterType.Exclude); foreach (FileReference CandidateBuild in BuildSearch.ApplyToDirectory(new DirectoryReference(SplitPath.Item1), false)) { string BaseBuildPath = CandidateBuild.FullName.Replace(MetaDataFilename, ""); Match Match = Pattern.Match(BaseBuildPath); if (Match.Success) { int MatchCL = int.Parse(Match.Result("${CL}")); if (IsValidCL(MatchCL, BuildType, LocalSync)) { ValidBuilds.Add(new LooseSharedCookedBuild { CL = MatchCL, Path = new DirectoryReference(BaseBuildPath), Platform = TargetPlatform }); } } } return ValidBuilds; } private static bool IsValidCL(int CL, SharedCookType BuildType, BuildVersion Version) { if (BuildType == SharedCookType.Exact && CL == Version.Changelist) { return true; } else if (BuildType == SharedCookType.Content && CL >= Version.EffectiveCompatibleChangelist && CL <= Version.Changelist) { return true; } else if (BuildType == SharedCookType.Any && CL <= Version.Changelist) { return true; } return false; } private static Regex RegexFromWildcards(string Path, BuildVersion Version, string TargetPlatform) { string Pattern = Path.Replace(@"\", @"\\"); Pattern = Pattern.Replace("[BRANCHNAME]", Version.BranchName.Replace(@"+", @"\+")); Pattern = Pattern.Replace("[PLATFORM]", TargetPlatform); Pattern = Pattern.Replace("[CL]", @"(?\d+)"); return new Regex(Pattern); } private static Tuple SplitOnFixedPrefix(string Path) { int IndexOfFirstParam = Path.IndexOf("["); int PrefixStart = Path.LastIndexOf(@"\", IndexOfFirstParam); return new Tuple(Path.Substring(0, PrefixStart), Path.Substring(PrefixStart)); } public interface ISharedCookedBuild { int CL { get; set; } string Platform { get; set; } bool CopyBuild(DirectoryReference InstallPath); } private class ManifestSharedCookedBuild : ISharedCookedBuild { public int CL { get; set; } public string Platform { get; set; } public FileReference Manifest { get; set; } public bool CopyBuild(DirectoryReference InstallPath) { Logger.LogInformation("Installing shared cooked build from manifest: {Arg0} to {Arg1}", Manifest.FullName, InstallPath.FullName); DirectoryReference PlatformInstallPath = DirectoryReference.Combine(InstallPath, Platform.ToString()); FileReference PreviousManifest = FileReference.Combine(PlatformInstallPath, ".build", "Current.manifest"); FileReference BPTI = FileReference.Combine(Unreal.RootDirectory, "Engine", "Restricted", "NotForLicensees", "Binaries", "Win64", "BuildPatchToolInstaller.exe"); if (!FileReference.Exists(BPTI)) { Logger.LogInformation("Could not locate BuildPatchToolInstaller.exe"); return false; } bool PreviousManifestExists = FileReference.Exists(PreviousManifest); if (!PreviousManifestExists && DirectoryReference.Exists(PlatformInstallPath)) { DirectoryReference.Delete(PlatformInstallPath, true); } IProcessResult Result = CommandUtils.Run(BPTI.FullName, string.Format("-Manifest={0} -OutputDir={1} -stdout -GenericConsoleOutput", Manifest.FullName, PlatformInstallPath.FullName), null, CommandUtils.ERunOptions.Default); if (Result.ExitCode != 0) { Logger.LogWarning("Failed to install manifest {Arg0} to {Arg1}", Manifest.FullName, PlatformInstallPath.FullName); return false; } FileReference SyncedBuildFile = new FileReference(CommandUtils.CombinePaths(PlatformInstallPath.FullName, SyncedBuildFileName)); FileReference.WriteAllLines(SyncedBuildFile, new string[] { CL.ToString(), Manifest.FullName }); return true; } } private class LooseSharedCookedBuild : ISharedCookedBuild { public int CL { get; set; } public string Platform { get; set; } public DirectoryReference Path { get; set; } public bool CopyBuild(DirectoryReference InstallPath) { Logger.LogInformation("Copying shared cooked build from stage directory: {Arg0} to {Arg1}", Path.FullName, InstallPath.FullName); // Delete existing if (DirectoryReference.Exists(InstallPath)) { DirectoryReference.Delete(InstallPath, true); } DirectoryReference.CreateDirectory(InstallPath); // Copy new if (!CommandUtils.CopyDirectory_NoExceptions(Path.FullName, InstallPath.FullName)) { Logger.LogWarning("Failed to copy {Arg0} -> {Arg1}", Path.FullName, InstallPath.FullName); return false; } FileReference SyncedBuildFile = new FileReference(CommandUtils.CombinePaths(InstallPath.FullName, SyncedBuildFileName)); FileReference.WriteAllLines(SyncedBuildFile, new string[] { CL.ToString(), Path.FullName }); return true; } } private class ExistingSharedCookedBuild : ISharedCookedBuild { public int CL { get; set; } public string Platform { get; set; } public bool CopyBuild(DirectoryReference InstallPath) { Logger.LogInformation("Using previously synced shared cooked build"); return true; } } }