// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text; using EpicGames.Core; using Microsoft.Extensions.Logging; using UnrealBuildBase; namespace UnrealBuildTool { /// /// Caches include dependency information to speed up preprocessing on subsequent runs. /// [DebuggerDisplay("{Location}")] class ActionHistoryLayer { /// /// Version number to check /// const int CurrentVersion = 3; /// /// Path to store the cache data to. /// public FileReference Location { get; } /// /// The Attributes used to produce files, keyed by the absolute file paths. /// ConcurrentDictionary OutputItemToAttributeHash = new ConcurrentDictionary(); /// /// Whether the dependency cache is dirty and needs to be saved. /// bool bModified; /// /// Constructor /// /// File to store this history in /// Logger for output public ActionHistoryLayer(FileReference Location, ILogger Logger) { this.Location = Location; if (FileReference.Exists(Location)) { Load(Logger); } } /// /// Attempts to load this action history from disk /// void Load(ILogger Logger) { try { using (BinaryArchiveReader Reader = new BinaryArchiveReader(Location)) { int Version = Reader.ReadInt(); if (Version != CurrentVersion) { Logger.LogDebug("Unable to read action history from {Location}; version {Version} vs current {CurrentVersion}", Location, Version, CurrentVersion); return; } OutputItemToAttributeHash = new ConcurrentDictionary(Reader.ReadDictionary(() => Reader.ReadFileItem()!, () => Reader.ReadIoHash() ?? IoHash.Zero)!); } } catch (Exception Ex) { Logger.LogWarning("Unable to read {Location}. See log for additional information.", Location); Logger.LogDebug("{Ex}", ExceptionUtils.FormatExceptionDetails(Ex)); } } /// /// Saves this action history to disk /// public void Save() { if (bModified) { DirectoryReference.CreateDirectory(Location.Directory); using (BinaryArchiveWriter Writer = new BinaryArchiveWriter(Location)) { Writer.WriteInt(CurrentVersion); Writer.WriteDictionary(OutputItemToAttributeHash, Key => Writer.WriteFileItem(Key), Value => Writer.WriteIoHash(Value)); } bModified = false; } } /// /// Computes the case-invariant IoHash for a string /// /// The text to hash /// IoHash of the string static IoHash ComputeHash(string Text) => IoHash.Compute(Encoding.Unicode.GetBytes(Text.ToUpperInvariant())); /// /// Gets the producing attributes for the given file /// /// The output file to look for /// Receives the Attributes used to produce this file /// True if Attributes have changed and is updated, false otherwise public bool UpdateProducingCommandLine(FileItem File, string Attributes) { IoHash NewHash = ComputeHash(Attributes); for (; ; ) { if (OutputItemToAttributeHash.TryAdd(File, NewHash)) { // If this is a new entry we're done bModified = true; return true; } else { if (OutputItemToAttributeHash.TryGetValue(File, out IoHash OldHash)) { if (NewHash == OldHash) { // hashes are the same, no update needed return false; } else { // Try to update with the new value if (OutputItemToAttributeHash.TryUpdate(File, NewHash, OldHash)) { bModified = true; return true; } } } } } } /// /// Gets the location for the engine action history /// /// Target name being built /// The platform being built /// Type of the target being built /// The target architecture(s) /// Path to the engine action history for this target public static FileReference GetEngineLocation(string TargetName, UnrealTargetPlatform Platform, TargetType TargetType, UnrealArchitectures Architectures) { string AppName; if (TargetType == TargetType.Program) { AppName = TargetName; } else { AppName = UEBuildTarget.GetAppNameForTargetType(TargetType); } return FileReference.Combine(Unreal.EngineDirectory, UEBuildTarget.GetPlatformIntermediateFolder(Platform, Architectures, false), AppName, "ActionHistory.bin"); } /// /// Gets the location of the project action history /// /// Path to the project file /// Platform being built /// Name of the target being built /// The target architecture(s) /// Path to the project action history public static FileReference GetProjectLocation(FileReference ProjectFile, string TargetName, UnrealTargetPlatform Platform, UnrealArchitectures Architectures) { return FileReference.Combine(ProjectFile.Directory, UEBuildTarget.GetPlatformIntermediateFolder(Platform, Architectures, false), TargetName, "ActionHistory.dat"); } /// /// Enumerates all the locations of action history files for the given target /// /// Project file for the target being built /// Name of the target /// Platform being built /// The target type /// The target architecture(s) /// Dependency cache hierarchy for the given project public static IEnumerable GetFilesToClean(FileReference ProjectFile, string TargetName, UnrealTargetPlatform Platform, TargetType TargetType, UnrealArchitectures Architectures) { if (ProjectFile == null || !Unreal.IsEngineInstalled()) { yield return GetEngineLocation(TargetName, Platform, TargetType, Architectures); } if (ProjectFile != null) { yield return GetProjectLocation(ProjectFile, TargetName, Platform, Architectures); } } } /// /// Information about actions producing artifacts under a particular directory /// [DebuggerDisplay("{BaseDir}")] class ActionHistoryPartition { /// /// The base directory for this partition /// public DirectoryReference BaseDir { get; } /// /// Used to ensure exclusive access to the layers list /// readonly object LockObject = new object(); /// /// Map of filename to layer /// IReadOnlyList Layers = new List(); /// /// Construct a new partition /// /// The base directory for this partition public ActionHistoryPartition(DirectoryReference BaseDir) { this.BaseDir = BaseDir; } /// /// Attempt to update the producing command line for the given file /// /// The file to update /// The new attributes /// Logger for output /// True if the attributes were updated, false otherwise public bool UpdateProducingAttributes(FileItem File, string Attributes, ILogger Logger) { FileReference LayerLocation = GetLayerLocationForFile(File.Location); ActionHistoryLayer? Layer = Layers.FirstOrDefault(x => x.Location == LayerLocation); if (Layer == null) { lock (LockObject) { Layer = Layers.FirstOrDefault(x => x.Location == LayerLocation); if (Layer == null) { Layer = new ActionHistoryLayer(LayerLocation, Logger); List NewLayers = new List(Layers); NewLayers.Add(Layer); Layers = NewLayers; } } } return Layer.UpdateProducingCommandLine(File, Attributes); } /// /// Get the path to the action history layer to use for the given file /// /// Path to the file to use /// Path to the file public FileReference GetLayerLocationForFile(FileReference Location) { int Offset = BaseDir.FullName.Length; for (; ; ) { int NameOffset = Offset + 1; // Get the next directory separator Offset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, NameOffset + 1); if (Offset == -1) { break; } // Get the length of the name int NameLength = Offset - NameOffset; // Try to find Binaries// in the path if (MatchPathFragment(Location, NameOffset, NameLength, "Binaries")) { int PlatformOffset = Offset + 1; int PlatformEndOffset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, PlatformOffset); if (PlatformEndOffset != -1) { string PlatformName = Location.FullName.Substring(PlatformOffset, PlatformEndOffset - PlatformOffset); return FileReference.Combine(BaseDir, "Intermediate", "Build", PlatformName, "ActionHistory.bin"); } } // Try to find /Intermediate/Build/// in the path if (MatchPathFragment(Location, NameOffset, NameLength, "Intermediate")) { int BuildOffset = Offset + 1; int BuildEndOffset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, BuildOffset); if (BuildEndOffset != -1 && MatchPathFragment(Location, BuildOffset, BuildEndOffset - BuildOffset, "Build")) { // Skip the platform, target/app name, and configuration int EndOffset = BuildEndOffset; for (int Idx = 0; ; Idx++) { EndOffset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, EndOffset + 1); if (EndOffset == -1) { break; } if (Idx == 2) { return FileReference.Combine(BaseDir, Location.FullName.Substring(NameOffset, EndOffset - NameOffset), "ActionHistory.bin"); } } } } } return FileReference.Combine(BaseDir, "Intermediate", "Build", "ActionHistory.bin"); } /// /// Attempts to match a substring of a path with the given fragment /// /// Path to match against /// Offset of the substring to match /// Length of the substring to match /// The path fragment /// True if the substring matches static bool MatchPathFragment(FileReference Location, int Offset, int Length, string Fragment) { return Length == Fragment.Length && String.Compare(Location.FullName, Offset, Fragment, 0, Fragment.Length, FileReference.Comparison) == 0; } /// /// Saves the modified layers /// public void Save() { foreach (ActionHistoryLayer Layer in Layers) { Layer.Save(); } } } /// /// A collection of ActionHistory layers /// class ActionHistory { /// /// The lock object for this history /// object LockObject = new object(); /// /// List of partitions /// List Partitions = new List(); /// /// Constructor /// public ActionHistory() { Partitions.Add(new ActionHistoryPartition(Unreal.EngineDirectory)); } /// /// Reads a cache from the given location, or creates it with the given settings /// /// Base directory for files that this cache should store data for /// Reference to a dependency cache with the given settings public void Mount(DirectoryReference BaseDir) { lock (LockObject) { ActionHistoryPartition? Partition = Partitions.FirstOrDefault(x => x.BaseDir == BaseDir); if (Partition == null) { Partition = new ActionHistoryPartition(BaseDir); Partitions.Add(Partition); } } } /// /// Gets the producing command line for the given file /// /// The output file to look for /// Receives the Attributes used to produce this file /// Logger for output /// True if the output item exists public bool UpdateProducingAttributes(FileItem File, string Attributes, ILogger Logger) { foreach (ActionHistoryPartition Partition in Partitions) { if (File.Location.IsUnderDirectory(Partition.BaseDir)) { return Partition.UpdateProducingAttributes(File, Attributes, Logger); } } Logger.LogWarning("File {FileLocation} is not under any action history root directory", File.Location); return false; } /// /// Saves all layers of this action history /// public void Save() { lock (LockObject) { foreach (ActionHistoryPartition Partition in Partitions) { Partition.Save(); } } } } }