// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using EpicGames.Core;
using Microsoft.Extensions.Logging;
using UnrealBuildBase;
namespace UnrealBuildTool
{
///
/// Caches include dependency information to speed up preprocessing on subsequent runs.
///
[DebuggerDisplay("{Location}")]
class ActionHistoryLayer
{
///
/// Version number to check
///
const int CurrentVersion = 3;
///
/// Path to store the cache data to.
///
public FileReference Location
{
get;
}
///
/// The Attributes used to produce files, keyed by the absolute file paths.
///
ConcurrentDictionary OutputItemToAttributeHash = new ConcurrentDictionary();
///
/// Whether the dependency cache is dirty and needs to be saved.
///
bool bModified;
///
/// Constructor
///
/// File to store this history in
/// Logger for output
public ActionHistoryLayer(FileReference Location, ILogger Logger)
{
this.Location = Location;
if (FileReference.Exists(Location))
{
Load(Logger);
}
}
///
/// Attempts to load this action history from disk
///
void Load(ILogger Logger)
{
try
{
using (BinaryArchiveReader Reader = new BinaryArchiveReader(Location))
{
int Version = Reader.ReadInt();
if (Version != CurrentVersion)
{
Logger.LogDebug("Unable to read action history from {Location}; version {Version} vs current {CurrentVersion}", Location, Version, CurrentVersion);
return;
}
OutputItemToAttributeHash = new ConcurrentDictionary(Reader.ReadDictionary(() => Reader.ReadFileItem()!, () => Reader.ReadIoHash() ?? IoHash.Zero)!);
}
}
catch (Exception Ex)
{
Logger.LogWarning("Unable to read {Location}. See log for additional information.", Location);
Logger.LogDebug("{Ex}", ExceptionUtils.FormatExceptionDetails(Ex));
}
}
///
/// Saves this action history to disk
///
public void Save()
{
if (bModified)
{
DirectoryReference.CreateDirectory(Location.Directory);
using (BinaryArchiveWriter Writer = new BinaryArchiveWriter(Location))
{
Writer.WriteInt(CurrentVersion);
Writer.WriteDictionary(OutputItemToAttributeHash, Key => Writer.WriteFileItem(Key), Value => Writer.WriteIoHash(Value));
}
bModified = false;
}
}
///
/// Computes the case-invariant IoHash for a string
///
/// The text to hash
/// IoHash of the string
static IoHash ComputeHash(string Text) => IoHash.Compute(Encoding.Unicode.GetBytes(Text.ToUpperInvariant()));
///
/// Gets the producing attributes for the given file
///
/// The output file to look for
/// Receives the Attributes used to produce this file
/// True if Attributes have changed and is updated, false otherwise
public bool UpdateProducingCommandLine(FileItem File, string Attributes)
{
IoHash NewHash = ComputeHash(Attributes);
for (; ; )
{
if (OutputItemToAttributeHash.TryAdd(File, NewHash))
{
// If this is a new entry we're done
bModified = true;
return true;
}
else
{
if (OutputItemToAttributeHash.TryGetValue(File, out IoHash OldHash))
{
if (NewHash == OldHash)
{
// hashes are the same, no update needed
return false;
}
else
{
// Try to update with the new value
if (OutputItemToAttributeHash.TryUpdate(File, NewHash, OldHash))
{
bModified = true;
return true;
}
}
}
}
}
}
///
/// Gets the location for the engine action history
///
/// Target name being built
/// The platform being built
/// Type of the target being built
/// The target architecture(s)
/// Path to the engine action history for this target
public static FileReference GetEngineLocation(string TargetName, UnrealTargetPlatform Platform, TargetType TargetType, UnrealArchitectures Architectures)
{
string AppName;
if (TargetType == TargetType.Program)
{
AppName = TargetName;
}
else
{
AppName = UEBuildTarget.GetAppNameForTargetType(TargetType);
}
return FileReference.Combine(Unreal.EngineDirectory, UEBuildTarget.GetPlatformIntermediateFolder(Platform, Architectures, false), AppName, "ActionHistory.bin");
}
///
/// Gets the location of the project action history
///
/// Path to the project file
/// Platform being built
/// Name of the target being built
/// The target architecture(s)
/// Path to the project action history
public static FileReference GetProjectLocation(FileReference ProjectFile, string TargetName, UnrealTargetPlatform Platform, UnrealArchitectures Architectures)
{
return FileReference.Combine(ProjectFile.Directory, UEBuildTarget.GetPlatformIntermediateFolder(Platform, Architectures, false), TargetName, "ActionHistory.dat");
}
///
/// Enumerates all the locations of action history files for the given target
///
/// Project file for the target being built
/// Name of the target
/// Platform being built
/// The target type
/// The target architecture(s)
/// Dependency cache hierarchy for the given project
public static IEnumerable GetFilesToClean(FileReference ProjectFile, string TargetName, UnrealTargetPlatform Platform, TargetType TargetType, UnrealArchitectures Architectures)
{
if (ProjectFile == null || !Unreal.IsEngineInstalled())
{
yield return GetEngineLocation(TargetName, Platform, TargetType, Architectures);
}
if (ProjectFile != null)
{
yield return GetProjectLocation(ProjectFile, TargetName, Platform, Architectures);
}
}
}
///
/// Information about actions producing artifacts under a particular directory
///
[DebuggerDisplay("{BaseDir}")]
class ActionHistoryPartition
{
///
/// The base directory for this partition
///
public DirectoryReference BaseDir { get; }
///
/// Used to ensure exclusive access to the layers list
///
readonly object LockObject = new object();
///
/// Map of filename to layer
///
IReadOnlyList Layers = new List();
///
/// Construct a new partition
///
/// The base directory for this partition
public ActionHistoryPartition(DirectoryReference BaseDir)
{
this.BaseDir = BaseDir;
}
///
/// Attempt to update the producing command line for the given file
///
/// The file to update
/// The new attributes
/// Logger for output
/// True if the attributes were updated, false otherwise
public bool UpdateProducingAttributes(FileItem File, string Attributes, ILogger Logger)
{
FileReference LayerLocation = GetLayerLocationForFile(File.Location);
ActionHistoryLayer? Layer = Layers.FirstOrDefault(x => x.Location == LayerLocation);
if (Layer == null)
{
lock (LockObject)
{
Layer = Layers.FirstOrDefault(x => x.Location == LayerLocation);
if (Layer == null)
{
Layer = new ActionHistoryLayer(LayerLocation, Logger);
List NewLayers = new List(Layers);
NewLayers.Add(Layer);
Layers = NewLayers;
}
}
}
return Layer.UpdateProducingCommandLine(File, Attributes);
}
///
/// Get the path to the action history layer to use for the given file
///
/// Path to the file to use
/// Path to the file
public FileReference GetLayerLocationForFile(FileReference Location)
{
int Offset = BaseDir.FullName.Length;
for (; ; )
{
int NameOffset = Offset + 1;
// Get the next directory separator
Offset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, NameOffset + 1);
if (Offset == -1)
{
break;
}
// Get the length of the name
int NameLength = Offset - NameOffset;
// Try to find Binaries// in the path
if (MatchPathFragment(Location, NameOffset, NameLength, "Binaries"))
{
int PlatformOffset = Offset + 1;
int PlatformEndOffset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, PlatformOffset);
if (PlatformEndOffset != -1)
{
string PlatformName = Location.FullName.Substring(PlatformOffset, PlatformEndOffset - PlatformOffset);
return FileReference.Combine(BaseDir, "Intermediate", "Build", PlatformName, "ActionHistory.bin");
}
}
// Try to find /Intermediate/Build/// in the path
if (MatchPathFragment(Location, NameOffset, NameLength, "Intermediate"))
{
int BuildOffset = Offset + 1;
int BuildEndOffset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, BuildOffset);
if (BuildEndOffset != -1 && MatchPathFragment(Location, BuildOffset, BuildEndOffset - BuildOffset, "Build"))
{
// Skip the platform, target/app name, and configuration
int EndOffset = BuildEndOffset;
for (int Idx = 0; ; Idx++)
{
EndOffset = Location.FullName.IndexOf(Path.DirectorySeparatorChar, EndOffset + 1);
if (EndOffset == -1)
{
break;
}
if (Idx == 2)
{
return FileReference.Combine(BaseDir, Location.FullName.Substring(NameOffset, EndOffset - NameOffset), "ActionHistory.bin");
}
}
}
}
}
return FileReference.Combine(BaseDir, "Intermediate", "Build", "ActionHistory.bin");
}
///
/// Attempts to match a substring of a path with the given fragment
///
/// Path to match against
/// Offset of the substring to match
/// Length of the substring to match
/// The path fragment
/// True if the substring matches
static bool MatchPathFragment(FileReference Location, int Offset, int Length, string Fragment)
{
return Length == Fragment.Length && String.Compare(Location.FullName, Offset, Fragment, 0, Fragment.Length, FileReference.Comparison) == 0;
}
///
/// Saves the modified layers
///
public void Save()
{
foreach (ActionHistoryLayer Layer in Layers)
{
Layer.Save();
}
}
}
///
/// A collection of ActionHistory layers
///
class ActionHistory
{
///
/// The lock object for this history
///
object LockObject = new object();
///
/// List of partitions
///
List Partitions = new List();
///
/// Constructor
///
public ActionHistory()
{
Partitions.Add(new ActionHistoryPartition(Unreal.EngineDirectory));
}
///
/// Reads a cache from the given location, or creates it with the given settings
///
/// Base directory for files that this cache should store data for
/// Reference to a dependency cache with the given settings
public void Mount(DirectoryReference BaseDir)
{
lock (LockObject)
{
ActionHistoryPartition? Partition = Partitions.FirstOrDefault(x => x.BaseDir == BaseDir);
if (Partition == null)
{
Partition = new ActionHistoryPartition(BaseDir);
Partitions.Add(Partition);
}
}
}
///
/// Gets the producing command line for the given file
///
/// The output file to look for
/// Receives the Attributes used to produce this file
/// Logger for output
/// True if the output item exists
public bool UpdateProducingAttributes(FileItem File, string Attributes, ILogger Logger)
{
foreach (ActionHistoryPartition Partition in Partitions)
{
if (File.Location.IsUnderDirectory(Partition.BaseDir))
{
return Partition.UpdateProducingAttributes(File, Attributes, Logger);
}
}
Logger.LogWarning("File {FileLocation} is not under any action history root directory", File.Location);
return false;
}
///
/// Saves all layers of this action history
///
public void Save()
{
lock (LockObject)
{
foreach (ActionHistoryPartition Partition in Partitions)
{
Partition.Save();
}
}
}
}
}