// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; using EpicGames.Serialization; namespace UnrealBuildTool.Artifacts { /// /// Artifacts can exist in different directory roots. /// public enum ArtifactDirectoryTree { /// /// Absolute path /// Absolute, /// /// Input/Output exists in the engine directory tree /// Engine, /// /// Input/Outputs exists in the project directory tree /// Project, } /// /// Represents a single artifact file /// /// Directory tree containing the artifact /// Name of the artifact /// Hash of the artifact contents public readonly record struct ArtifactFile(ArtifactDirectoryTree Tree, Utf8String Name, IoHash ContentHash) { /// /// The full path of the artifact /// /// Mapping object to use instead of embedded object /// Full path of the artifact public string GetFullPath(IArtifactDirectoryMapping? mapping) { if (Tree == ArtifactDirectoryTree.Absolute) { return Name.ToString(); } if (mapping == null) { throw new ApplicationException("Attempt to get the full path without mapping interface set"); } string directory = mapping.GetDirectory(Tree); return Path.Combine(directory, Name.ToString()); } } /// /// Collection of inputs and outputs for an action /// /// The hash of the primary input and the environment /// The unique hash for all inputs and the environment /// Information about all inputs /// Information about all outputs public record struct ArtifactAction(IoHash Key, IoHash ActionKey, ArtifactFile[] Inputs, ArtifactFile[] Outputs) { /// /// Directory mapping object /// public IArtifactDirectoryMapping? DirectoryMapping { get; set; } = null; /// public readonly override int GetHashCode() { return HashCode.Combine(Key.GetHashCode(), ActionKey.GetHashCode(), Inputs.GetHashCode(), Outputs.GetHashCode()); } } /// /// Collection of helper extension methods for working with IMemoryReader/Writer /// static class ArtifactSerializationExtensions { /// /// Read an artifact structure /// /// Source reader /// Read artifact structure public static ArtifactFile ReadArtifact(this IMemoryReader reader) { ArtifactDirectoryTree tree = (ArtifactDirectoryTree)reader.ReadUInt8(); Utf8String name = reader.ReadUtf8String(); IoHash contentHash = reader.ReadIoHash(); return new ArtifactFile(tree, name, contentHash); } /// /// Write an artifact structure /// /// Destination writer /// Artifact to be written public static void WriteArtifact(this IMemoryWriter writer, ArtifactFile artifact) { writer.WriteUInt8((byte)artifact.Tree); writer.WriteUtf8String(artifact.Name); writer.WriteIoHash(artifact.ContentHash); } /// /// Read an artifact action structure /// /// Source reader /// Read artifact action structure public static ArtifactAction ReadArtifactAction(this IMemoryReader reader) { IoHash key = reader.ReadIoHash(); IoHash actionKey = reader.ReadIoHash(); ArtifactFile[] inputs = reader.ReadVariableLengthArray(() => reader.ReadArtifact()); ArtifactFile[] outputs = reader.ReadVariableLengthArray(() => reader.ReadArtifact()); return new ArtifactAction(key, actionKey, inputs, outputs); } /// /// Write an artifact action structure /// /// Destination writer /// Artifact action to be written public static void WriteArtifactAction(this IMemoryWriter writer, ArtifactAction artifactAction) { writer.WriteIoHash(artifactAction.Key); writer.WriteIoHash(artifactAction.ActionKey); writer.WriteVariableLengthArray(artifactAction.Inputs, x => writer.WriteArtifact(x)); writer.WriteVariableLengthArray(artifactAction.Outputs, x => writer.WriteArtifact(x)); } } /// /// Represents the state of the cache. It is expect that after construction, the cache /// can be in pending state, but this is optional. From the pending state, the cache /// becomes available or unavailable. /// public enum ArtifactCacheState { /// /// The cache is still initializing /// Pending, /// /// There has been some form of cache failure and it is unavailable /// Unavailable, /// /// The cache is functional and ready to process requests /// Available, } /// /// Interface used to map a directory tree to an actual directory /// public interface IArtifactDirectoryMapping { /// /// Return the directory root for the given tree /// /// Tree in question /// Directory root public string GetDirectory(ArtifactDirectoryTree tree); } /// /// Interface for querying and adding artifacts /// public interface IArtifactCache { /// /// Return true if the cache is ready to process requests /// /// public ArtifactCacheState State { get; } /// /// Return task that waits for the cache to be ready /// /// State of the artifact cache public Task WaitForReadyAsync(); /// /// Given a collection of partial keys return all matching artifacts /// /// Source file key /// Token to be used to cancel operations /// Collection of all known artifacts public Task QueryArtifactActionsAsync(IoHash[] partialKeys, CancellationToken cancellationToken); /// /// Query the actual contents of a group of artifacts /// /// Actions to be read /// Token to be used to cancel operations /// Dictionary of the artifacts public Task QueryArtifactOutputsAsync(ArtifactAction[] artifactActions, CancellationToken cancellationToken); /// /// Save new artifact to the cache /// /// Collection of artifacts to be saved /// Token to be used to cancel operations /// Asynchronous task objects public Task SaveArtifactActionsAsync(ArtifactAction[] artifactActions, CancellationToken cancellationToken); /// /// Flush all updates in the cache asynchronously. /// /// Token to be used to cancel operations /// Asynchronous task objects public Task FlushChangesAsync(CancellationToken cancellationToken); } /// /// Struct containing results from artifact fetch /// /// Is set to true if succeeded in fetching artifacts /// Contains log lines if any /// Process exit code (expected to be 0 in almost all cases as errors aren't cached) public readonly record struct ActionArtifactResult(bool Success, List LogLines, int ExitCode = 0); /// /// Interface for action specific support of artifacts. /// interface IActionArtifactCache { /// /// Underlying artifact cache /// public IArtifactCache ArtifactCache { get; } /// /// If true, reads will be serviced /// public bool EnableReads { get; set; } /// /// If true, writes will be propagated to storage /// public bool EnableWrites { get; set; } /// /// If true, log all cache misses. Defaults to false. /// public bool LogCacheMisses { get; set; } /// /// Root location of the engine /// public DirectoryReference? EngineRoot { get; set; } /// /// Array of extra directory roots when handling relative directories. /// The array MUST be consistent between runs since only the index is saved. /// public DirectoryReference[]? DirectoryRoots { get; set; } /// /// Complete an action from existing cached data /// /// Action to be completed /// Token to be used to cancel operations /// True if it has been completed, false if not public Task CompleteActionFromCacheAsync(LinkedAction action, CancellationToken cancellationToken); /// /// Save the output for a completed action /// /// Completed action /// Token to be used to cancel operations /// Asynchronous task object public Task ActionCompleteAsync(LinkedAction action, CancellationToken cancellationToken); /// /// Flush all updates in the cache asynchronously. /// /// Token to be used to cancel operations /// Asynchronous task object public Task FlushChangesAsync(CancellationToken cancellationToken); } }