// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; #pragma warning disable CA1716 // Do not use 'imports' as variable name namespace EpicGames.Horde.Storage { /// /// Interface for a writer of node objects /// public interface IBlobWriter : IMemoryWriter, IAsyncDisposable { /// /// Options for serialization /// BlobSerializerOptions Options { get; } /// /// Accessor for the memory written to the current blob /// ReadOnlyMemory WrittenMemory { get; } /// /// Adds an alias to the blob currently being written /// /// Name of the alias /// Rank to use when finding blobs by alias /// Inline data to store with the alias void AddAlias(string name, int rank, ReadOnlyMemory data = default); /// /// Flush any pending nodes to storage /// /// Cancellation token for the operation Task FlushAsync(CancellationToken cancellationToken = default); /// /// Create another writer instance, allowing multiple threads to write in parallel. /// /// New writer instance IBlobWriter Fork(); /// /// Finish writing a blob that has been written into the output buffer. /// /// Type of the node that was written /// Cancellation token for the operation /// Handle to the written node ValueTask CompleteAsync(BlobType type, CancellationToken cancellationToken = default); /// /// Finish writing a blob that has been written into the output buffer. /// /// Type of the node that was written /// Cancellation token for the operation /// Handle to the written node ValueTask> CompleteAsync(BlobType type, CancellationToken cancellationToken = default); /// /// Writes a reference to another blob. NOTE: This does not write anything to the underlying output stream, which prevents the data forming a Merkle tree /// unless guaranteed uniqueness via a hash being written separately. /// /// Referenced blob void WriteBlobHandleDangerous(IBlobRef handle); /// /// Writes a reference to another blob. The blob's hash is serialized to the output stream. /// /// Referenced blob void WriteBlobRef(IHashedBlobRef blobRef); } /// /// Information about an alias to be added alongside a blob /// /// Name of the alias /// Rank of the alias /// Inline data to be stored for the alias public record class AliasInfo(string Name, int Rank, ReadOnlyMemory Data); /// /// Base class for implementations. /// public abstract class BlobWriter : IBlobWriter { Memory _memory; readonly List _aliases = new List(); readonly List _imports = new List(); readonly BlobSerializerOptions _options; int _length; /// public int Length => _length; /// public BlobSerializerOptions Options => _options; /// public ReadOnlyMemory WrittenMemory => _memory.Slice(0, _length); /// /// Constructor /// /// protected BlobWriter(BlobSerializerOptions? options) { _options = options ?? BlobSerializerOptions.Default; } /// /// Computes the hash of the written data /// public IoHash ComputeHash() => IoHash.Compute(_memory.Span.Slice(0, _length)); /// public void WriteBlobHandleDangerous(IBlobRef target) { _imports.Add(target); } /// /// Writes a handle to another node /// public void WriteBlobRef(IHashedBlobRef target) { this.WriteIoHash(target.Hash); WriteBlobHandleDangerous(target); } /// public Span GetSpan(int sizeHint = 0) => GetMemory(sizeHint).Span; /// public Memory GetMemory(int sizeHint = 0) { int newLength = _length + Math.Max(sizeHint, 1); if (newLength > _memory.Length) { newLength = _length + Math.Max(sizeHint, 1024); _memory = GetOutputBuffer(_length, Math.Max(_memory.Length * 2, newLength)); } return _memory.Slice(_length); } /// public void Advance(int length) => _length += length; /// /// Request a new buffer to write to /// /// Size of data written to the current buffer /// Desired size for the buffer /// New buffer public abstract Memory GetOutputBuffer(int usedSize, int desiredSize); /// /// Write the current blob to storage /// /// Type of the blob /// Size of the blob to write /// References to other blobs /// Aliases for the new blob /// Cancellation token for the operation /// New buffer public abstract ValueTask WriteBlobAsync(BlobType type, int size, IReadOnlyList imports, IReadOnlyList aliases, CancellationToken cancellationToken); /// public void AddAlias(string name, int rank, ReadOnlyMemory data) => _aliases.Add(new AliasInfo(name, rank, data)); /// public abstract Task FlushAsync(CancellationToken cancellationToken = default); /// public abstract IBlobWriter Fork(); /// public async ValueTask CompleteAsync(BlobType type, CancellationToken cancellationToken = default) { IHashedBlobRef blobRef = await WriteBlobAsync(type, _length, _imports, _aliases, cancellationToken); _memory = default; _length = 0; _imports.Clear(); _aliases.Clear(); return blobRef; } /// public async ValueTask> CompleteAsync(BlobType type, CancellationToken cancellationToken = default) { IoHash hash = IoHash.Compute(_memory.Span.Slice(0, _length)); IHashedBlobRef blobRef = await CompleteAsync(type, cancellationToken); return HashedBlobRef.Create(hash, blobRef, _options); } /// public abstract ValueTask DisposeAsync(); } /// /// Implementation of which discards any written data /// public sealed class NullBlobWriter : BlobWriter { class BlobRef : IHashedBlobRef { public IoHash Hash { get; } public IBlobRef Innermost => throw new NotImplementedException(); public BlobRef(IoHash hash) => Hash = hash; public ValueTask FlushAsync(CancellationToken cancellationToken = default) => default; public ValueTask ReadBlobDataAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); public bool TryGetLocator([NotNullWhen(true)] out BlobLocator locator) { locator = new BlobLocator(Hash.ToUtf8String()); return true; } } readonly ChunkedMemoryWriter _memoryWriter; readonly BlobSerializerOptions _options; /// /// Constructor /// public NullBlobWriter(BlobSerializerOptions options) : base(options) { _memoryWriter = new ChunkedMemoryWriter(); _options = options; } /// public override ValueTask DisposeAsync() { GC.SuppressFinalize(this); _memoryWriter.Dispose(); return new ValueTask(); } /// public override ValueTask WriteBlobAsync(BlobType type, int size, IReadOnlyList imports, IReadOnlyList aliases, CancellationToken cancellationToken) { Memory memory = _memoryWriter.GetMemoryAndAdvance(size); IoHash hash = IoHash.Compute(memory.Span); _memoryWriter.Clear(); return new ValueTask(new BlobRef(hash)); } /// public override Memory GetOutputBuffer(int usedSize, int desiredSize) => _memoryWriter.GetMemory(usedSize, desiredSize); /// public override Task FlushAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; /// public override IBlobWriter Fork() => new NullBlobWriter(_options); } /// /// Implementation of which just buffers data in memory /// public class MemoryBlobWriter : BlobWriter { class BlobRef : IHashedBlobRef { readonly int _index; readonly IoHash _hash; readonly BlobData _data; public BlobRef(int index, BlobData data) { _index = index; _hash = IoHash.Compute(data.Data.Span); _data = data; } public IoHash Hash => _hash; public int Index => _index; public IBlobRef Innermost => this; public ValueTask FlushAsync(CancellationToken cancellationToken = default) => default; public ValueTask ReadBlobDataAsync(CancellationToken cancellationToken = default) => new ValueTask(_data); public bool TryGetLocator([NotNullWhen(true)] out BlobLocator locator) => throw new NotSupportedException(); } readonly ChunkedMemoryWriter _memoryWriter; int _nextIndex; /// /// Constructor /// public MemoryBlobWriter(BlobSerializerOptions options) : base(options) { _memoryWriter = new ChunkedMemoryWriter(); } /// public override ValueTask DisposeAsync() { GC.SuppressFinalize(this); _memoryWriter.Dispose(); return new ValueTask(); } /// /// Clears the contents of this writer /// public void Clear() => _memoryWriter.Clear(); /// public override Task FlushAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; /// public override IBlobWriter Fork() => new MemoryBlobWriter(Options); /// public override Memory GetOutputBuffer(int usedSize, int desiredSize) => _memoryWriter.GetMemory(usedSize, desiredSize); /// public override ValueTask WriteBlobAsync(BlobType type, int size, IReadOnlyList imports, IReadOnlyList aliases, CancellationToken cancellationToken) { Memory memory = _memoryWriter.GetMemoryAndAdvance(size); BlobData data = new BlobData(type, memory.ToArray(), imports.ToArray()); return new ValueTask(new BlobRef(++_nextIndex, data)); } /// /// Helper function to get the index of a blob /// public static int GetIndex(IHashedBlobRef handle) => ((BlobRef)handle.Innermost).Index; } }