// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Core;
#pragma warning disable CA1716 // Do not use 'imports' as variable name
namespace EpicGames.Horde.Storage
{
///
/// Interface for a writer of node objects
///
public interface IBlobWriter : IMemoryWriter, IAsyncDisposable
{
///
/// Options for serialization
///
BlobSerializerOptions Options { get; }
///
/// Accessor for the memory written to the current blob
///
ReadOnlyMemory WrittenMemory { get; }
///
/// Adds an alias to the blob currently being written
///
/// Name of the alias
/// Rank to use when finding blobs by alias
/// Inline data to store with the alias
void AddAlias(string name, int rank, ReadOnlyMemory data = default);
///
/// Flush any pending nodes to storage
///
/// Cancellation token for the operation
Task FlushAsync(CancellationToken cancellationToken = default);
///
/// Create another writer instance, allowing multiple threads to write in parallel.
///
/// New writer instance
IBlobWriter Fork();
///
/// Finish writing a blob that has been written into the output buffer.
///
/// Type of the node that was written
/// Cancellation token for the operation
/// Handle to the written node
ValueTask CompleteAsync(BlobType type, CancellationToken cancellationToken = default);
///
/// Finish writing a blob that has been written into the output buffer.
///
/// Type of the node that was written
/// Cancellation token for the operation
/// Handle to the written node
ValueTask> CompleteAsync(BlobType type, CancellationToken cancellationToken = default);
///
/// Writes a reference to another blob. NOTE: This does not write anything to the underlying output stream, which prevents the data forming a Merkle tree
/// unless guaranteed uniqueness via a hash being written separately.
///
/// Referenced blob
void WriteBlobHandleDangerous(IBlobRef handle);
///
/// Writes a reference to another blob. The blob's hash is serialized to the output stream.
///
/// Referenced blob
void WriteBlobRef(IHashedBlobRef blobRef);
}
///
/// Information about an alias to be added alongside a blob
///
/// Name of the alias
/// Rank of the alias
/// Inline data to be stored for the alias
public record class AliasInfo(string Name, int Rank, ReadOnlyMemory Data);
///
/// Base class for implementations.
///
public abstract class BlobWriter : IBlobWriter
{
Memory _memory;
readonly List _aliases = new List();
readonly List _imports = new List();
readonly BlobSerializerOptions _options;
int _length;
///
public int Length => _length;
///
public BlobSerializerOptions Options => _options;
///
public ReadOnlyMemory WrittenMemory => _memory.Slice(0, _length);
///
/// Constructor
///
///
protected BlobWriter(BlobSerializerOptions? options)
{
_options = options ?? BlobSerializerOptions.Default;
}
///
/// Computes the hash of the written data
///
public IoHash ComputeHash() => IoHash.Compute(_memory.Span.Slice(0, _length));
///
public void WriteBlobHandleDangerous(IBlobRef target)
{
_imports.Add(target);
}
///
/// Writes a handle to another node
///
public void WriteBlobRef(IHashedBlobRef target)
{
this.WriteIoHash(target.Hash);
WriteBlobHandleDangerous(target);
}
///
public Span GetSpan(int sizeHint = 0) => GetMemory(sizeHint).Span;
///
public Memory GetMemory(int sizeHint = 0)
{
int newLength = _length + Math.Max(sizeHint, 1);
if (newLength > _memory.Length)
{
newLength = _length + Math.Max(sizeHint, 1024);
_memory = GetOutputBuffer(_length, Math.Max(_memory.Length * 2, newLength));
}
return _memory.Slice(_length);
}
///
public void Advance(int length) => _length += length;
///
/// Request a new buffer to write to
///
/// Size of data written to the current buffer
/// Desired size for the buffer
/// New buffer
public abstract Memory GetOutputBuffer(int usedSize, int desiredSize);
///
/// Write the current blob to storage
///
/// Type of the blob
/// Size of the blob to write
/// References to other blobs
/// Aliases for the new blob
/// Cancellation token for the operation
/// New buffer
public abstract ValueTask WriteBlobAsync(BlobType type, int size, IReadOnlyList imports, IReadOnlyList aliases, CancellationToken cancellationToken);
///
public void AddAlias(string name, int rank, ReadOnlyMemory data)
=> _aliases.Add(new AliasInfo(name, rank, data));
///
public abstract Task FlushAsync(CancellationToken cancellationToken = default);
///
public abstract IBlobWriter Fork();
///
public async ValueTask CompleteAsync(BlobType type, CancellationToken cancellationToken = default)
{
IHashedBlobRef blobRef = await WriteBlobAsync(type, _length, _imports, _aliases, cancellationToken);
_memory = default;
_length = 0;
_imports.Clear();
_aliases.Clear();
return blobRef;
}
///
public async ValueTask> CompleteAsync(BlobType type, CancellationToken cancellationToken = default)
{
IoHash hash = IoHash.Compute(_memory.Span.Slice(0, _length));
IHashedBlobRef blobRef = await CompleteAsync(type, cancellationToken);
return HashedBlobRef.Create(hash, blobRef, _options);
}
///
public abstract ValueTask DisposeAsync();
}
///
/// Implementation of which discards any written data
///
public sealed class NullBlobWriter : BlobWriter
{
class BlobRef : IHashedBlobRef
{
public IoHash Hash { get; }
public IBlobRef Innermost
=> throw new NotImplementedException();
public BlobRef(IoHash hash)
=> Hash = hash;
public ValueTask FlushAsync(CancellationToken cancellationToken = default)
=> default;
public ValueTask ReadBlobDataAsync(CancellationToken cancellationToken = default)
=> throw new NotSupportedException();
public bool TryGetLocator([NotNullWhen(true)] out BlobLocator locator)
{
locator = new BlobLocator(Hash.ToUtf8String());
return true;
}
}
readonly ChunkedMemoryWriter _memoryWriter;
readonly BlobSerializerOptions _options;
///
/// Constructor
///
public NullBlobWriter(BlobSerializerOptions options)
: base(options)
{
_memoryWriter = new ChunkedMemoryWriter();
_options = options;
}
///
public override ValueTask DisposeAsync()
{
GC.SuppressFinalize(this);
_memoryWriter.Dispose();
return new ValueTask();
}
///
public override ValueTask WriteBlobAsync(BlobType type, int size, IReadOnlyList imports, IReadOnlyList aliases, CancellationToken cancellationToken)
{
Memory memory = _memoryWriter.GetMemoryAndAdvance(size);
IoHash hash = IoHash.Compute(memory.Span);
_memoryWriter.Clear();
return new ValueTask(new BlobRef(hash));
}
///
public override Memory GetOutputBuffer(int usedSize, int desiredSize)
=> _memoryWriter.GetMemory(usedSize, desiredSize);
///
public override Task FlushAsync(CancellationToken cancellationToken = default)
=> Task.CompletedTask;
///
public override IBlobWriter Fork()
=> new NullBlobWriter(_options);
}
///
/// Implementation of which just buffers data in memory
///
public class MemoryBlobWriter : BlobWriter
{
class BlobRef : IHashedBlobRef
{
readonly int _index;
readonly IoHash _hash;
readonly BlobData _data;
public BlobRef(int index, BlobData data)
{
_index = index;
_hash = IoHash.Compute(data.Data.Span);
_data = data;
}
public IoHash Hash
=> _hash;
public int Index
=> _index;
public IBlobRef Innermost
=> this;
public ValueTask FlushAsync(CancellationToken cancellationToken = default)
=> default;
public ValueTask ReadBlobDataAsync(CancellationToken cancellationToken = default)
=> new ValueTask(_data);
public bool TryGetLocator([NotNullWhen(true)] out BlobLocator locator)
=> throw new NotSupportedException();
}
readonly ChunkedMemoryWriter _memoryWriter;
int _nextIndex;
///
/// Constructor
///
public MemoryBlobWriter(BlobSerializerOptions options)
: base(options)
{
_memoryWriter = new ChunkedMemoryWriter();
}
///
public override ValueTask DisposeAsync()
{
GC.SuppressFinalize(this);
_memoryWriter.Dispose();
return new ValueTask();
}
///
/// Clears the contents of this writer
///
public void Clear()
=> _memoryWriter.Clear();
///
public override Task FlushAsync(CancellationToken cancellationToken = default)
=> Task.CompletedTask;
///
public override IBlobWriter Fork()
=> new MemoryBlobWriter(Options);
///
public override Memory GetOutputBuffer(int usedSize, int desiredSize)
=> _memoryWriter.GetMemory(usedSize, desiredSize);
///
public override ValueTask WriteBlobAsync(BlobType type, int size, IReadOnlyList imports, IReadOnlyList aliases, CancellationToken cancellationToken)
{
Memory memory = _memoryWriter.GetMemoryAndAdvance(size);
BlobData data = new BlobData(type, memory.ToArray(), imports.ToArray());
return new ValueTask(new BlobRef(++_nextIndex, data));
}
///
/// Helper function to get the index of a blob
///
public static int GetIndex(IHashedBlobRef handle)
=> ((BlobRef)handle.Innermost).Index;
}
}