// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using EpicGames.Core;
#pragma warning disable CA1028 // Enum Storage should be Int32
namespace EpicGames.Serialization
{
static class CbPackageConstants
{
public const uint CbPackageHeaderMagic = 0xaa77aacc;
}
///
/// Header in the CbPackage format
///
/// see CbPackage definition in Zen https://github.com/EpicGames/zen/blob/main/zenhttp/include/zenhttp/httpshared.h
struct CbPackageHeader
{
private CbPackageHeader(uint attachmentCount, uint reserved1 = 0, uint reserved2 = 0)
{
HeaderMagic = CbPackageConstants.CbPackageHeaderMagic;
AttachmentCount = attachmentCount;
Reserved1 = reserved1;
Reserved2 = reserved2;
}
public uint HeaderMagic { get; set; }
public uint AttachmentCount { get; set; }
public uint Reserved1 { get; set; }
public uint Reserved2 { get; set; }
private const int Length = sizeof(uint) * 4;
public static async Task ReadAsync(Stream s)
{
// using async reading of stream because Asp.net requires this
byte[] buf = new byte[Length];
int readBytes = await s.ReadAsync(buf, 0, Length);
if (readBytes != Length)
{
throw new Exception($"Was not able to read package header, not enough bytes. Bytes read: {readBytes}");
}
using MemoryStream ms = new MemoryStream(buf);
using BinaryReader reader = new BinaryReader(ms);
uint headerMagic = reader.ReadUInt32();
if (headerMagic != CbPackageConstants.CbPackageHeaderMagic)
{
throw new Exception($"Magic did not match, expected {CbPackageConstants.CbPackageHeaderMagic} got {headerMagic}");
}
uint attachmentCount = reader.ReadUInt32();
uint reserved1 = reader.ReadUInt32();
uint reserved2 = reader.ReadUInt32();
return new CbPackageHeader(attachmentCount, reserved1, reserved2);
}
public readonly void Write(Stream stream)
{
using BinaryWriter writer = new BinaryWriter(stream, Encoding.ASCII, leaveOpen: true);
writer.Write(HeaderMagic);
writer.Write(AttachmentCount);
writer.Write(Reserved1);
writer.Write(Reserved2);
}
}
///
/// Per attachment entry flags
///
[Flags]
public enum CbPackageAttachmentFlags : uint
{
///
/// Is marshaled using compressed buffer storage format
///
IsCompressed = (1u << 0),
///
/// Is compact binary object
///
IsObject = (1u << 1),
///
/// Is error (compact binary formatted) object
///
IsError = (1u << 2),
}
///
/// Header for each attachment in the package
///
public struct CbPackageAttachmentEntry
{
///
/// The size of the attachment
///
public ulong PayloadSize { get; set; }
///
/// Flags description the attachment
///
public CbPackageAttachmentFlags Flags { get; set; }
///
/// The hash of the attachment
///
public IoHash AttachmentHash { get; set; }
///
/// Reads a package attachment struct from a stream
///
/// The stream to read from
/// The read package attachment
internal static async Task ReadAsync(Stream s)
{
byte[] buf = new byte[Length];
int readBytes = await s.ReadAsync(buf, 0, Length);
if (readBytes != Length)
{
throw new Exception($"Was not able to read package attachment entry, not enough bytes. Bytes read: {readBytes}");
}
using MemoryStream ms = new MemoryStream(buf);
using BinaryReader reader = new BinaryReader(ms);
ulong payloadSize = reader.ReadUInt64();
uint flags = reader.ReadUInt32();
byte[] hashBytes = reader.ReadBytes(20);
return new CbPackageAttachmentEntry
{
PayloadSize = payloadSize,
Flags = (CbPackageAttachmentFlags)flags,
AttachmentHash = new IoHash(hashBytes),
};
}
private const int Length = IoHash.NumBytes + sizeof(ulong) + sizeof(uint);
///
/// Write a package attachment entry to the stream
///
///
internal readonly void Write(Stream stream)
{
using BinaryWriter writer = new BinaryWriter(stream, Encoding.ASCII, leaveOpen: true);
writer.Write(PayloadSize);
writer.Write((uint)Flags);
writer.Write(AttachmentHash.ToByteArray());
}
}
///
/// Reads a CbPackage (a bundle of both a cb object and its attachments)
///
public class CbPackageReader
{
#pragma warning disable IDE0052 // Remove unread private members
private readonly CbPackageHeader _header;
#pragma warning restore IDE0052 // Remove unread private members
private readonly Stream _attachmentsStream;
private readonly List _attachmentEntries = [];
private CbPackageReader(CbPackageHeader header, CbObject rootObject, IoHash rootHash, List attachmentEntries, Stream attachmentsStream)
{
_header = header;
_attachmentEntries = attachmentEntries;
_attachmentsStream = attachmentsStream;
RootObject = rootObject;
RootHash = rootHash;
}
private static async Task ReadStreamAsync(Stream s, int count)
{
int index = 0;
byte[] bytes = new byte[count];
do
{
int n = await s.ReadAsync(new Memory(bytes, index, count - index))
.ConfigureAwait(false);
if (n == 0)
{
throw new Exception("Unexpected end of stream");
}
index += n;
} while (index < count);
return bytes;
}
///
/// Create a package reader from a stream, used to stream in the attachments of the package
///
/// A stream
/// A package reader instance
public static async Task CreateAsync(Stream s)
{
CbPackageHeader header = await CbPackageHeader.ReadAsync(s);
List entries = [];
// we expect a extra entry for the root object
uint attachmentsCount = header.AttachmentCount + 1;
for (int i = 0; i < attachmentsCount; i++)
{
CbPackageAttachmentEntry entry = await CbPackageAttachmentEntry.ReadAsync(s);
entries.Add(entry);
}
if (entries.Count < 1)
{
throw new Exception("Did not find any root object entry when reading CbPackage");
}
// the first object is assumed to be a CbObject, the root object which references the other attachments
CbPackageAttachmentEntry rootObject = entries.First();
entries.RemoveAt(0);
if (!rootObject.Flags.HasFlag(CbPackageAttachmentFlags.IsObject))
{
throw new Exception("First attachment must be a CbObject for a package");
}
if (rootObject.Flags.HasFlag(CbPackageAttachmentFlags.IsError))
{
throw new Exception("First attachment was a error object");
}
if (rootObject.Flags.HasFlag(CbPackageAttachmentFlags.IsCompressed))
{
// TODO: We could support this being a compressed buffer and just remove it
throw new Exception("First attachment must not be compressed");
}
if (rootObject.PayloadSize > Int32.MaxValue)
{
throw new Exception($"Package attachments larger then {Int32.MaxValue} not supported");
}
byte[] rootObjectBytes = await ReadStreamAsync(s, (int)rootObject.PayloadSize);
return new CbPackageReader(header, new CbObject(rootObjectBytes), rootObject.AttachmentHash, entries, s);
}
///
/// Iterates over the attachments, returning the attachment entry and the attachment (in memory)
///
///
public async IAsyncEnumerable<(CbPackageAttachmentEntry, byte[])> IterateAttachmentsAsync()
{
// close the stream after we have iterated the attachments as there should be nothing left in it
await using Stream s = _attachmentsStream;
foreach (CbPackageAttachmentEntry entry in _attachmentEntries)
{
if (entry.PayloadSize > Int32.MaxValue)
{
throw new Exception($"Package attachments larger then {Int32.MaxValue} not supported");
}
byte[] blob = await ReadStreamAsync(s, (int)entry.PayloadSize);
yield return (entry, blob);
}
}
///
/// The CbObject that is the root object (that references the other attachments in the package)
///
public CbObject RootObject { get; }
///
/// The hash of the root object
///
public IoHash RootHash { get; }
}
///
/// Builds a in-memory representation of a CbPackage
///
public sealed class CbPackageBuilder : IDisposable
{
private readonly List<(CbPackageAttachmentEntry, Stream)> _streamAttachments = [];
///
/// Constructor for the package builder
///
public CbPackageBuilder()
{
}
///
public void Dispose()
{
foreach ((CbPackageAttachmentEntry _, Stream stream) in _streamAttachments)
{
stream.Dispose();
}
}
///
/// Add a attachment to the package builder using in memory buffer
///
/// The hash of the attachment
/// The flags that apply to the attachment
/// In-memory buffer of the attachment
///
public void AddAttachment(IoHash attachmentHash, CbPackageAttachmentFlags flags, byte[] blobMemory)
{
CbPackageAttachmentEntry entry = new CbPackageAttachmentEntry()
{
AttachmentHash = attachmentHash,
Flags = flags,
PayloadSize = (ulong)blobMemory.LongLength
};
_streamAttachments.Add((entry, new MemoryStream(blobMemory)));
}
///
/// Add a attachment to the package builder from a stream, will be cache in memory for the lifetime of the package builder
///
/// The hash of the attachment
/// The flags that apply to the attachment
/// The stream to read the attachment from
/// The count of bytes to read from the stream
///
public void AddAttachment(IoHash attachmentHash, CbPackageAttachmentFlags flags, Stream stream, ulong length)
{
CbPackageAttachmentEntry entry = new CbPackageAttachmentEntry()
{
AttachmentHash = attachmentHash,
Flags = flags,
PayloadSize = length
};
_streamAttachments.Add((entry, stream));
}
///
/// Generate a contiguous buffer of the cb package
///
///
public async Task ToByteArrayAsync()
{
MemoryStream packageBuffer = new MemoryStream();
if (_streamAttachments.Count == 0)
{
throw new Exception("Expected the first attachment to be a CbObject but found no attachment");
}
// we will overwrite this header again after building the package builder
CbPackageHeader header = new CbPackageHeader
{
HeaderMagic = CbPackageConstants.CbPackageHeaderMagic,
AttachmentCount = (uint)_streamAttachments.Count - 1, // the root object does not count as a attachment
Reserved1 = 0,
Reserved2 = 0,
};
header.Write(packageBuffer);
foreach ((CbPackageAttachmentEntry entry, Stream _) in _streamAttachments)
{
entry.Write(packageBuffer);
}
foreach ((CbPackageAttachmentEntry _, Stream stream) in _streamAttachments)
{
await stream.CopyToAsync(packageBuffer);
}
return packageBuffer.ToArray();
}
}
}