// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; namespace EpicGames.Core { /// /// Static read-only utf8 strings for parsing log events /// public static class LogEventPropertyName { #pragma warning disable CS1591 // Missing documentation public static readonly Utf8String Time = new Utf8String("time"); public static readonly Utf8String Level = new Utf8String("level"); public static readonly Utf8String Id = new Utf8String("id"); public static readonly Utf8String Line = new Utf8String("line"); public static readonly Utf8String LineCount = new Utf8String("lineCount"); public static readonly Utf8String Message = new Utf8String("message"); public static readonly Utf8String Format = new Utf8String("format"); public static readonly Utf8String Properties = new Utf8String("properties"); public static readonly Utf8String Type = new Utf8String("$type"); public static readonly Utf8String Text = new Utf8String("$text"); public static readonly Utf8String File = new Utf8String("file"); // For source file / asset types public static readonly Utf8String Identifier = new Utf8String("identifier"); // For symbols public static readonly Utf8String RelativePath = new Utf8String("relativePath"); public static readonly Utf8String DepotPath = new Utf8String("depotPath"); public static readonly Utf8String Target = new Utf8String("target"); // For hyperlinks public static readonly Utf8String Exception = new Utf8String("exception"); public static readonly Utf8String Trace = new Utf8String("trace"); public static readonly Utf8String InnerException = new Utf8String("innerException"); public static readonly Utf8String InnerExceptions = new Utf8String("innerExceptions"); #pragma warning restore CS1591 // Missing documentation } /// /// Epic representation of a log event. Can be serialized to/from Json for the Horde dashboard, and passed directly through ILogger interfaces. /// [JsonConverter(typeof(LogEventConverter))] public class LogEvent : IEnumerable> { /// /// Time that the event was emitted /// public DateTime Time { get; set; } /// /// The log level /// public LogLevel Level { get; set; } /// /// Unique id associated with this event. See for possible values. /// public EventId Id { get; set; } /// /// Index of the line within a multi-line message /// public int LineIndex { get; set; } /// /// Number of lines in the message /// public int LineCount { get; set; } /// /// The formatted message /// public string Message { get; set; } /// /// Message template string /// public string? Format { get; set; } /// /// Map of property name to value /// public IEnumerable>? Properties { get; set; } /// /// The exception value /// public LogException? Exception { get; } class MergedPropertyList : IEnumerable> { readonly HashSet _names = []; readonly List> _properties = []; public void AddRange(IEnumerable> properties) { foreach (KeyValuePair property in properties) { if (_names.Add(property.Key)) { _properties.Add(property); } } } /// public IEnumerator> GetEnumerator() => _properties.GetEnumerator(); /// IEnumerator IEnumerable.GetEnumerator() => _properties.GetEnumerator(); } static readonly JsonSerializerOptions s_jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; /// /// Constructor /// public LogEvent(DateTime time, LogLevel level, EventId eventId, string message, string? format, IEnumerable>? properties, LogException? exception) : this(time, level, eventId, 0, 1, message, format, properties, exception) { } /// /// Constructor /// public LogEvent(DateTime time, LogLevel level, EventId eventId, int lineIndex, int lineCount, string message, string? format, IEnumerable>? properties, LogException? exception) { Time = time; Level = level; Id = eventId; LineIndex = lineIndex; LineCount = lineCount; Message = message; Format = format; Properties = properties; Exception = exception; } /// /// Add a new property to this event /// /// Name of the property to add /// Value for the property public void AddProperty(string name, object? value) => AddProperties([KeyValuePair.Create(name, value)]); /// /// Add new properties to this event /// /// Properties to add public void AddProperties(IEnumerable> properties) { if (Properties == null) { Properties = properties; } else { MergedPropertyList? list = Properties as MergedPropertyList; if (list == null) { list = new MergedPropertyList(); list.AddRange(Properties); Properties = list; } list.AddRange(properties); } } /// /// Gets an untyped property with the given name /// /// /// public object GetProperty(string name) { object? value; if (TryGetProperty(name, out value)) { return value; } throw new KeyNotFoundException($"Property {name} not found"); } /// /// Gets a property with the given name /// /// /// Name of the property /// public T GetProperty(string name) => (T)GetProperty(name); /// /// Finds a property with the given name /// /// Name of the property /// Value for the property, on success /// True if the property was found, false otherwise public bool TryGetProperty(string name, [NotNullWhen(true)] out object? value) { if (Properties != null) { foreach (KeyValuePair pair in Properties) { if (pair.Key.Equals(name, StringComparison.Ordinal) && pair.Value != null) { value = pair.Value; return true; } } } value = null; return false; } /// /// Finds a typed property with the given name /// /// Type of the property to receive /// Name of the property /// Value for the property, on success /// True if the property was found, false otherwise public bool TryGetProperty(string name, [NotNullWhen(true)] out T value) { object? untypedValue; if (TryGetProperty(name, out untypedValue) && untypedValue is T typedValue) { value = typedValue; return true; } else { value = default!; return false; } } /// /// Read a log event from a utf-8 encoded json byte array /// /// /// public static LogEvent Read(ReadOnlySpan data) { Utf8JsonReader reader = new Utf8JsonReader(data); reader.Read(); return Read(ref reader); } /// /// Read a log event from Json /// /// The Json reader /// New log event #pragma warning disable CA1045 // Do not pass types by reference public static LogEvent Read(ref Utf8JsonReader reader) #pragma warning restore CA1045 // Do not pass types by reference { DateTime time = new DateTime(0); LogLevel level = LogLevel.None; EventId eventId = new EventId(0); int line = 0; int lineCount = 1; string message = String.Empty; string? format = null; Dictionary? properties = null; LogException? exception = null; ReadOnlySpan propertyName; for (; JsonExtensions.TryReadNextPropertyName(ref reader, out propertyName); reader.Skip()) { if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.Time.Span)) { time = reader.GetDateTime(); } else if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.Level.Span)) { level = Enum.Parse(reader.GetString()!); } else if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.Id.Span)) { eventId = reader.GetInt32(); } else if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.Line.Span)) { line = reader.GetInt32(); } else if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.LineCount.Span)) { lineCount = reader.GetInt32(); } else if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.Message.Span)) { message = reader.GetString()!; } else if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.Format.Span)) { format = reader.GetString()!; } else if (Utf8StringComparer.OrdinalIgnoreCase.Equals(propertyName, LogEventPropertyName.Properties.Span)) { properties = ReadProperties(ref reader); } } return new LogEvent(time, level, eventId, line, lineCount, message, format, properties, exception); } static Dictionary ReadProperties(ref Utf8JsonReader reader) { Dictionary properties = []; ReadOnlySpan propertyName; for (; JsonExtensions.TryReadNextPropertyName(ref reader, out propertyName); reader.Skip()) { string name = Encoding.UTF8.GetString(propertyName); object? value = ReadPropertyValue(ref reader); properties.Add(name, value); } return properties; } static object? ReadPropertyValue(ref Utf8JsonReader reader) { switch (reader.TokenType) { case JsonTokenType.Null: return null; case JsonTokenType.True: return true; case JsonTokenType.False: return false; case JsonTokenType.StartArray: return ReadArrayPropertyValue(ref reader); case JsonTokenType.StartObject: return ReadObjectPropertyValue(ref reader); case JsonTokenType.String: return reader.GetString()!; case JsonTokenType.Number: if (reader.TryGetInt32(out int intValue)) { return intValue; } else if (reader.TryGetDouble(out double doubleValue)) { return doubleValue; } else { return Encoding.UTF8.GetString(reader.ValueSpan); } default: throw new InvalidOperationException("Unhandled property type"); } } static object ReadArrayPropertyValue(ref Utf8JsonReader reader) { List result = []; while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) { result.Add(ReadPropertyValue(ref reader)); } return result; } static object ReadObjectPropertyValue(ref Utf8JsonReader reader) { // Read all the properties Dictionary? properties = []; for (; JsonExtensions.TryReadNextPropertyName(ref reader, out ReadOnlySpan propertyName); reader.Skip()) { properties.Add(new Utf8String(propertyName.ToArray()), ReadPropertyValue(ref reader)); } // Check if we can convert it to a LogValue if (properties.TryGetValue(LogEventPropertyName.Type, out object? type) && type is string typeStr && properties.TryGetValue(LogEventPropertyName.Text, out object? text) && text is string textStr) { properties.Remove(LogEventPropertyName.Type); properties.Remove(LogEventPropertyName.Text); if (properties.Count == 0) { return new LogValue(new Utf8String(typeStr), textStr); } else { return new LogValue(new Utf8String(typeStr), textStr, properties); } } return properties; } /// /// Writes a log event to Json /// /// public void Write(Utf8JsonWriter writer) { writer.WriteStartObject(); writer.WriteString(LogEventPropertyName.Time.Span, Time.ToString("s", CultureInfo.InvariantCulture)); writer.WriteString(LogEventPropertyName.Level.Span, Level.ToString()); writer.WriteString(LogEventPropertyName.Message.Span, Message); if (Id.Id != 0) { writer.WriteNumber(LogEventPropertyName.Id.Span, Id.Id); } if (LineIndex > 0) { writer.WriteNumber(LogEventPropertyName.Line.Span, LineIndex); } if (LineCount > 1) { writer.WriteNumber(LogEventPropertyName.LineCount.Span, LineCount); } if (Format != null) { writer.WriteString(LogEventPropertyName.Format.Span, Format); } if (Properties != null && Properties.Any()) { writer.WriteStartObject(LogEventPropertyName.Properties.Span); foreach ((string name, object? value) in Properties!) { if (!name.Equals(MessageTemplate.FormatPropertyName, StringComparison.Ordinal)) { if (name.StartsWith('@')) { writer.WritePropertyName(name[1..]); JsonSerializer.Serialize(writer, value, value?.GetType() ?? typeof(object), s_jsonSerializerOptions); } else { writer.WritePropertyName(name); LogValueFormatter.Format(value, writer); } } } writer.WriteEndObject(); } if (Exception != null) { writer.WriteStartObject(LogEventPropertyName.Exception.Span); WriteException(ref writer, Exception); writer.WriteEndObject(); } writer.WriteEndObject(); } /// /// Writes an exception to a json object /// /// Writer to receive the exception data /// The exception static void WriteException(ref Utf8JsonWriter writer, LogException exception) { writer.WriteString("message", exception.Message); writer.WriteString("trace", exception.Trace); if (exception.InnerException != null) { writer.WriteStartObject("innerException"); WriteException(ref writer, exception.InnerException); writer.WriteEndObject(); } if (exception.InnerExceptions != null) { writer.WriteStartArray("innerExceptions"); for (int idx = 0; idx < 16 && idx < exception.InnerExceptions.Count; idx++) // Cap number of exceptions returned to avoid huge messages { LogException innerException = exception.InnerExceptions[idx]; writer.WriteStartObject(); WriteException(ref writer, innerException); writer.WriteEndObject(); } writer.WriteEndArray(); } } /// /// Create a new log event /// public static LogEvent Create(LogLevel level, string format, params object[] args) => Create(level, KnownLogEvents.None, null, format, args); /// /// Create a new log event /// public static LogEvent Create(LogLevel level, EventId eventId, string format, params object[] args) => Create(level, eventId, null, format, args); /// /// Create a new log event /// public static LogEvent Create(LogLevel level, EventId eventId, Exception? exception, string format, params object[] args) { Dictionary properties = []; MessageTemplate.ParsePropertyValues(format, args, properties); string message = MessageTemplate.Render(format, properties!); return new LogEvent(DateTime.UtcNow, level, eventId, message, format, properties, LogException.FromException(exception)); } /// /// Creates a log event from an ILogger parameters /// public static LogEvent FromState(LogLevel level, EventId eventId, TState state, Exception? exception, Func formatter) { _ = formatter; if (state is LogEvent logEvent) { return logEvent; } if (state is JsonLogEvent jsonLogEvent) { return Read(jsonLogEvent.Data.Span); } DateTime time = DateTime.UtcNow; // Try to log the event IEnumerable>? values = state as IEnumerable>; string? format = values?.FirstOrDefault(x => x.Key.Equals(MessageTemplate.FormatPropertyName, StringComparison.Ordinal)).Value?.ToString(); string message = (format == null)? formatter(state, exception) : MessageTemplate.Render(format, values); return new LogEvent(time, level, eventId, message, format, values, LogException.FromException(exception)); } /// /// Enumerates all the properties in this object /// /// Property pairs public IEnumerator> GetEnumerator() { if (Format != null) { yield return new KeyValuePair(MessageTemplate.FormatPropertyName, Format.ToString()); } if (Properties != null) { foreach ((string name, object? value) in Properties) { if (!name.Equals(MessageTemplate.FormatPropertyName, StringComparison.OrdinalIgnoreCase)) { yield return new KeyValuePair(name, value?.ToString()); } } } } /// /// Enumerates all the properties in this object /// /// Property pairs System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() { foreach (KeyValuePair pair in this) { yield return pair; } } /// /// Serialize a message template to JOSN /// public byte[] ToJsonBytes() { ArrayBufferWriter buffer = new ArrayBufferWriter(); using (Utf8JsonWriter writer = new Utf8JsonWriter(buffer)) { Write(writer); } return buffer.WrittenSpan.ToArray(); } /// /// Serialize a message template to JOSN /// public string ToJson() { ArrayBufferWriter buffer = new ArrayBufferWriter(); using (Utf8JsonWriter writer = new Utf8JsonWriter(buffer)) { Write(writer); } return Encoding.UTF8.GetString(buffer.WrittenSpan); } /// public override string ToString() => Message; } /// /// Information about an exception in a log event /// public sealed class LogException { /// /// Exception message /// public string Message { get; set; } /// /// Stack trace for the exception /// public string Trace { get; set; } /// /// Optional inner exception information /// public LogException? InnerException { get; set; } /// /// Multiple inner exceptions, in the case of an /// public List InnerExceptions { get; } = []; /// /// Constructor /// /// /// public LogException(string message, string trace) { Message = message; Trace = trace; } /// /// Constructor /// /// [return: NotNullIfNotNull(nameof(exception))] public static LogException? FromException(Exception? exception) { LogException? result = null; if (exception != null) { result = new LogException(exception.Message, exception.StackTrace ?? String.Empty); if (exception.InnerException != null) { result.InnerException = FromException(exception.InnerException); } AggregateException? aggregateException = exception as AggregateException; if (aggregateException != null && aggregateException.InnerExceptions.Count > 0) { for (int idx = 0; idx < 16 && idx < aggregateException.InnerExceptions.Count; idx++) // Cap number of exceptions returned to avoid huge messages { LogException innerException = FromException(aggregateException.InnerExceptions[idx]); result.InnerExceptions.Add(innerException); } } } return result; } } /// /// Interface for a log event sink /// public interface ILogEventSink { /// /// Process log event /// /// Log event void ProcessEvent(LogEvent logEvent); } /// /// Simple filtering log event sink with a callback for convenience /// public class FilteringEventSink : ILogEventSink { /// /// Log events received /// public IReadOnlyList LogEvents => _logEvents; private readonly List _logEvents = []; private readonly int[] _includeEventIds; private readonly Action? _eventCallback; /// /// Constructor /// /// Event IDs to include /// Optional callback function for each event received public FilteringEventSink(int[] includeEventIds, Action? eventCallback = null) { _includeEventIds = includeEventIds; _eventCallback = eventCallback; } /// public void ProcessEvent(LogEvent logEvent) { if (_includeEventIds.Any(x => x == logEvent.Id.Id)) { _logEvents.Add(logEvent); _eventCallback?.Invoke(logEvent); } } } /// /// Converter for serialization of instances to Json streams /// public class LogEventConverter : JsonConverter { /// public override LogEvent Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { return LogEvent.Read(ref reader); } /// public override void Write(Utf8JsonWriter writer, LogEvent value, JsonSerializerOptions options) { value.Write(writer); } } }