// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics.Metrics;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Horde.Storage;
using Jupiter.Common;
using Jupiter.Implementation.Blob;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using OpenTelemetry.Trace;
namespace Jupiter.Implementation
{
public class MetricsState
{
public Task? CalculateMetricsTask { get; set; } = null;
}
public class MetricsServiceSettings
{
///
/// Set to enable calculation of metrics in the background. Adds load to database so only enable these if you intend to use it.
///
public bool Enabled { get; set; } = false;
public TimeSpan PollFrequency { get; set; } = TimeSpan.FromHours(6);
}
public class MetricsService : PollingService
{
private readonly IOptionsMonitor _settings;
private readonly IReferencesStore _referencesStore;
private volatile bool _alreadyPolling;
private readonly ILogger _logger;
private readonly IServiceProvider _provider;
public MetricsService(IOptionsMonitor settings, IReferencesStore referencesStore, ILogger logger, IServiceProvider provider) : base(serviceName: nameof(MetricsService), settings.CurrentValue.PollFrequency, new MetricsState(), logger, startAtRandomTime: false)
{
_settings = settings;
_referencesStore = referencesStore;
_logger = logger;
_provider = provider;
}
protected override bool ShouldStartPolling()
{
return _settings.CurrentValue.Enabled;
}
public override async Task OnPollAsync(MetricsState state, CancellationToken cancellationToken)
{
if (_alreadyPolling)
{
return false;
}
_alreadyPolling = true;
try
{
if (!state.CalculateMetricsTask?.IsCompleted ?? false)
{
return false;
}
if (state.CalculateMetricsTask != null)
{
await state.CalculateMetricsTask;
}
state.CalculateMetricsTask = DoCalculateMetricsAsync(state, cancellationToken);
return true;
}
finally
{
_alreadyPolling = false;
}
}
private async Task DoCalculateMetricsAsync(MetricsState _, CancellationToken cancellationToken)
{
MetricsCalculator calculator = ActivatorUtilities.CreateInstance(_provider);
_logger.LogInformation("Attempting to calculate metrics. ");
try
{
await foreach (NamespaceId ns in _referencesStore.GetNamespacesAsync(cancellationToken).WithCancellation(cancellationToken))
{
await foreach (BucketId bucket in _referencesStore.GetBucketsAsync(ns, cancellationToken).WithCancellation(cancellationToken))
{
DateTime start = DateTime.UtcNow;
_logger.LogInformation("Calculating stats for {Namespace} {Bucket}", ns, bucket);
await calculator.CalculateStatsForBucketAsync(ns, bucket, cancellationToken);
TimeSpan duration = DateTime.UtcNow - start;
_logger.LogInformation("Stats calculated for {Namespace} {Bucket} took {Duration}", ns, bucket, duration);
}
}
}
catch (Exception e)
{
_logger.LogError("Error calculating metrics. {Exception}", e);
}
}
}
public class MetricsCalculator
{
private readonly IBlobIndex _blobIndex;
private readonly ILogger _logger;
private readonly Tracer _tracer;
private readonly Gauge _blobSizeAvgGauge;
private readonly Gauge _blobSizeMinGauge;
private readonly Gauge _blobSizeMaxGauge;
private readonly Gauge _refsInBucketGauge;
private readonly Gauge _blobSizeCountGauge;
private readonly Gauge _blobSizeTotalGauge;
public MetricsCalculator(IBlobIndex blobIndex, Meter meter, ILogger logger, Tracer tracer)
{
_blobIndex = blobIndex;
_logger = logger;
_tracer = tracer;
_blobSizeAvgGauge = meter.CreateGauge("blobstats.bucket_size.avg");
_blobSizeMinGauge = meter.CreateGauge("blobstats.bucket_size.min");
_blobSizeMaxGauge = meter.CreateGauge("blobstats.bucket_size.max");
_blobSizeCountGauge = meter.CreateGauge("blobstats.bucket_size.count");
_blobSizeTotalGauge = meter.CreateGauge("blobstats.bucket_size.sum");
_refsInBucketGauge = meter.CreateGauge("blobstats.refs_in_bucket");
}
public async Task CalculateStatsForBucketAsync(NamespaceId ns, BucketId bucket, CancellationToken cancellationToken = default)
{
using TelemetrySpan removeBlobScope = _tracer.StartActiveSpan("metrics.calculate")
.SetAttribute("operation.name", "metrics.calculate")
.SetAttribute("resource.name", $"{ns}.{bucket}");
KeyValuePair[] tags = new[] { new KeyValuePair("Bucket", bucket.ToString()), new KeyValuePair("Namespace", ns.ToString()) };
BucketStats stats = await _blobIndex.CalculateBucketStatisticsAsync(ns, bucket, cancellationToken);
_blobSizeAvgGauge.Record(stats.AvgSize, tags);
_blobSizeMinGauge.Record(stats.SmallestBlobFound, tags);
_blobSizeMaxGauge.Record(stats.LargestBlob, tags);
_blobSizeCountGauge.Record(stats.CountOfBlobs, tags);
_refsInBucketGauge.Record(stats.CountOfRefs, tags);
_blobSizeTotalGauge.Record(stats.TotalSize, tags);
_logger.LogInformation("Stats calculated for {Namespace} {Bucket}. {CountOfRefs} {CountOfBlobs} {TotalSize} {AvgSize} {MaxSize} {MinSize}",
ns, bucket, stats.CountOfRefs, stats.CountOfBlobs, stats.TotalSize, stats.AvgSize, stats.LargestBlob, stats.SmallestBlobFound);
return stats;
}
}
}