// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Net.Mime; using System.Threading; using System.Threading.Tasks; using EpicGames.Horde.Storage; using Jupiter.Implementation.LeaderElection; using k8s; using k8s.Models; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Jupiter.Implementation { public class PeerBlobStore : IBlobStore { private readonly IPeerServiceDiscovery _serviceDiscovery; private readonly IHttpClientFactory _httpClientFactory; private readonly IServiceCredentials _serviceCredentials; private readonly ILogger _logger; public PeerBlobStore(IPeerServiceDiscovery serviceDiscovery, IHttpClientFactory httpClientFactory, IServiceCredentials serviceCredentials, ILogger logger) { _serviceDiscovery = serviceDiscovery; _httpClientFactory = httpClientFactory; _serviceCredentials = serviceCredentials; _logger = logger; } private async Task BuildHttpRequestAsync(HttpMethod method, Uri uri) { string? token = await _serviceCredentials.GetTokenAsync(); HttpRequestMessage request = new HttpRequestMessage(method, uri); if (!string.IsNullOrEmpty(token)) { request.Headers.Add("Authorization", $"{_serviceCredentials.GetAuthenticationScheme()} {token}"); } return request; } private async Task DoGetObjectAsync(string instance, NamespaceId ns, BlobId blob) { try { string filesystemLayerName = nameof(FileSystemStore); using HttpRequestMessage getObjectRequest = await BuildHttpRequestAsync(HttpMethod.Get, new Uri($"api/v1/blobs/{ns}/{blob}?storageLayers={filesystemLayerName}", UriKind.Relative)); getObjectRequest.Headers.Add("Accept", MediaTypeNames.Application.Octet); HttpResponseMessage response = await GetHttpClient(instance).SendAsync(getObjectRequest); if (response.StatusCode == HttpStatusCode.NotFound) { return null; } response.EnsureSuccessStatusCode(); long? contentLength = response.Content.Headers.ContentLength; if (contentLength == null) { _logger.LogWarning("Content length missing in response from peer blob store. This is not supported, ignoring response"); return null; } return new BlobContents(await response.Content.ReadAsStreamAsync(), contentLength.Value); } catch (Exception e) { _logger.LogWarning(e, "Exception when attempting to fetch blob {Blob} in namespace {Namespace} from instance {Instance}", blob, ns, instance); } return null; } public async Task GetObjectAsync(NamespaceId ns, BlobId blob, LastAccessTrackingFlags flags = LastAccessTrackingFlags.DoTracking, bool supportsRedirectUri = false) { List> tasks = new(); await foreach (string instance in _serviceDiscovery.FindOtherInstances()) { Task task = DoGetObjectAsync(instance, ns, blob); tasks.Add(task); } while (tasks.Count != 0) { Task finishedTask = await Task.WhenAny(tasks); BlobContents? result = await finishedTask; if (result != null) { return result; } tasks.Remove(finishedTask); } throw new BlobNotFoundException(ns, blob); } private async Task DoExistsAsync(string instance, NamespaceId ns, BlobId blob) { try { string filesystemLayerName = nameof(FileSystemStore); using HttpRequestMessage headObjectRequest = await BuildHttpRequestAsync(HttpMethod.Head, new Uri($"api/v1/blobs/{ns}/{blob}?storageLayers={filesystemLayerName}", UriKind.Relative)); HttpResponseMessage response = await GetHttpClient(instance).SendAsync(headObjectRequest, CancellationToken.None); if (response.StatusCode == HttpStatusCode.NotFound) { return false; } response.EnsureSuccessStatusCode(); return true; } catch (Exception e) { _logger.LogWarning(e, "Exception when attempting to fetch blob {Blob} in namespace {Namespace} from instance {Instance}", blob, ns, instance); } return null; } public async Task ExistsAsync(NamespaceId ns, BlobId blob, bool forceCheck = false) { List> tasks = new(); await foreach (string instance in _serviceDiscovery.FindOtherInstances()) { Task task = DoExistsAsync(instance, ns, blob); tasks.Add(task); } while (tasks.Count != 0) { Task finishedTask = await Task.WhenAny(tasks); bool? result = await finishedTask; if (result != null) { return result.Value; } tasks.Remove(finishedTask); } return false; } private HttpClient GetHttpClient(string instance) { HttpClient httpClient = _httpClientFactory.CreateClient(instance); httpClient.BaseAddress = new Uri($"http://{instance}"); // for these connections to be useful they need to be fast - so timeout quickly if we can not establish the connection httpClient.Timeout = TimeSpan.FromSeconds(1.0); return httpClient; } public Task GetObjectByRedirectAsync(NamespaceId ns, BlobId identifier) { // not supported return Task.FromResult(null); } public Task GetObjectMetadataAsync(NamespaceId ns, BlobId blobId) { // Do not call into other instances to determine metadata as we lack a endpoint for it throw new BlobNotFoundException(ns, blobId); } public Task CopyBlobAsync(NamespaceId ns, NamespaceId targetNamespace, BlobId blobId) { throw new NotImplementedException(); } public Task PutObjectWithRedirectAsync(NamespaceId ns, BlobId identifier) { // not supported return Task.FromResult(null); } public Task PutObjectAsync(NamespaceId ns, byte[] blob, BlobId identifier) { // not applicable return Task.FromResult(identifier); } public Task PutObjectAsync(NamespaceId ns, ReadOnlyMemory blob, BlobId identifier) { // not applicable return Task.FromResult(identifier); } public Task PutObjectAsync(NamespaceId ns, Stream content, BlobId identifier) { // not applicable return Task.FromResult(identifier); } public Task DeleteObjectAsync(NamespaceId ns, BlobId blob) { // not applicable return Task.CompletedTask; } public Task DeleteObjectAsync(IEnumerable namespaces, BlobId blob) { // not applicable return Task.CompletedTask; } public Task DeleteNamespaceAsync(NamespaceId ns) { // not applicable return Task.CompletedTask; } public IAsyncEnumerable<(BlobId, DateTime)> ListObjectsAsync(NamespaceId ns) { // not applicable return AsyncEnumerable.Empty<(BlobId, DateTime)>(); } } public interface IPeerServiceDiscovery { public IAsyncEnumerable FindOtherInstances(); } public class StaticPeerServiceDiscoverySettings { [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Used by serialization")] public List Peers { get; set; } = new List(); } public sealed class StaticPeerServiceDiscovery : IPeerServiceDiscovery { private readonly IOptionsMonitor _settings; public StaticPeerServiceDiscovery(IOptionsMonitor settings) { _settings = settings; } public async IAsyncEnumerable FindOtherInstances() { await Task.CompletedTask; foreach (string peer in _settings.CurrentValue.Peers) { yield return peer; } } } public sealed class KubernetesPeerServiceDiscovery : IPeerServiceDiscovery, IDisposable { private readonly IOptionsMonitor _leaderSettings; private readonly Kubernetes _client; private DateTime _podEnumerationValidUntil = DateTime.Now; private List? _lastPodEnumeration; public KubernetesPeerServiceDiscovery(IOptionsMonitor leaderSettings) { _leaderSettings = leaderSettings; KubernetesClientConfiguration config = KubernetesClientConfiguration.InClusterConfig(); _client = new Kubernetes(config); } public async IAsyncEnumerable FindOtherInstances() { if (_lastPodEnumeration != null && _podEnumerationValidUntil > DateTime.Now) { foreach (string pod in _lastPodEnumeration) { yield return pod; } yield break; } _lastPodEnumeration = await EnumeratePodsAsync().ToListAsync(); _podEnumerationValidUntil = DateTime.Now.AddMinutes(5); foreach (string pod in _lastPodEnumeration) { yield return pod; } } private async IAsyncEnumerable EnumeratePodsAsync() { string currentPodName = _leaderSettings.CurrentValue.PodName; V1PodList podList = await _client.ListNamespacedPodAsync(_leaderSettings.CurrentValue.Namespace, labelSelector: _leaderSettings.CurrentValue.PeerPodLabelSelector); foreach (V1Pod pod in podList.Items) { if (pod.Status.Phase != "Running") { continue; } // skip the current pod if (pod.Name() == currentPodName) { continue; } string ip = pod.Status.PodIP; if (string.IsNullOrEmpty(ip)) { continue; } // we typically expose the container on port 80 yield return $"{ip}:80"; } } private void Dispose(bool disposing) { if (disposing) { _client.Dispose(); } } public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } } }