665 lines
30 KiB
C#
665 lines
30 KiB
C#
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using System.Net.Security;
|
|
using System.Runtime.InteropServices;
|
|
using System.Security.Authentication;
|
|
using System.Threading;
|
|
using Amazon;
|
|
using Amazon.Extensions.NETCore.Setup;
|
|
using Amazon.Runtime;
|
|
using Amazon.S3;
|
|
using Amazon.SecretsManager;
|
|
using Cassandra;
|
|
using Jupiter.Common.Implementation;
|
|
using Jupiter.Controllers;
|
|
using Jupiter.Implementation;
|
|
using Jupiter.Implementation.Blob;
|
|
using Jupiter.Implementation.LeaderElection;
|
|
using Jupiter.Implementation.Objects;
|
|
using Jupiter.Implementation.Replication;
|
|
using Jupiter.Implementation.TransactionLog;
|
|
using Microsoft.AspNetCore.Authorization;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Microsoft.Extensions.Logging;
|
|
using Microsoft.Extensions.Options;
|
|
using StatsdClient;
|
|
|
|
namespace Jupiter
|
|
{
|
|
// ReSharper disable once ClassNeverInstantiated.Global
|
|
public class JupiterStartup : BaseStartup
|
|
{
|
|
public JupiterStartup(IConfiguration configuration) : base(configuration, CreateLogger<JupiterStartup>())
|
|
{
|
|
string? ddAgentHost = System.Environment.GetEnvironmentVariable("DD_AGENT_HOST");
|
|
if (!string.IsNullOrEmpty(ddAgentHost))
|
|
{
|
|
Logger.LogInformation("Initializing Dogstatsd to connect to: {DatadogAgentHost}", ddAgentHost);
|
|
StatsdConfig dogstatsdConfig = new StatsdConfig
|
|
{
|
|
StatsdServerName = ddAgentHost,
|
|
StatsdPort = 8125,
|
|
};
|
|
|
|
DogStatsd.Configure(dogstatsdConfig);
|
|
}
|
|
}
|
|
|
|
protected override void OnAddAuthorization(AuthorizationOptions authorizationOptions, List<string> defaultSchemes)
|
|
{
|
|
|
|
}
|
|
|
|
protected override void OnAddService(IServiceCollection services)
|
|
{
|
|
services.AddHttpClient(Options.DefaultName, (provider, client) =>
|
|
{
|
|
static string GetPlatformName()
|
|
{
|
|
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
|
|
{
|
|
return "Linux";
|
|
}
|
|
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
|
|
{
|
|
return "Windows";
|
|
}
|
|
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
|
|
{
|
|
return "OSX";
|
|
}
|
|
else
|
|
{
|
|
throw new NotSupportedException("Unknown OS when formatting platform name");
|
|
}
|
|
}
|
|
IVersionFile? versionFile = provider.GetService<IVersionFile>();
|
|
string engineVersion = "5.1"; // TODO: we should read this from the version file
|
|
string? version = versionFile?.VersionString ?? "unknown";
|
|
string platformName = GetPlatformName();
|
|
client.DefaultRequestHeaders.TryAddWithoutValidation("UserAgent", $"UnrealEngine/{engineVersion} ({platformName}) Jupiter/{version}");
|
|
});
|
|
|
|
services.AddOptions<UnrealCloudDDCSettings>().Bind(Configuration.GetSection("UnrealCloudDDC")).ValidateDataAnnotations();
|
|
services.AddOptions<MongoSettings>().Bind(Configuration.GetSection("Mongo")).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<S3Settings>().Bind(Configuration.GetSection("S3")).ValidateDataAnnotations();
|
|
services.AddOptions<AzureSettings>().Bind(Configuration.GetSection("Azure")).ValidateDataAnnotations();
|
|
services.AddOptions<FilesystemSettings>().Bind(Configuration.GetSection("Filesystem")).ValidateDataAnnotations();
|
|
services.AddOptions<DebugSettings>().Bind(Configuration.GetSection("Debug")).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<NginxSettings>().Bind(Configuration.GetSection("Nginx")).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<SymbolsSettings>().Bind(Configuration.GetSection("Symbols")).ValidateDataAnnotations();
|
|
services.AddOptions<BuildsSettings>().Bind(Configuration.GetSection("Builds")).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<ConsistencyCheckSettings>().Bind(Configuration.GetSection("ConsistencyCheck")).ValidateDataAnnotations();
|
|
services.AddOptions<ReferenceResolverSettings>().Bind(Configuration.GetSection("ReferenceResolver")).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<BufferedPayloadOptions>().Bind(Configuration.GetSection("PayloadBuffering")).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<UpstreamRelaySettings>().Configure(o => Configuration.GetSection("Upstream").Bind(o)).ValidateDataAnnotations();
|
|
services.AddOptions<ClusterSettings>().Configure(o => Configuration.GetSection("Cluster").Bind(o)).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<GCSettings>().Configure(o => Configuration.GetSection("GC").Bind(o)).ValidateDataAnnotations();
|
|
services.AddOptions<ReplicationSettings>().Configure(o => Configuration.GetSection("Replication").Bind(o)).ValidateDataAnnotations();
|
|
services.AddOptions<ServiceCredentialSettings>().Configure(o => Configuration.GetSection("ServiceCredentials").Bind(o)).ValidateDataAnnotations();
|
|
services.AddOptions<SnapshotSettings>().Configure(o => Configuration.GetSection("Snapshot").Bind(o)).ValidateDataAnnotations();
|
|
services.AddOptions<MetricsServiceSettings>().Configure(o => Configuration.GetSection("Metrics").Bind(o)).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<ScyllaSettings>().Configure(o => Configuration.GetSection("Scylla").Bind(o)).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<KubernetesLeaderElectionSettings>().Configure(o => Configuration.GetSection("Kubernetes").Bind(o)).ValidateDataAnnotations();
|
|
services.AddOptions<StaticPeerServiceDiscoverySettings>().Configure(o => Configuration.GetSection("ServiceDiscovery").Bind(o)).ValidateDataAnnotations();
|
|
|
|
services.AddOptions<MemoryCacheReferencesSettings>().Configure(o => Configuration.GetSection("CacheRef").Bind(o)).ValidateDataAnnotations();
|
|
services.AddOptions<MemoryCacheContentIdSettings>().Configure(o => Configuration.GetSection("CacheContentId").Bind(o)).ValidateDataAnnotations();
|
|
|
|
services.AddSingleton(typeof(CompressedBufferUtils), CreateCompressedBufferUtils);
|
|
|
|
services.AddSingleton<AWSCredentials>(provider =>
|
|
{
|
|
AWSCredentialsSettings awsSettings = provider.GetService<IOptionsMonitor<AWSCredentialsSettings>>()!.CurrentValue;
|
|
|
|
return AWSCredentialsHelper.GetCredentials(awsSettings, "Jupiter");
|
|
});
|
|
services.AddSingleton<BufferedPayloadFactory>();
|
|
services.AddSingleton<ReplicationLogFactory>();
|
|
|
|
services.AddSingleton<BlobCleanupService>();
|
|
services.AddHostedService<BlobCleanupService>(p => p.GetService<BlobCleanupService>()!);
|
|
|
|
services.AddSingleton(serviceType: typeof(IRefService), typeof(ObjectService));
|
|
services.AddSingleton(serviceType: typeof(IReferencesStore), ObjectStoreFactory);
|
|
services.AddSingleton(serviceType: typeof(IReferenceResolver), typeof(ReferenceResolver));
|
|
services.AddSingleton(serviceType: typeof(IContentIdStore), ContentIdStoreFactory);
|
|
|
|
services.AddSingleton(serviceType: typeof(IBlobIndex), BlobIndexFactory);
|
|
services.AddSingleton(serviceType: typeof(IAmazonS3), CreateS3);
|
|
|
|
services.AddSingleton(serviceType: typeof(IBlockStore), BlockIndexFactory);
|
|
services.AddSingleton(serviceType: typeof(IBuildStore), BuildStoreFactory);
|
|
|
|
services.AddSingleton<AmazonS3Store>();
|
|
services.AddSingleton<FileSystemStore>();
|
|
services.AddSingleton<AzureBlobStore>();
|
|
services.AddSingleton<PeerBlobStore>();
|
|
services.AddSingleton<MemoryBlobStore>();
|
|
services.AddSingleton<RelayBlobStore>();
|
|
|
|
services.AddSingleton<OrphanBlobCleanupRefs>();
|
|
|
|
services.AddSingleton(typeof(IBlobService), typeof(BlobService));
|
|
services.AddSingleton(serviceType: typeof(IScyllaSessionManager), ScyllaFactory);
|
|
|
|
services.AddSingleton(serviceType: typeof(IReplicationLog), ReplicationLogFactory);
|
|
|
|
services.AddSingleton<LastAccessTrackerReference>();
|
|
services.AddSingleton(serviceType: typeof(ILastAccessCache<LastAccessRecord>), p => p.GetService<LastAccessTrackerReference>()!);
|
|
services.AddSingleton(serviceType: typeof(ILastAccessTracker<LastAccessRecord>), p => p.GetService<LastAccessTrackerReference>()!);
|
|
|
|
services.AddSingleton(serviceType: typeof(ILeaderElection), CreateLeaderElection);
|
|
|
|
services.AddTransient<IRequestHelper, RequestHelper>();
|
|
services.AddSingleton<BufferedPayloadFactory>();
|
|
services.AddSingleton(Configuration);
|
|
|
|
services.AddSingleton<FormatResolver>();
|
|
services.AddSingleton<NginxRedirectHelper>();
|
|
|
|
services.AddSingleton(serviceType: typeof(ISecretResolver), typeof(SecretResolver));
|
|
services.AddSingleton(typeof(IAmazonSecretsManager), CreateAWSSecretsManager);
|
|
|
|
services.AddSingleton<LastAccessServiceReferences>();
|
|
services.AddHostedService<LastAccessServiceReferences>(p => p.GetService<LastAccessServiceReferences>()!);
|
|
|
|
services.AddSingleton<IServiceCredentials, ServiceCredentials>(p => ActivatorUtilities.CreateInstance<ServiceCredentials>(p));
|
|
|
|
services.AddSingleton<ReplicationService>();
|
|
services.AddHostedService<ReplicationService>(p => p.GetService<ReplicationService>()!);
|
|
|
|
services.AddSingleton<ReplicationSnapshotService>();
|
|
services.AddHostedService<ReplicationSnapshotService>(p => p.GetService<ReplicationSnapshotService>()!);
|
|
|
|
services.AddSingleton<BlobStoreConsistencyCheckService>();
|
|
services.AddHostedService<BlobStoreConsistencyCheckService>(p => p.GetService<BlobStoreConsistencyCheckService>()!);
|
|
|
|
services.AddSingleton<BlobIndexConsistencyCheckService>();
|
|
services.AddHostedService<BlobIndexConsistencyCheckService>(p => p.GetService<BlobIndexConsistencyCheckService>()!);
|
|
|
|
services.AddSingleton<RefStoreConsistencyCheckService>();
|
|
services.AddHostedService<RefStoreConsistencyCheckService>(p => p.GetService<RefStoreConsistencyCheckService>()!);
|
|
|
|
services.AddSingleton<BuildStoreConsistencyCheckService>();
|
|
services.AddHostedService<BuildStoreConsistencyCheckService>(p => p.GetService<BuildStoreConsistencyCheckService>()!);
|
|
|
|
services.AddSingleton<MetricsService>();
|
|
services.AddHostedService<MetricsService>(p => p.GetService<MetricsService>()!);
|
|
|
|
services.AddSingleton(typeof(IPeerStatusService), typeof(PeerStatusService));
|
|
services.AddHostedService<PeerStatusService>(p => (PeerStatusService)p.GetService<IPeerStatusService>()!);
|
|
|
|
services.AddTransient(typeof(IRefCleanup), typeof(RefLastAccessCleanup));
|
|
|
|
services.AddTransient(typeof(VersionFile), typeof(VersionFile));
|
|
|
|
services.AddSingleton<RefCleanupService>();
|
|
services.AddHostedService<RefCleanupService>(p => p.GetService<RefCleanupService>()!);
|
|
|
|
// This will technically create a new settings object, but as we do not dynamically update it the settings will reflect what we actually want
|
|
UnrealCloudDDCSettings settings = services.BuildServiceProvider().GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
|
|
if (settings.LeaderElectionImplementation == UnrealCloudDDCSettings.LeaderElectionImplementations.Kubernetes)
|
|
{
|
|
// add the kubernetes leader instance under its actual type as well to make it easier to find
|
|
services.AddSingleton<KubernetesLeaderElection>(p => (KubernetesLeaderElection)p.GetService<ILeaderElection>()!);
|
|
services.AddHostedService<KubernetesLeaderElection>(p => p.GetService<KubernetesLeaderElection>()!);
|
|
}
|
|
|
|
services.AddSingleton<BlobStoreConsistencyCheckService>();
|
|
services.AddSingleton(typeof(IPeerServiceDiscovery), ServiceDiscoveryFactory);
|
|
}
|
|
|
|
private IBlockStore BlockIndexFactory(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
switch (settings.BuildStoreImplementation)
|
|
{
|
|
case UnrealCloudDDCSettings.BuildStoreImplementations.Memory:
|
|
return ActivatorUtilities.CreateInstance<MemoryBlockStore>(provider);
|
|
case UnrealCloudDDCSettings.BuildStoreImplementations.Scylla:
|
|
return ActivatorUtilities.CreateInstance<ScyllaBlockStore>(provider);
|
|
default:
|
|
throw new NotImplementedException($"Unknown build index implementation: {settings.BuildStoreImplementation}");
|
|
}
|
|
}
|
|
|
|
private IBuildStore BuildStoreFactory(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
switch (settings.BuildStoreImplementation)
|
|
{
|
|
case UnrealCloudDDCSettings.BuildStoreImplementations.Memory:
|
|
return ActivatorUtilities.CreateInstance<MemoryBuildStore>(provider);
|
|
case UnrealCloudDDCSettings.BuildStoreImplementations.Scylla:
|
|
return ActivatorUtilities.CreateInstance<ScyllaBuildStore>(provider);
|
|
default:
|
|
throw new NotImplementedException($"Unknown build store implementation: {settings.BuildStoreImplementation}");
|
|
}
|
|
}
|
|
|
|
private IBlobIndex BlobIndexFactory(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
switch (settings.BlobIndexImplementation)
|
|
{
|
|
case UnrealCloudDDCSettings.BlobIndexImplementations.Memory:
|
|
return ActivatorUtilities.CreateInstance<MemoryBlobIndex>(provider);
|
|
case UnrealCloudDDCSettings.BlobIndexImplementations.Scylla:
|
|
return ActivatorUtilities.CreateInstance<ScyllaBlobIndex>(provider);
|
|
case UnrealCloudDDCSettings.BlobIndexImplementations.Mongo:
|
|
return ActivatorUtilities.CreateInstance<MongoBlobIndex>(provider);
|
|
case UnrealCloudDDCSettings.BlobIndexImplementations.Cache:
|
|
return ActivatorUtilities.CreateInstance<CachedBlobIndex>(provider);
|
|
default:
|
|
throw new NotImplementedException($"Unknown blob index implementation: {settings.BlobIndexImplementation}");
|
|
}
|
|
}
|
|
|
|
private IPeerServiceDiscovery ServiceDiscoveryFactory(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
switch (settings.ServiceDiscoveryImplementation)
|
|
{
|
|
case UnrealCloudDDCSettings.ServiceDiscoveryImplementations.Kubernetes:
|
|
return ActivatorUtilities.CreateInstance<KubernetesPeerServiceDiscovery>(provider);
|
|
case UnrealCloudDDCSettings.ServiceDiscoveryImplementations.Static:
|
|
return ActivatorUtilities.CreateInstance<StaticPeerServiceDiscovery>(provider);
|
|
default:
|
|
throw new NotImplementedException($"Unknown service discovery implementation: {settings.ServiceDiscoveryImplementation}");
|
|
}
|
|
}
|
|
|
|
private IAmazonSecretsManager CreateAWSSecretsManager(IServiceProvider provider)
|
|
{
|
|
AWSCredentials awsCredentials = provider.GetService<AWSCredentials>()!;
|
|
AWSOptions awsOptions = Configuration.GetAWSOptions();
|
|
awsOptions.Credentials = awsCredentials;
|
|
|
|
IAmazonSecretsManager serviceClient = awsOptions.CreateServiceClient<IAmazonSecretsManager>();
|
|
return serviceClient;
|
|
}
|
|
|
|
private CompressedBufferUtils CreateCompressedBufferUtils(IServiceProvider provider)
|
|
{
|
|
return ActivatorUtilities.CreateInstance<CompressedBufferUtils>(provider);
|
|
}
|
|
|
|
private object ContentIdStoreFactory(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
IContentIdStore store = settings.ContentIdStoreImplementation switch
|
|
{
|
|
UnrealCloudDDCSettings.ContentIdStoreImplementations.Memory => ActivatorUtilities
|
|
.CreateInstance<MemoryContentIdStore>(provider),
|
|
UnrealCloudDDCSettings.ContentIdStoreImplementations.Scylla => ActivatorUtilities
|
|
.CreateInstance<ScyllaContentIdStore>(provider),
|
|
UnrealCloudDDCSettings.ContentIdStoreImplementations.Mongo => ActivatorUtilities
|
|
.CreateInstance<MongoContentIdStore>(provider),
|
|
UnrealCloudDDCSettings.ContentIdStoreImplementations.Cache => ActivatorUtilities
|
|
.CreateInstance<CacheContentIdStore>(provider),
|
|
_ => throw new NotImplementedException(
|
|
$"Unknown content id store implementation: {settings.ContentIdStoreImplementation}")
|
|
};
|
|
|
|
MemoryCacheContentIdSettings memoryCacheSettings = provider.GetService<IOptionsMonitor<MemoryCacheContentIdSettings>>()!.CurrentValue;
|
|
|
|
if (memoryCacheSettings.Enabled)
|
|
{
|
|
store = ActivatorUtilities.CreateInstance<MemoryCachedContentIdStore>(provider, store);
|
|
}
|
|
return store;
|
|
}
|
|
|
|
private IScyllaSessionManager ScyllaFactory(IServiceProvider provider)
|
|
{
|
|
ScyllaSettings settings = provider.GetService<IOptionsMonitor<ScyllaSettings>>()!.CurrentValue!;
|
|
ISecretResolver secretResolver = provider.GetService<ISecretResolver>()!;
|
|
|
|
using Serilog.Extensions.Logging.SerilogLoggerProvider serilogLoggerProvider = new();
|
|
Diagnostics.AddLoggerProvider(serilogLoggerProvider);
|
|
|
|
string? connectionString = secretResolver.Resolve(settings.ConnectionString);
|
|
if (string.IsNullOrEmpty(connectionString))
|
|
{
|
|
throw new Exception("Connection string has to be specified");
|
|
}
|
|
|
|
CassandraConnectionStringBuilder cassandraConnectionStringBuilder = new CassandraConnectionStringBuilder(connectionString);
|
|
if (string.IsNullOrEmpty(cassandraConnectionStringBuilder.DefaultKeyspace))
|
|
{
|
|
throw new Exception("Default Keyspace has to be specified");
|
|
}
|
|
|
|
// Configure the builder with your cluster's contact points
|
|
Builder clusterBuilder = Cluster.Builder()
|
|
.WithConnectionString(connectionString)
|
|
.WithLoadBalancingPolicy(Policies.NewDefaultLoadBalancingPolicy(settings.LocalDatacenterName))
|
|
.WithPoolingOptions(PoolingOptions.Create().SetMaxConnectionsPerHost(HostDistance.Local, settings.MaxConnectionForLocalHost).SetMaxRequestsPerConnection(settings.MaxRequestsPerConnection))
|
|
.WithExecutionProfiles(options =>
|
|
options.WithProfile("default", builder => builder.WithConsistencyLevel(ConsistencyLevel.LocalOne)));
|
|
|
|
if (settings.ReadTimeout != -1)
|
|
{
|
|
clusterBuilder.SocketOptions.SetReadTimeoutMillis(settings.ReadTimeout);
|
|
}
|
|
if (settings.UseAzureCosmosDB)
|
|
{
|
|
CassandraConnectionStringBuilder connectionStringBuilder = new CassandraConnectionStringBuilder(connectionString);
|
|
string[] contactPoints = connectionStringBuilder.ContactPoints;
|
|
|
|
// Connect to cassandra cluster using TLSv1.2.
|
|
if (settings.UseSSL)
|
|
{
|
|
SSLOptions sslOptions = new SSLOptions(SslProtocols.None, false,
|
|
(sender, certificate, chain, sslPolicyErrors) =>
|
|
{
|
|
if (sslPolicyErrors == SslPolicyErrors.None)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
Logger.LogError("Certificate error: {Errors}", sslPolicyErrors);
|
|
// Do not allow this client to communicate with unauthenticated servers.
|
|
return false;
|
|
});
|
|
|
|
// Prepare a map to resolve the host name from the IP address.
|
|
Dictionary<IPAddress, string> hostNameByIp = new Dictionary<IPAddress, string>();
|
|
foreach (string contactPoint in contactPoints)
|
|
{
|
|
IPAddress[] resolvedIps = Dns.GetHostAddresses(contactPoint);
|
|
foreach (IPAddress resolvedIp in resolvedIps)
|
|
{
|
|
hostNameByIp[resolvedIp] = contactPoint;
|
|
}
|
|
}
|
|
|
|
sslOptions.SetHostNameResolver((ipAddress) =>
|
|
{
|
|
if (hostNameByIp.TryGetValue(ipAddress, out string? resolvedName))
|
|
{
|
|
return resolvedName;
|
|
}
|
|
IPHostEntry hostEntry = Dns.GetHostEntry(ipAddress.ToString());
|
|
return hostEntry.HostName;
|
|
});
|
|
|
|
clusterBuilder = clusterBuilder.WithSSL(sslOptions);
|
|
}
|
|
}
|
|
Cluster cluster = clusterBuilder.Build();
|
|
|
|
Dictionary<string, string> replicationStrategy = ReplicationStrategies.CreateSimpleStrategyReplicationProperty(2);
|
|
if (settings.KeyspaceReplicationStrategy != null)
|
|
{
|
|
replicationStrategy = settings.KeyspaceReplicationStrategy;
|
|
Logger.LogInformation("Scylla Replication strategy for replicated keyspace is set to {@ReplicationStrategy}", replicationStrategy);
|
|
}
|
|
|
|
ISession replicatedSession;
|
|
const int MaxRetryAttempts = 100;
|
|
int countOfAttempts = 0;
|
|
while (true)
|
|
{
|
|
try
|
|
{
|
|
countOfAttempts += 1;
|
|
replicatedSession = cluster.ConnectAndCreateDefaultKeyspaceIfNotExists(replicationStrategy);
|
|
break;
|
|
}
|
|
catch (NoHostAvailableException)
|
|
{
|
|
Logger.LogWarning("Failed to connect to scylla, waiting a while and will then retry. Attempt {AttemptNum} of {NumAttempts}", countOfAttempts, MaxRetryAttempts);
|
|
// wait for a few seconds before attempt to connect again
|
|
Thread.Sleep(5000);
|
|
if (countOfAttempts >= MaxRetryAttempts)
|
|
{
|
|
Logger.LogError("Failed to connect to Scylla after {NumAttempts} attempts, giving up.", countOfAttempts);
|
|
throw;
|
|
}
|
|
}
|
|
}
|
|
|
|
string keyspace = replicatedSession.Keyspace;
|
|
if (!settings.AvoidSchemaChanges)
|
|
{
|
|
replicatedSession.Execute(new SimpleStatement("CREATE TYPE IF NOT EXISTS blob_identifier (hash blob)"));
|
|
replicatedSession.Execute(new SimpleStatement("CREATE TYPE IF NOT EXISTS object_reference (bucket text, key text)"));
|
|
}
|
|
|
|
replicatedSession.UserDefinedTypes.Define(UdtMap.For<ScyllaBlobIdentifier>("blob_identifier", keyspace));
|
|
replicatedSession.UserDefinedTypes.Define(UdtMap.For<ScyllaObjectReference>("object_reference", keyspace));
|
|
|
|
string localKeyspaceName = $"{keyspace}_local_{settings.LocalKeyspaceSuffix}";
|
|
|
|
Dictionary<string, string> replicationStrategyLocal = ReplicationStrategies.CreateSimpleStrategyReplicationProperty(2);
|
|
if (settings.LocalKeyspaceReplicationStrategy != null)
|
|
{
|
|
replicationStrategyLocal = settings.LocalKeyspaceReplicationStrategy;
|
|
Logger.LogInformation("Scylla Replication strategy for local keyspace is set to {@ReplicationStrategy}", replicationStrategyLocal);
|
|
}
|
|
|
|
// the local keyspace is never replicated so we do not support controlling how the replication strategy is setup
|
|
replicatedSession.CreateKeyspaceIfNotExists(localKeyspaceName, replicationStrategyLocal);
|
|
ISession localSession = cluster.Connect(localKeyspaceName);
|
|
|
|
if (!settings.AvoidSchemaChanges)
|
|
{
|
|
localSession.Execute(new SimpleStatement("CREATE TYPE IF NOT EXISTS blob_identifier (hash blob)"));
|
|
}
|
|
localSession.UserDefinedTypes.Define(UdtMap.For<ScyllaBlobIdentifier>("blob_identifier", localKeyspaceName));
|
|
|
|
bool isScylla = !settings.UseAzureCosmosDB;
|
|
return new ScyllaSessionManager(replicatedSession, localSession, isScylla, !isScylla);
|
|
}
|
|
|
|
private IReferencesStore ObjectStoreFactory(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
IReferencesStore store = settings.ReferencesDbImplementation switch
|
|
{
|
|
UnrealCloudDDCSettings.ReferencesDbImplementations.Memory => ActivatorUtilities.CreateInstance<MemoryReferencesStore>(provider),
|
|
UnrealCloudDDCSettings.ReferencesDbImplementations.Scylla => ActivatorUtilities.CreateInstance<ScyllaReferencesStore>(provider),
|
|
UnrealCloudDDCSettings.ReferencesDbImplementations.Mongo => ActivatorUtilities.CreateInstance<MongoReferencesStore>(provider),
|
|
UnrealCloudDDCSettings.ReferencesDbImplementations.Cache => ActivatorUtilities.CreateInstance<CacheReferencesStore>(provider),
|
|
_ => throw new NotImplementedException(
|
|
$"Unknown references db implementation: {settings.ReferencesDbImplementation}")
|
|
};
|
|
|
|
MemoryCacheReferencesSettings memoryCacheSettings = provider.GetService<IOptionsMonitor<MemoryCacheReferencesSettings>>()!.CurrentValue;
|
|
|
|
if (memoryCacheSettings.Enabled)
|
|
{
|
|
store = ActivatorUtilities.CreateInstance<MemoryCachedReferencesStore>(provider, store);
|
|
}
|
|
|
|
return store;
|
|
}
|
|
|
|
private ILeaderElection CreateLeaderElection(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
if (settings.LeaderElectionImplementation == UnrealCloudDDCSettings.LeaderElectionImplementations.Kubernetes)
|
|
{
|
|
return ActivatorUtilities.CreateInstance<KubernetesLeaderElection>(provider);
|
|
}
|
|
else if (settings.LeaderElectionImplementation == UnrealCloudDDCSettings.LeaderElectionImplementations.Static)
|
|
{
|
|
// hard coded leader election that assumes it is always the leader
|
|
return ActivatorUtilities.CreateInstance<LeaderElectionStub>(provider, true);
|
|
}
|
|
else if (settings.LeaderElectionImplementation == UnrealCloudDDCSettings.LeaderElectionImplementations.Disabled)
|
|
{
|
|
// disabled leader election means we are never the leader
|
|
return ActivatorUtilities.CreateInstance<LeaderElectionStub>(provider, false);
|
|
}
|
|
else
|
|
{
|
|
throw new NotImplementedException($"Unknown leader election set {settings.LeaderElectionImplementation}");
|
|
}
|
|
}
|
|
|
|
private IAmazonS3 CreateS3(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue!;
|
|
|
|
if (settings.StorageImplementations == null)
|
|
{
|
|
throw new ArgumentException("No storage implementation set");
|
|
}
|
|
|
|
bool isS3InUse = settings.StorageImplementations.Any(x =>
|
|
string.Equals(x, UnrealCloudDDCSettings.StorageBackendImplementations.S3.ToString(), StringComparison.OrdinalIgnoreCase));
|
|
|
|
if (isS3InUse)
|
|
{
|
|
S3Settings s3Settings = provider.GetService<IOptionsMonitor<S3Settings>>()!.CurrentValue!;
|
|
AWSCredentials awsCredentials = provider.GetService<AWSCredentials>()!;
|
|
AWSOptions awsOptions = Configuration.GetAWSOptions();
|
|
if (s3Settings.ConnectionString.ToUpper() != "AWS")
|
|
{
|
|
awsOptions.DefaultClientConfig.ServiceURL = s3Settings.ConnectionString;
|
|
}
|
|
|
|
awsOptions.Credentials = awsCredentials;
|
|
|
|
IAmazonS3 serviceClient = awsOptions.CreateServiceClient<IAmazonS3>();
|
|
|
|
if (serviceClient.Config is AmazonS3Config c)
|
|
{
|
|
if (s3Settings.ForceAWSPathStyle)
|
|
{
|
|
c.ForcePathStyle = true;
|
|
// this works around a issue with presigned urls when using minio to emulate S3 due to AWS SDK not defaulting to signature v4 for presigned urls outside of us-east-1
|
|
// see https://github.com/minio/minio/issues/19887
|
|
AWSConfigsS3.UseSignatureVersion4 = true;
|
|
}
|
|
|
|
if (s3Settings.UseArnRegion.HasValue)
|
|
{
|
|
c.UseArnRegion = s3Settings.UseArnRegion.Value;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
throw new NotImplementedException();
|
|
}
|
|
|
|
return serviceClient;
|
|
}
|
|
|
|
return null!;
|
|
}
|
|
|
|
private IReplicationLog ReplicationLogFactory(IServiceProvider provider)
|
|
{
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue;
|
|
IReplicationLog replicationLog = settings.ReplicationLogWriterImplementation switch
|
|
{
|
|
UnrealCloudDDCSettings.ReplicationLogWriterImplementations.Scylla =>
|
|
ActivatorUtilities.CreateInstance<ScyllaReplicationLog>(provider),
|
|
UnrealCloudDDCSettings.ReplicationLogWriterImplementations.Memory =>
|
|
ActivatorUtilities.CreateInstance<MemoryReplicationLog>(provider),
|
|
_ => throw new NotImplementedException()
|
|
};
|
|
|
|
MemoryCacheReplicationLogSettings memoryCacheSettings = provider.GetService<IOptionsMonitor<MemoryCacheReplicationLogSettings>>()!.CurrentValue;
|
|
|
|
if (memoryCacheSettings.Enabled)
|
|
{
|
|
replicationLog = ActivatorUtilities.CreateInstance<MemoryCachedReplicationLog>(provider, replicationLog);
|
|
}
|
|
|
|
return replicationLog;
|
|
}
|
|
|
|
protected override void OnAddHealthChecks(IServiceCollection services, IHealthChecksBuilder healthChecks)
|
|
{
|
|
ServiceProvider provider = services.BuildServiceProvider();
|
|
UnrealCloudDDCSettings settings = provider.GetService<IOptionsMonitor<UnrealCloudDDCSettings>>()!.CurrentValue;
|
|
|
|
foreach (UnrealCloudDDCSettings.StorageBackendImplementations impl in settings.GetStorageImplementations())
|
|
{
|
|
switch (impl)
|
|
{
|
|
case UnrealCloudDDCSettings.StorageBackendImplementations.S3:
|
|
|
|
// The S3 health checks are disabled as they do not support dynamic credentials similar to the issue with dynamo
|
|
/*AWSCredentials awsCredentials = provider.GetService<AWSCredentials>();
|
|
S3Settings s3Settings = provider.GetService<IOptionsMonitor<S3Settings>>()!.CurrentValue;
|
|
healthChecks.AddS3(options =>
|
|
{
|
|
options.BucketName = s3Settings.BucketName;
|
|
options.S3Config = provider.GetService<IAmazonS3>().Config as AmazonS3Config;
|
|
options.Credentials = awsCredentials;
|
|
}, tags: new[] {"services"});*/
|
|
break;
|
|
case UnrealCloudDDCSettings.StorageBackendImplementations.Azure:
|
|
// Health checks for Azure are disabled as the connection string will vary based on the namespace used
|
|
/*AzureSettings azureSettings = provider.GetService<IOptionsMonitor<AzureSettings>>()!.CurrentValue;
|
|
healthChecks.AddAzureBlobStorage(AzureBlobStore.GetConnectionString(azureSettings, provider), tags: new[] {"services"});*/
|
|
|
|
break;
|
|
|
|
case UnrealCloudDDCSettings.StorageBackendImplementations.FileSystem:
|
|
healthChecks.AddDiskStorageHealthCheck(options =>
|
|
{
|
|
FilesystemSettings filesystemSettings = provider.GetService<IOptionsMonitor<FilesystemSettings>>()!.CurrentValue;
|
|
string? driveRoot = Path.GetPathRoot(PathUtil.ResolvePath(filesystemSettings.RootDir));
|
|
if (!string.IsNullOrEmpty(driveRoot))
|
|
{
|
|
options.AddDrive(driveRoot);
|
|
}
|
|
});
|
|
break;
|
|
case UnrealCloudDDCSettings.StorageBackendImplementations.Memory:
|
|
case UnrealCloudDDCSettings.StorageBackendImplementations.Relay:
|
|
case UnrealCloudDDCSettings.StorageBackendImplementations.Peer:
|
|
break;
|
|
default:
|
|
throw new ArgumentOutOfRangeException("Unhandled storage impl " + impl);
|
|
}
|
|
}
|
|
|
|
healthChecks.AddCheck<LastAccessServiceCheck>("LastAccessServiceCheck", tags: new[] { "services" });
|
|
|
|
healthChecks.AddCheck<ReplicatorServiceCheck>("ReplicatorServiceCheck", tags: new[] { "services" });
|
|
healthChecks.AddCheck<ReplicationSnapshotServiceCheck>("ReplicationSnapshotServiceCheck", tags: new[] { "services" });
|
|
|
|
GCSettings gcSettings = provider.GetService<IOptionsMonitor<GCSettings>>()!.CurrentValue;
|
|
|
|
if (gcSettings.CleanOldRefRecords)
|
|
{
|
|
healthChecks.AddCheck<RefCleanupServiceCheck>("RefCleanupCheck", tags: new[] { "services" });
|
|
}
|
|
|
|
if (gcSettings.BlobCleanupServiceEnabled)
|
|
{
|
|
healthChecks.AddCheck<BlobCleanupServiceCheck>("BlobStoreCheck", tags: new[] { "services" });
|
|
}
|
|
|
|
if (settings.LeaderElectionImplementation == UnrealCloudDDCSettings.LeaderElectionImplementations.Kubernetes)
|
|
{
|
|
healthChecks.AddCheck<KubernetesLeaderServiceCheck>("KubernetesLeaderService", tags: new[] { "services" });
|
|
}
|
|
}
|
|
}
|
|
}
|