// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StackExchange.Redis; namespace HordeServer.Server { /// /// Service containing an async task that allows long polling operations to complete early if the server is shutting down /// public sealed class LifetimeService : ILifetimeService, IHostedService, IAsyncDisposable { /// /// Writer for log output /// readonly ILogger _logger; /// /// Task source for the server stopping /// readonly TaskCompletionSource _stoppingTaskCompletionSource; // /// // /// Task source for the server stopping // /// // readonly TaskCompletionSource _preStoppingTaskCompletionSource; /// /// Registration token for the stopping event /// readonly CancellationTokenRegistration _registration; readonly IHostApplicationLifetime _lifetime; readonly IMongoService _mongoService; readonly IRedisService _redisService; readonly ITicker _ticker; /// readonly int? _shutdownMemoryThreshold = null; /* /// /// Max time to wait for any outstanding requests to finish /// readonly TimeSpan RequestGracefulTimeout = TimeSpan.FromMinutes(5); /// /// Initial delay before attempting the shutdown. This to ensure any load balancers/ingress will detect /// the server is unavailable to serve new requests in the event of no outstanding requests to wait for. /// readonly TimeSpan InitialStoppingDelay = TimeSpan.FromSeconds(35); */ /// /// Constructor /// /// Server settings /// Application lifetime interface /// Current ASP.NET environment /// Database singleton service /// Redis singleton service /// /// Logging interface public LifetimeService(IOptionsMonitor settings, IHostApplicationLifetime lifetime, IHostEnvironment env, IMongoService mongoService, IRedisService redisService, IClock clock, ILogger logger) { _shutdownMemoryThreshold = settings.CurrentValue.ShutdownMemoryThreshold; _lifetime = lifetime; _mongoService = mongoService; _redisService = redisService; _logger = logger; _stoppingTaskCompletionSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); // _preStoppingTaskCompletionSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); if (env.IsProduction() || env.IsDevelopment()) { _registration = lifetime.ApplicationStopping.Register(ApplicationStopping); } _ticker = clock.AddTicker(TimeSpan.FromMinutes(5), CheckMemoryUsageAsync, _logger); } /// public async ValueTask DisposeAsync() { await _registration.DisposeAsync(); await _ticker.DisposeAsync(); } /// public async Task StartAsync(CancellationToken cancellationToken) { await _ticker.StartAsync(); } /// public async Task StopAsync(CancellationToken cancellationToken) { await _ticker.StopAsync(); } private ValueTask CheckMemoryUsageAsync(CancellationToken cancellationToken) { if (_shutdownMemoryThreshold == null || _shutdownMemoryThreshold <= 0) { return ValueTask.CompletedTask; } // Force a garbage collection and wait for it to complete for a more accurate reading // Can be a heavy operation but this ticker method is run infrequently GC.Collect(); GC.WaitForPendingFinalizers(); long totalMemoryUsageMb = GC.GetTotalMemory(forceFullCollection: false) / 1024 / 1024; if (totalMemoryUsageMb > _shutdownMemoryThreshold) { _logger.LogWarning("Memory usage exceeded {MemoryThreshold} MB. Stopping process...", _shutdownMemoryThreshold); _lifetime.StopApplication(); } return ValueTask.CompletedTask; } /// /// Callback for the application stopping /// void ApplicationStopping() { _logger.LogInformation("Shutdown/SIGTERM signal received"); IsPreStopping = true; IsStopping = true; // _preStoppingTaskCompletionSource.TrySetResult(true); _stoppingTaskCompletionSource.TrySetResult(true); /* int shutdownDelayMs = 30 * 1000; _logger.LogInformation("Delaying shutdown by sleeping {ShutdownDelayMs} ms...", shutdownDelayMs); Thread.Sleep(shutdownDelayMs); _logger.LogInformation("Server process now shutting down..."); */ /* if (PreStoppingTaskCompletionSource.TrySetResult(true)) { WaitAndTriggerStoppingTask = Task.Run(() => ExecStoppingTask()); Logger.LogInformation("App is stopping. Waiting an initial {InitialDelay} secs before waiting on any requests...", (int)InitialStoppingDelay.TotalSeconds); Thread.Sleep(InitialStoppingDelay); Logger.LogInformation("Blocking shutdown for up to {MaxGraceTimeout} secs until all request have finished...", (int)RequestGracefulTimeout.TotalSeconds); DateTime StartTime = DateTime.UtcNow; do { RequestTrackerService.LogRequestsInProgress(); Thread.Sleep(5000); } while (DateTime.UtcNow < StartTime + RequestGracefulTimeout && RequestTrackerService.GetRequestsInProgress().Count > 0); if (RequestTrackerService.GetRequestsInProgress().Count == 0) { Logger.LogInformation("All open requests finished gracefully after {TimeTaken} secs", (DateTime.UtcNow - StartTime).TotalSeconds); } else { Logger.LogInformation("One or more requests did not finish within the grace period of {TimeTaken} secs. Shutdown will now resume with risk of interrupting those requests!", (DateTime.UtcNow - StartTime).TotalSeconds); RequestTrackerService.LogRequestsInProgress(); } } */ } /// /// Returns true if the server is stopping /// public bool IsStopping { get; private set; } /// /// Returns true if the server is stopping, but may not be removed from the load balancer yet /// public bool IsPreStopping { get; private set; } /// /// Gets an awaitable task for the server stopping /// public Task StoppingTask => _stoppingTaskCompletionSource.Task; /// /// Check if MongoDB can be reached /// /// True if communication works public async Task IsMongoDbConnectionHealthyAsync() { using CancellationTokenSource cancelSource = new CancellationTokenSource(10000); bool isHealthy = false; try { await _mongoService.Database.ListCollectionNamesAsync(null, cancelSource.Token); isHealthy = true; } catch (Exception e) { _logger.LogError(e, "MongoDB call failed during health check"); } return isHealthy; } /// /// Check if Redis can be reached /// /// True if communication works public async Task IsRedisConnectionHealthyAsync() { using CancellationTokenSource cancelSource = new CancellationTokenSource(10000); bool isHealthy = false; try { string key = "HordeLifetimeService-Health-Check"; IDatabase redis = _redisService.GetDatabase(); await redis.StringSetAsync(key, "ok"); await redis.StringGetAsync(key); isHealthy = true; } catch (Exception e) { _logger.LogError(e, "Redis call failed during health check"); } return isHealthy; } } }