// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Net; using System.Net.Http; using System.Threading.Tasks; using EpicGames.Core; using Microsoft.Extensions.Http; using Microsoft.Extensions.Logging; using Polly; using Polly.Extensions.Http; using Polly.Retry; using Polly.Timeout; namespace EpicGames.Horde { /// /// Concrete implementation of which manages the lifetime of the instance. /// class HordeHttpMessageHandler : IHordeHttpMessageHandler, IDisposable { readonly HttpMessageHandler _instance; /// public HttpMessageHandler Instance => _instance; /// /// Constructor /// /// public HordeHttpMessageHandler(ILogger logger) { _instance = new SocketsHttpHandler(); _instance = new PolicyHttpMessageHandler(request => CreateDefaultTimeoutRetryPolicy(request, logger)) { InnerHandler = _instance }; _instance = new PolicyHttpMessageHandler(request => CreateHordeTransientErrorPolicy(request, logger)) { InnerHandler = _instance }; } /// public void Dispose() => _instance.Dispose(); /// /// Create a default timeout retry policy /// static IAsyncPolicy CreateDefaultTimeoutRetryPolicy(HttpRequestMessage request, ILogger logger) { // Wait 30 seconds for operations to timeout Task OnTimeoutAsync(Context context, TimeSpan timespan, Task timeoutTask) { logger.LogWarning(KnownLogEvents.Systemic_Horde_Http, "{Method} {Url} timed out after {Time}s.", request.Method, request.RedactedRequestUri(), (int)timespan.TotalSeconds); return Task.CompletedTask; } AsyncTimeoutPolicy timeoutPolicy = Policy.TimeoutAsync(60, OnTimeoutAsync); // Retry twice after a timeout void OnRetry(Exception ex, TimeSpan timespan) { logger.LogWarning(KnownLogEvents.Systemic_Horde_Http, ex, "{Method} {Url} retrying after {Time}s.", request.Method, request.RedactedRequestUri(), timespan.TotalSeconds); } TimeSpan[] retryTimes = new[] { TimeSpan.FromSeconds(5.0), TimeSpan.FromSeconds(10.0), TimeSpan.FromSeconds(15.0) }; AsyncRetryPolicy retryPolicy = Policy.Handle().WaitAndRetryAsync(retryTimes, OnRetry); return retryPolicy.WrapAsync(timeoutPolicy); } /// /// Create an error and timeout retry policy taking into account Horde's use of 503 errors /// static IAsyncPolicy CreateHordeTransientErrorPolicy(HttpRequestMessage request, ILogger logger) { Task OnTimeoutAsync(DelegateResult outcome, TimeSpan timespan, int retryAttempt, Context context) { logger.LogInformation(KnownLogEvents.Systemic_Horde_Http, "{Method} {Url} failed ({Result}). Delaying for {DelayMs}ms (attempt #{RetryNum}).", request.Method, request.RedactedRequestUri(), outcome.Result?.StatusCode, timespan.TotalMilliseconds, retryAttempt); return Task.CompletedTask; } TimeSpan[] retryTimes = new[] { TimeSpan.FromSeconds(1.0), TimeSpan.FromSeconds(5.0), TimeSpan.FromSeconds(10.0), TimeSpan.FromSeconds(30.0), TimeSpan.FromSeconds(30.0) }; // Policy for transient errors is the same as HttpPolicyExtensions.HandleTransientHttpError(), but excludes HttpStatusCode.ServiceUnavailable (which is used as a response // when allocating compute resources when none are available). This pathway is handled explicitly on the application side. return Policy .Handle() .OrResult(x => (x.StatusCode >= HttpStatusCode.InternalServerError && x.StatusCode != HttpStatusCode.ServiceUnavailable) || x.StatusCode == HttpStatusCode.RequestTimeout) .WaitAndRetryAsync(retryTimes, OnTimeoutAsync); } /// /// Create a default retry policy /// public static IAsyncPolicy CreateDefaultTransientErrorPolicy() { return HttpPolicyExtensions .HandleTransientHttpError() .OrResult(response => response.StatusCode == HttpStatusCode.ServiceUnavailable) .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))); } } }