// Copyright Epic Games, Inc. All Rights Reserved. #pragma once #include "Containers/Array.h" #include "Containers/Set.h" #include "Containers/UnrealString.h" #include "CoreMinimal.h" #include "HAL/PlatformCrt.h" #include "HAL/PlatformMath.h" #include "HttpRequestAdapter.h" #include "Interfaces/IHttpRequest.h" #include "Logging/LogVerbosity.h" #include "Misc/Optional.h" #include "Templates/Atomic.h" #include "Templates/SharedPointer.h" #include "Templates/UnrealTemplate.h" #include "UObject/NameTypes.h" /** * Helpers of various types for the retry system */ namespace FHttpRetrySystem { class FManager; typedef uint32 RetryLimitCountType; typedef double RetryTimeoutRelativeSecondsType; inline RetryLimitCountType RetryLimitCount(uint32 Value) { return Value; } inline RetryTimeoutRelativeSecondsType RetryTimeoutRelativeSeconds(double Value) { return Value; } template IntrinsicType TZero(); template <> inline float TZero() { return 0.0f; } template <> inline RetryLimitCountType TZero() { return RetryLimitCount(0); } template <> inline RetryTimeoutRelativeSecondsType TZero() { return RetryTimeoutRelativeSeconds(0.0); } typedef TOptional FRandomFailureRateSetting; typedef TOptional FRetryLimitCountSetting; typedef TOptional FRetryTimeoutRelativeSecondsSetting; typedef TSet FRetryResponseCodes; typedef TSet FRetryVerbs; struct FRetryDomains { FRetryDomains(TArray&& InDomains) : Domains(MoveTemp(InDomains)) , ActiveIndex(0) {} /** The domains to use */ TArray Domains; /** * Index into Domains to attempt * Domains are cycled through on some errors, and when we succeed on one domain, we remain on that domain until that domain results in an error */ TAtomic ActiveIndex; }; typedef TSharedPtr FRetryDomainsPtr; /** * Model for computing exponential backoff using the formula: Base**(CurrentRetryAttempt + Bias) * Then applying jitter to the backoff. Jitter application is performed by selecting a random value in the [Min, Max] range and multiplying it against the computed backoff. * Half jitter can be implemented using { 0.5, 1.0 }, which becomes Backoff' = Backoff * Rand(0.5, 1.0) = Backoff/2 + Rand(0, Backoff/2) * Full jitter can be implemented using { 0.0, 1.0 }, which becomes Backoff' = Backoff * Rand(0.0, 1.0) = Rand(0.0, Backoff) * No jitter can be implemented using { 0.0, 0.0 } or any pair that where Min > Max. Backoff' = Backoff */ struct FExponentialBackoffCurve { /** Exponential backoff base */ float Base = 2.0f; /** Exponential backoff bias added to the current retry number */ float ExponentBias = 1.0f; /** Exponential backoff jitter coefficient minimum value. Defaults to half jitter */ float MinCoefficient = 0.5f; /** Exponential backoff jitter coefficient maximum value. Defaults to half jitter */ float MaxCoefficient = 1.0f; /** Max back off seconds */ float MaxBackoffSeconds = 60.0f; bool IsValid() const; float Compute(uint32 RetryNumber) const; }; /** * Read the number of seconds a HTTP request is throttled for from the response * @param Response the HTTP response to read the value from * @return If found, the number of seconds the request is rate limited for. If not found, an unset TOptional */ TOptional HTTP_API ReadThrottledTimeFromResponseInSeconds(FHttpResponsePtr Response); }; namespace FHttpRetrySystem { /** * class FRequest is what the retry system accepts as inputs */ class FRequest : public FHttpRequestAdapterBase { public: struct EStatus { enum Type { NotStarted = 0, Processing, ProcessingLockout, Cancelled, FailedRetry, FailedTimeout, Succeeded }; }; public: // IHttpRequest interface HTTP_API virtual bool ProcessRequest() override; HTTP_API virtual void CancelRequest() override; // FRequest EStatus::Type GetRetryStatus() const { return RetryStatus; } protected: friend class FManager; HTTP_API FRequest( TSharedRef InManager, const TSharedRef& HttpRequest, const FRetryLimitCountSetting& InRetryLimitCountOverride = FRetryLimitCountSetting(), const FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsOverride = FRetryTimeoutRelativeSecondsSetting(), const FRetryResponseCodes& InRetryResponseCodes = FRetryResponseCodes(), const FRetryVerbs& InRetryVerbs = FRetryVerbs(), const FRetryDomainsPtr& InRetryDomains = FRetryDomainsPtr(), const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorOverride = FRetryLimitCountSetting(), const FExponentialBackoffCurve& InExponentialBackoffCurve = FExponentialBackoffCurve() ); void HttpOnRequestProgress(FHttpRequestPtr InHttpRequest, uint64 BytesSent, uint64 BytesRcv); void HttpOnProcessRequestComplete(FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded); void HttpOnStatusCodeReceived(FHttpRequestPtr Request, int32 StatusCode); void HttpOnHeaderReceived(FHttpRequestPtr Request, const FString& HeaderName, const FString& NewHeaderValue); /** Update our HTTP request's URL's domain from our RetryDomains */ void SetUrlFromRetryDomains(); /** Move to the next retry domain from our RetryDomains */ void MoveToNextRetryDomain(); void BindAdaptorDelegates(); EStatus::Type RetryStatus; FRetryLimitCountSetting RetryLimitCountOverride; FRetryLimitCountSetting RetryLimitCountForConnectionErrorOverride; FRetryTimeoutRelativeSecondsSetting RetryTimeoutRelativeSecondsOverride; FRetryResponseCodes RetryResponseCodes; FRetryVerbs RetryVerbs; FRetryDomainsPtr RetryDomains; /** The current index in RetryDomains we are attempting */ int32 RetryDomainsIndex = 0; /** The original URL before replacing anything from RetryDomains */ FString OriginalUrl; TWeakPtr RetryManager; /** Save the last response before the retry */ FHttpResponsePtr LastResponse; bool bLastSucceeded = false; /** Exponential backoff curve */ FExponentialBackoffCurve RetryExponentialBackoffCurve; bool bBoundAdaptorDelegates = false; }; } namespace FHttpRetrySystem { class FManager : public TSharedFromThis { public: // FManager HTTP_API FManager( const FRetryLimitCountSetting& InRetryLimitCountDefault, const FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsDefault, const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorDefault = FRetryLimitCountSetting() ); /** * Create a new http request with retries */ HTTP_API TSharedRef CreateRequest( const FRetryLimitCountSetting& InRetryLimitCountOverride = FRetryLimitCountSetting(), const FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsOverride = FRetryTimeoutRelativeSecondsSetting(), const FRetryResponseCodes& InRetryResponseCodes = FRetryResponseCodes(), const FRetryVerbs& InRetryVerbs = FRetryVerbs(), const FRetryDomainsPtr& InRetryDomains = FRetryDomainsPtr(), const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorOverride = FRetryLimitCountSetting(), const FExponentialBackoffCurve & InExponentialBackoffCurve = FExponentialBackoffCurve() ); HTTP_API virtual ~FManager(); void SetRandomFailureRate(float Value) { RandomFailureRate = FRandomFailureRateSetting(Value); } void SetDefaultRetryLimit(uint32 Value) { RetryLimitCountDefault = FRetryLimitCountSetting(Value); } // @return Block the current process until all requests are flushed, or timeout has elapsed HTTP_API void BlockUntilFlushed(float TimeoutSec); protected: friend class FRequest; struct FHttpRetryRequestEntry { FHttpRetryRequestEntry(TSharedRef& InRequest); bool bShouldCancel; uint32 CurrentRetryCount; uint32 CurrentRetryCountForConnectionError; double RequestStartTimeAbsoluteSeconds; double LockoutEndTimeAbsoluteSeconds; TSharedRef Request; }; class FHttpLogVerbosityTracker { public: FHttpLogVerbosityTracker(); ~FHttpLogVerbosityTracker(); /** Mark that a request is being retried */ void IncrementRetriedRequests(); /** Mark that a retried request is no longer being retried */ void DecrementRetriedRequests(); static FHttpLogVerbosityTracker& Get(); protected: /** Update settings from config */ void UpdateSettingsFromConfig(); void OnConfigSectionsChanged(const FString& IniFilename, const TSet& SectionName); /** Number of requests that are in a retried state. When this is non-zero, verbosity will be adjusted. */ int32 NumRetriedRequests = 0; /** DecrementRetriedRequests can be called from game thread or http thread depends on the http request thread policy, make sure it's thread-safe */ FCriticalSection NumRetriedRequestsLock; /** Verbosity to restore to when there are no requests being retried */ ELogVerbosity::Type OriginalVerbosity = ELogVerbosity::Error; /** Config driven target verbosity to set to when requests are being retried. NoLogging means the verbosity will not be modified. */ ELogVerbosity::Type TargetVerbosity = ELogVerbosity::NoLogging; }; bool ProcessRequest(TSharedRef& HttpRequest); void CancelRequest(TSharedRef& HttpRequest); // @return true if there is a no formal response to the request // @TODO return true if a variety of 5xx errors are the result of a formal response bool ShouldRetry(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const; // @return true if retry chances have not been exhausted bool CanRetry(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const; // @return true if the retry request has timed out bool HasTimedOut(const FHttpRetryRequestEntry& HttpRetryRequestEntry, const double NowAbsoluteSeconds); /** * Retry an HTTP request * @param RequestEntry request to retry */ void RetryHttpRequest(FHttpRetryRequestEntry& RequestEntry); /** * Retry an HTTP request with delay * @param RequestEntry request retry * @param InDelay the delay to wait before retrying * @param bWasSucceeded was the request succeeded before retry */ void RetryHttpRequestWithDelay(FManager::FHttpRetryRequestEntry& RequestEntry, float InDelay, bool bWasSucceeded); void HttpRequestTimeoutAfterDelay(FManager::FHttpRetryRequestEntry& RequestEntry, bool bWasSucceeded, float Delay); // @return number of seconds to lockout for float GetLockoutPeriodSeconds(const FHttpRetryRequestEntry& HttpRetryRequestEntry); bool RetryLimitForConnectionErrorIsSet(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const; bool CanRetryForConnectionError(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const; bool CanRetryInGeneral(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const; // Default configuration for the retry system FRandomFailureRateSetting RandomFailureRate; FRetryLimitCountSetting RetryLimitCountDefault; FRetryLimitCountSetting RetryLimitCountForConnectionErrorDefault; FRetryTimeoutRelativeSecondsSetting RetryTimeoutRelativeSecondsDefault; TArray RequestList; FCriticalSection RequestListLock; }; }