Files
UnrealEngine/Engine/Source/Runtime/Online/HTTP/Private/HttpRetrySystem.cpp
2025-05-18 13:04:45 +08:00

768 lines
26 KiB
C++

// Copyright Epic Games, Inc. All Rights Reserved.
#include "HttpRetrySystem.h"
#include "AutoRTFM.h"
#include "GenericPlatform/HttpRequestCommon.h"
#include "HAL/ConsoleManager.h"
#include "HAL/PlatformTime.h"
#include "HAL/PlatformProcess.h"
#include "HAL/LowLevelMemTracker.h"
#include "Math/RandomStream.h"
#include "HttpModule.h"
#include "Http.h"
#include "HttpManager.h"
#include "HttpThread.h"
#include "Stats/Stats.h"
#include "Misc/ConfigCacheIni.h"
#include "Misc/CoreDelegates.h"
LLM_DEFINE_TAG(HTTP);
namespace FHttpRetrySystem
{
TOptional<double> ReadThrottledTimeFromResponseInSeconds(FHttpResponsePtr Response)
{
TOptional<double> LockoutPeriod;
// Check if there was a Retry-After header
if (Response.IsValid())
{
FString RetryAfter = Response->GetHeader(TEXT("Retry-After"));
if (!RetryAfter.IsEmpty())
{
if (RetryAfter.IsNumeric())
{
// seconds
LockoutPeriod.Emplace(FCString::Atof(*RetryAfter));
}
else
{
// http date
FDateTime UTCServerTime;
if (FDateTime::ParseHttpDate(RetryAfter, UTCServerTime))
{
const FDateTime UTCNow = FDateTime::UtcNow();
LockoutPeriod.Emplace((UTCServerTime - UTCNow).GetTotalSeconds());
}
}
}
else
{
FString RateLimitReset = Response->GetHeader(TEXT("X-Rate-Limit-Reset"));
if (!RateLimitReset.IsEmpty())
{
// UTC seconds
const FDateTime UTCServerTime = FDateTime::FromUnixTimestamp(FCString::Atoi64(*RateLimitReset));
const FDateTime UTCNow = FDateTime::UtcNow();
LockoutPeriod.Emplace((UTCServerTime - UTCNow).GetTotalSeconds());
}
}
}
return LockoutPeriod;
}
bool FHttpRetrySystem::FExponentialBackoffCurve::IsValid() const
{
return Base > 1.0f
&& ExponentBias >= 0.0f
&& MinCoefficient <= MaxCoefficient
&& MaxCoefficient > 0.001f
&& MinCoefficient >= 0.0f;
}
float FHttpRetrySystem::FExponentialBackoffCurve::Compute(uint32 RetryNumber) const
{
float BackOff = FMath::Pow(Base, static_cast<float>(RetryNumber) + ExponentBias);
const float Coefficient = IsValid() ? FMath::RandRange(MinCoefficient, MaxCoefficient) : 1.0f;
return FMath::Min(BackOff * Coefficient, MaxBackoffSeconds);
}
}
FHttpRetrySystem::FRequest::FRequest(
TSharedRef<FManager> InManager,
const TSharedRef<IHttpRequest, ESPMode::ThreadSafe>& HttpRequest,
const FHttpRetrySystem::FRetryLimitCountSetting& InRetryLimitCountOverride,
const FHttpRetrySystem::FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsOverride,
const FHttpRetrySystem::FRetryResponseCodes& InRetryResponseCodes,
const FHttpRetrySystem::FRetryVerbs& InRetryVerbs,
const FHttpRetrySystem::FRetryDomainsPtr& InRetryDomains,
const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorOverride,
const FExponentialBackoffCurve& InExponentialBackoffCurve
)
: FHttpRequestAdapterBase(HttpRequest)
, RetryStatus(FHttpRetrySystem::FRequest::EStatus::NotStarted)
, RetryLimitCountOverride(InRetryLimitCountOverride)
, RetryLimitCountForConnectionErrorOverride(InRetryLimitCountForConnectionErrorOverride)
, RetryTimeoutRelativeSecondsOverride(InRetryTimeoutRelativeSecondsOverride)
, RetryResponseCodes(InRetryResponseCodes)
, RetryVerbs(InRetryVerbs)
, RetryDomains(InRetryDomains)
, RetryManager(InManager)
, RetryExponentialBackoffCurve(InExponentialBackoffCurve)
{
// if the InRetryTimeoutRelativeSecondsOverride override is being used the value cannot be negative
check(!(InRetryTimeoutRelativeSecondsOverride.IsSet()) || (InRetryTimeoutRelativeSecondsOverride.GetValue() >= 0.0));
if (RetryDomains.IsValid())
{
if (RetryDomains->Domains.Num() == 0)
{
// If there are no domains to cycle through, go through the simpler path
RetryDomains.Reset();
}
else
{
// Start with the active index
RetryDomainsIndex = RetryDomains->ActiveIndex;
check(RetryDomains->Domains.IsValidIndex(RetryDomainsIndex));
}
}
}
void FHttpRetrySystem::FRequest::BindAdaptorDelegates()
{
if (!bBoundAdaptorDelegates)
{
bBoundAdaptorDelegates = true;
// Can't BindRaw&Unbind from ctor&dtor because with thread-safe delegate, it can cause issue when delete this request during complete callback, then unbind the callback
HttpRequest->OnProcessRequestComplete().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnProcessRequestComplete);
HttpRequest->OnRequestProgress64().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnRequestProgress);
HttpRequest->OnStatusCodeReceived().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnStatusCodeReceived);
HttpRequest->OnHeaderReceived().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnHeaderReceived);
}
}
bool FHttpRetrySystem::FRequest::ProcessRequest()
{
TSharedRef<FRequest> RetryRequest = StaticCastSharedRef<FRequest>(AsShared());
OriginalUrl = HttpRequest->GetURL();
if (RetryDomains.IsValid() && !RetryDomains->Domains.IsEmpty())
{
FString OriginalUrlDomainAndPort = FPlatformHttp::GetUrlDomainAndPort(OriginalUrl);
int32 Index = RetryDomains->Domains.Find(OriginalUrlDomainAndPort);
if (Index == INDEX_NONE)
{
RetryDomains->Domains.Insert(MoveTemp(OriginalUrlDomainAndPort), 0);
}
else if (Index > 0)
{
RetryDomains->Domains.RemoveAt(Index);
RetryDomains->Domains.Insert(MoveTemp(OriginalUrlDomainAndPort), 0);
}
}
// The ActiveIndex inside FRetryDomains could have been increased before, so to skip the domains failed to connect
if (RetryDomains.IsValid())
{
SetUrlFromRetryDomains();
}
BindAdaptorDelegates();
TSharedPtr<FManager> RetryManagerPtr = RetryManager.Pin();
if (ensure(RetryManagerPtr))
{
return RetryManagerPtr->ProcessRequest(RetryRequest);
}
else
{
return false;
}
}
void FHttpRetrySystem::FRequest::SetUrlFromRetryDomains()
{
check(RetryDomains.IsValid());
FString OriginalUrlDomainAndPort = FPlatformHttp::GetUrlDomainAndPort(OriginalUrl);
if (!OriginalUrlDomainAndPort.IsEmpty())
{
const FString Url(OriginalUrl.Replace(*OriginalUrlDomainAndPort, *RetryDomains->Domains[RetryDomainsIndex]));
HttpRequest->SetURL(Url);
}
}
void FHttpRetrySystem::FRequest::MoveToNextRetryDomain()
{
check(RetryDomains.IsValid());
const int32 NextDomainIndex = (RetryDomainsIndex + 1) % RetryDomains->Domains.Num();
if (RetryDomains->ActiveIndex.CompareExchange(RetryDomainsIndex, NextDomainIndex))
{
RetryDomainsIndex = NextDomainIndex;
}
SetUrlFromRetryDomains();
}
void FHttpRetrySystem::FRequest::CancelRequest()
{
TSharedRef<FRequest, ESPMode::ThreadSafe> RetryRequest = StaticCastSharedRef<FRequest>(AsShared());
BindAdaptorDelegates();
if (TSharedPtr<FManager> RetryManagerPtr = RetryManager.Pin())
{
RetryManagerPtr->CancelRequest(RetryRequest);
}
else
{
HttpRequest->CancelRequest();
}
}
void FHttpRetrySystem::FRequest::HttpOnRequestProgress(FHttpRequestPtr InHttpRequest, uint64 BytesSent, uint64 BytesRcv)
{
OnRequestProgress64().ExecuteIfBound(AsShared(), BytesSent, BytesRcv);
}
void FHttpRetrySystem::FRequest::HttpOnProcessRequestComplete(FHttpRequestPtr InHttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded)
{
TSharedPtr<FManager> RetryManagerPtr = RetryManager.Pin();
if (!RetryManagerPtr)
{
return;
}
TSharedRef<FRequest> SelfPtr = StaticCastSharedRef<FRequest>(AsShared()); // In case no ref after removing from RetryManager
{
FScopeLock ScopeLock(&RetryManagerPtr->RequestListLock);
uint32 EntryIndex = RetryManagerPtr->RequestList.IndexOfByPredicate([this](const FManager::FHttpRetryRequestEntry& Entry) { return Entry.Request == AsShared(); });
if (ensure(EntryIndex != INDEX_NONE))
{
FManager::FHttpRetryRequestEntry& HttpRetryRequestEntry = RetryManagerPtr->RequestList[EntryIndex];
if (RetryStatus == FHttpRetrySystem::FRequest::EStatus::Cancelled)
{
// Do nothing here
}
else if (GetStatus() == EHttpRequestStatus::Failed)
{
if (GetFailureReason() == EHttpFailureReason::ConnectionError && RetryDomains.IsValid())
{
MoveToNextRetryDomain();
}
if (GetFailureReason() == EHttpFailureReason::TimedOut)
{
RetryStatus = FHttpRetrySystem::FRequest::EStatus::FailedTimeout;
}
else
{
RetryStatus = FHttpRetrySystem::FRequest::EStatus::FailedRetry;
}
}
else
{
RetryStatus = FHttpRetrySystem::FRequest::EStatus::Succeeded;
}
if (RetryStatus != FHttpRetrySystem::FRequest::EStatus::Cancelled &&
RetryStatus != FHttpRetrySystem::FRequest::EStatus::FailedTimeout &&
RetryManagerPtr->ShouldRetry(HttpRetryRequestEntry) &&
RetryManagerPtr->CanRetry(HttpRetryRequestEntry))
{
const double NowAbsoluteSeconds = FPlatformTime::Seconds();
float LockoutPeriod = RetryManagerPtr->GetLockoutPeriodSeconds(HttpRetryRequestEntry);
RetryStatus = FHttpRetrySystem::FRequest::EStatus::ProcessingLockout;
{
QUICK_SCOPE_CYCLE_COUNTER(STAT_FHttpRetrySystem_FManager_Update_OnRequestWillRetry);
OnRequestWillRetry().ExecuteIfBound(HttpRetryRequestEntry.Request, GetResponse(), LockoutPeriod);
}
RetryManagerPtr->RetryHttpRequestWithDelay(HttpRetryRequestEntry, LockoutPeriod, bSucceeded);
return;
}
if (HttpRetryRequestEntry.CurrentRetryCount > 0)
{
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::Get().DecrementRetriedRequests();
}
RetryManagerPtr->RequestList.RemoveAtSwap(EntryIndex);
}
}
FHttpResponsePtr ResultResponse = HttpResponse;
bool bResultSucceeded = bSucceeded;
if (RetryStatus == FHttpRetrySystem::FRequest::EStatus::FailedTimeout && LastResponse != nullptr)
{
// Last response is better than nothing when it's timeout
ResultResponse = LastResponse;
LastResponse.Reset();
bResultSucceeded = bLastSucceeded;
}
LLM_SCOPE_BYTAG(HTTP);
OnProcessRequestComplete().ExecuteIfBound(SelfPtr, ResultResponse, bResultSucceeded);
ClearTimeout();
}
void FHttpRetrySystem::FRequest::HttpOnStatusCodeReceived(FHttpRequestPtr Request, int32 StatusCode)
{
TSharedRef<FRequest> SelfPtr = StaticCastSharedRef<FRequest>(AsShared());
OnStatusCodeReceived().ExecuteIfBound(SelfPtr, StatusCode);
}
void FHttpRetrySystem::FRequest::HttpOnHeaderReceived(FHttpRequestPtr Request, const FString& HeaderName, const FString& NewHeaderValue)
{
TSharedRef<FRequest> SelfPtr = StaticCastSharedRef<FRequest>(AsShared());
OnHeaderReceived().ExecuteIfBound(SelfPtr, HeaderName, NewHeaderValue);
}
FHttpRetrySystem::FManager::FManager(
const FRetryLimitCountSetting& InRetryLimitCountDefault,
const FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsDefault,
const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorDefault
)
: RandomFailureRate(FRandomFailureRateSetting())
, RetryLimitCountDefault(InRetryLimitCountDefault)
, RetryLimitCountForConnectionErrorDefault(InRetryLimitCountForConnectionErrorDefault)
, RetryTimeoutRelativeSecondsDefault(InRetryTimeoutRelativeSecondsDefault)
{
check(FHttpModule::Get().GetHttpManager().GetThread());
}
FHttpRetrySystem::FManager::~FManager()
{
FScopeLock ScopeLock(&RequestListLock);
// Decrement retried request for log verbosity tracker
for (const FHttpRetryRequestEntry& Request : RequestList)
{
if (Request.CurrentRetryCount > 0)
{
FHttpLogVerbosityTracker::Get().DecrementRetriedRequests();
}
}
}
TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe> FHttpRetrySystem::FManager::CreateRequest(
const FRetryLimitCountSetting& InRetryLimitCountOverride,
const FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsOverride,
const FRetryResponseCodes& InRetryResponseCodes,
const FRetryVerbs& InRetryVerbs,
const FRetryDomainsPtr& InRetryDomains,
const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorOverride,
const FExponentialBackoffCurve& InExponentialBackoffCurve
)
{
return MakeShareable(new FRequest(
AsShared(),
FHttpModule::Get().CreateRequest(),
InRetryLimitCountOverride,
InRetryTimeoutRelativeSecondsOverride,
InRetryResponseCodes,
InRetryVerbs,
InRetryDomains,
InRetryLimitCountForConnectionErrorOverride,
InExponentialBackoffCurve
));
}
bool FHttpRetrySystem::FManager::ShouldRetry(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
{
FHttpResponsePtr Response = HttpRetryRequestEntry.Request->GetResponse();
if (Response)
{
return HttpRetryRequestEntry.Request->RetryResponseCodes.Contains(Response->GetResponseCode());
}
// ONLY continue to check retry if no response. If there is any response, it means at least the http
// connection was established, we shouldn't attempt to retry. Otherwise request may be sent (and
// processed) twice
// Safe check
if (HttpRetryRequestEntry.Request->GetStatus() != EHttpRequestStatus::Failed)
{
// This shouldn't happen when response is null, but just in case
return false;
}
// Should retry if couldn't connect at all
if (HttpRetryRequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError)
{
return true;
}
// Should retry for idempotent verbs if there is network error
const FName Verb = FName(*HttpRetryRequestEntry.Request->GetVerb());
if (!HttpRetryRequestEntry.Request->RetryVerbs.IsEmpty())
{
return HttpRetryRequestEntry.Request->RetryVerbs.Contains(Verb);
}
// Be default, we will also allow retry for GET and HEAD requests even if they may duplicate on the server
static const TSet<FName> DefaultRetryVerbs(TArray<FName>({ FName(TEXT("GET")), FName(TEXT("HEAD")) }));
return DefaultRetryVerbs.Contains(Verb);
}
bool FHttpRetrySystem::FManager::RetryLimitForConnectionErrorIsSet(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
{
return HttpRetryRequestEntry.Request->RetryLimitCountForConnectionErrorOverride.IsSet() || RetryLimitCountForConnectionErrorDefault.IsSet();
}
bool FHttpRetrySystem::FManager::CanRetryForConnectionError(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
{
uint32 RetryLimitForConnectionError = HttpRetryRequestEntry.Request->RetryLimitCountForConnectionErrorOverride.Get(RetryLimitCountForConnectionErrorDefault.Get(0));
return HttpRetryRequestEntry.CurrentRetryCountForConnectionError < RetryLimitForConnectionError;
}
bool FHttpRetrySystem::FManager::CanRetryInGeneral(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
{
uint32 RetryLimit = HttpRetryRequestEntry.Request->RetryLimitCountOverride.Get(RetryLimitCountDefault.Get(0));
return HttpRetryRequestEntry.CurrentRetryCount < RetryLimit;
}
bool FHttpRetrySystem::FManager::CanRetry(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
{
if (HttpRetryRequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError && RetryLimitForConnectionErrorIsSet(HttpRetryRequestEntry))
{
return CanRetryForConnectionError(HttpRetryRequestEntry);
}
return CanRetryInGeneral(HttpRetryRequestEntry);
}
bool FHttpRetrySystem::FManager::HasTimedOut(const FHttpRetryRequestEntry& HttpRetryRequestEntry, const double NowAbsoluteSeconds)
{
bool bResult = false;
bool bShouldTestRetryTimeout = false;
double RetryTimeoutAbsoluteSeconds = HttpRetryRequestEntry.RequestStartTimeAbsoluteSeconds;
if (HttpRetryRequestEntry.Request->RetryTimeoutRelativeSecondsOverride.IsSet())
{
bShouldTestRetryTimeout = true;
RetryTimeoutAbsoluteSeconds += HttpRetryRequestEntry.Request->RetryTimeoutRelativeSecondsOverride.GetValue();
}
else if (RetryTimeoutRelativeSecondsDefault.IsSet())
{
bShouldTestRetryTimeout = true;
RetryTimeoutAbsoluteSeconds += RetryTimeoutRelativeSecondsDefault.GetValue();
}
if (bShouldTestRetryTimeout)
{
if (NowAbsoluteSeconds >= RetryTimeoutAbsoluteSeconds)
{
bResult = true;
}
}
return bResult;
}
void FHttpRetrySystem::FManager::RetryHttpRequest(FHttpRetryRequestEntry& RequestEntry)
{
// if this fails the HttpRequest's state will be failed which will cause the retry logic to kick(as expected)
if (RequestEntry.CurrentRetryCount == 0)
{
FHttpLogVerbosityTracker::Get().IncrementRetriedRequests();
}
++RequestEntry.CurrentRetryCount;
if (RequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError)
{
++RequestEntry.CurrentRetryCountForConnectionError;
}
RequestEntry.Request->RetryStatus = FRequest::EStatus::Processing;
if (const FHttpResponsePtr Response = RequestEntry.Request->GetResponse())
{
if (const int32 ResponseCode = Response->GetResponseCode(); ResponseCode < 400)
{
// 1XX, 2XX, and 3XX are non error responses, regular log level
UE_LOG(LogHttp, Log, TEXT("Retry %d on %s with response %d"),
RequestEntry.CurrentRetryCount,
*(RequestEntry.Request->GetURL()),
ResponseCode);
}
else
{
// 4XX, 5XX are error responses, warning log level
UE_LOG(LogHttp, Warning, TEXT("Retry %d on %s with response %d"),
RequestEntry.CurrentRetryCount,
*(RequestEntry.Request->GetURL()),
ResponseCode);
}
}
else
{
// We don't know the response code, default to warning log level
UE_LOG(LogHttp, Warning, TEXT("Retry %d on %s"), RequestEntry.CurrentRetryCount, *(RequestEntry.Request->GetURL()));
}
RequestEntry.Request->HttpRequest->ProcessRequest();
}
void FHttpRetrySystem::FManager::RetryHttpRequestWithDelay(FManager::FHttpRetryRequestEntry& RequestEntry, float InDelay, bool bWasSucceeded)
{
// Timeout during lock out to keep existing behavior
float TimeoutOrDefault = RequestEntry.Request->GetTimeout().Get(FHttpModule::Get().GetHttpTotalTimeout());
if (TimeoutOrDefault != 0)
{
float TimeElapsedForTheRequest = FPlatformTime::Seconds() - RequestEntry.RequestStartTimeAbsoluteSeconds;
float WillTimeoutInDelay = TimeoutOrDefault - TimeElapsedForTheRequest;
if (WillTimeoutInDelay < InDelay)
{
HttpRequestTimeoutAfterDelay(RequestEntry, bWasSucceeded, WillTimeoutInDelay);
return;
}
}
// Delay and start
TWeakPtr<FRequest> RequestWeakPtr(RequestEntry.Request);
FHttpModule::Get().GetHttpManager().AddHttpThreadTask([RequestWeakPtr, bWasSucceeded]() {
if (TSharedPtr<FRequest> RequestPtr = RequestWeakPtr.Pin())
{
if (TSharedPtr<FManager> RetryManagerPtr = RequestPtr->RetryManager.Pin())
{
FScopeLock ScopeLock(&RetryManagerPtr->RequestListLock);
// Check if it's still there in case it has been cancelled during the delay period
uint32 EntryIndex = RetryManagerPtr->RequestList.IndexOfByPredicate([RequestPtr](const FManager::FHttpRetryRequestEntry& Entry) { return Entry.Request == RequestPtr; });
if (EntryIndex != INDEX_NONE)
{
FManager::FHttpRetryRequestEntry* HttpRetryRequestEntry = &RetryManagerPtr->RequestList[EntryIndex];
// TODO: Move this into RetryHttpRequest after stabilizing the flow with CVarHttpRetrySystemNonGameThreadSupportEnabled on
HttpRetryRequestEntry->Request->LastResponse = HttpRetryRequestEntry->Request->GetResponse();
HttpRetryRequestEntry->Request->bLastSucceeded = bWasSucceeded;
RetryManagerPtr->RetryHttpRequest(*HttpRetryRequestEntry);
}
}
}
}, InDelay);
}
void FHttpRetrySystem::FManager::HttpRequestTimeoutAfterDelay(FManager::FHttpRetryRequestEntry& RequestEntry, bool bWasSucceeded, float Delay)
{
TWeakPtr<FRequest> RequestWeakPtr(RequestEntry.Request);
TFunction<void()> Callback([RequestWeakPtr, bWasSucceeded]() {
if (TSharedPtr<FRequest> RequestPtr = RequestWeakPtr.Pin())
{
if (TSharedPtr<FManager> RetryManagerPtr = RequestPtr->RetryManager.Pin())
{
FScopeLock ScopeLock(&RetryManagerPtr->RequestListLock);
uint32 EntryIndex = RetryManagerPtr->RequestList.IndexOfByPredicate([RequestPtr](const FManager::FHttpRetryRequestEntry& Entry) { return Entry.Request == RequestPtr; });
if (EntryIndex != INDEX_NONE)
{
FManager::FHttpRetryRequestEntry* HttpRetryRequestEntry = &RetryManagerPtr->RequestList[EntryIndex];
if (HttpRetryRequestEntry->CurrentRetryCount > 0)
{
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::Get().DecrementRetriedRequests();
}
RetryManagerPtr->RequestList.RemoveAtSwap(EntryIndex);
}
}
// Same as existing behavior, when timeout during lock out period, it fails with result of last request before lockout
RequestPtr->OnProcessRequestComplete().ExecuteIfBound(RequestPtr, RequestPtr->GetResponse(), bWasSucceeded);
}
});
if (RequestEntry.Request->GetDelegateThreadPolicy() == EHttpRequestDelegateThreadPolicy::CompleteOnGameThread)
{
FHttpModule::Get().GetHttpManager().AddGameThreadTask(MoveTemp(Callback), Delay);
}
else
{
FHttpModule::Get().GetHttpManager().AddHttpThreadTask(MoveTemp(Callback), Delay);
}
}
float FHttpRetrySystem::FManager::GetLockoutPeriodSeconds(const FHttpRetryRequestEntry& HttpRetryRequestEntry)
{
float LockoutPeriod = 0.0f;
TOptional<double> ResponseLockoutPeriod = FHttpRetrySystem::ReadThrottledTimeFromResponseInSeconds(HttpRetryRequestEntry.Request->GetResponse());
if (ResponseLockoutPeriod.IsSet())
{
LockoutPeriod = static_cast<float>(ResponseLockoutPeriod.GetValue());
}
if (LockoutPeriod <= 0.0f)
{
const bool bFailedToConnect = (HttpRetryRequestEntry.Request->GetStatus() == EHttpRequestStatus::Failed && HttpRetryRequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError);
const bool bHasRetryDomains = HttpRetryRequestEntry.Request->RetryDomains.IsValid();
// Skip the lockout period if we failed to connect to a domain and we have other domains to try
if (bFailedToConnect && bHasRetryDomains)
{
return 0.0f;
}
// The first time through this function, the CurrentRetryCount is 0, the second time it's 1, etc. We automatically add 1 to make the input into the backoff function line up with expectations.
LockoutPeriod = HttpRetryRequestEntry.Request->RetryExponentialBackoffCurve.Compute(HttpRetryRequestEntry.CurrentRetryCount+1);
}
return LockoutPeriod;
}
static FRandomStream TempRandomStream(4435261);
FHttpRetrySystem::FManager::FHttpRetryRequestEntry::FHttpRetryRequestEntry(TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe>& InRequest)
: bShouldCancel(false)
, CurrentRetryCount(0)
, CurrentRetryCountForConnectionError(0)
, RequestStartTimeAbsoluteSeconds(FPlatformTime::Seconds())
, Request(InRequest)
{}
bool FHttpRetrySystem::FManager::ProcessRequest(TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe>& HttpRetryRequest)
{
QUICK_SCOPE_CYCLE_COUNTER(STAT_FHttpRetrySystem_FManager_ProcessRequest);
// Let the request trigger timeout by itself, instead of ticking it in retry system
if (HttpRetryRequest->RetryTimeoutRelativeSecondsOverride.IsSet())
{
HttpRetryRequest->SetTimeout(HttpRetryRequest->RetryTimeoutRelativeSecondsOverride.GetValue());
}
else if (RetryTimeoutRelativeSecondsDefault.IsSet())
{
HttpRetryRequest->SetTimeout(RetryTimeoutRelativeSecondsDefault.GetValue());
}
FScopeLock ScopeLock(&RequestListLock);
RequestList.Add(FHttpRetryRequestEntry(HttpRetryRequest));
HttpRetryRequest->RetryStatus = FHttpRetrySystem::FRequest::EStatus::Processing;
HttpRetryRequest->HttpRequest->ProcessRequest();
return true;
}
void FHttpRetrySystem::FManager::CancelRequest(TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe>& HttpRetryRequest)
{
UE_AUTORTFM_ONCOMMIT(this, HttpRetryRequest)
{
QUICK_SCOPE_CYCLE_COUNTER(STAT_FHttpRetrySystem_FManager_CancelRequest);
FScopeLock ScopeLock(&RequestListLock);
// Find the existing request entry if is was previously processed.
bool bFound = false;
for (int32 i = 0; i < RequestList.Num(); ++i)
{
FHttpRetryRequestEntry& EntryRef = RequestList[i];
if (EntryRef.Request == HttpRetryRequest)
{
EntryRef.bShouldCancel = true;
bFound = true;
}
}
// If we did not find the entry, likely auth failed for the request, in which case ProcessRequest does not get called.
// Adding it to the list and flagging as cancel will process it on next tick.
if (!bFound)
{
FHttpRetryRequestEntry RetryRequestEntry(HttpRetryRequest);
RetryRequestEntry.bShouldCancel = true;
RequestList.Add(RetryRequestEntry);
}
HttpRetryRequest->HttpRequest->CancelRequest();
HttpRetryRequest->RetryStatus = FHttpRetrySystem::FRequest::EStatus::Cancelled;
};
}
/* This should only be used when shutting down or suspending, to make sure
all pending HTTP requests are flushed to the network */
void FHttpRetrySystem::FManager::BlockUntilFlushed(float InTimeoutSec)
{
const float SleepInterval = 0.016;
float TimeElapsed = 0.0f;
while (TimeElapsed < InTimeoutSec)
{
{
FScopeLock ScopeLock(&RequestListLock);
if (RequestList.IsEmpty())
{
break;
}
}
FHttpModule::Get().GetHttpManager().Tick(SleepInterval);
FPlatformProcess::Sleep(SleepInterval);
TimeElapsed += SleepInterval;
}
}
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker& FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::Get()
{
static FHttpLogVerbosityTracker Tracker;
return Tracker;
}
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::FHttpLogVerbosityTracker()
{
UpdateSettingsFromConfig();
FCoreDelegates::TSOnConfigSectionsChanged().AddRaw(this, &FHttpLogVerbosityTracker::OnConfigSectionsChanged);
}
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::~FHttpLogVerbosityTracker()
{
FCoreDelegates::TSOnConfigSectionsChanged().RemoveAll(this);
}
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::IncrementRetriedRequests()
{
FScopeLock ScopeLock(&NumRetriedRequestsLock);
++NumRetriedRequests;
if (NumRetriedRequests == 1)
{
OriginalVerbosity = UE_GET_LOG_VERBOSITY(LogHttp);
if (TargetVerbosity != ELogVerbosity::NoLogging)
{
UE_LOG(LogHttp, Warning, TEXT("HttpRetry: Increasing log verbosity from %s to %s due to requests being retried"), ToString(OriginalVerbosity), ToString(TargetVerbosity));
//UE_SET_LOG_VERBOSITY(LogHttp, TargetVerbosity); // Macro requires the value to be a ELogVerbosity constant
#if !NO_LOGGING
LogHttp.SetVerbosity(TargetVerbosity);
#endif
}
}
}
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::DecrementRetriedRequests()
{
FScopeLock ScopeLock(&NumRetriedRequestsLock);
--NumRetriedRequests;
check(NumRetriedRequests >= 0);
if (NumRetriedRequests == 0)
{
UE_LOG(LogHttp, Warning, TEXT("HttpRetry: Resetting log verbosity to %s due to requests being retried"), ToString(OriginalVerbosity));
//UE_SET_LOG_VERBOSITY(LogHttp, OriginalVerbosity); // Macro requires the value to be a ELogVerbosity constant
#if !NO_LOGGING
LogHttp.SetVerbosity(OriginalVerbosity);
#endif
}
}
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::UpdateSettingsFromConfig()
{
FString TargetVerbosityAsString;
if (GConfig->GetString(TEXT("HTTP.Retry"), TEXT("RetryManagerVerbosityLevel"), TargetVerbosityAsString, GEngineIni))
{
TargetVerbosity = ParseLogVerbosityFromString(TargetVerbosityAsString);
}
else
{
TargetVerbosity = ELogVerbosity::NoLogging;
}
}
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::OnConfigSectionsChanged(const FString& IniFilename, const TSet<FString>& SectionName)
{
if (IniFilename == GEngineIni && SectionName.Contains(TEXT("HTTP.Retry")))
{
UpdateSettingsFromConfig();
}
}