768 lines
26 KiB
C++
768 lines
26 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "HttpRetrySystem.h"
|
|
#include "AutoRTFM.h"
|
|
#include "GenericPlatform/HttpRequestCommon.h"
|
|
#include "HAL/ConsoleManager.h"
|
|
#include "HAL/PlatformTime.h"
|
|
#include "HAL/PlatformProcess.h"
|
|
#include "HAL/LowLevelMemTracker.h"
|
|
#include "Math/RandomStream.h"
|
|
#include "HttpModule.h"
|
|
#include "Http.h"
|
|
#include "HttpManager.h"
|
|
#include "HttpThread.h"
|
|
#include "Stats/Stats.h"
|
|
#include "Misc/ConfigCacheIni.h"
|
|
#include "Misc/CoreDelegates.h"
|
|
|
|
LLM_DEFINE_TAG(HTTP);
|
|
|
|
namespace FHttpRetrySystem
|
|
{
|
|
|
|
TOptional<double> ReadThrottledTimeFromResponseInSeconds(FHttpResponsePtr Response)
|
|
{
|
|
TOptional<double> LockoutPeriod;
|
|
// Check if there was a Retry-After header
|
|
if (Response.IsValid())
|
|
{
|
|
FString RetryAfter = Response->GetHeader(TEXT("Retry-After"));
|
|
if (!RetryAfter.IsEmpty())
|
|
{
|
|
if (RetryAfter.IsNumeric())
|
|
{
|
|
// seconds
|
|
LockoutPeriod.Emplace(FCString::Atof(*RetryAfter));
|
|
}
|
|
else
|
|
{
|
|
// http date
|
|
FDateTime UTCServerTime;
|
|
if (FDateTime::ParseHttpDate(RetryAfter, UTCServerTime))
|
|
{
|
|
const FDateTime UTCNow = FDateTime::UtcNow();
|
|
LockoutPeriod.Emplace((UTCServerTime - UTCNow).GetTotalSeconds());
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
FString RateLimitReset = Response->GetHeader(TEXT("X-Rate-Limit-Reset"));
|
|
if (!RateLimitReset.IsEmpty())
|
|
{
|
|
// UTC seconds
|
|
const FDateTime UTCServerTime = FDateTime::FromUnixTimestamp(FCString::Atoi64(*RateLimitReset));
|
|
const FDateTime UTCNow = FDateTime::UtcNow();
|
|
LockoutPeriod.Emplace((UTCServerTime - UTCNow).GetTotalSeconds());
|
|
}
|
|
}
|
|
}
|
|
return LockoutPeriod;
|
|
}
|
|
|
|
bool FHttpRetrySystem::FExponentialBackoffCurve::IsValid() const
|
|
{
|
|
return Base > 1.0f
|
|
&& ExponentBias >= 0.0f
|
|
&& MinCoefficient <= MaxCoefficient
|
|
&& MaxCoefficient > 0.001f
|
|
&& MinCoefficient >= 0.0f;
|
|
}
|
|
|
|
float FHttpRetrySystem::FExponentialBackoffCurve::Compute(uint32 RetryNumber) const
|
|
{
|
|
float BackOff = FMath::Pow(Base, static_cast<float>(RetryNumber) + ExponentBias);
|
|
const float Coefficient = IsValid() ? FMath::RandRange(MinCoefficient, MaxCoefficient) : 1.0f;
|
|
return FMath::Min(BackOff * Coefficient, MaxBackoffSeconds);
|
|
}
|
|
|
|
}
|
|
|
|
FHttpRetrySystem::FRequest::FRequest(
|
|
TSharedRef<FManager> InManager,
|
|
const TSharedRef<IHttpRequest, ESPMode::ThreadSafe>& HttpRequest,
|
|
const FHttpRetrySystem::FRetryLimitCountSetting& InRetryLimitCountOverride,
|
|
const FHttpRetrySystem::FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsOverride,
|
|
const FHttpRetrySystem::FRetryResponseCodes& InRetryResponseCodes,
|
|
const FHttpRetrySystem::FRetryVerbs& InRetryVerbs,
|
|
const FHttpRetrySystem::FRetryDomainsPtr& InRetryDomains,
|
|
const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorOverride,
|
|
const FExponentialBackoffCurve& InExponentialBackoffCurve
|
|
)
|
|
: FHttpRequestAdapterBase(HttpRequest)
|
|
, RetryStatus(FHttpRetrySystem::FRequest::EStatus::NotStarted)
|
|
, RetryLimitCountOverride(InRetryLimitCountOverride)
|
|
, RetryLimitCountForConnectionErrorOverride(InRetryLimitCountForConnectionErrorOverride)
|
|
, RetryTimeoutRelativeSecondsOverride(InRetryTimeoutRelativeSecondsOverride)
|
|
, RetryResponseCodes(InRetryResponseCodes)
|
|
, RetryVerbs(InRetryVerbs)
|
|
, RetryDomains(InRetryDomains)
|
|
, RetryManager(InManager)
|
|
, RetryExponentialBackoffCurve(InExponentialBackoffCurve)
|
|
{
|
|
// if the InRetryTimeoutRelativeSecondsOverride override is being used the value cannot be negative
|
|
check(!(InRetryTimeoutRelativeSecondsOverride.IsSet()) || (InRetryTimeoutRelativeSecondsOverride.GetValue() >= 0.0));
|
|
|
|
if (RetryDomains.IsValid())
|
|
{
|
|
if (RetryDomains->Domains.Num() == 0)
|
|
{
|
|
// If there are no domains to cycle through, go through the simpler path
|
|
RetryDomains.Reset();
|
|
}
|
|
else
|
|
{
|
|
// Start with the active index
|
|
RetryDomainsIndex = RetryDomains->ActiveIndex;
|
|
check(RetryDomains->Domains.IsValidIndex(RetryDomainsIndex));
|
|
}
|
|
}
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::BindAdaptorDelegates()
|
|
{
|
|
if (!bBoundAdaptorDelegates)
|
|
{
|
|
bBoundAdaptorDelegates = true;
|
|
|
|
// Can't BindRaw&Unbind from ctor&dtor because with thread-safe delegate, it can cause issue when delete this request during complete callback, then unbind the callback
|
|
HttpRequest->OnProcessRequestComplete().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnProcessRequestComplete);
|
|
HttpRequest->OnRequestProgress64().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnRequestProgress);
|
|
HttpRequest->OnStatusCodeReceived().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnStatusCodeReceived);
|
|
HttpRequest->OnHeaderReceived().BindThreadSafeSP(this, &FHttpRetrySystem::FRequest::HttpOnHeaderReceived);
|
|
}
|
|
}
|
|
|
|
bool FHttpRetrySystem::FRequest::ProcessRequest()
|
|
{
|
|
TSharedRef<FRequest> RetryRequest = StaticCastSharedRef<FRequest>(AsShared());
|
|
|
|
OriginalUrl = HttpRequest->GetURL();
|
|
|
|
if (RetryDomains.IsValid() && !RetryDomains->Domains.IsEmpty())
|
|
{
|
|
FString OriginalUrlDomainAndPort = FPlatformHttp::GetUrlDomainAndPort(OriginalUrl);
|
|
int32 Index = RetryDomains->Domains.Find(OriginalUrlDomainAndPort);
|
|
if (Index == INDEX_NONE)
|
|
{
|
|
RetryDomains->Domains.Insert(MoveTemp(OriginalUrlDomainAndPort), 0);
|
|
}
|
|
else if (Index > 0)
|
|
{
|
|
RetryDomains->Domains.RemoveAt(Index);
|
|
RetryDomains->Domains.Insert(MoveTemp(OriginalUrlDomainAndPort), 0);
|
|
}
|
|
}
|
|
|
|
// The ActiveIndex inside FRetryDomains could have been increased before, so to skip the domains failed to connect
|
|
if (RetryDomains.IsValid())
|
|
{
|
|
SetUrlFromRetryDomains();
|
|
}
|
|
|
|
BindAdaptorDelegates();
|
|
|
|
TSharedPtr<FManager> RetryManagerPtr = RetryManager.Pin();
|
|
|
|
if (ensure(RetryManagerPtr))
|
|
{
|
|
return RetryManagerPtr->ProcessRequest(RetryRequest);
|
|
}
|
|
else
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::SetUrlFromRetryDomains()
|
|
{
|
|
check(RetryDomains.IsValid());
|
|
FString OriginalUrlDomainAndPort = FPlatformHttp::GetUrlDomainAndPort(OriginalUrl);
|
|
if (!OriginalUrlDomainAndPort.IsEmpty())
|
|
{
|
|
const FString Url(OriginalUrl.Replace(*OriginalUrlDomainAndPort, *RetryDomains->Domains[RetryDomainsIndex]));
|
|
HttpRequest->SetURL(Url);
|
|
}
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::MoveToNextRetryDomain()
|
|
{
|
|
check(RetryDomains.IsValid());
|
|
const int32 NextDomainIndex = (RetryDomainsIndex + 1) % RetryDomains->Domains.Num();
|
|
if (RetryDomains->ActiveIndex.CompareExchange(RetryDomainsIndex, NextDomainIndex))
|
|
{
|
|
RetryDomainsIndex = NextDomainIndex;
|
|
}
|
|
SetUrlFromRetryDomains();
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::CancelRequest()
|
|
{
|
|
TSharedRef<FRequest, ESPMode::ThreadSafe> RetryRequest = StaticCastSharedRef<FRequest>(AsShared());
|
|
|
|
BindAdaptorDelegates();
|
|
|
|
if (TSharedPtr<FManager> RetryManagerPtr = RetryManager.Pin())
|
|
{
|
|
RetryManagerPtr->CancelRequest(RetryRequest);
|
|
}
|
|
else
|
|
{
|
|
HttpRequest->CancelRequest();
|
|
}
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::HttpOnRequestProgress(FHttpRequestPtr InHttpRequest, uint64 BytesSent, uint64 BytesRcv)
|
|
{
|
|
OnRequestProgress64().ExecuteIfBound(AsShared(), BytesSent, BytesRcv);
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::HttpOnProcessRequestComplete(FHttpRequestPtr InHttpRequest, FHttpResponsePtr HttpResponse, bool bSucceeded)
|
|
{
|
|
TSharedPtr<FManager> RetryManagerPtr = RetryManager.Pin();
|
|
if (!RetryManagerPtr)
|
|
{
|
|
return;
|
|
}
|
|
|
|
TSharedRef<FRequest> SelfPtr = StaticCastSharedRef<FRequest>(AsShared()); // In case no ref after removing from RetryManager
|
|
|
|
|
|
{
|
|
FScopeLock ScopeLock(&RetryManagerPtr->RequestListLock);
|
|
|
|
uint32 EntryIndex = RetryManagerPtr->RequestList.IndexOfByPredicate([this](const FManager::FHttpRetryRequestEntry& Entry) { return Entry.Request == AsShared(); });
|
|
if (ensure(EntryIndex != INDEX_NONE))
|
|
{
|
|
FManager::FHttpRetryRequestEntry& HttpRetryRequestEntry = RetryManagerPtr->RequestList[EntryIndex];
|
|
|
|
if (RetryStatus == FHttpRetrySystem::FRequest::EStatus::Cancelled)
|
|
{
|
|
// Do nothing here
|
|
}
|
|
else if (GetStatus() == EHttpRequestStatus::Failed)
|
|
{
|
|
if (GetFailureReason() == EHttpFailureReason::ConnectionError && RetryDomains.IsValid())
|
|
{
|
|
MoveToNextRetryDomain();
|
|
}
|
|
|
|
if (GetFailureReason() == EHttpFailureReason::TimedOut)
|
|
{
|
|
RetryStatus = FHttpRetrySystem::FRequest::EStatus::FailedTimeout;
|
|
}
|
|
else
|
|
{
|
|
RetryStatus = FHttpRetrySystem::FRequest::EStatus::FailedRetry;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
RetryStatus = FHttpRetrySystem::FRequest::EStatus::Succeeded;
|
|
}
|
|
|
|
if (RetryStatus != FHttpRetrySystem::FRequest::EStatus::Cancelled &&
|
|
RetryStatus != FHttpRetrySystem::FRequest::EStatus::FailedTimeout &&
|
|
RetryManagerPtr->ShouldRetry(HttpRetryRequestEntry) &&
|
|
RetryManagerPtr->CanRetry(HttpRetryRequestEntry))
|
|
{
|
|
const double NowAbsoluteSeconds = FPlatformTime::Seconds();
|
|
float LockoutPeriod = RetryManagerPtr->GetLockoutPeriodSeconds(HttpRetryRequestEntry);
|
|
|
|
RetryStatus = FHttpRetrySystem::FRequest::EStatus::ProcessingLockout;
|
|
|
|
{
|
|
QUICK_SCOPE_CYCLE_COUNTER(STAT_FHttpRetrySystem_FManager_Update_OnRequestWillRetry);
|
|
OnRequestWillRetry().ExecuteIfBound(HttpRetryRequestEntry.Request, GetResponse(), LockoutPeriod);
|
|
}
|
|
|
|
RetryManagerPtr->RetryHttpRequestWithDelay(HttpRetryRequestEntry, LockoutPeriod, bSucceeded);
|
|
return;
|
|
}
|
|
|
|
if (HttpRetryRequestEntry.CurrentRetryCount > 0)
|
|
{
|
|
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::Get().DecrementRetriedRequests();
|
|
}
|
|
|
|
RetryManagerPtr->RequestList.RemoveAtSwap(EntryIndex);
|
|
}
|
|
}
|
|
|
|
FHttpResponsePtr ResultResponse = HttpResponse;
|
|
bool bResultSucceeded = bSucceeded;
|
|
if (RetryStatus == FHttpRetrySystem::FRequest::EStatus::FailedTimeout && LastResponse != nullptr)
|
|
{
|
|
// Last response is better than nothing when it's timeout
|
|
ResultResponse = LastResponse;
|
|
LastResponse.Reset();
|
|
|
|
bResultSucceeded = bLastSucceeded;
|
|
}
|
|
|
|
LLM_SCOPE_BYTAG(HTTP);
|
|
OnProcessRequestComplete().ExecuteIfBound(SelfPtr, ResultResponse, bResultSucceeded);
|
|
|
|
ClearTimeout();
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::HttpOnStatusCodeReceived(FHttpRequestPtr Request, int32 StatusCode)
|
|
{
|
|
TSharedRef<FRequest> SelfPtr = StaticCastSharedRef<FRequest>(AsShared());
|
|
OnStatusCodeReceived().ExecuteIfBound(SelfPtr, StatusCode);
|
|
}
|
|
|
|
void FHttpRetrySystem::FRequest::HttpOnHeaderReceived(FHttpRequestPtr Request, const FString& HeaderName, const FString& NewHeaderValue)
|
|
{
|
|
TSharedRef<FRequest> SelfPtr = StaticCastSharedRef<FRequest>(AsShared());
|
|
OnHeaderReceived().ExecuteIfBound(SelfPtr, HeaderName, NewHeaderValue);
|
|
}
|
|
|
|
FHttpRetrySystem::FManager::FManager(
|
|
const FRetryLimitCountSetting& InRetryLimitCountDefault,
|
|
const FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsDefault,
|
|
const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorDefault
|
|
)
|
|
: RandomFailureRate(FRandomFailureRateSetting())
|
|
, RetryLimitCountDefault(InRetryLimitCountDefault)
|
|
, RetryLimitCountForConnectionErrorDefault(InRetryLimitCountForConnectionErrorDefault)
|
|
, RetryTimeoutRelativeSecondsDefault(InRetryTimeoutRelativeSecondsDefault)
|
|
{
|
|
check(FHttpModule::Get().GetHttpManager().GetThread());
|
|
}
|
|
|
|
FHttpRetrySystem::FManager::~FManager()
|
|
{
|
|
FScopeLock ScopeLock(&RequestListLock);
|
|
|
|
// Decrement retried request for log verbosity tracker
|
|
for (const FHttpRetryRequestEntry& Request : RequestList)
|
|
{
|
|
if (Request.CurrentRetryCount > 0)
|
|
{
|
|
FHttpLogVerbosityTracker::Get().DecrementRetriedRequests();
|
|
}
|
|
}
|
|
}
|
|
|
|
TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe> FHttpRetrySystem::FManager::CreateRequest(
|
|
const FRetryLimitCountSetting& InRetryLimitCountOverride,
|
|
const FRetryTimeoutRelativeSecondsSetting& InRetryTimeoutRelativeSecondsOverride,
|
|
const FRetryResponseCodes& InRetryResponseCodes,
|
|
const FRetryVerbs& InRetryVerbs,
|
|
const FRetryDomainsPtr& InRetryDomains,
|
|
const FRetryLimitCountSetting& InRetryLimitCountForConnectionErrorOverride,
|
|
const FExponentialBackoffCurve& InExponentialBackoffCurve
|
|
)
|
|
{
|
|
return MakeShareable(new FRequest(
|
|
AsShared(),
|
|
FHttpModule::Get().CreateRequest(),
|
|
InRetryLimitCountOverride,
|
|
InRetryTimeoutRelativeSecondsOverride,
|
|
InRetryResponseCodes,
|
|
InRetryVerbs,
|
|
InRetryDomains,
|
|
InRetryLimitCountForConnectionErrorOverride,
|
|
InExponentialBackoffCurve
|
|
));
|
|
}
|
|
|
|
bool FHttpRetrySystem::FManager::ShouldRetry(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
|
|
{
|
|
FHttpResponsePtr Response = HttpRetryRequestEntry.Request->GetResponse();
|
|
if (Response)
|
|
{
|
|
return HttpRetryRequestEntry.Request->RetryResponseCodes.Contains(Response->GetResponseCode());
|
|
}
|
|
|
|
// ONLY continue to check retry if no response. If there is any response, it means at least the http
|
|
// connection was established, we shouldn't attempt to retry. Otherwise request may be sent (and
|
|
// processed) twice
|
|
|
|
// Safe check
|
|
if (HttpRetryRequestEntry.Request->GetStatus() != EHttpRequestStatus::Failed)
|
|
{
|
|
// This shouldn't happen when response is null, but just in case
|
|
return false;
|
|
}
|
|
|
|
// Should retry if couldn't connect at all
|
|
if (HttpRetryRequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
// Should retry for idempotent verbs if there is network error
|
|
const FName Verb = FName(*HttpRetryRequestEntry.Request->GetVerb());
|
|
|
|
if (!HttpRetryRequestEntry.Request->RetryVerbs.IsEmpty())
|
|
{
|
|
return HttpRetryRequestEntry.Request->RetryVerbs.Contains(Verb);
|
|
}
|
|
|
|
// Be default, we will also allow retry for GET and HEAD requests even if they may duplicate on the server
|
|
static const TSet<FName> DefaultRetryVerbs(TArray<FName>({ FName(TEXT("GET")), FName(TEXT("HEAD")) }));
|
|
return DefaultRetryVerbs.Contains(Verb);
|
|
}
|
|
|
|
bool FHttpRetrySystem::FManager::RetryLimitForConnectionErrorIsSet(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
|
|
{
|
|
return HttpRetryRequestEntry.Request->RetryLimitCountForConnectionErrorOverride.IsSet() || RetryLimitCountForConnectionErrorDefault.IsSet();
|
|
}
|
|
|
|
bool FHttpRetrySystem::FManager::CanRetryForConnectionError(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
|
|
{
|
|
uint32 RetryLimitForConnectionError = HttpRetryRequestEntry.Request->RetryLimitCountForConnectionErrorOverride.Get(RetryLimitCountForConnectionErrorDefault.Get(0));
|
|
return HttpRetryRequestEntry.CurrentRetryCountForConnectionError < RetryLimitForConnectionError;
|
|
}
|
|
|
|
bool FHttpRetrySystem::FManager::CanRetryInGeneral(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
|
|
{
|
|
uint32 RetryLimit = HttpRetryRequestEntry.Request->RetryLimitCountOverride.Get(RetryLimitCountDefault.Get(0));
|
|
return HttpRetryRequestEntry.CurrentRetryCount < RetryLimit;
|
|
}
|
|
|
|
bool FHttpRetrySystem::FManager::CanRetry(const FHttpRetryRequestEntry& HttpRetryRequestEntry) const
|
|
{
|
|
if (HttpRetryRequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError && RetryLimitForConnectionErrorIsSet(HttpRetryRequestEntry))
|
|
{
|
|
return CanRetryForConnectionError(HttpRetryRequestEntry);
|
|
}
|
|
|
|
return CanRetryInGeneral(HttpRetryRequestEntry);
|
|
}
|
|
|
|
bool FHttpRetrySystem::FManager::HasTimedOut(const FHttpRetryRequestEntry& HttpRetryRequestEntry, const double NowAbsoluteSeconds)
|
|
{
|
|
bool bResult = false;
|
|
|
|
bool bShouldTestRetryTimeout = false;
|
|
double RetryTimeoutAbsoluteSeconds = HttpRetryRequestEntry.RequestStartTimeAbsoluteSeconds;
|
|
if (HttpRetryRequestEntry.Request->RetryTimeoutRelativeSecondsOverride.IsSet())
|
|
{
|
|
bShouldTestRetryTimeout = true;
|
|
RetryTimeoutAbsoluteSeconds += HttpRetryRequestEntry.Request->RetryTimeoutRelativeSecondsOverride.GetValue();
|
|
}
|
|
else if (RetryTimeoutRelativeSecondsDefault.IsSet())
|
|
{
|
|
bShouldTestRetryTimeout = true;
|
|
RetryTimeoutAbsoluteSeconds += RetryTimeoutRelativeSecondsDefault.GetValue();
|
|
}
|
|
|
|
if (bShouldTestRetryTimeout)
|
|
{
|
|
if (NowAbsoluteSeconds >= RetryTimeoutAbsoluteSeconds)
|
|
{
|
|
bResult = true;
|
|
}
|
|
}
|
|
|
|
return bResult;
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::RetryHttpRequest(FHttpRetryRequestEntry& RequestEntry)
|
|
{
|
|
// if this fails the HttpRequest's state will be failed which will cause the retry logic to kick(as expected)
|
|
if (RequestEntry.CurrentRetryCount == 0)
|
|
{
|
|
FHttpLogVerbosityTracker::Get().IncrementRetriedRequests();
|
|
}
|
|
++RequestEntry.CurrentRetryCount;
|
|
if (RequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError)
|
|
{
|
|
++RequestEntry.CurrentRetryCountForConnectionError;
|
|
}
|
|
RequestEntry.Request->RetryStatus = FRequest::EStatus::Processing;
|
|
|
|
if (const FHttpResponsePtr Response = RequestEntry.Request->GetResponse())
|
|
{
|
|
if (const int32 ResponseCode = Response->GetResponseCode(); ResponseCode < 400)
|
|
{
|
|
// 1XX, 2XX, and 3XX are non error responses, regular log level
|
|
UE_LOG(LogHttp, Log, TEXT("Retry %d on %s with response %d"),
|
|
RequestEntry.CurrentRetryCount,
|
|
*(RequestEntry.Request->GetURL()),
|
|
ResponseCode);
|
|
}
|
|
else
|
|
{
|
|
// 4XX, 5XX are error responses, warning log level
|
|
UE_LOG(LogHttp, Warning, TEXT("Retry %d on %s with response %d"),
|
|
RequestEntry.CurrentRetryCount,
|
|
*(RequestEntry.Request->GetURL()),
|
|
ResponseCode);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// We don't know the response code, default to warning log level
|
|
UE_LOG(LogHttp, Warning, TEXT("Retry %d on %s"), RequestEntry.CurrentRetryCount, *(RequestEntry.Request->GetURL()));
|
|
}
|
|
|
|
RequestEntry.Request->HttpRequest->ProcessRequest();
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::RetryHttpRequestWithDelay(FManager::FHttpRetryRequestEntry& RequestEntry, float InDelay, bool bWasSucceeded)
|
|
{
|
|
// Timeout during lock out to keep existing behavior
|
|
float TimeoutOrDefault = RequestEntry.Request->GetTimeout().Get(FHttpModule::Get().GetHttpTotalTimeout());
|
|
if (TimeoutOrDefault != 0)
|
|
{
|
|
float TimeElapsedForTheRequest = FPlatformTime::Seconds() - RequestEntry.RequestStartTimeAbsoluteSeconds;
|
|
float WillTimeoutInDelay = TimeoutOrDefault - TimeElapsedForTheRequest;
|
|
if (WillTimeoutInDelay < InDelay)
|
|
{
|
|
HttpRequestTimeoutAfterDelay(RequestEntry, bWasSucceeded, WillTimeoutInDelay);
|
|
return;
|
|
}
|
|
}
|
|
|
|
// Delay and start
|
|
TWeakPtr<FRequest> RequestWeakPtr(RequestEntry.Request);
|
|
FHttpModule::Get().GetHttpManager().AddHttpThreadTask([RequestWeakPtr, bWasSucceeded]() {
|
|
if (TSharedPtr<FRequest> RequestPtr = RequestWeakPtr.Pin())
|
|
{
|
|
if (TSharedPtr<FManager> RetryManagerPtr = RequestPtr->RetryManager.Pin())
|
|
{
|
|
FScopeLock ScopeLock(&RetryManagerPtr->RequestListLock);
|
|
// Check if it's still there in case it has been cancelled during the delay period
|
|
uint32 EntryIndex = RetryManagerPtr->RequestList.IndexOfByPredicate([RequestPtr](const FManager::FHttpRetryRequestEntry& Entry) { return Entry.Request == RequestPtr; });
|
|
if (EntryIndex != INDEX_NONE)
|
|
{
|
|
FManager::FHttpRetryRequestEntry* HttpRetryRequestEntry = &RetryManagerPtr->RequestList[EntryIndex];
|
|
// TODO: Move this into RetryHttpRequest after stabilizing the flow with CVarHttpRetrySystemNonGameThreadSupportEnabled on
|
|
HttpRetryRequestEntry->Request->LastResponse = HttpRetryRequestEntry->Request->GetResponse();
|
|
HttpRetryRequestEntry->Request->bLastSucceeded = bWasSucceeded;
|
|
RetryManagerPtr->RetryHttpRequest(*HttpRetryRequestEntry);
|
|
}
|
|
}
|
|
}
|
|
}, InDelay);
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::HttpRequestTimeoutAfterDelay(FManager::FHttpRetryRequestEntry& RequestEntry, bool bWasSucceeded, float Delay)
|
|
{
|
|
TWeakPtr<FRequest> RequestWeakPtr(RequestEntry.Request);
|
|
TFunction<void()> Callback([RequestWeakPtr, bWasSucceeded]() {
|
|
if (TSharedPtr<FRequest> RequestPtr = RequestWeakPtr.Pin())
|
|
{
|
|
if (TSharedPtr<FManager> RetryManagerPtr = RequestPtr->RetryManager.Pin())
|
|
{
|
|
FScopeLock ScopeLock(&RetryManagerPtr->RequestListLock);
|
|
uint32 EntryIndex = RetryManagerPtr->RequestList.IndexOfByPredicate([RequestPtr](const FManager::FHttpRetryRequestEntry& Entry) { return Entry.Request == RequestPtr; });
|
|
if (EntryIndex != INDEX_NONE)
|
|
{
|
|
FManager::FHttpRetryRequestEntry* HttpRetryRequestEntry = &RetryManagerPtr->RequestList[EntryIndex];
|
|
if (HttpRetryRequestEntry->CurrentRetryCount > 0)
|
|
{
|
|
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::Get().DecrementRetriedRequests();
|
|
}
|
|
RetryManagerPtr->RequestList.RemoveAtSwap(EntryIndex);
|
|
}
|
|
}
|
|
|
|
// Same as existing behavior, when timeout during lock out period, it fails with result of last request before lockout
|
|
RequestPtr->OnProcessRequestComplete().ExecuteIfBound(RequestPtr, RequestPtr->GetResponse(), bWasSucceeded);
|
|
}
|
|
});
|
|
|
|
if (RequestEntry.Request->GetDelegateThreadPolicy() == EHttpRequestDelegateThreadPolicy::CompleteOnGameThread)
|
|
{
|
|
FHttpModule::Get().GetHttpManager().AddGameThreadTask(MoveTemp(Callback), Delay);
|
|
}
|
|
else
|
|
{
|
|
FHttpModule::Get().GetHttpManager().AddHttpThreadTask(MoveTemp(Callback), Delay);
|
|
}
|
|
}
|
|
|
|
float FHttpRetrySystem::FManager::GetLockoutPeriodSeconds(const FHttpRetryRequestEntry& HttpRetryRequestEntry)
|
|
{
|
|
float LockoutPeriod = 0.0f;
|
|
TOptional<double> ResponseLockoutPeriod = FHttpRetrySystem::ReadThrottledTimeFromResponseInSeconds(HttpRetryRequestEntry.Request->GetResponse());
|
|
if (ResponseLockoutPeriod.IsSet())
|
|
{
|
|
LockoutPeriod = static_cast<float>(ResponseLockoutPeriod.GetValue());
|
|
}
|
|
|
|
if (LockoutPeriod <= 0.0f)
|
|
{
|
|
const bool bFailedToConnect = (HttpRetryRequestEntry.Request->GetStatus() == EHttpRequestStatus::Failed && HttpRetryRequestEntry.Request->GetFailureReason() == EHttpFailureReason::ConnectionError);
|
|
const bool bHasRetryDomains = HttpRetryRequestEntry.Request->RetryDomains.IsValid();
|
|
// Skip the lockout period if we failed to connect to a domain and we have other domains to try
|
|
if (bFailedToConnect && bHasRetryDomains)
|
|
{
|
|
return 0.0f;
|
|
}
|
|
// The first time through this function, the CurrentRetryCount is 0, the second time it's 1, etc. We automatically add 1 to make the input into the backoff function line up with expectations.
|
|
LockoutPeriod = HttpRetryRequestEntry.Request->RetryExponentialBackoffCurve.Compute(HttpRetryRequestEntry.CurrentRetryCount+1);
|
|
}
|
|
|
|
return LockoutPeriod;
|
|
}
|
|
|
|
static FRandomStream TempRandomStream(4435261);
|
|
|
|
FHttpRetrySystem::FManager::FHttpRetryRequestEntry::FHttpRetryRequestEntry(TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe>& InRequest)
|
|
: bShouldCancel(false)
|
|
, CurrentRetryCount(0)
|
|
, CurrentRetryCountForConnectionError(0)
|
|
, RequestStartTimeAbsoluteSeconds(FPlatformTime::Seconds())
|
|
, Request(InRequest)
|
|
{}
|
|
|
|
bool FHttpRetrySystem::FManager::ProcessRequest(TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe>& HttpRetryRequest)
|
|
{
|
|
QUICK_SCOPE_CYCLE_COUNTER(STAT_FHttpRetrySystem_FManager_ProcessRequest);
|
|
|
|
// Let the request trigger timeout by itself, instead of ticking it in retry system
|
|
if (HttpRetryRequest->RetryTimeoutRelativeSecondsOverride.IsSet())
|
|
{
|
|
HttpRetryRequest->SetTimeout(HttpRetryRequest->RetryTimeoutRelativeSecondsOverride.GetValue());
|
|
}
|
|
else if (RetryTimeoutRelativeSecondsDefault.IsSet())
|
|
{
|
|
HttpRetryRequest->SetTimeout(RetryTimeoutRelativeSecondsDefault.GetValue());
|
|
}
|
|
|
|
FScopeLock ScopeLock(&RequestListLock);
|
|
RequestList.Add(FHttpRetryRequestEntry(HttpRetryRequest));
|
|
HttpRetryRequest->RetryStatus = FHttpRetrySystem::FRequest::EStatus::Processing;
|
|
HttpRetryRequest->HttpRequest->ProcessRequest();
|
|
|
|
return true;
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::CancelRequest(TSharedRef<FHttpRetrySystem::FRequest, ESPMode::ThreadSafe>& HttpRetryRequest)
|
|
{
|
|
UE_AUTORTFM_ONCOMMIT(this, HttpRetryRequest)
|
|
{
|
|
QUICK_SCOPE_CYCLE_COUNTER(STAT_FHttpRetrySystem_FManager_CancelRequest);
|
|
|
|
FScopeLock ScopeLock(&RequestListLock);
|
|
|
|
// Find the existing request entry if is was previously processed.
|
|
bool bFound = false;
|
|
for (int32 i = 0; i < RequestList.Num(); ++i)
|
|
{
|
|
FHttpRetryRequestEntry& EntryRef = RequestList[i];
|
|
|
|
if (EntryRef.Request == HttpRetryRequest)
|
|
{
|
|
EntryRef.bShouldCancel = true;
|
|
bFound = true;
|
|
}
|
|
}
|
|
// If we did not find the entry, likely auth failed for the request, in which case ProcessRequest does not get called.
|
|
// Adding it to the list and flagging as cancel will process it on next tick.
|
|
if (!bFound)
|
|
{
|
|
FHttpRetryRequestEntry RetryRequestEntry(HttpRetryRequest);
|
|
RetryRequestEntry.bShouldCancel = true;
|
|
RequestList.Add(RetryRequestEntry);
|
|
}
|
|
|
|
HttpRetryRequest->HttpRequest->CancelRequest();
|
|
HttpRetryRequest->RetryStatus = FHttpRetrySystem::FRequest::EStatus::Cancelled;
|
|
};
|
|
}
|
|
|
|
/* This should only be used when shutting down or suspending, to make sure
|
|
all pending HTTP requests are flushed to the network */
|
|
void FHttpRetrySystem::FManager::BlockUntilFlushed(float InTimeoutSec)
|
|
{
|
|
const float SleepInterval = 0.016;
|
|
float TimeElapsed = 0.0f;
|
|
|
|
while (TimeElapsed < InTimeoutSec)
|
|
{
|
|
{
|
|
FScopeLock ScopeLock(&RequestListLock);
|
|
if (RequestList.IsEmpty())
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
FHttpModule::Get().GetHttpManager().Tick(SleepInterval);
|
|
|
|
FPlatformProcess::Sleep(SleepInterval);
|
|
TimeElapsed += SleepInterval;
|
|
}
|
|
}
|
|
|
|
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker& FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::Get()
|
|
{
|
|
static FHttpLogVerbosityTracker Tracker;
|
|
return Tracker;
|
|
}
|
|
|
|
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::FHttpLogVerbosityTracker()
|
|
{
|
|
UpdateSettingsFromConfig();
|
|
FCoreDelegates::TSOnConfigSectionsChanged().AddRaw(this, &FHttpLogVerbosityTracker::OnConfigSectionsChanged);
|
|
}
|
|
|
|
FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::~FHttpLogVerbosityTracker()
|
|
{
|
|
FCoreDelegates::TSOnConfigSectionsChanged().RemoveAll(this);
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::IncrementRetriedRequests()
|
|
{
|
|
FScopeLock ScopeLock(&NumRetriedRequestsLock);
|
|
|
|
++NumRetriedRequests;
|
|
if (NumRetriedRequests == 1)
|
|
{
|
|
OriginalVerbosity = UE_GET_LOG_VERBOSITY(LogHttp);
|
|
if (TargetVerbosity != ELogVerbosity::NoLogging)
|
|
{
|
|
UE_LOG(LogHttp, Warning, TEXT("HttpRetry: Increasing log verbosity from %s to %s due to requests being retried"), ToString(OriginalVerbosity), ToString(TargetVerbosity));
|
|
//UE_SET_LOG_VERBOSITY(LogHttp, TargetVerbosity); // Macro requires the value to be a ELogVerbosity constant
|
|
#if !NO_LOGGING
|
|
LogHttp.SetVerbosity(TargetVerbosity);
|
|
#endif
|
|
}
|
|
}
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::DecrementRetriedRequests()
|
|
{
|
|
FScopeLock ScopeLock(&NumRetriedRequestsLock);
|
|
|
|
--NumRetriedRequests;
|
|
check(NumRetriedRequests >= 0);
|
|
if (NumRetriedRequests == 0)
|
|
{
|
|
UE_LOG(LogHttp, Warning, TEXT("HttpRetry: Resetting log verbosity to %s due to requests being retried"), ToString(OriginalVerbosity));
|
|
//UE_SET_LOG_VERBOSITY(LogHttp, OriginalVerbosity); // Macro requires the value to be a ELogVerbosity constant
|
|
#if !NO_LOGGING
|
|
LogHttp.SetVerbosity(OriginalVerbosity);
|
|
#endif
|
|
}
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::UpdateSettingsFromConfig()
|
|
{
|
|
FString TargetVerbosityAsString;
|
|
if (GConfig->GetString(TEXT("HTTP.Retry"), TEXT("RetryManagerVerbosityLevel"), TargetVerbosityAsString, GEngineIni))
|
|
{
|
|
TargetVerbosity = ParseLogVerbosityFromString(TargetVerbosityAsString);
|
|
}
|
|
else
|
|
{
|
|
TargetVerbosity = ELogVerbosity::NoLogging;
|
|
}
|
|
}
|
|
|
|
void FHttpRetrySystem::FManager::FHttpLogVerbosityTracker::OnConfigSectionsChanged(const FString& IniFilename, const TSet<FString>& SectionName)
|
|
{
|
|
if (IniFilename == GEngineIni && SectionName.Contains(TEXT("HTTP.Retry")))
|
|
{
|
|
UpdateSettingsFromConfig();
|
|
}
|
|
}
|