757 lines
24 KiB
C++
757 lines
24 KiB
C++
// Copyright Epic Games, Inc. All Rights Reserved.
|
|
|
|
#include "Experimental/BuildServerInterface.h"
|
|
|
|
#include "Async/Mutex.h"
|
|
#include "Containers/StringView.h"
|
|
#include "DesktopPlatformModule.h"
|
|
#include "Experimental/ZenServerInterface.h"
|
|
#include "HAL/CriticalSection.h"
|
|
#include "HAL/FileManager.h"
|
|
#include "Http/HttpClient.h"
|
|
#include "Http/HttpHostBuilder.h"
|
|
#include "Misc/App.h"
|
|
#include "Misc/ConfigCacheIni.h"
|
|
#include "Misc/MonitoredProcess.h"
|
|
#include "Misc/Paths.h"
|
|
#include "Misc/ScopeExit.h"
|
|
#include "Misc/ScopeLock.h"
|
|
#include "Misc/ScopeRWLock.h"
|
|
#include "Misc/StringBuilder.h"
|
|
#include "Serialization/Archive.h"
|
|
#include "Serialization/CompactBinary.h"
|
|
#include "Serialization/CompactBinarySerialization.h"
|
|
#include "Serialization/CompactBinaryValidation.h"
|
|
#include "Serialization/CompactBinaryWriter.h"
|
|
#include "String/LexFromString.h"
|
|
#include "Tasks/Task.h"
|
|
|
|
#if WITH_SSL
|
|
#include "Ssl.h"
|
|
#endif
|
|
|
|
#define UE_BUILDSERVERINSTANCE_MAX_FAILED_LOGIN_ATTEMPTS 16
|
|
|
|
namespace UE::Zen::Build
|
|
{
|
|
|
|
DEFINE_LOG_CATEGORY_STATIC(LogBuildServiceInstance, Log, All);
|
|
|
|
namespace Private
|
|
{
|
|
class FAccessToken
|
|
{
|
|
public:
|
|
void SetToken(FStringView Scheme, FStringView Token);
|
|
inline uint32 GetSerial() const { return Serial.load(std::memory_order_relaxed); }
|
|
friend FAnsiStringBuilderBase& operator<<(FAnsiStringBuilderBase& Builder, const FAccessToken& Token);
|
|
|
|
private:
|
|
mutable FRWLock Lock;
|
|
TArray<ANSICHAR> Header;
|
|
std::atomic<uint32> Serial;
|
|
};
|
|
|
|
FAnsiStringBuilderBase& operator<<(FAnsiStringBuilderBase& Builder, const Private::FAccessToken& Token)
|
|
{
|
|
FReadScopeLock ReadLock(Token.Lock);
|
|
return Builder.Append(Token.Header);
|
|
}
|
|
|
|
void FAccessToken::SetToken(const FStringView Scheme, const FStringView Token)
|
|
{
|
|
FWriteScopeLock WriteLock(Lock);
|
|
const int32 TokenLen = FPlatformString::ConvertedLength<ANSICHAR>(Token.GetData(), Token.Len());
|
|
|
|
Header.Empty(TokenLen);
|
|
|
|
const int32 TokenIndex = Header.AddUninitialized(TokenLen);
|
|
FPlatformString::Convert(Header.GetData() + TokenIndex, TokenLen, Token.GetData(), Token.Len());
|
|
Serial.fetch_add(1, std::memory_order_relaxed);
|
|
}
|
|
} // namespace Private
|
|
|
|
bool LoadFromCompactBinary(FCbFieldView Field, FBuildServiceInstance::FBuildRecord& OutBuildRecord)
|
|
{
|
|
if (!Field.IsObject())
|
|
{
|
|
return false;
|
|
}
|
|
bool bOk = true;
|
|
FString BuildIdString;
|
|
FCbObjectId::ByteArray BuildIdBytes;
|
|
bOk &= LoadFromCompactBinary(Field["buildId"], BuildIdString);
|
|
bOk &= UE::String::HexToBytes(BuildIdString, BuildIdBytes) == sizeof(BuildIdBytes);
|
|
OutBuildRecord.BuildId = FCbObjectId(BuildIdBytes);
|
|
|
|
FCbFieldView MetadataField = Field["metadata"];
|
|
if (!MetadataField.IsObject())
|
|
{
|
|
return false;
|
|
}
|
|
|
|
OutBuildRecord.Metadata = FCbObject::Clone(MetadataField.AsObjectView());
|
|
return bOk;
|
|
}
|
|
|
|
bool
|
|
FServiceSettings::ReadFromConfig()
|
|
{
|
|
check(GConfig && GConfig->IsReadyForUse());
|
|
FString Config;
|
|
if (!GConfig->GetString(TEXT("StorageServers"), TEXT("Cloud"), Config, GEngineIni))
|
|
{
|
|
return false;
|
|
}
|
|
bool bRetVal = false;
|
|
|
|
bRetVal |= FParse::Value(*Config, TEXT("Host="), Host);
|
|
bRetVal |= FParse::Value(*Config, TEXT("OAuthProviderIdentifier="), OAuthProviderIdentifier);
|
|
|
|
FParse::Value(*Config, TEXT("AuthScheme="), AuthScheme);
|
|
if (AuthScheme.IsEmpty())
|
|
{
|
|
AuthScheme = "Bearer";
|
|
}
|
|
|
|
return bRetVal;
|
|
}
|
|
|
|
bool
|
|
FServiceSettings::ReadFromURL(FStringView InstanceURL)
|
|
{
|
|
Host = InstanceURL;
|
|
return true;
|
|
}
|
|
|
|
FBuildServiceInstance::FBuildServiceInstance()
|
|
{
|
|
Settings.ReadFromConfig();
|
|
Initialize();
|
|
}
|
|
|
|
FBuildServiceInstance::FBuildServiceInstance(FStringView InstanceURL)
|
|
{
|
|
Settings.ReadFromURL(InstanceURL);
|
|
Initialize();
|
|
}
|
|
|
|
FBuildServiceInstance::~FBuildServiceInstance()
|
|
{
|
|
bBuildTransferThreadStopping = true;
|
|
if (BuildTransferThread.IsJoinable())
|
|
{
|
|
BuildTransferThread.Join();
|
|
}
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::Connect(FOnConnectionComplete&& OnConnectionComplete)
|
|
{
|
|
EConnectionState ExpectedConnectionState = EConnectionState::NotStarted;
|
|
if (!ConnectionState.compare_exchange_weak(ExpectedConnectionState, EConnectionState::ConnectionInProgress))
|
|
{
|
|
if (OnConnectionComplete)
|
|
{
|
|
OnConnectionComplete(ExpectedConnectionState);
|
|
}
|
|
return;
|
|
}
|
|
|
|
UE::Tasks::Launch(UE_SOURCE_LOCATION, [this, OnConnectionComplete = MoveTemp(OnConnectionComplete)]()
|
|
{
|
|
TAnsiStringBuilder<256> ResolvedHost;
|
|
double ResolvedLatency;
|
|
FHttpHostBuilder HostBuilder;
|
|
HostBuilder.AddFromString(Settings.GetHost());
|
|
if (HostBuilder.ResolveHost(/* Warning timeout */ 1.0, 4.0 /* Max duration timeout*/, ResolvedHost, ResolvedLatency))
|
|
{
|
|
EffectiveDomain.Reset();
|
|
EffectiveDomain.Append(ResolvedHost);
|
|
|
|
if (AcquireAccessToken())
|
|
{
|
|
ConnectionState.store(EConnectionState::ConnectionSucceeded, std::memory_order_relaxed);
|
|
if (OnConnectionComplete)
|
|
{
|
|
OnConnectionComplete(EConnectionState::ConnectionSucceeded);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unable to acquire access token."));
|
|
ConnectionState.store(EConnectionState::ConnectionFailed, std::memory_order_relaxed);
|
|
if (OnConnectionComplete)
|
|
{
|
|
OnConnectionComplete(EConnectionState::ConnectionFailed);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// even if we fail to resolve a host to use the returned host will at least contain the first of the possible hosts which we can attempt to use
|
|
EffectiveDomain.Reset();
|
|
EffectiveDomain.Append(ResolvedHost);
|
|
|
|
FString HostCandidates = HostBuilder.GetHostCandidatesString();
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unable to resolve best host candidate to use, most likely none of the suggested hosts was reachable. Attempted hosts were: '%s' ."), *HostCandidates);
|
|
ConnectionState.store(EConnectionState::ConnectionFailed, std::memory_order_relaxed);
|
|
if (OnConnectionComplete)
|
|
{
|
|
OnConnectionComplete(EConnectionState::ConnectionFailed);
|
|
}
|
|
}
|
|
|
|
});
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::RefreshNamespacesAndBuckets(FOnRefreshNamespacesAndBucketsComplete&& InOnRefreshNamespacesAndBucketsComplete)
|
|
{
|
|
{
|
|
FWriteScopeLock _(NamespacesAndBucketsLock);
|
|
NamespacesAndBuckets.Empty();
|
|
}
|
|
if (ConnectionState.load(std::memory_order_relaxed) != EConnectionState::ConnectionSucceeded)
|
|
{
|
|
CallOnRefreshNamespacesAndBucketsComplete(MoveTemp(InOnRefreshNamespacesAndBucketsComplete));
|
|
return;
|
|
}
|
|
|
|
FString OutputFilePath = FPaths::CreateTempFilename(FPlatformProcess::UserTempDir(), TEXT("BuildContainers"), TEXT(".cbo"));
|
|
FPaths::MakePlatformFilename(OutputFilePath);
|
|
|
|
FString CommandLineArgs = FString::Printf(TEXT("builds list-namespaces --recursive \"%s\""),
|
|
*OutputFilePath);
|
|
|
|
InvokeZenUtility(CommandLineArgs, nullptr,
|
|
[this, OutputFilePath = MoveTemp(OutputFilePath), InOnRefreshNamespacesAndBucketsComplete = MoveTemp(InOnRefreshNamespacesAndBucketsComplete)](bool bSuccessful) mutable
|
|
{
|
|
if (bSuccessful)
|
|
{
|
|
TUniquePtr<FArchive> Ar(IFileManager::Get().CreateFileReader(*OutputFilePath));
|
|
if (!Ar)
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Missing output file from zen utility when gathering build data: '%s'"), *OutputFilePath);
|
|
CallOnRefreshNamespacesAndBucketsComplete(MoveTemp(InOnRefreshNamespacesAndBucketsComplete));
|
|
return;
|
|
}
|
|
|
|
FCbObject OutputObject = LoadCompactBinary(*Ar).AsObject();
|
|
{
|
|
FWriteScopeLock _(NamespacesAndBucketsLock);
|
|
for (FCbFieldView ResultField : OutputObject["results"])
|
|
{
|
|
FString Namespace = *WriteToString<64>(ResultField["name"].AsString());
|
|
for (FCbFieldView Item : ResultField["items"])
|
|
{
|
|
NamespacesAndBuckets.Add(Namespace, *WriteToString<64>(Item.AsString()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
CallOnRefreshNamespacesAndBucketsComplete(MoveTemp(InOnRefreshNamespacesAndBucketsComplete));
|
|
});
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::ListBuilds(FStringView Namespace, FStringView Bucket, FOnListBuildsComplete&& InOnListBuildsComplete)
|
|
{
|
|
if (ConnectionState.load(std::memory_order_relaxed) != EConnectionState::ConnectionSucceeded)
|
|
{
|
|
if (InOnListBuildsComplete)
|
|
{
|
|
InOnListBuildsComplete({});
|
|
}
|
|
return;
|
|
}
|
|
|
|
FString QueryFilePath = FPaths::CreateTempFilename(FPlatformProcess::UserTempDir(), TEXT("BuildListQuery"), TEXT(".cbo"));
|
|
FPaths::MakePlatformFilename(QueryFilePath);
|
|
|
|
FString OutputFilePath = FPaths::CreateTempFilename(FPlatformProcess::UserTempDir(), TEXT("BuildList"), TEXT(".cbo"));
|
|
FPaths::MakePlatformFilename(OutputFilePath);
|
|
|
|
FString CommandLineArgs = FString::Printf(TEXT("builds list --namespace \"%s\" --bucket \"%s\" \"%s\" \"%s\""),
|
|
*WriteToString<64>(Namespace), *WriteToString<64>(Bucket), *QueryFilePath, *OutputFilePath);
|
|
|
|
auto PreInvoke = [QueryFilePath](FString& CommandLineArgs)
|
|
{
|
|
TUniquePtr<FArchive> QueryFileArchive(IFileManager::Get().CreateFileWriter(*QueryFilePath, FILEWRITE_NoFail));
|
|
FCbWriter Writer;
|
|
Writer.BeginObject("query");
|
|
Writer.EndObject();
|
|
Writer.Save(*QueryFileArchive);
|
|
};
|
|
|
|
InvokeZenUtility(CommandLineArgs, MoveTemp(PreInvoke),
|
|
[OutputFilePath = MoveTemp(OutputFilePath), InOnListBuildsComplete = MoveTemp(InOnListBuildsComplete)](bool bSuccessful) mutable
|
|
{
|
|
if (!bSuccessful)
|
|
{
|
|
if (InOnListBuildsComplete)
|
|
{
|
|
InOnListBuildsComplete({});
|
|
}
|
|
return;
|
|
}
|
|
|
|
TUniquePtr<FArchive> Ar(IFileManager::Get().CreateFileReader(*OutputFilePath));
|
|
if (!Ar)
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Missing output file from zen utility when gathering build data: '%s'"), *OutputFilePath);
|
|
if (InOnListBuildsComplete)
|
|
{
|
|
InOnListBuildsComplete({});
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (InOnListBuildsComplete)
|
|
{
|
|
FCbObject OutputObject = LoadCompactBinary(*Ar).AsObject();
|
|
TArray<FBuildRecord> BuildRecords;
|
|
for (FCbFieldView ResultField : OutputObject["results"].AsArrayView())
|
|
{
|
|
FBuildRecord BuildRecord;
|
|
if (LoadFromCompactBinary(ResultField, BuildRecord))
|
|
{
|
|
BuildRecords.Emplace(MoveTemp(BuildRecord));
|
|
}
|
|
}
|
|
InOnListBuildsComplete(MoveTemp(BuildRecords));
|
|
}
|
|
|
|
});
|
|
}
|
|
|
|
FBuildServiceInstance::FBuildTransfer
|
|
FBuildServiceInstance::StartBuildTransfer(const FCbObjectId& BuildId, FStringView DestinationFolder, FStringView Namespace, FStringView Bucket)
|
|
{
|
|
const FString ZenUtility = UE::Zen::GetLocalInstallUtilityPath();
|
|
TSharedPtr<FBuildTransferCommand> NewTransferCommand = MakeShared<FBuildTransferCommand>();
|
|
NewTransferCommand->Type = EBuildTransferType::Files;
|
|
NewTransferCommand->BuildId = BuildId;
|
|
NewTransferCommand->Namespace = Namespace;
|
|
NewTransferCommand->Bucket = Bucket;
|
|
NewTransferCommand->Destination = DestinationFolder;
|
|
BuildTransferThreadCommands.Enqueue(NewTransferCommand.ToSharedRef());
|
|
KickBuildTransferThread();
|
|
|
|
FBuildTransfer NewTransfer;
|
|
NewTransfer.BuildTransferCommand = NewTransferCommand;
|
|
return NewTransfer;
|
|
}
|
|
|
|
FBuildServiceInstance::FBuildTransfer
|
|
FBuildServiceInstance::StartOplogBuildTransfer(const FCbObjectId& BuildId, FStringView DestinationProjectId, FStringView DestinationOplogId, FStringView ProjectFilePath, FStringView Namespace, FStringView Bucket)
|
|
{
|
|
const FString ZenUtility = UE::Zen::GetLocalInstallUtilityPath();
|
|
TSharedPtr<FBuildTransferCommand> NewTransferCommand = MakeShared<FBuildTransferCommand>();
|
|
NewTransferCommand->Type = EBuildTransferType::Oplog;
|
|
NewTransferCommand->BuildId = BuildId;
|
|
NewTransferCommand->Namespace = Namespace;
|
|
NewTransferCommand->Bucket = Bucket;
|
|
NewTransferCommand->Destination = *WriteToString<64>(DestinationProjectId, TEXT("/"), DestinationOplogId);
|
|
NewTransferCommand->ProjectFilePath = ProjectFilePath;
|
|
BuildTransferThreadCommands.Enqueue(NewTransferCommand.ToSharedRef());
|
|
KickBuildTransferThread();
|
|
|
|
FBuildTransfer NewTransfer;
|
|
NewTransfer.BuildTransferCommand = NewTransferCommand;
|
|
return NewTransfer;
|
|
}
|
|
|
|
FString
|
|
FBuildServiceInstance::FBuildTransfer::GetDescription() const
|
|
{
|
|
if (!BuildTransferCommand)
|
|
{
|
|
return TEXT("null");
|
|
}
|
|
|
|
FReadScopeLock _(BuildTransferCommand->Lock);
|
|
int32 Index = 0;
|
|
if (BuildTransferCommand->Output.FindLastChar(TCHAR('\n'), Index))
|
|
{
|
|
return BuildTransferCommand->Output.RightChop(Index+1);
|
|
}
|
|
return BuildTransferCommand->Output;
|
|
}
|
|
|
|
FBuildServiceInstance::EBuildTransferStatus
|
|
FBuildServiceInstance::FBuildTransfer::GetStatus() const
|
|
{
|
|
if (!BuildTransferCommand)
|
|
{
|
|
return EBuildTransferStatus::Invalid;
|
|
}
|
|
|
|
FReadScopeLock _(BuildTransferCommand->Lock);
|
|
return BuildTransferCommand->Status;
|
|
}
|
|
|
|
FBuildServiceInstance::EConnectionState
|
|
FBuildServiceInstance::GetConnectionState() const
|
|
{
|
|
return ConnectionState.load(std::memory_order_relaxed);
|
|
}
|
|
|
|
FAnsiStringView
|
|
FBuildServiceInstance::GetEffectiveDomain() const
|
|
{
|
|
return EffectiveDomain;
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::Initialize()
|
|
{
|
|
#if WITH_SSL
|
|
// Load SSL module during HTTP module's StatupModule() to make sure module manager figures out the dependencies correctly
|
|
// and doesn't unload SSL before unloading HTTP module at exit
|
|
FSslModule::Get();
|
|
#endif
|
|
FDesktopPlatformModule::TryGet();
|
|
}
|
|
|
|
bool
|
|
FBuildServiceInstance::AcquireAccessToken()
|
|
{
|
|
if (GetEffectiveDomain().StartsWith("http://localhost"))
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Log, TEXT("Skipping authorization for connection to localhost."));
|
|
return true;
|
|
}
|
|
|
|
LoginAttempts++;
|
|
|
|
// Avoid spamming this if the service is down.
|
|
if (FailedLoginAttempts > UE_BUILDSERVERINSTANCE_MAX_FAILED_LOGIN_ATTEMPTS)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
TRACE_CPUPROFILER_EVENT_SCOPE(BuildServiceInstance_AcquireAccessToken);
|
|
|
|
// In case many requests wants to update the token at the same time
|
|
// get the current serial while we wait to take the CS.
|
|
const uint32 WantsToUpdateTokenSerial = Access ? Access->GetSerial() : 0;
|
|
|
|
FScopeLock Lock(&AccessCs);
|
|
|
|
// If the token was updated while we waited to take the lock, then it should now be valid.
|
|
if (Access && Access->GetSerial() > WantsToUpdateTokenSerial)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
FString AccessTokenString;
|
|
FDateTime TokenExpiresAt;
|
|
bool bWasInteractiveLogin = false;
|
|
|
|
IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::TryGet();
|
|
if (DesktopPlatform && DesktopPlatform->GetOidcAccessToken(FPaths::RootDir(), {}, Settings.GetOAuthProviderIdentifier(), FApp::IsUnattended(), GWarn, AccessTokenString, TokenExpiresAt, bWasInteractiveLogin))
|
|
{
|
|
if (bWasInteractiveLogin)
|
|
{
|
|
InteractiveLoginAttempts++;
|
|
}
|
|
|
|
const double ExpiryTimeSeconds = (TokenExpiresAt - FDateTime::UtcNow()).GetTotalSeconds();
|
|
UE_LOG(LogBuildServiceInstance, Display,
|
|
TEXT("OidcToken: Logged in to HTTP DDC services. Expires at %s which is in %.0f seconds."),
|
|
*TokenExpiresAt.ToString(), ExpiryTimeSeconds);
|
|
SetAccessTokenAndUnlock(Lock, AccessTokenString, ExpiryTimeSeconds);
|
|
return true;
|
|
}
|
|
else if (DesktopPlatform)
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("OidcToken: Failed to log in to HTTP services."));
|
|
FailedLoginAttempts++;
|
|
return false;
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("OidcToken: Use of OAuthProviderIdentifier requires that the target depend on DesktopPlatform."));
|
|
FailedLoginAttempts++;
|
|
return false;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::SetAccessTokenAndUnlock(FScopeLock& Lock, FStringView Token, double RefreshDelay)
|
|
{
|
|
// Cache the expired refresh handle.
|
|
FTSTicker::FDelegateHandle ExpiredRefreshAccessTokenHandle = MoveTemp(RefreshAccessTokenHandle);
|
|
RefreshAccessTokenHandle.Reset();
|
|
|
|
if (!Access)
|
|
{
|
|
Access = MakeUnique<Private::FAccessToken>();
|
|
}
|
|
Access->SetToken(Settings.GetAuthScheme(), Token);
|
|
|
|
constexpr double RefreshGracePeriod = 20.0f;
|
|
if (RefreshDelay > RefreshGracePeriod)
|
|
{
|
|
// Schedule a refresh of the token ahead of expiry time (this will not work in commandlets)
|
|
if (!IsRunningCommandlet())
|
|
{
|
|
RefreshAccessTokenHandle = FTSTicker::GetCoreTicker().AddTicker(FTickerDelegate::CreateLambda(
|
|
[this](float DeltaTime)
|
|
{
|
|
AcquireAccessToken();
|
|
return false;
|
|
}
|
|
), float(FMath::Min(RefreshDelay - RefreshGracePeriod, MAX_flt)));
|
|
}
|
|
|
|
// Schedule a forced refresh of the token when the scheduled refresh is starved or unavailable.
|
|
RefreshAccessTokenTime = FPlatformTime::Seconds() + RefreshDelay - RefreshGracePeriod * 0.5f;
|
|
}
|
|
else
|
|
{
|
|
RefreshAccessTokenTime = 0.0;
|
|
}
|
|
|
|
// Reset failed login attempts, the service is indeed alive.
|
|
FailedLoginAttempts = 0;
|
|
|
|
// Unlock the critical section before attempting to remove the expired refresh handle.
|
|
// The associated ticker delegate could already be executing, which could cause a
|
|
// hang in RemoveTicker when the critical section is locked.
|
|
Lock.Unlock();
|
|
if (ExpiredRefreshAccessTokenHandle.IsValid())
|
|
{
|
|
FTSTicker::GetCoreTicker().RemoveTicker(MoveTemp(ExpiredRefreshAccessTokenHandle));
|
|
}
|
|
}
|
|
|
|
FString
|
|
FBuildServiceInstance::GetAccessToken() const
|
|
{
|
|
TAnsiStringBuilder<128> AccessTokenBuilder;
|
|
if (Access.IsValid())
|
|
{
|
|
AccessTokenBuilder << *Access;
|
|
}
|
|
return FString(AccessTokenBuilder);
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::InvokeZenUtility(FStringView InCommandLineArgs, FPreZenUtilityInvocation&& PreInvocation, FOnZenUtilityInvocationComplete&& OnComplete)
|
|
{
|
|
UE::Tasks::Launch(UE_SOURCE_LOCATION,
|
|
[this, InCommandLineArgs = FString(InCommandLineArgs), PreInvocation = MoveTemp(PreInvocation), OnComplete = MoveTemp(OnComplete)]() mutable
|
|
{
|
|
FString ZenUtilityPath = UE::Zen::GetLocalInstallUtilityPath();
|
|
|
|
FString CommandLineArgs = FString::Printf(TEXT("%s --host %s --access-token \"%s\""),
|
|
*InCommandLineArgs, *WriteToString<64>(EffectiveDomain), *GetAccessToken());
|
|
|
|
if (PreInvocation)
|
|
{
|
|
PreInvocation(CommandLineArgs);
|
|
}
|
|
|
|
FString AbsoluteUtilityPath = FPaths::ConvertRelativePathToFull(ZenUtilityPath);
|
|
FMonitoredProcess MonitoredUtilityProcess(AbsoluteUtilityPath, *CommandLineArgs, FPaths::GetPath(AbsoluteUtilityPath), true);
|
|
if (!MonitoredUtilityProcess.Launch())
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Failed to launch zen utility to gather build data: '%s'."), *AbsoluteUtilityPath);
|
|
if (OnComplete)
|
|
{
|
|
OnComplete(false);
|
|
}
|
|
return;
|
|
}
|
|
|
|
const uint64 StartTime = FPlatformTime::Cycles64();
|
|
while (MonitoredUtilityProcess.Update())
|
|
{
|
|
double Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
|
|
if (Duration > 120.0)
|
|
{
|
|
MonitoredUtilityProcess.Cancel(true);
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Cancelled launch of zen utility for gathering build data: '%s' due to timeout."), *AbsoluteUtilityPath);
|
|
|
|
// Wait for execution to be terminated
|
|
while (MonitoredUtilityProcess.Update())
|
|
{
|
|
Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
|
|
if (Duration > 15.0)
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Cancelled launch of zen utility for gathering build data: '%s'. Failed waiting for termination."), *AbsoluteUtilityPath);
|
|
break;
|
|
}
|
|
FPlatformProcess::Sleep(0.2f);
|
|
}
|
|
|
|
FString OutputString = MonitoredUtilityProcess.GetFullOutputWithoutDelegate();
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Launch of zen utility for gathering build data: '%s' failed. Output: '%s'"), *AbsoluteUtilityPath, *OutputString);
|
|
if (OnComplete)
|
|
{
|
|
OnComplete(false);
|
|
}
|
|
return;
|
|
}
|
|
FPlatformProcess::Sleep(0.1f);
|
|
}
|
|
|
|
FString OutputString = MonitoredUtilityProcess.GetFullOutputWithoutDelegate();
|
|
if (MonitoredUtilityProcess.GetReturnCode() != 0)
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unexpected return code after launch of zen utility for gathering build data: '%s' (%d). Output: '%s'"), *AbsoluteUtilityPath, MonitoredUtilityProcess.GetReturnCode(), *OutputString);
|
|
if (OnComplete)
|
|
{
|
|
OnComplete(false);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (OnComplete)
|
|
{
|
|
OnComplete(true);
|
|
}
|
|
});
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::CallOnRefreshNamespacesAndBucketsComplete(FOnRefreshNamespacesAndBucketsComplete&& InOnRefreshNamespacesAndBucketsComplete)
|
|
{
|
|
if (InOnRefreshNamespacesAndBucketsComplete)
|
|
{
|
|
InOnRefreshNamespacesAndBucketsComplete();
|
|
}
|
|
|
|
RefreshNamespacesAndBucketsComplete.Broadcast();
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::KickBuildTransferThread()
|
|
{
|
|
if (!bBuildTransferThreadStarting.load(std::memory_order_relaxed) && !bBuildTransferThreadStarting.exchange(true, std::memory_order_relaxed))
|
|
{
|
|
BuildTransferThread = FThread(TEXT("BuildTransfer"), [this] { BuildTransferThreadLoop(); }, 128 * 1024);
|
|
}
|
|
}
|
|
|
|
void
|
|
FBuildServiceInstance::BuildTransferThreadLoop()
|
|
{
|
|
while (!BuildTransferThreadCommands.IsEmpty() || !bBuildTransferThreadStopping.load(std::memory_order_relaxed))
|
|
{
|
|
TOptional<TSharedRef<FBuildTransferCommand>> OptionalCommand = BuildTransferThreadCommands.Dequeue();
|
|
if (!OptionalCommand)
|
|
{
|
|
FPlatformProcess::Sleep(0.2f);
|
|
continue;
|
|
}
|
|
TSharedRef<FBuildTransferCommand> TransferCommand(OptionalCommand.GetValue());
|
|
|
|
FString ZenUtilityPath = FPaths::ConvertRelativePathToFull(UE::Zen::GetLocalInstallUtilityPath());
|
|
FString OidcTokenExecutableFilename = FPaths::ConvertRelativePathToFull(FDesktopPlatformModule::TryGet()->GetOidcTokenExecutableFilename(FPaths::RootDir()));
|
|
|
|
FString CommandLineArgs;
|
|
switch (TransferCommand->Type)
|
|
{
|
|
case EBuildTransferType::Files:
|
|
CommandLineArgs = FString::Printf(TEXT("builds download --host %s --local-path \"%s\" --namespace %s --bucket %s --build-id %s --oidctoken-exe-path \"%s\""),
|
|
*WriteToString<64>(EffectiveDomain), *FPaths::ConvertRelativePathToFull(TransferCommand->Destination), *TransferCommand->Namespace, *TransferCommand->Bucket,
|
|
*WriteToString<64>(TransferCommand->BuildId), *OidcTokenExecutableFilename);
|
|
break;
|
|
case EBuildTransferType::Oplog:
|
|
{
|
|
UE::Zen::FZenLocalServiceRunContext RunContext;
|
|
if (UE::Zen::TryGetLocalServiceRunContext(RunContext) && !TransferCommand->ProjectFilePath.IsEmpty())
|
|
{
|
|
UE::Zen::StartLocalService(RunContext);
|
|
}
|
|
FString DestinationProjectId, DestinationOplogId;
|
|
FString SplitDelim(TEXT("/"));
|
|
TransferCommand->Destination.Split(SplitDelim, &DestinationProjectId, &DestinationOplogId);
|
|
CommandLineArgs = FString::Printf(TEXT("oplog-import %s %s --clean --builds %s --namespace %s --bucket %s --builds-id %s --access-token \"%s\""),
|
|
*DestinationProjectId, *DestinationOplogId, *WriteToString<64>(EffectiveDomain),
|
|
*TransferCommand->Namespace, *TransferCommand->Bucket,
|
|
*WriteToString<64>(TransferCommand->BuildId), *GetAccessToken());
|
|
}
|
|
break;
|
|
}
|
|
|
|
FMonitoredProcess MonitoredUtilityProcess(ZenUtilityPath, *CommandLineArgs, FPaths::GetPath(ZenUtilityPath), true);
|
|
MonitoredUtilityProcess.OnOutput().BindSPLambda(TransferCommand, [TransferCommand](FString Output)
|
|
{
|
|
FWriteScopeLock _(TransferCommand->Lock);
|
|
TransferCommand->Output += TEXT("\n");
|
|
TransferCommand->Output += Output;
|
|
});
|
|
|
|
{
|
|
FWriteScopeLock _(TransferCommand->Lock);
|
|
TransferCommand->Status = EBuildTransferStatus::Active;
|
|
}
|
|
if (!MonitoredUtilityProcess.Launch())
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Failed to launch zen utility to download build: '%s'."), *ZenUtilityPath);
|
|
FWriteScopeLock _(TransferCommand->Lock);
|
|
TransferCommand->Status = EBuildTransferStatus::Failed;
|
|
continue;
|
|
}
|
|
|
|
const uint64 StartTime = FPlatformTime::Cycles64();
|
|
while (MonitoredUtilityProcess.Update())
|
|
{
|
|
if (bBuildTransferThreadStopping.load(std::memory_order_relaxed))
|
|
{
|
|
return;
|
|
}
|
|
double Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
|
|
if (TransferCommand->bCancelRequested.load(std::memory_order_relaxed))
|
|
{
|
|
MonitoredUtilityProcess.Cancel(true);
|
|
UE_LOG(LogBuildServiceInstance, Display, TEXT("Canceled launch of zen utility for downloading build data: '%s'."), *ZenUtilityPath);
|
|
|
|
// Wait for execution to be terminated
|
|
while (MonitoredUtilityProcess.Update())
|
|
{
|
|
if (bBuildTransferThreadStopping.load(std::memory_order_relaxed))
|
|
{
|
|
return;
|
|
}
|
|
Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
|
|
if (Duration > 15.0)
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Canceled launch of zen utility for downloading build data: '%s'. Failed waiting for termination."), *ZenUtilityPath);
|
|
break;
|
|
}
|
|
FPlatformProcess::Sleep(0.2f);
|
|
}
|
|
|
|
UE_LOG(LogBuildServiceInstance, Display, TEXT("Launch of zen utility for downloading build data: '%s' canceled."), *ZenUtilityPath);
|
|
FWriteScopeLock _(TransferCommand->Lock);
|
|
TransferCommand->Status = EBuildTransferStatus::Canceled;
|
|
continue;
|
|
}
|
|
FPlatformProcess::Sleep(0.1f);
|
|
}
|
|
|
|
FString OutputString = MonitoredUtilityProcess.GetFullOutputWithoutDelegate();
|
|
if (MonitoredUtilityProcess.GetReturnCode() == 0)
|
|
{
|
|
FWriteScopeLock _(TransferCommand->Lock);
|
|
TransferCommand->ReturnCode = MonitoredUtilityProcess.GetReturnCode();
|
|
TransferCommand->Status = EBuildTransferStatus::Succeeded;
|
|
}
|
|
else
|
|
{
|
|
UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unexpected return code after launch of zen utility for downloading build data: '%s' (%d). Output: '%s'"), *ZenUtilityPath, MonitoredUtilityProcess.GetReturnCode(), *TransferCommand->Output);
|
|
FWriteScopeLock _(TransferCommand->Lock);
|
|
TransferCommand->ReturnCode = MonitoredUtilityProcess.GetReturnCode();
|
|
TransferCommand->Status = EBuildTransferStatus::Failed;
|
|
}
|
|
}
|
|
}
|
|
|
|
} // namespace UE::StorageService::Build
|