// Copyright Epic Games, Inc. All Rights Reserved.

#include "Experimental/BuildServerInterface.h"

#include "Async/Mutex.h"
#include "Containers/StringView.h"
#include "DesktopPlatformModule.h"
#include "Experimental/ZenServerInterface.h"
#include "HAL/CriticalSection.h"
#include "HAL/FileManager.h"
#include "Http/HttpClient.h"
#include "Http/HttpHostBuilder.h"
#include "Misc/App.h"
#include "Misc/ConfigCacheIni.h"
#include "Misc/MonitoredProcess.h"
#include "Misc/Paths.h"
#include "Misc/ScopeExit.h"
#include "Misc/ScopeLock.h"
#include "Misc/ScopeRWLock.h"
#include "Misc/StringBuilder.h"
#include "Serialization/Archive.h"
#include "Serialization/CompactBinary.h"
#include "Serialization/CompactBinarySerialization.h"
#include "Serialization/CompactBinaryValidation.h"
#include "Serialization/CompactBinaryWriter.h"
#include "String/LexFromString.h"
#include "Tasks/Task.h"

#if WITH_SSL
#include "Ssl.h"
#endif

#define UE_BUILDSERVERINSTANCE_MAX_FAILED_LOGIN_ATTEMPTS 16

namespace UE::Zen::Build
{

DEFINE_LOG_CATEGORY_STATIC(LogBuildServiceInstance, Log, All);

namespace Private
{
class FAccessToken
{
public:
	void SetToken(FStringView Scheme, FStringView Token);
	inline uint32 GetSerial() const { return Serial.load(std::memory_order_relaxed); }
	friend FAnsiStringBuilderBase& operator<<(FAnsiStringBuilderBase& Builder, const FAccessToken& Token);

private:
	mutable FRWLock Lock;
	TArray<ANSICHAR> Header;
	std::atomic<uint32> Serial;
};

FAnsiStringBuilderBase& operator<<(FAnsiStringBuilderBase& Builder, const Private::FAccessToken& Token)
{
	FReadScopeLock ReadLock(Token.Lock);
	return Builder.Append(Token.Header);
}

void FAccessToken::SetToken(const FStringView Scheme, const FStringView Token)
{
	FWriteScopeLock WriteLock(Lock);
	const int32 TokenLen = FPlatformString::ConvertedLength<ANSICHAR>(Token.GetData(), Token.Len());

	Header.Empty(TokenLen);

	const int32 TokenIndex = Header.AddUninitialized(TokenLen);
	FPlatformString::Convert(Header.GetData() + TokenIndex, TokenLen, Token.GetData(), Token.Len());
	Serial.fetch_add(1, std::memory_order_relaxed);
}
} // namespace Private

bool LoadFromCompactBinary(FCbFieldView Field, FBuildServiceInstance::FBuildRecord& OutBuildRecord)
{
	if (!Field.IsObject())
	{
		return false;
	}
	bool bOk = true;
	FString BuildIdString;
	FCbObjectId::ByteArray BuildIdBytes;
	bOk &= LoadFromCompactBinary(Field["buildId"], BuildIdString);
	bOk &= UE::String::HexToBytes(BuildIdString, BuildIdBytes) == sizeof(BuildIdBytes);
	OutBuildRecord.BuildId = FCbObjectId(BuildIdBytes);

	FCbFieldView MetadataField = Field["metadata"];
	if (!MetadataField.IsObject())
	{
		return false;
	}

	OutBuildRecord.Metadata = FCbObject::Clone(MetadataField.AsObjectView());
	return bOk;
}

bool
FServiceSettings::ReadFromConfig()
{
	check(GConfig && GConfig->IsReadyForUse());
	FString Config;
	if (!GConfig->GetString(TEXT("StorageServers"), TEXT("Cloud"), Config, GEngineIni))
	{
		return false;
	}
	bool bRetVal = false;

	bRetVal |= FParse::Value(*Config, TEXT("Host="), Host);
	bRetVal |= FParse::Value(*Config, TEXT("OAuthProviderIdentifier="), OAuthProviderIdentifier);

	FParse::Value(*Config, TEXT("AuthScheme="), AuthScheme);
	if (AuthScheme.IsEmpty())
	{
		AuthScheme = "Bearer";
	}

	return bRetVal;
}

bool
FServiceSettings::ReadFromURL(FStringView InstanceURL)
{
	Host = InstanceURL;
	return true;
}

FBuildServiceInstance::FBuildServiceInstance()
{
	Settings.ReadFromConfig();
	Initialize();
}

FBuildServiceInstance::FBuildServiceInstance(FStringView InstanceURL)
{
	Settings.ReadFromURL(InstanceURL);
	Initialize();
}

FBuildServiceInstance::~FBuildServiceInstance()
{
	bBuildTransferThreadStopping = true;
	if (BuildTransferThread.IsJoinable())
	{
		BuildTransferThread.Join();
	}
}

void
FBuildServiceInstance::Connect(FOnConnectionComplete&& OnConnectionComplete)
{
	EConnectionState ExpectedConnectionState = EConnectionState::NotStarted;
	if (!ConnectionState.compare_exchange_weak(ExpectedConnectionState, EConnectionState::ConnectionInProgress))
	{
		if (OnConnectionComplete)
		{
			OnConnectionComplete(ExpectedConnectionState);
		}
		return;
	}

	UE::Tasks::Launch(UE_SOURCE_LOCATION, [this, OnConnectionComplete = MoveTemp(OnConnectionComplete)]()
	{
		TAnsiStringBuilder<256> ResolvedHost;
		double ResolvedLatency;
		FHttpHostBuilder HostBuilder;
		HostBuilder.AddFromString(Settings.GetHost());
		if (HostBuilder.ResolveHost(/* Warning timeout */ 1.0, 4.0 /* Max duration timeout*/, ResolvedHost, ResolvedLatency))
		{
			EffectiveDomain.Reset();
			EffectiveDomain.Append(ResolvedHost);

			if (AcquireAccessToken())
			{
				ConnectionState.store(EConnectionState::ConnectionSucceeded, std::memory_order_relaxed);
				if (OnConnectionComplete)
				{
					OnConnectionComplete(EConnectionState::ConnectionSucceeded);
				}
			}
			else
			{
				UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unable to acquire access token."));
				ConnectionState.store(EConnectionState::ConnectionFailed, std::memory_order_relaxed);
				if (OnConnectionComplete)
				{
					OnConnectionComplete(EConnectionState::ConnectionFailed);
				}
			}
		}
		else
		{
			// even if we fail to resolve a host to use the returned host will at least contain the first of the possible hosts which we can attempt to use
			EffectiveDomain.Reset();
			EffectiveDomain.Append(ResolvedHost);

			FString HostCandidates = HostBuilder.GetHostCandidatesString();
			UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unable to resolve best host candidate to use, most likely none of the suggested hosts was reachable. Attempted hosts were: '%s' ."), *HostCandidates);
			ConnectionState.store(EConnectionState::ConnectionFailed, std::memory_order_relaxed);
			if (OnConnectionComplete)
			{
				OnConnectionComplete(EConnectionState::ConnectionFailed);
			}
		}

	});
}

void
FBuildServiceInstance::RefreshNamespacesAndBuckets(FOnRefreshNamespacesAndBucketsComplete&& InOnRefreshNamespacesAndBucketsComplete)
{
	{
		FWriteScopeLock _(NamespacesAndBucketsLock);
		NamespacesAndBuckets.Empty();
	}
	if (ConnectionState.load(std::memory_order_relaxed) != EConnectionState::ConnectionSucceeded)
	{
		CallOnRefreshNamespacesAndBucketsComplete(MoveTemp(InOnRefreshNamespacesAndBucketsComplete));
		return;
	}

	FString OutputFilePath = FPaths::CreateTempFilename(FPlatformProcess::UserTempDir(), TEXT("BuildContainers"), TEXT(".cbo"));
	FPaths::MakePlatformFilename(OutputFilePath);

	FString CommandLineArgs = FString::Printf(TEXT("builds list-namespaces --recursive \"%s\""),
		*OutputFilePath);

	InvokeZenUtility(CommandLineArgs, nullptr,
	[this, OutputFilePath = MoveTemp(OutputFilePath), InOnRefreshNamespacesAndBucketsComplete = MoveTemp(InOnRefreshNamespacesAndBucketsComplete)](bool bSuccessful) mutable
	{
		if (bSuccessful)
		{
			TUniquePtr<FArchive> Ar(IFileManager::Get().CreateFileReader(*OutputFilePath));
			if (!Ar)
			{
				UE_LOG(LogBuildServiceInstance, Warning, TEXT("Missing output file from zen utility when gathering build data: '%s'"), *OutputFilePath);
				CallOnRefreshNamespacesAndBucketsComplete(MoveTemp(InOnRefreshNamespacesAndBucketsComplete));
				return;
			}

			FCbObject OutputObject = LoadCompactBinary(*Ar).AsObject();
			{
				FWriteScopeLock _(NamespacesAndBucketsLock);
				for (FCbFieldView ResultField : OutputObject["results"])
				{
					FString Namespace = *WriteToString<64>(ResultField["name"].AsString());
					for (FCbFieldView Item : ResultField["items"])
					{
						NamespacesAndBuckets.Add(Namespace, *WriteToString<64>(Item.AsString()));
					}
				}
			}
		}
		CallOnRefreshNamespacesAndBucketsComplete(MoveTemp(InOnRefreshNamespacesAndBucketsComplete));
	});
}

void
FBuildServiceInstance::ListBuilds(FStringView Namespace, FStringView Bucket, FOnListBuildsComplete&& InOnListBuildsComplete)
{
	if (ConnectionState.load(std::memory_order_relaxed) != EConnectionState::ConnectionSucceeded)
	{
		if (InOnListBuildsComplete)
		{
			InOnListBuildsComplete({});
		}
		return;
	}

	FString QueryFilePath = FPaths::CreateTempFilename(FPlatformProcess::UserTempDir(), TEXT("BuildListQuery"), TEXT(".cbo"));
	FPaths::MakePlatformFilename(QueryFilePath);

	FString OutputFilePath = FPaths::CreateTempFilename(FPlatformProcess::UserTempDir(), TEXT("BuildList"), TEXT(".cbo"));
	FPaths::MakePlatformFilename(OutputFilePath);

	FString CommandLineArgs = FString::Printf(TEXT("builds list --namespace \"%s\" --bucket \"%s\" \"%s\" \"%s\""),
		*WriteToString<64>(Namespace), *WriteToString<64>(Bucket), *QueryFilePath, *OutputFilePath);

	auto PreInvoke = [QueryFilePath](FString& CommandLineArgs)
	{
		TUniquePtr<FArchive> QueryFileArchive(IFileManager::Get().CreateFileWriter(*QueryFilePath, FILEWRITE_NoFail));
		FCbWriter Writer;
		Writer.BeginObject("query");
		Writer.EndObject();
		Writer.Save(*QueryFileArchive);
	};

	InvokeZenUtility(CommandLineArgs, MoveTemp(PreInvoke),
	[OutputFilePath = MoveTemp(OutputFilePath), InOnListBuildsComplete = MoveTemp(InOnListBuildsComplete)](bool bSuccessful) mutable
	{
		if (!bSuccessful)
		{
			if (InOnListBuildsComplete)
			{
				InOnListBuildsComplete({});
			}
			return;
		}

		TUniquePtr<FArchive> Ar(IFileManager::Get().CreateFileReader(*OutputFilePath));
		if (!Ar)
		{
			UE_LOG(LogBuildServiceInstance, Warning, TEXT("Missing output file from zen utility when gathering build data: '%s'"), *OutputFilePath);
			if (InOnListBuildsComplete)
			{
				InOnListBuildsComplete({});
			}
			return;
		}

		if (InOnListBuildsComplete)
		{
			FCbObject OutputObject = LoadCompactBinary(*Ar).AsObject();
			TArray<FBuildRecord> BuildRecords;
			for (FCbFieldView ResultField : OutputObject["results"].AsArrayView())
			{
				FBuildRecord BuildRecord;
				if (LoadFromCompactBinary(ResultField, BuildRecord))
				{
					BuildRecords.Emplace(MoveTemp(BuildRecord));
				}
			}
			InOnListBuildsComplete(MoveTemp(BuildRecords));
		}

	});
}

FBuildServiceInstance::FBuildTransfer
FBuildServiceInstance::StartBuildTransfer(const FCbObjectId& BuildId, FStringView DestinationFolder, FStringView Namespace, FStringView Bucket)
{
	const FString ZenUtility = UE::Zen::GetLocalInstallUtilityPath();
	TSharedPtr<FBuildTransferCommand> NewTransferCommand = MakeShared<FBuildTransferCommand>();
	NewTransferCommand->Type = EBuildTransferType::Files;
	NewTransferCommand->BuildId = BuildId;
	NewTransferCommand->Namespace = Namespace;
	NewTransferCommand->Bucket = Bucket;
	NewTransferCommand->Destination = DestinationFolder;
	BuildTransferThreadCommands.Enqueue(NewTransferCommand.ToSharedRef());
	KickBuildTransferThread();

	FBuildTransfer NewTransfer;
	NewTransfer.BuildTransferCommand = NewTransferCommand;
	return NewTransfer;
}

FBuildServiceInstance::FBuildTransfer
FBuildServiceInstance::StartOplogBuildTransfer(const FCbObjectId& BuildId, FStringView DestinationProjectId, FStringView DestinationOplogId, FStringView ProjectFilePath, FStringView Namespace, FStringView Bucket)
{
	const FString ZenUtility = UE::Zen::GetLocalInstallUtilityPath();
	TSharedPtr<FBuildTransferCommand> NewTransferCommand = MakeShared<FBuildTransferCommand>();
	NewTransferCommand->Type = EBuildTransferType::Oplog;
	NewTransferCommand->BuildId = BuildId;
	NewTransferCommand->Namespace = Namespace;
	NewTransferCommand->Bucket = Bucket;
	NewTransferCommand->Destination = *WriteToString<64>(DestinationProjectId, TEXT("/"), DestinationOplogId);
	NewTransferCommand->ProjectFilePath = ProjectFilePath;
	BuildTransferThreadCommands.Enqueue(NewTransferCommand.ToSharedRef());
	KickBuildTransferThread();

	FBuildTransfer NewTransfer;
	NewTransfer.BuildTransferCommand = NewTransferCommand;
	return NewTransfer;
}

FString
FBuildServiceInstance::FBuildTransfer::GetDescription() const
{
	if (!BuildTransferCommand)
	{
		return TEXT("null");
	}

	FReadScopeLock _(BuildTransferCommand->Lock);
	int32 Index = 0;
	if (BuildTransferCommand->Output.FindLastChar(TCHAR('\n'), Index))
	{
		return BuildTransferCommand->Output.RightChop(Index+1);
	}
	return BuildTransferCommand->Output;
}

FBuildServiceInstance::EBuildTransferStatus
FBuildServiceInstance::FBuildTransfer::GetStatus() const
{
	if (!BuildTransferCommand)
	{
		return EBuildTransferStatus::Invalid;
	}

	FReadScopeLock _(BuildTransferCommand->Lock);
	return BuildTransferCommand->Status;
}

FBuildServiceInstance::EConnectionState
FBuildServiceInstance::GetConnectionState() const
{
	return ConnectionState.load(std::memory_order_relaxed);
}

FAnsiStringView
FBuildServiceInstance::GetEffectiveDomain() const
{
	return EffectiveDomain;
}

void
FBuildServiceInstance::Initialize()
{
#if WITH_SSL
	// Load SSL module during HTTP module's StatupModule() to make sure module manager figures out the dependencies correctly
	// and doesn't unload SSL before unloading HTTP module at exit
	FSslModule::Get();
#endif
	FDesktopPlatformModule::TryGet();
}

bool
FBuildServiceInstance::AcquireAccessToken()
{
	if (GetEffectiveDomain().StartsWith("http://localhost"))
	{
		UE_LOG(LogBuildServiceInstance, Log, TEXT("Skipping authorization for connection to localhost."));
		return true;
	}

	LoginAttempts++;

	// Avoid spamming this if the service is down.
	if (FailedLoginAttempts > UE_BUILDSERVERINSTANCE_MAX_FAILED_LOGIN_ATTEMPTS)
	{
		return false;
	}

	TRACE_CPUPROFILER_EVENT_SCOPE(BuildServiceInstance_AcquireAccessToken);

	// In case many requests wants to update the token at the same time
	// get the current serial while we wait to take the CS.
	const uint32 WantsToUpdateTokenSerial = Access ? Access->GetSerial() : 0;

	FScopeLock Lock(&AccessCs);

	// If the token was updated while we waited to take the lock, then it should now be valid.
	if (Access && Access->GetSerial() > WantsToUpdateTokenSerial)
	{
		return true;
	}

	FString AccessTokenString;
	FDateTime TokenExpiresAt;
	bool bWasInteractiveLogin = false;

	IDesktopPlatform* DesktopPlatform = FDesktopPlatformModule::TryGet();
	if (DesktopPlatform && DesktopPlatform->GetOidcAccessToken(FPaths::RootDir(), {}, Settings.GetOAuthProviderIdentifier(), FApp::IsUnattended(), GWarn, AccessTokenString, TokenExpiresAt, bWasInteractiveLogin))
	{
		if (bWasInteractiveLogin)
		{
			InteractiveLoginAttempts++;
		}

		const double ExpiryTimeSeconds = (TokenExpiresAt - FDateTime::UtcNow()).GetTotalSeconds();
		UE_LOG(LogBuildServiceInstance, Display,
			TEXT("OidcToken: Logged in to HTTP DDC services. Expires at %s which is in %.0f seconds."),
			*TokenExpiresAt.ToString(), ExpiryTimeSeconds);
		SetAccessTokenAndUnlock(Lock, AccessTokenString, ExpiryTimeSeconds);
		return true;
	}
	else if (DesktopPlatform)
	{
		UE_LOG(LogBuildServiceInstance, Warning, TEXT("OidcToken: Failed to log in to HTTP services."));
		FailedLoginAttempts++;
		return false;
	}
	else
	{
		UE_LOG(LogBuildServiceInstance, Warning, TEXT("OidcToken: Use of OAuthProviderIdentifier requires that the target depend on DesktopPlatform."));
		FailedLoginAttempts++;
		return false;
	}
	return false;
}

void
FBuildServiceInstance::SetAccessTokenAndUnlock(FScopeLock& Lock, FStringView Token, double RefreshDelay)
{
	// Cache the expired refresh handle.
	FTSTicker::FDelegateHandle ExpiredRefreshAccessTokenHandle = MoveTemp(RefreshAccessTokenHandle);
	RefreshAccessTokenHandle.Reset();

	if (!Access)
	{
		Access = MakeUnique<Private::FAccessToken>();
	}
	Access->SetToken(Settings.GetAuthScheme(), Token);

	constexpr double RefreshGracePeriod = 20.0f;
	if (RefreshDelay > RefreshGracePeriod)
	{
		// Schedule a refresh of the token ahead of expiry time (this will not work in commandlets)
		if (!IsRunningCommandlet())
		{
			RefreshAccessTokenHandle = FTSTicker::GetCoreTicker().AddTicker(FTickerDelegate::CreateLambda(
				[this](float DeltaTime)
				{
					AcquireAccessToken();
					return false;
				}
			), float(FMath::Min(RefreshDelay - RefreshGracePeriod, MAX_flt)));
		}

		// Schedule a forced refresh of the token when the scheduled refresh is starved or unavailable.
		RefreshAccessTokenTime = FPlatformTime::Seconds() + RefreshDelay - RefreshGracePeriod * 0.5f;
	}
	else
	{
		RefreshAccessTokenTime = 0.0;
	}

	// Reset failed login attempts, the service is indeed alive.
	FailedLoginAttempts = 0;

	// Unlock the critical section before attempting to remove the expired refresh handle.
	// The associated ticker delegate could already be executing, which could cause a
	// hang in RemoveTicker when the critical section is locked.
	Lock.Unlock();
	if (ExpiredRefreshAccessTokenHandle.IsValid())
	{
		FTSTicker::GetCoreTicker().RemoveTicker(MoveTemp(ExpiredRefreshAccessTokenHandle));
	}
}

FString
FBuildServiceInstance::GetAccessToken() const
{
	TAnsiStringBuilder<128> AccessTokenBuilder;
	if (Access.IsValid())
	{
		AccessTokenBuilder << *Access;
	}
	return FString(AccessTokenBuilder);
}

void
FBuildServiceInstance::InvokeZenUtility(FStringView InCommandLineArgs, FPreZenUtilityInvocation&& PreInvocation, FOnZenUtilityInvocationComplete&& OnComplete)
{
	UE::Tasks::Launch(UE_SOURCE_LOCATION,
	[this, InCommandLineArgs = FString(InCommandLineArgs), PreInvocation = MoveTemp(PreInvocation), OnComplete = MoveTemp(OnComplete)]() mutable
	{
 		FString ZenUtilityPath = UE::Zen::GetLocalInstallUtilityPath();

		FString CommandLineArgs = FString::Printf(TEXT("%s --host %s --access-token \"%s\""),
			*InCommandLineArgs, *WriteToString<64>(EffectiveDomain), *GetAccessToken());

		if (PreInvocation)
		{
			PreInvocation(CommandLineArgs);
		}

		FString AbsoluteUtilityPath = FPaths::ConvertRelativePathToFull(ZenUtilityPath);
		FMonitoredProcess MonitoredUtilityProcess(AbsoluteUtilityPath, *CommandLineArgs, FPaths::GetPath(AbsoluteUtilityPath), true);
		if (!MonitoredUtilityProcess.Launch())
		{
			UE_LOG(LogBuildServiceInstance, Warning, TEXT("Failed to launch zen utility to gather build data: '%s'."), *AbsoluteUtilityPath);
			if (OnComplete)
			{
				OnComplete(false);
			}
			return;
		}

		const uint64 StartTime = FPlatformTime::Cycles64();
		while (MonitoredUtilityProcess.Update())
		{
			double Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
			if (Duration > 120.0)
			{
				MonitoredUtilityProcess.Cancel(true);
				UE_LOG(LogBuildServiceInstance, Warning, TEXT("Cancelled launch of zen utility for gathering build data: '%s' due to timeout."), *AbsoluteUtilityPath);

				// Wait for execution to be terminated
				while (MonitoredUtilityProcess.Update())
				{
					Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
					if (Duration > 15.0)
					{
						UE_LOG(LogBuildServiceInstance, Warning, TEXT("Cancelled launch of zen utility for gathering build data: '%s'. Failed waiting for termination."), *AbsoluteUtilityPath);
						break;
					}
					FPlatformProcess::Sleep(0.2f);
				}

				FString OutputString = MonitoredUtilityProcess.GetFullOutputWithoutDelegate();
				UE_LOG(LogBuildServiceInstance, Warning, TEXT("Launch of zen utility for gathering build data: '%s' failed. Output: '%s'"), *AbsoluteUtilityPath, *OutputString);
				if (OnComplete)
				{
					OnComplete(false);
				}
				return;
			}
			FPlatformProcess::Sleep(0.1f);
		}

		FString OutputString = MonitoredUtilityProcess.GetFullOutputWithoutDelegate();
		if (MonitoredUtilityProcess.GetReturnCode() != 0)
		{
			UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unexpected return code after launch of zen utility for gathering build data: '%s' (%d). Output: '%s'"), *AbsoluteUtilityPath, MonitoredUtilityProcess.GetReturnCode(), *OutputString);
			if (OnComplete)
			{
				OnComplete(false);
			}
			return;
		}

		if (OnComplete)
		{
			OnComplete(true);
		}
	});
}

void
FBuildServiceInstance::CallOnRefreshNamespacesAndBucketsComplete(FOnRefreshNamespacesAndBucketsComplete&& InOnRefreshNamespacesAndBucketsComplete)
{
	if (InOnRefreshNamespacesAndBucketsComplete)
	{
		InOnRefreshNamespacesAndBucketsComplete();
	}

	RefreshNamespacesAndBucketsComplete.Broadcast();
}

void
FBuildServiceInstance::KickBuildTransferThread()
{
	if (!bBuildTransferThreadStarting.load(std::memory_order_relaxed) && !bBuildTransferThreadStarting.exchange(true, std::memory_order_relaxed))
	{
		BuildTransferThread = FThread(TEXT("BuildTransfer"), [this] { BuildTransferThreadLoop(); }, 128 * 1024);
	}
}

void
FBuildServiceInstance::BuildTransferThreadLoop()
{
	while (!BuildTransferThreadCommands.IsEmpty() || !bBuildTransferThreadStopping.load(std::memory_order_relaxed))
	{
		TOptional<TSharedRef<FBuildTransferCommand>> OptionalCommand = BuildTransferThreadCommands.Dequeue();
		if (!OptionalCommand)
		{
			FPlatformProcess::Sleep(0.2f);
			continue;
		}
		TSharedRef<FBuildTransferCommand> TransferCommand(OptionalCommand.GetValue());

 		FString ZenUtilityPath = FPaths::ConvertRelativePathToFull(UE::Zen::GetLocalInstallUtilityPath());
 		FString OidcTokenExecutableFilename = FPaths::ConvertRelativePathToFull(FDesktopPlatformModule::TryGet()->GetOidcTokenExecutableFilename(FPaths::RootDir()));

		FString CommandLineArgs;
		switch (TransferCommand->Type)
		{
		case EBuildTransferType::Files:
			CommandLineArgs = FString::Printf(TEXT("builds download --host %s --local-path \"%s\" --namespace %s --bucket %s --build-id %s --oidctoken-exe-path \"%s\""),
				*WriteToString<64>(EffectiveDomain), *FPaths::ConvertRelativePathToFull(TransferCommand->Destination), *TransferCommand->Namespace, *TransferCommand->Bucket,
				*WriteToString<64>(TransferCommand->BuildId), *OidcTokenExecutableFilename);
			break;
		case EBuildTransferType::Oplog:
			{
				UE::Zen::FZenLocalServiceRunContext RunContext;
				if (UE::Zen::TryGetLocalServiceRunContext(RunContext) && !TransferCommand->ProjectFilePath.IsEmpty())
				{
					UE::Zen::StartLocalService(RunContext);
				}
				FString DestinationProjectId, DestinationOplogId;
				FString SplitDelim(TEXT("/"));
				TransferCommand->Destination.Split(SplitDelim, &DestinationProjectId, &DestinationOplogId);
				CommandLineArgs = FString::Printf(TEXT("oplog-import %s %s --clean --builds %s --namespace %s --bucket %s --builds-id %s --access-token \"%s\""),
					*DestinationProjectId, *DestinationOplogId, *WriteToString<64>(EffectiveDomain),
					*TransferCommand->Namespace, *TransferCommand->Bucket,
					*WriteToString<64>(TransferCommand->BuildId), *GetAccessToken());
			}
			break;
		}

		FMonitoredProcess MonitoredUtilityProcess(ZenUtilityPath, *CommandLineArgs, FPaths::GetPath(ZenUtilityPath), true);
		MonitoredUtilityProcess.OnOutput().BindSPLambda(TransferCommand, [TransferCommand](FString Output)
		{
			FWriteScopeLock _(TransferCommand->Lock);
			TransferCommand->Output += TEXT("\n");
			TransferCommand->Output += Output;
		});

		{
			FWriteScopeLock _(TransferCommand->Lock);
			TransferCommand->Status = EBuildTransferStatus::Active;
		}
		if (!MonitoredUtilityProcess.Launch())
		{
			UE_LOG(LogBuildServiceInstance, Warning, TEXT("Failed to launch zen utility to download build: '%s'."), *ZenUtilityPath);
			FWriteScopeLock _(TransferCommand->Lock);
			TransferCommand->Status = EBuildTransferStatus::Failed;
			continue;
		}

		const uint64 StartTime = FPlatformTime::Cycles64();
		while (MonitoredUtilityProcess.Update())
		{
			if (bBuildTransferThreadStopping.load(std::memory_order_relaxed))
			{
				return;
			}
			double Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
			if (TransferCommand->bCancelRequested.load(std::memory_order_relaxed))
			{
				MonitoredUtilityProcess.Cancel(true);
				UE_LOG(LogBuildServiceInstance, Display, TEXT("Canceled launch of zen utility for downloading build data: '%s'."), *ZenUtilityPath);

				// Wait for execution to be terminated
				while (MonitoredUtilityProcess.Update())
				{
					if (bBuildTransferThreadStopping.load(std::memory_order_relaxed))
					{
						return;
					}
					Duration = FPlatformTime::ToSeconds64(FPlatformTime::Cycles64() - StartTime);
					if (Duration > 15.0)
					{
						UE_LOG(LogBuildServiceInstance, Warning, TEXT("Canceled launch of zen utility for downloading build data: '%s'. Failed waiting for termination."), *ZenUtilityPath);
						break;
					}
					FPlatformProcess::Sleep(0.2f);
				}

				UE_LOG(LogBuildServiceInstance, Display, TEXT("Launch of zen utility for downloading build data: '%s' canceled."), *ZenUtilityPath);
				FWriteScopeLock _(TransferCommand->Lock);
				TransferCommand->Status = EBuildTransferStatus::Canceled;
				continue;
			}
			FPlatformProcess::Sleep(0.1f);
		}

		FString OutputString = MonitoredUtilityProcess.GetFullOutputWithoutDelegate();
		if (MonitoredUtilityProcess.GetReturnCode() == 0)
		{
			FWriteScopeLock _(TransferCommand->Lock);
			TransferCommand->ReturnCode = MonitoredUtilityProcess.GetReturnCode();
			TransferCommand->Status = EBuildTransferStatus::Succeeded;
		}
		else
		{
			UE_LOG(LogBuildServiceInstance, Warning, TEXT("Unexpected return code after launch of zen utility for downloading build data: '%s' (%d). Output: '%s'"), *ZenUtilityPath, MonitoredUtilityProcess.GetReturnCode(), *TransferCommand->Output);
			FWriteScopeLock _(TransferCommand->Lock);
			TransferCommand->ReturnCode = MonitoredUtilityProcess.GetReturnCode();
			TransferCommand->Status = EBuildTransferStatus::Failed;
		}
	}
}

} // namespace UE::StorageService::Build