// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections; using System.Collections.Generic; using System.IO; using System.Reflection; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using EpicGames.Core; using EpicGames.OIDC; using EpicGames.Serialization; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace UnrealGameSync { /// /// A build found during search /// /// /// /// public class FoundBuildResponse(string buildName, string buildId, string commit) { public string BuildId { get; init; } = buildId; public string Commit { get; init; } = commit; public string Name { get; set; } = buildName; } /// /// Interface for interacting with UE Cloud Storage /// public interface ICloudStorage { /// /// True if the config file enables cloud storage /// /// Config file to lookup cloud storage settings in /// bool IsEnabled(ConfigFile configFile); /// /// Download a build from UE Cloud Storage based on an uri /// /// The uri /// The directory to write the build to /// Progress reporting /// A cancellation token /// Task DownloadBuildFromUriAsync(Uri uri, DirectoryReference outputDir, IProgress progress, CancellationToken cancellationToken); /// /// Download a build using all the required fields /// /// The host url /// The namespace /// The bucket /// The id of the build /// The directory to write the build to /// A optional directory were zen cli should create its state. In the output dir by default /// Progress reporting /// A cancellation token /// Task DownloadBuildAsync(string host, string namespaceId, string bucketId, string buildId, DirectoryReference outputDir, DirectoryReference? zenStateDirectory, IProgress progress, CancellationToken cancellationToken); /// /// Search for builds in UE Cloud Storage /// /// The host url /// The namespace /// The bucket /// CbObject that describes the query, see UE Cloud Storage documentation /// A cancellation token /// IAsyncEnumerable FindBuildAsync(string host, string namespaceId, string bucketId, CbObject query, CancellationToken cancellationToken); } class CloudStorage(ILogger logger, ITokenStore tokenStore) : ICloudStorage, IDisposable { private OidcTokenManager? _tokenManager; private string? _providerName; private readonly SemaphoreSlim _createOidcTokenSemaphore = new SemaphoreSlim(1, 1); public bool IsEnabled(ConfigFile configFile) { ConfigSection? section = configFile.FindSection("CloudStorage"); if (section == null) { return false; } return section.GetValue("Enabled", false); } public async Task CreateOidcTokenManagerAsync(Uri uri) { ClientAuthConfigurationV1? remoteAuthConfig = await ProviderConfigurationFactory.ReadRemoteAuthConfigurationAsync(uri, ProviderConfigurationFactory.DefaultEncryptionKey); if (remoteAuthConfig == null) { throw new NotImplementedException("Unable to fetch remote auth configuration"); } IConfiguration config = ProviderConfigurationFactory.BindOptions(remoteAuthConfig); _tokenManager = OidcTokenManager.CreateTokenManager(config, tokenStore); _providerName = remoteAuthConfig.DefaultProvider; } public async Task DownloadBuildFromUriAsync(Uri uri, DirectoryReference outputDir, IProgress progress, CancellationToken cancellationToken) { string host = $"{uri.Scheme}://{uri.Host}"; string path = uri.AbsolutePath; // remove api path if specified path = path.Replace("api/v2/builds/", "", StringComparison.OrdinalIgnoreCase); string[] components = path.Split('/', StringSplitOptions.RemoveEmptyEntries); if (components.Length != 3) { logger.LogError("Failed to base uri path {Uri}, did not match expected format of namespace/bucket/buildId", path); return true; } string namespaceId = components[0]; string bucketId = components[1]; string buildId = components[2]; return await DownloadBuildAsync(host, namespaceId, bucketId, buildId, outputDir, zenStateDirectory: null, progress, cancellationToken); } public async Task DownloadBuildAsync(string host, string namespaceId, string bucketId, string buildId, DirectoryReference outputDir, DirectoryReference? zenStateDirectory, IProgress progress, CancellationToken cancellationToken) { string? accessToken = await GetAccessToken(new Uri(host)); if (accessToken == null) { logger.LogError("Unable to determine ugs root directory so unable to find zen executable."); return true; } string zenStateArg = zenStateDirectory == null ? "" : $" --zen-folder-path {zenStateDirectory} "; string cmdline = $"builds download --host {host} --namespace {namespaceId} --bucket {bucketId} {zenStateArg} --plain-progress \"{outputDir}\" {buildId}"; return await RunZenCli(cmdline, accessToken, progress, cancellationToken); } public async Task RunZenCli(string cmdline, string accessToken, IProgress progress, CancellationToken cancellationToken) { string accessTokenEnvVar = "UE-CloudDataCacheAccessToken"; // pass the access token via environment variable to avoid showing it on the cli cmdline += $" --access-token-env {accessTokenEnvVar} "; progress.Report("Connecting to server..."); string? ugsDir = Path.GetDirectoryName(Assembly.GetEntryAssembly()?.Location); if (ugsDir == null) { logger.LogError("Unable to determine ugs root directory so unable to find zen executable."); return true; } FileInfo zenExe; switch (RuntimePlatform.Current) { case RuntimePlatform.Type.Windows: zenExe = new FileInfo(Path.Combine(ugsDir, "Binaries/Win64/zen.exe")); break; case RuntimePlatform.Type.Linux: zenExe = new FileInfo(Path.Combine(ugsDir, "Binaries/Linux/zen")); break; case RuntimePlatform.Type.Mac: zenExe = new FileInfo(Path.Combine(ugsDir, "Binaries/Mac/zen")); break; default: throw new ArgumentOutOfRangeException($"Unknown runtime platform {RuntimePlatform.Current}"); } if (!zenExe.Exists) { logger.LogError("Unable to locate zen executable at {Path} unable to run it.", zenExe); return true; } // we need to propagate the environment while also adding the access token we want to override Dictionary processEnv = new Dictionary(); foreach (DictionaryEntry entry in Environment.GetEnvironmentVariables()) { if (entry is { Key: string k, Value: string v }) { processEnv[k] = v; } } processEnv[accessTokenEnvVar] = accessToken; logger.LogInformation("Running zen cli '{ZenPath} {ZenArgs}' to download build.", zenExe.FullName, cmdline); int exitCode = await Utility.ExecuteProcessAsync(zenExe.FullName, zenExe.DirectoryName, cmdline, s => { progress.Report(s); logger.LogInformation("{Line}", s); }, processEnv, cancellationToken); if (exitCode != 0) { logger.LogError("Error running {App} {Args} shutdown with exitcode {ExitCode}", zenExe, cmdline, exitCode); } return exitCode != 0; } public async IAsyncEnumerable FindBuildAsync(string host, string namespaceId, string bucketId, CbObject query, [EnumeratorCancellation] CancellationToken cancellationToken) { string? accessToken = await GetAccessToken(new Uri(host)); if (accessToken == null) { throw new Exception("Unable to list builds as no access token was found"); } string tempInputFile = Path.GetTempFileName(); string tempOutputFile = Path.GetTempFileName(); tempInputFile = Path.ChangeExtension(tempInputFile, "cbo"); tempOutputFile = Path.ChangeExtension(tempOutputFile, "cbo"); try { await File.WriteAllBytesAsync(tempInputFile, query.GetView().ToArray(), cancellationToken); string cmdline = $"builds list --host {host} --namespace {namespaceId} --bucket {bucketId} --query-path {tempInputFile} --result-path {tempOutputFile}"; Progress progress = new Progress(); bool failed = await RunZenCli(cmdline, accessToken, progress, cancellationToken); if (failed) { logger.LogError("Failed to list builds, zen non-zero exit code"); yield break; } byte[] b = await File.ReadAllBytesAsync(tempOutputFile, cancellationToken); CbObject o = new CbObject(b); foreach (CbField result in o["results"].AsArray()) { CbObject metadata = result["metadata"].AsObject(); if (metadata.Equals(CbObject.Empty)) { // we require metadata to be able to know which commit this build belongs to continue; } string buildName = metadata["name"].AsString(); string buildId = result["buildId"].AsString(); CbField commitField = metadata["commit"]; if (commitField.Equals(CbField.Empty)) { // we require metadata to be able to know which commit this build belongs to continue; } string commit = ""; if (commitField.IsInteger()) { commit = commitField.AsInt64().ToString(); } if (commitField.IsFloat()) { // due to jsons problematic number types we sometimes get doubles for integers, these are likely changelists and thus whole numbers so we just cast them commit = ((long)commitField.AsDouble()).ToString(); } else if (commitField.IsString()) { commit = commitField.AsString(); } if (String.IsNullOrEmpty(commit) ) { // if there is no commit specified we can not know which changelist this PCB belong to continue; } yield return new FoundBuildResponse(buildName, buildId, commit); } } finally { if (File.Exists(tempInputFile)) { File.Delete(tempInputFile); } if (File.Exists(tempOutputFile)) { File.Delete(tempOutputFile); } } } private async Task GetAccessToken(Uri uri, bool allowInteractiveLogin = true) { if (_tokenManager == null) { await _createOidcTokenSemaphore.WaitAsync(); try { // check to see if it wasn't created while we waited for the semaphore #pragma warning disable CA1508 if (_tokenManager == null) #pragma warning restore CA1508 { await CreateOidcTokenManagerAsync(uri); } } finally { _createOidcTokenSemaphore.Release(); } } OidcTokenInfo tokenInfo; try { tokenInfo = await _tokenManager!.GetAccessToken(_providerName!, CancellationToken.None); } catch (NotLoggedInException) { if (allowInteractiveLogin) { tokenInfo = await _tokenManager!.LoginAsync(_providerName!, CancellationToken.None); } else { logger.LogError("Failed to get a access token"); return null; } } return tokenInfo.AccessToken; } private void Dispose(bool disposing) { if (disposing) { _createOidcTokenSemaphore.Dispose(); } } public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } ~CloudStorage() { Dispose(false); } } /// /// Extension methods for Cloud Storage /// public static class CloudStorageExtensions { /// /// Adds Horde-related services with the default settings /// /// Collection to register services with public static void AddCloudStorage(this IServiceCollection serviceCollection) { serviceCollection.AddLogging(); serviceCollection.AddSingleton(sp => TokenStoreFactory.CreateTokenStore()); serviceCollection.AddSingleton(); } /// /// Adds Horde-related services /// /// Collection to register services with /// Callback to configure options public static void AddCloudStorage(this IServiceCollection serviceCollection, Action configureHorde) { serviceCollection.Configure(configureHorde); AddCloudStorage(serviceCollection); } } public class CloudStorageOptions { } }