// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using EpicGames.Core; using Microsoft.Extensions.Logging; using UnrealBuildBase; namespace UnrealBuildTool { /// /// Defines an interface which allows querying the working set. Files which are in the working set are excluded from unity builds, to improve iterative compile times. /// interface ISourceFileWorkingSet : IDisposable { /// /// Checks if the given file is part of the working set /// /// File to check /// True if the file is part of the working set, false otherwise abstract bool Contains(FileItem File); } /// /// Implementation of ISourceFileWorkingSet which does not contain any files /// class EmptySourceFileWorkingSet : ISourceFileWorkingSet { /// /// Dispose of the current instance. /// public void Dispose() { } /// /// Checks if the given file is part of the working set /// /// File to check /// True if the file is part of the working set, false otherwise public bool Contains(FileItem File) { return false; } } /// /// Queries the working set for files tracked by Perforce. /// class PerforceSourceFileWorkingSet : ISourceFileWorkingSet { /// /// Dispose of the current instance. /// public void Dispose() { } /// /// Checks if the given file is part of the working set /// /// File to check /// True if the file is part of the working set, false otherwise public bool Contains(FileItem File) { // Generated .cpp files should never be treated as part of the working set if (File.HasExtension(".gen.cpp")) { return false; } // Check if the file is read-only try { return !File.Attributes.HasFlag(FileAttributes.ReadOnly); } catch (FileNotFoundException) { return false; } } } /// /// Queries the working set for files tracked by Git. /// class GitSourceFileWorkingSet : ISourceFileWorkingSet { DirectoryReference RootDir; Process? BackgroundProcess; HashSet Files; List Directories; List ErrorOutput; GitSourceFileWorkingSet? Inner; ILogger Logger; /// /// Constructor /// /// Path to the Git executable /// Root directory to run queries from (typically the directory containing the .git folder, to ensure all subfolders can be searched) /// An inner working set. This allows supporting multiple Git repositories (one containing the engine, another containing the project, for example) /// Logger for output public GitSourceFileWorkingSet(string GitPath, DirectoryReference RootDir, GitSourceFileWorkingSet? Inner, ILogger Logger) { this.RootDir = RootDir; Files = new HashSet(); Directories = new List(); ErrorOutput = new List(); this.Inner = Inner; this.Logger = Logger; Logger.LogInformation("Using 'git status' to determine working set for adaptive non-unity build ({RootDir}).", RootDir); BackgroundProcess = new Process(); BackgroundProcess.StartInfo.FileName = GitPath; BackgroundProcess.StartInfo.Arguments = "--no-optional-locks status --porcelain"; BackgroundProcess.StartInfo.WorkingDirectory = RootDir.FullName; BackgroundProcess.StartInfo.RedirectStandardOutput = true; BackgroundProcess.StartInfo.RedirectStandardError = true; BackgroundProcess.StartInfo.UseShellExecute = false; BackgroundProcess.ErrorDataReceived += ErrorDataReceived; BackgroundProcess.OutputDataReceived += OutputDataReceived; try { BackgroundProcess.Start(); BackgroundProcess.BeginErrorReadLine(); BackgroundProcess.BeginOutputReadLine(); } catch { BackgroundProcess.Dispose(); BackgroundProcess = null; } } /// /// Terminates the background process. /// private void TerminateBackgroundProcess() { if (BackgroundProcess != null) { if (!BackgroundProcess.HasExited) { try { BackgroundProcess.Kill(); } catch { } } WaitForBackgroundProcess(); } } /// /// Waits for the background to terminate. /// private void WaitForBackgroundProcess() { if (BackgroundProcess != null) { if (!BackgroundProcess.WaitForExit(500)) { Logger.LogInformation("Waiting for 'git status' command to complete"); } if (!BackgroundProcess.WaitForExit(15000)) { Logger.LogInformation("Terminating git child process due to timeout"); try { BackgroundProcess.Kill(); } catch { } } BackgroundProcess.WaitForExit(); BackgroundProcess.Dispose(); BackgroundProcess = null; } } /// /// Dispose of this object /// public void Dispose() { TerminateBackgroundProcess(); Inner?.Dispose(); } /// /// Checks if the given file is part of the working set /// /// File to check /// True if the file is part of the working set, false otherwise public bool Contains(FileItem File) { WaitForBackgroundProcess(); if (Files.Contains(File.Location) || Directories.Any(x => File.Location.IsUnderDirectory(x))) { return true; } if (Inner != null && Inner.Contains(File)) { return true; } return false; } /// /// Parse output text from Git /// void OutputDataReceived(object Sender, DataReceivedEventArgs Args) { if (Args.Data != null && Args.Data.Length > 3 && Args.Data[2] == ' ') { int MinIdx = 3; int MaxIdx = Args.Data.Length; while (MinIdx < MaxIdx && Char.IsWhiteSpace(Args.Data[MinIdx])) { MinIdx++; } while (MinIdx < MaxIdx && Char.IsWhiteSpace(Args.Data[MaxIdx - 1])) { MaxIdx--; } int ArrowIdx = Args.Data.IndexOf(" -> ", MinIdx, MaxIdx - MinIdx); if (ArrowIdx == -1) { AddPath(Args.Data.Substring(MinIdx, MaxIdx - MinIdx)); } else { AddPath(Args.Data.Substring(MinIdx, ArrowIdx - MinIdx)); int ArrowEndIdx = ArrowIdx + 4; AddPath(Args.Data.Substring(ArrowEndIdx, MaxIdx - ArrowEndIdx)); } } } /// /// Handle error output text from Git /// void ErrorDataReceived(object Sender, DataReceivedEventArgs Args) { if (Args.Data != null) { ErrorOutput.Add(Args.Data); } } /// /// Add a path to the working set /// /// Path to be added void AddPath(string Path) { if (Path.EndsWith("/")) { Directories.Add(DirectoryReference.Combine(RootDir, Path)); } else { Files.Add(FileReference.Combine(RootDir, Path)); } } } /// /// Utility class for ISourceFileWorkingSet /// static class SourceFileWorkingSet { enum ProviderType { None, Default, Perforce, Git }; /// /// Sets the provider to use for determining the working set. /// [XmlConfigFile] static ProviderType Provider = ProviderType.Default; /// /// Sets the path to use for the repository. Interpreted relative to the Unreal Engine root directory (the folder above the Engine folder) -- if relative. /// [XmlConfigFile] public static string? RepositoryPath = null; /// /// Sets the path to use for the Git executable. Defaults to "git" (assuming it is in the PATH). /// [XmlConfigFile] public static string GitPath = "git"; /// /// Create an ISourceFileWorkingSet instance suitable for the given project or root directory /// /// The root directory /// The project directories /// Logger for output /// Working set instance for the given directory public static ISourceFileWorkingSet Create(DirectoryReference RootDir, IEnumerable ProjectDirs, ILogger Logger) { if (Provider == ProviderType.None || ProjectFileGenerator.bGenerateProjectFiles) { return new EmptySourceFileWorkingSet(); } else if (Provider == ProviderType.Git) { ISourceFileWorkingSet? WorkingSet; if (!String.IsNullOrEmpty(RepositoryPath)) { WorkingSet = new GitSourceFileWorkingSet(GitPath, DirectoryReference.Combine(RootDir, RepositoryPath), null, Logger); } else if (!TryCreateGitWorkingSet(RootDir, ProjectDirs, Logger, out WorkingSet)) { WorkingSet = new GitSourceFileWorkingSet(GitPath, RootDir, null, Logger); } return WorkingSet; } else if (Provider == ProviderType.Perforce) { return new PerforceSourceFileWorkingSet(); } else { ISourceFileWorkingSet? WorkingSet; if (TryCreateGitWorkingSet(RootDir, ProjectDirs, Logger, out WorkingSet)) { return WorkingSet; } else if (TryCreatePerforceWorkingSet(RootDir, ProjectDirs, Logger, out WorkingSet)) { return WorkingSet; } } return new EmptySourceFileWorkingSet(); } static bool TryCreateGitWorkingSet(DirectoryReference RootDir, IEnumerable ProjectDirs, ILogger Logger, [NotNullWhen(true)] out ISourceFileWorkingSet? OutWorkingSet) { GitSourceFileWorkingSet? WorkingSet = null; // Create the working set for the engine directory if (DirectoryReference.Exists(DirectoryReference.Combine(RootDir, ".git")) || FileReference.Exists(FileReference.Combine(RootDir, ".git"))) { WorkingSet = new GitSourceFileWorkingSet(GitPath, RootDir, WorkingSet, Logger); } // Try to create a working set for the project directory foreach (DirectoryReference ProjectDir in ProjectDirs) { if (WorkingSet == null || !ProjectDir.IsUnderDirectory(RootDir)) { if (DirectoryReference.Exists(DirectoryReference.Combine(ProjectDir, ".git")) || FileReference.Exists(FileReference.Combine(ProjectDir, ".git"))) { WorkingSet = new GitSourceFileWorkingSet(GitPath, ProjectDir, WorkingSet, Logger); } else if (DirectoryReference.Exists(DirectoryReference.Combine(ProjectDir.ParentDirectory!, ".git")) || FileReference.Exists(FileReference.Combine(ProjectDir.ParentDirectory!, ".git"))) { WorkingSet = new GitSourceFileWorkingSet(GitPath, ProjectDir.ParentDirectory!, WorkingSet, Logger); } } } // Set the output value OutWorkingSet = WorkingSet; return OutWorkingSet != null; } static bool TryCreatePerforceWorkingSet(DirectoryReference RootDir, IEnumerable ProjectDirs, ILogger Logger, [NotNullWhen(true)] out ISourceFileWorkingSet? OutWorkingSet) { PerforceSourceFileWorkingSet? WorkingSet = null; // If an installed engine, or the root directory contains any read-only files assume this is a perforce working set if (Unreal.IsEngineInstalled() || DirectoryReference.EnumerateFiles(RootDir).Any(x => x.ToFileInfo().Attributes.HasFlag(FileAttributes.ReadOnly))) { WorkingSet = new PerforceSourceFileWorkingSet(); } // Set the output value OutWorkingSet = WorkingSet; return OutWorkingSet != null; } } }