// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using EpicGames.Core;
using Microsoft.Extensions.Logging;
using UnrealBuildBase;
namespace UnrealBuildTool
{
///
/// Defines an interface which allows querying the working set. Files which are in the working set are excluded from unity builds, to improve iterative compile times.
///
interface ISourceFileWorkingSet : IDisposable
{
///
/// Checks if the given file is part of the working set
///
/// File to check
/// True if the file is part of the working set, false otherwise
abstract bool Contains(FileItem File);
}
///
/// Implementation of ISourceFileWorkingSet which does not contain any files
///
class EmptySourceFileWorkingSet : ISourceFileWorkingSet
{
///
/// Dispose of the current instance.
///
public void Dispose()
{
}
///
/// Checks if the given file is part of the working set
///
/// File to check
/// True if the file is part of the working set, false otherwise
public bool Contains(FileItem File)
{
return false;
}
}
///
/// Queries the working set for files tracked by Perforce.
///
class PerforceSourceFileWorkingSet : ISourceFileWorkingSet
{
///
/// Dispose of the current instance.
///
public void Dispose()
{
}
///
/// Checks if the given file is part of the working set
///
/// File to check
/// True if the file is part of the working set, false otherwise
public bool Contains(FileItem File)
{
// Generated .cpp files should never be treated as part of the working set
if (File.HasExtension(".gen.cpp"))
{
return false;
}
// Check if the file is read-only
try
{
return !File.Attributes.HasFlag(FileAttributes.ReadOnly);
}
catch (FileNotFoundException)
{
return false;
}
}
}
///
/// Queries the working set for files tracked by Git.
///
class GitSourceFileWorkingSet : ISourceFileWorkingSet
{
DirectoryReference RootDir;
Process? BackgroundProcess;
HashSet Files;
List Directories;
List ErrorOutput;
GitSourceFileWorkingSet? Inner;
ILogger Logger;
///
/// Constructor
///
/// Path to the Git executable
/// Root directory to run queries from (typically the directory containing the .git folder, to ensure all subfolders can be searched)
/// An inner working set. This allows supporting multiple Git repositories (one containing the engine, another containing the project, for example)
/// Logger for output
public GitSourceFileWorkingSet(string GitPath, DirectoryReference RootDir, GitSourceFileWorkingSet? Inner, ILogger Logger)
{
this.RootDir = RootDir;
Files = new HashSet();
Directories = new List();
ErrorOutput = new List();
this.Inner = Inner;
this.Logger = Logger;
Logger.LogInformation("Using 'git status' to determine working set for adaptive non-unity build ({RootDir}).", RootDir);
BackgroundProcess = new Process();
BackgroundProcess.StartInfo.FileName = GitPath;
BackgroundProcess.StartInfo.Arguments = "--no-optional-locks status --porcelain";
BackgroundProcess.StartInfo.WorkingDirectory = RootDir.FullName;
BackgroundProcess.StartInfo.RedirectStandardOutput = true;
BackgroundProcess.StartInfo.RedirectStandardError = true;
BackgroundProcess.StartInfo.UseShellExecute = false;
BackgroundProcess.ErrorDataReceived += ErrorDataReceived;
BackgroundProcess.OutputDataReceived += OutputDataReceived;
try
{
BackgroundProcess.Start();
BackgroundProcess.BeginErrorReadLine();
BackgroundProcess.BeginOutputReadLine();
}
catch
{
BackgroundProcess.Dispose();
BackgroundProcess = null;
}
}
///
/// Terminates the background process.
///
private void TerminateBackgroundProcess()
{
if (BackgroundProcess != null)
{
if (!BackgroundProcess.HasExited)
{
try
{
BackgroundProcess.Kill();
}
catch
{
}
}
WaitForBackgroundProcess();
}
}
///
/// Waits for the background to terminate.
///
private void WaitForBackgroundProcess()
{
if (BackgroundProcess != null)
{
if (!BackgroundProcess.WaitForExit(500))
{
Logger.LogInformation("Waiting for 'git status' command to complete");
}
if (!BackgroundProcess.WaitForExit(15000))
{
Logger.LogInformation("Terminating git child process due to timeout");
try
{
BackgroundProcess.Kill();
}
catch
{
}
}
BackgroundProcess.WaitForExit();
BackgroundProcess.Dispose();
BackgroundProcess = null;
}
}
///
/// Dispose of this object
///
public void Dispose()
{
TerminateBackgroundProcess();
Inner?.Dispose();
}
///
/// Checks if the given file is part of the working set
///
/// File to check
/// True if the file is part of the working set, false otherwise
public bool Contains(FileItem File)
{
WaitForBackgroundProcess();
if (Files.Contains(File.Location) || Directories.Any(x => File.Location.IsUnderDirectory(x)))
{
return true;
}
if (Inner != null && Inner.Contains(File))
{
return true;
}
return false;
}
///
/// Parse output text from Git
///
void OutputDataReceived(object Sender, DataReceivedEventArgs Args)
{
if (Args.Data != null && Args.Data.Length > 3 && Args.Data[2] == ' ')
{
int MinIdx = 3;
int MaxIdx = Args.Data.Length;
while (MinIdx < MaxIdx && Char.IsWhiteSpace(Args.Data[MinIdx]))
{
MinIdx++;
}
while (MinIdx < MaxIdx && Char.IsWhiteSpace(Args.Data[MaxIdx - 1]))
{
MaxIdx--;
}
int ArrowIdx = Args.Data.IndexOf(" -> ", MinIdx, MaxIdx - MinIdx);
if (ArrowIdx == -1)
{
AddPath(Args.Data.Substring(MinIdx, MaxIdx - MinIdx));
}
else
{
AddPath(Args.Data.Substring(MinIdx, ArrowIdx - MinIdx));
int ArrowEndIdx = ArrowIdx + 4;
AddPath(Args.Data.Substring(ArrowEndIdx, MaxIdx - ArrowEndIdx));
}
}
}
///
/// Handle error output text from Git
///
void ErrorDataReceived(object Sender, DataReceivedEventArgs Args)
{
if (Args.Data != null)
{
ErrorOutput.Add(Args.Data);
}
}
///
/// Add a path to the working set
///
/// Path to be added
void AddPath(string Path)
{
if (Path.EndsWith("/"))
{
Directories.Add(DirectoryReference.Combine(RootDir, Path));
}
else
{
Files.Add(FileReference.Combine(RootDir, Path));
}
}
}
///
/// Utility class for ISourceFileWorkingSet
///
static class SourceFileWorkingSet
{
enum ProviderType
{
None,
Default,
Perforce,
Git
};
///
/// Sets the provider to use for determining the working set.
///
[XmlConfigFile]
static ProviderType Provider = ProviderType.Default;
///
/// Sets the path to use for the repository. Interpreted relative to the Unreal Engine root directory (the folder above the Engine folder) -- if relative.
///
[XmlConfigFile]
public static string? RepositoryPath = null;
///
/// Sets the path to use for the Git executable. Defaults to "git" (assuming it is in the PATH).
///
[XmlConfigFile]
public static string GitPath = "git";
///
/// Create an ISourceFileWorkingSet instance suitable for the given project or root directory
///
/// The root directory
/// The project directories
/// Logger for output
/// Working set instance for the given directory
public static ISourceFileWorkingSet Create(DirectoryReference RootDir, IEnumerable ProjectDirs, ILogger Logger)
{
if (Provider == ProviderType.None || ProjectFileGenerator.bGenerateProjectFiles)
{
return new EmptySourceFileWorkingSet();
}
else if (Provider == ProviderType.Git)
{
ISourceFileWorkingSet? WorkingSet;
if (!String.IsNullOrEmpty(RepositoryPath))
{
WorkingSet = new GitSourceFileWorkingSet(GitPath, DirectoryReference.Combine(RootDir, RepositoryPath), null, Logger);
}
else if (!TryCreateGitWorkingSet(RootDir, ProjectDirs, Logger, out WorkingSet))
{
WorkingSet = new GitSourceFileWorkingSet(GitPath, RootDir, null, Logger);
}
return WorkingSet;
}
else if (Provider == ProviderType.Perforce)
{
return new PerforceSourceFileWorkingSet();
}
else
{
ISourceFileWorkingSet? WorkingSet;
if (TryCreateGitWorkingSet(RootDir, ProjectDirs, Logger, out WorkingSet))
{
return WorkingSet;
}
else if (TryCreatePerforceWorkingSet(RootDir, ProjectDirs, Logger, out WorkingSet))
{
return WorkingSet;
}
}
return new EmptySourceFileWorkingSet();
}
static bool TryCreateGitWorkingSet(DirectoryReference RootDir, IEnumerable ProjectDirs, ILogger Logger, [NotNullWhen(true)] out ISourceFileWorkingSet? OutWorkingSet)
{
GitSourceFileWorkingSet? WorkingSet = null;
// Create the working set for the engine directory
if (DirectoryReference.Exists(DirectoryReference.Combine(RootDir, ".git")) || FileReference.Exists(FileReference.Combine(RootDir, ".git")))
{
WorkingSet = new GitSourceFileWorkingSet(GitPath, RootDir, WorkingSet, Logger);
}
// Try to create a working set for the project directory
foreach (DirectoryReference ProjectDir in ProjectDirs)
{
if (WorkingSet == null || !ProjectDir.IsUnderDirectory(RootDir))
{
if (DirectoryReference.Exists(DirectoryReference.Combine(ProjectDir, ".git")) || FileReference.Exists(FileReference.Combine(ProjectDir, ".git")))
{
WorkingSet = new GitSourceFileWorkingSet(GitPath, ProjectDir, WorkingSet, Logger);
}
else if (DirectoryReference.Exists(DirectoryReference.Combine(ProjectDir.ParentDirectory!, ".git")) || FileReference.Exists(FileReference.Combine(ProjectDir.ParentDirectory!, ".git")))
{
WorkingSet = new GitSourceFileWorkingSet(GitPath, ProjectDir.ParentDirectory!, WorkingSet, Logger);
}
}
}
// Set the output value
OutWorkingSet = WorkingSet;
return OutWorkingSet != null;
}
static bool TryCreatePerforceWorkingSet(DirectoryReference RootDir, IEnumerable ProjectDirs, ILogger Logger, [NotNullWhen(true)] out ISourceFileWorkingSet? OutWorkingSet)
{
PerforceSourceFileWorkingSet? WorkingSet = null;
// If an installed engine, or the root directory contains any read-only files assume this is a perforce working set
if (Unreal.IsEngineInstalled() || DirectoryReference.EnumerateFiles(RootDir).Any(x => x.ToFileInfo().Attributes.HasFlag(FileAttributes.ReadOnly)))
{
WorkingSet = new PerforceSourceFileWorkingSet();
}
// Set the output value
OutWorkingSet = WorkingSet;
return OutWorkingSet != null;
}
}
}