// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using EpicGames.Core;
namespace EpicGames.Perforce.Managed
{
///
/// Stores the state of a directory in the workspace
///
class WorkspaceDirectoryInfo
{
///
/// The parent directory
///
public WorkspaceDirectoryInfo? ParentDirectory { get; }
///
/// Name of this directory
///
public Utf8String Name { get; }
///
/// Digest of the matching stream directory info with the base path. This should be set to zero if the workspace is modified.
///
public IoHash StreamDirectoryDigest { get; set; }
///
/// Map of name to file
///
public Dictionary NameToFile { get; set; }
///
/// Map of name to subdirectory
///
public Dictionary NameToSubDirectory { get; set; }
///
/// Constructor
///
///
public WorkspaceDirectoryInfo(DirectoryReference rootDir)
: this(null, new Utf8String(rootDir.FullName), null)
{
}
///
/// Constructor
///
/// The parent directory
/// Name of this directory
/// The corresponding stream digest
public WorkspaceDirectoryInfo(WorkspaceDirectoryInfo? parentDirectory, Utf8String name, StreamTreeRef? treeRef)
{
ParentDirectory = parentDirectory;
Name = name;
StreamDirectoryDigest = (treeRef == null) ? IoHash.Zero : treeRef.ComputeHash();
NameToFile = new Dictionary(Utf8StringComparer.Ordinal);
NameToSubDirectory = new Dictionary(FileUtils.PlatformPathComparerUtf8);
}
///
/// Adds a file to the workspace
///
/// Relative path to the file, using forward slashes, and without a leading slash
/// Length of the file on disk
/// Last modified time of the file
/// Whether the file is read only
/// Unique identifier for the server content
public void AddFile(Utf8String path, long length, long lastModifiedTicks, bool readOnly, FileContentId contentId)
{
StreamDirectoryDigest = IoHash.Zero;
int idx = path.Span.IndexOf((byte)'/');
if (idx == -1)
{
NameToFile[path] = new WorkspaceFileInfo(this, path, length, lastModifiedTicks, readOnly, contentId);
}
else
{
Utf8String name = path.Slice(0, idx);
WorkspaceDirectoryInfo? subDirectory;
if (!NameToSubDirectory.TryGetValue(name, out subDirectory))
{
subDirectory = new WorkspaceDirectoryInfo(this, name, null);
NameToSubDirectory[name] = subDirectory;
}
subDirectory.AddFile(path.Slice(idx + 1), length, lastModifiedTicks, readOnly, contentId);
}
}
///
/// Create a flat list of files in this workspace
///
/// List of files
public List GetFiles()
{
List files = new List();
GetFilesInternal(files);
return files;
}
///
/// Internal helper method for recursing through the tree to build a file list
///
///
private void GetFilesInternal(List files)
{
files.AddRange(NameToFile.Values);
foreach (KeyValuePair pair in NameToSubDirectory)
{
pair.Value.GetFilesInternal(files);
}
}
///
/// Refresh the state of the workspace on disk
///
/// Whether to remove files that are not part of the stream
/// Number of concurrent workers to use when refreshing
public async Task<(FileInfo[] filesToDelete, DirectoryInfo[] directoriesToDelete)> RefreshAsync(bool removeUntracked, int numWorkers)
{
ConcurrentQueue concurrentFilesToDelete = new ConcurrentQueue();
ConcurrentQueue concurrentDirectoriesToDelete = new ConcurrentQueue();
using AsyncThreadPoolWorkQueue queue = new(numWorkers);
await queue.EnqueueAsync(_ => RefreshAsync(new DirectoryInfo(GetFullName()), removeUntracked, concurrentFilesToDelete, concurrentDirectoriesToDelete, queue));
await queue.ExecuteAsync();
return (concurrentFilesToDelete.ToArray(), concurrentDirectoriesToDelete.ToArray());
}
///
/// Recursive method for querying the workspace state
///
///
///
///
///
///
async Task RefreshAsync(DirectoryInfo info, bool removeUntracked, ConcurrentQueue filesToDelete, ConcurrentQueue directoriesToDelete, AsyncThreadPoolWorkQueue queue)
{
// Recurse through subdirectories
Dictionary newNameToSubDirectory = new Dictionary(NameToSubDirectory.Count, NameToSubDirectory.Comparer);
// Figure out which files have changed.
Dictionary newNameToFile = new Dictionary(NameToFile.Count, NameToFile.Comparer);
foreach (FileSystemInfo fileSystemInfo in info.EnumerateFileSystemInfos())
{
if (fileSystemInfo is DirectoryInfo subDirectoryInfo)
{
WorkspaceDirectoryInfo? subDirectory;
if (NameToSubDirectory.TryGetValue(new Utf8String(subDirectoryInfo.Name), out subDirectory))
{
newNameToSubDirectory.Add(subDirectory.Name, subDirectory);
await queue.EnqueueAsync(_ => subDirectory.RefreshAsync(subDirectoryInfo, removeUntracked, filesToDelete, directoriesToDelete, queue));
}
else if (removeUntracked)
{
directoriesToDelete.Enqueue(subDirectoryInfo);
}
}
else
{
FileInfo file = (FileInfo)fileSystemInfo;
WorkspaceFileInfo? stagedFile;
if (NameToFile.TryGetValue(new Utf8String(file.Name), out stagedFile))
{
if (stagedFile.MatchesAttributes(file))
{
newNameToFile.Add(stagedFile.Name, stagedFile);
}
else
{
filesToDelete.Enqueue(file);
}
}
else
{
if (removeUntracked)
{
filesToDelete.Enqueue(file);
}
}
}
}
NameToSubDirectory = newNameToSubDirectory;
// If the file state has changed, clear the directory hashes
if (NameToFile.Count != newNameToFile.Count)
{
for (WorkspaceDirectoryInfo? directory = this; directory != null && directory.StreamDirectoryDigest != IoHash.Zero; directory = directory.ParentDirectory)
{
directory.StreamDirectoryDigest = IoHash.Zero;
}
}
// Update the new file list
NameToFile = newNameToFile;
}
///
/// Builds a list of differences from the working directory
///
///
public async Task FindDifferencesAsync(int numWorkers)
{
ConcurrentQueue paths = new();
using AsyncThreadPoolWorkQueue queue = new(numWorkers);
await queue.EnqueueAsync(_ => FindDifferencesAsync(new DirectoryInfo(GetFullName()), "/", paths, queue));
await queue.ExecuteAsync();
return paths.OrderBy(x => x).ToArray();
}
///
/// Helper method for finding differences from the working directory
///
///
///
///
///
async Task FindDifferencesAsync(DirectoryInfo directory, string path, ConcurrentQueue paths, AsyncThreadPoolWorkQueue queue)
{
// Recurse through subdirectories
HashSet remainingSubDirectoryNames = new HashSet(NameToSubDirectory.Keys);
foreach (DirectoryInfo subDirectory in directory.EnumerateDirectories())
{
WorkspaceDirectoryInfo? stagedSubDirectory;
if (NameToSubDirectory.TryGetValue(new Utf8String(subDirectory.Name), out stagedSubDirectory))
{
remainingSubDirectoryNames.Remove(new Utf8String(subDirectory.Name));
await queue.EnqueueAsync(_ => stagedSubDirectory.FindDifferencesAsync(subDirectory, String.Format("{0}{1}/", path, subDirectory.Name), paths, queue));
continue;
}
paths.Enqueue(String.Format("+{0}{1}/...", path, subDirectory.Name));
}
foreach (Utf8String remainingSubDirectoryName in remainingSubDirectoryNames)
{
paths.Enqueue(String.Format("-{0}{1}/...", path, remainingSubDirectoryName));
}
// Search through files
HashSet remainingFileNames = new HashSet(NameToFile.Keys);
foreach (FileInfo file in directory.EnumerateFiles())
{
WorkspaceFileInfo? stagedFile;
if (!NameToFile.TryGetValue(new Utf8String(file.Name), out stagedFile))
{
paths.Enqueue(String.Format("+{0}{1}", path, file.Name));
}
else if (!stagedFile.MatchesAttributes(file))
{
paths.Enqueue(String.Format("!{0}{1}", path, file.Name));
remainingFileNames.Remove(new Utf8String(file.Name));
}
else
{
remainingFileNames.Remove(new Utf8String(file.Name));
}
}
foreach (Utf8String remainingFileName in remainingFileNames)
{
paths.Enqueue(String.Format("-{0}{1}", path, remainingFileName));
}
}
///
/// Get the full path to this directory
///
///
public string GetFullName()
{
StringBuilder builder = new StringBuilder();
AppendFullPath(builder);
return builder.ToString();
}
///
/// Get the path to this directory
///
///
public DirectoryReference GetLocation()
{
return new DirectoryReference(GetFullName());
}
///
/// Append the client path, using native directory separators, to the given string builder
///
///
public void AppendClientPath(StringBuilder builder)
{
if (ParentDirectory != null)
{
ParentDirectory.AppendClientPath(builder);
builder.Append(Name);
builder.Append(Path.DirectorySeparatorChar);
}
}
///
/// Append the path for this directory to the given string builder
///
///
public void AppendFullPath(StringBuilder builder)
{
if (ParentDirectory != null)
{
ParentDirectory.AppendFullPath(builder);
builder.Append(Path.DirectorySeparatorChar);
}
builder.Append(Name);
}
///
public override string ToString()
{
return GetFullName();
}
}
///
/// Extension methods for WorkspaceDirectoryInfo
///
static class WorkspaceDirectoryInfoExtensions
{
public static void ReadWorkspaceDirectoryInfo(this MemoryReader reader, WorkspaceDirectoryInfo directoryInfo, ManagedWorkspaceVersion version)
{
if (version < ManagedWorkspaceVersion.AddDigest)
{
directoryInfo.StreamDirectoryDigest = IoHash.Zero;
}
else if (version < ManagedWorkspaceVersion.AddDigestIoHash)
{
reader.ReadFixedLengthBytes(Sha1.Length);
directoryInfo.StreamDirectoryDigest = IoHash.Zero;
}
else
{
directoryInfo.StreamDirectoryDigest = reader.ReadIoHash();
}
int numFiles = reader.ReadInt32();
for (int idx = 0; idx < numFiles; idx++)
{
WorkspaceFileInfo fileInfo = reader.ReadWorkspaceFileInfo(directoryInfo);
directoryInfo.NameToFile.Add(fileInfo.Name, fileInfo);
}
int numSubDirectories = reader.ReadInt32();
for (int idx = 0; idx < numSubDirectories; idx++)
{
Utf8String name = reader.ReadNullTerminatedUtf8String();
WorkspaceDirectoryInfo subDirectory = new WorkspaceDirectoryInfo(directoryInfo, name, null);
reader.ReadWorkspaceDirectoryInfo(subDirectory, version);
directoryInfo.NameToSubDirectory[subDirectory.Name] = subDirectory;
}
}
public static void WriteWorkspaceDirectoryInfo(this MemoryWriter writer, WorkspaceDirectoryInfo directoryInfo)
{
writer.WriteIoHash(directoryInfo.StreamDirectoryDigest);
writer.WriteInt32(directoryInfo.NameToFile.Count);
foreach (WorkspaceFileInfo file in directoryInfo.NameToFile.Values)
{
writer.WriteWorkspaceFileInfo(file);
}
writer.WriteInt32(directoryInfo.NameToSubDirectory.Count);
foreach (WorkspaceDirectoryInfo subDirectory in directoryInfo.NameToSubDirectory.Values)
{
writer.WriteNullTerminatedUtf8String(subDirectory.Name);
writer.WriteWorkspaceDirectoryInfo(subDirectory);
}
}
public static int GetSerializedSize(this WorkspaceDirectoryInfo directoryInfo)
{
return Digest.Length + sizeof(int) + directoryInfo.NameToFile.Values.Sum(x => x.GetSerializedSize()) + sizeof(int) + directoryInfo.NameToSubDirectory.Values.Sum(x => x.Name.GetNullTerminatedSize() + x.GetSerializedSize());
}
}
}