// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using EpicGames.Core; namespace EpicGames.Perforce.Managed { /// /// Stores the state of a directory in the workspace /// class WorkspaceDirectoryInfo { /// /// The parent directory /// public WorkspaceDirectoryInfo? ParentDirectory { get; } /// /// Name of this directory /// public Utf8String Name { get; } /// /// Digest of the matching stream directory info with the base path. This should be set to zero if the workspace is modified. /// public IoHash StreamDirectoryDigest { get; set; } /// /// Map of name to file /// public Dictionary NameToFile { get; set; } /// /// Map of name to subdirectory /// public Dictionary NameToSubDirectory { get; set; } /// /// Constructor /// /// public WorkspaceDirectoryInfo(DirectoryReference rootDir) : this(null, new Utf8String(rootDir.FullName), null) { } /// /// Constructor /// /// The parent directory /// Name of this directory /// The corresponding stream digest public WorkspaceDirectoryInfo(WorkspaceDirectoryInfo? parentDirectory, Utf8String name, StreamTreeRef? treeRef) { ParentDirectory = parentDirectory; Name = name; StreamDirectoryDigest = (treeRef == null) ? IoHash.Zero : treeRef.ComputeHash(); NameToFile = new Dictionary(Utf8StringComparer.Ordinal); NameToSubDirectory = new Dictionary(FileUtils.PlatformPathComparerUtf8); } /// /// Adds a file to the workspace /// /// Relative path to the file, using forward slashes, and without a leading slash /// Length of the file on disk /// Last modified time of the file /// Whether the file is read only /// Unique identifier for the server content public void AddFile(Utf8String path, long length, long lastModifiedTicks, bool readOnly, FileContentId contentId) { StreamDirectoryDigest = IoHash.Zero; int idx = path.Span.IndexOf((byte)'/'); if (idx == -1) { NameToFile[path] = new WorkspaceFileInfo(this, path, length, lastModifiedTicks, readOnly, contentId); } else { Utf8String name = path.Slice(0, idx); WorkspaceDirectoryInfo? subDirectory; if (!NameToSubDirectory.TryGetValue(name, out subDirectory)) { subDirectory = new WorkspaceDirectoryInfo(this, name, null); NameToSubDirectory[name] = subDirectory; } subDirectory.AddFile(path.Slice(idx + 1), length, lastModifiedTicks, readOnly, contentId); } } /// /// Create a flat list of files in this workspace /// /// List of files public List GetFiles() { List files = new List(); GetFilesInternal(files); return files; } /// /// Internal helper method for recursing through the tree to build a file list /// /// private void GetFilesInternal(List files) { files.AddRange(NameToFile.Values); foreach (KeyValuePair pair in NameToSubDirectory) { pair.Value.GetFilesInternal(files); } } /// /// Refresh the state of the workspace on disk /// /// Whether to remove files that are not part of the stream /// Number of concurrent workers to use when refreshing public async Task<(FileInfo[] filesToDelete, DirectoryInfo[] directoriesToDelete)> RefreshAsync(bool removeUntracked, int numWorkers) { ConcurrentQueue concurrentFilesToDelete = new ConcurrentQueue(); ConcurrentQueue concurrentDirectoriesToDelete = new ConcurrentQueue(); using AsyncThreadPoolWorkQueue queue = new(numWorkers); await queue.EnqueueAsync(_ => RefreshAsync(new DirectoryInfo(GetFullName()), removeUntracked, concurrentFilesToDelete, concurrentDirectoriesToDelete, queue)); await queue.ExecuteAsync(); return (concurrentFilesToDelete.ToArray(), concurrentDirectoriesToDelete.ToArray()); } /// /// Recursive method for querying the workspace state /// /// /// /// /// /// async Task RefreshAsync(DirectoryInfo info, bool removeUntracked, ConcurrentQueue filesToDelete, ConcurrentQueue directoriesToDelete, AsyncThreadPoolWorkQueue queue) { // Recurse through subdirectories Dictionary newNameToSubDirectory = new Dictionary(NameToSubDirectory.Count, NameToSubDirectory.Comparer); // Figure out which files have changed. Dictionary newNameToFile = new Dictionary(NameToFile.Count, NameToFile.Comparer); foreach (FileSystemInfo fileSystemInfo in info.EnumerateFileSystemInfos()) { if (fileSystemInfo is DirectoryInfo subDirectoryInfo) { WorkspaceDirectoryInfo? subDirectory; if (NameToSubDirectory.TryGetValue(new Utf8String(subDirectoryInfo.Name), out subDirectory)) { newNameToSubDirectory.Add(subDirectory.Name, subDirectory); await queue.EnqueueAsync(_ => subDirectory.RefreshAsync(subDirectoryInfo, removeUntracked, filesToDelete, directoriesToDelete, queue)); } else if (removeUntracked) { directoriesToDelete.Enqueue(subDirectoryInfo); } } else { FileInfo file = (FileInfo)fileSystemInfo; WorkspaceFileInfo? stagedFile; if (NameToFile.TryGetValue(new Utf8String(file.Name), out stagedFile)) { if (stagedFile.MatchesAttributes(file)) { newNameToFile.Add(stagedFile.Name, stagedFile); } else { filesToDelete.Enqueue(file); } } else { if (removeUntracked) { filesToDelete.Enqueue(file); } } } } NameToSubDirectory = newNameToSubDirectory; // If the file state has changed, clear the directory hashes if (NameToFile.Count != newNameToFile.Count) { for (WorkspaceDirectoryInfo? directory = this; directory != null && directory.StreamDirectoryDigest != IoHash.Zero; directory = directory.ParentDirectory) { directory.StreamDirectoryDigest = IoHash.Zero; } } // Update the new file list NameToFile = newNameToFile; } /// /// Builds a list of differences from the working directory /// /// public async Task FindDifferencesAsync(int numWorkers) { ConcurrentQueue paths = new(); using AsyncThreadPoolWorkQueue queue = new(numWorkers); await queue.EnqueueAsync(_ => FindDifferencesAsync(new DirectoryInfo(GetFullName()), "/", paths, queue)); await queue.ExecuteAsync(); return paths.OrderBy(x => x).ToArray(); } /// /// Helper method for finding differences from the working directory /// /// /// /// /// async Task FindDifferencesAsync(DirectoryInfo directory, string path, ConcurrentQueue paths, AsyncThreadPoolWorkQueue queue) { // Recurse through subdirectories HashSet remainingSubDirectoryNames = new HashSet(NameToSubDirectory.Keys); foreach (DirectoryInfo subDirectory in directory.EnumerateDirectories()) { WorkspaceDirectoryInfo? stagedSubDirectory; if (NameToSubDirectory.TryGetValue(new Utf8String(subDirectory.Name), out stagedSubDirectory)) { remainingSubDirectoryNames.Remove(new Utf8String(subDirectory.Name)); await queue.EnqueueAsync(_ => stagedSubDirectory.FindDifferencesAsync(subDirectory, String.Format("{0}{1}/", path, subDirectory.Name), paths, queue)); continue; } paths.Enqueue(String.Format("+{0}{1}/...", path, subDirectory.Name)); } foreach (Utf8String remainingSubDirectoryName in remainingSubDirectoryNames) { paths.Enqueue(String.Format("-{0}{1}/...", path, remainingSubDirectoryName)); } // Search through files HashSet remainingFileNames = new HashSet(NameToFile.Keys); foreach (FileInfo file in directory.EnumerateFiles()) { WorkspaceFileInfo? stagedFile; if (!NameToFile.TryGetValue(new Utf8String(file.Name), out stagedFile)) { paths.Enqueue(String.Format("+{0}{1}", path, file.Name)); } else if (!stagedFile.MatchesAttributes(file)) { paths.Enqueue(String.Format("!{0}{1}", path, file.Name)); remainingFileNames.Remove(new Utf8String(file.Name)); } else { remainingFileNames.Remove(new Utf8String(file.Name)); } } foreach (Utf8String remainingFileName in remainingFileNames) { paths.Enqueue(String.Format("-{0}{1}", path, remainingFileName)); } } /// /// Get the full path to this directory /// /// public string GetFullName() { StringBuilder builder = new StringBuilder(); AppendFullPath(builder); return builder.ToString(); } /// /// Get the path to this directory /// /// public DirectoryReference GetLocation() { return new DirectoryReference(GetFullName()); } /// /// Append the client path, using native directory separators, to the given string builder /// /// public void AppendClientPath(StringBuilder builder) { if (ParentDirectory != null) { ParentDirectory.AppendClientPath(builder); builder.Append(Name); builder.Append(Path.DirectorySeparatorChar); } } /// /// Append the path for this directory to the given string builder /// /// public void AppendFullPath(StringBuilder builder) { if (ParentDirectory != null) { ParentDirectory.AppendFullPath(builder); builder.Append(Path.DirectorySeparatorChar); } builder.Append(Name); } /// public override string ToString() { return GetFullName(); } } /// /// Extension methods for WorkspaceDirectoryInfo /// static class WorkspaceDirectoryInfoExtensions { public static void ReadWorkspaceDirectoryInfo(this MemoryReader reader, WorkspaceDirectoryInfo directoryInfo, ManagedWorkspaceVersion version) { if (version < ManagedWorkspaceVersion.AddDigest) { directoryInfo.StreamDirectoryDigest = IoHash.Zero; } else if (version < ManagedWorkspaceVersion.AddDigestIoHash) { reader.ReadFixedLengthBytes(Sha1.Length); directoryInfo.StreamDirectoryDigest = IoHash.Zero; } else { directoryInfo.StreamDirectoryDigest = reader.ReadIoHash(); } int numFiles = reader.ReadInt32(); for (int idx = 0; idx < numFiles; idx++) { WorkspaceFileInfo fileInfo = reader.ReadWorkspaceFileInfo(directoryInfo); directoryInfo.NameToFile.Add(fileInfo.Name, fileInfo); } int numSubDirectories = reader.ReadInt32(); for (int idx = 0; idx < numSubDirectories; idx++) { Utf8String name = reader.ReadNullTerminatedUtf8String(); WorkspaceDirectoryInfo subDirectory = new WorkspaceDirectoryInfo(directoryInfo, name, null); reader.ReadWorkspaceDirectoryInfo(subDirectory, version); directoryInfo.NameToSubDirectory[subDirectory.Name] = subDirectory; } } public static void WriteWorkspaceDirectoryInfo(this MemoryWriter writer, WorkspaceDirectoryInfo directoryInfo) { writer.WriteIoHash(directoryInfo.StreamDirectoryDigest); writer.WriteInt32(directoryInfo.NameToFile.Count); foreach (WorkspaceFileInfo file in directoryInfo.NameToFile.Values) { writer.WriteWorkspaceFileInfo(file); } writer.WriteInt32(directoryInfo.NameToSubDirectory.Count); foreach (WorkspaceDirectoryInfo subDirectory in directoryInfo.NameToSubDirectory.Values) { writer.WriteNullTerminatedUtf8String(subDirectory.Name); writer.WriteWorkspaceDirectoryInfo(subDirectory); } } public static int GetSerializedSize(this WorkspaceDirectoryInfo directoryInfo) { return Digest.Length + sizeof(int) + directoryInfo.NameToFile.Values.Sum(x => x.GetSerializedSize()) + sizeof(int) + directoryInfo.NameToSubDirectory.Values.Sum(x => x.Name.GetNullTerminatedSize() + x.GetSerializedSize()); } } }