Files
UnrealEngine/Engine/Source/Programs/Unsync/Private/UnsyncCore.cpp
2025-05-18 13:04:45 +08:00

2298 lines
68 KiB
C++

// Copyright Epic Games, Inc. All Rights Reserved.
#include "UnsyncCommon.h"
#include "UnsyncChunking.h"
#include "UnsyncCompression.h"
#include "UnsyncCore.h"
#include "UnsyncDiff.h"
#include "UnsyncFile.h"
#include "UnsyncHashTable.h"
#include "UnsyncHttp.h"
#include "UnsyncProgress.h"
#include "UnsyncProxy.h"
#include "UnsyncScan.h"
#include "UnsyncScavenger.h"
#include "UnsyncScheduler.h"
#include "UnsyncSerialization.h"
#include "UnsyncTarget.h"
#include "UnsyncThread.h"
#include "UnsyncUtil.h"
#include "UnsyncVersion.h"
#include "UnsyncFilter.h"
#include "UnsyncSource.h"
#include "UnsyncPack.h"
#include <condition_variable>
#include <filesystem>
#include <mutex>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <optional>
UNSYNC_THIRD_PARTY_INCLUDES_START
#include <blake3.h>
#include <md5-sse2.h>
UNSYNC_THIRD_PARTY_INCLUDES_END
namespace unsync {
bool GDryRun = false;
bool GExperimental = false;
bool GExperimentalStreaming = false;
FBlock128
ToBlock128(const FGenericBlock& GenericBlock)
{
FBlock128 Result;
Result.HashStrong = GenericBlock.HashStrong.ToHash128();
Result.HashWeak = GenericBlock.HashWeak;
Result.Offset = GenericBlock.Offset;
Result.Size = GenericBlock.Size;
return Result;
}
std::vector<FBlock128>
ToBlock128(FGenericBlockArray& GenericBlocks)
{
std::vector<FBlock128> Result;
Result.reserve(GenericBlocks.size());
for (const FGenericBlock& It : GenericBlocks)
{
Result.push_back(ToBlock128(It));
}
return Result;
}
std::vector<FCopyCommand>
OptimizeNeedList(const std::vector<FNeedBlock>& Input, uint64 MaxMergedBlockSize)
{
std::vector<FCopyCommand> Result;
Result.reserve(Input.size());
for (const FNeedBlock& Block : Input)
{
FCopyCommand Cmd;
Cmd.SourceOffset = Block.SourceOffset;
Cmd.TargetOffset = Block.TargetOffset;
Cmd.Size = Block.Size;
Result.push_back(Cmd);
}
std::sort(Result.begin(), Result.end(), FCopyCommand::FCompareBySourceOffset());
for (uint64 I = 1; I < Result.size(); ++I)
{
FCopyCommand& PrevBlock = Result[I - 1];
FCopyCommand& ThisBlock = Result[I];
if (PrevBlock.SourceOffset + PrevBlock.Size == ThisBlock.SourceOffset &&
PrevBlock.TargetOffset + PrevBlock.Size == ThisBlock.TargetOffset && PrevBlock.Size + ThisBlock.Size <= MaxMergedBlockSize)
{
ThisBlock.SourceOffset = PrevBlock.SourceOffset;
ThisBlock.TargetOffset = PrevBlock.TargetOffset;
ThisBlock.Size += PrevBlock.Size;
UNSYNC_ASSERT(ThisBlock.Size <= MaxMergedBlockSize);
PrevBlock.Size = 0;
}
}
for (uint64 I = 0; I < Result.size(); ++I)
{
UNSYNC_ASSERT(Result[I].Size <= MaxMergedBlockSize);
}
auto It = std::remove_if(Result.begin(), Result.end(), [](const FCopyCommand& Block) { return Block.Size == 0; });
Result.erase(It, Result.end());
return Result;
}
FReadSchedule
BuildReadSchedule(const std::vector<FNeedBlock>& Blocks)
{
FReadSchedule Result;
Result.Blocks = OptimizeNeedList(Blocks);
std::sort(Result.Blocks.begin(), Result.Blocks.end(), [](const FCopyCommand& A, const FCopyCommand& B) {
if (A.Size == B.Size)
{
return A.SourceOffset < B.SourceOffset;
}
else
{
return A.Size < B.Size;
}
});
for (uint64 I = 0; I < Result.Blocks.size(); ++I)
{
Result.Requests.push_back(I);
}
return Result;
}
bool
IsSynchronized(const FNeedList& NeedList, const FGenericBlockArray& SourceBlocks)
{
if (NeedList.Source.size() != 0)
{
return false;
}
if (NeedList.Base.size() != SourceBlocks.size())
{
return false;
}
if (NeedList.Sequence.size() != SourceBlocks.size())
{
return false;
}
for (uint64 I = 0; I < SourceBlocks.size(); ++I)
{
if (NeedList.Sequence[I] != SourceBlocks[I].HashStrong.ToHash128()) // #wip-widehash
{
return false;
}
}
return true;
}
bool
ValidateTarget(FIOReader& Reader, const FNeedList& NeedList, EStrongHashAlgorithmID StrongHasher)
{
FGenericBlockArray ValidationBlocks;
for (const FNeedBlock& It : NeedList.Source)
{
FGenericBlock Block;
Block.Size = CheckedNarrow(It.Size);
Block.Offset = It.TargetOffset;
Block.HashStrong = It.Hash;
ValidationBlocks.push_back(Block);
}
for (const FNeedBlock& It : NeedList.Base)
{
FGenericBlock Block;
Block.Size = CheckedNarrow(It.Size);
Block.Offset = It.TargetOffset;
Block.HashStrong = It.Hash;
ValidationBlocks.push_back(Block);
}
std::sort(ValidationBlocks.begin(),
ValidationBlocks.end(),
[](const FGenericBlock& A, const FGenericBlock& B) { return A.Offset < B.Offset; });
return ValidateTarget(Reader, ValidationBlocks, StrongHasher);
}
bool
ValidateTarget(FIOReader& Reader, const FGenericBlockArray& ValidationBlocks, EStrongHashAlgorithmID StrongHasher)
{
const uint64 TotalStreamBytes = Reader.GetSize();
std::atomic<uint64> NumInvalidBlocks = {};
FSchedulerSemaphore IoSemaphore(*GScheduler, 16);
FTaskGroup TaskGroup = GScheduler->CreateTaskGroup(&IoSemaphore);
FLogProgressScope ValidationProgressLogger(TotalStreamBytes, ELogProgressUnits::MB);
// Inherit verbosity and indentation from parent theread
// TODO: make a helper that sets verbosity and indentation automatically
const bool bLogVerbose = GLogVerbose;
const uint32 LogIndent = GLogIndent;
uint64 MaxBatchSizeBytes = 8_MB;
uint64 BatchBegin = 0;
uint64 BatchSizeBytes = 0;
std::unique_ptr<FAsyncReader> AsyncReader = Reader.CreateAsyncReader();
for (uint64 BlockIndex = 0; BlockIndex < ValidationBlocks.size(); ++BlockIndex)
{
const FGenericBlock& CurrBlock = ValidationBlocks[BlockIndex];
if (BlockIndex > 0)
{
const FGenericBlock& PrevBlock = ValidationBlocks[BlockIndex - 1];
if (PrevBlock.Offset + PrevBlock.Size != CurrBlock.Offset)
{
UNSYNC_ERROR(L"Found block at unexpected offset");
return false;
}
}
BatchSizeBytes += CurrBlock.Size;
if (BlockIndex + 1 < ValidationBlocks.size() && BatchSizeBytes + ValidationBlocks[BlockIndex + 1].Size < MaxBatchSizeBytes)
{
continue;
}
UNSYNC_ASSERT(BatchSizeBytes <= MaxBatchSizeBytes || BatchBegin == BlockIndex);
const uint64 ReadOffset = ValidationBlocks[BatchBegin].Offset;
UNSYNC_ASSERT(BlockIndex + 1 == ValidationBlocks.size() ||
(ReadOffset + BatchSizeBytes) == ValidationBlocks[BlockIndex + 1].Offset);
auto ReadCallback = [StrongHasher,
bLogVerbose,
LogIndent,
BatchBegin,
BatchEnd = BlockIndex + 1,
BatchSizeBytes,
&NumInvalidBlocks,
&TaskGroup,
&ValidationProgressLogger,
&ValidationBlocks](FIOBuffer CmdBuffer, uint64 CmdSourceOffset, uint64 CmdReadSize, uint64 CmdUserData) {
if (CmdReadSize != BatchSizeBytes)
{
UNSYNC_ERROR(L"Expected to read %lld bytes, but read %lld", BatchSizeBytes, CmdReadSize);
NumInvalidBlocks++;
return;
}
TaskGroup.run([CmdBuffer = std::make_shared<FIOBuffer>(std::move(CmdBuffer)),
BatchBegin,
BatchEnd,
StrongHasher,
bLogVerbose,
LogIndent,
&NumInvalidBlocks,
&ValidationProgressLogger,
&ValidationBlocks]() {
FLogIndentScope IndentScope(LogIndent, true);
FLogVerbosityScope VerbosityScope(bLogVerbose);
const uint64 FirstBlockOffset = ValidationBlocks[BatchBegin].Offset;
for (uint64 I = BatchBegin; I < BatchEnd; ++I)
{
const FGenericBlock& Block = ValidationBlocks[I];
const uint64 BlockBufferOffset = Block.Offset - FirstBlockOffset;
FGenericHash Hash = ComputeHash(CmdBuffer->GetData() + BlockBufferOffset, Block.Size, StrongHasher);
if (Hash != Block.HashStrong)
{
UNSYNC_ERROR(L"Found block hash mismatch at offset %llu", llu(BlockBufferOffset));
NumInvalidBlocks++;
return;
}
ValidationProgressLogger.Add(Block.Size);
}
});
};
AsyncReader->EnqueueRead(ReadOffset, BatchSizeBytes, 0, ReadCallback);
if (NumInvalidBlocks)
{
break;
}
BatchSizeBytes = 0;
BatchBegin = BlockIndex + 1;
}
AsyncReader->Flush();
TaskGroup.wait();
ValidationProgressLogger.Complete();
return NumInvalidBlocks == 0;
}
static FBuildTargetParams
GetBuildTargetParams(const FSyncFileOptions& Options)
{
FBuildTargetParams Result;
Result.StrongHasher = Options.Algorithm.StrongHashAlgorithmId;
Result.ProxyPool = Options.ProxyPool;
Result.BlockCache = Options.BlockCache;
Result.ScavengeDatabase = Options.ScavengeDatabase;
if (IsFileSystemSource(Options.SourceType))
{
Result.SourceType = FBuildTargetParams::ESourceType::File;
}
else
{
Result.SourceType = FBuildTargetParams::ESourceType::Server;
}
return Result;
}
FFileSyncResult
SyncFile(const FNeedList& NeedList,
const FPath& SourceFilePath,
const FGenericBlockArray& SourceBlocks,
FIOReader& BaseDataReader,
const FPath& TargetFilePath,
const FSyncFileOptions& Options)
{
UNSYNC_LOG_INDENT;
FFileSyncResult Result;
if (Options.SourceType == ESourceType::Unknown)
{
Result.Status = EFileSyncStatus::ErrorInvalidParameters;
UNSYNC_ERROR(L"Sync source type must be specified");
return Result;
}
uint64 NeedFromSource = ComputeSize(NeedList.Source);
uint64 NeedFromBase = ComputeSize(NeedList.Base);
UNSYNC_VERBOSE(L"Need from source %.2f MB, from base: %.2f MB", SizeMb(NeedFromSource), SizeMb(NeedFromBase));
const FFileAttributes TargetFileAttributes = GetFileAttrib(TargetFilePath);
if (!TargetFileAttributes.bValid && NeedList.Sequence.empty())
{
UNSYNC_VERBOSE(L"Creating empty file '%ls'", TargetFilePath.wstring().c_str());
if (GDryRun)
{
Result.Status = EFileSyncStatus::Ok;
}
else
{
FPath TargetFileParent = TargetFilePath.parent_path();
if (!PathExists(TargetFileParent))
{
CreateDirectories(TargetFileParent);
}
auto TargetFile = FNativeFile(TargetFilePath, EFileMode::CreateWriteOnly, 0);
if (TargetFile.IsValid())
{
Result.Status = EFileSyncStatus::Ok;
}
else
{
Result.Status = EFileSyncStatus::ErrorTargetFileCreate;
Result.SystemErrorCode = std::error_code(TargetFile.GetError(), std::system_category());
}
}
}
else if (!IsSynchronized(NeedList, SourceBlocks))
{
LogStatus(TargetFilePath.wstring().c_str(), L"Initializing");
FPath TempTargetFilePath = TargetFilePath;
TempTargetFilePath.replace_extension(TargetFilePath.extension().wstring() + L".tmp");
const FNeedListSize TargetFileSizeInfo = ComputeNeedListSize(NeedList);
FBuffer TargetFileBuffer;
std::unique_ptr<FIOReaderWriter> TargetFile;
if (GDryRun)
{
if (Options.bValidateTargetFiles)
{
TargetFileBuffer.Resize(TargetFileSizeInfo.TotalBytes);
TargetFile = std::make_unique<FMemReaderWriter>(TargetFileBuffer.Data(), TargetFileBuffer.Size());
}
else
{
TargetFile = std::make_unique<FNullReaderWriter>(TargetFileSizeInfo.TotalBytes);
}
}
else
{
FPath TargetFileParent = TempTargetFilePath.parent_path();
if (!PathExists(TargetFileParent))
{
CreateDirectories(TargetFileParent);
}
TargetFile = std::make_unique<FNativeFile>(TempTargetFilePath, EFileMode::CreateWriteOnly, TargetFileSizeInfo.TotalBytes);
if (TargetFile->GetError() != 0)
{
UNSYNC_FATAL(L"Failed to create output file '%ls'. %hs",
TempTargetFilePath.wstring().c_str(),
FormatSystemErrorMessage(TargetFile->GetError()).c_str());
}
}
LogStatus(TargetFilePath.wstring().c_str(), L"Patching");
FDeferredOpenReader SourceFile(
[SourceFilePath, TargetFilePath, Options]() -> std::unique_ptr<FIOReader>
{
if (IsFileSystemSource(Options.SourceType))
{
UNSYNC_VERBOSE(L"Opening source file '%ls'", SourceFilePath.wstring().c_str());
LogStatus(TargetFilePath.wstring().c_str(), L"Opening source file");
return std::unique_ptr<FNativeFile>(new FNativeFile(SourceFilePath, EFileMode::ReadOnlyUnbuffered));
}
else
{
UNSYNC_ERROR(L"Sync source is not directly accessible");
return std::unique_ptr<FIOReader>(new FNullReaderWriter(FNullReaderWriter::FInvalid()));
}
});
FBuildTargetParams BuildParams = GetBuildTargetParams(Options);
FBuildTargetResult BuildResult = BuildTarget(*TargetFile, SourceFile, BaseDataReader, NeedList, BuildParams);
Result.SourceBytes = BuildResult.SourceBytes;
Result.BaseBytes = BuildResult.BaseBytes;
if (!BuildResult.bSuccess)
{
Result.Status = EFileSyncStatus::ErrorBuildTargetFailed;
return Result;
}
if (Options.bValidateTargetFiles)
{
LogStatus(TargetFilePath.wstring().c_str(), L"Verifying");
UNSYNC_VERBOSE(L"Verifying patched file '%ls'", TargetFilePath.wstring().c_str());
UNSYNC_LOG_INDENT;
if (!GDryRun)
{
// Reopen the file in unuffered read mode for optimal reading performance
TargetFile = nullptr;
TargetFile = std::make_unique<FNativeFile>(TempTargetFilePath, EFileMode::ReadOnlyUnbuffered);
}
if (TargetFileSizeInfo.TotalBytes > 0)
{
if (!ValidateTarget(*TargetFile, NeedList, Options.Algorithm.StrongHashAlgorithmId))
{
Result.Status = EFileSyncStatus::ErrorValidation;
return Result;
}
}
}
if (GDryRun)
{
Result.Status = EFileSyncStatus::Ok;
}
else
{
LogStatus(TargetFilePath.wstring().c_str(), L"Finalizing");
UNSYNC_VERBOSE(L"Finalizing target file '%ls'", TargetFilePath.wstring().c_str());
BaseDataReader.Close();
if (TargetFile)
{
TargetFile->Close();
}
if (GetFileAttrib(TargetFilePath).bReadOnly)
{
UNSYNC_VERBOSE(L"Clearing read-only flag from target file '%ls'", TargetFilePath.wstring().c_str());
bool bClearReadOnlyOk = SetFileReadOnly(TargetFilePath, false);
if (!bClearReadOnlyOk)
{
UNSYNC_ERROR(L"Failed to clear read-only flag from '%ls'", TargetFilePath.wstring().c_str());
}
}
std::error_code ErrorCode = {};
FileRename(TempTargetFilePath, TargetFilePath, ErrorCode);
if (ErrorCode.value() == 0)
{
Result.Status = EFileSyncStatus::Ok;
}
else
{
Result.Status = EFileSyncStatus::ErrorFinalRename;
Result.SystemErrorCode = ErrorCode;
}
}
const uint64 ExpectedSourceBytes = ComputeSize(NeedList.Source);
const uint64 ExpectedBaseBytes = ComputeSize(NeedList.Base);
const uint64 ActualProcessedBytes = BuildResult.SourceBytes + BuildResult.BaseBytes;
const uint64 ExpectedProcessedBytes = ExpectedSourceBytes + ExpectedBaseBytes;
if (ActualProcessedBytes != ExpectedProcessedBytes)
{
Result.Status = EFileSyncStatus::ErrorValidation;
UNSYNC_ERROR(L"Failed to patch file '%ls'. Expected to write %llu bytes, but actually wrote %llu bytes.",
TargetFilePath.wstring().c_str(),
llu(ExpectedProcessedBytes),
llu(ActualProcessedBytes));
}
}
else
{
UNSYNC_VERBOSE(L"Target file '%ls' already synchronized", TargetFilePath.wstring().c_str());
Result.Status = EFileSyncStatus::Ok;
Result.BaseBytes = NeedFromBase;
}
return Result;
}
FFileSyncResult
SyncFile(const FPath& SourceFilePath,
const FGenericBlockArray& SourceBlocks,
FIOReader& BaseDataReader,
const FPath& TargetFilePath,
const FSyncFileOptions& Options)
{
UNSYNC_LOG_INDENT;
UNSYNC_VERBOSE(L"Computing difference for target '%ls' (base size: %.2f MB)",
TargetFilePath.wstring().c_str(),
SizeMb(BaseDataReader.GetSize()));
FNeedList NeedList = DiffBlocks(BaseDataReader,
Options.BlockSize,
Options.Algorithm.WeakHashAlgorithmId,
Options.Algorithm.StrongHashAlgorithmId,
SourceBlocks);
return SyncFile(NeedList, SourceFilePath, SourceBlocks, BaseDataReader, TargetFilePath, Options);
}
FFileSyncResult
SyncFile(const FPath& SourceFilePath, const FPath& BaseFilePath, const FPath& TargetFilePath, const FSyncFileOptions& InOptions)
{
UNSYNC_LOG_INDENT;
FSyncFileOptions Options = InOptions; // This may be modified by LoadBlocks()
FFileSyncResult Result;
FNativeFile BaseFile(BaseFilePath, EFileMode::ReadOnlyUnbuffered);
if (!BaseFile.IsValid())
{
BaseFile.Close();
UNSYNC_VERBOSE(L"Full copy required for '%ls' (base does not exist)", BaseFilePath.wstring().c_str());
std::error_code ErrorCode;
bool bCopyOk = FileCopy(SourceFilePath, TargetFilePath, ErrorCode);
if (bCopyOk)
{
Result.Status = EFileSyncStatus::Ok;
}
else
{
Result.Status = EFileSyncStatus::ErrorFullCopy;
Result.SystemErrorCode = ErrorCode;
}
Result.SourceBytes = GetFileAttrib(SourceFilePath).Size;
return Result;
}
FGenericBlockArray SourceBlocks;
FPath BlockFilename = BaseFilePath.wstring() + std::wstring(L".unsync");
UNSYNC_VERBOSE(L"Loading block manifest from '%ls'", BlockFilename.wstring().c_str());
if (LoadBlocks(SourceBlocks, Options.BlockSize, BlockFilename.c_str()))
{
UNSYNC_VERBOSE(L"Loaded blocks: %d", uint32(SourceBlocks.size()));
}
else
{
UNSYNC_VERBOSE(L"Full copy required (manifest file does not exist or is invalid)");
std::error_code ErrorCode;
bool bCopyOk = FileCopy(SourceFilePath, TargetFilePath, ErrorCode);
if (bCopyOk)
{
Result.Status = EFileSyncStatus::Ok;
}
else
{
Result.Status = EFileSyncStatus::ErrorFullCopy;
Result.SystemErrorCode = ErrorCode;
}
Result.SourceBytes = GetFileAttrib(SourceFilePath).Size;
return Result;
}
return SyncFile(SourceFilePath, SourceBlocks, BaseFile, TargetFilePath, Options);
}
struct FPendingFileRename
{
std::wstring Old;
std::wstring New;
};
// Updates the target directory manifest filename case to be consistent with reference.
// Internally we always perform case-sensitive path comparisons, however on non-case-sensitive filesystems some local files may be renamed
// to a mismatching case. We can update the locally-generated manifest to take the case from the reference manifest for equivalent paths.
// Returns a list of files that should be renamed on disk.
static std::vector<FPendingFileRename>
FixManifestFileNameCases(FDirectoryManifest& TargetDirectoryManifest, const FDirectoryManifest& ReferenceManifest)
{
// Build a lookup table of lowercase -> original file names and detect potential case conflicts (which will explode on Windows and Mac)
std::unordered_map<std::wstring, std::wstring> ReferenceFileNamesLowerCase;
bool bFoundCaseConflicts = false;
for (auto& ReferenceManifestEntry : ReferenceManifest.Files)
{
std::wstring FileNameLowerCase = StringToLower(ReferenceManifestEntry.first);
auto InsertResult =
ReferenceFileNamesLowerCase.insert(std::pair<std::wstring, std::wstring>(FileNameLowerCase, ReferenceManifestEntry.first));
if (!InsertResult.second)
{
UNSYNC_WARNING(L"Found file name case conflict: '%ls'", ReferenceManifestEntry.first.c_str());
bFoundCaseConflicts = true;
}
}
if (bFoundCaseConflicts)
{
UNSYNC_WARNING(L"File name case conflicts will result in issues on case-insensitive systems, such as Windows and macOS.");
}
// Find inconsistently-cased files and add them to a list to be fixed up
std::vector<FPendingFileRename> FixupEntries;
for (auto& TargetManifestEntry : TargetDirectoryManifest.Files)
{
const std::wstring& TargetFileName = TargetManifestEntry.first;
if (ReferenceManifest.Files.find(TargetFileName) == ReferenceManifest.Files.end())
{
std::wstring TargetFileNameLowerCase = StringToLower(TargetFileName);
auto ReferenceIt = ReferenceFileNamesLowerCase.find(TargetFileNameLowerCase);
if (ReferenceIt != ReferenceFileNamesLowerCase.end())
{
FixupEntries.push_back({TargetFileName, ReferenceIt->second});
}
}
}
// Re-add file manifests under the correct names
for (const FPendingFileRename& Entry : FixupEntries)
{
auto It = TargetDirectoryManifest.Files.find(Entry.Old);
UNSYNC_ASSERT(It != TargetDirectoryManifest.Files.end());
FFileManifest Manifest;
std::swap(It->second, Manifest);
TargetDirectoryManifest.Files.erase(Entry.Old);
TargetDirectoryManifest.Files.insert(std::pair(Entry.New, std::move(Manifest)));
}
return FixupEntries;
}
// Takes a list of file names that require case fixup and performs the necessary renaming.
// Handles renaming of intermediate directories as well as the leaf files.
// Quite wasteful in terms of mallocs, but doesn't matter since we're about to touch the file system anyway.
static bool
FixFileNameCases(const FPath& RootPath, const std::vector<FPendingFileRename>& PendingRenames)
{
std::vector<FPendingFileRename> UniqueRenames;
std::unordered_set<FPath::string_type> UniqueRenamesSet;
// Build a rename schedule, with only unique entries (taking subdirectories into account)
for (const FPendingFileRename& Entry : PendingRenames)
{
UNSYNC_ASSERTF(StringToLower(Entry.Old) == StringToLower(Entry.New),
L"FixFileNameCases expects inputs that are different only by case. Old: '%ls', New: '%ls'",
Entry.Old.c_str(),
Entry.New.c_str());
FPath OldPath = Entry.Old;
FPath NewPath = Entry.New;
auto ItOld = OldPath.begin();
auto ItNew = NewPath.begin();
FPath OldPathPart;
FPath NewPathPart;
while (ItOld != OldPath.end())
{
OldPathPart /= *ItOld;
NewPathPart /= *ItNew;
if (*ItOld != *ItNew)
{
auto InsertResult = UniqueRenamesSet.insert(OldPathPart.native());
if (InsertResult.second)
{
UniqueRenames.push_back({OldPathPart.wstring(), NewPathPart.wstring()});
}
}
++ItOld;
++ItNew;
}
}
std::sort(UniqueRenames.begin(), UniqueRenames.end(), [](const FPendingFileRename& A, const FPendingFileRename& B) {
return A.Old < B.Old;
});
// Perform actual renaming
for (const FPendingFileRename& Entry : UniqueRenames)
{
FPath OldPath = RootPath / Entry.Old;
FPath NewPath = RootPath / Entry.New;
std::error_code ErrorCode;
if (GDryRun)
{
UNSYNC_VERBOSE(L"Renaming '%ls' -> '%ls' (skipped due to dry run mode)", Entry.Old.c_str(), Entry.New.c_str());
}
else
{
UNSYNC_VERBOSE(L"Renaming '%ls' -> '%ls'", Entry.Old.c_str(), Entry.New.c_str());
FileRename(OldPath, NewPath, ErrorCode);
}
if (ErrorCode)
{
UNSYNC_VERBOSE(L"Failed to rename file. System error code %d: %hs", ErrorCode.value(), ErrorCode.message().c_str());
return false;
}
}
return true;
}
// Delete files from target directory that are not in the source directory manifest
static void
DeleteUnnecessaryFiles(const FPath& TargetDirectory,
const FDirectoryManifest& TargetDirectoryManifest,
const FDirectoryManifest& ReferenceManifest,
const FSyncFilter* SyncFilter)
{
auto ShouldCleanup = [SyncFilter](const FPath& Filename) -> bool {
if (SyncFilter)
{
return SyncFilter->ShouldCleanup(Filename);
}
else
{
return true;
}
};
for (const auto& TargetManifestEntry : TargetDirectoryManifest.Files)
{
const std::wstring& TargetFileName = TargetManifestEntry.first;
auto Cleanup = [&ShouldCleanup, &TargetDirectory](const std::wstring& TargetFileName, const wchar_t* Reason) {
FPath FilePath = TargetDirectory / TargetFileName;
if (!ShouldCleanup(TargetFileName))
{
UNSYNC_VERBOSE2(L"Skipped deleting '%ls' (excluded by cleanup filter)", FilePath.wstring().c_str());
return;
}
if (GDryRun)
{
UNSYNC_VERBOSE(L"Deleting '%ls' (%ls, skipped due to dry run mode)", FilePath.wstring().c_str(), Reason);
}
else
{
UNSYNC_VERBOSE(L"Deleting '%ls' (%ls)", FilePath.wstring().c_str(), Reason);
std::error_code ErrorCode = {};
FileRemove(FilePath, ErrorCode);
if (ErrorCode)
{
UNSYNC_VERBOSE(L"System error code %d: %hs", ErrorCode.value(), ErrorCode.message().c_str());
}
}
};
if (ReferenceManifest.Files.find(TargetFileName) == ReferenceManifest.Files.end())
{
Cleanup(TargetFileName, L"not in manifest");
}
else if (!SyncFilter->ShouldSync(TargetFileName))
{
Cleanup(TargetFileName, L"excluded from sync");
}
}
}
FPath
ToPath(const std::wstring_view& Str)
{
#if UNSYNC_PLATFORM_UNIX
// TODO: ensure that all serialized path separators are unix style ('/')
std::wstring Temp = std::wstring(Str);
std::replace(Temp.begin(), Temp.end(), L'\\', L'/');
return FPath(Temp);
#else // UNSYNC_PLATFORM_UNIX
return FPath(Str);
#endif // UNSYNC_PLATFORM_UNIX
}
struct FPooledProxy
{
FPooledProxy(FProxyPool& InProxyPool) : ProxyPool(InProxyPool) { Proxy = ProxyPool.Alloc(); }
~FPooledProxy() { ProxyPool.Dealloc(std::move(Proxy)); }
const FProxy& operator->() const { return *Proxy; }
FProxyPool& ProxyPool;
std::unique_ptr<FProxy> Proxy;
};
struct FRemoteFileInfo
{
FPath Path;
ProxyQuery::FDirectoryListingEntry Entry;
};
struct FFoundManifest
{
FRemoteFileInfo Manifest;
std::vector<FRemoteFileInfo> PackDataFiles;
std::vector<FRemoteFileInfo> PackIndexFiles;
};
static TResult<FFoundManifest>
FindUnsyncManifest(FProxyFileSystem& FileSystem)
{
using FDirectoryListing = ProxyQuery::FDirectoryListing;
using FDirectoryListingEntry = ProxyQuery::FDirectoryListingEntry;
TResult<FDirectoryListing> RootDirectoryListingResult = FileSystem.ListDirectory("");
UNSYNC_RETURN_ON_ERROR(RootDirectoryListingResult);
std::optional<FDirectoryListingEntry> FoundUnsyncSubdirectory;
std::optional<FDirectoryListingEntry> FoundUnsyncManifestDotfile;
for (const FDirectoryListingEntry& Entry : RootDirectoryListingResult.GetData().Entries)
{
if (Entry.Name == ".unsyncmanifest" && !Entry.bDirectory)
{
FoundUnsyncManifestDotfile = Entry;
}
if (Entry.Name == ".unsync" && Entry.bDirectory)
{
FoundUnsyncSubdirectory = Entry;
}
}
if (FoundUnsyncManifestDotfile && !FoundUnsyncSubdirectory)
{
FFoundManifest Result;
Result.Manifest.Path = FoundUnsyncManifestDotfile->Name;
Result.Manifest.Entry = *FoundUnsyncManifestDotfile;
return ResultOk(Result);
}
if (FoundUnsyncSubdirectory)
{
std::string UnsyncDirectory = ".unsync";
TResult<FDirectoryListing> UnsyncDirectoryListingResult = FileSystem.ListDirectory(UnsyncDirectory);
UNSYNC_RETURN_ON_ERROR(UnsyncDirectoryListingResult);
FFoundManifest Result;
for (const FDirectoryListingEntry& Entry : UnsyncDirectoryListingResult.GetData().Entries)
{
if (Entry.Name == "manifest.bin" && !Entry.bDirectory)
{
Result.Manifest.Path = FPath(UnsyncDirectory) / Entry.Name;
Result.Manifest.Entry = Entry;
}
if (Entry.Name == "pack" && Entry.bDirectory)
{
FPath PackDirectory = FPath(UnsyncDirectory) / "pack";
TResult<FDirectoryListing> PackDirectoryListring = FileSystem.ListDirectory(ToString(PackDirectory));
if (const FDirectoryListing* Listing = PackDirectoryListring.TryData())
{
for (const FDirectoryListingEntry& PackEntry : Listing->Entries)
{
if (PackEntry.bDirectory)
{
continue;
}
FRemoteFileInfo Info;
Info.Path = PackDirectory / PackEntry.Name;
Info.Entry = PackEntry;
if (PackEntry.Name.ends_with(".unsync_pack"))
{
Result.PackDataFiles.push_back(Info);
}
if (PackEntry.Name.ends_with(".unsync_index"))
{
Result.PackIndexFiles.push_back(Info);
}
}
}
}
}
if (!Result.Manifest.Path.empty())
{
return ResultOk(std::move(Result));
}
}
return AppError("Could not find unsync manifest file");
}
static bool
CopyFileIfPossiblyDifferent(FProxyFileSystem& FileSystem,
const FRemoteFileInfo& Source,
const FPath& Target,
EFileMode TargetFileMode = EFileMode::CreateWriteOnly)
{
FFileAttributes TargetAttr = GetFileAttrib(Target);
if (Source.Entry.Size != TargetAttr.Size || Source.Entry.Mtime != TargetAttr.Mtime)
{
UNSYNC_VERBOSE(L"Reading '%ls'", Source.Path.wstring().c_str());
TResult<FBuffer> FileBuffer = FileSystem.ReadFile(ToString(Source.Path));
if (FileBuffer.IsError())
{
LogError(FileBuffer.GetError(), L"Failed to read source file");
return false;
}
if (FileBuffer->Size() != Source.Entry.Size)
{
UNSYNC_ERROR(L"Read file size mismatch. Expected %llu, actual %llu.", llu(Source.Entry.Size), llu(FileBuffer->Size()));
return false;
}
const bool bFileWritten = WriteBufferToFile(Target, *FileBuffer, TargetFileMode);
if (!bFileWritten)
{
UNSYNC_ERROR(L"Failed to write file '%ls'", Target.wstring().c_str());
return false;
}
const bool bAllowInDryRun = true;
if (Source.Entry.Mtime)
{
SetFileMtime(Target, Source.Entry.Mtime, bAllowInDryRun);
}
}
return true;
}
static bool
LoadAndMergeSourceManifest(FDirectoryManifest& Output,
std::vector<FPackIndexDatabase>& OutIndexFiles,
FProxyFileSystem& ProxyFileSystem,
const FPath& SourcePath,
const FPath& TempPath,
FSyncFilter* SyncFilter,
bool bCaseSensitiveTargetFileSystem)
{
UNSYNC_VERBOSE2(L"LoadAndMergeSourceManifest: '%ls'", SourcePath.wstring().c_str());
FDirectoryManifest LoadedManifest;
FPath SourceManifestRelativePath = FPath(".unsync") / "manifest.bin";
FPath SourceManifestPath = SourcePath / SourceManifestRelativePath;
FHash128 SourcePathHash =
HashBlake3Bytes<FHash128>((const uint8*)SourcePath.native().c_str(), SourcePath.native().length() * sizeof(SourcePath.native()[0]));
std::string SourcePathHashStr = BytesToHexString(SourcePathHash.Data, sizeof(SourcePathHash.Data));
FPath CachedManifestPath = TempPath / SourcePathHashStr;
LogGlobalStatus(L"Caching source manifest");
UNSYNC_VERBOSE(L"Caching source manifest");
UNSYNC_LOG_INDENT;
UNSYNC_VERBOSE(L"Source '%ls'", SourceManifestPath.wstring().c_str());
UNSYNC_VERBOSE(L"Target '%ls'", CachedManifestPath.wstring().c_str());
TResult<FFoundManifest> FindManifestResult = FindUnsyncManifest(ProxyFileSystem);
if (FindManifestResult.IsError())
{
LogError(FindManifestResult.GetError(), L"Failed to find remote manifest");
return false;
}
if (!CopyFileIfPossiblyDifferent(ProxyFileSystem,
FindManifestResult->Manifest,
CachedManifestPath,
EFileMode::CreateReadWrite | EFileMode::IgnoreDryRun))
{
return false;
}
if (!LoadDirectoryManifest(LoadedManifest, SourcePath, CachedManifestPath))
{
UNSYNC_ERROR(L"Failed to load source directory manifest '%ls'", SourceManifestPath.wstring().c_str());
return false;
}
if (!FindManifestResult->PackIndexFiles.empty())
{
UNSYNC_VERBOSE(L"Loading pack index database");
UNSYNC_LOG_INDENT;
std::unordered_set<FPathStringView> FoundPackFiles;
for (const FRemoteFileInfo& PackFileInfo : FindManifestResult->PackDataFiles)
{
FoundPackFiles.insert(PackFileInfo.Path.native());
}
for (const FRemoteFileInfo& IndexFileInfo : FindManifestResult->PackIndexFiles)
{
FPath PackDataFilePath = IndexFileInfo.Path;
PackDataFilePath.replace_extension(".unsync_pack");
if (!FoundPackFiles.contains(PackDataFilePath.native()))
{
UNSYNC_WARNING(L"Could not find pack file '%ls'", PackDataFilePath.wstring().c_str());
continue;
}
UNSYNC_VERBOSE(L"Reading '%ls'", IndexFileInfo.Path.wstring().c_str());
TResult<FBuffer> FileBuffer = ProxyFileSystem.ReadFile(ToString(IndexFileInfo.Path));
if (FileBuffer.IsError())
{
LogError(FileBuffer.GetError(), L"Failed to read remote file");
return false;
}
FMemReader Reader(*FileBuffer);
FIOReaderStream Stream(Reader);
FPackIndexDatabase IndexFile;
IndexFile.IndexPath = SourcePath / IndexFileInfo.Path;
IndexFile.DataPath = SourcePath / PackDataFilePath;
if (LoadPackIndexDatabase(IndexFile, Stream))
{
OutIndexFiles.emplace_back(std::move(IndexFile));
}
}
}
if (Output.IsValid() && !AlgorithmOptionsCompatible(Output.Algorithm, LoadedManifest.Algorithm))
{
UNSYNC_ERROR(L"Can't merge manifest '%ls' as it uses different algorithm options", SourcePath.wstring().c_str());
return false;
}
return MergeManifests(Output, LoadedManifest, bCaseSensitiveTargetFileSystem);
}
struct FFileSyncTaskBatch
{
std::vector<const FFileSyncTask*> FileTasks;
uint64 TotalSizeBytes = 0;
uint64 NeedBytesFromSource = 0;
std::unique_ptr<FBlockCache> CreateBlockCache(FProxyPool& ProxyPool, EStrongHashAlgorithmID StrongHasher) const
{
FTimePoint TimeBegin = TimePointNow();
std::unique_ptr<FBlockCache> Result = std::make_unique<FBlockCache>();
Result->BlockData.Resize(NeedBytesFromSource);
uint64 OutputCursor = 0;
THashSet<FHash128> UniqueBlockSet;
std::vector<FNeedBlock> UniqueNeedBlocks;
for (const FFileSyncTask* Task : FileTasks)
{
for (const FNeedBlock& Block : Task->NeedList.Source)
{
if (UniqueBlockSet.insert(Block.Hash.ToHash128()).second)
{
UniqueNeedBlocks.push_back(Block);
}
}
}
Result->BlockMap.reserve(UniqueNeedBlocks.size());
GScheduler->NetworkSemaphore.Acquire(false);
std::unique_ptr<FProxy> Proxy = ProxyPool.Alloc();
if (Proxy)
{
auto DownloadCallback =
[&OutputCursor, &Result, &UniqueBlockSet, StrongHasher](const FDownloadedBlock& Block, FHash128 BlockHash)
{
if (OutputCursor + Block.DecompressedSize <= Result->BlockData.Size())
{
if (UniqueBlockSet.find(BlockHash) != UniqueBlockSet.end())
{
FMutBufferView OutputView = Result->BlockData.MutView(OutputCursor, Block.DecompressedSize);
bool bOk = true;
if (Block.bCompressed)
{
bOk = Decompress(Block.Data, Block.CompressedSize, OutputView.Data, OutputView.Size);
}
else
{
memcpy(OutputView.Data, Block.Data, OutputView.Size);
}
if (bOk)
{
const FHash128 ActualBlockHash = ComputeHash(OutputView.Data, OutputView.Size, StrongHasher).ToHash128();
bOk = BlockHash == ActualBlockHash;
}
if (bOk)
{
Result->BlockMap[BlockHash] = FBufferView{OutputView.Data, OutputView.Size};
OutputCursor += Block.DecompressedSize;
}
else
{
UNSYNC_WARNING(L"Received a corrupt block");
}
}
else
{
UNSYNC_WARNING(L"Received a block with unexpected hash");
}
}
};
FDownloadResult DownloadResult =
Proxy->Download(MakeView<FNeedBlock>(UniqueNeedBlocks.data(), UniqueNeedBlocks.size()), DownloadCallback);
const uint64 NumExpected = UniqueNeedBlocks.size();
const uint64 NumDownloaded = Result->BlockMap.size();
if (NumExpected != NumDownloaded)
{
THashSet<FHash128> MissingBlocks = UniqueBlockSet;
for (const auto& It : Result->BlockMap)
{
MissingBlocks.erase(It.first);
}
if (MissingBlocks.size() <= 10)
{
std::string MissingBlockStr;
for (const FHash128& Hash : MissingBlocks)
{
if (!MissingBlockStr.empty())
{
MissingBlockStr += ", ";
}
MissingBlockStr += HashToHexString(Hash);
}
UNSYNC_WARNING(
L"Could not download all required data while building block cache. "
L"Blocks expected: %llu, actual: %llu. Missing blocks: %hs",
llu(NumExpected),
llu(NumDownloaded),
MissingBlockStr.c_str());
}
else
{
UNSYNC_WARNING(
L"Could not download all required data while building block cache. "
L"Blocks expected: %llu, actual: %llu.",
llu(NumExpected),
llu(NumDownloaded));
}
}
UNSYNC_UNUSED(DownloadResult);
}
ProxyPool.Dealloc(std::move(Proxy));
GScheduler->NetworkSemaphore.Release();
Result->InitDuration = TimePointNow() - TimeBegin;
return Result;
}
};
bool // TODO: return a TResult
SyncDirectory(const FSyncDirectoryOptions& SyncOptions)
{
FProxyPool DummyProxyPool;
FProxyPool& ProxyPool = SyncOptions.ProxyPool ? *SyncOptions.ProxyPool : DummyProxyPool;
FTimePoint TimeBegin = TimePointNow();
const bool bFileSystemSource = SyncOptions.SourceType == ESourceType::FileSystem;
const bool bServerSource =
SyncOptions.SourceType == ESourceType::Server || SyncOptions.SourceType == ESourceType::ServerWithManifestId;
UNSYNC_ASSERT(bFileSystemSource || bServerSource);
const FPath SourcePath = bFileSystemSource ? std::filesystem::absolute(SyncOptions.Source) : SyncOptions.Source;
const FPath BasePath = std::filesystem::absolute(SyncOptions.Base);
const FPath TargetPath = std::filesystem::absolute(SyncOptions.Target);
FSyncFilter* SyncFilter = SyncOptions.SyncFilter;
bool bSourceManifestOk = true;
UNSYNC_LOG_INDENT;
if (SyncOptions.bCleanup)
{
UNSYNC_LOG(L"Unnecessary files will be deleted after sync (cleanup mode)");
}
const FPath BaseManifestRoot = BasePath / ".unsync";
const FPath BaseManifestPath = BaseManifestRoot / "manifest.bin";
const FPath TargetManifestRoot = TargetPath / ".unsync";
const FPath TargetManifestPath = TargetManifestRoot / "manifest.bin";
const FPath TargetTempPath = TargetManifestRoot / "temp";
const bool bTempDirectoryExists = (PathExists(TargetTempPath) && IsDirectory(TargetTempPath)) || CreateDirectories(TargetTempPath);
if (!bTempDirectoryExists)
{
UNSYNC_ERROR(L"Failed to create temporary working directory");
return false;
}
// Delete oldest cached manifest files if there are more than N
{
UNSYNC_VERBOSE(L"Cleaning temporary directory");
UNSYNC_LOG_INDENT;
const uint32 MaxFilesToKeep = uint32(5 + SyncOptions.Overlays.size());
DeleteOldFilesInDirectory(TargetTempPath, MaxFilesToKeep);
}
const FPath LogFilePath = TargetManifestRoot / L"unsync.log";
const FLogFileScope LogFileScope(LogFilePath.wstring().c_str());
SetCrashDumpPath(TargetManifestRoot);
auto ShouldSync = [SyncFilter](const FPath& Filename) -> bool {
if (SyncFilter)
{
return SyncFilter->ShouldSync(Filename);
}
else
{
return true;
}
};
FDirectoryManifest SourceDirectoryManifest;
FPath SourceManifestTempPath;
const bool bCaseSensitiveTargetFileSystem = IsCaseSensitiveFileSystem(TargetTempPath);
FTimingLogger ManifestLoadTimingLogger("Manifest load time", ELogLevel::Info);
std::vector<FPackIndexDatabase> PackIndexFiles;
std::vector<FPath> AllSources;
AllSources.push_back(SourcePath);
for (const FPath& OverlayPath : SyncOptions.Overlays)
{
AllSources.push_back(OverlayPath);
}
auto ResolvePath = [SyncFilter](const FPath& Filename) -> FPath { return SyncFilter ? SyncFilter->Resolve(Filename) : Filename; };
// Used to build block request map when syncing from multiple sources.
THashMap<FHash256, uint32> FileSourceIdMap;
if (SyncOptions.SourceType == ESourceType::ServerWithManifestId)
{
if (!ProxyPool.IsValid())
{
UNSYNC_ERROR(L"Remote server connection is required when syncing by manifest ID");
return false;
}
std::unique_ptr<FProxy> Proxy = ProxyPool.Alloc();
uint32 SourceIndex = 0;
for (const FPath& ThisSourcePath : AllSources)
{
std::string SourceManifestName = ConvertWideToUtf8(ThisSourcePath.wstring());
FHash128 SourcePathHash = HashBlake3String<FHash128>(SourceManifestName);
std::string SourcePathHashStr = BytesToHexString(SourcePathHash.Data, sizeof(SourcePathHash.Data));
FPath CachedManifestPath = TargetTempPath / SourcePathHashStr;
FPath EmptyPath; // no physical path for downloaded manifests
FDirectoryManifest LoadedManifest;
if (!PathExists(CachedManifestPath) || !LoadDirectoryManifest(LoadedManifest, EmptyPath, CachedManifestPath))
{
LogGlobalStatus(L"Caching source manifest");
UNSYNC_VERBOSE(L"Caching source manifest");
UNSYNC_LOG_INDENT;
UNSYNC_VERBOSE(L"Source '%hs'", SourceManifestName.c_str());
UNSYNC_VERBOSE(L"Target '%ls'", CachedManifestPath.wstring().c_str());
TResult<FDirectoryManifest> DownloadResult = Proxy->DownloadManifest(SourceManifestName);
if (FDirectoryManifest* Manifest = DownloadResult.TryData())
{
std::swap(LoadedManifest, *Manifest);
}
else
{
LogError(DownloadResult.GetError(), L"Failed to download manifest");
UNSYNC_BREAK_ON_ERROR;
return false;
}
const bool bAllowInDryRun = true;
SaveDirectoryManifest(LoadedManifest, CachedManifestPath, bAllowInDryRun);
}
for (const auto& It : LoadedManifest.Files)
{
FHash256 NameHash = HashBlake3String<FHash256>(It.first);
FileSourceIdMap[NameHash] = SourceIndex;
}
bSourceManifestOk = MergeManifests(SourceDirectoryManifest, LoadedManifest, bCaseSensitiveTargetFileSystem);
if (!bSourceManifestOk)
{
break;
}
++SourceIndex;
}
ProxyPool.Dealloc(std::move(Proxy)); // TODO: RAII helper for pooled proxy connections
}
else if (!SyncOptions.SourceManifestOverride.empty())
{
bSourceManifestOk = LoadDirectoryManifest(SourceDirectoryManifest, SourcePath, SyncOptions.SourceManifestOverride);
// TODO: load pack index files
if (!bSourceManifestOk)
{
UNSYNC_ERROR(L"Could not load explicit manifest file");
return false;
}
}
else
{
for (const FPath& ThisSourcePath : AllSources)
{
std::unique_ptr<FProxyFileSystem> ProxyFileSystem;
if (bServerSource)
{
const FRemoteProtocolFeatures& RemoteFeatures = ProxyPool.GetFeatures();
if (!RemoteFeatures.bDirectoryListing)
{
UNSYNC_ERROR(L"Remote server does not support directory listing");
return false;
}
if (!RemoteFeatures.bFileDownload)
{
UNSYNC_ERROR(L"Remote server does not support direct file downloads");
return false;
}
ProxyFileSystem = std::make_unique<FRemoteFileSystem>(ToString(ThisSourcePath), ProxyPool);
}
else
{
ProxyFileSystem = std::make_unique<FPhysicalFileSystem>(ThisSourcePath);
}
if (!LoadAndMergeSourceManifest(SourceDirectoryManifest,
PackIndexFiles,
*ProxyFileSystem,
ThisSourcePath,
TargetTempPath,
SyncFilter,
bCaseSensitiveTargetFileSystem))
{
return false;
}
}
}
{
UNSYNC_VERBOSE(L"Loaded manifest properties:");
UNSYNC_LOG_INDENT;
FDirectoryManifestInfo ManifestInfo = GetManifestInfo(SourceDirectoryManifest, false /*bGenerateSignature*/);
LogManifestInfo(ELogLevel::Debug, ManifestInfo);
if (ProxyPool.RemoteDesc.Protocol == EProtocolFlavor::Jupiter && ManifestInfo.NumMacroBlocks == 0)
{
UNSYNC_ERROR(L"Manifest must contain macro blocks when using Jupiter");
return false;
}
}
ManifestLoadTimingLogger.Finish();
FTimingLogger TargetManifestTimingLogger("Target directory manifest generation time", ELogLevel::Info);
UNSYNC_LOG(L"Creating manifest for directory '%ls'", TargetPath.wstring().c_str());
// Propagate algorithm selection from source
const FAlgorithmOptions Algorithm = SourceDirectoryManifest.Algorithm;
FComputeBlocksParams LightweightManifestParams;
LightweightManifestParams.Algorithm = Algorithm;
LightweightManifestParams.bNeedBlocks = false;
LightweightManifestParams.BlockSize = 0;
FDirectoryManifest TargetDirectoryManifest = CreateDirectoryManifest(TargetPath, LightweightManifestParams);
TargetManifestTimingLogger.Finish();
if (!bCaseSensitiveTargetFileSystem)
{
std::vector<FPendingFileRename> PendingRenames;
PendingRenames = FixManifestFileNameCases(TargetDirectoryManifest, SourceDirectoryManifest);
if (!PendingRenames.empty())
{
UNSYNC_VERBOSE(L"Fixing inconsistent case of target files");
UNSYNC_LOG_INDENT;
if (!FixFileNameCases(TargetPath, PendingRenames))
{
return false;
}
}
}
uint32 StatSkipped = 0;
uint32 StatFullCopy = 0;
uint32 StatPartialCopy = 0;
std::atomic<uint64> NumFailedTasks = {};
std::atomic<uint64> StatSourceBytes = {};
std::atomic<uint64> StatBaseBytes = {};
std::vector<FFileSyncTask> AllFileTasks;
LogGlobalStatus(L"Scanning base directory");
UNSYNC_LOG(L"Scanning base directory");
FFileAttributeCache BaseAttribCache = CreateFileAttributeCache(BasePath, SyncFilter);
UNSYNC_LOG(L"Base files: %d", (uint32)BaseAttribCache.Map.size());
FFileAttributeCache SourceAttribCache;
if (bFileSystemSource && SyncOptions.bValidateSourceFiles)
{
LogGlobalStatus(L"Scanning source directory");
UNSYNC_LOG(L"Scanning source directory");
SourceAttribCache = CreateFileAttributeCache(SourcePath, SyncFilter);
}
// If variable blocks are used and we already have a manifest file from previous sync,
// then we can compute difference quickly based only on file timestamps and previously computed chunks.
FDirectoryManifest BaseDirectoryManifest;
bool bBaseDirectoryManifestValid = false;
bool bQuickDifferencePossible = false;
if (!SyncOptions.bFullDifference && SourceDirectoryManifest.Algorithm.ChunkingAlgorithmId == EChunkingAlgorithmID::VariableBlocks &&
PathExists(BaseManifestPath))
{
bBaseDirectoryManifestValid = LoadDirectoryManifest(BaseDirectoryManifest, BasePath, BaseManifestPath);
if (bBaseDirectoryManifestValid && AlgorithmOptionsCompatible(SourceDirectoryManifest.Algorithm, TargetDirectoryManifest.Algorithm))
{
bQuickDifferencePossible = true;
}
}
if (bQuickDifferencePossible)
{
UNSYNC_LOG(L"Quick file difference is allowed (use --full-diff option to override)");
}
uint64 TotalSourceSize = 0;
for (const auto& SourceManifestIt : SourceDirectoryManifest.Files)
{
const std::wstring& SourceFilename = SourceManifestIt.first;
const FHash256 SourceFilenameHash = HashBlake3String<FHash256>(SourceFilename);
if (!ShouldSync(SourceFilename))
{
StatSkipped++;
UNSYNC_VERBOSE2(L"Skipped '%ls' (excluded by sync filter)", SourceManifestIt.first.c_str());
continue;
}
const FFileManifest& SourceFileManifest = SourceManifestIt.second;
TotalSourceSize += SourceFileManifest.Size;
bool bTargetFileAttributesMatch = false;
auto TargetManifestIt = TargetDirectoryManifest.Files.find(SourceFilename);
if (TargetManifestIt != TargetDirectoryManifest.Files.end())
{
const FFileManifest& TargetFileManifest = TargetManifestIt->second;
if (SourceFileManifest.Size == TargetFileManifest.Size && SourceFileManifest.Mtime == TargetFileManifest.Mtime)
{
bTargetFileAttributesMatch = true;
}
}
if (bTargetFileAttributesMatch && !SyncOptions.bFullDifference)
{
UNSYNC_VERBOSE2(L"Skipped '%ls' (up to date)", SourceManifestIt.first.c_str());
StatSkipped++;
continue;
}
FPath SourceFilePath = SourceManifestIt.second.CurrentPath;
FPath BaseFilePath = BasePath / ToPath(SourceManifestIt.first);
FPath TargetFilePath = TargetPath / ToPath(SourceManifestIt.first);
FPath ResolvedSourceFilePath = ResolvePath(SourceFilePath);
uint32 SourceId = 0;
{
FHash256 NameHash = HashBlake3String<FHash256>(SourceFilename);
auto FoundSourceId = FileSourceIdMap.find(NameHash);
if (FoundSourceId != FileSourceIdMap.end())
{
SourceId = FoundSourceId->second;
}
}
if (bFileSystemSource && SyncOptions.bValidateSourceFiles)
{
FFileAttributes SourceFileAttrib = GetFileAttrib(ResolvedSourceFilePath, &SourceAttribCache);
if (!SourceFileAttrib.bValid)
{
UNSYNC_ERROR(L"Source file '%ls' is declared in manifest but does not exist. Manifest may be wrong or out of date.",
SourceFilePath.wstring().c_str());
bSourceManifestOk = false;
}
if (bSourceManifestOk && SourceFileAttrib.Size != SourceFileManifest.Size)
{
UNSYNC_ERROR(
L"Source file '%ls' size (%lld bytes) does not match the manifest (%lld bytes). Manifest may be wrong or out of date.",
SourceFilePath.wstring().c_str(),
SourceFileAttrib.Size,
SourceFileManifest.Size);
bSourceManifestOk = false;
}
if (bSourceManifestOk && SourceFileAttrib.Mtime != SourceFileManifest.Mtime)
{
UNSYNC_ERROR(
L"Source file '%ls' modification time (%lld) does not match the manifest (%lld). Manifest may be wrong or out of date.",
SourceFilePath.wstring().c_str(),
SourceFileAttrib.Mtime,
SourceFileManifest.Mtime);
bSourceManifestOk = false;
}
}
if (bSourceManifestOk)
{
FFileAttributes BaseFileAttrib = GetCachedFileAttrib(BaseFilePath, BaseAttribCache);
if (!BaseFileAttrib.bValid)
{
UNSYNC_VERBOSE2(L"Dirty file: '%ls' (no base data)", SourceFilename.c_str());
StatFullCopy++;
}
else
{
if (bTargetFileAttributesMatch && SyncOptions.bFullDifference)
{
UNSYNC_VERBOSE2(L"Dirty file: '%ls' (forced by --full-diff)", SourceManifestIt.first.c_str());
}
else
{
UNSYNC_VERBOSE2(L"Dirty file: '%ls'", SourceManifestIt.first.c_str());
}
StatPartialCopy++;
if (bFileSystemSource && SyncOptions.bValidateSourceFiles && !SourceAttribCache.Exists(ResolvedSourceFilePath) &&
!PathExists(ResolvedSourceFilePath))
{
UNSYNC_VERBOSE(L"Source file '%ls' does not exist", SourceFilePath.wstring().c_str());
continue;
}
}
FFileSyncTask Task;
Task.OriginalSourceFilePath = std::move(SourceFilePath);
Task.ResolvedSourceFilePath = std::move(ResolvedSourceFilePath);
Task.BaseFilePath = std::move(BaseFilePath);
Task.TargetFilePath = std::move(TargetFilePath);
Task.SourceManifest = &SourceFileManifest;
Task.SourceId = SourceId;
if (bQuickDifferencePossible)
{
UNSYNC_ASSERT(bBaseDirectoryManifestValid);
auto BaseManifestIt = BaseDirectoryManifest.Files.find(SourceFilename);
if (BaseManifestIt != BaseDirectoryManifest.Files.end())
{
const FFileManifest& BaseFileManifest = BaseManifestIt->second;
if (BaseFileManifest.Mtime == BaseFileAttrib.Mtime && BaseFileManifest.Size == BaseFileAttrib.Size)
{
if (ValidateBlockListT(BaseFileManifest.Blocks))
{
Task.BaseManifest = &BaseFileManifest;
}
}
}
}
AllFileTasks.push_back(Task);
}
}
if (SourceDirectoryManifest.Files.empty())
{
UNSYNC_ERROR(L"Source directory manifest is empty");
bSourceManifestOk = false;
}
if (!bSourceManifestOk)
{
return false;
}
LogGlobalStatus(L"Computing difference");
UNSYNC_LOG(L"Computing difference ...");
uint64 EstimatedNeedBytesFromSource = 0;
uint64 EstimatedNeedBytesFromBase = 0;
uint64 TotalSyncSizeBytes = 0;
{
UNSYNC_LOG_INDENT;
auto TimeDiffBegin = TimePointNow();
auto DiffTask = [Algorithm, bQuickDifferencePossible](FFileSyncTask& Item) {
FLogVerbosityScope VerbosityScope(false); // turn off logging from threads
const FGenericBlockArray& SourceBlocks = Item.SourceManifest->Blocks;
if (Item.IsBaseValid() && PathExists(Item.BaseFilePath))
{
FNativeFile BaseFile(Item.BaseFilePath, EFileMode::ReadOnlyUnbuffered);
uint32 SourceBlockSize = Item.SourceManifest->BlockSize;
UNSYNC_VERBOSE(L"Computing difference for target '%ls' (base size: %.2f MB)",
Item.BaseFilePath.wstring().c_str(),
SizeMb(BaseFile.GetSize()));
if (bQuickDifferencePossible && Item.BaseManifest)
{
Item.NeedList = DiffManifestBlocks(Item.SourceManifest->Blocks, Item.BaseManifest->Blocks);
}
else if (Algorithm.ChunkingAlgorithmId == EChunkingAlgorithmID::FixedBlocks)
{
Item.NeedList =
DiffBlocks(BaseFile, SourceBlockSize, Algorithm.WeakHashAlgorithmId, Algorithm.StrongHashAlgorithmId, SourceBlocks);
}
else if (Algorithm.ChunkingAlgorithmId == EChunkingAlgorithmID::VariableBlocks)
{
Item.NeedList = DiffBlocksVariable(BaseFile,
SourceBlockSize,
Algorithm.WeakHashAlgorithmId,
Algorithm.StrongHashAlgorithmId,
SourceBlocks);
}
else
{
UNSYNC_FATAL(L"Unexpected file difference calculation mode");
}
}
else
{
Item.NeedList.Sequence.reserve(SourceBlocks.size());
Item.NeedList.Source.reserve(SourceBlocks.size());
for (const FGenericBlock& Block : SourceBlocks)
{
FNeedBlock NeedBlock;
NeedBlock.Size = Block.Size;
NeedBlock.SourceOffset = Block.Offset;
NeedBlock.TargetOffset = Block.Offset;
NeedBlock.Hash = Block.HashStrong;
Item.NeedList.Source.push_back(NeedBlock);
Item.NeedList.Sequence.push_back(NeedBlock.Hash.ToHash128()); // #wip-widehash
}
}
Item.NeedBytesFromSource = ComputeSize(Item.NeedList.Source);
Item.NeedBytesFromBase = ComputeSize(Item.NeedList.Base);
Item.TotalSizeBytes = Item.SourceManifest->Size;
UNSYNC_ASSERT(Item.NeedBytesFromSource + Item.NeedBytesFromBase == Item.TotalSizeBytes);
};
ParallelForEach(AllFileTasks, DiffTask);
auto TimeDiffEnd = TimePointNow();
double Duration = DurationSec(TimeDiffBegin, TimeDiffEnd);
UNSYNC_LOG(L"Difference complete in %.3f sec", Duration);
for (FFileSyncTask& Item : AllFileTasks)
{
EstimatedNeedBytesFromSource += Item.NeedBytesFromSource;
EstimatedNeedBytesFromBase += Item.NeedBytesFromBase;
TotalSyncSizeBytes += Item.TotalSizeBytes;
}
UNSYNC_LOG(L"Total need from source: %.2f MB", SizeMb(EstimatedNeedBytesFromSource));
UNSYNC_LOG(L"Total need from base: %.2f MB", SizeMb(EstimatedNeedBytesFromBase));
uint64 AvailableDiskBytes = SyncOptions.bCheckAvailableSpace ? GetAvailableDiskSpace(TargetPath) : ~0ull;
if (TotalSyncSizeBytes > AvailableDiskBytes)
{
UNSYNC_ERROR(
L"Sync requires %.0f MB (%llu bytes) of disk space, but only %.0f MB (%llu bytes) is available. "
L"Use --no-space-validation flag to suppress this check.",
SizeMb(TotalSyncSizeBytes),
TotalSyncSizeBytes,
SizeMb(AvailableDiskBytes),
AvailableDiskBytes);
return false;
}
}
GGlobalProgressCurrent = 0;
GGlobalProgressTotal =
EstimatedNeedBytesFromSource * GLOBAL_PROGRESS_SOURCE_SCALE + EstimatedNeedBytesFromBase * GLOBAL_PROGRESS_BASE_SCALE;
std::unique_ptr<FScavengeDatabase> ScavengeDatabase;
if (!SyncOptions.ScavengeRoot.empty())
{
UNSYNC_LOG(L"Scavenging blocks from existing data sets");
UNSYNC_LOG_INDENT;
FTimePoint ScavengeDbTimeBegin = TimePointNow();
TArrayView<FFileSyncTask> AllFileTasksView = MakeView(AllFileTasks.data(), AllFileTasks.size());
ScavengeDatabase = std::unique_ptr<FScavengeDatabase>(FScavengeDatabase::BuildFromFileSyncTasks(SyncOptions, AllFileTasksView));
double Duration = DurationSec(ScavengeDbTimeBegin, TimePointNow());
UNSYNC_LOG(L"Done in %.3f sec", Duration);
}
LogGlobalProgress();
if (ProxyPool.IsValid())
{
LogGlobalStatus(L"Connecting to server");
UNSYNC_LOG(L"Connecting to %hs server '%hs:%d' ...",
ToString(ProxyPool.RemoteDesc.Protocol),
ProxyPool.RemoteDesc.Host.Address.c_str(),
ProxyPool.RemoteDesc.Host.Port);
UNSYNC_LOG_INDENT;
std::unique_ptr<FProxy> Proxy = ProxyPool.Alloc();
if (Proxy.get() && Proxy->IsValid())
{
// TODO: report TLS status
// ESocketSecurity security = proxy->get_socket_security();
// UNSYNC_LOG(L"Connection established (security: %hs)", ToString(security));
UNSYNC_LOG(L"Connection established");
UNSYNC_LOG(L"Building block request map");
const bool bProxyHasData = Proxy->Contains(SourceDirectoryManifest);
ProxyPool.Dealloc(std::move(Proxy));
if (bProxyHasData)
{
FBlockRequestMap BlockRequestMap;
BlockRequestMap.Init(SourceDirectoryManifest.Algorithm.StrongHashAlgorithmId, AllSources);
for (const FFileSyncTask& Task : AllFileTasks)
{
BlockRequestMap.AddFileBlocks(Task.SourceId, Task.OriginalSourceFilePath, Task.ResolvedSourceFilePath, *Task.SourceManifest);
}
// Override loose file blocks with pack files
for (const FPackIndexDatabase& Pack : PackIndexFiles)
{
FPath ResolvedDataPackPath = ResolvePath(Pack.DataPath);
BlockRequestMap.AddPackBlocks(Pack.DataPath, ResolvedDataPackPath, MakeView(Pack.Entries));
}
ProxyPool.SetRequestMap(std::move(BlockRequestMap));
}
else
{
UNSYNC_WARNING(L"Remote server does not have the data referenced by manifest");
ProxyPool.Invalidate();
}
}
else
{
Proxy = nullptr;
ProxyPool.Invalidate();
}
}
else
{
// TODO: bail out if remote connection is required for the download,
// such as when downloading data purely from Jupiter.
UNSYNC_VERBOSE(L"Attempting to sync without remote server connection");
}
LogGlobalStatus(L"Copying files");
UNSYNC_LOG(L"Copying files ...");
{
// Throttle background tasks by trying to keep them to some sensible memory budget. Best effort only, not a hard limit.
const uint64 BackgroundTaskMemoryBudget = SyncOptions.BackgroundTaskMemoryBudget;
const uint64 TargetTotalSizePerTaskBatch = BackgroundTaskMemoryBudget;
const uint64 MaxFilesPerTaskBatch = SyncOptions.MaxFilesPerTask;
UNSYNC_VERBOSE2(L"Background task memory budget: %llu GB", BackgroundTaskMemoryBudget >> 30);
struct FBackgroundTaskResult
{
FPath TargetFilePath;
FFileSyncResult SyncResult;
bool bIsPartialCopy = false;
};
std::deque<FFileSyncTaskBatch> SyncTaskList;
std::mutex BackgroundTaskStatMutex;
std::vector<FBackgroundTaskResult> BackgroundTaskResults;
// Tasks are sorted by download size and processed by multiple threads.
// Large downloads are processed on the foreground thread and small ones on the background.
std::sort(AllFileTasks.begin(), AllFileTasks.end(), [](const FFileSyncTask& A, const FFileSyncTask& B) {
return A.NeedBytesFromSource < B.NeedBytesFromSource;
});
// Blocks for multiple files can be downloaded in one request.
// Group small file tasks into batches to reduce the number of individual download requests.
{
const uint64 MaxBatchDownloadSize = 4_MB;
FFileSyncTaskBatch CurrentBatch;
for (const FFileSyncTask& FileTask : AllFileTasks)
{
bool bShouldBreakBatch = false;
if (!CurrentBatch.FileTasks.empty())
{
if (CurrentBatch.NeedBytesFromSource + FileTask.NeedBytesFromSource > MaxBatchDownloadSize)
{
bShouldBreakBatch = true;
}
else if (CurrentBatch.FileTasks.size() >= MaxFilesPerTaskBatch)
{
bShouldBreakBatch = true;
}
else if (CurrentBatch.TotalSizeBytes >= TargetTotalSizePerTaskBatch)
{
bShouldBreakBatch = true;
}
}
if (bShouldBreakBatch)
{
SyncTaskList.push_back(std::move(CurrentBatch));
CurrentBatch = {};
}
CurrentBatch.FileTasks.push_back(&FileTask);
CurrentBatch.NeedBytesFromSource += FileTask.NeedBytesFromSource;
CurrentBatch.TotalSizeBytes += FileTask.TotalSizeBytes;
}
if (CurrentBatch.FileTasks.size())
{
SyncTaskList.push_back(std::move(CurrentBatch));
}
}
// Validate batching
{
uint64 TotalSyncSizeBatched = 0;
uint64 TotalFilesBatched = 0;
for (const FFileSyncTaskBatch& Batch : SyncTaskList)
{
TotalSyncSizeBatched += Batch.TotalSizeBytes;
TotalFilesBatched += Batch.FileTasks.size();
}
UNSYNC_ASSERT(TotalFilesBatched == AllFileTasks.size());
UNSYNC_ASSERT(TotalSyncSizeBatched == TotalSyncSizeBytes);
}
auto FileSyncTaskBody = [Algorithm,
&SyncOptions,
&StatSourceBytes,
&StatBaseBytes,
&BackgroundTaskStatMutex,
&BackgroundTaskResults,
&NumFailedTasks,
&ScavengeDatabase,
&ProxyPool](const FFileSyncTask& Item, FBlockCache* BlockCache, bool bBackground) {
UNSYNC_VERBOSE(L"Copy '%ls' (%ls)", Item.TargetFilePath.wstring().c_str(), (Item.NeedBytesFromBase) ? L"partial" : L"full");
FDeferredOpenReader BaseFile(
[&Item]
{
if (Item.IsBaseValid())
{
UNSYNC_VERBOSE(L"Opening base file '%ls'", Item.BaseFilePath.wstring().c_str());
LogStatus(Item.BaseFilePath.wstring().c_str(), L"Opening base file");
return std::unique_ptr<FIOReader>(new FNativeFile(Item.BaseFilePath, EFileMode::ReadOnlyUnbuffered));
}
else
{
return std::unique_ptr<FIOReader>(new FNullReaderWriter(0));
}
});
const FGenericBlockArray& SourceBlocks = Item.SourceManifest->Blocks;
uint32 SourceBlockSize = Item.SourceManifest->BlockSize;
FSyncFileOptions SyncFileOptions;
SyncFileOptions.Algorithm = Algorithm;
SyncFileOptions.BlockSize = SourceBlockSize;
SyncFileOptions.ProxyPool = &ProxyPool;
SyncFileOptions.BlockCache = BlockCache;
SyncFileOptions.ScavengeDatabase = ScavengeDatabase.get();
SyncFileOptions.bValidateTargetFiles = SyncOptions.bValidateTargetFiles;
SyncFileOptions.SourceType = SyncOptions.SourceType;
FFileSyncResult SyncResult =
SyncFile(Item.NeedList, Item.ResolvedSourceFilePath, SourceBlocks, BaseFile, Item.TargetFilePath, SyncFileOptions);
LogStatus(Item.TargetFilePath.wstring().c_str(), SyncResult.Succeeded() ? L"Succeeded" : L"Failed");
if (SyncResult.Succeeded())
{
StatSourceBytes += SyncResult.SourceBytes;
StatBaseBytes += SyncResult.BaseBytes;
UNSYNC_ASSERT(SyncResult.SourceBytes + SyncResult.BaseBytes == Item.TotalSizeBytes);
if (!GDryRun)
{
BaseFile.Close();
if (Item.SourceManifest->Mtime)
{
SetFileMtime(Item.TargetFilePath, Item.SourceManifest->Mtime);
}
if (Item.SourceManifest->bReadOnly)
{
SetFileReadOnly(Item.TargetFilePath, true);
}
if (Item.SourceManifest->bIsExecutable)
{
SetFileExecutable(Item.TargetFilePath, true);
}
}
if (bBackground)
{
FBackgroundTaskResult Result;
Result.TargetFilePath = Item.TargetFilePath;
Result.SyncResult = SyncResult;
Result.bIsPartialCopy = Item.NeedBytesFromBase != 0;
std::lock_guard<std::mutex> LockGuard(BackgroundTaskStatMutex);
BackgroundTaskResults.push_back(Result);
}
}
else
{
if (SyncResult.SystemErrorCode.value())
{
UNSYNC_ERROR(L"Sync failed from '%ls' to '%ls'. Status: %ls, system error code: %d %hs",
Item.ResolvedSourceFilePath.wstring().c_str(),
Item.TargetFilePath.wstring().c_str(),
ToString(SyncResult.Status),
SyncResult.SystemErrorCode.value(),
SyncResult.SystemErrorCode.message().c_str());
}
else
{
UNSYNC_ERROR(L"Sync failed from '%ls' to '%ls'. Status: %ls.",
Item.ResolvedSourceFilePath.wstring().c_str(),
Item.TargetFilePath.wstring().c_str(),
ToString(SyncResult.Status));
}
NumFailedTasks++;
}
};
std::atomic<uint64> NumBackgroundTasks = {};
std::atomic<uint64> NumForegroundTasks = {};
FTaskGroup BackgroundTaskGroup = GScheduler->CreateTaskGroup();
FTaskGroup ForegroundTaskGroup = GScheduler->CreateTaskGroup();
std::atomic<uint64> BackgroundTaskMemory = {};
std::atomic<uint64> RemainingSourceBytes = EstimatedNeedBytesFromSource;
std::mutex SchedulerMutex;
std::condition_variable SchedulerEvent;
while (!SyncTaskList.empty())
{
if (NumForegroundTasks == 0)
{
FFileSyncTaskBatch LocalTaskBatch = std::move(SyncTaskList.back());
SyncTaskList.pop_back();
++NumForegroundTasks;
RemainingSourceBytes -= LocalTaskBatch.NeedBytesFromSource;
ForegroundTaskGroup.run([TaskBatch = std::move(LocalTaskBatch),
&SchedulerEvent,
&NumForegroundTasks,
&FileSyncTaskBody,
&ProxyPool,
Algorithm,
LogVerbose = GLogVerbose]() {
FLogVerbosityScope VerbosityScope(LogVerbose);
std::unique_ptr<FBlockCache> BlockCache;
if (TaskBatch.FileTasks.size() > 1 && ProxyPool.IsValid())
{
BlockCache = TaskBatch.CreateBlockCache(ProxyPool, Algorithm.StrongHashAlgorithmId);
}
for (const FFileSyncTask* Task : TaskBatch.FileTasks)
{
FileSyncTaskBody(*Task, BlockCache.get(), false);
}
--NumForegroundTasks;
SchedulerEvent.notify_one();
});
continue;
}
const uint32 MaxBackgroundTasks = std::min<uint32>(8, GMaxThreads - 1);
if (NumBackgroundTasks < MaxBackgroundTasks && (SyncTaskList.front().NeedBytesFromSource < RemainingSourceBytes / 4) &&
(BackgroundTaskMemory + SyncTaskList.front().TotalSizeBytes < BackgroundTaskMemoryBudget))
{
FFileSyncTaskBatch LocalTaskBatch = std::move(SyncTaskList.front());
SyncTaskList.pop_front();
BackgroundTaskMemory += LocalTaskBatch.TotalSizeBytes;
++NumBackgroundTasks;
RemainingSourceBytes -= LocalTaskBatch.NeedBytesFromSource;
BackgroundTaskGroup.run([TaskBatch = std::move(LocalTaskBatch),
&SchedulerEvent,
&NumBackgroundTasks,
&FileSyncTaskBody,
&BackgroundTaskMemory,
&ProxyPool,
Algorithm]() {
FLogVerbosityScope VerbosityScope(false); // turn off logging from background threads
std::unique_ptr<FBlockCache> BlockCache;
if (TaskBatch.FileTasks.size() > 1 && ProxyPool.IsValid())
{
BlockCache = TaskBatch.CreateBlockCache(ProxyPool, Algorithm.StrongHashAlgorithmId);
}
for (const FFileSyncTask* Task : TaskBatch.FileTasks)
{
FileSyncTaskBody(*Task, BlockCache.get(), true);
}
BackgroundTaskMemory -= TaskBatch.TotalSizeBytes;
--NumBackgroundTasks;
SchedulerEvent.notify_one();
});
continue;
}
if (GScheduler->ExecuteTasksUntilIdle())
{
continue;
}
std::unique_lock<std::mutex> SchedulerLock(SchedulerMutex);
SchedulerEvent.wait(SchedulerLock);
}
ForegroundTaskGroup.wait();
if (NumBackgroundTasks != 0)
{
UNSYNC_LOG(L"Waiting for background tasks to complete");
}
BackgroundTaskGroup.wait();
UNSYNC_ASSERT(RemainingSourceBytes == 0);
bool bAllBackgroundTasksSucceeded = true;
uint32 NumBackgroundSyncFiles = 0;
uint64 DownloadedBackgroundBytes = 0;
for (const FBackgroundTaskResult& Item : BackgroundTaskResults)
{
if (Item.SyncResult.Succeeded())
{
UNSYNC_VERBOSE2(L"Copied '%ls' (%ls, background)",
Item.TargetFilePath.wstring().c_str(),
Item.bIsPartialCopy ? L"partial" : L"full");
++NumBackgroundSyncFiles;
DownloadedBackgroundBytes += Item.SyncResult.SourceBytes;
}
else
{
bAllBackgroundTasksSucceeded = false;
}
}
if (NumBackgroundSyncFiles)
{
UNSYNC_VERBOSE(L"Background file copies: %d (%.2f MB)", NumBackgroundSyncFiles, SizeMb(DownloadedBackgroundBytes));
}
if (!bAllBackgroundTasksSucceeded)
{
for (const FBackgroundTaskResult& Item : BackgroundTaskResults)
{
if (!Item.SyncResult.Succeeded())
{
UNSYNC_ERROR(L"Failed to copy file '%ls' on background task. Status: %ls, system error code: %d %hs",
Item.TargetFilePath.wstring().c_str(),
ToString(Item.SyncResult.Status),
Item.SyncResult.SystemErrorCode.value(),
Item.SyncResult.SystemErrorCode.message().c_str());
}
}
UNSYNC_ERROR(L"Background file copy process failed!");
}
}
const bool bSyncSucceeded = NumFailedTasks.load() == 0;
if (bSyncSucceeded && SyncOptions.bCleanup)
{
UNSYNC_LOG(L"Deleting unnecessary files");
UNSYNC_LOG_INDENT;
DeleteUnnecessaryFiles(TargetPath, TargetDirectoryManifest, SourceDirectoryManifest, SyncFilter);
}
// Save the source directory manifest on success.
// It can be used to speed up the diffing process during next sync.
if (bSyncSucceeded && !GDryRun)
{
bool bSaveOk = SaveDirectoryManifest(SourceDirectoryManifest, TargetManifestPath);
if (!bSaveOk)
{
UNSYNC_ERROR(L"Failed to save manifest after sync");
}
}
UNSYNC_LOG(L"Skipped files: %d, full copies: %d, partial copies: %d", StatSkipped, StatFullCopy, StatPartialCopy);
UNSYNC_LOG(L"Copied from source: %.2f MB, copied from base: %.2f MB", SizeMb(StatSourceBytes), SizeMb(StatBaseBytes));
UNSYNC_LOG(L"Sync completed %ls", bSyncSucceeded ? L"successfully" : L"with errors (see log for details)");
double ElapsedSeconds = DurationSec(TimeBegin, TimePointNow());
UNSYNC_VERBOSE2(L"Sync time: %.2f seconds", ElapsedSeconds);
if (ProxyPool.IsValid() && ProxyPool.GetFeatures().bTelemetry)
{
FTelemetryEventSyncComplete Event;
Event.ClientVersion = GetVersionString();
Event.Session = ProxyPool.GetSessionId();
Event.Source = ConvertWideToUtf8(SourcePath.wstring());
Event.ClientHostNameHash = GetAnonymizedMachineIdString();
Event.TotalBytes = TotalSourceSize;
Event.SourceBytes = StatSourceBytes;
Event.BaseBytes = StatBaseBytes;
Event.SkippedFiles = StatSkipped;
Event.FullCopyFiles = StatFullCopy;
Event.PartialCopyFiles = StatPartialCopy;
Event.Elapsed = ElapsedSeconds;
Event.bSuccess = bSyncSucceeded;
ProxyPool.SendTelemetryEvent(Event);
}
return bSyncSucceeded;
}
FNeedListSize
ComputeNeedListSize(const FNeedList& NeedList)
{
FNeedListSize Result = {};
for (const FNeedBlock& Block : NeedList.Base)
{
Result.TotalBytes += Block.Size;
Result.BaseBytes += Block.Size;
}
for (const FNeedBlock& Block : NeedList.Source)
{
Result.TotalBytes += Block.Size;
Result.SourceBytes += Block.Size;
}
return Result;
}
const wchar_t*
ToString(EFileSyncStatus Status)
{
switch (Status)
{
default:
return L"UNKNOWN";
case EFileSyncStatus::Ok:
return L"Ok";
case EFileSyncStatus::ErrorUnknown:
return L"Unknown error";
case EFileSyncStatus::ErrorFullCopy:
return L"Full file copy failed";
case EFileSyncStatus::ErrorValidation:
return L"Patched file validation failed";
case EFileSyncStatus::ErrorFinalRename:
return L"Final file rename failed";
case EFileSyncStatus::ErrorTargetFileCreate:
return L"Target file creation failed";
case EFileSyncStatus::ErrorBuildTargetFailed:
return L"Failed to build target";
}
}
} // namespace unsync