// Copyright Epic Games, Inc. All Rights Reserved. #include "UnsyncCmdInfo.h" #include "UnsyncFile.h" #include "UnsyncManifest.h" #include "UnsyncFilter.h" #include "UnsyncSerialization.h" #include #if __has_include() # include #endif namespace unsync { static void FilterManifest(const FSyncFilter& SyncFilter, FDirectoryManifest& Manifest) { auto It = Manifest.Files.begin(); while (It != Manifest.Files.end()) { if (SyncFilter.ShouldSync(It->first)) { ++It; } else { It = Manifest.Files.erase(It); } } } static THashMap BuildBlockMap(const FDirectoryManifest& Manifest, bool bNeedMacroBlocks) { THashMap Result; for (const auto& It : Manifest.Files) { const FFileManifest& File = It.second; if (bNeedMacroBlocks) { for (const FGenericBlock& Block : File.MacroBlocks) { Result[Block.HashStrong] = Block; } } else { for (const FGenericBlock& Block : File.Blocks) { Result[Block.HashStrong] = Block; } } } return Result; } void LogManifestDiff(ELogLevel LogLevel, const FDirectoryManifest& ManifestA, const FDirectoryManifest& ManifestB) { THashMap BlocksA = BuildBlockMap(ManifestA, false); THashMap BlocksB = BuildBlockMap(ManifestB, false); THashMap MacroBlocksA = BuildBlockMap(ManifestA, true); THashMap MacroBlocksB = BuildBlockMap(ManifestB, true); uint32 NumCommonBlocks = 0; uint64 TotalCommonBlockSize = 0; uint64 TotalSizeA = 0; uint64 TotalSizeB = 0; uint64 PatchSizeFromAtoB = 0; for (const auto& ItA : BlocksA) { TotalSizeA += ItA.second.Size; auto ItB = BlocksB.find(ItA.first); if (ItB != BlocksB.end()) { NumCommonBlocks++; TotalCommonBlockSize += ItA.second.Size; } } for (const auto& ItB : BlocksB) { TotalSizeB += ItB.second.Size; if (BlocksA.find(ItB.first) == BlocksA.end()) { PatchSizeFromAtoB += ItB.second.Size; } } uint32 NumCommonMacroBlocks = 0; uint64 TotalCommonMacroBlockSize = 0; for (const auto& ItA : MacroBlocksA) { auto ItB = MacroBlocksB.find(ItA.first); if (ItB != MacroBlocksB.end()) { NumCommonMacroBlocks++; TotalCommonMacroBlockSize += ItA.second.Size; } } LogPrintf(LogLevel, L"Common macro blocks: %d, %.3f MB\n", NumCommonMacroBlocks, SizeMb(TotalCommonMacroBlockSize)); LogPrintf(LogLevel, L"Common blocks: %d, %.3f MB (%.2f%% of A, %.2f%% of B)\n", NumCommonBlocks, SizeMb(TotalCommonBlockSize), 100.0 * double(TotalCommonBlockSize) / double(TotalSizeA), 100.0 * double(TotalCommonBlockSize) / double(TotalSizeB)); LogPrintf(LogLevel, L"Patch size: %.3f MB\n", SizeMb(PatchSizeFromAtoB)); } static void LogDecodedManifestJson(const FDirectoryManifest& Manifest) { std::wstring Output; Output += L"{\n"; // main object const std::string_view StrongHash = ToString(Manifest.Algorithm.StrongHashAlgorithmId); const std::string_view WeakHash = ToString(Manifest.Algorithm.WeakHashAlgorithmId); const std::string_view Chunking = ToString(Manifest.Algorithm.ChunkingAlgorithmId); FormatJsonKeyValueStr(Output, L"type", L"unsync_manifest", L",\n"); FormatJsonKeyValueStr(Output, L"hash_strong", ConvertUtf8ToWide(StrongHash), L",\n"); FormatJsonKeyValueStr(Output, L"hash_weak", ConvertUtf8ToWide(WeakHash), L",\n"); FormatJsonKeyValueStr(Output, L"chunking", ConvertUtf8ToWide(Chunking), L",\n"); Output += fmt::format(LR"("files": [)"); Output += L"\n"; uint64 FileIndex = 0; for (const auto& FileIt : Manifest.Files) { const std::wstring& Name = FileIt.first; const FFileManifest FileManifest = FileIt.second; if (FileIndex != 0) { Output += L",\n"; } Output += L"{"; FormatJsonKeyValueStr(Output, L"name", Name, L","); FormatJsonKeyValueBool(Output, L"read_only", FileManifest.bReadOnly, L", "); FormatJsonKeyValueBool(Output, L"executable", FileManifest.bIsExecutable, L", "); FormatJsonKeyValueUInt(Output, L"mtime", FileManifest.Mtime, L", "); FormatJsonKeyValueUInt(Output, L"size", FileManifest.Size, L", "); FormatJsonKeyValueUInt(Output, L"block_size", FileManifest.BlockSize, L", "); FormatJsonKeyValueUInt(Output, L"num_blocks", FileManifest.Blocks.size(), L", "); FormatJsonKeyValueUInt(Output, L"num_macro_blocks", FileManifest.MacroBlocks.size(), L","); Output += L"\"blocks\": "; FormatJsonBlockArray(Output, FileManifest.Blocks); Output += L","; Output += L"\"macro_blocks\": "; FormatJsonBlockArray(Output, FileManifest.MacroBlocks); Output += L"}"; ++FileIndex; } Output += L"]\n"; // files Output += L"}\n"; // main object LogPrintf(ELogLevel::MachineReadable, Output.c_str()); } int32 CmdInfo(const FCmdInfoOptions& Options) { FPath DirectoryManifestPathA = IsDirectory(Options.InputA) ? (Options.InputA / ".unsync" / "manifest.bin") : Options.InputA; FPath DirectoryManifestPathB = IsDirectory(Options.InputB) ? (Options.InputB / ".unsync" / "manifest.bin") : Options.InputB; FDirectoryManifest ManifestA; bool bManifestAValid = LoadDirectoryManifest(ManifestA, Options.InputA, DirectoryManifestPathA); if (!bManifestAValid) { return 1; } LogPrintf(ELogLevel::Info, L"Manifest A: %ls\n", DirectoryManifestPathA.wstring().c_str()); if (Options.SyncFilter) { FilterManifest(*Options.SyncFilter, ManifestA); } { UNSYNC_LOG_INDENT; LogManifestInfo(ELogLevel::Info, ManifestA); } if (Options.bListFiles) { UNSYNC_LOG_INDENT; LogManifestFiles(ELogLevel::Info, ManifestA); } if (Options.bDecode) { LogDecodedManifestJson(ManifestA); return 0; } if (Options.InputB.empty()) { return 0; } LogPrintf(ELogLevel::Info, L"\n"); FDirectoryManifest ManifestB; bool bManifestBValid = LoadDirectoryManifest(ManifestB, Options.InputB, DirectoryManifestPathB); if (!bManifestBValid) { return 1; } LogPrintf(ELogLevel::Info, L"Manifest B: %ls\n", DirectoryManifestPathB.wstring().c_str()); if (Options.SyncFilter) { FilterManifest(*Options.SyncFilter, ManifestB); } { UNSYNC_LOG_INDENT; LogManifestInfo(ELogLevel::Info, ManifestB); } if (Options.bListFiles) { UNSYNC_LOG_INDENT; LogManifestFiles(ELogLevel::Info, ManifestB); } LogPrintf(ELogLevel::Info, L"\n"); LogPrintf(ELogLevel::Info, L"Difference:\n"); { UNSYNC_LOG_INDENT; LogManifestDiff(ELogLevel::Info, ManifestA, ManifestB); } return 0; } } // namespace unsync