// Copyright Epic Games, Inc. All Rights Reserved.

#include "ChunkDownloader.h"

#include "ChunkDownloaderLog.h"
#include "Async/AsyncWork.h"
#include "HAL/FileManager.h"
#include "HttpModule.h"
#include "Misc/CoreDelegates.h"
#include "Interfaces/IHttpRequest.h"
#include "Misc/SecureHash.h"
#include "Interfaces/IHttpResponse.h"
#include "UObject/UObjectGlobals.h"
#include "Misc/ConfigCacheIni.h"
#include "Download.h"
#include "Modules/ModuleManager.h"

#if PLATFORM_ANDROID || PLATFORM_IOS
#include "HAL/PlatformApplicationMisc.h"
#endif

#define LOCTEXT_NAMESPACE "ChunkDownloader"

static const FString EMBEDDED_MANIFEST = TEXT("EmbeddedManifest.txt");
static const FString LOCAL_MANIFEST = TEXT("LocalManifest.txt");
static const FString CACHED_BUILD_MANIFEST = TEXT("CachedBuildManifest.txt");
static const FString BUILD_ID_KEY = TEXT("BUILD_ID");

//static 
bool FChunkDownloader::CheckFileSha1Hash(const FString& FullPathOnDisk, const FString& Sha1HashStr)
{
	IFileHandle* FilePtr = IPlatformFile::GetPlatformPhysical().OpenRead(*FullPathOnDisk);
	if (FilePtr == nullptr)
	{
		UE_LOG(LogChunkDownloader, Error, TEXT("Unable to open %s for hash verify."), *FullPathOnDisk);
		return false;
	}

	// create a SHA1 reader
	FSHA1 HashContext;

	// read in 64K chunks to prevent raising the memory high water mark too much
	{
		static const int64 FILE_BUFFER_SIZE = 64 * 1024;
		uint8 Buffer[FILE_BUFFER_SIZE];
		int64 FileSize = FilePtr->Size();
		for (int64 Pointer = 0; Pointer<FileSize;)
		{
			// how many bytes to read in this iteration
			int64 SizeToRead = FileSize - Pointer;
			if (SizeToRead > FILE_BUFFER_SIZE)
			{
				SizeToRead = FILE_BUFFER_SIZE;
			}

			// read dem bytes
			if (!FilePtr->Read(Buffer, SizeToRead))
			{
				UE_LOG(LogChunkDownloader, Error, TEXT("Read error while validating '%s' at offset %lld."), *FullPathOnDisk, Pointer);

				// don't forget to close
				delete FilePtr;
				return false;
			}
			Pointer += SizeToRead;

			// update the hash
			HashContext.Update(Buffer, SizeToRead);
		}

		// done with the file
		delete FilePtr;
	}

	// close up shop
	HashContext.Final();
	uint8 FinalHash[FSHA1::DigestSize];
	HashContext.GetHash(FinalHash);

	// build the hash string we just computed
	FString LocalHashStr = TEXT("SHA1:");
	for (int Idx = 0; Idx < 20; Idx++)
	{
		LocalHashStr += FString::Printf(TEXT("%02X"), FinalHash[Idx]);
	}
	return Sha1HashStr == LocalHashStr;
}

static bool WriteStringAsUtf8TextFile(const FString& FileText, const FString& FilePath)
{
	// convert to UTF8
	FTCHARToUTF8 PakFileUtf8(*FileText);

	// open the file for writing
	bool bSuccess = false;
	IFileHandle* ManifestFile = IPlatformFile::GetPlatformPhysical().OpenWrite(*FilePath);
	if (ManifestFile != nullptr)
	{
		// write to the file
		if (ManifestFile->Write(reinterpret_cast<const uint8*>(PakFileUtf8.Get()), PakFileUtf8.Length()))
		{
			UE_LOG(LogChunkDownloader, Log, TEXT("Wrote to %s"), *FilePath);
			bSuccess = true;
		}
		else
		{
			UE_LOG(LogChunkDownloader, Error, TEXT("Write error writing to %s"), *FilePath);
		}

		// close the file
		delete ManifestFile;
	}
	else
	{
		UE_LOG(LogChunkDownloader, Error, TEXT("Unable open %s for writing."), *FilePath);
	}
	return bSuccess;
}

////////////////////////////////////////////////////////////////////////////////////////////

class FChunkDownloader::FMultiCallback
{
public:
	FMultiCallback(const FCallback& Callback)
		: OuterCallback(Callback)
	{
		IndividualCb = [this](bool bSuccess) {
			// update stats
			--NumPending;
			if (bSuccess)
				++NumSucceeded;
			else
				++NumFailed;

			// if we're the last one, trigger the outer callback
			if (NumPending <= 0)
			{
				check(NumPending == 0);
				if (OuterCallback)
				{
					OuterCallback(NumFailed <= 0);
				}

				// done with this
				delete this;
			}
		};
	}

	inline const FCallback& AddPending()
	{ 
		++NumPending;
		return IndividualCb;
	}

	inline int GetNumPending() const { return NumPending; }

	void Abort()
	{
		check(NumPending == 0);
		delete this;
	}

private:
	~FMultiCallback() {}

	int NumPending = 0;
	int NumSucceeded = 0;
	int NumFailed = 0;
	FCallback IndividualCb;
	FCallback OuterCallback;
};

////////////////////////////////////////////////////////////////////////////////////////////

class FChunkDownloader::FPakMountWork : public FNonAbandonableTask
{
public:
	friend class FAsyncTask<FPakMountWork>;

	void DoWork()
	{
		// try to mount the pak file
		if (FCoreDelegates::MountPak.IsBound())
		{
			uint32 PakReadOrder = PakFiles.Num();
			for (const TSharedRef<FPakFile>& PakFile : PakFiles)
			{
				FString FullPathOnDisk = (PakFile->bIsEmbedded ? EmbeddedFolder : CacheFolder) / PakFile->Entry.FileName;
				IPakFile* MountedPak = FCoreDelegates::MountPak.Execute(FullPathOnDisk, PakReadOrder);

#if !UE_BUILD_SHIPPING
				if (!MountedPak)
				{
					// This can fail because of the sandbox system - which the pak system doesn't understand.
					FString SandboxedPath = IFileManager::Get().ConvertToAbsolutePathForExternalAppForRead(*FullPathOnDisk);
					MountedPak = FCoreDelegates::MountPak.Execute(SandboxedPath, PakReadOrder);
				}
#endif

				if (MountedPak)
				{
					// record that we successfully mounted this pak file
					MountedPakFiles.Add(PakFile);
					--PakReadOrder;
				}
				else
				{
					UE_LOG(LogChunkDownloader, Error, TEXT("Unable to mount %s from chunk %d (mount operation failed)"), *FullPathOnDisk, ChunkId);
				}
			}
		}
		else
		{
			UE_LOG(LogChunkDownloader, Error, TEXT("Unable to mount chunk %d (no FCoreDelegates::MountPak bound)"), ChunkId);
		}
	}

	FORCEINLINE TStatId GetStatId() const
	{
		RETURN_QUICK_DECLARE_CYCLE_STAT(FPakMountWork, STATGROUP_ThreadPoolAsyncTasks);
	}

public: // inputs

	int32 ChunkId;

	// folders to save pak files into on disk
	FString CacheFolder;
	FString EmbeddedFolder;

	// mount these IN ORDER
	TArray<TSharedRef<FPakFile>> PakFiles;

	// callbacks
	TArray<FCallback> PostMountCallbacks;

public: // results

	// files which were successfully mounted
	TArray<TSharedRef<FPakFile>> MountedPakFiles;
};

////////////////////////////////////////////////////////////////////////////////////////////

FChunkDownloader::FChunkDownloader()
{
}

FChunkDownloader::~FChunkDownloader()
{
	// this will be true unless we forgot to have Finalize called.
	check(PakFiles.Num() <= 0);
}

static TArray<FPakFileEntry> ParseManifest(const FString& ManifestPath, TMap<FString,FString>* Properties = nullptr)
{
	int32 ExpectedEntries = -1;
	TArray<FPakFileEntry> Entries;
	IFileHandle* ManifestFile = IPlatformFile::GetPlatformPhysical().OpenRead(*ManifestPath);
	if (ManifestFile != nullptr)
	{
		const uint64 FileSize = ManifestFile->Size();
		if (FileSize > 0)
		{
			UE_LOG(LogChunkDownloader, Log, TEXT("Found manifest at %s"), *ManifestPath);

			// read the whole file into a buffer (expecting UTF-8 so null terminate)
			char* FileBuffer = new char[FileSize + 8]; // little extra since we're forcing null term in places outside bounds of a field
			if (ManifestFile->Read((uint8*)FileBuffer, FileSize))
			{
				FileBuffer[FileSize] = '\0';

				// make buffers for stuff read from each line
				char NameBuffer[512] = { 0 };
				uint64 FinalFileLen = 0;
				char VersionBuffer[512] = { 0 };
				int32 ChunkId = -1;
				char RelativeUrl[2048] = { 0 };

				// line end
				uint64 LineNum = 0;
				uint64 NextLineStart = 0;
				while (NextLineStart < FileSize)
				{
					uint64 LineStart = NextLineStart;
					++LineNum;

					// find the end of the line
					uint64 LineEnd = LineStart;
					while (LineEnd < FileSize && FileBuffer[LineEnd] != '\n' && FileBuffer[LineEnd] != '\r')
					{
						++LineEnd;
					}

					// find the end of the line
					NextLineStart = LineEnd + 1;
					while (NextLineStart < FileSize && (FileBuffer[NextLineStart] == '\n' || FileBuffer[NextLineStart] == '\r'))
					{
						++NextLineStart;
					}

					// see if this is a property
					if (FileBuffer[LineStart] == '$')
					{
						// parse the line
						char* NameStart = &FileBuffer[LineStart+1];
						char* NameEnd = FCStringAnsi::Strstr(NameStart, " = ");
						if (NameEnd != nullptr)
						{
							char* ValueStart = NameEnd + 3;
							char* ValueEnd = &FileBuffer[LineEnd];
							*NameEnd = '\0';
							*ValueEnd = '\0';

							FString Name = FUTF8ToTCHAR(NameStart, NameEnd-NameStart + 1).Get();
							FString Value = FUTF8ToTCHAR(ValueStart, ValueEnd-ValueStart + 1).Get();
							if (Properties != nullptr)
							{
								Properties->Add(Name,Value);
							}

							if (Name == TEXT("NUM_ENTRIES"))
							{
								ExpectedEntries = FCString::Atoi(*Value);
							}
						}
						continue;
					}

					// parse the line
#if PLATFORM_WINDOWS || PLATFORM_MICROSOFT
					if (!ensure(sscanf_s(&FileBuffer[LineStart], "%511[^\t]\t%llu\t%511[^\t]\t%d\t%2047[^\r\n]", 
						NameBuffer, (int)sizeof(NameBuffer), 
						&FinalFileLen, 
						VersionBuffer, (int)sizeof(VersionBuffer), 
						&ChunkId, 
						RelativeUrl, (int)sizeof(RelativeUrl)
					) == 5))
#else
					if (!ensure(sscanf(&FileBuffer[LineStart], "%511[^\t]\t%llu\t%511[^\t]\t%d\t%2047[^\r\n]", 
						NameBuffer, 
						&FinalFileLen, 
						VersionBuffer, 
						&ChunkId, 
						RelativeUrl
					) == 5))
#endif
					{
						UE_LOG(LogChunkDownloader, Error, TEXT("Manifest parse error at %s:%" UINT64_FMT), *ManifestPath, LineNum);
						continue;
					}

					// add a new pak file entry
					FPakFileEntry Entry;
					Entry.FileName = UTF8_TO_TCHAR(NameBuffer);
					Entry.FileSize = FinalFileLen;
					Entry.FileVersion = UTF8_TO_TCHAR(VersionBuffer);
					if (ChunkId >= 0)
					{
						Entry.ChunkId = ChunkId;
						Entry.RelativeUrl = UTF8_TO_TCHAR(RelativeUrl);
					}
					Entries.Add(Entry);
				}

				// all done
				delete[] FileBuffer;
			}
			else
			{
				UE_LOG(LogChunkDownloader, Error, TEXT("Read error loading manifest at %s"), *ManifestPath);
			}
		}
		else
		{
			UE_LOG(LogChunkDownloader, Log, TEXT("Empty manifest found at %s"), *ManifestPath);
		}
		
		// close the file
		delete ManifestFile;
	}
	else
	{
		UE_LOG(LogChunkDownloader, Log, TEXT("No manifest found at %s"), *ManifestPath);
	}

	if (ExpectedEntries >= 0 && ExpectedEntries != Entries.Num())
	{
		UE_LOG(LogChunkDownloader, Error, TEXT("Corrupt manifest at %s (expected %d entries, got %d)"), *ManifestPath, ExpectedEntries, Entries.Num());
		Entries.Empty();
		if (Properties != nullptr)
		{
			Properties->Empty();
		}
	}

	return Entries;
}

void FChunkDownloader::Initialize(const FString& InPlatformName, int32 TargetDownloadsInFlightIn)
{
	check(PakFiles.Num() == 0); // this means we didn't call Finalize
	check(!InPlatformName.IsEmpty());
	UE_LOG(LogChunkDownloader, Display, TEXT("Initializing with platform='%s'"), *InPlatformName);
    
    FPlatformMisc::AddAdditionalRootDirectory(FPaths::Combine(*FPaths::ProjectPersistentDownloadDir(), TEXT("pakcache")));

	// save platform name
	PlatformName = InPlatformName;

	// save target concurrency
	TargetDownloadsInFlight = TargetDownloadsInFlightIn;
	check(TargetDownloadsInFlight >= 1);

	// figure out our base dirs
	CacheFolder = FPaths::ProjectPersistentDownloadDir() / TEXT("PakCache/");
	EmbeddedFolder = FPaths::ProjectContentDir() / TEXT("EmbeddedPaks/");

	// make sure the cache folder exists
	IFileManager& FileManager = IFileManager::Get();
	if (!FileManager.MakeDirectory(*CacheFolder, true))
	{
		UE_LOG(LogChunkDownloader, Error, TEXT("Unable to create cache folder at '%s'"), *CacheFolder);
	}

	// see what's in the embedded chunks folder
	EmbeddedPaks.Empty();
	for (const FPakFileEntry& Entry : ParseManifest(EmbeddedFolder / EMBEDDED_MANIFEST))
	{
		// just index these
		EmbeddedPaks.Add(Entry.FileName, Entry);
	}

	// enumerate the caches dir and delete anything not in the local manifest
	TArray<FString> StrayFiles;
	FileManager.FindFiles(StrayFiles, *CacheFolder, TEXT("*.pak"));

	// load the LocalManifest to see what we've got on disk
	TArray<FPakFileEntry> LocalManifest = ParseManifest(CacheFolder / LOCAL_MANIFEST);
	if (LocalManifest.Num() > 0)
	{
		// make entries in PakFileInfo for each thing in the local cache (will fill in when BuildManifest is loaded)
		for (const FPakFileEntry& Entry : LocalManifest)
		{
			// make a new file info
			TSharedRef<FPakFile> FileInfo = MakeShared<FPakFile>();

			// copy over entry fields (more will be filled in by BuildManifest)
			FileInfo->Entry = Entry;

			// see if there's a partial or cached file
			FString LocalPath = CacheFolder / Entry.FileName;
			int64 SizeOnDiskInt = FileManager.FileSize(*LocalPath);
			if (SizeOnDiskInt > 0)
			{
				FileInfo->SizeOnDisk = (uint64)SizeOnDiskInt;
				if (FileInfo->SizeOnDisk > Entry.FileSize)
				{
					// abort adding this file info (it's too big, we'll delete it)
					UE_LOG(LogChunkDownloader, Warning, TEXT("Found '%s' on disk with size larger than LocalManifest indicates"), *LocalPath);
					bNeedsManifestSave = true;
					continue;
				}

				// see if this is a fullly cached file
				if (FileInfo->SizeOnDisk == Entry.FileSize)
				{
					// consider size match to be fully downloaded
					FileInfo->bIsCached = true;
				}

				// add the info
				PakFiles.Add(Entry.FileName, FileInfo);
			}
			else
			{
				// remove this from the local manifest and resave (may be that we crashed before the file download successfully started)
				UE_LOG(LogChunkDownloader, Log, TEXT("'%s' appears in LocalManifest but is not on disk (not necessarily a problem)"), *LocalPath);
				bNeedsManifestSave = true;
			}

			// remove from StrayFiles
			StrayFiles.RemoveSingle(Entry.FileName);
		}
	}

	// delete any stray files that weren't in the local manifest
	for (FString Orphan : StrayFiles)
	{
		bNeedsManifestSave = true;
		FString FullPathOnDisk = CacheFolder / Orphan;
		UE_LOG(LogChunkDownloader, Log, TEXT("Deleting orphaned file '%s'"), *FullPathOnDisk);
		if (!ensure(FileManager.Delete(*FullPathOnDisk)))
		{
			// log an error (best we can do)
			UE_LOG(LogChunkDownloader, Error, TEXT("Unable to delete '%s'"), *FullPathOnDisk);
		}
	}

	// resave the local manifest
	SaveLocalManifest(false);
}

bool FChunkDownloader::LoadCachedBuild(const FString& DeploymentName)
{
	// try to re-populate ContentBuildId and the cached manifest
	TMap<FString, FString> CachedManifestProps;
	TArray<FPakFileEntry> CachedManifest = ParseManifest(CacheFolder / CACHED_BUILD_MANIFEST, &CachedManifestProps);
	const FString* BuildId = CachedManifestProps.Find(BUILD_ID_KEY);
	if (BuildId == nullptr || BuildId->IsEmpty())
	{
		return false;
	}

	SetContentBuildId(DeploymentName, *BuildId);
	LoadManifest(CachedManifest);
	return true;
}

void FChunkDownloader::SetContentBuildId(const FString& DeploymentName, const FString& NewContentBuildId)
{
	// save the content build id
	ContentBuildId = NewContentBuildId;
	LastDeploymentName = DeploymentName;
	UE_LOG(LogChunkDownloader, Display, TEXT("Deployment = %s, ContentBuildId = %s"), *DeploymentName, *ContentBuildId);

	// read CDN urls from deployment configs
	TArray<FString> CdnBaseUrls;
	FString ConfigSectionName = FString::Printf(TEXT("/Script/Plugins.ChunkDownloader %s"), *DeploymentName);
	GConfig->GetArray(*ConfigSectionName, TEXT("CdnBaseUrls"), CdnBaseUrls, GGameIni);
	if (CdnBaseUrls.Num() <= 0)
	{
		// fall back to generic config
		GConfig->GetArray(TEXT("/Script/Plugins.ChunkDownloader"), TEXT("CdnBaseUrls"), CdnBaseUrls, GGameIni);
		if (CdnBaseUrls.Num() <= 0)
		{
			UE_LOG(LogChunkDownloader, Warning, TEXT("No CDN base URLs configured in [%s]. Chunk downloading will only be able to use embedded cache."), *ConfigSectionName);
		}
	}

	// combine CdnBaseUrls with ContentBuildId
	BuildBaseUrls.Empty();
	for (int32 i=0,n=CdnBaseUrls.Num();i<n;++i)
	{
		const FString& BaseUrl = CdnBaseUrls[i];
		check(!BaseUrl.IsEmpty());
		FString BuildUrl = BaseUrl / ContentBuildId;
		UE_LOG(LogChunkDownloader, Display, TEXT("ContentBaseUrl[%d] = %s"), i, *BuildUrl);
		BuildBaseUrls.Add(BuildUrl);
	}
}

void FChunkDownloader::UpdateBuild(const FString& DeploymentName, const FString& ContentBuildIdIn, const FCallback& Callback)
{
	check(!ContentBuildIdIn.IsEmpty());

	// if the build ID hasn't changed, there's no work to do
	if (ContentBuildIdIn == ContentBuildId && LastDeploymentName == DeploymentName)
	{
		ExecuteNextTick(Callback, true);
		return;
	}
	SetContentBuildId(DeploymentName, ContentBuildIdIn);

	// no overlapped UpdateBuild calls allowed, and Callback is required
	check(!UpdateBuildCallback);
	check(Callback);
	UpdateBuildCallback = Callback;

	// start the load/download process
	TryLoadBuildManifest(0);
}

void FChunkDownloader::Finalize()
{
	UE_LOG(LogChunkDownloader, Display, TEXT("Finalizing."));

	// wait for all mounts to finish
	WaitForMounts();

	// update the mount tasks (queues up callbacks)
	ensure(UpdateMountTasks(0.0f) == false);

	// cancel all downloads
	for (const auto& It : PakFiles)
	{
		const TSharedRef<FPakFile>& File = It.Value;
		if (File->Download.IsValid())
		{
			CancelDownload(File, false);
		}
	}

	// unmount all mounted chunks (best effort)
	for (const auto& It : Chunks)
	{
		const TSharedRef<FChunk>& Chunk = It.Value;
		if (Chunk->bIsMounted)
		{
			// unmount the paks (in reverse order)
			for (int32 i=Chunk->PakFiles.Num()-1;i >= 0; --i)
			{
				const TSharedRef<FPakFile>& PakFile = Chunk->PakFiles[i];
				UnmountPakFile(PakFile);
			}

			// clear the flag
			Chunk->bIsMounted = false;
		}
	}

	// clear pak files and chunks
	PakFiles.Empty();
	Chunks.Empty();

	// cancel any pending manifest request
	if (ManifestRequest.IsValid())
	{
		ManifestRequest->CancelRequest();
		ManifestRequest.Reset();
	}

	// any loading mode is de-facto complete
	if (PostLoadCallbacks.Num() > 0)
	{
		TArray<FCallback> Callbacks = MoveTemp(PostLoadCallbacks);
		PostLoadCallbacks.Empty();
		for (const auto& Callback : Callbacks)
		{
			ExecuteNextTick(Callback, false);
		}
	}

	// update is also de-facto complete
	if (UpdateBuildCallback)
	{
		FCallback Callback = MoveTemp(UpdateBuildCallback);
		ExecuteNextTick(Callback, false);
	}

	// clear out the content build id
	ContentBuildId.Empty();
}

void FChunkDownloader::SaveLocalManifest(bool bForce)
{
	if (bForce || bNeedsManifestSave)
	{
		// build the whole file into an FString (wish we could stream it out)
		int32 NumEntries = 0;
		for (const auto& It : PakFiles)
		{
			if (!It.Value->bIsEmbedded)
			{
				if (It.Value->SizeOnDisk > 0 || It.Value->Download.IsValid())
				{
					++NumEntries;
				}
			}
		}
		FString PakFileText = FString::Printf(TEXT("$NUM_ENTRIES = %d\n"), NumEntries);
		for (const auto& It : PakFiles)
		{
			if (!It.Value->bIsEmbedded)
			{
				if (It.Value->SizeOnDisk > 0 || It.Value->Download.IsValid())
				{
					// local manifest
					const FPakFileEntry& PakFile = It.Value->Entry;
					PakFileText += FString::Printf(TEXT("%s\t%llu\t%s\t-1\t/\n"), *PakFile.FileName, PakFile.FileSize, *PakFile.FileVersion);
				}
			}
		}

		// write the file
		FString ManifestPath = CacheFolder / LOCAL_MANIFEST;
		if (WriteStringAsUtf8TextFile(PakFileText, ManifestPath))
		{
			// mark that we have saved
			bNeedsManifestSave = false;
		}
	}
}

void FChunkDownloader::WaitForMounts()
{
	bool bWaiting = false;

	for (const auto& It : Chunks)
	{
		const TSharedRef<FChunk>& Chunk = It.Value;
		if (Chunk->MountTask != nullptr)
		{
			if (!bWaiting)
			{
				UE_LOG(LogChunkDownloader, Display, TEXT("Waiting for chunk mounts to complete..."));
				bWaiting = true;
			}

			// wait for the async task to end
			Chunk->MountTask->EnsureCompletion(true);

			// complete the task on the main thread
			CompleteMountTask(*Chunk);
			check(Chunk->MountTask == nullptr);
		}
	}

	if (bWaiting)
	{
		UE_LOG(LogChunkDownloader, Display, TEXT("...chunk mounts finished."));
	}
}

void FChunkDownloader::CancelDownload(const TSharedRef<FPakFile>& PakFile, bool bResult)
{
	if (PakFile->Download.IsValid())
	{
		// cancel the download itself
		PakFile->Download->Cancel(bResult);
		check(!PakFile->Download.IsValid());
	}
}

void FChunkDownloader::UnmountPakFile(const TSharedRef<FPakFile>& PakFile)
{
	// if it's already unmounted, don't do anything
	if (PakFile->bIsMounted)
	{
		// unmount
		if (ensure(FCoreDelegates::OnUnmountPak.IsBound()))
		{
			FString FullPathOnDisk = (PakFile->bIsEmbedded ? EmbeddedFolder : CacheFolder) / PakFile->Entry.FileName;
			if (ensure(FCoreDelegates::OnUnmountPak.Execute(FullPathOnDisk)))
			{
				// clear the mounted flag
				PakFile->bIsMounted = false;
			}
			else
			{
				UE_LOG(LogChunkDownloader, Error, TEXT("Unable to unmount %s"), *FullPathOnDisk);
			}
		}
		else
		{
			UE_LOG(LogChunkDownloader, Error, TEXT("Unable to unmount %s because no OnUnmountPak is bound"), *PakFile->Entry.FileName);
		}
	}
}

FChunkDownloader::EChunkStatus FChunkDownloader::GetChunkStatus(int32 ChunkId) const
{
	// do we know about this chunk at all?
	const TSharedRef<FChunk>* ChunkPtr = Chunks.Find(ChunkId);
	if (ChunkPtr == nullptr)
	{
		return EChunkStatus::Unknown;
	}
	const FChunk& Chunk = **ChunkPtr;

	// if it has no pak files, treat it the same as not found (shouldn't happen)
	if (!ensure(Chunk.PakFiles.Num() > 0))
	{
		return EChunkStatus::Unknown;
	}

	// see if it's fully mounted
	if (Chunk.bIsMounted)
	{
		return EChunkStatus::Mounted;
	}

	// count the number of paks in flight vs local
	int32 NumPaks = Chunk.PakFiles.Num(), NumCached = 0, NumDownloading = 0;
	for (const TSharedRef<FPakFile>& PakFile : Chunk.PakFiles)
	{
		if (PakFile->bIsCached)
		{
			++NumCached;
		}
		else if (PakFile->Download.IsValid())
		{
			++NumDownloading;
		}
	}

	if (NumCached >= NumPaks)
	{
		// all cached
		return EChunkStatus::Cached;
	}
	else if (NumCached + NumDownloading >= NumPaks)
	{
		// some downloads still in progress
		return EChunkStatus::Downloading;
	}
	else if (NumCached + NumDownloading > 0)
	{
		// any progress at all? (might be paused or partially preserved from manifest update)
		return EChunkStatus::Partial;
	}

	// nothing
	return EChunkStatus::Remote;
}

void FChunkDownloader::GetAllChunkIds(TArray<int32>& OutChunkIds) const
{
	Chunks.GetKeys(OutChunkIds);
}

// static
void FChunkDownloader::DumpLoadedChunks()
{
#if !WITH_EDITOR
	TSharedRef<FChunkDownloader> ChunkDownloader = FChunkDownloader::GetChecked();

	TArray<int32> ChunkIdList;
	ChunkDownloader->GetAllChunkIds(ChunkIdList);

	UE_LOG(LogChunkDownloader, Display, TEXT("Dumping loaded chunk status\n--------------------------"));
	for (int32 ChunkId : ChunkIdList)
	{
		auto ChunkStatus = ChunkDownloader->GetChunkStatus(ChunkId);
		UE_LOG(LogChunkDownloader, Display, TEXT("Chunk #%d => %s"), ChunkId, ChunkStatusToString(ChunkStatus));
	}
#endif
}

//static 
const TCHAR* FChunkDownloader::ChunkStatusToString(EChunkStatus Status)
{
	switch (Status)
	{
	case EChunkStatus::Mounted: return TEXT("Mounted");
	case EChunkStatus::Cached: return TEXT("Cached");
	case EChunkStatus::Downloading: return TEXT("Downloading");
	case EChunkStatus::Partial: return TEXT("Partial");
	case EChunkStatus::Remote: return TEXT("Remote");
	case EChunkStatus::Unknown: return TEXT("Unknown");
	default: return TEXT("Invalid");
	}
}

int FChunkDownloader::FlushCache()
{
	IFileManager& FileManager = IFileManager::Get();

	// wait for all mounts to finish
	WaitForMounts();

	UE_LOG(LogChunkDownloader, Display, TEXT("Flushing chunk caches at %s"), *CacheFolder);
	int FilesDeleted = 0, FilesSkipped = 0;
	for (const auto& It : Chunks)
	{
		const TSharedRef<FChunk>& Chunk = It.Value;
		check(Chunk->MountTask == nullptr); // we waited for mounts

		// cancel background downloads
		bool bDownloadPending = false;
		for (const TSharedRef<FPakFile>& PakFile : Chunk->PakFiles)
		{
			if (PakFile->Download.IsValid() && !PakFile->Download->HasCompleted())
			{
				// skip paks that are being downloaded
				bDownloadPending = true;
				break;
			}
		}

		// skip chunks that have a foreground download pending
		if (bDownloadPending)
		{
			for (const TSharedRef<FPakFile>& PakFile : Chunk->PakFiles)
			{
				if (PakFile->SizeOnDisk > 0)
				{
					// log that we skipped this one
					UE_LOG(LogChunkDownloader, Warning, TEXT("Could not flush %s (chunk %d) due to download in progress."), *PakFile->Entry.FileName, Chunk->ChunkId);
					++FilesSkipped;
				}
			}
		}
		else
		{
			// delete paks
			for (const TSharedRef<FPakFile>& PakFile : Chunk->PakFiles)
			{
				if (PakFile->SizeOnDisk > 0 && !PakFile->bIsEmbedded)
				{
					// log that we deleted this one
					FString FullPathOnDisk = CacheFolder / PakFile->Entry.FileName;
					if (ensure(FileManager.Delete(*FullPathOnDisk)))
					{
						UE_LOG(LogChunkDownloader, Log, TEXT("Deleted %s (chunk %d)."), *FullPathOnDisk, Chunk->ChunkId);
						++FilesDeleted;

						// flag uncached (may have been partial)
						PakFile->bIsCached = false;
						PakFile->SizeOnDisk = 0;
						bNeedsManifestSave = true;
					}
					else
					{
						// log an error (best we can do)
						UE_LOG(LogChunkDownloader, Error, TEXT("Unable to delete %s"), *FullPathOnDisk);
						++FilesSkipped;
					}
				}
			}
		}
	}

	// resave the manifest
	SaveLocalManifest(false);

	UE_LOG(LogChunkDownloader, Display, TEXT("Chunk cache flush complete. %d files deleted. %d files skipped."), FilesDeleted, FilesSkipped);
	return FilesSkipped;
}

int FChunkDownloader::ValidateCache()
{
	IFileManager& FileManager = IFileManager::Get();

	// wait for all mounts to finish
	WaitForMounts();

	UE_LOG(LogChunkDownloader, Display, TEXT("Starting inline chunk validation."));
	int ValidFiles = 0, InvalidFiles = 0, SkippedFiles = 0;
	for (const auto& It : PakFiles)
	{
		const TSharedRef<FPakFile>& PakFile = It.Value;
		if (PakFile->bIsCached && !PakFile->bIsEmbedded)
		{
			// we know how to validate certain hash versions
			bool bFileIsValid = false;
			if (PakFile->Entry.FileVersion.StartsWith(TEXT("SHA1:")))
			{
				// check the sha1 hash
				bFileIsValid = CheckFileSha1Hash(CacheFolder / PakFile->Entry.FileName, PakFile->Entry.FileVersion);
			}
			else
			{
				// we don't know how to validate this version format
				UE_LOG(LogChunkDownloader, Warning, TEXT("Unable to validate %s with version '%s'."), *PakFile->Entry.FileName, *PakFile->Entry.FileVersion);
				++SkippedFiles;
				continue;
			}

			// see if it's valid or not
			if (bFileIsValid)
			{
				// log valid
				UE_LOG(LogChunkDownloader, Log, TEXT("%s matches hash '%s'."), *PakFile->Entry.FileName, *PakFile->Entry.FileVersion);
				++ValidFiles;
			}
			else
			{
				// log invalid
				UE_LOG(LogChunkDownloader, Warning, TEXT("%s does NOT match hash '%s'."), *PakFile->Entry.FileName, *PakFile->Entry.FileVersion);
				++InvalidFiles;

				// delete invalid files
				FString FullPathOnDisk = CacheFolder / PakFile->Entry.FileName;
				if (ensure(FileManager.Delete(*FullPathOnDisk)))
				{
					UE_LOG(LogChunkDownloader, Log, TEXT("Deleted invalid pak %s (chunk %d)."), *FullPathOnDisk, PakFile->Entry.ChunkId);
					PakFile->bIsCached = false;
					PakFile->SizeOnDisk = 0;
					bNeedsManifestSave = true;
				}
			}
		}
	}

	// resave the manifest
	SaveLocalManifest(false);

	UE_LOG(LogChunkDownloader, Display, TEXT("Chunk validation complete. %d valid, %d invalid, %d skipped"), ValidFiles, InvalidFiles, SkippedFiles);
	return InvalidFiles;
}

void FChunkDownloader::BeginLoadingMode(const FCallback& Callback)
{
	check(Callback); // you can't start loading mode without a valid callback

	// see if we're already in loading mode
	if (PostLoadCallbacks.Num() > 0)
	{
		UE_LOG(LogChunkDownloader, Log, TEXT("JoinLoadingMode"));

		// just wait on the existing loading mode to finish
		PostLoadCallbacks.Add(Callback);
		return;
	}

	// start loading mode
	UE_LOG(LogChunkDownloader, Log, TEXT("BeginLoadingMode"));
#if PLATFORM_ANDROID || PLATFORM_IOS
	FPlatformApplicationMisc::ControlScreensaver(FPlatformApplicationMisc::Disable);
#endif

	// reset stats
	LoadingModeStats.LastError = FText();
	LoadingModeStats.BytesDownloaded = 0;
	LoadingModeStats.FilesDownloaded = 0;
	LoadingModeStats.ChunksMounted = 0;
	LoadingModeStats.LoadingStartTime = FDateTime::UtcNow();
	ComputeLoadingStats(); // recompute before binding callback in case there's nothing queued yet

	// set the callback
	PostLoadCallbacks.Add(Callback);
	LoadingCompleteLatch = 0;

	// compute again next frame (if nothing's queued by then, we'll fire the callback
	TWeakPtr<FChunkDownloader> WeakThisPtr = AsShared();
	FTSTicker::GetCoreTicker().AddTicker(FTickerDelegate::CreateLambda([WeakThisPtr](float dts) {
		TSharedPtr<FChunkDownloader> SharedThis = WeakThisPtr.Pin();
		if (!SharedThis.IsValid() || SharedThis->PostLoadCallbacks.Num() <= 0)
		{
			return false; // stop ticking
		}
		return SharedThis->UpdateLoadingMode();
	}));
}

bool FChunkDownloader::UpdateLoadingMode()
{
	// recompute loading stats
	ComputeLoadingStats();

	// check for the end of loading mode
	if (LoadingModeStats.FilesDownloaded >= LoadingModeStats.TotalFilesToDownload &&
		LoadingModeStats.ChunksMounted >= LoadingModeStats.TotalChunksToMount)
	{
		// make sure loading's been done for at least 2 frames before firing the callback
		// this adds a negligible amount of time to the loading screen but gives dependent loads a chance to queue
		static const int32 NUM_CONSECUTIVE_IDLE_FRAMES_FOR_LOADING_COMPLETION = 5;
		if (++LoadingCompleteLatch >= NUM_CONSECUTIVE_IDLE_FRAMES_FOR_LOADING_COMPLETION)
		{
			// end loading mode
			UE_LOG(LogChunkDownloader, Log, TEXT("EndLoadingMode (%d files downloaded, %d chunks mounted)"), LoadingModeStats.FilesDownloaded, LoadingModeStats.ChunksMounted);
#if PLATFORM_ANDROID || PLATFORM_IOS
			FPlatformApplicationMisc::ControlScreensaver(FPlatformApplicationMisc::Enable);
#endif

			// fire any loading mode completion callbacks
			TArray<FCallback> Callbacks = MoveTemp(PostLoadCallbacks);
			if (Callbacks.Num() > 0)
			{
				PostLoadCallbacks.Empty(); // shouldn't be necessary due to MoveTemp but just in case
				for (const auto& Callback : Callbacks)
				{
					Callback(LoadingModeStats.LastError.IsEmpty());
				}
			}
			return false; // stop ticking
		}
	}
	else
	{
		// reset the latch
		LoadingCompleteLatch = 0;
	}

	return true; // keep ticking
}

void FChunkDownloader::ComputeLoadingStats()
{
	LoadingModeStats.TotalBytesToDownload = LoadingModeStats.BytesDownloaded;
	LoadingModeStats.TotalFilesToDownload = LoadingModeStats.FilesDownloaded;
	LoadingModeStats.TotalChunksToMount = LoadingModeStats.ChunksMounted;

	// loop over all chunks
	for (const auto& It : Chunks)
	{
		const TSharedRef<FChunk>& Chunk = It.Value;

		// if it's mounting, add files to mount
		if (Chunk->MountTask != nullptr)
		{
			++LoadingModeStats.TotalChunksToMount;
		}
	}

	// check downloads
	for (const TSharedRef<FPakFile>& PakFile : DownloadRequests)
	{
		++LoadingModeStats.TotalFilesToDownload;
		if (PakFile->Download.IsValid())
		{
			LoadingModeStats.TotalBytesToDownload += PakFile->Entry.FileSize - PakFile->Download->GetProgress();
		}
		else
		{
			LoadingModeStats.TotalBytesToDownload += PakFile->Entry.FileSize;
		}
	}
}

void FChunkDownloader::ExecuteNextTick(const FCallback& Callback, bool bSuccess)
{
	if (Callback)
	{
		FTSTicker::GetCoreTicker().AddTicker(FTickerDelegate::CreateLambda([Callback, bSuccess](float dts) {
			Callback(bSuccess);
			return false;
		}));
	}
}

void FChunkDownloader::TryLoadBuildManifest(int TryNumber)
{
	// load the local build manifest
	TMap<FString, FString> CachedManifestProps;
	TArray<FPakFileEntry> CachedManifest = ParseManifest(CacheFolder / CACHED_BUILD_MANIFEST, &CachedManifestProps);

	// see if the BUILD_ID property matches
	if (CachedManifestProps.FindOrAdd(BUILD_ID_KEY) != ContentBuildId)
	{
		// if we have no CDN configured, we're done
		if (BuildBaseUrls.Num() <= 0)
		{
			UE_LOG(LogChunkDownloader, Error, TEXT("Unable to download build manifest. No CDN urls configured."));
			LoadingModeStats.LastError = LOCTEXT("UnableToDownloadManifest", "Unable to download build manifest. (NoCDN)");

			// execute and clear the callback
			FCallback Callback = MoveTemp(UpdateBuildCallback);
			ExecuteNextTick(Callback, false);
			return;
		}

		// fast path the first try
		if (TryNumber <= 0)
		{
			// download it
			TryDownloadBuildManifest(TryNumber);
			return;
		}

		// compute delay before re-starting download
		float SecondsToDelay = TryNumber * 5.0f;
		if (SecondsToDelay > 60)
		{
			SecondsToDelay = 60;
		}

		// set a ticker to delay
		UE_LOG(LogChunkDownloader, Log, TEXT("Will re-attempt manifest download in %f seconds"), SecondsToDelay);
		TWeakPtr<FChunkDownloader> WeakThisPtr = AsShared();
		FTSTicker::GetCoreTicker().AddTicker(FTickerDelegate::CreateLambda([WeakThisPtr, TryNumber](float Unused) {
			TSharedPtr<FChunkDownloader> SharedThis = WeakThisPtr.Pin();
			if (SharedThis.IsValid())
			{
				SharedThis->TryDownloadBuildManifest(TryNumber);
			}
			return false;
		}), SecondsToDelay);
		return;
	}

	// cached build manifest is up to date, load this one
	LoadManifest(CachedManifest);

	// execute and clear the callback
	FCallback Callback = MoveTemp(UpdateBuildCallback);
	ExecuteNextTick(Callback, true);
}

void FChunkDownloader::TryDownloadBuildManifest(int TryNumber)
{
	check(BuildBaseUrls.Num() > 0);

	// download the manifest from CDN, then load it
	FString ManifestFileName = FString::Printf(TEXT("BuildManifest-%s.txt"), *PlatformName);
	FString Url = BuildBaseUrls[TryNumber % BuildBaseUrls.Num()] / ManifestFileName;
	UE_LOG(LogChunkDownloader, Log, TEXT("Downloading build manifest (attempt #%d) from %s"), TryNumber+1, *Url);

	// download the manifest from the root CDN
	FHttpModule& HttpModule = FModuleManager::LoadModuleChecked<FHttpModule>("HTTP");
	check(!ManifestRequest.IsValid());
	ManifestRequest = HttpModule.Get().CreateRequest();
	ManifestRequest->SetURL(Url);
	ManifestRequest->SetVerb(TEXT("GET"));
	TWeakPtr<FChunkDownloader> WeakThisPtr = AsShared();
	FString CachedManifestFullPath = CacheFolder / CACHED_BUILD_MANIFEST;
	ManifestRequest->OnProcessRequestComplete().BindLambda([WeakThisPtr, TryNumber, CachedManifestFullPath](FHttpRequestPtr HttpRequest, FHttpResponsePtr HttpResponse, bool bSuccess) {
		// if successful, save
		FText LastError;
		if (bSuccess && HttpResponse.IsValid())
		{
			const int32 HttpStatus = HttpResponse->GetResponseCode();
			if (EHttpResponseCodes::IsOk(HttpStatus))
			{
				// Save the manifest to a file
				if (!WriteStringAsUtf8TextFile(HttpResponse->GetContentAsString(), CachedManifestFullPath))
				{
					UE_LOG(LogChunkDownloader, Error, TEXT("Failed to write manifest to '%s'"), *CachedManifestFullPath);
					LastError = FText::Format(LOCTEXT("FailedToWriteManifest", "[Try {0}] Failed to write manifest."), FText::AsNumber(TryNumber));
				}
			}
			else
			{
				UE_LOG(LogChunkDownloader, Error, TEXT("HTTP %d while downloading manifest from '%s'"), HttpStatus, *HttpRequest->GetURL());
				LastError = FText::Format(LOCTEXT("ManifestHttpError_FailureCode", "[Try {0}] Manifest download failed (HTTP {1})"), FText::AsNumber(TryNumber), FText::AsNumber(HttpStatus));
			}
		}
		else
		{
			UE_LOG(LogChunkDownloader, Error, TEXT("HTTP connection issue while downloading manifest '%s'"), *HttpRequest->GetURL());
			LastError = FText::Format(LOCTEXT("ManifestHttpError_Generic", "[Try {0}] Connection issues downloading manifest. Check your network connection..."), FText::AsNumber(TryNumber));
		}

		// try to load it
		TSharedPtr<FChunkDownloader> SharedThis = WeakThisPtr.Pin();
		if (!SharedThis.IsValid())
		{
			UE_LOG(LogChunkDownloader, Warning, TEXT("FChunkDownloader was destroyed while downloading manifest '%s'"), *HttpRequest->GetURL());
			return;
		}
		SharedThis->ManifestRequest.Reset();
		SharedThis->LoadingModeStats.LastError = LastError; // ok with this clearing the error on success
		SharedThis->TryLoadBuildManifest(TryNumber + 1);
	});
	ManifestRequest->ProcessRequest();
}

// block waiting for any pending mounts to finish
// then collect garbage to clean up any references to chunks.
// then create entries for any new chunks
// for any chunks that change, cancel downloads and unmount invalid paks (and any after invalid paks).
// If a changed chunk was mounted, then inline mount all non-mounted paks in the new list (in order)  
// then unload any chunks that no longer exist (cancel downloads and unmount all paks)
void FChunkDownloader::LoadManifest(const TArray<FPakFileEntry>& ManifestPakFiles)
{
	UE_LOG(LogChunkDownloader, Display, TEXT("Beginning manifest load."));

	// wait for all mounts to finish
	WaitForMounts();

	// trigger garbage collection (give any unmounts which are about to happen a good chance of success)
	CollectGarbage(RF_NoFlags);

	// group the manifest paks by chunk ID (maintain ordering)
	TMap<int32,TArray<FPakFileEntry>> Manifest;
	for (const FPakFileEntry& FileEntry : ManifestPakFiles)
	{
		check(FileEntry.ChunkId >= 0);
		Manifest.FindOrAdd(FileEntry.ChunkId).Add(FileEntry);
	}

	// copy old chunk map (we will reuse any that still exist)
	TMap<int32,TSharedRef<FChunk>> OldChunks = MoveTemp(Chunks);
	TMap<FString,TSharedRef<FPakFile>> OldPakFiles = MoveTemp(PakFiles);

	// loop over the new chunks
	int32 NumChunks = 0, NumPaks = 0;
	for (const auto& It : Manifest)
	{
		int32 ChunkId = It.Key;
		
		// keep track of new chunk and old pak files
		TSharedPtr<FChunk> Chunk;
		TArray<TSharedRef<FPakFile>> PrevPakList;

		// create or reuse the chunk
		TSharedRef<FChunk>* OldChunk = OldChunks.Find(ChunkId);
		if (OldChunk != nullptr)
		{
			// move over the old chunk
			Chunk = *OldChunk;
			check(Chunk->ChunkId == ChunkId);

			// don't clean it up later
			OldChunks.Remove(ChunkId);

			// move out OldPakFiles
			PrevPakList = MoveTemp(Chunk->PakFiles);
		}
		else
		{
			// make a brand new chunk
			Chunk = MakeShared<FChunk>();
			Chunk->ChunkId = ChunkId;
		}

		// add the chunk to the new map
		Chunks.Add(Chunk->ChunkId, Chunk.ToSharedRef());

		// find or create new pak files
		check(Chunk->PakFiles.Num() == 0);
		for (const FPakFileEntry& FileEntry : It.Value)
		{
			// see if there's an existing file for this one
			const TSharedRef<FPakFile>* ExistingFilePtr = OldPakFiles.Find(FileEntry.FileName);
			if (ExistingFilePtr != nullptr)
			{
				const TSharedRef<FPakFile>& ExistingFile = *ExistingFilePtr;
				if (ExistingFile->Entry.FileVersion == FileEntry.FileVersion)
				{
					// if version matched, size should too
					check(ExistingFile->Entry.FileSize == FileEntry.FileSize);

					// update and add to list (may populate ChunkId and RelativeUrl if we loaded from cache)
					ExistingFile->Entry = FileEntry;
					Chunk->PakFiles.Add(ExistingFile);
					PakFiles.Add(ExistingFile->Entry.FileName, ExistingFile);

					// remove from old pak files list
					OldPakFiles.Remove(FileEntry.FileName);
					continue;
				}
			}

			// create a new entry
			TSharedRef<FPakFile> NewFile = MakeShared<FPakFile>();
			NewFile->Entry = FileEntry;
			Chunk->PakFiles.Add(NewFile);
			PakFiles.Add(NewFile->Entry.FileName, NewFile);

			// see if it matches an embedded pak file
			const FPakFileEntry* CachedEntry = EmbeddedPaks.Find(FileEntry.FileName);
			if (CachedEntry != nullptr && CachedEntry->FileVersion == FileEntry.FileVersion)
			{
				NewFile->bIsEmbedded = true;
				NewFile->bIsCached = true;
				NewFile->SizeOnDisk = CachedEntry->FileSize;
			}
		}

		// log the chunk and pak file count
		UE_LOG(LogChunkDownloader, Verbose, TEXT("Found chunk %d (%d pak files)."), ChunkId, Chunk->PakFiles.Num());
		++NumChunks;
		NumPaks += Chunk->PakFiles.Num();

		// if the chunk is already mounted, we want to unmount any invalid data
		check(Chunk->MountTask == nullptr); // we already waited for mounts to finish
		if (Chunk->bIsMounted)
		{
			// see if all the existing pak files match to the new manifest (means it can stay mounted)
			// this is a common case so we're trying to be more efficient here
			int LongestCommonPrefix = 0;
			for (int i=0;i < PrevPakList.Num() && i < Chunk->PakFiles.Num();++i,++LongestCommonPrefix)
			{
				if (Chunk->PakFiles[i]->Entry.FileVersion != PrevPakList[i]->Entry.FileVersion)
				{
					break;
				}
			}

			// if they don't all match we need to remount
			if (LongestCommonPrefix != PrevPakList.Num() || LongestCommonPrefix != Chunk->PakFiles.Num())
			{
				// this chunk is no longer fully mounted
				Chunk->bIsMounted = false;

				// unmount any old paks that didn't match (reverse order)
				for (int i= PrevPakList.Num()-1;i>=0;--i)
				{
					UnmountPakFile(PrevPakList[i]);
				}

				// unmount any new paks that didn't match (may have changed position) (reverse order)
				// any new pak files unmounted will be re-mounted (in the right order) if this chunk is requested again
				for (int i=Chunk->PakFiles.Num()-1;i>=0;--i)
				{
					UnmountPakFile(Chunk->PakFiles[i]);
				}
			}
		}
	}

	// any files still left in OldPakFiles should be cancelled, unmounted, and deleted
	IFileManager& FileManager = IFileManager::Get();
	for (const auto& It : OldPakFiles)
	{
		const TSharedRef<FPakFile>& File = It.Value;
		UE_LOG(LogChunkDownloader, Log, TEXT("Removing orphaned pak file %s (was chunk %d)."), *File->Entry.FileName, File->Entry.ChunkId);

		// cancel downloads of pak files that are no longer valid
		if (File->Download.IsValid())
		{
			// treat these cancellations as successful since the pak is no longer needed (we've successfully downloaded nothing)
			CancelDownload(File, true);
		}

		// if a chunk completely disappeared we may need to clean up its mounts this way (otherwise would have been taken care of above)
		if (File->bIsMounted)
		{
			UnmountPakFile(File);
		}

		// delete any locally cached file
		if (File->SizeOnDisk > 0 && !File->bIsEmbedded)
		{
			bNeedsManifestSave = true;
			FString FullPathOnDisk = CacheFolder / File->Entry.FileName;
			if (!ensure(FileManager.Delete(*FullPathOnDisk)))
			{
				UE_LOG(LogChunkDownloader, Error, TEXT("Failed to delete orphaned pak %s."), *FullPathOnDisk);
			}
		}
	}

	// resave the manifest
	SaveLocalManifest(false);

	// log end
	check(ManifestPakFiles.Num() == NumPaks);
	UE_LOG(LogChunkDownloader, Display, TEXT("Manifest load complete. %d chunks with %d pak files."), NumChunks, NumPaks);
}

void FChunkDownloader::DownloadChunkInternal(const FChunk& Chunk, const FCallback& Callback, int32 Priority)
{
	UE_LOG(LogChunkDownloader, Log, TEXT("Chunk %d download requested."), Chunk.ChunkId);

	// see if we need to download anything at all
	bool bNeedsDownload = false;
	for (const auto& PakFile : Chunk.PakFiles)
	{
		if (!PakFile->bIsCached)
		{
			bNeedsDownload = true;
			break;
		}
	}
	if (!bNeedsDownload)
	{
		ExecuteNextTick(Callback, true);
		return;
	}

	// make sure we have CDN configured
	if (BuildBaseUrls.Num() <= 0)
	{
		UE_LOG(LogChunkDownloader, Error, TEXT("Unable to download Chunk %d (no CDN urls)."), Chunk.ChunkId);
		ExecuteNextTick(Callback, false);
		return;
	}

	// download all pak files that aren't already cached
	FMultiCallback* MultiCallback = new FMultiCallback(Callback);
	for (const auto& PakFile : Chunk.PakFiles)
	{
		if (!PakFile->bIsCached)
		{
			DownloadPakFileInternal(PakFile, MultiCallback->AddPending(), Priority);
		}
	}
	check(MultiCallback->GetNumPending() > 0);
} //-V773

void FChunkDownloader::MountChunkInternal(FChunk& Chunk, const FCallback& Callback)
{
	check(!Chunk.bIsMounted);

	// see if there's already a mount pending
	if (Chunk.MountTask != nullptr)
	{
		// join with the existing callbacks
		if (Callback)
		{
			Chunk.MountTask->GetTask().PostMountCallbacks.Add(Callback);
		}
		return;
	}

	// see if we need to trigger any downloads
	bool bAllPaksCached = true;
	for (const auto& PakFile : Chunk.PakFiles)
	{
		if (!PakFile->bIsCached)
		{
			bAllPaksCached = false;
			break;
		}
	}

	if (bAllPaksCached)
	{
		// if all pak files are cached, mount now
		UE_LOG(LogChunkDownloader, Log, TEXT("Chunk %d mount requested (%d pak sequence)."), Chunk.ChunkId, Chunk.PakFiles.Num());

		// spin up a background task to mount the pak file
		check(Chunk.MountTask == nullptr);
		Chunk.MountTask = new FMountTask();

		// configure the task
		FPakMountWork& MountWork = Chunk.MountTask->GetTask();
		MountWork.ChunkId = Chunk.ChunkId;
		MountWork.CacheFolder = CacheFolder;
		MountWork.EmbeddedFolder = EmbeddedFolder;
		for (const TSharedRef<FPakFile>& PakFile : Chunk.PakFiles)
		{
			if (!PakFile->bIsMounted)
			{
				MountWork.PakFiles.Add(PakFile);
			}
		}
		if (Callback)
		{
			MountWork.PostMountCallbacks.Add(Callback);
		}

		// start as a background task
		Chunk.MountTask->StartBackgroundTask();

		// start a per-frame ticker until mounts are finished
		if (!MountTicker.IsValid())
		{
			MountTicker = FTSTicker::GetCoreTicker().AddTicker(FTickerDelegate::CreateSP(this, &FChunkDownloader::UpdateMountTasks));
		}
	}
	else
	{
		// queue up pak file downloads
		TWeakPtr<FChunkDownloader> WeakThisPtr = AsShared();
		int32 ChunkId = Chunk.ChunkId;
		DownloadChunkInternal(Chunk, [WeakThisPtr, ChunkId, Callback](bool bDownloadSuccess) {
			// if the download failed, we can't mount
			if (bDownloadSuccess)
			{
				TSharedPtr<FChunkDownloader> SharedThis = WeakThisPtr.Pin();
				if (SharedThis.IsValid())
				{
					// if all chunks are downloaded, do the mount again (this will pick up any changes and continue downloading if needed)
					SharedThis->MountChunk(ChunkId, Callback);
					return;
				}
			}

			// if anything went wrong, fire the callback now
			if (Callback)
			{
				Callback(false);
			}
		}, MAX_int32);
	}
}

void FChunkDownloader::DownloadPakFileInternal(const TSharedRef<FPakFile>& PakFile, const FCallback& Callback, int32 Priority)
{
	check(BuildBaseUrls.Num() > 0);

	// increase priority if it's updated
	if (Priority > PakFile->Priority)
	{
		// if the download has already started this won't really change anything
		PakFile->Priority = Priority;
	}

	// just piggyback on the existing post-download callback
	if (Callback)
	{
		PakFile->PostDownloadCallbacks.Add(Callback);
	}

	// see if the download is already started
	if (PakFile->Download.IsValid())
	{
		// nothing to do then (we already added our callback)
		return;
	}

	// add it to the downloading set
	DownloadRequests.AddUnique(PakFile);
	DownloadRequests.StableSort([](const TSharedRef<FPakFile>& A, const TSharedRef<FPakFile>& B) {
		return A->Priority < B->Priority;
	});

	// start the first N pak files in flight
	IssueDownloads();
}

void FChunkDownloader::IssueDownloads()
{
	for (int32 i = 0; i < DownloadRequests.Num() && i < TargetDownloadsInFlight; ++i)
	{
		TSharedRef<FPakFile> DownloadPakFile = DownloadRequests[i];
		if (DownloadPakFile->Download.IsValid())
		{
			// already downloading
			continue;
		}

		// log that we're starting a download
		UE_LOG(LogChunkDownloader, Log, TEXT("Pak file %s download requested (%s)."),
			*DownloadPakFile->Entry.FileName,
			*DownloadPakFile->Entry.RelativeUrl
		);
		bNeedsManifestSave = true;

		// make a new download (platform specific)
		DownloadPakFile->Download = MakeShared<FDownload>(AsShared(), DownloadPakFile);
		DownloadPakFile->Download->Start();
	}
}

void FChunkDownloader::CompleteMountTask(FChunk& Chunk)
{
	check(Chunk.MountTask != nullptr);
	check(Chunk.MountTask->IsDone());

	// increment chunks mounted
	++LoadingModeStats.ChunksMounted;

	// remove the mount
	FMountTask* Mount = Chunk.MountTask;
	Chunk.MountTask = nullptr;

	// get the work
	const FPakMountWork& MountWork = Mount->GetTask();

	// update bIsMounted on paks that actually succeeded
	for (const TSharedRef<FPakFile>& PakFile : MountWork.MountedPakFiles)
	{
		PakFile->bIsMounted = true;
	}

	// update bIsMounted on the chunk
	bool bAllPaksMounted = true;
	for (const TSharedRef<FPakFile>& PakFile : Chunk.PakFiles)
	{
		if (!PakFile->bIsMounted)
		{
			LoadingModeStats.LastError = FText::Format(LOCTEXT("FailedToMount", "Failed to mount {0}."), FText::FromString(PakFile->Entry.FileName));
			bAllPaksMounted = false;
			break;
		}
	}
	Chunk.bIsMounted = bAllPaksMounted;
	if (Chunk.bIsMounted)
	{
		UE_LOG(LogChunkDownloader, Log, TEXT("Chunk %d mount succeeded."), Chunk.ChunkId);
	}
	else
	{
		UE_LOG(LogChunkDownloader, Error, TEXT("Chunk %d mount failed."), Chunk.ChunkId);
	}

	// trigger the post-mount callbacks
	for (const FCallback& Callback : MountWork.PostMountCallbacks)
	{
		ExecuteNextTick(Callback, bAllPaksMounted);
	}

	// also trigger the multicast event
	OnChunkMounted.Broadcast(Chunk.ChunkId, bAllPaksMounted);

	// finally delete the task
	delete Mount;

	// recompute loading stats
	ComputeLoadingStats();
}

bool FChunkDownloader::UpdateMountTasks(float dts)
{
	bool bMountsPending = false;

	for (const auto& It : Chunks)
	{
		const TSharedRef<FChunk>& Chunk = It.Value;
		if (Chunk->MountTask != nullptr)
		{
			if (Chunk->MountTask->IsDone())
			{
				// complete it
				CompleteMountTask(*Chunk);
			}
			else
			{
				// mount still pending
				bMountsPending = true;
			}
		}
	}

	if (!bMountsPending)
	{
		MountTicker.Reset();
	}
	return bMountsPending; // keep ticking
}

void FChunkDownloader::DownloadChunk(int32 ChunkId, const FCallback& Callback, int32 Priority)
{
	// look up the chunk
	TSharedRef<FChunk>* ChunkPtr = Chunks.Find(ChunkId);
	if (ChunkPtr == nullptr || (*ChunkPtr)->PakFiles.Num() <= 0)
	{
		// a chunk that doesn't exist or one with no pak files are both considered "complete" for the purposes of this call
		// use GetChunkStatus to differentiate from chunks that mounted successfully
		UE_LOG(LogChunkDownloader, Warning, TEXT("Ignoring download request for chunk %d (no mapped pak files)."), ChunkId);
		ExecuteNextTick(Callback, true);
		return;
	}
	const FChunk& Chunk = **ChunkPtr;

	// if all the paks are cached, just succeed
	if (Chunk.IsCached())
	{
		ExecuteNextTick(Callback, true);
		return;
	}

	// queue the download
	DownloadChunkInternal(Chunk, Callback, Priority);

	// resave manifest if needed
	SaveLocalManifest(false);
	ComputeLoadingStats();
}

void FChunkDownloader::DownloadChunks(const TArray<int32>& ChunkIds, const FCallback& Callback, int32 Priority)
{
	// convert to chunk references
	TArray<TSharedRef<FChunk>> ChunksToDownload;
	for (int32 ChunkId : ChunkIds)
	{
		TSharedRef<FChunk>* ChunkPtr = Chunks.Find(ChunkId);
		if (ChunkPtr != nullptr)
		{
			TSharedRef<FChunk>& ChunkRef = *ChunkPtr;
			if (ChunkRef->PakFiles.Num() > 0)
			{
				if (!ChunkRef->IsCached())
				{
					ChunksToDownload.Add(ChunkRef);
				}
				continue;
			}
		}
		UE_LOG(LogChunkDownloader, Warning, TEXT("Ignoring download request for chunk %d (no mapped pak files)."), ChunkId);
	}

	// make sure there are some chunks to mount (saves a frame)
	if (ChunksToDownload.Num() <= 0)
	{
		// trivial success
		ExecuteNextTick(Callback, true);
		return;
	}

	// if there's no callback for some reason, avoid a bunch of boilerplate
#ifndef PVS_STUDIO // Build machine refuses to disable this warning
	if (Callback)
	{
		// loop over chunks and issue individual callback
		FMultiCallback* MultiCallback = new FMultiCallback(Callback);
		for (const TSharedRef<FChunk>& Chunk : ChunksToDownload)
		{
			DownloadChunkInternal(*Chunk, MultiCallback->AddPending(), Priority);
		}
		check(MultiCallback->GetNumPending() > 0);
	} //-V773
	else
	{
		// no need to manage callbacks
		for (const TSharedRef<FChunk>& Chunk : ChunksToDownload)
		{
			DownloadChunkInternal(*Chunk, FCallback(), Priority);
		}
	}
#endif

	// resave manifest if needed
	SaveLocalManifest(false);
	ComputeLoadingStats();
}

void FChunkDownloader::MountChunk(int32 ChunkId, const FCallback& Callback)
{
	// look up the chunk
	TSharedRef<FChunk>* ChunkPtr = Chunks.Find(ChunkId);
	if (ChunkPtr == nullptr || (*ChunkPtr)->PakFiles.Num() <= 0)
	{
		// a chunk that doesn't exist or one with no pak files are both considered "complete" for the purposes of this call
		// use GetChunkStatus to differentiate from chunks that mounted successfully
		UE_LOG(LogChunkDownloader, Warning, TEXT("Ignoring mount request for chunk %d (no mapped pak files)."), ChunkId);
		ExecuteNextTick(Callback, true);
		return;
	}
	FChunk& Chunk = **ChunkPtr;

	// see if we're mounted already
	if (Chunk.bIsMounted)
	{
		// trivial success
		ExecuteNextTick(Callback, true);
		return;
	}

	// mount the chunk
	MountChunkInternal(Chunk, Callback);

	// resave manifest if needed
	SaveLocalManifest(false);
	ComputeLoadingStats();
}

void FChunkDownloader::MountChunks(const TArray<int32>& ChunkIds, const FCallback& Callback)
{
	// convert to chunk references
	TArray<TSharedRef<FChunk>> ChunksToMount;
	for (int32 ChunkId : ChunkIds)
	{
		TSharedRef<FChunk>* ChunkPtr = Chunks.Find(ChunkId);
		if (ChunkPtr != nullptr)
		{
			TSharedRef<FChunk>& ChunkRef = *ChunkPtr;
			if (ChunkRef->PakFiles.Num() > 0)
			{
				if (!ChunkRef->bIsMounted)
				{
					ChunksToMount.Add(ChunkRef);
				}
				continue;
			}
		}
		UE_LOG(LogChunkDownloader, Warning, TEXT("Ignoring mount request for chunk %d (no mapped pak files)."), ChunkId);
	}

	// make sure there are some chunks to mount (saves a frame)
	if (ChunksToMount.Num() <= 0)
	{
		// trivial success
		ExecuteNextTick(Callback, true);
		return;
	}

	// if there's no callback for some reason, avoid a bunch of boilerplate
#ifndef PVS_STUDIO // Build machine refuses to disable this warning
	if (Callback)
	{
		// loop over chunks and issue individual callback
		FMultiCallback* MultiCallback = new FMultiCallback(Callback);
		for (const TSharedRef<FChunk>& Chunk : ChunksToMount)
		{
			MountChunkInternal(*Chunk, MultiCallback->AddPending());
		}
		check(MultiCallback->GetNumPending() > 0);
	} //-V773
	else
	{
		// no need to manage callbacks
		for (const TSharedRef<FChunk>& Chunk : ChunksToMount)
		{
			MountChunkInternal(*Chunk, FCallback());
		}
	}
#endif

	// resave manifest if needed
	SaveLocalManifest(false);
	ComputeLoadingStats();
}

#undef LOCTEXT_NAMESPACE