// Copyright Epic Games, Inc. All Rights Reserved.

#include "ZenStoreHttpClient.h"

#include "Algo/BinarySearch.h"
#include "Algo/IsSorted.h"
#include "Algo/Sort.h"
#include "Misc/ScopeRWLock.h"
#include "Misc/App.h"
#include "Misc/CommandLine.h"
#include "HAL/FileManager.h"

#if PLATFORM_DESKTOP

#include "Async/Async.h"
#include "HAL/PlatformFileManager.h"
#include "Http/HttpClient.h"
#include "IO/IoHash.h"
#include "Misc/Paths.h"
#include "Misc/ScopeLock.h"
#include "Misc/SecureHash.h"
#include "Misc/StringBuilder.h"
#include "ProfilingDebugging/CpuProfilerTrace.h"
#include "Serialization/BufferArchive.h"
#include "Serialization/CompactBinaryWriter.h"
#include "Serialization/LargeMemoryWriter.h"
#include "Tasks/Task.h"
#include "ZenAsyncCbPackageReceiver.h"
#include "ZenCbPackageReceiver.h"
#include "ZenSerialization.h"
#include "ZenServerHttp.h"

DEFINE_LOG_CATEGORY_STATIC(LogZenStore, Log, All);

namespace UE
{
namespace Zen::Internal
{
struct FZenHttpClientState
{
	THttpUniquePtr<IHttpConnectionPool> ConnectionPool;
	TUniquePtr<FHttpRequestQueue> RequestQueue;
};
}

const uint32 FZenStoreHttpClient::PoolEntryCount = 32;

std::atomic<uint32> FZenStoreHttpClient::FSaltGenerator::GOpCounter(0);

FZenStoreHttpClient::FSaltGenerator::FSaltGenerator()
	: SaltBase(FGenericPlatformProcess::GetCurrentProcessId() + 0x9e3779b9u)
{
}

FZenStoreHttpClient::FZenStoreHttpClient()
{
	InitCommon();
}

FZenStoreHttpClient::FZenStoreHttpClient(FStringView HostName, uint16 Port)
: ZenService(HostName.IsEmpty() ? FStringView() : *WriteToString<64>(TEXT("http://"), HostName, TEXT(":"), Port))
{
	InitCommon();
}

FZenStoreHttpClient::FZenStoreHttpClient(UE::Zen::FServiceSettings&& InSettings)
: ZenService(MoveTemp(InSettings))
{
	InitCommon();
}

FZenStoreHttpClient::~FZenStoreHttpClient()
{
}

void
FZenStoreHttpClient::InitCommon()
{
	RequestPool = MakeUnique<Zen::FZenHttpRequestPool>(ZenService.GetInstance().GetEndpoint(), PoolEntryCount);
	ClientState = MakeUnique<Zen::Internal::FZenHttpClientState>();

	const uint32 MaxConnections = uint32(FMath::Clamp(FPlatformMisc::NumberOfCoresIncludingHyperthreads(), 8, 64));
	constexpr uint32 RequestPoolSize = 128;
	constexpr uint32 RequestPoolOverflowSize = 128;

	FHttpConnectionPoolParams ConnectionPoolParams;
	ConnectionPoolParams.MaxConnections = MaxConnections;
	ConnectionPoolParams.MinConnections = MaxConnections;
	ClientState->ConnectionPool = IHttpManager::Get().CreateConnectionPool(ConnectionPoolParams);

	FHttpClientParams ClientParams;
	ClientParams.MaxRequests = RequestPoolSize + RequestPoolOverflowSize;
	ClientParams.MinRequests = RequestPoolSize;
	ClientParams.LowSpeedLimit = 1;
	ClientParams.LowSpeedTime = 25;
	ClientParams.bBypassProxy = true;
	ClientState->RequestQueue = MakeUnique<FHttpRequestQueue>(*ClientState->ConnectionPool, ClientParams);
}

bool
FZenStoreHttpClient::TryCreateProject(FStringView InProjectId,
	FStringView InParentProjectId,
	FStringView InOplogId,
	FStringView ServerRoot,
	FStringView EngineRoot,
	FStringView ProjectRoot,
	FStringView ProjectFilePath)
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_Initialize);

	UE_LOG(LogZenStore, Display, TEXT("Establishing oplog '%s/%s'"), *FString(InProjectId), *FString(InOplogId));

	// Establish project

	{
		// Create the connection request with no logging of errors; our caller will handle logging the connection error
		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get(), false /* bLogErrors */);

		TStringBuilder<128> ProjectUri;
		ProjectUri << "/prj/" << InProjectId;
		TArray64<uint8> GetBuffer;

		// TODO: how to handle failure here? This is probably the most likely point of failure
		// if the service is not up or not responding

		if (Download(*Request, ProjectUri, &GetBuffer, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
		{
			UE_LOG(LogZenStore, Display, TEXT("Zen project '%s' already exists"), *FString(InProjectId));
			bConnectionSucceeded = true;
		}
		else
		{
			Request->Reset();

			FCbWriter ProjInfo;
			ProjInfo.BeginObject();
			ProjInfo << "id" << InProjectId;
			if (!InParentProjectId.IsEmpty())
			{
				ProjInfo << "parentid" << InParentProjectId;
			}
			ProjInfo << "root" << ServerRoot;
			ProjInfo << "engine" << EngineRoot;
			ProjInfo << "project" << ProjectRoot;
			ProjInfo << "projectfile" << ProjectFilePath;
			ProjInfo.EndObject();

			if (!Post(*Request, ProjectUri, ProjInfo.Save().AsObject()))
			{
				UE_LOG(LogZenStore, Display, TEXT("Zen project '%s' creation FAILED"), *FString(InProjectId));
				bConnectionSucceeded = false;

				// TODO: how to recover / handle this?
			}
			else if (Request->GetResponseCode() == 201)
			{
				UE_LOG(LogZenStore, Display, TEXT("Zen project '%s' created"), *FString(InProjectId));
				bConnectionSucceeded = true;
			}
			else
			{
				UE_LOG(LogZenStore, Warning, TEXT("Zen project '%s' creation returned success but not HTTP 201"), *FString(InProjectId));
				bConnectionSucceeded = true;
			}
		}
	}

	ProjectPath = WriteToString<128>("/prj/", InProjectId);
	OplogPath = WriteToString<128>("/prj/", InProjectId, "/oplog/", InOplogId);
	OplogRpcPath = WriteToAnsiString<128>("/prj/", InProjectId, "/oplog/", InOplogId, "/rpc");
	OplogNewEntryPath = WriteToString<128>("/prj/", InProjectId, "/oplog/", InOplogId, "/new");
	OplogPrepNewEntryPath = WriteToString<128>("/prj/", InProjectId, "/oplog/", InOplogId, "/prep");

	bAllowRead = true;
	bAllowEdit = true;

	return bConnectionSucceeded;
}

bool FZenStoreHttpClient::TryDeleteOplog(FStringView InProjectId, FStringView InOplogId)
{
	if (!IsConnected())
	{
		return false;
	}
	UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());

	UE_LOG(LogZenStore, Display, TEXT("Deleting oplog '%s/%s' if it exists"), *FString(InProjectId), *FString(InOplogId));
	bool Success = Delete(*Request, OplogPath);
	return Success;
}

bool FZenStoreHttpClient::TryCreateOplog(FStringView InProjectId, FStringView InOplogId, FStringView InOplogLifetimeMarkerPath)
{
	if (!IsConnected())
	{
		return false;
	}

	UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());

	TArray64<uint8> GetBuffer;
	FCbObjectView OplogInfo;
	if (Download(*Request, OplogPath, &GetBuffer, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
	{
		UE_LOG(LogZenStore, Display, TEXT("Zen oplog '%s/%s' already exists"), *FString(InProjectId), *FString(InOplogId));

		OplogInfo = FCbObjectView(GetBuffer.GetData());
	}
	else
	{
		FCbWriter Writer;
		Writer.BeginObject();
		Writer.AddString("gcpath", InOplogLifetimeMarkerPath);
		Writer.EndObject();

		FCbFieldIterator OplogCreateInfo = Writer.Save();

		Request->Reset();

		if (!Post(*Request, OplogPath, OplogCreateInfo.AsObjectView()))
		{
			UE_LOG(LogZenStore, Error, TEXT("Zen oplog '%s/%s' creation FAILED"), *FString(InProjectId), *FString(InOplogId));
			// Demote the connection status back to not connected
			bConnectionSucceeded = false;
			return false;
		}
		else if (Request->GetResponseCode() == 201)
		{
			UE_LOG(LogZenStore, Display, TEXT("Zen oplog '%s/%s' created"), *FString(InProjectId), *FString(InOplogId));
		}
		else
		{
			UE_LOG(LogZenStore, Warning, TEXT("Zen oplog '%s/%s' creation returned success but not HTTP 201"), *FString(InProjectId), *FString(InOplogId));
		}

		// Issue another GET to retrieve information

		GetBuffer.Reset();
		Request->Reset();
		if (Download(*Request, OplogPath, &GetBuffer, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
		{
			OplogInfo = FCbObjectView(GetBuffer.GetData());
		}
	}

	TempDirPath = FUTF8ToTCHAR(OplogInfo["tempdir"].AsString());
	return true;
}

void FZenStoreHttpClient::InitializeReadOnly(FStringView InProjectId, FStringView InOplogId)
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_InitializeReadOnly);

	UE_LOG(LogZenStore, Display, TEXT("Establishing oplog %s / %s"), *FString(InProjectId), *FString(InOplogId));

	// Establish project

	{
		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());

		TArray64<uint8> GetBuffer;

		// TODO: how to handle failure here? This is probably the most likely point of failure
		// if the service is not up or not responding

		if (!Download(*Request, WriteToString<128>("/prj/", InProjectId), &GetBuffer, Zen::EContentType::CbObject) || Request->GetResponseCode() != 200)
		{
			UE_LOG(LogZenStore, Fatal, TEXT("Zen project '%s' not found"), *FString(InProjectId));
			bConnectionSucceeded = false;
		}
		else
		{
			bConnectionSucceeded = true;
		}
	}

	// Establish oplog
	check(IsConnected());

	{
		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());

		OplogPath = WriteToString<128>("/prj/", InProjectId, "/oplog/", InOplogId);
		ProjectPath = WriteToString<128>("/prj/", InProjectId);

		TArray64<uint8> GetBuffer;
		if (!Download(*Request, OplogPath, &GetBuffer, Zen::EContentType::CbObject) || Request->GetResponseCode() != 200)
		{
			UE_LOG(LogZenStore, Fatal, TEXT("Zen oplog '%s'/'%s' not found"), *FString(InProjectId), *FString(InOplogId));
		}
	}

	bAllowRead = true;
}

bool FZenStoreHttpClient::Download(Zen::FZenHttpRequest& Request, FStringView Uri, TArray64<uint8>* Buffer, Zen::EContentType AcceptType)
{
	Zen::FZenHttpRequest::Result Res = Request.PerformBlockingDownload(Uri, Buffer, AcceptType);
	if (ShouldRecoverAndRetry(Request))
	{
		Request.Reset();
		Res = Request.PerformBlockingDownload(Uri, Buffer, AcceptType);
	}
	return Res == Zen::FZenHttpRequest::Result::Success;
}

bool FZenStoreHttpClient::Post(Zen::FZenHttpRequest& Request, FStringView Uri, FCbObjectView Obj)
{
	Zen::FZenHttpRequest::Result Res = Request.PerformBlockingPost(Uri, Obj);
	if (ShouldRecoverAndRetry(Request))
	{
		Request.Reset();
		Res = Request.PerformBlockingPost(Uri, Obj);
	}
	return Res == Zen::FZenHttpRequest::Result::Success;
}

bool FZenStoreHttpClient::Post(Zen::FZenHttpRequest& Request, FStringView Uri, FMemoryView Payload)
{
	Zen::FZenHttpRequest::Result Res = Request.PerformBlockingPost(Uri, Payload);
	if (ShouldRecoverAndRetry(Request))
	{
		Request.Reset();
		Res = Request.PerformBlockingPost(Uri, Payload);
	}
	return Res == Zen::FZenHttpRequest::Result::Success;
}

void FZenStoreHttpClient::SetRequestUri(IHttpRequest& Request, FAnsiStringView Path)
{
	using namespace Zen;

	check(Path.Len() && Path[0] == '/');

	const FZenServiceEndpoint& Endpoint = ZenService.GetInstance().GetEndpoint();

	TAnsiStringBuilder<128> Uri;
	Uri << Endpoint.GetURL();
	Uri << Path;
	Request.SetUri(Uri);

	if (Endpoint.GetSocketType() != FZenServiceEndpoint::ESocketType::Unix)
	{
		return;
	}

	FStringView UnixSocketPath = Endpoint.GetName();
	const auto& AsAnsi = StringCast<ANSICHAR>(UnixSocketPath.GetData(), UnixSocketPath.Len());
	Request.SetUnixSocketPath(AsAnsi);
}

void FZenStoreHttpClient::Rpc(FAnsiStringView Uri, FCbObject RpcRequestObject, FCbPackage& OutResponse)
{
	THttpUniquePtr<IHttpRequest> Request = ClientState->RequestQueue->CreateRequest({});
	if (!Request)
	{
		return;
	}

	constexpr int MaxAttempts = 2;
	int Attempt = 0;
	bool bRetry = false;
	SetRequestUri(*Request, Uri);
	Request->SetMethod(EHttpMethod::Post);
	Request->AddAcceptType(EHttpMediaType::CbPackage);
	Request->SetContentType(EHttpMediaType::CbObject);
	Request->SetBody(RpcRequestObject.GetBuffer().MakeOwned());
	do
	{
		Zen::FCbPackageReceiver Receiver(OutResponse);
		THttpUniquePtr<IHttpResponse> HttpResponse;
		Request->Send(&Receiver, HttpResponse);
		int StatusCode = HttpResponse->GetStatusCode();

		bRetry = Receiver.ShouldRecoverAndRetry(ZenService.GetInstance(), *HttpResponse) && (++Attempt < MaxAttempts);
	}
	while (bRetry);
}

void FZenStoreHttpClient::RpcAsync(FAnsiStringView Uri, FCbObject RpcRequestObject, TUniqueFunction<void(const FCbPackage& Response)>&& ResponseCallback)
{
	THttpUniquePtr<IHttpRequest> Request = ClientState->RequestQueue->CreateRequest({});
	if (!Request)
	{
		FCbPackage Response;
		ResponseCallback(Response);
		return;
	}

	SetRequestUri(*Request, Uri);
	Request->SetMethod(EHttpMethod::Post);
	Request->AddAcceptType(EHttpMediaType::CbPackage);
	Request->SetContentType(EHttpMediaType::CbObject);
	Request->SetBody(RpcRequestObject.GetBuffer().MakeOwned());
	// PackageReceiver will delete itself in the completion callback
	Zen::FAsyncCbPackageReceiver* PackageReceiver = new Zen::FAsyncCbPackageReceiver(MoveTemp(Request), ZenService.GetInstance(),
		[ResponseCallback = MoveTemp(ResponseCallback)]
		(Zen::FAsyncCbPackageReceiver* Receiver, const THttpUniquePtr<IHttpResponse>& HttpResponse, const FCbPackage& Response) mutable
		{
			// Launch a task for the completion function since it can execute arbitrary code.
			Tasks::Launch(TEXT("FZenStoreHttpClient::RpcAsync"), [Receiver, &Response, InnerResponseCallback = MoveTemp(ResponseCallback)]
			{
				InnerResponseCallback(Response);
				delete Receiver;
			});
		},
		/*MaxAttempts*/ 2);
	PackageReceiver->SendAsync();
} //-V773

bool FZenStoreHttpClient::Delete(Zen::FZenHttpRequest& Request, FStringView Uri)
{
	Zen::FZenHttpRequest::Result Res = Request.PerformBlockingDelete(Uri);
	if (ShouldRecoverAndRetry(Request))
	{
		Request.Reset();
		Res = Request.PerformBlockingDelete(Uri);
	}
	return Res == Zen::FZenHttpRequest::Result::Success;
}

bool FZenStoreHttpClient::ShouldRecoverAndRetry(Zen::FZenHttpRequest& Request)
{
	if (!ZenService.GetInstance().IsServiceRunningLocally())
	{
		return false;
	}

	const int ConnectError = 7;
	const int SSLConnectError = 35;
	const int TimeoutError = 28;

	if ((Request.GetResponseCode() == ConnectError) ||
		(Request.GetResponseCode() == SSLConnectError) ||
		(Request.GetResponseCode() == TimeoutError))
	{
		return true;
	}

	return false;
}

TIoStatusOr<uint64> FZenStoreHttpClient::AppendOp(FCbPackage OpEntry)
{
	check(bAllowEdit);

	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_AppendOp);

	{
		TRACE_CPUPROFILER_EVENT_SCOPE(Zen_AppendOp_Async);
		FLargeMemoryWriter SerializedPackage;

		const int32 Salt = SaltGen.Next();
		bool bIsUsingTempFiles = false;

		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());

		if (TempDirPath.IsEmpty())
		{
			// Old-style with all attachments by value

			UE::Zen::OpLog::SaveCbPackage(OpEntry, SerializedPackage);
		}
		else
		{
			TConstArrayView<FCbAttachment> Attachments = OpEntry.GetAttachments();

			// Prep phase

			TSet<FIoHash> NeedChunks;

			if (!Attachments.IsEmpty())
			{
				FCbWriter Writer;
				Writer.BeginObject();
				Writer.BeginArray("have");

				for (const FCbAttachment& Attachment : Attachments)
				{
					Writer.AddHash(Attachment.GetHash());
				}

				Writer.EndArray();
				Writer.EndObject();

				FCbFieldIterator Prep = Writer.Save();

				bool IsOk = false;			
				if (Post(*Request, OplogPrepNewEntryPath, Prep.AsObjectView()))
				{
					FCbObjectView NeedObject;

					if (Request->GetResponseCode() == 200)
					{
						NeedObject = FCbObjectView(Request->GetResponseBuffer().GetData());

						for (auto& Entry : NeedObject["need"])
						{
							NeedChunks.Add(Entry.AsHash());
						}

						IsOk = true;
					}
				}

				if (!IsOk)
				{
					UE_LOG(LogZenStore, Warning, TEXT("Unable to check for needed chunks, assuming all are needed"));
					for (const FCbAttachment& Attachment : Attachments)
					{
						NeedChunks.Add(Attachment.GetHash());
					}
				}
			}

			// This uses a slight variation for package attachment serialization
			// by writing larger attachments to a file and referencing it in the
			// core object. Small attachments are serialized inline as normal

			FCbWriter Writer;

			FCbObject PackageObj = OpEntry.GetObject();
			const FIoHash PackageObjHash = PackageObj.GetHash();

			Writer.AddObject(PackageObj);
			Writer.AddObjectAttachment(PackageObjHash);

			// Send phase

			const bool bCanUseLocalTempFile = ZenService.GetInstance().IsServiceRunningLocally();

			for (const FCbAttachment& Attachment : Attachments)
			{
				if (!Attachment.IsCompressedBinary())
				{
					return TIoStatusOr<uint64>((FIoStatus)(FIoStatusBuilder(EIoErrorCode::CompressionError) << TEXT("Attachment is not compressed")));
				}

				const FIoHash AttachmentHash = Attachment.GetHash();
				bool bIsSerialized = false;

				if (NeedChunks.Contains(AttachmentHash))
				{
					FSharedBuffer AttachmentData = Attachment.AsCompressedBinary().GetCompressed().ToShared();
					if (bCanUseLocalTempFile && AttachmentData.GetSize() >= StandaloneThresholdBytes)
					{
						// Write to temporary file. To avoid race conditions we derive
						// the file name from a salt value and the attachment hash

						FIoHash AttachmentSpec[] { FIoHash::HashBuffer(&Salt, sizeof Salt), AttachmentHash };
						FIoHash AttachmentId = FIoHash::HashBuffer(MakeMemoryView(AttachmentSpec));

						FString TempFilePath = TempDirPath / LexToString(AttachmentId);
						IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();

						if (IFileHandle* FileHandle = PlatformFile.OpenWrite(*TempFilePath))
						{
							bool WriteOK = FileHandle->Write((const uint8*)AttachmentData.GetData(), AttachmentData.GetSize());
							delete FileHandle;
							if (WriteOK)
							{
								Writer.AddHash(AttachmentHash);
								bIsSerialized = true;
								bIsUsingTempFiles = true;
							}
							else
							{
								(void)PlatformFile.DeleteFile(*TempFilePath);
								// Take the slow path if we can't write the payload file in the large attachment directory
								UE_LOG(LogZenStore, Warning, TEXT("Could not write to file '%s', taking slow path for large attachment"), *TempFilePath);
							}
						}
						else
						{
							// Take the slow path if we can't open the payload file in the large attachment directory
							UE_LOG(LogZenStore, Warning, TEXT("Could not create file '%s', taking slow path for large attachment"), *TempFilePath);
						}
					}

					if (!bIsSerialized)
					{
						UE::Zen::OpLog::SaveCbAttachment(Attachment, Writer);
					}
				}
				else
				{
					Writer.AddHash(AttachmentHash);
				}
			}
			Writer.AddNull();

			Writer.Save(SerializedPackage);
		}

		UE_LOG(LogZenStore, Verbose, TEXT("Package size: %" UINT64_FMT), SerializedPackage.TotalSize());

		TStringBuilder<64> NewOpPostUri;
		NewOpPostUri << OplogNewEntryPath;

		if (bIsUsingTempFiles)
		{
			NewOpPostUri << "?salt=" << Salt;
		}

		Request->Reset();

		if (Post(*Request, NewOpPostUri, SerializedPackage.GetView()))
		{
			return TIoStatusOr<uint64>(SerializedPackage.TotalSize());
		}
		else
		{
			return TIoStatusOr<uint64>((FIoStatus)(FIoStatusBuilder(EIoErrorCode::Unknown) << TEXT("Append OpLog failed, NewOpLogPath='") << OplogNewEntryPath << TEXT("'")));
		}
	}
}

TIoStatusOr<uint64> FZenStoreHttpClient::GetChunkSize(const FIoChunkId& Id)
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_GetChunkSize);

	check(bAllowRead);

	UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());
	TStringBuilder<128> ChunkUri;
	ChunkUri << OplogPath << '/' << Id << "/info";
	if (Download(*Request, ChunkUri, nullptr, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
	{
		FCbObjectView ResponseObj = Request->GetResponseAsObject();
		const uint64 ChunkSize = ResponseObj["size"].AsUInt64(0);
		return ChunkSize;
	}
	else if (int ResponseCode = Request->GetResponseCode(); ResponseCode != 404)
	{
		UE_LOG(LogZenStore, Verbose, TEXT("Request '%s' failed. Error: %s. Status: %s (%d). \"%s\""), *ChunkUri, *FString(Request->GetError()), *FString(UE::Zen::FZenHttpRequest::LexResponseCodeToString(ResponseCode)), ResponseCode, *Request->GetResponseAsString());
	}
	return FIoStatus(EIoErrorCode::NotFound);
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadChunk(const FIoChunkId& Id, uint64 Offset, uint64 Size)
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_ReadChunk);
	TStringBuilder<128> ChunkUri;
	ChunkUri << OplogPath << '/' << Id;
	return ReadOpLogUri(ChunkUri, Offset, Size);
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadChunk(const FIoHash& RawHash, uint64 Offset, uint64 Size)
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_ReadChunk);
	TStringBuilder<128> ChunkUri;
	ChunkUri << OplogPath << '/' << RawHash;
	return ReadOpLogUri(ChunkUri, Offset, Size);
}

static void SerializeRequest(FCbWriter& Writer, const FIoChunkId& ChunkId)
{
	Writer << "Oid" << ChunkId;
}

static void SerializeRequest(FCbWriter& Writer, const FIoHash& RawHash)
{
	Writer << "RawHash" << RawHash;
}

static void DeserializeResponse(const FCbObjectView ChunkObject, FIoChunkId& ChunkId)
{
	FCbObjectId ObjectId = ChunkObject[ANSITEXTVIEW("Id")].AsObjectId();
	ChunkId.Set(ObjectId.GetView());
}

static void DeserializeResponse(const FCbObjectView ChunkObject, FIoHash& RawHash)
{
	RawHash = ChunkObject[ANSITEXTVIEW("Id")].AsHash();
}

static const TCHAR* DescribeRequest(const FIoChunkId& ChunkId)
{
	return TEXT("Oid");
}

static const TCHAR* DescribeRequest(const FIoHash& RawHash)
{
	return TEXT("RawHash");
}

template<bool bAsync, typename T>
void FZenStoreHttpClient::ReadChunks_Internal(TConstArrayView<T> Requests, TUniqueFunction<void(const T& Id, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_ReadChunksAsync_Internal);
	FCbWriter Writer;
	Writer.BeginObject();
	Writer << "method" << "getchunks";

	Writer.BeginObject("Request");
	Writer.BeginArray("Chunks");

	TSet<T> InProgressRequests;
	for (const T& Request : Requests)
	{
		InProgressRequests.Add(Request);
	}
	for (const T& Request : InProgressRequests)
	{
		Writer.BeginObject();
		SerializeRequest(Writer, Request);
		Writer.EndObject();
	}

	Writer.EndArray();
	Writer.EndObject();
	Writer.EndObject();

	auto OnGetChunksResponse = [InProgressRequests = MoveTemp(InProgressRequests)]
	(TUniqueFunction<void(const T& Id, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback, const FCbPackage& ResultPackage) mutable
	{
		if (!ResultPackage)
		{
			for (const T& InProgressRequest : InProgressRequests)
			{
				ChunkCallback(InProgressRequest, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::NotFound) << TEXT("Transport/request failure, ")
					<< DescribeRequest(InProgressRequest) << TEXT("=") << *WriteToString<32>(InProgressRequest)));
			}
			return;
		}

		const FCbObjectView ResultObjectView(ResultPackage.GetObject());
		for (FCbFieldView ChunkField : ResultObjectView[ANSITEXTVIEW("Chunks")])
		{
			const FCbObjectView ChunkObject(ChunkField.AsObjectView());
			T Response;
			DeserializeResponse(ChunkObject, Response);

			ON_SCOPE_EXIT
			{
				InProgressRequests.Remove(Response);
			};

			const FIoHash RawHash = ChunkObject[ANSITEXTVIEW("RawHash")].AsHash();
			if (const FCbAttachment* Attachment = ResultPackage.FindAttachment(RawHash))
			{
				if (const FCompressedBuffer& Compressed = Attachment->AsCompressedBinary(); !Compressed.IsNull())
				{
					FIoBuffer Decompressed(Compressed.GetRawSize());
					if (!FCompressedBufferReader(Compressed).TryDecompressTo(Decompressed.GetMutableView()))
					{
						ChunkCallback(Response, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::CompressionError) << TEXT("Failed to decompress, ")
							<< DescribeRequest(Response) << TEXT("=") << *WriteToString<32>(Response)));
					}
					else
					{
						// Completion status - success
						ChunkCallback(Response, Decompressed);
					}
				}
				else if (const FSharedBuffer& Uncompressed = Attachment->AsBinary(); !Uncompressed.IsNull())
				{
					// Completion status - success
					ChunkCallback(Response, FIoBuffer(FIoBuffer::Clone, Uncompressed.GetData(), Uncompressed.GetSize()));
				}
				else
				{
					ChunkCallback(Response, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::ReadError) << TEXT("Unexpected attachment type, ")
						<< DescribeRequest(Response) << TEXT("=") << *WriteToString<32>(Response)));
				}
			}
			else
			{
				ChunkCallback(Response, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::NotFound) << TEXT("Attachment not found, ")
					<< DescribeRequest(Response) << TEXT("=") << *WriteToString<32>(Response)));
			}
		}

		for (const T& InProgressRequest : InProgressRequests)
		{
			ChunkCallback(InProgressRequest, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::NotFound) << TEXT("Missing response, ")
				<< DescribeRequest(InProgressRequest) << TEXT("=") << *WriteToString<32>(InProgressRequest)));
		}
	};

	if constexpr (bAsync)
	{
		RpcAsync(OplogRpcPath, Writer.Save().AsObject(),
			[ChunkCallback = MoveTemp(ChunkCallback), OnGetChunksResponse = MoveTemp(OnGetChunksResponse)]
			(const FCbPackage& Response) mutable
			{
				OnGetChunksResponse(MoveTemp(ChunkCallback), Response);
			});
	}
	else
	{
		FCbPackage ResultPackage;
		Rpc(OplogRpcPath, Writer.Save().AsObject(), ResultPackage);
		OnGetChunksResponse(MoveTemp(ChunkCallback), MoveTemp(ResultPackage));
	}
}

void FZenStoreHttpClient::ReadChunks(TConstArrayView<FIoChunkId> Ids, TUniqueFunction<void(const FIoChunkId& Id, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	return ReadChunks_Internal<false, FIoChunkId>(Ids, MoveTemp(ChunkCallback));
}

void FZenStoreHttpClient::ReadChunks(TConstArrayView<FIoHash> RawHashes, TUniqueFunction<void(const FIoHash& RawHash, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	return ReadChunks_Internal<false, FIoHash>(RawHashes, MoveTemp(ChunkCallback));
}

void FZenStoreHttpClient::ReadChunksAsync(TConstArrayView<FIoChunkId> Ids, TUniqueFunction<void(const FIoChunkId& Id, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	return ReadChunks_Internal<true, FIoChunkId>(Ids, MoveTemp(ChunkCallback));
}

void FZenStoreHttpClient::ReadChunksAsync(TConstArrayView<FIoHash> RawHashes, TUniqueFunction<void(const FIoHash& RawHash, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	return ReadChunks_Internal<true, FIoHash>(RawHashes, MoveTemp(ChunkCallback));
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadOpLogAttachment(FStringView Id)
{
	TStringBuilder<128> ChunkUri;
	ChunkUri << OplogPath << '/' << Id;
	return ReadOpLogUri(ChunkUri);
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadOpLogUri(FStringBuilderBase& ChunkUri, uint64 Offset, uint64 Size)
{
	check(bAllowRead);

	UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());
	TArray64<uint8> GetBuffer;

	bool bHaveQuery = false;

	auto AppendQueryDelimiter = [&bHaveQuery, &ChunkUri]
	{
		if (bHaveQuery)
		{
			ChunkUri.AppendChar(TEXT('&'));
		}
		else
		{
			ChunkUri.AppendChar(TEXT('?'));
			bHaveQuery = true;
		}
	};

	if (Offset)
	{
		AppendQueryDelimiter();
		ChunkUri.Appendf(TEXT("offset=%" UINT64_FMT), Offset);
	}

	if (Size != ~uint64(0))
	{
		AppendQueryDelimiter();
		ChunkUri.Appendf(TEXT("size=%" UINT64_FMT), Size);
	}

	if (Download(*Request, ChunkUri, &GetBuffer, Zen::EContentType::CompressedBinary) && Request->GetResponseCode() == 200)
	{
		if (FCompressedBuffer Compressed = FCompressedBuffer::FromCompressed(FSharedBuffer::MakeView(GetBuffer.GetData(), GetBuffer.Num())))
		{
			uint64 CompressedOffset = 0;
			if (Offset > 0)
			{
				uint64 BlockSize = 0;
				ECompressedBufferCompressor Compressor;
				ECompressedBufferCompressionLevel CompressionLevel;
				if  (!Compressed.TryGetCompressParameters(Compressor, CompressionLevel, BlockSize))
				{
					return FIoStatus(EIoErrorCode::CompressionError);
				}

				if (BlockSize > 0)
				{
					CompressedOffset = Offset % BlockSize;
				}
			}

			FIoBuffer Decompressed(Compressed.GetRawSize());
			if (!FCompressedBufferReader(Compressed).TryDecompressTo(Decompressed.GetMutableView(), CompressedOffset))
			{
				return FIoStatus(EIoErrorCode::CompressionError);
			}

			return Decompressed;
		}
		else
		{
			return FIoBuffer(FIoBuffer::Clone, GetBuffer.GetData(), GetBuffer.Num());
		}
	}
	return FIoStatus(EIoErrorCode::NotFound);
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetProjectInfo()
{
#if WITH_EDITOR
	EAsyncExecution ThreadPool = EAsyncExecution::LargeThreadPool;
#else
	EAsyncExecution ThreadPool = EAsyncExecution::ThreadPool;
#endif
	return Async(ThreadPool, [this]
	{
		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());

		TArray64<uint8> GetBuffer;
		if (Download(*Request, ProjectPath, &GetBuffer, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
		{
			FCbObjectView Response(GetBuffer.GetData());
			return TIoStatusOr<FCbObject>(FCbObject::Clone(Response));
		}
		else
		{
			return TIoStatusOr<FCbObject>(FIoStatus(EIoErrorCode::NotFound));
		}
	});
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetOplog(bool bTrimByReferencedSet)
{
#if WITH_EDITOR
	EAsyncExecution ThreadPool = EAsyncExecution::LargeThreadPool;
#else
	EAsyncExecution ThreadPool = EAsyncExecution::ThreadPool;
#endif
	return Async(ThreadPool, [this, bTrimByReferencedSet]
	{
		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());
		
		TStringBuilder<128> Uri;
		Uri << OplogPath << TEXT("/entries");
		Uri << TEXT("?") << TEXT("trim_by_referencedset=") << (bTrimByReferencedSet ? TEXT("true") : TEXT("false"));

		TArray64<uint8> GetBuffer;
		if (Download(*Request, Uri, &GetBuffer, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
		{
			FCbObjectView Response(GetBuffer.GetData());
			return TIoStatusOr<FCbObject>(FCbObject::Clone(Response));
		}
		else
		{
			return TIoStatusOr<FCbObject>(FIoStatus(EIoErrorCode::NotFound));
		}
	});
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetFiles()
{
#if WITH_EDITOR
	EAsyncExecution ThreadPool = EAsyncExecution::LargeThreadPool;
#else
	EAsyncExecution ThreadPool = EAsyncExecution::ThreadPool;
#endif
	return Async(ThreadPool, [this]
	{
		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());
		
		TStringBuilder<128> Uri;
		Uri << OplogPath << "/files";

		TArray64<uint8> GetBuffer;
		if (Download(*Request, Uri, &GetBuffer, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
		{
			FCbObjectView Response(GetBuffer.GetData());
			return TIoStatusOr<FCbObject>(FCbObject::Clone(Response));
		}
		else
		{
			return TIoStatusOr<FCbObject>(FIoStatus(EIoErrorCode::NotFound));
		}
	});
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetChunkInfos()
{
#if WITH_EDITOR
	EAsyncExecution ThreadPool = EAsyncExecution::LargeThreadPool;
#else
	EAsyncExecution ThreadPool = EAsyncExecution::ThreadPool;
#endif
	return Async(ThreadPool, [this]
	{
		UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());
		
		TStringBuilder<128> Uri;
		Uri << OplogPath << "/chunkinfos";

		TArray64<uint8> GetBuffer;
		if (Download(*Request, Uri, &GetBuffer, Zen::EContentType::CbObject) && Request->GetResponseCode() == 200)
		{
			FCbObjectView Response(GetBuffer.GetData());
			return TIoStatusOr<FCbObject>(FCbObject::Clone(Response));
		}
		else
		{
			return TIoStatusOr<FCbObject>(FIoStatus(EIoErrorCode::NotFound));
		}
	});
}

void 
FZenStoreHttpClient::StartBuildPass()
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_StartBuildPass);

	check(bAllowEdit);
}

TIoStatusOr<uint64>
FZenStoreHttpClient::EndBuildPass(FCbPackage OpEntry)
{
	TRACE_CPUPROFILER_EVENT_SCOPE(ZenStoreHttp_EndBuildPass);

	check(bAllowEdit);

	FLargeMemoryWriter SerializedPackage;
	UE::Zen::OpLog::SaveCbPackage(OpEntry, SerializedPackage);

	UE_LOG(LogZenStore, Verbose, TEXT("Package size: %lld"), SerializedPackage.TotalSize());

	UE::Zen::FZenScopedRequestPtr Request(RequestPool.Get());

	FMemoryView Payload { SerializedPackage.GetData(), (uint64)SerializedPackage.TotalSize()};
	if (Post(*Request, OplogNewEntryPath, Payload))
	{
		return static_cast<uint64>(Payload.GetSize());
	}
	else
	{
		return (FIoStatus)(FIoStatusBuilder(EIoErrorCode::Unknown) << TEXT("End build pass failed, NewOpLogPath='") << OplogNewEntryPath << TEXT("'"));
	}
}

} // UE

#else // not desktop platform

namespace UE {
namespace Zen {
	struct FZenHttpRequestPool
	{
	};
}

FZenStoreHttpClient::FZenStoreHttpClient()
{
}

FZenStoreHttpClient::FZenStoreHttpClient(const FStringView InHostName, uint16 InPort)
{
}

FZenStoreHttpClient::FZenStoreHttpClient(UE::Zen::FServiceSettings&& InSettings)
{
}

FZenStoreHttpClient::~FZenStoreHttpClient()
{
}

bool FZenStoreHttpClient::TryCreateProject(FStringView InProjectId, FStringView InOplogId, FStringView ServerRoot,
	FStringView EngineRoot,	FStringView ProjectRoot,
	FStringView ProjectFilePath)
{
	return false;
}

bool FZenStoreHttpClient::TryDeleteOplog(FStringView InProjectId, FStringView InOplogId)
{
	return false;
}

bool FZenStoreHttpClient::TryCreateOplog(FStringView InProjectId, FStringView InOplogId, FStringView InOplogLifetimeMarkerPath)
{
	return false;
}

void FZenStoreHttpClient::InitializeReadOnly(FStringView InProjectId, FStringView InOplogId)
{
}

TIoStatusOr<uint64> FZenStoreHttpClient::GetChunkSize(const FIoChunkId& Id)
{
	return 0;
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadChunk(const FIoChunkId& Id, uint64 Offset, uint64 Size)
{
	return FIoBuffer();
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadChunk(const FIoHash& RawHash, uint64 Offset, uint64 Size)
{
	return FIoBuffer();
}

void FZenStoreHttpClient::ReadChunks(TConstArrayView<FIoChunkId> Ids, TUniqueFunction<void(const FIoChunkId& Id, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	for (const FIoChunkId& Id : Ids)
	{
		ChunkCallback(Id, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::Unknown) << TEXT("Unsupported, Oid=") << *WriteToString<32>(Id)));
	}
}

void FZenStoreHttpClient::ReadChunks(TConstArrayView<FIoHash> RawHashes, TUniqueFunction<void(const FIoHash& RawHash, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	for (const FIoHash& RawHash : RawHashes)
	{
		ChunkCallback(RawHash, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::Unknown) << TEXT("Unsupported, RawHash=") << *WriteToString<32>(RawHash)));
	}
}

void FZenStoreHttpClient::ReadChunksAsync(TConstArrayView<FIoChunkId> Ids, TUniqueFunction<void(const FIoChunkId& Id, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	for (const FIoChunkId& Id : Ids)
	{
		ChunkCallback(Id, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::Unknown) << TEXT("Unsupported, Oid=") << *WriteToString<32>(Id)));
	}
}

void FZenStoreHttpClient::ReadChunksAsync(TConstArrayView<FIoHash> RawHashes, TUniqueFunction<void(const FIoHash& RawHash, TIoStatusOr<FIoBuffer> Result)>&& ChunkCallback)
{
	for (const FIoHash& RawHash : RawHashes)
	{
		ChunkCallback(RawHash, (FIoStatus)(FIoStatusBuilder(EIoErrorCode::Unknown) << TEXT("Unsupported, RawHash=") << *WriteToString<32>(RawHash)));
	}
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadOpLogAttachment(FStringView Id)
{
	return FIoBuffer();
}

TIoStatusOr<FIoBuffer> FZenStoreHttpClient::ReadOpLogUri(FStringBuilderBase& ChunkUri, uint64 Offset, uint64 Size)
{
	return FIoBuffer();
}

void FZenStoreHttpClient::StartBuildPass()
{
}

TIoStatusOr<uint64> FZenStoreHttpClient::EndBuildPass(FCbPackage OpEntry)
{
	return FIoStatus(EIoErrorCode::Unknown);
}

TIoStatusOr<uint64> FZenStoreHttpClient::AppendOp(FCbPackage OpEntry)
{
	return TIoStatusOr<uint64>();
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetProjectInfo()
{
	return TFuture<TIoStatusOr<FCbObject>>();
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetOplog(bool bTrimByReferencedSet)
{
	return TFuture<TIoStatusOr<FCbObject>>();
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetFiles()
{
	return TFuture<TIoStatusOr<FCbObject>>();
}

TFuture<TIoStatusOr<FCbObject>> FZenStoreHttpClient::GetChunkInfos()
{
	return TFuture<TIoStatusOr<FCbObject>>();
}

}

#endif // desktop platform

namespace UE
{

bool FZenStoreHttpClient::IsConnected() const
{
	return bConnectionSucceeded;
}

namespace Zen::Private
{

TArray<TUniquePtr<UTF8CHAR[]>> AttachmentIds;
FRWLock AttachmentIdsLock;

}

const UTF8CHAR* FZenStoreHttpClient::FindOrAddAttachmentId(FUtf8StringView AttachmentText)
{
	FRWScopeLock AttachmentIdScopeLock(Zen::Private::AttachmentIdsLock, SLT_ReadOnly);
	bool bReadOnly = true;

	for (;;)
	{
		int32 Index = Algo::LowerBound(Zen::Private::AttachmentIds, AttachmentText,
			[](const TUniquePtr<UTF8CHAR[]>& Existing, FUtf8StringView AttachmentText)
			{
				return FUtf8StringView(Existing.Get()).Compare(AttachmentText, ESearchCase::IgnoreCase) < 0;
			});
		UTF8CHAR* Existing = nullptr;
		if (Index != Zen::Private::AttachmentIds.Num())
		{
			Existing = Zen::Private::AttachmentIds[Index].Get();
		}
 		if (Existing == nullptr || !FUtf8StringView(Existing).Equals(AttachmentText, ESearchCase::IgnoreCase))
		{
			if (bReadOnly)
			{
				AttachmentIdScopeLock.ReleaseReadOnlyLockAndAcquireWriteLock_USE_WITH_CAUTION();
				bReadOnly = false;
				continue;
			}

			auto StrDupNew = [](const UTF8CHAR* Data, int32 Len)
			{
				UTF8CHAR* Dup = new UTF8CHAR[Len + 1];
				FMemory::Memcpy(Dup, Data, Len * sizeof(UTF8CHAR));
				Dup[Len] = UTF8CHAR('\0');
				return Dup;
			};
			Existing = StrDupNew(AttachmentText.GetData(), AttachmentText.Len());
			Zen::Private::AttachmentIds.Insert(TUniquePtr<UTF8CHAR[]>(Existing), Index);
		}
		return Existing;
	}
}

const UTF8CHAR* FZenStoreHttpClient::FindAttachmentId(FUtf8StringView AttachmentText)
{
	FReadScopeLock AttachmentIdScopeLock(Zen::Private::AttachmentIdsLock);

	int32 Index = Algo::LowerBound(Zen::Private::AttachmentIds, AttachmentText,
		[](const TUniquePtr<UTF8CHAR[]>& Existing, FUtf8StringView AttachmentText)
		{
			return FUtf8StringView(Existing.Get()).Compare(AttachmentText, ESearchCase::IgnoreCase) < 0;
		});
	if (Index == Zen::Private::AttachmentIds.Num())
	{
		return nullptr;
	}
	const UTF8CHAR* Existing = Zen::Private::AttachmentIds[Index].Get();
	if (!FUtf8StringView(Existing).Equals(AttachmentText, ESearchCase::IgnoreCase))
	{
		return nullptr;
	}
	return Existing;
}

}