// Copyright Epic Games, Inc. All Rights Reserved.

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Security;
using System.Reflection;
using System.Runtime.InteropServices;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Core;
using EpicGames.Redis.Utility;
using HordeServer.Utilities;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using MongoDB.Driver;
using MongoDB.Driver.Core.Events;
using MongoDB.Driver.Linq;
using OpenTelemetry.Trace;
using StackExchange.Redis;

namespace HordeServer.Server
{
	/// <summary>
	/// Task used to update indexes
	/// </summary>
	record class MongoUpgradeTask(Func<CancellationToken, Task> UpgradeAsync, TaskCompletionSource CompletionSource);

	/// <summary>
	/// Singleton for accessing the database
	/// </summary>
	public sealed class MongoService : IMongoService, IHealthCheck, IAsyncDisposable
	{
		/// <summary>
		/// The database instance
		/// </summary>
		public IMongoDatabase Database { get; private set; }

		/// <summary>
		/// Collection of singleton documents
		/// </summary>
		IMongoCollection<BsonDocument> SingletonsV1 { get; }

		/// <summary>
		/// Collection of singleton documents
		/// </summary>
		IMongoCollection<BsonDocument> SingletonsV2 { get; }

		/// <summary>
		/// Settings for the application
		/// </summary>
		ServerSettings Settings { get; }

		/// <summary>
		/// Logger for this instance
		/// </summary>
		readonly ILogger<MongoService> _logger;

		/// <summary>
		/// Access the database in a read-only mode (don't create indices or modify content)
		/// </summary>
		public bool ReadOnlyMode { get; }

		/// <summary>
		/// The mongo process group
		/// </summary>
		ManagedProcessGroup? _mongoProcessGroup;

		/// <summary>
		/// The mongo process
		/// </summary>
		ManagedProcess? _mongoProcess;

		/// <summary>
		/// Task to read from the mongo process
		/// </summary>
		Task? _mongoOutputTask;

		/// <summary>
		/// Factory for creating logger instances
		/// </summary>
		readonly ILoggerFactory _loggerFactory;

		/// <summary>
		/// Tracer
		/// </summary>
		readonly Tracer _tracer;

		/// <summary>
		/// Default port for MongoDB connections
		/// </summary>
		const int DefaultMongoPort = 27017;

		static readonly RedisKey s_schemaLockKey = new RedisKey("server/schema-upgrade/lock");

		readonly MongoClient _client;
		readonly IRedisService _redisService;
#pragma warning disable CA2213 // Disposable fields should be disposed
		readonly SemaphoreSlim _upgradeSema = new SemaphoreSlim(1);
#pragma warning restore CA2213 // Disposable fields should be disposed
		CancellationTokenSource _upgradeCancellationSource = new CancellationTokenSource();
		readonly Dictionary<string, Task> _collectionUpgradeTasks = new Dictionary<string, Task>(StringComparer.Ordinal);
		readonly Task<bool> _setSchemaVersionTask;

		static string? s_existingInstance;

		/// <summary>
		/// Constructor
		/// </summary>
		public MongoService(IOptions<ServerSettings> settingsSnapshot, IRedisService redisService, MongoCommandTracer mongoTracer, Tracer tracer, ILogger<MongoService> logger, ILoggerFactory loggerFactory)
		{
			if (s_existingInstance != null)
			{
				throw new Exception("Existing instance on MongoService!");
			}

			s_existingInstance = Environment.StackTrace;

			Settings = settingsSnapshot.Value;
			_redisService = redisService;
			_tracer = tracer;
			_logger = logger;
			_loggerFactory = loggerFactory;

			try
			{
				ReadOnlyMode = Settings.MongoReadOnlyMode;
				if (Settings.MongoPublicCertificate != null)
				{
					X509Store localTrustStore = new X509Store(StoreName.Root);
					try
					{
						localTrustStore.Open(OpenFlags.ReadWrite);

						X509Certificate2Collection collection = ImportCertificateBundle(Settings.MongoPublicCertificate);
						foreach (X509Certificate2 certificate in collection)
						{
							_logger.LogInformation("Importing certificate for {Subject}", certificate.Subject);
						}

						localTrustStore.AddRange(collection);
					}
					finally
					{
						localTrustStore.Close();
					}
				}

				string? connectionString = Settings.MongoConnectionString;
				if (connectionString == null)
				{
					if (IsPortInUse(DefaultMongoPort))
					{
						connectionString = "mongodb://localhost:27017";
					}
					else if (TryStartMongoServer(_logger))
					{
						connectionString = "mongodb://localhost:27017/?readPreference=primary&appname=Horde&ssl=false";
					}
					else
					{
						throw new Exception($"Unable to connect to MongoDB server. Setup a MongoDB server and set the connection string in {ServerApp.ServerConfigFile}");
					}
				}

				MongoClientSettings mongoSettings = MongoClientSettings.FromConnectionString(connectionString);
				mongoSettings.ClusterConfigurator = clusterBuilder =>
				{
					if (_logger.IsEnabled(LogLevel.Trace))
					{
						clusterBuilder.Subscribe<CommandStartedEvent>(ev => TraceMongoCommand(ev.Command));
					}

					if (settingsSnapshot.Value.OpenTelemetry.Enabled)
					{
						mongoTracer.Register(clusterBuilder);
					}
				};

				mongoSettings.LinqProvider = LinqProvider.V2;
				mongoSettings.SslSettings = new SslSettings();
				mongoSettings.SslSettings.ServerCertificateValidationCallback = CertificateValidationCallBack;

				// Increase defaults to handle larger deployments of Horde interfacing with MongoDB
				// Making room for slower running commands potentially hogging the queue 
				mongoSettings.MinConnectionPoolSize = 50; // Default is 0
				mongoSettings.MaxConnectionPoolSize = 300; // Default is 100
#pragma warning disable CS0618 // Type or member is obsolete
				mongoSettings.WaitQueueSize = 5000;
#pragma warning restore CS0618 // Type or member is obsolete
				mongoSettings.MaxConnecting = 10; // Default is 2

				//TestSslConnection(MongoSettings.Server.Host, MongoSettings.Server.Port, Logger);

				_client = new MongoClient(mongoSettings);
				Database = _client.GetDatabase(Settings.MongoDatabaseName);

				SingletonsV1 = GetCollection<BsonDocument>("Singletons");
				SingletonsV2 = GetCollection<BsonDocument>("SingletonsV2");
			}
			catch (Exception ex)
			{
				_logger.LogError(ex, "Exception while initializing MongoService");
				throw;
			}

			_setSchemaVersionTask = SetSchemaVersionAsync(ServerApp.Version, CancellationToken.None);
		}

		internal const int CtrlCEvent = 0;

		[DllImport("kernel32.dll")]
		internal static extern bool GenerateConsoleCtrlEvent(int eventId, int processGroupId);

		/// <inheritdoc/>
		public async ValueTask DisposeAsync()
		{
			if (_upgradeCancellationSource != null)
			{
				await _upgradeCancellationSource.CancelAsync();

				if (_collectionUpgradeTasks.Count > 0)
				{
					_logger.LogInformation("Waiting for upgrade tasks to cancel...");
					try
					{
						await Task.WhenAll(_collectionUpgradeTasks.Values);
					}
					catch (Exception ex)
					{
						_logger.LogInformation(ex, "Discarded upgrade task exception: {Message}", ex.Message);
					}
				}

				_upgradeCancellationSource.Dispose();
				_upgradeCancellationSource = null!;
			}

			if (_mongoProcess != null)
			{
				_logger.LogInformation("Stopping MongoDB...");
				try
				{
					_logger.LogDebug("  Sent shutdown command");
					IMongoDatabase adminDb = _client.GetDatabase("admin");
					await adminDb.RunCommandAsync(new JsonCommand<BsonDocument>("{shutdown: 1}"));
				}
				catch
				{
					// Ignore errors due to connection termination
				}

				try
				{
					_logger.LogInformation("  Waiting for MongoDB to exit");
					await _mongoProcess.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(5.0));

					_mongoProcess.Dispose();
					_mongoProcess = null;

					if (_mongoOutputTask != null)
					{
						_logger.LogInformation("  Waiting for logger task");
						await _mongoOutputTask;
						_mongoOutputTask = null;
					}
				}
				catch (Exception ex)
				{
					_logger.LogInformation(ex, "  Unable to terminate mongo process: {Message}", ex.Message);
				}
				_logger.LogInformation("Done");
			}

			if (_mongoProcessGroup != null)
			{
				_mongoProcessGroup.Dispose();
				_mongoProcessGroup = null;
			}

			//_upgradeSema.Dispose();
			s_existingInstance = null;
		}

		/// <summary>
		/// Checks if the given port is in use
		/// </summary>
		/// <param name="portNumber"></param>
		/// <returns></returns>
		static bool IsPortInUse(int portNumber)
		{
			IPGlobalProperties ipGlobalProperties = IPGlobalProperties.GetIPGlobalProperties();

			IPEndPoint[] listeners = ipGlobalProperties.GetActiveTcpListeners();
			if (listeners.Any(x => x.Port == portNumber))
			{
				return true;
			}

			return false;
		}

		/// <summary>
		/// Attempts to start a local instance of MongoDB
		/// </summary>
		/// <param name="logger"></param>
		/// <returns></returns>
		bool TryStartMongoServer(ILogger logger)
		{
			if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
			{
				return false;
			}

			FileReference mongoExe = FileReference.Combine(ServerApp.AppDir, "ThirdParty", "Mongo", "mongod.exe");
			if (!FileReference.Exists(mongoExe))
			{
				logger.LogWarning("Unable to find Mongo executable.");
				return false;
			}

			DirectoryReference mongoDir = DirectoryReference.Combine(ServerApp.DataDir, "Mongo");

			DirectoryReference mongoDataDir = DirectoryReference.Combine(mongoDir, "Data");
			DirectoryReference.CreateDirectory(mongoDataDir);

			FileReference mongoLogFile = FileReference.Combine(mongoDir, "mongod.log");

			FileReference configFile = FileReference.Combine(mongoDir, "mongod.conf");

			// Check mongo log and config files for non-latin characters, mongodb.exe cannout read these paths
			if (Regex.IsMatch(mongoLogFile.ToString(), "[^a-zA-Z0-9:\\\\\\/.\\-_]"))
			{
				logger.LogError("MongoDB log file contains non-latin characters, which mongodb.exe cannot read: {MongoLogFile}", mongoLogFile.ToString());
			}

			if (Regex.IsMatch(configFile.ToString(), "[^a-zA-Z0-9:\\\\\\/.\\-_]"))
			{
				logger.LogError("MongoDB config file contains non-latin characters, which mongodb.exe cannot read: {MongoConfigFile}", configFile.ToString());
			}

			if (!FileReference.Exists(configFile))
			{
				DirectoryReference.CreateDirectory(configFile.Directory);
				using (StreamWriter writer = new StreamWriter(configFile.FullName))
				{
					writer.WriteLine("# mongod.conf");
					writer.WriteLine();
					writer.WriteLine("# for documentation of all options, see:");
					writer.WriteLine("# http://docs.mongodb.org/manual/reference/configuration-options/");
					writer.WriteLine();
					writer.WriteLine("storage:");
					writer.WriteLine("    dbPath: {0}", mongoDataDir.FullName);
					writer.WriteLine();
					writer.WriteLine("net:");
					writer.WriteLine("    port: {0}", DefaultMongoPort);
					writer.WriteLine("    bindIp: 127.0.0.1");
				}
			}

			_mongoProcessGroup = new ManagedProcessGroup();
			try
			{
				_mongoProcess = new ManagedProcess(_mongoProcessGroup, mongoExe.FullName, $"--config \"{configFile}\"", null, null, ProcessPriorityClass.Normal);
				_mongoProcess.StdIn.Close();
				_mongoOutputTask = Task.Run(() => RelayMongoOutputAsync());
				return true;
			}
			catch (Exception ex)
			{
				logger.LogWarning(ex, "Unable to start Mongo server process");
				return false;
			}
		}

		/// <summary>
		/// Copies output from the mongo process to the logger
		/// </summary>
		/// <returns></returns>
		async Task RelayMongoOutputAsync()
		{
			ILogger mongoLogger = _loggerFactory.CreateLogger("MongoDB");

			Dictionary<string, ILogger> channelLoggers = new Dictionary<string, ILogger>();
			for (; ; )
			{
				string? line = await _mongoProcess!.ReadLineAsync();
				if (line == null)
				{
					break;
				}
				if (line.Length > 0)
				{
					Match match = Regex.Match(line, @"^\s*[^\s]+\s+([A-Z])\s+([^\s]+)\s+(.*)");
					if (match.Success)
					{
						ILogger? channelLogger;
						if (!channelLoggers.TryGetValue(match.Groups[2].Value, out channelLogger))
						{
							channelLogger = _loggerFactory.CreateLogger($"MongoDB.{match.Groups[2].Value}");
							channelLoggers.Add(match.Groups[2].Value, channelLogger);
						}
						channelLogger.Log(ParseMongoLogLevel(match.Groups[1].Value), "{Message}", match.Groups[3].Value.TrimEnd());
					}
					else
					{
						mongoLogger.Log(LogLevel.Information, "{Message}", line);
					}
				}
			}
			mongoLogger.LogInformation("Exit code {ExitCode}", _mongoProcess.ExitCode);
		}

		static LogLevel ParseMongoLogLevel(string text)
		{
			if (text.Equals("I", StringComparison.Ordinal))
			{
				return LogLevel.Information;
			}
			else if (text.Equals("E", StringComparison.Ordinal))
			{
				return LogLevel.Error;
			}
			else
			{
				return LogLevel.Warning;
			}
		}

		/// <summary>
		/// Logs a mongodb command, removing any fields we don't care about
		/// </summary>
		/// <param name="command">The command document</param>
		void TraceMongoCommand(BsonDocument command)
		{
			List<string> names = new List<string>();
			List<string> values = new List<string>();

			foreach (BsonElement element in command)
			{
				if (element.Value != null && !element.Name.Equals("$db", StringComparison.Ordinal) && !element.Name.Equals("lsid", StringComparison.Ordinal))
				{
					names.Add($"{element.Name}: {{{values.Count}}}");
					values.Add(element.Value.ToString()!);
				}
			}

#pragma warning disable CA2254 // Template should be a static expression
			_logger.LogTrace($"MongoDB: {String.Join(", ", names)}", values.ToArray());
#pragma warning restore CA2254 // Template should be a static expression
		}

		/// <summary>
		/// Imports one or more certificates from a single PEM file
		/// </summary>
		/// <param name="fileName">File to import</param>
		/// <returns>Collection of certificates</returns>
		static X509Certificate2Collection ImportCertificateBundle(string fileName)
		{
			X509Certificate2Collection collection = new X509Certificate2Collection();

			string text = File.ReadAllText(fileName);
			for (int offset = 0; offset < text.Length;)
			{
				int nextOffset = text.IndexOf("-----BEGIN CERTIFICATE-----", offset + 1, StringComparison.Ordinal);
				if (nextOffset == -1)
				{
					nextOffset = text.Length;
				}

				string certificateText = text.Substring(offset, nextOffset - offset);
				collection.Add(new X509Certificate2(Encoding.UTF8.GetBytes(certificateText)));

				offset = nextOffset;
			}

			return collection;
		}

		/// <summary>
		/// Provides additional diagnostic information for SSL certificate validation
		/// </summary>
		/// <param name="sender"></param>
		/// <param name="certificate"></param>
		/// <param name="chain"></param>
		/// <param name="sslPolicyErrors"></param>
		/// <returns>True if the certificate is allowed, false otherwise</returns>
		bool CertificateValidationCallBack(object sender, X509Certificate? certificate, X509Chain? chain, SslPolicyErrors sslPolicyErrors)
		{
			// If the certificate is a valid, signed certificate, return true.
			if (sslPolicyErrors == SslPolicyErrors.None)
			{
				return true;
			}

			// Generate diagnostic information
			StringBuilder builder = new StringBuilder();
			if (sender != null)
			{
				string senderInfo = StringUtils.Indent(sender.ToString() ?? String.Empty, "    ");
				builder.Append(CultureInfo.InvariantCulture, $"\nSender:\n{senderInfo}");
			}
			if (certificate != null)
			{
				builder.Append(CultureInfo.InvariantCulture, $"\nCertificate: {certificate.Subject}");
			}
			if (chain != null)
			{
				if (chain.ChainStatus != null && chain.ChainStatus.Length > 0)
				{
					builder.Append("\nChain status:");
					foreach (X509ChainStatus status in chain.ChainStatus)
					{
						builder.Append(CultureInfo.InvariantCulture, $"\n  {status.StatusInformation}");
					}
				}
				if (chain.ChainElements != null)
				{
					builder.Append("\nChain elements:");
					for (int idx = 0; idx < chain.ChainElements.Count; idx++)
					{
						X509ChainElement element = chain.ChainElements[idx];
						builder.Append(CultureInfo.InvariantCulture, $"\n  {idx,4} - Certificate: {element.Certificate.Subject}");
						if (element.ChainElementStatus != null && element.ChainElementStatus.Length > 0)
						{
							foreach (X509ChainStatus status in element.ChainElementStatus)
							{
								builder.Append(CultureInfo.InvariantCulture, $"\n         Status: {status.StatusInformation} ({status.Status})");
							}
						}
						if (!String.IsNullOrEmpty(element.Information))
						{
							builder.Append(CultureInfo.InvariantCulture, $"\n         Info: {element.Information}");
						}
					}
				}
			}

			// Print out additional diagnostic information
			_logger.LogError("TLS certificate validation failed ({Errors}).{AdditionalInfo}", sslPolicyErrors, StringUtils.Indent(builder.ToString(), "    "));
			return false;
		}

		/// <summary>
		/// Get the MongoDB client
		/// </summary>
		/// <returns>A MongoDB client instance</returns>
		public MongoClient GetClient()
		{
			return _client;
		}

		/// <summary>
		/// Get a MongoDB collection from database
		/// </summary>
		/// <param name="name">Name of collection</param>
		/// <typeparam name="T">A MongoDB document</typeparam>
		/// <returns></returns>
		public IMongoCollection<T> GetCollection<T>(string name)
		{
			return GetCollection<T>(name, Enumerable.Empty<MongoIndex<T>>());
		}

		/// <summary>
		/// Get a MongoDB collection from database with a single index
		/// </summary>
		/// <param name="name">Name of collection</param>
		/// <param name="keysFunc">Method to configure keys for the collection</param>
		/// <param name="unique">Whether a unique index is required</param>
		/// <typeparam name="T">A MongoDB document</typeparam>
		/// <returns></returns>
		public IMongoCollection<T> GetCollection<T>(string name, Func<IndexKeysDefinitionBuilder<T>, IndexKeysDefinition<T>> keysFunc, bool unique = false)
		{
			List<MongoIndex<T>> indexes = new List<MongoIndex<T>>();
			indexes.Add(MongoIndex.Create<T>(keysFunc, unique));
			return GetCollection(name, indexes);
		}

		/// <summary>
		/// Get a MongoDB collection from database
		/// </summary>
		/// <param name="name">Name of collection</param>
		/// <param name="indexes">Indexes for the collection</param>
		/// <typeparam name="T">A MongoDB document</typeparam>
		/// <returns></returns>
		public IMongoCollection<T> GetCollection<T>(string name, IEnumerable<MongoIndex<T>> indexes)
		{
			IMongoCollection<T> collection = Database.GetCollection<T>(name);

			Task? upgradeTask = Task.CompletedTask;
			if (indexes.Any())
			{
				lock (_collectionUpgradeTasks)
				{
					if (!_collectionUpgradeTasks.TryGetValue(name, out upgradeTask))
					{
						_logger.LogDebug("Queuing update for collection {Name}", name);

						MongoIndex<T>[] indexesCopy = indexes.ToArray();
						upgradeTask = Task.Run(() => UpdateIndexesAsync(name, collection, indexesCopy, CancellationToken.None));

						_collectionUpgradeTasks.Add(name, upgradeTask);
					}
				}
			}

			return new MongoTracingCollection<T>(Database.GetCollection<T>(name), upgradeTask, _tracer);
		}

		private async Task UpdateIndexesAsync<T>(string collectionName, IMongoCollection<T> collection, MongoIndex<T>[] newIndexes, CancellationToken cancellationToken)
		{
			// Check we're allowed to upgrade the DB
			if (!await _setSchemaVersionTask)
			{
				return;
			}

			int attemptIdx = 1;
			for (; ; )
			{
				try
				{
					await UpdateIndexesInternalAsync<T>(collectionName, collection, newIndexes, cancellationToken);
					break;
				}
				catch (Exception ex) when (attemptIdx < 3)
				{
					_logger.LogError(ex, "Error updating indexes ({Message}) - retrying (attempt {Attempt})", ex.Message, attemptIdx);
					attemptIdx++;
				}
			}
		}

		private async Task UpdateIndexesInternalAsync<T>(string collectionName, IMongoCollection<T> collection, MongoIndex<T>[] newIndexes, CancellationToken cancellationToken)
		{
			// Find all the current indexes, excluding the default
			Dictionary<string, MongoIndex> nameToExistingIndex = new Dictionary<string, MongoIndex>(StringComparer.Ordinal);
			using (IAsyncCursor<BsonDocument> cursor = await collection.Indexes.ListAsync(cancellationToken))
			{
				while (await cursor.MoveNextAsync(cancellationToken))
				{
					foreach (BsonDocument document in cursor.Current)
					{
						MongoIndex indexDocument = BsonSerializer.Deserialize<MongoIndex>(document);
						if (!indexDocument.Name.Equals("_id_", StringComparison.Ordinal))
						{
							nameToExistingIndex.Add(indexDocument.Name, indexDocument);
						}
					}
				}
			}

			// Figure out which indexes to drop
			List<MongoIndex<T>> createIndexes = new List<MongoIndex<T>>();
			HashSet<string> removeIndexNames = new HashSet<string>(nameToExistingIndex.Keys, StringComparer.Ordinal);
			foreach (MongoIndex<T> newIndex in newIndexes)
			{
				MongoIndex? existingIndex;
				if (nameToExistingIndex.TryGetValue(newIndex.Name, out existingIndex))
				{
					if (existingIndex.Equals(newIndex))
					{
						removeIndexNames.Remove(newIndex.Name);
						continue;
					}
				}
				createIndexes.Add(newIndex);
			}

			if (removeIndexNames.Count > 0 || createIndexes.Count > 0)
			{
				await _upgradeSema.WaitAsync(cancellationToken);
				try
				{
					// Aquire a lock for updating the DB
					await using RedisLock schemaLock = new(_redisService.GetDatabase(), s_schemaLockKey);
					while (!await schemaLock.AcquireAsync(TimeSpan.FromMinutes(5.0)))
					{
						_logger.LogDebug("Unable to acquire lock for upgrade task; pausing for 1s");
						await Task.Delay(TimeSpan.FromSeconds(1.0), cancellationToken);
					}

					_logger.LogDebug("Updating indexes for collection {CollectionName}", collectionName);

					// Drop any indexes that are no longer needed
					foreach (string removeIndexName in removeIndexNames)
					{
						_logger.LogWarning("Collection {CollectionName} index {IndexName} is no longer required and should be removed. This may adversely affect performance.", collectionName, removeIndexName);
					}

					// Create all the new indexes
					foreach (MongoIndex<T> createIndex in createIndexes)
					{
						if (ReadOnlyMode)
						{
							_logger.LogWarning("Would create index {CollectionName}.{IndexName} - skipping due to read-only setting.", collectionName, createIndex.Name);
						}
						else
						{
							_logger.LogInformation("Creating index {IndexName} in {CollectionName}", createIndex.Name, collectionName);

							CreateIndexOptions<T> options = new CreateIndexOptions<T>();
							options.Name = createIndex.Name;
							options.Unique = createIndex.Unique;
							options.Sparse = createIndex.Sparse;
							options.Background = true;

							CreateIndexModel<T> model = new CreateIndexModel<T>(createIndex.Keys, options);

							try
							{
								string result = await collection.Indexes.CreateOneAsync(model, cancellationToken: cancellationToken);
								_logger.LogInformation("Created index {IndexName}", result);
							}
							catch (Exception ex)
							{
								_logger.LogError(ex, "Unable to create index {IndexName}: {Message}", createIndex.Name, ex.Message);
								throw;
							}
						}
					}
				}
				finally
				{
					_upgradeSema.Release();
				}

				_logger.LogInformation("Finished updating indexes for collection {CollectionName}", collectionName);
			}
		}

		async Task<bool> SetSchemaVersionAsync(SemVer schemaVersion, CancellationToken cancellationToken)
		{
			// Check we're not downgrading the data
			for (; ; )
			{
				MongoSchemaDocument currentSchema = await GetSingletonAsync<MongoSchemaDocument>(cancellationToken);
				if (!String.IsNullOrEmpty(currentSchema.Version))
				{
					SemVer currentVersion = SemVer.Parse(currentSchema.Version);
					if (schemaVersion < currentVersion)
					{
						_logger.LogInformation("Ignoring upgrade command; server is older than current schema version ({ProgramVer} < {CurrentVer})", ServerApp.Version, currentVersion);
						return false;
					}
					if (schemaVersion == currentVersion)
					{
						return true;
					}
				}

				_logger.LogInformation("Upgrading schema version {OldVersion} -> {NewVersion}", currentSchema.Version, schemaVersion.ToString());
				currentSchema.Version = schemaVersion.ToString();

				if (ReadOnlyMode)
				{
					return false;
				}
				if (await TryUpdateSingletonAsync(currentSchema, cancellationToken))
				{
					return true;
				}
			}
		}

		class SingletonInfo<T> where T : SingletonBase
		{
			public static readonly SingletonDocumentAttribute Attribute = GetAttribute();

			static SingletonDocumentAttribute GetAttribute()
			{
				SingletonDocumentAttribute? attribute = typeof(T).GetCustomAttribute<SingletonDocumentAttribute>();
				if (attribute == null)
				{
					throw new Exception($"Type {typeof(T).Name} is missing a {nameof(SingletonDocumentAttribute)} annotation");
				}
				return attribute;
			}
		}

		/// <summary>
		/// Gets a singleton document by id
		/// </summary>
		/// <returns>The document</returns>
		public Task<T> GetSingletonAsync<T>(CancellationToken cancellationToken) where T : SingletonBase, new()
		{
			return GetSingletonAsync(() => new T(), cancellationToken);
		}

		/// <summary>
		/// Gets a singleton document by id
		/// </summary>
		/// <param name="constructor">Method to use to construct a new object</param>
		/// <param name="cancellationToken">Cancellation token for the operation</param>
		/// <returns>The document</returns>
		public async Task<T> GetSingletonAsync<T>(Func<T> constructor, CancellationToken cancellationToken) where T : SingletonBase, new()
		{
			SingletonDocumentAttribute attribute = SingletonInfo<T>.Attribute;

			FilterDefinition<BsonDocument> filter = new BsonDocument(new BsonElement("_id", attribute.Id));
			for (; ; )
			{
				BsonDocument? document = await SingletonsV2.Find(filter).FirstOrDefaultAsync(cancellationToken);
				if (document != null)
				{
					T item = BsonSerializer.Deserialize<T>(document);
					item.PostLoad();
					return item;
				}

				T? newItem = null;
				if (attribute.LegacyId != null)
				{
					BsonDocument? legacyDocument = await SingletonsV1.Find(new BsonDocument(new BsonElement("_id", ObjectId.Parse(attribute.LegacyId)))).FirstOrDefaultAsync(cancellationToken);
					if (legacyDocument != null)
					{
						legacyDocument.Remove("_id");
						newItem = BsonSerializer.Deserialize<T>(legacyDocument);
						newItem.PostLoad();
					}
				}
				newItem ??= constructor();

				newItem.Id = new SingletonId(attribute.Id);
				await SingletonsV2.InsertOneIgnoreDuplicatesAsync(newItem.ToBsonDocument(), cancellationToken);
			}
		}

		/// <summary>
		/// Updates a singleton
		/// </summary>
		/// <typeparam name="T"></typeparam>
		/// <param name="updater"></param>
		/// <param name="cancellationToken">Cancellation token for the operation</param>
		/// <returns></returns>
		public Task UpdateSingletonAsync<T>(Action<T> updater, CancellationToken cancellationToken) where T : SingletonBase, new()
		{
			bool Update(T instance)
			{
				updater(instance);
				return true;
			}
			return UpdateSingletonAsync<T>(Update, cancellationToken);
		}

		/// <summary>
		/// Updates a singleton
		/// </summary>
		/// <typeparam name="T"></typeparam>
		/// <param name="updater"></param>
		/// <param name="cancellationToken">Cancellation token for the operation</param>
		/// <returns></returns>
		public async Task UpdateSingletonAsync<T>(Func<T, bool> updater, CancellationToken cancellationToken) where T : SingletonBase, new()
		{
			for (; ; )
			{
				T document = await GetSingletonAsync(() => new T(), cancellationToken);
				if (!updater(document))
				{
					break;
				}
				if (await TryUpdateSingletonAsync(document, cancellationToken))
				{
					break;
				}
			}
		}

		/// <summary>
		/// Attempts to update a singleton object
		/// </summary>
		/// <param name="singletonObject">The singleton object</param>
		/// <param name="cancellationToken">Cancellation token for the operation</param>
		/// <returns>True if the singleton document was updated</returns>
		public async Task<bool> TryUpdateSingletonAsync<T>(T singletonObject, CancellationToken cancellationToken) where T : SingletonBase
		{
			int prevRevision = singletonObject.Revision++;

			BsonDocument filter = new BsonDocument { new BsonElement("_id", singletonObject.Id.ToString()), new BsonElement(nameof(SingletonBase.Revision), prevRevision) };
			try
			{
				ReplaceOneResult result = await SingletonsV2.ReplaceOneAsync(filter, singletonObject.ToBsonDocument(), new ReplaceOptions { IsUpsert = true }, cancellationToken);
				return result.MatchedCount > 0;
			}
			catch (MongoWriteException ex)
			{
				// Duplicate key error occurs if filter fails to match because revision is not the same.
				if (ex.WriteError != null && ex.WriteError.Category == ServerErrorCategory.DuplicateKey)
				{
					return false;
				}
				else
				{
					throw;
				}
			}
		}

		/// <inheritdoc/>
		public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
		{
			try
			{
				await Database.RunCommandAsync((Command<BsonDocument>)"{ping:1}", cancellationToken: cancellationToken);
				return HealthCheckResult.Healthy();
			}
			catch (Exception ex)
			{
				return HealthCheckResult.Unhealthy("Unable to ping MongoDB", ex);
			}
		}
	}

	/// <summary>
	/// Stores the version number of the latest server instance to have upgraded the database schema
	/// </summary>
	[SingletonDocument("mongo-schema", "62470d8508d48eddfac7b55d")]
	public class MongoSchemaDocument : SingletonBase
	{
		/// <summary>
		/// Current version number
		/// </summary>
		public string? Version { get; set; }
	}
}