// Copyright Epic Games, Inc. All Rights Reserved. using System; using System.IO; using System.Linq; using System.Data; using System.Collections.Generic; using MySql.Data.MySqlClient; using System.Text.Json; using System.Text.RegularExpressions; using AutomationTool; namespace Gauntlet { /// /// Hold information about the telemetry context /// public interface ITelemetryContext { object GetProperty(string Name); } /// /// Interface to drive Database submission /// public interface IDatabaseDriver where Data : class { /// /// Submit collection of object to target Database, use TestContext to complete data modeling. /// /// /// /// bool SubmitDataItems(IEnumerable DataItems, ITelemetryContext Context); /// /// Execute a Query through the database driver /// /// /// DataSet ExecuteQuery(string Query); /// /// Fetch collection of object from target Database, uses Conditions to narrow down pool of data. /// /// /// DataSet FetchData(ITelemetryContext Conditions); } /// /// Interface for Database Configuration /// public interface IDatabaseConfig where Data : class { void LoadConfig(string ConfigFilePath); IDatabaseDriver GetDriver(); } /// /// Interface for data type MySQL Configuration /// public interface IMySQLConfig where Data : class { /// /// Get Target Table name based on data type /// string GetTableName(); /// /// Get Target Table columns based on data type /// IEnumerable GetTableColumns(); /// /// Format the data for target table based on data type /// /// Data to format /// ITelemetryContext of the telemetry data /// IEnumerable FormatDataForTable(Data InData, ITelemetryContext InContext); } public class MySQLDriver : IDatabaseDriver where Data : class { protected MySQLConfig Config; public MySQLDriver(MySQLConfig InConfig) { Config = InConfig; if (string.IsNullOrEmpty(Config.ConfigString)) { throw new AutomationException(string.Format("Database Driver '{0}' not configured.", this.GetType().FullName)); } if (string.IsNullOrEmpty(Config.DatabaseName)) { throw new AutomationException(string.Format("Database Driver '{0}' not configured properly, missing Database name.", this.GetType().FullName)); } } public override string ToString() { return string.Format("{0} with config '{1}'", Config.GetConfigValue("Server"), Config.GetType().FullName); } public DataSet ExecuteQuery(string SqlQuery) { return MySqlHelper.ExecuteDataset(Config.ConfigString, SqlQuery); } public bool Insert(string Table, IEnumerable Columns, IEnumerable> Rows) { foreach (var Chunk in ChunkIt(Rows)) { string SqlQuery = string.Format( "INSERT INTO `{0}`.{1} ({2}) VALUES {3};SELECT LAST_INSERT_ID()", Config.DatabaseName, Table, string.Join(", ", Columns), string.Join(", ", Chunk.Select(R => string.Format("({0})", string.Join(", ", R.Select(V => string.Format("'{0}'", V)))))) ); if (MySqlHelper.ExecuteScalar(Config.ConfigString, SqlQuery) == null) { return false; } } return true; } private IEnumerable> ChunkIt(IEnumerable ToChunk, int ChunkSize = 1000) { return ToChunk.Select((v, i) => new { Value = v, Index = i }).GroupBy(x => x.Index / ChunkSize).Select(g => g.Select(x => x.Value)); } public bool SubmitDataItems(IEnumerable DataRows, ITelemetryContext Context) { if (Config is IMySQLConfig DataConfig) { if(!PreSubmitQuery(DataRows, Context)) { Log.Error("Fail MySQL '{0}' pre-submit query.", Config.GetType().FullName); return false; } bool Success = Insert( DataConfig.GetTableName(), DataConfig.GetTableColumns(), DataRows.Select(D => DataConfig.FormatDataForTable(D, Context)) ); if(Success && !PostSubmitQuery(DataRows, Context)) { Log.Error("Fail MySQL '{0}' post-submit query.", Config.GetType().FullName); return false; } return Success; } else { Log.Error("MySQL configuration '{0}' does not known how to handle {1}.", Config.GetType().FullName, typeof(Data).FullName); } return false; } public virtual bool PreSubmitQuery(IEnumerable DataRows, ITelemetryContext Context) { return Config.PreSubmitQuery(this, DataRows, Context); } public virtual bool PostSubmitQuery(IEnumerable DataRows, ITelemetryContext Context) { return Config.PostSubmitQuery(this, DataRows, Context); } public DataSet FetchData(ITelemetryContext Context) { return MySqlHelper.ExecuteDataset(Config.ConfigString, Config.GetFetchDataQuery(Context)); } } public abstract class MySQLConfig : IDatabaseConfig, IMySQLConfig where Data : class { public string ConfigString { get; protected set; } public string DatabaseName { get; protected set; } protected Dictionary KeyValuePairs = null; public virtual void LoadConfig(string ConfigFilePath) { ConfigString = string.Empty; DatabaseName = string.Empty; KeyValuePairs = null; if (File.Exists(ConfigFilePath)) { using (StreamReader ConnectionReader = new StreamReader(ConfigFilePath)) { ConfigString = ConnectionReader.ReadLine(); } if (string.IsNullOrEmpty(ConfigString)) { Log.Warning("Properly found config file, but couldn't read a valid connection string."); return; } else { Log.Info("Found MySQL connection string from config file."); } } else { Log.Error("Could not find connection string config file at '{0}'.", ConfigFilePath); return; } DatabaseName = GetConfigValue("database"); if (string.IsNullOrEmpty(DatabaseName)) { Log.Warning("Missing MySQL Database name in config file '{0}'.", ConfigFilePath); return; } } public string GetConfigValue(string Key) { if (string.IsNullOrEmpty(ConfigString)) { return null; } if (KeyValuePairs == null) { KeyValuePairs = ConfigString.Split(';').Where(KeyValue => KeyValue.Contains('=')) .Select(KeyValue => KeyValue.Split('=', 2)) .ToDictionary( KeyValue => KeyValue[0].Trim(), KeyValue => KeyValue[1].Trim(), StringComparer.InvariantCultureIgnoreCase ); } string FoundValue; if (KeyValuePairs.TryGetValue(Key, out FoundValue)) { return FoundValue; } return string.Empty; } public IDatabaseDriver GetDriver() { return new MySQLDriver(this); } /// /// Override to add pre-submit query. /// /// /// /// /// public virtual bool PreSubmitQuery(IDatabaseDriver Driver, IEnumerable DataRows, ITelemetryContext Context) { return true; } /// /// Override to add post-submit query. /// /// /// /// /// public virtual bool PostSubmitQuery(IDatabaseDriver Driver, IEnumerable DataRows, ITelemetryContext Context) { return true; } /// /// Get Target Table name based on data type /// public abstract string GetTableName(); /// /// Get Target Table columns based on data type /// public abstract IEnumerable GetTableColumns(); /// /// Format the data for target table based on data type /// /// public abstract IEnumerable FormatDataForTable(Data InData, ITelemetryContext InContext); /// /// Return the query to fetch data /// /// /// public virtual string GetFetchDataQuery(ITelemetryContext InContext) { return null; } } public class JsonHttpRequestDriver : IDatabaseDriver where Data : class { protected JsonHttpRequestConfig Config; public JsonHttpRequestDriver(JsonHttpRequestConfig InConfig) { Config = InConfig; if (string.IsNullOrEmpty(Config.Host)) { throw new AutomationException(string.Format("Database Driver '{0}' host not configured.", this.GetType().FullName)); } if (Config.Auth == null) { throw new AutomationException(string.Format("Database Driver '{0}' authorization header not configured.", this.GetType().FullName)); } } public virtual bool SubmitDataItems(IEnumerable DataItems, ITelemetryContext Context) { var Rows = Config.FormatData(DataItems, Context); foreach (var Chunk in ChunkIt(Rows)) { string JsonString = JsonSerializer.Serialize(Chunk); var Result = ExecuteQuery(JsonString); if (Result == null) { return false; } } return true; } private IEnumerable> ChunkIt(IEnumerable ToChunk, int ChunkSize = 1000) { return ToChunk.Select((v, i) => new { Value = v, Index = i }).GroupBy(x => x.Index / ChunkSize).Select(g => g.Select(x => x.Value)); } public virtual DataSet ExecuteQuery(string Query) { // Send data var Auth = HttpRequest.Authentication(Config.Auth.target, Config.Auth.token); var Connection = new HttpRequest.Connection(Config.Host, Auth); var Response = Connection.PostJson(Config.Path, Query); if (!Response.IsSuccessStatusCode) { Log.Error("Error sending data to host. Status={0}.\n{1}", Response.StatusCode, Response.Content); return null; } // Format response var Data = new DataSet(); DataTable Table = new DataTable(); Table.Columns.Add("Status", typeof(int)); Table.Columns.Add("Content", typeof(string)); Table.Rows.Add(new object[] { Response.StatusCode, Response.Content }); Table.AcceptChanges(); Data.Tables.Add(Table); return Data; } public DataSet FetchData(ITelemetryContext Conditions) { return new DataSet(); } public override string ToString() { return string.Format("{0}/{1} with config '{2}'", Config.Host, Config.Path ?? "", Config.GetType().FullName); } } public abstract class JsonHttpRequestConfig : IDatabaseConfigwhere Data : class { public string Host; public string Path; public JsonHttpRequestConfig.Auth Auth; public virtual void LoadConfig(string ConfigFilePath) { JsonHttpRequestConfig Config = null; Host = string.Empty; Path = string.Empty; Auth = null; if (File.Exists(ConfigFilePath)) { try { Config = JsonHttpRequestConfig.LoadFromFile(ConfigFilePath); Log.Info("Found Http Request connection config from file."); } catch (Exception Ex) { Log.Warning("Properly found config file, but couldn't read a valid json schema.\n{0}", Ex); return; } } else { Log.Error("Could not find Http Request connection config file at '{0}'.", ConfigFilePath); return; } var Url = new Regex(@"^(https?://[^/]+/?)(.+)?$"); var Match = Url.Match(Config.endpoint); var Success = Match.Success; if (!Success) { Log.Warning("Properly found config file, but couldn't parse http endpoint."); return; } Host = Match.Groups[1].Value; Path = Match.Groups[2].Value; Auth = Config.authorization; } /// /// Format the data for target data type /// /// public abstract IEnumerable FormatData(IEnumerable DataItems, ITelemetryContext Context); public virtual IDatabaseDriver GetDriver() { JsonHttpRequestDriver Driver = new JsonHttpRequestDriver(this); return Driver; } } public class JsonHttpRequestConfig { public class Auth { public string target { get; set; } public string token { get; set; } } public string endpoint { get; set; } public Auth authorization { get; set; } public static JsonHttpRequestConfig LoadFromFile(string FilePath) { JsonSerializerOptions Options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true }; string JsonString = File.ReadAllText(FilePath); return JsonSerializer.Deserialize(JsonString, Options); } } public class DatabaseConfigManager where Data : class { protected static IEnumerable> Configs; static DatabaseConfigManager() { Configs = Gauntlet.Utils.InterfaceHelpers.FindImplementations>(true); } public static IDatabaseConfig GetConfigByName(string Name) { return Configs.Where(C => string.Equals(C.GetType().Name, Name, StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); } } }