// Copyright (C) Microsoft. All rights reserved. // Copyright Epic Games, Inc. All Rights Reserved. using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Text; using System.Diagnostics; using System.Collections; using System.Text.Json; using System.Data; using CSVStats; namespace RegressionsReport { class Version { // Major.Minor.Bugfix private static string VersionString = "1.1.0"; public static string Get() { return VersionString; } }; class Program : CommandLineTool { static string defaultBaseHTML = @" Regressions Report

Regressions Report

"; static string toolInformation = "RegressionsReport v" + Version.Get() + "\n" + "Required Arguments:\n" + " -csvFile - Path to input CSV file\n" + " -o - Path to output directory\n" + " -thresholds - Path to JSON file containing thresholds\n" + "Optional Arguments:\n" + " -filename - Will set output filename (Default: \"highlights.html\")" + " -insertAfterTag - Insert the report after first
tag in base HTML file (Default: \"highlights\")\n" + " -base - Will set base HTML file for report\n" + " -dumpContents - Will dump the summary contents to a JSON file\n" + " -testName - Will set the name of the test\n" + ""; // Run the main tool void Run(string[] args) { ReadCommandLine(args); // Check for help flag if(GetBoolArg("help")) { WriteLine(toolInformation); return; } // Check for invalid usage bool requiredArguments = !GetBoolArg("csvFile") || !GetBoolArg("o") || !GetBoolArg("thresholds"); if (requiredArguments) { WriteLine("Missing required arguments, see -help flag"); WriteLine(toolInformation); return; } // Run tool WriteLine("RegressionsReport v" + Version.Get()); // Set flag arguments string inputFile = GetArg("csvFile"); string outputPath = GetArg("o"); string thresholdsFile = GetArg("thresholds"); string outputName = GetArg("filename", "highlights.html"); string insertAfterTag = GetArg("insertAfterTag", "highlights"); string baseFile = GetArg("base", defaultBaseHTML); string dumpContentsFile = GetArg("dumpContents", ""); string testName = GetArg("testName", "PerfTests"); // Read CSV file into DataTable DataTable dataTable = ReadCsv(inputFile); // Need at least two columns to compare // TODO: Just show something in the report to signify this instead of throwing error if(dataTable.Rows.Count < 2) { WriteLine("Need at least two columns in the table"); return; } // Read thresholds data from JSON file var thresholds = JsonSerializer.Deserialize>>(File.ReadAllText(thresholdsFile)); // Compute changes between newest CLs DataRow previousRow = dataTable.Rows[0]; DataRow newRow = dataTable.Rows[1]; var percentageChanges = ComputePercentageChanges(newRow, previousRow, dataTable.Columns, thresholds); // Identify regressions var significantRegressions = new Dictionary(); var minorRegressions = new Dictionary(); foreach (var change in percentageChanges) { double significantThreshold = (double) thresholds["significant"][change.Key]; double minorThreshold = (double) thresholds["minor"][change.Key]; if (Math.Abs(change.Value) > significantThreshold) { significantRegressions[change.Key] = change.Value; } else if (Math.Abs(change.Value) > minorThreshold) { minorRegressions[change.Key] = change.Value; } } // Generate regression message string regressionMessage; if(significantRegressions.Any()) { regressionMessage = "This CL showed significant regressions."; } else if (minorRegressions.Any()) { regressionMessage = "This CL showed minor regressions."; } else { regressionMessage = "This CL did not have any meaningful regressions."; } // Generate report lines var significantLines = GenerateRegressionLines(significantRegressions, percentageChanges, previousRow, newRow, "significant"); var minorLines = GenerateRegressionLines(minorRegressions, percentageChanges, previousRow, newRow, "minor"); // Generate HTML report content string reportContent = GenerateReportContent(newRow, regressionMessage, significantLines, minorLines, percentageChanges); // Write to HTML file WriteHtmlReport(outputPath, outputName, baseFile, insertAfterTag, reportContent); // Dump contents to JSON file if flag is present if (!string.IsNullOrEmpty(dumpContentsFile)) { DumpContentsToJson(dumpContentsFile, previousRow, newRow, significantRegressions, percentageChanges, testName); } } // Creates a DataTable from reading in a CSV file static DataTable ReadCsv(string filePath) { DataTable dataTable = new DataTable(); using (var reader = new StreamReader(filePath)) { bool isFirstRow = true; while (!reader.EndOfStream) { var line = reader.ReadLine(); var values = line.Split(','); // Check if its the first row of the CSV if (isFirstRow) { // Add each column name to the DataTable foreach (var column in values) { dataTable.Columns.Add(column); } isFirstRow = false; } else { // Add to row to DataTable dataTable.Rows.Add(values); } } } return dataTable; } // Compute perctange change between two rows in the DataTable based on some threshold values static Dictionary ComputePercentageChanges(DataRow newRow, DataRow oldRow, DataColumnCollection columns, Dictionary> thresholds) { // Calculate perctange change in specified columns var percentageChanges = new Dictionary(); for (int i = 0; i < columns.Count; i++) { string columnName = columns[i].ColumnName; // Only compute change between columns entries specified within the JSON file if(thresholds["significant"].ContainsKey(columnName) || thresholds["minor"].ContainsKey(columnName)) { double newValue = Convert.ToDouble(newRow[columnName]); double oldValue = Convert.ToDouble(oldRow[columnName]); if (oldValue != 0) { double percentageChange = ((newValue - oldValue) / oldValue) * 100; percentageChanges[columnName] = percentageChange; } else { percentageChanges[columnName] = double.PositiveInfinity; } } } return percentageChanges; } // Generates output lines for each regression that has occurred static List GenerateRegressionLines(Dictionary regressions, Dictionary changes, DataRow oldRow, DataRow newRow, string category) { return regressions.Select(reg => $"{category} regression in terms of {reg.Key} - Change of {changes[reg.Key]:F2}%, from {oldRow[reg.Key]} to {newRow[reg.Key]}" ).ToList(); } // Generates the new resulting content for the HTML report static string GenerateReportContent(DataRow newRow, string message, List significantLines, List minorLines, Dictionary percentageChanges) { var significantHtml = string.Join("", significantLines.Select(line => $"
  • {line}
    • ")); var minorHtml = string.Join("", minorLines.Select(line => $"
  • {line}
    • ")); var detailsHtml = string.Join("", percentageChanges.Select(change => $"
  • {change.Key} changed by {change.Value:F2}% to {newRow[change.Key]}
    • " )); return $@"

    Report on {newRow["buildversion"]}

    {message}

    Significant Regressions

      {significantHtml}

    Minor Regressions

      {minorHtml}
    ▼ Click to see all details
    "; } // Writes the new new report HTML report static void WriteHtmlReport(string outputPath, string outputName, string baseHtml, string insertAfterTag, string reportContent) { string outputFilePath = Path.Combine(outputPath, outputName); string content; // Append if the file exists, otherwise use the base template if (File.Exists(outputFilePath)) { content = File.ReadAllText(outputFilePath); } else { content = baseHtml; } // Append the report after the first instance of the specified tag string insertTag = $"
    "; int insertIndex = content.IndexOf(insertTag) + insertTag.Length; content = content.Insert(insertIndex, reportContent); File.WriteAllText(outputFilePath, content); } // Creates a JSON file with a content summary of the regression report it generated static void DumpContentsToJson(string filePath, DataRow oldRow, DataRow newRow, Dictionary significantRegressions, Dictionary percentageChanges, string testName) { // Store regression information in a dictionary var regressions = new Dictionary(); foreach (var regression in significantRegressions) { string stat = regression.Key; regressions[stat] = new { percentage_change = $"{percentageChanges[stat]:F2}%", original_value = oldRow[stat].ToString(), new_value = newRow[stat].ToString() }; } // Create new JSON object var jsonObject = new { had_regression = significantRegressions.Count != 0, commit = $"CL {newRow["buildversion"]}", test = testName, html_path = filePath.Replace(".json", ".html"), regressions = regressions }; // Write JSON object to path provided string jsonString = JsonSerializer.Serialize(jsonObject, new JsonSerializerOptions { WriteIndented = true }); File.WriteAllText(filePath, jsonString); } static int Main(string[] args) { Program program = new Program(); if (Debugger.IsAttached) { program.Run(args); } else { try { program.Run(args); } catch (System.Exception e) { Console.WriteLine("[ERROR] " + e.Message); return 1; } } return 0; } } }