private static void Compare(string baseResultsPath, string newResultsPath, string savePath) { if (baseResultsPath == string.Empty) { Console.WriteLine("Please generate a baseline benchmark results file," + "or provide a path for a baseline benchmark results file."); return; } if (newResultsPath == string.Empty) { Console.WriteLine("Please provide a path to a benchmark results file to compare against the baseline."); return; } // Create Model comparer Console.WriteLine("\nComparison of Model tests: \n"); var baseModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), "DynamoPerformanceTests.DynamoModelPerformanceTestBase-report.csv"); var newModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), "DynamoPerformanceTests.DynamoModelPerformanceTestBase-report.csv"); var modelSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), "DynamoPerformanceTests.Comparison-Model.csv"); var modelComparer = new ResultsComparer(baseModelPath, newModelPath, modelSavePath); // Create View comparer Console.WriteLine("\nComparison of View tests: \n"); var baseViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), "DynamoPerformanceTests.DynamoViewPerformanceTestBase-report.csv"); var newViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), "DynamoPerformanceTests.DynamoViewPerformanceTestBase-report.csv"); var viewSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), "DynamoPerformanceTests.Comparison-View.csv"); var viewComparer = new ResultsComparer(baseViewPath, newViewPath, viewSavePath); }
private static void Compare(string baseResultsPath, string newResultsPath, string savePath) { if (baseResultsPath == string.Empty) { Console.WriteLine("Please generate a baseline benchmark results file," + "or provide a path for a baseline benchmark results file."); return; } if (newResultsPath == string.Empty) { Console.WriteLine("Please provide a path to a benchmark results file to compare against the baseline."); return; } ResultsComparer modelComparer = null; ResultsComparer viewComparer = null; try { // Create Model comparer Console.WriteLine("\nComparison of Model tests: \n"); var baseModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), modelTestBaseReport); var newModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), modelTestBaseReport); var modelSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), modelTestComparison); modelComparer = new ResultsComparer(baseModelPath, newModelPath, modelSavePath); // Create View comparer Console.WriteLine("\nComparison of View tests: \n"); var baseViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), viewTestBaseReport); var newViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), viewTestBaseReport); var viewSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), viewTestComparison); viewComparer = new ResultsComparer(baseViewPath, newViewPath, viewSavePath); } //catch here - likely we could not create a view comparer if view comparison did not run. catch (Exception e) { Console.WriteLine($"exception while comparing results {e} {Environment.NewLine} {e.Message}"); } //return result of comparison if (modelComparer != null) { if (modelComparer.ComparisonData.Any(x => x.ResultState == ResultsComparer.ComparisonResultState.FAIL)) { Console.WriteLine("Comparison failed, some model performance benchmarks failed. Please see log above for details."); Environment.Exit((int)ExitCode.ComparisonFailure); } } if (viewComparer != null) { if (viewComparer.ComparisonData.Any(x => x.ResultState == ResultsComparer.ComparisonResultState.FAIL)) { Console.WriteLine("Comparison failed, some view performance benchmarks failed. Please see log above for details."); Environment.Exit((int)ExitCode.ComparisonFailure); } } }
private Dictionary <string, BenchmarkResult> ImportResultsCSV(string csvPath) { // Get csv csvPath = PerformanceTestHelper.GetFullPath(csvPath); TextFieldParser parser = new TextFieldParser(csvPath); parser.HasFieldsEnclosedInQuotes = true; parser.SetDelimiters(","); // Get csv header var header = parser.ReadFields(); // Check that columns we care about exist var columnNames = new string[] { "Method", "Graph", "Mean", "Error", "StdDev" }; var missingColumns = columnNames.Where(c => !header.Contains(c)); if (missingColumns.Any()) { throw new Exception(string.Format("The csv file at {0} does not contain the following required columns: {1}.", csvPath, string.Join(", ", missingColumns))); } // Get indices for columns we care about var iMethod = Array.IndexOf(header, columnNames[0]); var iGraph = Array.IndexOf(header, columnNames[1]); var iMean = Array.IndexOf(header, columnNames[2]); var iError = Array.IndexOf(header, columnNames[3]); var iStdDev = Array.IndexOf(header, columnNames[4]); // Parse results var benchmarkResults = new Dictionary <string, BenchmarkResult>(); while (!parser.EndOfData) { // Get full benchmark data from csv var brLine = parser.ReadFields(); // Get the values we care about var method = brLine[iMethod]; var graph = brLine[iGraph]; var mean = Convert.ToDouble(Regex.Replace(brLine[iMean], "[^0-9|.|,]", ""), CultureInfo.InvariantCulture); var error = Convert.ToDouble(Regex.Replace(brLine[iError], "[^0-9|.|,]", ""), CultureInfo.InvariantCulture); var stdDev = Convert.ToDouble(Regex.Replace(brLine[iStdDev], "[^0-9|.|,]", ""), CultureInfo.InvariantCulture); var meanUnits = new string(brLine[iMean].ToCharArray().Where(c => Char.IsLetter(c)).ToArray()); var errorUnits = new string(brLine[iError].ToCharArray().Where(c => Char.IsLetter(c)).ToArray()); var stdDevUnits = new string(brLine[iStdDev].ToCharArray().Where(c => Char.IsLetter(c)).ToArray()); // Store the benchmark based on its 'Method' and 'Graph' values benchmarkResults[method + graph] = (new BenchmarkResult(method, graph, mean, meanUnits, error, errorUnits, stdDev, stdDevUnits)); } return(benchmarkResults); }