private static void Compare(string baseResultsPath, string newResultsPath, string savePath)
{
if (baseResultsPath == string.Empty)
{
Console.WriteLine("Please generate a baseline benchmark results file," +
"or provide a path for a baseline benchmark results file.");
return;
}
if (newResultsPath == string.Empty)
{
Console.WriteLine("Please provide a path to a benchmark results file to compare against the baseline.");
return;
}
// Create Model comparer
Console.WriteLine("\nComparison of Model tests: \n");
var baseModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), "DynamoPerformanceTests.DynamoModelPerformanceTestBase-report.csv");
var newModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), "DynamoPerformanceTests.DynamoModelPerformanceTestBase-report.csv");
var modelSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), "DynamoPerformanceTests.Comparison-Model.csv");
var modelComparer = new ResultsComparer(baseModelPath, newModelPath, modelSavePath);
// Create View comparer
Console.WriteLine("\nComparison of View tests: \n");
var baseViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), "DynamoPerformanceTests.DynamoViewPerformanceTestBase-report.csv");
var newViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), "DynamoPerformanceTests.DynamoViewPerformanceTestBase-report.csv");
var viewSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), "DynamoPerformanceTests.Comparison-View.csv");
var viewComparer = new ResultsComparer(baseViewPath, newViewPath, viewSavePath);
}
private static void Compare(string baseResultsPath, string newResultsPath, string savePath)
{
if (baseResultsPath == string.Empty)
{
Console.WriteLine("Please generate a baseline benchmark results file," +
"or provide a path for a baseline benchmark results file.");
return;
}
if (newResultsPath == string.Empty)
{
Console.WriteLine("Please provide a path to a benchmark results file to compare against the baseline.");
return;
}
ResultsComparer modelComparer = null;
ResultsComparer viewComparer = null;
try
{
// Create Model comparer
Console.WriteLine("\nComparison of Model tests: \n");
var baseModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), modelTestBaseReport);
var newModelPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), modelTestBaseReport);
var modelSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), modelTestComparison);
modelComparer = new ResultsComparer(baseModelPath, newModelPath, modelSavePath);
// Create View comparer
Console.WriteLine("\nComparison of View tests: \n");
var baseViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(baseResultsPath), viewTestBaseReport);
var newViewPath = Path.Combine(PerformanceTestHelper.GetFullPath(newResultsPath), viewTestBaseReport);
var viewSavePath = Path.Combine(PerformanceTestHelper.GetFullPath(savePath), viewTestComparison);
viewComparer = new ResultsComparer(baseViewPath, newViewPath, viewSavePath);
}
//catch here - likely we could not create a view comparer if view comparison did not run.
catch (Exception e)
{
Console.WriteLine($"exception while comparing results {e} {Environment.NewLine} {e.Message}");
}
//return result of comparison
if (modelComparer != null)
{
if (modelComparer.ComparisonData.Any(x => x.ResultState == ResultsComparer.ComparisonResultState.FAIL))
{
Console.WriteLine("Comparison failed, some model performance benchmarks failed. Please see log above for details.");
Environment.Exit((int)ExitCode.ComparisonFailure);
}
}
if (viewComparer != null)
{
if (viewComparer.ComparisonData.Any(x => x.ResultState == ResultsComparer.ComparisonResultState.FAIL))
{
Console.WriteLine("Comparison failed, some view performance benchmarks failed. Please see log above for details.");
Environment.Exit((int)ExitCode.ComparisonFailure);
}
}
}
private Dictionary <string, BenchmarkResult> ImportResultsCSV(string csvPath)
{
// Get csv
csvPath = PerformanceTestHelper.GetFullPath(csvPath);
TextFieldParser parser = new TextFieldParser(csvPath);
parser.HasFieldsEnclosedInQuotes = true;
parser.SetDelimiters(",");
// Get csv header
var header = parser.ReadFields();
// Check that columns we care about exist
var columnNames = new string[] { "Method", "Graph", "Mean", "Error", "StdDev" };
var missingColumns = columnNames.Where(c => !header.Contains(c));
if (missingColumns.Any())
{
throw new Exception(string.Format("The csv file at {0} does not contain the following required columns: {1}.", csvPath, string.Join(", ", missingColumns)));
}
// Get indices for columns we care about
var iMethod = Array.IndexOf(header, columnNames[0]);
var iGraph = Array.IndexOf(header, columnNames[1]);
var iMean = Array.IndexOf(header, columnNames[2]);
var iError = Array.IndexOf(header, columnNames[3]);
var iStdDev = Array.IndexOf(header, columnNames[4]);
// Parse results
var benchmarkResults = new Dictionary <string, BenchmarkResult>();
while (!parser.EndOfData)
{
// Get full benchmark data from csv
var brLine = parser.ReadFields();
// Get the values we care about
var method = brLine[iMethod];
var graph = brLine[iGraph];
var mean = Convert.ToDouble(Regex.Replace(brLine[iMean], "[^0-9|.|,]", ""), CultureInfo.InvariantCulture);
var error = Convert.ToDouble(Regex.Replace(brLine[iError], "[^0-9|.|,]", ""), CultureInfo.InvariantCulture);
var stdDev = Convert.ToDouble(Regex.Replace(brLine[iStdDev], "[^0-9|.|,]", ""), CultureInfo.InvariantCulture);
var meanUnits = new string(brLine[iMean].ToCharArray().Where(c => Char.IsLetter(c)).ToArray());
var errorUnits = new string(brLine[iError].ToCharArray().Where(c => Char.IsLetter(c)).ToArray());
var stdDevUnits = new string(brLine[iStdDev].ToCharArray().Where(c => Char.IsLetter(c)).ToArray());
// Store the benchmark based on its 'Method' and 'Graph' values
benchmarkResults[method + graph] = (new BenchmarkResult(method, graph, mean, meanUnits, error, errorUnits, stdDev, stdDevUnits));
}
return(benchmarkResults);
}
public static void Main(string[] args)
{
var showHelp = false;
if (args.Length <= 0)
{
Console.WriteLine("Please specify a command.");
return;
}
// Default arguments
IConfig config = PerformanceTestHelper.getFastReleaseConfig();
var testDirectory = "../../../graphs/";
var baseResultsPath = string.Empty;
var newResultsPath = "BenchmarkDotNet.Artifacts/results/";
var saveComparisonPath = string.Empty;
// Command line options
var opts = new OptionSet()
{
{ "g=|graphs=", "Path to Directory containing test graphs. Defaults to 'Dynamo/tools/Performance/DynamoPerformanceTests/graphs/'", v => { testDirectory = v; } },
{ "b=|base=", "Path to Directory containing performance results files to use as comparison base. Defaults to 'BenchmarkDotNet.Artifacts/results/'", v => { baseResultsPath = v; } },
{ "n=|new=", "Path to Directory containing new performance results files to compare against the baseline", v => { newResultsPath = v; } },
{ "s=|save=", "Location to save comparison csv", v => { saveComparisonPath = v; } },
{ "h|help", "show this message and return", v => showHelp = v != null },
};
opts.Parse(args);
// Show help
if (showHelp)
{
ShowHelp(opts);
return;
}
// Get command
Command command;
var commandRecognized = Enum.TryParse(args[0], out command);
if (!commandRecognized)
{
Console.WriteLine("Command \"{0}\" not recognized.", args[0]);
return;
}
// Execute command
switch (command)
{
case Command.NonOptimizedBenchmark:
config = PerformanceTestHelper.getDebugConfig();
goto case Command.Benchmark;
case Command.DebugInProcess:
config = PerformanceTestHelper.getDebugInProcessConfig();
goto case Command.Benchmark;
case Command.Benchmark:
DynamoViewPerformanceTestBase.testDirectory = testDirectory;
var runSummaryWithUI = BenchmarkRunner.Run <DynamoViewPerformanceTestBase>(config);
goto case Command.ModelOnlyBenchmark;
case Command.ModelOnlyBenchmark:
DynamoModelPerformanceTestBase.testDirectory = testDirectory;
var runSummaryWithoutUI = BenchmarkRunner.Run <DynamoModelPerformanceTestBase>(config);
break;
case Command.StandardConfigModelOnlyBenchmark:
config = PerformanceTestHelper.getReleaseConfig();
goto case Command.ModelOnlyBenchmark;
case Command.Compare:
Compare(baseResultsPath, newResultsPath, saveComparisonPath);
break;
}
}
/// <summary>
/// Override this function to preload dlls into Dynamo library
/// </summary>
/// <param name="libraries">extra dlls to load</param>
protected override void GetLibrariesToPreload(List <string> libraries)
{
libraries.AddRange(PerformanceTestHelper.getDynamoDefaultLibs());
base.GetLibrariesToPreload(libraries);
}
/// <summary>
/// This is the entrance of Dynamo Performance Console App
/// </summary>
/// <param name="args"></param>
public static void Main(string[] args)
{
DirectoryInfo dir;
// Running with an input dir location:
// DynamoPerformanceTests.exe "C:\directory path\"
//
if (args.Length <= 0)
{
Console.WriteLine("Supply a path to a test directory containing DYN files");
}
else
{
try
{
dir = new DirectoryInfo(args[0]);
// Use helper to get debug config in order to run benchmarks on debug build of DynamoCore
// PerformanceTestHelper.getDebugConfig();
// Use helper to get debug in process config in order to debug benchmarks
// PerformanceTestHelper.getDebugInProcessConfig();
DynamoViewPerformanceTestBase.testDirectory = dir.FullName;
var runSummaryWithUI = BenchmarkRunner.Run <DynamoViewPerformanceTestBase>(PerformanceTestHelper.getFastReleaseConfig());
DynamoModelPerformanceTestBase.testDirectory = dir.FullName;
var runSummaryWithoutUI = BenchmarkRunner.Run <DynamoModelPerformanceTestBase>(PerformanceTestHelper.getFastReleaseConfig());
}
catch
{
Console.WriteLine("Not a valid path");
}
}
}
