private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
private void PrintTable(List<string[]> table, IBenchmarkLogger logger)
{
int rowCount = table.Count, colCount = table[0].Length;
int[] widths = new int[colCount];
bool[] areSame = new bool[colCount];
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
areSame[colIndex] = rowCount > 2 && colIndex < colCount - 3;
for (int rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1);
if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex])
areSame[colIndex] = false;
}
}
if (areSame.Any(s => s))
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (areSame[colIndex])
logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} ");
logger.NewLine();
logger.WriteLineInfo("```");
logger.NewLine();
}
table.Insert(1, widths.Select(w => new string('-', w)).ToArray());
foreach (var row in table)
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (!areSame[colIndex])
logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |");
logger.NewLine();
}
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo("```ini");
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = ReportExporterHelper.BuildTable(reports);
PrintTable(table, logger);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
private void EnsureDependancyInCorrectLocation(Type type, string outputDir)
{
var fileInfo = new FileInfo(type.Assembly.Location);
if (fileInfo.Name == "mscorlib.dll")
{
return;
}
var expectedLocation = Path.GetFullPath(Path.Combine(outputDir, "..\\" + fileInfo.Name));
if (File.Exists(expectedLocation) == false)
{
logger.WriteLineInfo("// File doesn't exist: {0}", expectedLocation);
logger.WriteLineInfo("// Actually at: {0}", fileInfo.FullName);
CopyFile(fileInfo.FullName, expectedLocation);
}
}
public void Export(IList<BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo("```ini");
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = ReportExporterHelper.BuildTable(reports);
// If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense
var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0];
PrintTable(table, logger, columnsToAlwaysShow);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
private void PrintTable(List<string[]> table, IBenchmarkLogger logger, string [] columnsToAlwaysShow)
{
if (table.Count == 0)
{
logger.WriteLineError("There are no found benchmarks");
logger.WriteLineInfo("```");
logger.NewLine();
return;
}
int rowCount = table.Count, colCount = table[0].Length;
var columnsToShowIndexes = columnsToAlwaysShow.Select(col => Array.IndexOf(table[0], col));
int[] widths = new int[colCount];
bool[] areSame = new bool[colCount];
for (int colIndex = 0; colIndex < colCount; colIndex++)
{
areSame[colIndex] = rowCount > 2 && colIndex < colCount - 3;
for (int rowIndex = 0; rowIndex < rowCount; rowIndex++)
{
widths[colIndex] = Math.Max(widths[colIndex], table[rowIndex][colIndex].Length + 1);
if (rowIndex > 1 && table[rowIndex][colIndex] != table[1][colIndex])
areSame[colIndex] = false;
}
}
if (areSame.Any(s => s))
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (areSame[colIndex] && columnsToShowIndexes.Contains(colIndex) == false)
logger.WriteInfo($"{table[0][colIndex]}={table[1][colIndex]} ");
logger.NewLine();
logger.WriteLineInfo("```");
logger.NewLine();
}
table.Insert(1, widths.Select(w => new string('-', w)).ToArray());
foreach (var row in table)
{
for (int colIndex = 0; colIndex < colCount; colIndex++)
if (!areSame[colIndex] || columnsToShowIndexes.Contains(colIndex))
logger.WriteStatistic(row[colIndex].PadLeft(widths[colIndex], ' ') + " |");
logger.NewLine();
}
}
private BenchmarkBuildResult Build(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain, BenchmarkGenerateResult generateResult)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Build(generateResult);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
{
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
}
}
logger.NewLine();
return(buildResult);
}
private BenchmarkGenerateResult Generate(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generate();
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.DirectoryPath)} = {generateResult.DirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
{
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
}
}
logger.NewLine();
return(generateResult);
}
private List <BenchmarkRunReport> Execute(IBenchmarkLogger logger, Benchmark benchmark, IList <string> importantPropertyNames, BenchmarkParameters parameters, IBenchmarkToolchainFacade toolchain, BenchmarkBuildResult buildResult)
{
logger.WriteLineInfo("// *** Execute ***");
var processCount = Math.Max(1, benchmark.Task.ProcessCount);
var runReports = new List <BenchmarkRunReport>();
for (int processNumber = 0; processNumber < processCount; processNumber++)
{
logger.WriteLineInfo($"// Run, Process: {processNumber + 1} / {processCount}");
if (parameters != null)
{
logger.WriteLineInfo($"// {parameters.ToInfo()}");
}
if (importantPropertyNames.Any())
{
logger.WriteInfo("// ");
foreach (var name in importantPropertyNames)
{
logger.WriteInfo($"{name}={benchmark.Properties.GetValue(name)} ");
}
logger.NewLine();
}
var execResult = toolchain.Execute(buildResult, parameters, Plugins.CompositeDiagnoser);
if (execResult.FoundExecutable)
{
var iterRunReports = execResult.Data.Select(line => BenchmarkRunReport.Parse(logger, line)).Where(r => r != null).ToList();
runReports.AddRange(iterRunReports);
}
else
{
logger.WriteLineError("Executable not found");
}
}
logger.NewLine();
return(runReports);
}
public void Export(IList <BenchmarkReport> reports, IBenchmarkLogger logger)
{
logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var table = BenchmarkExporterHelper.BuildTable(reports);
// If we have Benchmarks with ParametersSets, force the "Method" columns to be displayed, otherwise it doesn't make as much sense
var columnsToAlwaysShow = reports.Any(r => r.Benchmark.Task.ParametersSets != null) ? new[] { "Method" } : new string[0];
PrintTable(table, logger, columnsToAlwaysShow);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
logger.NewLine();
logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
{
logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
}
}
private List <BenchmarkReport> Run(List <Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
{
logger.WriteLineInfo($"// {benchmark.Description}");
}
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List <BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
{
logger.WriteLineError($"{warning.Message}");
}
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return(reports);
}
private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger, string competitionName)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var globalStopwatch = Stopwatch.StartNew();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.GetTargetRuns().Any())
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
}
}
logger.NewLine();
}
globalStopwatch.Stop();
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
logger.WriteLineHeader("// * Export *");
var files = Plugins.CompositeExporter.ExportToFile(reports, competitionName, Plugins.ResultExtenders);
foreach (var file in files)
logger.WriteLineInfo($" {file}");
logger.NewLine();
logger.WriteLineHeader("// * Detailed results *");
foreach (var report in reports)
{
logger.WriteLineInfo(report.Benchmark.Description);
logger.WriteLineStatistic(report.GetTargetRuns().GetStats().ToTimeStr());
logger.NewLine();
}
logger.WriteLineStatistic($"Total time: {globalStopwatch.Elapsed.TotalHours:00}:{globalStopwatch.Elapsed:mm\\:ss}");
logger.NewLine();
logger.WriteLineHeader("// * Summary *");
BenchmarkMarkdownExporter.Default.Export(reports, logger, Plugins.ResultExtenders);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// * Warnings * ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
private List<BenchmarkRunReport> Execute(IBenchmarkLogger logger, Benchmark benchmark, IList<string> importantPropertyNames, BenchmarkParameters parameters, IBenchmarkToolchainFacade toolchain, BenchmarkBuildResult buildResult)
{
logger.WriteLineInfo("// *** Execute ***");
var processCount = Math.Max(1, benchmark.Task.ProcessCount);
var runReports = new List<BenchmarkRunReport>();
for (int processNumber = 0; processNumber < processCount; processNumber++)
{
logger.WriteLineInfo($"// Run, Process: {processNumber + 1} / {processCount}");
if (parameters != null)
logger.WriteLineInfo($"// {parameters.ToInfo()}");
if (importantPropertyNames.Any())
{
logger.WriteInfo("// ");
foreach (var name in importantPropertyNames)
logger.WriteInfo($"{name}={benchmark.Properties.GetValue(name)} ");
logger.NewLine();
}
var execResult = toolchain.Execute(buildResult, parameters, Plugins.CompositeDiagnoser);
if (execResult.FoundExecutable)
{
var iterRunReports = execResult.Data.Select(line => BenchmarkRunReport.Parse(logger, line)).Where(r => r != null).ToList();
runReports.AddRange(iterRunReports);
}
else
{
logger.WriteLineError("Executable not found");
}
}
logger.NewLine();
return runReports;
}
private BenchmarkBuildResult Build(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain, BenchmarkGenerateResult generateResult)
{
logger.WriteLineInfo("// *** Build ***");
var buildResult = toolchain.Build(generateResult);
if (buildResult.IsBuildSuccess)
{
logger.WriteLineInfo("// Result = Success");
}
else
{
logger.WriteLineError("// Result = Failure");
if (buildResult.BuildException != null)
logger.WriteLineError($"// Exception: {buildResult.BuildException.Message}");
}
logger.NewLine();
return buildResult;
}
private BenchmarkGenerateResult Generate(IBenchmarkLogger logger, IBenchmarkToolchainFacade toolchain)
{
logger.WriteLineInfo("// *** Generate *** ");
var generateResult = toolchain.Generate();
if (generateResult.IsGenerateSuccess)
{
logger.WriteLineInfo("// Result = Success");
logger.WriteLineInfo($"// {nameof(generateResult.DirectoryPath)} = {generateResult.DirectoryPath}");
}
else
{
logger.WriteLineError("// Result = Failure");
if (generateResult.GenerateException != null)
logger.WriteLineError($"// Exception: {generateResult.GenerateException.Message}");
}
logger.NewLine();
return generateResult;
}