private List<BenchmarkReport> Run(List<Benchmark> benchmarks, IBenchmarkLogger logger)
{
logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
logger.WriteLineInfo($"// {benchmark.Description}");
logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(logger, benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(logger, benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
logger.NewLine();
}
logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
logger.NewLine();
BenchmarkMarkdownExporter.Default.Export(reports, logger);
var warnings = Plugins.CompositeAnalyser.Analyze(reports).ToList();
if (warnings.Count > 0)
{
logger.NewLine();
logger.WriteLineError("// *** Warnings *** ");
foreach (var warning in warnings)
logger.WriteLineError($"{warning.Message}");
}
logger.NewLine();
logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
internal IEnumerable<BenchmarkReport> Run(List<Benchmark> benchmarks)
{
Logger.WriteLineHeader("// ***** BenchmarkRunner: Start *****");
Logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
Logger.WriteLineInfo($"// {benchmark.Description}");
Logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
if (benchmark.Task.ParametersSets.IsEmpty())
{
var report = Run(benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
Logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
Logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
else
{
var parametersSets = benchmark.Task.ParametersSets;
foreach (var parameters in parametersSets.ToParameters())
{
var report = Run(benchmark, importantPropertyNames, parameters);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
Logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
Logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
}
}
Logger.NewLine();
}
Logger.WriteLineHeader("// ***** BenchmarkRunner: Finish *****");
Logger.NewLine();
ReportExporter.Export(reports, Logger);
Logger.NewLine();
Logger.WriteLineHeader("// ***** BenchmarkRunner: End *****");
return reports;
}
public IEnumerable<BenchmarkReport> RunCompetition(List<Benchmark> benchmarks)
{
benchmarks.Sort((a, b) => string.Compare((a.Task.Configuration.Caption + a.Target.Caption), b.Task.Configuration.Caption + b.Target.Caption, StringComparison.Ordinal));
Logger.WriteLineHeader("// ***** Competition: Start *****");
Logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
Logger.WriteLineInfo($"// {benchmark.Caption} {benchmark.Task.Settings.ToArgs()}");
Logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
var report = Run(benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
Logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
Logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
Logger.NewLine();
}
Logger.WriteLineHeader("// ***** Competition: Finish *****");
Logger.NewLine();
ReportExporter.Export(reports, Logger);
Logger.NewLine();
Logger.WriteLineHeader("// ***** Competition: End *****");
return reports;
}
public IEnumerable<BenchmarkReport> RunCompetition(List<Benchmark> benchmarks)
{
benchmarks.Sort((a, b) => string.Compare((a.Task.Configuration.Caption + a.Target.Caption), b.Task.Configuration.Caption + b.Target.Caption, StringComparison.Ordinal));
Logger.WriteLineHeader("// ***** Competition: Start *****");
Logger.WriteLineInfo("// Found benchmarks:");
foreach (var benchmark in benchmarks)
Logger.WriteLineInfo($"// {benchmark.Caption} {benchmark.Task.Settings.ToArgs()}");
Logger.NewLine();
var importantPropertyNames = benchmarks.Select(b => b.Properties).GetImportantNames();
var reports = new List<BenchmarkReport>();
foreach (var benchmark in benchmarks)
{
var report = Run(benchmark, importantPropertyNames);
reports.Add(report);
if (report.Runs.Count > 0)
{
var stat = new BenchmarkRunReportsStatistic("Target", report.Runs);
Logger.WriteLineResult($"AverageTime (ns/op): {stat.AverageTime}");
Logger.WriteLineResult($"OperationsPerSecond: {stat.OperationsPerSeconds}");
}
Logger.NewLine();
}
Logger.WriteLineHeader("// ***** Competition: Finish *****");
Logger.NewLine();
Logger.WriteLineInfo("```ini");
Logger.WriteLineInfo(EnvironmentHelper.GetFullEnvironmentInfo("Host", false));
var reportStats = reports.Where(r => r.Runs.Count > 0).Select(
r => new
{
r.Benchmark,
Stat = new BenchmarkRunReportsStatistic("Target", r.Runs)
}).ToList();
// Ensure uniform number formats and use of time units via these helpers.
var averageTimeStats = reportStats.Select(reportStat => reportStat.Stat);
var timeToStringFunc = GetTimeMeasurementFormattingFunc(averageTimeStats);
var opsPerSecToStringFunc = GetOpsPerSecFormattingFunc();
var table = new List<string[]> { new[] { "Type", "Method", "Mode", "Platform", "Jit", ".NET", "AvrTime", "StdDev", "op/s" } };
foreach (var reportStat in reportStats)
{
var b = reportStat.Benchmark;
string[] row = {
b.Target.Type.Name,
b.Target.Method.Name,
b.Task.Configuration.Mode.ToString(),
b.Task.Configuration.Platform.ToString(),
b.Task.Configuration.JitVersion.ToString(),
b.Task.Configuration.Framework.ToString(),
timeToStringFunc(reportStat.Stat.AverageTime.Median),
timeToStringFunc(reportStat.Stat.AverageTime.StandardDeviation),
opsPerSecToStringFunc(reportStat.Stat.OperationsPerSeconds.Median)
};
table.Add(row);
}
PrintTable(table);
var benchmarksWithTroubles = reports.Where(r => r.Runs.Count == 0).Select(r => r.Benchmark).ToList();
if (benchmarksWithTroubles.Count > 0)
{
Logger.NewLine();
Logger.WriteLineError("Benchmarks with troubles:");
foreach (var benchmarkWithTroubles in benchmarksWithTroubles)
Logger.WriteLineError(" " + benchmarkWithTroubles.Caption);
}
Logger.NewLine();
Logger.WriteLineHeader("// ***** Competition: End *****");
return reports;
}